From bca0e2e8c25bef8e579815ddd3ba4dbf33b91d95 Mon Sep 17 00:00:00 2001
From: Andrea Luzzardi <aluzzardi@gmail.com>
Date: Tue, 5 May 2015 14:05:19 -0700
Subject: [PATCH 1/6] integration: cleanup discovery tests

Signed-off-by: Andrea Luzzardi <aluzzardi@gmail.com>
---
 ...file-discovery.bats => discovery-file.bats} |  7 +++++--
 ...ken-discovery.bats => discovery-token.bats} | 17 ++++++++++-------
 .../{zk-discovery.bats => discovery-zk.bats}   | 18 +++++++++++-------
 test/integration/helpers.bash                  |  2 +-
 4 files changed, 27 insertions(+), 17 deletions(-)
 rename test/integration/{file-discovery.bats => discovery-file.bats} (95%)
 rename test/integration/{token-discovery.bats => discovery-token.bats} (66%)
 rename test/integration/{zk-discovery.bats => discovery-zk.bats} (65%)

diff --git a/test/integration/file-discovery.bats b/test/integration/discovery-file.bats
similarity index 95%
rename from test/integration/file-discovery.bats
rename to test/integration/discovery-file.bats
index d2ca80a658..15325e8b4c 100644
--- a/test/integration/file-discovery.bats
+++ b/test/integration/discovery-file.bats
@@ -5,10 +5,14 @@ load helpers
 # create a blank temp file for discovery
 DISCOVERY_FILE=$(mktemp)
 
+function setup() {
+	start_docker 2
+}
+
 function teardown() {
 	swarm_manage_cleanup
-	rm -f $DISCOVERY_FILE
 	stop_docker
+	rm -f $DISCOVERY_FILE
 }
 
 function setup_file_discovery() {
@@ -18,7 +22,6 @@ function setup_file_discovery() {
 }
 
 @test "file discovery" {
-	start_docker 2
 	setup_file_discovery
 	swarm_manage file://$DISCOVERY_FILE
 
diff --git a/test/integration/token-discovery.bats b/test/integration/discovery-token.bats
similarity index 66%
rename from test/integration/token-discovery.bats
rename to test/integration/discovery-token.bats
index 126424ca20..a7c143e4dc 100644
--- a/test/integration/token-discovery.bats
+++ b/test/integration/discovery-token.bats
@@ -6,6 +6,10 @@ function token_cleanup() {
 	curl -X DELETE https://discovery-stage.hub.docker.com/v1/clusters/$1
 }
 
+function setup() {
+	start_docker 2
+}
+
 function teardown() {
 	swarm_join_cleanup
 	swarm_manage_cleanup
@@ -13,18 +17,17 @@ function teardown() {
 }
 
 @test "token discovery" {
-	start_docker 2
-
-	TOKEN=$(swarm create)
+	run swarm create
 	[ "$status" -eq 0 ]
-	[[ ${TOKEN} =~ ^[0-9a-f]{32}$ ]]
+	[[ "$output" =~ ^[0-9a-f]{32}$ ]]
+	TOKEN="$output"
 
-	swarm_manage token://$TOKEN
-	swarm_join   token://$TOKEN
+	swarm_manage "token://$TOKEN"
+	swarm_join   "token://$TOKEN"
 
 	run docker_swarm info
 	echo $output
 	[[ "$output" == *"Nodes: 2"* ]]
 
-	token_cleanup $TOKEN
+	token_cleanup "$TOKEN"
 }
diff --git a/test/integration/zk-discovery.bats b/test/integration/discovery-zk.bats
similarity index 65%
rename from test/integration/zk-discovery.bats
rename to test/integration/discovery-zk.bats
index c21f3daed7..49c245bfc0 100644
--- a/test/integration/zk-discovery.bats
+++ b/test/integration/discovery-zk.bats
@@ -9,11 +9,18 @@ ZK_HOST=127.0.0.1:$(( ( RANDOM % 1000 )  + 7000 ))
 ZK_CONTAINER_NAME=swarm_integration_zk
 
 function start_zk() {
-	docker_host run --name $ZK_CONTAINER_NAME -p $ZK_HOST:2181 -d jplock/zookeeper
+	run docker run --name $ZK_CONTAINER_NAME -p $ZK_HOST:2181 -d jplock/zookeeper
+	[ "$status" -eq 0 ]
 }
 
 function stop_zk() {
-	docker_host rm -f -v $ZK_CONTAINER_NAME > /dev/null
+	run docker rm -f -v $ZK_CONTAINER_NAME
+	[ "$status" -eq 0 ]
+}
+
+function setup() {
+	start_zk
+	start_docker 2
 }
 
 function teardown() {
@@ -23,13 +30,10 @@ function teardown() {
 	stop_zk
 }
 
-@test "zk discovery" {
-	start_zk
-	start_docker 1
-
+@test "zookeeperk discovery" {
 	swarm_manage zk://${ZK_HOST}/test
 	swarm_join   zk://${ZK_HOST}/test
 
 	run docker_swarm info
-	[[ "$output" == *"Nodes: 1"* ]]
+	[[ "$output" == *"Nodes: 2"* ]]
 }
diff --git a/test/integration/helpers.bash b/test/integration/helpers.bash
index 76e85a7798..14197c291b 100644
--- a/test/integration/helpers.bash
+++ b/test/integration/helpers.bash
@@ -105,7 +105,7 @@ function swarm_join() {
 
 # Stops the manager.
 function swarm_manage_cleanup() {
-	kill $SWARM_PID
+	kill $SWARM_PID || true
 }
 
 # Clean up Swarm join processes

From bd3c5c3636c256f7290d3284498f3a7736eae732 Mon Sep 17 00:00:00 2001
From: Andrea Luzzardi <aluzzardi@gmail.com>
Date: Tue, 5 May 2015 18:35:09 -0700
Subject: [PATCH 2/6] Revert "use retry to wait for swarm join"

This reverts commit 89fe217e12c4f1c2879d274ee4ba7ad324c38371.

Signed-off-by: Andrea Luzzardi <aluzzardi@gmail.com>

Conflicts:
	test/integration/helpers.bash
---
 test/integration/helpers.bash | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/test/integration/helpers.bash b/test/integration/helpers.bash
index 14197c291b..d8fb4d1ba7 100644
--- a/test/integration/helpers.bash
+++ b/test/integration/helpers.bash
@@ -100,7 +100,24 @@ function swarm_join() {
 		SWARM_JOIN_PID[$i]=$!
 		((++i))
 	done
-	retry 30 1 [ -n "$(docker_swarm info | grep -q 'Nodes: $i')" ]
+	wait_until_swarm_joined $i
+}
+
+# Wait until a swarm instance joins the cluster.
+# Parameter $1 is number of nodes to check.
+function wait_until_swarm_joined {
+	local attempts=0
+	local max_attempts=10
+
+	until [ $attempts -ge $max_attempts ]; do
+		run docker -H $SWARM_HOST info
+		if [[ "${lines[3]}" == *"Nodes: $1"* ]]; then
+			break
+		fi 
+		echo "Checking if joined successfully for the $((++attempts)) time" >&2
+		sleep 1
+	done
+	[[ $attempts -lt $max_attempts ]]
 }
 
 # Stops the manager.

From 4ac91fd7cdfbf62cb48877b2e8c69538ce8d8d54 Mon Sep 17 00:00:00 2001
From: Andrea Luzzardi <aluzzardi@gmail.com>
Date: Tue, 5 May 2015 18:35:16 -0700
Subject: [PATCH 3/6] integration helpers: Revamped swarm join/manage.

- swarm_join can be called multiple times. Only new nodes will join the
  discovery.
- swarm_join won't return until the nodes are visible in discovery
  (swarm list).
- swarm_manage won't return until the nodes are visible in the manager.

Signed-off-by: Andrea Luzzardi <aluzzardi@gmail.com>
---
 test/integration/helpers.bash | 62 ++++++++++++++++++++---------------
 1 file changed, 36 insertions(+), 26 deletions(-)

diff --git a/test/integration/helpers.bash b/test/integration/helpers.bash
index d8fb4d1ba7..329687f207 100644
--- a/test/integration/helpers.bash
+++ b/test/integration/helpers.bash
@@ -69,7 +69,7 @@ function retry() {
 	done
 
 	echo "Command \"$@\" failed $attempts times. Output: $output"
-	[[ false ]]
+	false
 }
 
 # Waits until the given docker engine API becomes reachable.
@@ -77,6 +77,10 @@ function wait_until_reachable() {
 	retry 10 1 docker -H $1 info
 }
 
+function all_nodes_registered_in_swarm() {
+	docker_swarm info | grep -q "Nodes: ${#HOSTS[@]}"
+}
+
 # Start the swarm manager in background.
 function swarm_manage() {
 	local discovery
@@ -86,38 +90,44 @@ function swarm_manage() {
 		discovery="$@"
 	fi
 
-	$SWARM_BINARY manage -H $SWARM_HOST $discovery &
+	"$SWARM_BINARY" manage -H "$SWARM_HOST" --heartbeat=1 "$discovery" &
 	SWARM_PID=$!
-	wait_until_reachable $SWARM_HOST
+	wait_until_reachable "$SWARM_HOST"
+	retry 10 1 all_nodes_registered_in_swarm
 }
 
-# Start swarm join for every engine with the discovery as parameter
+# swarm join every engine created with `start_docker`.
+#
+# It will wait until all nodes are visible in discovery (`swarm list`) before
+# returning and will fail if that's not the case after a certain time.
+#
+# It can be called multiple times and will only join new engines started with
+# `start_docker` since the last `swarm_join` call.
 function swarm_join() {
-	local i=0
-	for h in ${HOSTS[@]}; do
-		echo "Swarm join #${i}: $h $@"
-		$SWARM_BINARY join --addr=$h "$@" &
+	local current=${#SWARM_JOIN_PID[@]}
+	local nodes=${#HOSTS[@]}
+	local addr="$1"
+	shift
+
+	# Start the engines.
+	local i
+	echo "current: $current | nodes: $nodes" > log
+	for ((i=current; i < nodes; i++)); do
+		local h="${HOSTS[$i]}"
+		echo "Swarm join #${i}: $h $addr"
+		"$SWARM_BINARY" join --addr="$h" "$addr" &
 		SWARM_JOIN_PID[$i]=$!
-		((++i))
 	done
-	wait_until_swarm_joined $i
+	retry 10 0.5 all_nodes_registered_in_discovery "$addr"
 }
 
-# Wait until a swarm instance joins the cluster.
-# Parameter $1 is number of nodes to check.
-function wait_until_swarm_joined {
-	local attempts=0
-	local max_attempts=10
+# Returns true if all nodes have joined the discovery.
+function all_nodes_registered_in_discovery() {
+	local joined=`swarm list "$1" | wc -l`
+	local total=${#HOSTS[@]}
 
-	until [ $attempts -ge $max_attempts ]; do
-		run docker -H $SWARM_HOST info
-		if [[ "${lines[3]}" == *"Nodes: $1"* ]]; then
-			break
-		fi 
-		echo "Checking if joined successfully for the $((++attempts)) time" >&2
-		sleep 1
-	done
-	[[ $attempts -lt $max_attempts ]]
+	echo "${joined} out of ${total} hosts joined discovery"
+	[ "$joined" -eq "$total" ]
 }
 
 # Stops the manager.
@@ -128,7 +138,7 @@ function swarm_manage_cleanup() {
 # Clean up Swarm join processes
 function swarm_join_cleanup() {
 	for pid in ${SWARM_JOIN_PID[@]}; do
-		kill $pid
+		kill $pid || true
 	done
 }
 
@@ -159,7 +169,7 @@ function start_docker() {
 
 	# Wait for the engines to be reachable.
 	for ((i=current; i < (current + instances); i++)); do
-		wait_until_reachable ${HOSTS[$i]}
+		wait_until_reachable "${HOSTS[$i]}"
 	done
 }
 

From 9124f3a0c7fb6dcc8b17521a314db383c998b5d4 Mon Sep 17 00:00:00 2001
From: Andrea Luzzardi <aluzzardi@gmail.com>
Date: Tue, 5 May 2015 19:13:57 -0700
Subject: [PATCH 4/6] cluster: Improved discovery duplication error messaging.

Signed-off-by: Andrea Luzzardi <aluzzardi@gmail.com>
---
 cluster/swarm/cluster.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cluster/swarm/cluster.go b/cluster/swarm/cluster.go
index ab6f49a3f3..b2934029d5 100644
--- a/cluster/swarm/cluster.go
+++ b/cluster/swarm/cluster.go
@@ -141,10 +141,10 @@ func (c *Cluster) newEntries(entries []*discovery.Entry) {
 
 				if old, exists := c.engines[engine.ID]; exists {
 					c.Unlock()
-					if old.IP != engine.IP {
-						log.Errorf("ID duplicated. %s shared by %s and %s", engine.ID, old.IP, engine.IP)
+					if old.Addr != engine.Addr {
+						log.Errorf("ID duplicated. %s shared by %s and %s", engine.ID, old.Addr, engine.Addr)
 					} else {
-						log.Errorf("node %q with IP %q is  already registered", engine.Name, engine.IP)
+						log.Debugf("node %q (name: %q) with address %q is already registered", engine.ID, engine.Name, engine.Addr)
 					}
 					return
 				}

From 3501e45f4fb5b22ac0f101eb0ff71700a16fb96b Mon Sep 17 00:00:00 2001
From: Andrea Luzzardi <aluzzardi@gmail.com>
Date: Tue, 5 May 2015 19:14:59 -0700
Subject: [PATCH 5/6] integration: Improved discovery testing.

Verify that swarm can see new nodes joining discovery.

Signed-off-by: Andrea Luzzardi <aluzzardi@gmail.com>
---
 test/integration/discovery-file.bats  | 21 +++++++++++---------
 test/integration/discovery-token.bats | 28 ++++++++++++++++-----------
 test/integration/discovery-zk.bats    | 24 ++++++++++++++---------
 3 files changed, 44 insertions(+), 29 deletions(-)

diff --git a/test/integration/discovery-file.bats b/test/integration/discovery-file.bats
index 15325e8b4c..0f8cd3521c 100644
--- a/test/integration/discovery-file.bats
+++ b/test/integration/discovery-file.bats
@@ -5,27 +5,30 @@ load helpers
 # create a blank temp file for discovery
 DISCOVERY_FILE=$(mktemp)
 
-function setup() {
-	start_docker 2
-}
-
 function teardown() {
 	swarm_manage_cleanup
 	stop_docker
-	rm -f $DISCOVERY_FILE
+	rm -f "$DISCOVERY_FILE"
 }
 
 function setup_file_discovery() {
+	rm -f "$DISCOVERY_FILE"
 	for host in ${HOSTS[@]}; do
 		echo "$host" >> $DISCOVERY_FILE
 	done
 }
 
 @test "file discovery" {
+	# Start 2 engines, register them in a file, then start swarm and make sure
+	# it sees them.
+	start_docker 2
 	setup_file_discovery
-	swarm_manage file://$DISCOVERY_FILE
+	swarm_manage "file://$DISCOVERY_FILE"
+	all_nodes_registered_in_swarm
 
-	run docker_swarm info
-	[ "$status" -eq 0 ]
-	[[ "$output" == *"Nodes: 2"* ]]
+	# Add another engine to the cluster, update the discovery file and make
+	# sure it's picked up by swarm.
+	start_docker 1
+	setup_file_discovery
+	retry 10 1 all_nodes_registered_in_swarm
 }
diff --git a/test/integration/discovery-token.bats b/test/integration/discovery-token.bats
index a7c143e4dc..f8802b7eaa 100644
--- a/test/integration/discovery-token.bats
+++ b/test/integration/discovery-token.bats
@@ -2,32 +2,38 @@
 
 load helpers
 
-function token_cleanup() {
-	curl -X DELETE https://discovery-stage.hub.docker.com/v1/clusters/$1
-}
+TOKEN=""
 
-function setup() {
-	start_docker 2
+function token_cleanup() {
+	[ -z "$TOKEN" ] && return
+	echo "Removing $TOKEN"
+	curl -X DELETE "https://discovery-stage.hub.docker.com/v1/clusters/$TOKEN"
 }
 
 function teardown() {
-	swarm_join_cleanup
 	swarm_manage_cleanup
+	swarm_join_cleanup
 	stop_docker
+	token_cleanup
 }
 
 @test "token discovery" {
+	# Create a cluster and validate the token.
 	run swarm create
 	[ "$status" -eq 0 ]
 	[[ "$output" =~ ^[0-9a-f]{32}$ ]]
 	TOKEN="$output"
 
-	swarm_manage "token://$TOKEN"
+	# Start 2 engines and make them join the cluster.
+	start_docker 2
 	swarm_join   "token://$TOKEN"
 
-	run docker_swarm info
-	echo $output
-	[[ "$output" == *"Nodes: 2"* ]]
+	# Start a manager and ensure it sees all the engines.
+	swarm_manage "token://$TOKEN"
+	all_nodes_registered_in_swarm
 
-	token_cleanup "$TOKEN"
+	# Add another engine to the cluster and make sure it's picked up by swarm.
+	start_docker 1
+	swarm_join   "token://$TOKEN"
+	retry 10 1 all_nodes_registered_in_swarm
 }
diff --git a/test/integration/discovery-zk.bats b/test/integration/discovery-zk.bats
index 49c245bfc0..7118170e52 100644
--- a/test/integration/discovery-zk.bats
+++ b/test/integration/discovery-zk.bats
@@ -9,31 +9,37 @@ ZK_HOST=127.0.0.1:$(( ( RANDOM % 1000 )  + 7000 ))
 ZK_CONTAINER_NAME=swarm_integration_zk
 
 function start_zk() {
-	run docker run --name $ZK_CONTAINER_NAME -p $ZK_HOST:2181 -d jplock/zookeeper
+	run docker_host run --name $ZK_CONTAINER_NAME -p $ZK_HOST:2181 -d jplock/zookeeper
 	[ "$status" -eq 0 ]
 }
 
 function stop_zk() {
-	run docker rm -f -v $ZK_CONTAINER_NAME
+	run docker_host rm -f -v $ZK_CONTAINER_NAME
 	[ "$status" -eq 0 ]
 }
 
 function setup() {
 	start_zk
-	start_docker 2
 }
 
 function teardown() {
-	swarm_join_cleanup
 	swarm_manage_cleanup
+	swarm_join_cleanup
 	stop_docker
 	stop_zk
 }
 
-@test "zookeeperk discovery" {
-	swarm_manage zk://${ZK_HOST}/test
-	swarm_join   zk://${ZK_HOST}/test
+@test "zookeeper discovery" {
+	# Start 2 engines and make them join the cluster.
+	start_docker 2
+	swarm_join   "zk://${ZK_HOST}/test"
 
-	run docker_swarm info
-	[[ "$output" == *"Nodes: 2"* ]]
+	# Start a manager and ensure it sees all the engines.
+	swarm_manage "zk://${ZK_HOST}/test"
+	all_nodes_registered_in_swarm
+
+	# Add another engine to the cluster and make sure it's picked up by swarm.
+	start_docker 1
+	swarm_join   "zk://${ZK_HOST}/test"
+	retry 30 1 all_nodes_registered_in_swarm
 }

From a2703e05621c72da805e2c01d18aae2ce06cff18 Mon Sep 17 00:00:00 2001
From: Andrea Luzzardi <aluzzardi@gmail.com>
Date: Tue, 5 May 2015 19:19:10 -0700
Subject: [PATCH 6/6] integration: helpers cleanup.

Signed-off-by: Andrea Luzzardi <aluzzardi@gmail.com>
---
 test/integration/discovery-file.bats  | 4 ++--
 test/integration/discovery-token.bats | 8 ++++----
 test/integration/discovery-zk.bats    | 8 ++++----
 test/integration/helpers.bash         | 9 +++++----
 4 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/test/integration/discovery-file.bats b/test/integration/discovery-file.bats
index 0f8cd3521c..98b91557c9 100644
--- a/test/integration/discovery-file.bats
+++ b/test/integration/discovery-file.bats
@@ -24,11 +24,11 @@ function setup_file_discovery() {
 	start_docker 2
 	setup_file_discovery
 	swarm_manage "file://$DISCOVERY_FILE"
-	all_nodes_registered_in_swarm
+	check_swarm_nodes
 
 	# Add another engine to the cluster, update the discovery file and make
 	# sure it's picked up by swarm.
 	start_docker 1
 	setup_file_discovery
-	retry 10 1 all_nodes_registered_in_swarm
+	retry 10 1 check_swarm_nodes
 }
diff --git a/test/integration/discovery-token.bats b/test/integration/discovery-token.bats
index f8802b7eaa..877954fbf9 100644
--- a/test/integration/discovery-token.bats
+++ b/test/integration/discovery-token.bats
@@ -26,14 +26,14 @@ function teardown() {
 
 	# Start 2 engines and make them join the cluster.
 	start_docker 2
-	swarm_join   "token://$TOKEN"
+	swarm_join "token://$TOKEN"
 
 	# Start a manager and ensure it sees all the engines.
 	swarm_manage "token://$TOKEN"
-	all_nodes_registered_in_swarm
+	check_swarm_nodes
 
 	# Add another engine to the cluster and make sure it's picked up by swarm.
 	start_docker 1
-	swarm_join   "token://$TOKEN"
-	retry 10 1 all_nodes_registered_in_swarm
+	swarm_join "token://$TOKEN"
+	retry 10 1 check_swarm_nodes
 }
diff --git a/test/integration/discovery-zk.bats b/test/integration/discovery-zk.bats
index 7118170e52..638e929fc8 100644
--- a/test/integration/discovery-zk.bats
+++ b/test/integration/discovery-zk.bats
@@ -32,14 +32,14 @@ function teardown() {
 @test "zookeeper discovery" {
 	# Start 2 engines and make them join the cluster.
 	start_docker 2
-	swarm_join   "zk://${ZK_HOST}/test"
+	swarm_join "zk://${ZK_HOST}/test"
 
 	# Start a manager and ensure it sees all the engines.
 	swarm_manage "zk://${ZK_HOST}/test"
-	all_nodes_registered_in_swarm
+	check_swarm_nodes
 
 	# Add another engine to the cluster and make sure it's picked up by swarm.
 	start_docker 1
-	swarm_join   "zk://${ZK_HOST}/test"
-	retry 30 1 all_nodes_registered_in_swarm
+	swarm_join "zk://${ZK_HOST}/test"
+	retry 30 1 check_swarm_nodes
 }
diff --git a/test/integration/helpers.bash b/test/integration/helpers.bash
index 329687f207..10e74753d8 100644
--- a/test/integration/helpers.bash
+++ b/test/integration/helpers.bash
@@ -77,7 +77,8 @@ function wait_until_reachable() {
 	retry 10 1 docker -H $1 info
 }
 
-function all_nodes_registered_in_swarm() {
+# Returns true if all nodes have joined the swarm.
+function check_swarm_nodes() {
 	docker_swarm info | grep -q "Nodes: ${#HOSTS[@]}"
 }
 
@@ -93,7 +94,7 @@ function swarm_manage() {
 	"$SWARM_BINARY" manage -H "$SWARM_HOST" --heartbeat=1 "$discovery" &
 	SWARM_PID=$!
 	wait_until_reachable "$SWARM_HOST"
-	retry 10 1 all_nodes_registered_in_swarm
+	retry 10 1 check_swarm_nodes
 }
 
 # swarm join every engine created with `start_docker`.
@@ -118,11 +119,11 @@ function swarm_join() {
 		"$SWARM_BINARY" join --addr="$h" "$addr" &
 		SWARM_JOIN_PID[$i]=$!
 	done
-	retry 10 0.5 all_nodes_registered_in_discovery "$addr"
+	retry 10 0.5 check_discovery_nodes "$addr"
 }
 
 # Returns true if all nodes have joined the discovery.
-function all_nodes_registered_in_discovery() {
+function check_discovery_nodes() {
 	local joined=`swarm list "$1" | wc -l`
 	local total=${#HOSTS[@]}