Add integration tests for engines departure with node removal and ephemeral behavior

Signed-off-by: Alexandre Beslic <abronan@docker.com>

test/integration/discovery/consul.bats

@@ -59,6 +59,31 @@ function teardown() {
 	retry 5 1 discovery_check_swarm_info
 }
 
+@test "consul discovery: check for engines departure" {
+	# The goal of this test is to ensure swarm can detect engines that
+	# are removed from the discovery and refresh info accordingly
+
+	# Start the store
+	start_store
+
+	# Start a manager with no engines.
+	swarm_manage "$DISCOVERY"
+	retry 10 1 discovery_check_swarm_info
+
+	# Add Engines to the cluster and make sure it's picked by swarm
+	start_docker 2
+	swarm_join "$DISCOVERY"
+	retry 5 1 discovery_check_swarm_list "$DISCOVERY"
+	retry 5 1 discovery_check_swarm_info
+
+	# Removes all the swarm agents
+	swarm_join_cleanup
+
+	# Check if previously registered engines are all gone
+	retry 5 1 discovery_check_swarm_list "$DISCOVERY"
+	retry 30 1 discovery_check_swarm_info_empty
+}
+
 @test "consul discovery: failure" {
 	# The goal of this test is to simulate a store failure and ensure discovery
 	# is resilient to it.

test/integration/discovery/discovery_helpers.bash

@@ -10,6 +10,11 @@ function discovery_check_swarm_info() {
 	docker_swarm info | grep -q "Nodes: $count"
 }
 
+# Returns true if swarm info outputs is empty (0 nodes).
+function discovery_check_swarm_info_empty() {
+	docker_swarm info | grep -q "Nodes: 0"
+}
+
 # Returns true if all nodes have joined the discovery.
 function discovery_check_swarm_list() {
 	local joined=`swarm list "$1" | wc -l`

test/integration/discovery/etcd.bats

@@ -64,6 +64,31 @@ function teardown() {
 	retry 5 1 discovery_check_swarm_info
 }
 
+@test "etcd discovery: check for engines departure" {
+	# The goal of this test is to ensure swarm can detect engines that
+	# are removed from the discovery and refresh info accordingly
+
+	# Start the store
+	start_store
+
+	# Start a manager with no engines.
+	swarm_manage "$DISCOVERY"
+	retry 10 1 discovery_check_swarm_info
+
+	# Add Engines to the cluster and make sure it's picked by swarm
+	start_docker 2
+	swarm_join "$DISCOVERY"
+	retry 5 1 discovery_check_swarm_list "$DISCOVERY"
+	retry 5 1 discovery_check_swarm_info
+
+	# Removes all the swarm agents
+	swarm_join_cleanup
+
+	# Check if nodes are all gone
+	retry 5 1 discovery_check_swarm_list "$DISCOVERY"
+	retry 15 1 discovery_check_swarm_info_empty
+}
+
 @test "etcd discovery: failure" {
 	# The goal of this test is to simulate a store failure and ensure discovery
 	# is resilient to it.

test/integration/discovery/zk.bats

@@ -59,6 +59,31 @@ function teardown() {
 	retry 10 1 discovery_check_swarm_info
 }
 
+@test "zk discovery: check for engines departure" {
+	# The goal of this test is to ensure swarm can detect engines that
+	# are removed from the discovery and refresh info accordingly
+
+	# Start the store
+	start_store
+
+	# Start 2 engines and make them join the cluster.
+	swarm_manage "$DISCOVERY"
+	retry 10 1 discovery_check_swarm_info
+
+	# Add Engines to the cluster and make sure it's picked by swarm
+	start_docker 2
+	swarm_join "$DISCOVERY"
+	retry 10 1 discovery_check_swarm_list "$DISCOVERY"
+	retry 10 1 discovery_check_swarm_info
+
+	# Removes all the swarm agents
+	swarm_join_cleanup
+
+	# Check if previously registered engines are all gone
+	retry 10 1 discovery_check_swarm_list "$DISCOVERY"
+	retry 20 1 discovery_check_swarm_info_empty
+}
+
 @test "zk discovery: failure" {
 	# The goal of this test is to simulate a store failure and ensure discovery
 	# is resilient to it.

test/integration/helpers.bash

@@ -117,7 +117,7 @@ function swarm_join() {
 	for ((i=current; i < nodes; i++)); do
 		local h="${HOSTS[$i]}"
 		echo "Swarm join #${i}: $h $addr"
-		"$SWARM_BINARY" -l debug join --heartbeat=1s --addr="$h" "$addr" &
+		"$SWARM_BINARY" -l debug join --heartbeat=1s --ttl=10s --addr="$h" "$addr" &
 		SWARM_JOIN_PID[$i]=$!
 	done
 }