fix: Replace a test container whose database does not become reachable, and remove every container that failed to start.

eventsourcingdb-testcontainers/src/main/java/io/eventsourcingdb/testcontainers/Container.java

@@ -12,6 +12,7 @@
 import java.util.ArrayList;
 import java.util.Base64;
 import java.util.List;
+import java.util.function.Consumer;
 import org.testcontainers.containers.GenericContainer;
 import org.testcontainers.containers.wait.strategy.Wait;
 import org.testcontainers.images.builder.Transferable;
@@ -38,12 +39,19 @@ public final class Container {
     private static final String IMAGE_NAME = "thenativeweb/eventsourcingdb";
     private static final String SIGNING_KEY_PATH = "/etc/esdb/signing-key.pem";
 
+    // How many containers start() starts at most, as it replaces each
+    // container whose database does not become reachable.
+    private static final int MAX_START_ATTEMPTS = 3;
+
     private String imageTag = "latest";
     private int internalPort = 3000;
     private String apiToken = "secret";
     private KeyPair signingKeyPair;
     private GenericContainer<?> container;
 
+    // Starts a container. It is a field only so that tests can replace it.
+    Consumer<GenericContainer<?>> starter = GenericContainer::start;
+
     /**
      * Creates a container that uses the {@code latest} tag of the official EventSourcingDB image, port 3000 inside the
      * container, and the API token {@code secret}, and does not sign events.
@@ -98,9 +106,47 @@ public Container withSigningKey() {
     /**
      * Starts the container, and waits until the instance answers pings.
      *
-     * @throws RuntimeException if the container does not start, e.g. because the image does not exist
+     * <p>Docker Desktop sometimes does not make the port it maps a container to reachable from the host. If the
+     * database in a running container does not become reachable, the container is replaced by a new one, up to three
+     * times. A container that fails to start is removed either way.
+     *
+     * @throws IllegalStateException if the database did not become reachable in any of three containers
+     * @throws RuntimeException if the container does not start for another reason, e.g. because the image does not
+     *     exist
      */
     public void start() {
+        for (var attempt = 1; ; attempt++) {
+            var container = newContainer();
+
+            try {
+                starter.accept(container);
+                this.container = container;
+                return;
+            } catch (RuntimeException ex) {
+                // Docker Desktop sometimes does not make the port it maps a
+                // container to reachable from the host. The container keeps
+                // running, but the database in it can never be reached, so it
+                // is replaced by a new one. Every other failure is thrown at
+                // once. Either way, the container that failed to start is
+                // removed, so that it is not left behind.
+                var isRunning = container.isRunning();
+                container.stop();
+
+                if (!isRunning) {
+                    throw ex;
+                }
+                if (attempt == MAX_START_ATTEMPTS) {
+                    throw new IllegalStateException(
+                            "failed to start container, the database did not become reachable in "
+                                    + MAX_START_ATTEMPTS
+                                    + " attempts",
+                            ex);
+                }
+            }
+        }
+    }
+
+    private GenericContainer<?> newContainer() {
         var command = new ArrayList<>(List.of(
                 "run",
                 "--api-token",
@@ -122,9 +168,8 @@ public void start() {
         }
 
         container.withCommand(command.toArray(String[]::new));
-        container.start();
 
-        this.container = container;
+        return container;
     }
 
     private static byte[] pem(PrivateKey privateKey) {

eventsourcingdb-testcontainers/src/test/java/io/eventsourcingdb/testcontainers/ContainerTest.java

@@ -7,9 +7,12 @@
 
 import io.eventsourcingdb.EventCandidate;
 import java.net.URI;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import org.junit.jupiter.api.Test;
+import org.testcontainers.DockerClientFactory;
+import org.testcontainers.containers.ContainerLaunchException;
 
 class ContainerTest {
     @Test
@@ -109,4 +112,90 @@ void stopsWithoutFailingIfItIsNotRunning() {
 
         assertFalse(container.isRunning());
     }
+
+    // Lists which of the given containers still exist in Docker, running or
+    // not.
+    private static List<String> existing(List<String> containerIds) {
+        return DockerClientFactory.instance()
+                .client()
+                .listContainersCmd()
+                .withShowAll(true)
+                .withIdFilter(containerIds)
+                .exec()
+                .stream()
+                .map(container -> container.getId())
+                .toList();
+    }
+
+    @Test
+    void replacesAContainerWhoseDatabaseDoesNotBecomeReachable() {
+        var containerIds = new ArrayList<String>();
+        var container = new Container().withImageTag(ImageTag.fromDockerfile());
+
+        // Docker Desktop sometimes does not make the port of a container
+        // reachable from the host. The container keeps running, but starting
+        // it fails, as it does here twice.
+        container.starter = genericContainer -> {
+            genericContainer.start();
+            containerIds.add(genericContainer.getContainerId());
+            if (containerIds.size() < 3) {
+                throw new ContainerLaunchException("the database did not become reachable");
+            }
+        };
+
+        container.start();
+        try {
+            assertEquals(3, containerIds.size());
+            assertEquals(List.of(containerIds.get(2)), existing(containerIds));
+            container.getClient().ping();
+        } finally {
+            container.stop();
+        }
+    }
+
+    @Test
+    void givesUpAfterThreeAttemptsAndSaysWhy() {
+        var containerIds = new ArrayList<String>();
+        var failures = new ArrayList<ContainerLaunchException>();
+        var container = new Container().withImageTag(ImageTag.fromDockerfile());
+
+        container.starter = genericContainer -> {
+            genericContainer.start();
+            containerIds.add(genericContainer.getContainerId());
+            var failure = new ContainerLaunchException("the database did not become reachable");
+            failures.add(failure);
+            throw failure;
+        };
+
+        var exception = assertThrows(IllegalStateException.class, container::start);
+
+        assertEquals(
+                "failed to start container, the database did not become reachable in 3 attempts",
+                exception.getMessage());
+        assertEquals(failures.getLast(), exception.getCause());
+        assertEquals(3, containerIds.size());
+        assertEquals(List.of(), existing(containerIds));
+        assertFalse(container.isRunning());
+    }
+
+    @Test
+    void throwsAnyOtherFailureAtOnceAndRemovesTheContainer() {
+        var containerIds = new ArrayList<String>();
+
+        // Without an API token, EventSourcingDB exits right away.
+        var container = new Container().withImageTag(ImageTag.fromDockerfile()).withApiToken("");
+        container.starter = genericContainer -> {
+            try {
+                genericContainer.start();
+            } finally {
+                containerIds.add(genericContainer.getContainerId());
+            }
+        };
+
+        assertThrows(ContainerLaunchException.class, container::start);
+
+        assertEquals(1, containerIds.size());
+        assertEquals(List.of(), existing(containerIds));
+        assertFalse(container.isRunning());
+    }
 }