fix(docker): recover from out-of-band container removal in persistent mode (salvage #36631) (#39415)

Salvage of #36631 (@annguyenNous), rebased onto current main with
regression tests added. Fixes #36266.

When a persistent Docker sandbox container is removed out-of-band (idle
reaper, `docker prune`, OOM kill, daemon restart), the gateway kept
issuing `docker exec` against the dead container ID, returning
"No such container" on every subsequent tool call — the agent was
permanently blocked until the gateway process restarted.

DockerEnvironment.execute() now detects the "No such container" /
"is not running" error after a non-zero exit (gated on
persist_across_processes) and calls _recreate_container(): it tries
label-based reuse first, falls back to a fresh container replaying the
same image + full all_run_args set, re-runs init_session(), and retries
the command once. A genuine non-zero exit is NOT misclassified as
container-gone.

Differs from #36631 as submitted: adds the tests the original lacked.
tests/tools/test_docker_environment.py covers _is_container_gone pattern
matching (incl. the negative/control case), the recover-and-retry path,
the persist_across_processes=False opt-out (no recovery), and the
ordinary-failure passthrough (no spurious recreation). _make_dummy_env
now forwards persist_across_processes.

Verified:
- Unit: 67/67 in test_docker_environment.py (4 new + existing).
- Live E2E against the real docker daemon: started a persistent
  container, `docker rm -f`'d it out-of-band, and the next execute()
  transparently recreated a fresh container and succeeded; a follow-up
  command worked in the recovered container; a real `exit N` passed
  through without triggering recovery.

Co-authored-by: annguyenNous <annguyenNous@users.noreply.github.com>

tools/environments/docker.py

@@ -537,6 +537,10 @@ class DockerEnvironment(BaseEnvironment):
         self._env = _normalize_env_dict(env)
         self._container_id: Optional[str] = None
         self._labels: dict[str, str] = {}
+        self._image: str = ""
+        self._container_name: str = ""
+        self._image_uses_s6_init: bool = False
+        self._all_run_args: list[str] = []
         logger.info(f"DockerEnvironment volumes: {volumes}")
         # Ensure volumes is a list (config.yaml could be malformed)
         if volumes is not None and not isinstance(volumes, list):
@@ -791,6 +795,12 @@ class DockerEnvironment(BaseEnvironment):
             "--label", f"hermes-task-id={task_label}",
             "--label", f"hermes-profile={profile_name}",
         ]
+        # Save args for container recreation on "No such container" recovery.
+        self._image = image
+        self._container_name = container_name
+        self._image_uses_s6_init = image_uses_s6_init
+        self._all_run_args = all_run_args
+
         self._labels = {
             "hermes-agent": "1",
             "hermes-task-id": task_label,
@@ -945,6 +955,117 @@ class DockerEnvironment(BaseEnvironment):
 
         return _popen_bash(cmd, stdin_data)
 
+    # ------------------------------------------------------------------
+    # "No such container" recovery (issue #36266)
+    # ------------------------------------------------------------------
+
+    _NO_CONTAINER_PATTERNS = (
+        "No such container",
+        "is not running",
+        "no such container",
+    )
+
+    def _is_container_gone(self, output: str) -> bool:
+        """Return True if the output indicates the container no longer exists."""
+        return any(p in output for p in self._NO_CONTAINER_PATTERNS)
+
+    def _recreate_container(self) -> bool:
+        """Recreate the container after it was removed out-of-band.
+
+        Tries label-based reuse first; if no existing container is found,
+        starts a fresh one with the same image and run-args.  Returns True
+        on success, False if recreation fails (caller should surface the
+        original error).
+        """
+        old_id = (self._container_id or "")[:12]
+        logger.warning(
+            "Container %s appears to be gone — attempting recovery", old_id,
+        )
+        self._container_id = None
+
+        # 1. Try label-based reuse (another process may have recreated it).
+        task_label = self._labels.get("hermes-task-id", "")
+        profile_label = self._labels.get("hermes-profile", "")
+        existing = self._find_reusable_container(task_label, profile_label)
+        if existing is not None:
+            cid, state = existing
+            if state == "running":
+                self._container_id = cid
+                logger.info("Recovery: reusing running container %s", cid[:12])
+            else:
+                try:
+                    subprocess.run(
+                        [self._docker_exe, "start", cid],
+                        capture_output=True, text=True, timeout=30, check=True,
+                    )
+                    self._container_id = cid
+                    logger.info("Recovery: restarted container %s", cid[:12])
+                except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
+                    logger.warning("Recovery: failed to start container %s: %s", cid[:12], e)
+
+        # 2. No reusable container — create a fresh one.
+        if not self._container_id:
+            if not self._image:
+                logger.error("Recovery: no saved image name, cannot recreate container")
+                return False
+            try:
+                import uuid as _uuid
+                new_name = f"hermes-{_uuid.uuid4().hex[:8]}"
+                init_args = [] if self._image_uses_s6_init else ["--init"]
+                label_args = []
+                for k, v in self._labels.items():
+                    label_args.extend(["--label", f"{k}={v}"])
+                run_cmd = [
+                    self._docker_exe, "run", "-d",
+                    *init_args,
+                    "--name", new_name,
+                    *label_args,
+                    "-w", self.cwd,
+                    *self._all_run_args,
+                    self._image,
+                    "sleep", "infinity",
+                ]
+                result = subprocess.run(
+                    run_cmd, capture_output=True, text=True, timeout=120, check=True,
+                )
+                self._container_id = result.stdout.strip()
+                self._container_name = new_name
+                logger.info(
+                    "Recovery: created fresh container %s (%s)",
+                    new_name, self._container_id[:12],
+                )
+            except (subprocess.CalledProcessError, subprocess.TimeoutExpired, OSError) as e:
+                logger.error("Recovery: failed to create new container: %s", e)
+                return False
+
+        # 3. Re-initialize session snapshot in the (re)created container.
+        try:
+            self._snapshot_ready = False
+            self.init_session()
+        except Exception as e:
+            logger.error("Recovery: init_session failed in new container: %s", e)
+            return False
+
+        logger.info("Recovery successful — new container %s", (self._container_id or "")[:12])
+        return True
+
+    def execute(self, command: str, cwd: str = "", **kwargs) -> dict:
+        """Execute a command, auto-recovering from dead containers.
+
+        If the container was removed out-of-band (idle reaper, docker prune,
+        OOM kill, daemon restart), detect the error and recreate the container
+        transparently before retrying once.
+        """
+        result = super().execute(command, cwd, **kwargs)
+        if (
+            result.get("returncode", 0) != 0
+            and self._is_container_gone(result.get("output", ""))
+            and self._persist_across_processes
+        ):
+            if self._recreate_container():
+                result = super().execute(command, cwd, **kwargs)
+        return result
+
     @staticmethod
     def _storage_opt_supported() -> bool:
         """Check if Docker's storage driver supports --storage-opt size=.