init: MindOS CLI 本地执行体（从 mindOSv2/mindos-cli 独立）
- 独立 pyproject.toml（pip install -e .） - vendor_hermes.sh 已改为显式路径模式（不再依赖相对目录） - 包含 hermes vendor 快照
This commit is contained in:
@@ -0,0 +1,13 @@
|
||||
"""Hermes execution environment backends.
|
||||
|
||||
Each backend provides the same interface (BaseEnvironment ABC) for running
|
||||
shell commands in a specific execution context: local, Docker, Singularity,
|
||||
SSH, Modal, or Daytona.
|
||||
|
||||
The terminal_tool.py factory (_create_environment) selects the backend
|
||||
based on the TERMINAL_ENV configuration.
|
||||
"""
|
||||
|
||||
from tools.environments.base import BaseEnvironment
|
||||
|
||||
__all__ = ["BaseEnvironment"]
|
||||
@@ -0,0 +1,579 @@
|
||||
"""Base class for all Hermes execution environment backends.
|
||||
|
||||
Unified spawn-per-call model: every command spawns a fresh ``bash -c`` process.
|
||||
A session snapshot (env vars, functions, aliases) is captured once at init and
|
||||
re-sourced before each command. CWD persists via in-band stdout markers (remote)
|
||||
or a temp file (local).
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from abc import ABC, abstractmethod
|
||||
from pathlib import Path
|
||||
from typing import IO, Callable, Protocol
|
||||
|
||||
from hermes_constants import get_hermes_home
|
||||
from tools.interrupt import is_interrupted
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Thread-local activity callback. The agent sets this before a tool call so
|
||||
# long-running _wait_for_process loops can report liveness to the gateway.
|
||||
_activity_callback_local = threading.local()
|
||||
|
||||
|
||||
def set_activity_callback(cb: Callable[[str], None] | None) -> None:
|
||||
"""Register a callback that _wait_for_process fires periodically."""
|
||||
_activity_callback_local.callback = cb
|
||||
|
||||
|
||||
def _get_activity_callback() -> Callable[[str], None] | None:
|
||||
return getattr(_activity_callback_local, "callback", None)
|
||||
|
||||
|
||||
def get_sandbox_dir() -> Path:
|
||||
"""Return the host-side root for all sandbox storage (Docker workspaces,
|
||||
Singularity overlays/SIF cache, etc.).
|
||||
|
||||
Configurable via TERMINAL_SANDBOX_DIR. Defaults to {HERMES_HOME}/sandboxes/.
|
||||
"""
|
||||
custom = os.getenv("TERMINAL_SANDBOX_DIR")
|
||||
if custom:
|
||||
p = Path(custom)
|
||||
else:
|
||||
p = get_hermes_home() / "sandboxes"
|
||||
p.mkdir(parents=True, exist_ok=True)
|
||||
return p
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shared constants and utilities
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _pipe_stdin(proc: subprocess.Popen, data: str) -> None:
|
||||
"""Write *data* to proc.stdin on a daemon thread to avoid pipe-buffer deadlocks."""
|
||||
|
||||
def _write():
|
||||
try:
|
||||
proc.stdin.write(data)
|
||||
proc.stdin.close()
|
||||
except (BrokenPipeError, OSError):
|
||||
pass
|
||||
|
||||
threading.Thread(target=_write, daemon=True).start()
|
||||
|
||||
|
||||
def _popen_bash(
|
||||
cmd: list[str], stdin_data: str | None = None, **kwargs
|
||||
) -> subprocess.Popen:
|
||||
"""Spawn a subprocess with standard stdout/stderr/stdin setup.
|
||||
|
||||
If *stdin_data* is provided, writes it asynchronously via :func:`_pipe_stdin`.
|
||||
Backends with special Popen needs (e.g. local's ``preexec_fn``) can bypass
|
||||
this and call :func:`_pipe_stdin` directly.
|
||||
"""
|
||||
proc = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
stdin=subprocess.PIPE if stdin_data is not None else subprocess.DEVNULL,
|
||||
text=True,
|
||||
**kwargs,
|
||||
)
|
||||
if stdin_data is not None:
|
||||
_pipe_stdin(proc, stdin_data)
|
||||
return proc
|
||||
|
||||
|
||||
def _load_json_store(path: Path) -> dict:
|
||||
"""Load a JSON file as a dict, returning ``{}`` on any error."""
|
||||
if path.exists():
|
||||
try:
|
||||
return json.loads(path.read_text())
|
||||
except Exception:
|
||||
pass
|
||||
return {}
|
||||
|
||||
|
||||
def _save_json_store(path: Path, data: dict) -> None:
|
||||
"""Write *data* as pretty-printed JSON to *path*."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, indent=2))
|
||||
|
||||
|
||||
def _file_mtime_key(host_path: str) -> tuple[float, int] | None:
|
||||
"""Return ``(mtime, size)`` for cache comparison, or ``None`` if unreadable."""
|
||||
try:
|
||||
st = Path(host_path).stat()
|
||||
return (st.st_mtime, st.st_size)
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ProcessHandle protocol
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ProcessHandle(Protocol):
|
||||
"""Duck type that every backend's _run_bash() must return.
|
||||
|
||||
subprocess.Popen satisfies this natively. SDK backends (Modal, Daytona)
|
||||
return _ThreadedProcessHandle which adapts their blocking calls.
|
||||
"""
|
||||
|
||||
def poll(self) -> int | None: ...
|
||||
def kill(self) -> None: ...
|
||||
def wait(self, timeout: float | None = None) -> int: ...
|
||||
|
||||
@property
|
||||
def stdout(self) -> IO[str] | None: ...
|
||||
|
||||
@property
|
||||
def returncode(self) -> int | None: ...
|
||||
|
||||
|
||||
class _ThreadedProcessHandle:
|
||||
"""Adapter for SDK backends (Modal, Daytona) that have no real subprocess.
|
||||
|
||||
Wraps a blocking ``exec_fn() -> (output_str, exit_code)`` in a background
|
||||
thread and exposes a ProcessHandle-compatible interface. An optional
|
||||
``cancel_fn`` is invoked on ``kill()`` for backend-specific cancellation
|
||||
(e.g. Modal sandbox.terminate, Daytona sandbox.stop).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
exec_fn: Callable[[], tuple[str, int]],
|
||||
cancel_fn: Callable[[], None] | None = None,
|
||||
):
|
||||
self._cancel_fn = cancel_fn
|
||||
self._done = threading.Event()
|
||||
self._returncode: int | None = None
|
||||
self._error: Exception | None = None
|
||||
|
||||
# Pipe for stdout — drain thread in _wait_for_process reads the read end.
|
||||
read_fd, write_fd = os.pipe()
|
||||
self._stdout = os.fdopen(read_fd, "r", encoding="utf-8", errors="replace")
|
||||
self._write_fd = write_fd
|
||||
|
||||
def _worker():
|
||||
try:
|
||||
output, exit_code = exec_fn()
|
||||
self._returncode = exit_code
|
||||
# Write output into the pipe so drain thread picks it up.
|
||||
try:
|
||||
os.write(self._write_fd, output.encode("utf-8", errors="replace"))
|
||||
except OSError:
|
||||
pass
|
||||
except Exception as exc:
|
||||
self._error = exc
|
||||
self._returncode = 1
|
||||
finally:
|
||||
try:
|
||||
os.close(self._write_fd)
|
||||
except OSError:
|
||||
pass
|
||||
self._done.set()
|
||||
|
||||
t = threading.Thread(target=_worker, daemon=True)
|
||||
t.start()
|
||||
|
||||
@property
|
||||
def stdout(self):
|
||||
return self._stdout
|
||||
|
||||
@property
|
||||
def returncode(self) -> int | None:
|
||||
return self._returncode
|
||||
|
||||
def poll(self) -> int | None:
|
||||
return self._returncode if self._done.is_set() else None
|
||||
|
||||
def kill(self):
|
||||
if self._cancel_fn:
|
||||
try:
|
||||
self._cancel_fn()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def wait(self, timeout: float | None = None) -> int:
|
||||
self._done.wait(timeout=timeout)
|
||||
return self._returncode
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CWD marker for remote backends
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _cwd_marker(session_id: str) -> str:
|
||||
return f"__HERMES_CWD_{session_id}__"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# BaseEnvironment
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class BaseEnvironment(ABC):
|
||||
"""Common interface and unified execution flow for all Hermes backends.
|
||||
|
||||
Subclasses implement ``_run_bash()`` and ``cleanup()``. The base class
|
||||
provides ``execute()`` with session snapshot sourcing, CWD tracking,
|
||||
interrupt handling, and timeout enforcement.
|
||||
"""
|
||||
|
||||
# Subclasses that embed stdin as a heredoc (Modal, Daytona) set this.
|
||||
_stdin_mode: str = "pipe" # "pipe" or "heredoc"
|
||||
|
||||
# Snapshot creation timeout (override for slow cold-starts).
|
||||
_snapshot_timeout: int = 30
|
||||
|
||||
def get_temp_dir(self) -> str:
|
||||
"""Return the backend temp directory used for session artifacts.
|
||||
|
||||
Most sandboxed backends use ``/tmp`` inside the target environment.
|
||||
LocalEnvironment overrides this on platforms like Termux where ``/tmp``
|
||||
may be missing and ``TMPDIR`` is the portable writable location.
|
||||
"""
|
||||
return "/tmp"
|
||||
|
||||
def __init__(self, cwd: str, timeout: int, env: dict = None):
|
||||
self.cwd = cwd
|
||||
self.timeout = timeout
|
||||
self.env = env or {}
|
||||
|
||||
self._session_id = uuid.uuid4().hex[:12]
|
||||
temp_dir = self.get_temp_dir().rstrip("/") or "/"
|
||||
self._snapshot_path = f"{temp_dir}/hermes-snap-{self._session_id}.sh"
|
||||
self._cwd_file = f"{temp_dir}/hermes-cwd-{self._session_id}.txt"
|
||||
self._cwd_marker = _cwd_marker(self._session_id)
|
||||
self._snapshot_ready = False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Abstract methods
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _run_bash(
|
||||
self,
|
||||
cmd_string: str,
|
||||
*,
|
||||
login: bool = False,
|
||||
timeout: int = 120,
|
||||
stdin_data: str | None = None,
|
||||
) -> ProcessHandle:
|
||||
"""Spawn a bash process to run *cmd_string*.
|
||||
|
||||
Returns a ProcessHandle (subprocess.Popen or _ThreadedProcessHandle).
|
||||
Must be overridden by every backend.
|
||||
"""
|
||||
raise NotImplementedError(f"{type(self).__name__} must implement _run_bash()")
|
||||
|
||||
@abstractmethod
|
||||
def cleanup(self):
|
||||
"""Release backend resources (container, instance, connection)."""
|
||||
...
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Session snapshot (init_session)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def init_session(self):
|
||||
"""Capture login shell environment into a snapshot file.
|
||||
|
||||
Called once after backend construction. On success, sets
|
||||
``_snapshot_ready = True`` so subsequent commands source the snapshot
|
||||
instead of running with ``bash -l``.
|
||||
"""
|
||||
# Full capture: env vars, functions (filtered), aliases, shell options.
|
||||
bootstrap = (
|
||||
f"export -p > {self._snapshot_path}\n"
|
||||
f"declare -f | grep -vE '^_[^_]' >> {self._snapshot_path}\n"
|
||||
f"alias -p >> {self._snapshot_path}\n"
|
||||
f"echo 'shopt -s expand_aliases' >> {self._snapshot_path}\n"
|
||||
f"echo 'set +e' >> {self._snapshot_path}\n"
|
||||
f"echo 'set +u' >> {self._snapshot_path}\n"
|
||||
f"pwd -P > {self._cwd_file} 2>/dev/null || true\n"
|
||||
f"printf '\\n{self._cwd_marker}%s{self._cwd_marker}\\n' \"$(pwd -P)\"\n"
|
||||
)
|
||||
try:
|
||||
proc = self._run_bash(bootstrap, login=True, timeout=self._snapshot_timeout)
|
||||
result = self._wait_for_process(proc, timeout=self._snapshot_timeout)
|
||||
self._snapshot_ready = True
|
||||
self._update_cwd(result)
|
||||
logger.info(
|
||||
"Session snapshot created (session=%s, cwd=%s)",
|
||||
self._session_id,
|
||||
self.cwd,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"init_session failed (session=%s): %s — "
|
||||
"falling back to bash -l per command",
|
||||
self._session_id,
|
||||
exc,
|
||||
)
|
||||
self._snapshot_ready = False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Command wrapping
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _wrap_command(self, command: str, cwd: str) -> str:
|
||||
"""Build the full bash script that sources snapshot, cd's, runs command,
|
||||
re-dumps env vars, and emits CWD markers."""
|
||||
escaped = command.replace("'", "'\\''")
|
||||
|
||||
parts = []
|
||||
|
||||
# Source snapshot (env vars from previous commands)
|
||||
if self._snapshot_ready:
|
||||
parts.append(f"source {self._snapshot_path} 2>/dev/null || true")
|
||||
|
||||
# cd to working directory — let bash expand ~ natively
|
||||
quoted_cwd = (
|
||||
shlex.quote(cwd) if cwd != "~" and not cwd.startswith("~/") else cwd
|
||||
)
|
||||
parts.append(f"cd {quoted_cwd} || exit 126")
|
||||
|
||||
# Run the actual command
|
||||
parts.append(f"eval '{escaped}'")
|
||||
parts.append("__hermes_ec=$?")
|
||||
|
||||
# Re-dump env vars to snapshot (last-writer-wins for concurrent calls)
|
||||
if self._snapshot_ready:
|
||||
parts.append(f"export -p > {self._snapshot_path} 2>/dev/null || true")
|
||||
|
||||
# Write CWD to file (local reads this) and stdout marker (remote parses this)
|
||||
parts.append(f"pwd -P > {self._cwd_file} 2>/dev/null || true")
|
||||
# Use a distinct line for the marker. The leading \n ensures
|
||||
# the marker starts on its own line even if the command doesn't
|
||||
# end with a newline (e.g. printf 'exact'). We'll strip this
|
||||
# injected newline in _extract_cwd_from_output.
|
||||
parts.append(
|
||||
f"printf '\\n{self._cwd_marker}%s{self._cwd_marker}\\n' \"$(pwd -P)\""
|
||||
)
|
||||
parts.append("exit $__hermes_ec")
|
||||
|
||||
return "\n".join(parts)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Stdin heredoc embedding (for SDK backends)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _embed_stdin_heredoc(command: str, stdin_data: str) -> str:
|
||||
"""Append stdin_data as a shell heredoc to the command string."""
|
||||
delimiter = f"HERMES_STDIN_{uuid.uuid4().hex[:12]}"
|
||||
return f"{command} << '{delimiter}'\n{stdin_data}\n{delimiter}"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Process lifecycle
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _wait_for_process(self, proc: ProcessHandle, timeout: int = 120) -> dict:
|
||||
"""Poll-based wait with interrupt checking and stdout draining.
|
||||
|
||||
Shared across all backends — not overridden.
|
||||
|
||||
Fires the ``activity_callback`` (if set on this instance) every 10s
|
||||
while the process is running so the gateway's inactivity timeout
|
||||
doesn't kill long-running commands.
|
||||
"""
|
||||
output_chunks: list[str] = []
|
||||
|
||||
def _drain():
|
||||
try:
|
||||
for line in proc.stdout:
|
||||
output_chunks.append(line)
|
||||
except UnicodeDecodeError:
|
||||
output_chunks.clear()
|
||||
output_chunks.append(
|
||||
"[binary output detected — raw bytes not displayable]"
|
||||
)
|
||||
except (ValueError, OSError):
|
||||
pass
|
||||
|
||||
drain_thread = threading.Thread(target=_drain, daemon=True)
|
||||
drain_thread.start()
|
||||
deadline = time.monotonic() + timeout
|
||||
_last_activity_touch = time.monotonic()
|
||||
_ACTIVITY_INTERVAL = 10.0 # seconds between activity touches
|
||||
|
||||
while proc.poll() is None:
|
||||
if is_interrupted():
|
||||
self._kill_process(proc)
|
||||
drain_thread.join(timeout=2)
|
||||
return {
|
||||
"output": "".join(output_chunks) + "\n[Command interrupted]",
|
||||
"returncode": 130,
|
||||
}
|
||||
if time.monotonic() > deadline:
|
||||
self._kill_process(proc)
|
||||
drain_thread.join(timeout=2)
|
||||
partial = "".join(output_chunks)
|
||||
timeout_msg = f"\n[Command timed out after {timeout}s]"
|
||||
return {
|
||||
"output": partial + timeout_msg
|
||||
if partial
|
||||
else timeout_msg.lstrip(),
|
||||
"returncode": 124,
|
||||
}
|
||||
# Periodic activity touch so the gateway knows we're alive
|
||||
_now = time.monotonic()
|
||||
if _now - _last_activity_touch >= _ACTIVITY_INTERVAL:
|
||||
_last_activity_touch = _now
|
||||
_cb = _get_activity_callback()
|
||||
if _cb:
|
||||
try:
|
||||
_elapsed = int(_now - (deadline - timeout))
|
||||
_cb(f"terminal command running ({_elapsed}s elapsed)")
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(0.2)
|
||||
|
||||
drain_thread.join(timeout=5)
|
||||
|
||||
try:
|
||||
proc.stdout.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return {"output": "".join(output_chunks), "returncode": proc.returncode}
|
||||
|
||||
def _kill_process(self, proc: ProcessHandle):
|
||||
"""Terminate a process. Subclasses may override for process-group kill."""
|
||||
try:
|
||||
proc.kill()
|
||||
except (ProcessLookupError, PermissionError, OSError):
|
||||
pass
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# CWD extraction
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _update_cwd(self, result: dict):
|
||||
"""Extract CWD from command output. Override for local file-based read."""
|
||||
self._extract_cwd_from_output(result)
|
||||
|
||||
def _extract_cwd_from_output(self, result: dict):
|
||||
"""Parse the __HERMES_CWD_{session}__ marker from stdout output.
|
||||
|
||||
Updates self.cwd and strips the marker from result["output"].
|
||||
Used by remote backends (Docker, SSH, Modal, Daytona, Singularity).
|
||||
"""
|
||||
output = result.get("output", "")
|
||||
marker = self._cwd_marker
|
||||
last = output.rfind(marker)
|
||||
if last == -1:
|
||||
return
|
||||
|
||||
# Find the opening marker before this closing one
|
||||
search_start = max(0, last - 4096) # CWD path won't be >4KB
|
||||
first = output.rfind(marker, search_start, last)
|
||||
if first == -1 or first == last:
|
||||
return
|
||||
|
||||
cwd_path = output[first + len(marker) : last].strip()
|
||||
if cwd_path:
|
||||
self.cwd = cwd_path
|
||||
|
||||
# Strip the marker line AND the \n we injected before it.
|
||||
# The wrapper emits: printf '\n__MARKER__%s__MARKER__\n'
|
||||
# So the output looks like: <cmd output>\n__MARKER__path__MARKER__\n
|
||||
# We want to remove everything from the injected \n onwards.
|
||||
line_start = output.rfind("\n", 0, first)
|
||||
if line_start == -1:
|
||||
line_start = first
|
||||
line_end = output.find("\n", last + len(marker))
|
||||
line_end = line_end + 1 if line_end != -1 else len(output)
|
||||
|
||||
result["output"] = output[:line_start] + output[line_end:]
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Hooks
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _before_execute(self) -> None:
|
||||
"""Hook called before each command execution.
|
||||
|
||||
Remote backends (SSH, Modal, Daytona) override this to trigger
|
||||
their FileSyncManager. Bind-mount backends (Docker, Singularity)
|
||||
and Local don't need file sync — the host filesystem is directly
|
||||
visible inside the container/process.
|
||||
"""
|
||||
pass
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Unified execute()
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def execute(
|
||||
self,
|
||||
command: str,
|
||||
cwd: str = "",
|
||||
*,
|
||||
timeout: int | None = None,
|
||||
stdin_data: str | None = None,
|
||||
) -> dict:
|
||||
"""Execute a command, return {"output": str, "returncode": int}."""
|
||||
self._before_execute()
|
||||
|
||||
exec_command, sudo_stdin = self._prepare_command(command)
|
||||
effective_timeout = timeout or self.timeout
|
||||
effective_cwd = cwd or self.cwd
|
||||
|
||||
# Merge sudo stdin with caller stdin
|
||||
if sudo_stdin is not None and stdin_data is not None:
|
||||
effective_stdin = sudo_stdin + stdin_data
|
||||
elif sudo_stdin is not None:
|
||||
effective_stdin = sudo_stdin
|
||||
else:
|
||||
effective_stdin = stdin_data
|
||||
|
||||
# Embed stdin as heredoc for backends that need it
|
||||
if effective_stdin and self._stdin_mode == "heredoc":
|
||||
exec_command = self._embed_stdin_heredoc(exec_command, effective_stdin)
|
||||
effective_stdin = None
|
||||
|
||||
wrapped = self._wrap_command(exec_command, effective_cwd)
|
||||
|
||||
# Use login shell if snapshot failed (so user's profile still loads)
|
||||
login = not self._snapshot_ready
|
||||
|
||||
proc = self._run_bash(
|
||||
wrapped, login=login, timeout=effective_timeout, stdin_data=effective_stdin
|
||||
)
|
||||
result = self._wait_for_process(proc, timeout=effective_timeout)
|
||||
self._update_cwd(result)
|
||||
|
||||
return result
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Shared helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def stop(self):
|
||||
"""Alias for cleanup (compat with older callers)."""
|
||||
self.cleanup()
|
||||
|
||||
def __del__(self):
|
||||
try:
|
||||
self.cleanup()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _prepare_command(self, command: str) -> tuple[str, str | None]:
|
||||
"""Transform sudo commands if SUDO_PASSWORD is available."""
|
||||
from tools.terminal_tool import _transform_sudo_command
|
||||
|
||||
return _transform_sudo_command(command)
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
"""Daytona cloud execution environment.
|
||||
|
||||
Uses the Daytona Python SDK to run commands in cloud sandboxes.
|
||||
Supports persistent sandboxes: when enabled, sandboxes are stopped on cleanup
|
||||
and resumed on next creation, preserving the filesystem across sessions.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import math
|
||||
import shlex
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
from tools.environments.base import (
|
||||
BaseEnvironment,
|
||||
_ThreadedProcessHandle,
|
||||
)
|
||||
from tools.environments.file_sync import (
|
||||
FileSyncManager,
|
||||
iter_sync_files,
|
||||
quoted_mkdir_command,
|
||||
quoted_rm_command,
|
||||
unique_parent_dirs,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DaytonaEnvironment(BaseEnvironment):
|
||||
"""Daytona cloud sandbox execution backend.
|
||||
|
||||
Spawn-per-call via _ThreadedProcessHandle wrapping blocking SDK calls.
|
||||
cancel_fn wired to sandbox.stop() for interrupt support.
|
||||
Shell timeout wrapper preserved (SDK timeout unreliable).
|
||||
"""
|
||||
|
||||
_stdin_mode = "heredoc"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image: str,
|
||||
cwd: str = "/home/daytona",
|
||||
timeout: int = 60,
|
||||
cpu: int = 1,
|
||||
memory: int = 5120,
|
||||
disk: int = 10240,
|
||||
persistent_filesystem: bool = True,
|
||||
task_id: str = "default",
|
||||
):
|
||||
requested_cwd = cwd
|
||||
super().__init__(cwd=cwd, timeout=timeout)
|
||||
|
||||
from daytona import (
|
||||
Daytona,
|
||||
CreateSandboxFromImageParams,
|
||||
DaytonaError,
|
||||
Resources,
|
||||
SandboxState,
|
||||
)
|
||||
|
||||
self._persistent = persistent_filesystem
|
||||
self._task_id = task_id
|
||||
self._SandboxState = SandboxState
|
||||
self._daytona = Daytona()
|
||||
self._sandbox = None
|
||||
self._lock = threading.Lock()
|
||||
|
||||
memory_gib = max(1, math.ceil(memory / 1024))
|
||||
disk_gib = max(1, math.ceil(disk / 1024))
|
||||
if disk_gib > 10:
|
||||
logger.warning(
|
||||
"Daytona: requested disk (%dGB) exceeds platform limit (10GB). "
|
||||
"Capping to 10GB.", disk_gib,
|
||||
)
|
||||
disk_gib = 10
|
||||
resources = Resources(cpu=cpu, memory=memory_gib, disk=disk_gib)
|
||||
|
||||
labels = {"hermes_task_id": task_id}
|
||||
sandbox_name = f"hermes-{task_id}"
|
||||
|
||||
if self._persistent:
|
||||
try:
|
||||
self._sandbox = self._daytona.get(sandbox_name)
|
||||
self._sandbox.start()
|
||||
logger.info("Daytona: resumed sandbox %s for task %s",
|
||||
self._sandbox.id, task_id)
|
||||
except DaytonaError:
|
||||
self._sandbox = None
|
||||
except Exception as e:
|
||||
logger.warning("Daytona: failed to resume sandbox for task %s: %s",
|
||||
task_id, e)
|
||||
self._sandbox = None
|
||||
|
||||
if self._sandbox is None:
|
||||
try:
|
||||
page = self._daytona.list(labels=labels, page=1, limit=1)
|
||||
if page.items:
|
||||
self._sandbox = page.items[0]
|
||||
self._sandbox.start()
|
||||
logger.info("Daytona: resumed legacy sandbox %s for task %s",
|
||||
self._sandbox.id, task_id)
|
||||
except Exception as e:
|
||||
logger.debug("Daytona: no legacy sandbox found for task %s: %s",
|
||||
task_id, e)
|
||||
self._sandbox = None
|
||||
|
||||
if self._sandbox is None:
|
||||
self._sandbox = self._daytona.create(
|
||||
CreateSandboxFromImageParams(
|
||||
image=image,
|
||||
name=sandbox_name,
|
||||
labels=labels,
|
||||
auto_stop_interval=0,
|
||||
resources=resources,
|
||||
)
|
||||
)
|
||||
logger.info("Daytona: created sandbox %s for task %s",
|
||||
self._sandbox.id, task_id)
|
||||
|
||||
# Detect remote home dir
|
||||
self._remote_home = "/root"
|
||||
try:
|
||||
home = self._sandbox.process.exec("echo $HOME").result.strip()
|
||||
if home:
|
||||
self._remote_home = home
|
||||
if requested_cwd in ("~", "/home/daytona"):
|
||||
self.cwd = home
|
||||
except Exception:
|
||||
pass
|
||||
logger.info("Daytona: resolved home to %s, cwd to %s", self._remote_home, self.cwd)
|
||||
|
||||
self._sync_manager = FileSyncManager(
|
||||
get_files_fn=lambda: iter_sync_files(f"{self._remote_home}/.hermes"),
|
||||
upload_fn=self._daytona_upload,
|
||||
delete_fn=self._daytona_delete,
|
||||
bulk_upload_fn=self._daytona_bulk_upload,
|
||||
)
|
||||
self._sync_manager.sync(force=True)
|
||||
self.init_session()
|
||||
|
||||
def _daytona_upload(self, host_path: str, remote_path: str) -> None:
|
||||
"""Upload a single file via Daytona SDK."""
|
||||
parent = str(Path(remote_path).parent)
|
||||
self._sandbox.process.exec(f"mkdir -p {parent}")
|
||||
self._sandbox.fs.upload_file(host_path, remote_path)
|
||||
|
||||
def _daytona_bulk_upload(self, files: list[tuple[str, str]]) -> None:
|
||||
"""Upload many files in a single HTTP call via Daytona SDK.
|
||||
|
||||
Uses ``sandbox.fs.upload_files()`` which batches all files into one
|
||||
multipart POST, avoiding per-file TLS/HTTP overhead (~580 files
|
||||
goes from ~5 min to <2 s).
|
||||
"""
|
||||
from daytona.common.filesystem import FileUpload
|
||||
|
||||
if not files:
|
||||
return
|
||||
|
||||
parents = unique_parent_dirs(files)
|
||||
if parents:
|
||||
self._sandbox.process.exec(quoted_mkdir_command(parents))
|
||||
|
||||
uploads = [
|
||||
FileUpload(source=host_path, destination=remote_path)
|
||||
for host_path, remote_path in files
|
||||
]
|
||||
self._sandbox.fs.upload_files(uploads)
|
||||
|
||||
def _daytona_delete(self, remote_paths: list[str]) -> None:
|
||||
"""Batch-delete remote files via SDK exec."""
|
||||
self._sandbox.process.exec(quoted_rm_command(remote_paths))
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Sandbox lifecycle
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _ensure_sandbox_ready(self) -> None:
|
||||
"""Restart sandbox if it was stopped (e.g., by a previous interrupt)."""
|
||||
self._sandbox.refresh_data()
|
||||
if self._sandbox.state in (self._SandboxState.STOPPED, self._SandboxState.ARCHIVED):
|
||||
self._sandbox.start()
|
||||
logger.info("Daytona: restarted sandbox %s", self._sandbox.id)
|
||||
|
||||
def _before_execute(self) -> None:
|
||||
"""Ensure sandbox is ready, then sync files via FileSyncManager."""
|
||||
with self._lock:
|
||||
self._ensure_sandbox_ready()
|
||||
self._sync_manager.sync()
|
||||
|
||||
def _run_bash(self, cmd_string: str, *, login: bool = False,
|
||||
timeout: int = 120,
|
||||
stdin_data: str | None = None):
|
||||
"""Return a _ThreadedProcessHandle wrapping a blocking Daytona SDK call."""
|
||||
sandbox = self._sandbox
|
||||
lock = self._lock
|
||||
|
||||
def cancel():
|
||||
with lock:
|
||||
try:
|
||||
sandbox.stop()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if login:
|
||||
shell_cmd = f"bash -l -c {shlex.quote(cmd_string)}"
|
||||
else:
|
||||
shell_cmd = f"bash -c {shlex.quote(cmd_string)}"
|
||||
|
||||
def exec_fn() -> tuple[str, int]:
|
||||
response = sandbox.process.exec(shell_cmd, timeout=timeout)
|
||||
return (response.result or "", response.exit_code)
|
||||
|
||||
return _ThreadedProcessHandle(exec_fn, cancel_fn=cancel)
|
||||
|
||||
def cleanup(self):
|
||||
with self._lock:
|
||||
if self._sandbox is None:
|
||||
return
|
||||
try:
|
||||
if self._persistent:
|
||||
self._sandbox.stop()
|
||||
logger.info("Daytona: stopped sandbox %s (filesystem preserved)",
|
||||
self._sandbox.id)
|
||||
else:
|
||||
self._daytona.delete(self._sandbox)
|
||||
logger.info("Daytona: deleted sandbox %s", self._sandbox.id)
|
||||
except Exception as e:
|
||||
logger.warning("Daytona: cleanup failed: %s", e)
|
||||
self._sandbox = None
|
||||
@@ -0,0 +1,560 @@
|
||||
"""Docker execution environment for sandboxed command execution.
|
||||
|
||||
Security hardened (cap-drop ALL, no-new-privileges, PID limits),
|
||||
configurable resource limits (CPU, memory, disk), and optional filesystem
|
||||
persistence via bind mounts.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import uuid
|
||||
from typing import Optional
|
||||
|
||||
from tools.environments.base import BaseEnvironment, _popen_bash
|
||||
from tools.environments.local import _HERMES_PROVIDER_ENV_BLOCKLIST
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Common Docker Desktop install paths checked when 'docker' is not in PATH.
|
||||
# macOS Intel: /usr/local/bin, macOS Apple Silicon (Homebrew): /opt/homebrew/bin,
|
||||
# Docker Desktop app bundle: /Applications/Docker.app/Contents/Resources/bin
|
||||
_DOCKER_SEARCH_PATHS = [
|
||||
"/usr/local/bin/docker",
|
||||
"/opt/homebrew/bin/docker",
|
||||
"/Applications/Docker.app/Contents/Resources/bin/docker",
|
||||
]
|
||||
|
||||
_docker_executable: Optional[str] = None # resolved once, cached
|
||||
_ENV_VAR_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
||||
|
||||
|
||||
def _normalize_forward_env_names(forward_env: list[str] | None) -> list[str]:
|
||||
"""Return a deduplicated list of valid environment variable names."""
|
||||
normalized: list[str] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
for item in forward_env or []:
|
||||
if not isinstance(item, str):
|
||||
logger.warning("Ignoring non-string docker_forward_env entry: %r", item)
|
||||
continue
|
||||
|
||||
key = item.strip()
|
||||
if not key:
|
||||
continue
|
||||
if not _ENV_VAR_NAME_RE.match(key):
|
||||
logger.warning("Ignoring invalid docker_forward_env entry: %r", item)
|
||||
continue
|
||||
if key in seen:
|
||||
continue
|
||||
|
||||
seen.add(key)
|
||||
normalized.append(key)
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
def _normalize_env_dict(env: dict | None) -> dict[str, str]:
|
||||
"""Validate and normalize a docker_env dict to {str: str}.
|
||||
|
||||
Filters out entries with invalid variable names or non-string values.
|
||||
"""
|
||||
if not env:
|
||||
return {}
|
||||
if not isinstance(env, dict):
|
||||
logger.warning("docker_env is not a dict: %r", env)
|
||||
return {}
|
||||
|
||||
normalized: dict[str, str] = {}
|
||||
for key, value in env.items():
|
||||
if not isinstance(key, str) or not _ENV_VAR_NAME_RE.match(key.strip()):
|
||||
logger.warning("Ignoring invalid docker_env key: %r", key)
|
||||
continue
|
||||
key = key.strip()
|
||||
if not isinstance(value, str):
|
||||
# Coerce simple scalar types (int, bool, float) to string;
|
||||
# reject complex types.
|
||||
if isinstance(value, (int, float, bool)):
|
||||
value = str(value)
|
||||
else:
|
||||
logger.warning("Ignoring non-string docker_env value for %r: %r", key, value)
|
||||
continue
|
||||
normalized[key] = value
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
def _load_hermes_env_vars() -> dict[str, str]:
|
||||
"""Load ~/.hermes/.env values without failing Docker command execution."""
|
||||
try:
|
||||
from hermes_cli.config import load_env
|
||||
|
||||
return load_env() or {}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def find_docker() -> Optional[str]:
|
||||
"""Locate the docker CLI binary.
|
||||
|
||||
Checks ``shutil.which`` first (respects PATH), then probes well-known
|
||||
install locations on macOS where Docker Desktop may not be in PATH
|
||||
(e.g. when running as a gateway service via launchd).
|
||||
|
||||
Returns the absolute path, or ``None`` if docker cannot be found.
|
||||
"""
|
||||
global _docker_executable
|
||||
if _docker_executable is not None:
|
||||
return _docker_executable
|
||||
|
||||
found = shutil.which("docker")
|
||||
if found:
|
||||
_docker_executable = found
|
||||
return found
|
||||
|
||||
for path in _DOCKER_SEARCH_PATHS:
|
||||
if os.path.isfile(path) and os.access(path, os.X_OK):
|
||||
_docker_executable = path
|
||||
logger.info("Found docker at non-PATH location: %s", path)
|
||||
return path
|
||||
|
||||
return None
|
||||
|
||||
|
||||
# Security flags applied to every container.
|
||||
# The container itself is the security boundary (isolated from host).
|
||||
# We drop all capabilities then add back the minimum needed:
|
||||
# DAC_OVERRIDE - root can write to bind-mounted dirs owned by host user
|
||||
# CHOWN/FOWNER - package managers (pip, npm, apt) need to set file ownership
|
||||
# Block privilege escalation and limit PIDs.
|
||||
# /tmp is size-limited and nosuid but allows exec (needed by pip/npm builds).
|
||||
_SECURITY_ARGS = [
|
||||
"--cap-drop", "ALL",
|
||||
"--cap-add", "DAC_OVERRIDE",
|
||||
"--cap-add", "CHOWN",
|
||||
"--cap-add", "FOWNER",
|
||||
"--security-opt", "no-new-privileges",
|
||||
"--pids-limit", "256",
|
||||
"--tmpfs", "/tmp:rw,nosuid,size=512m",
|
||||
"--tmpfs", "/var/tmp:rw,noexec,nosuid,size=256m",
|
||||
"--tmpfs", "/run:rw,noexec,nosuid,size=64m",
|
||||
]
|
||||
|
||||
|
||||
_storage_opt_ok: Optional[bool] = None # cached result across instances
|
||||
|
||||
|
||||
def _ensure_docker_available() -> None:
|
||||
"""Best-effort check that the docker CLI is available before use.
|
||||
|
||||
Reuses ``find_docker()`` so this preflight stays consistent with the rest of
|
||||
the Docker backend, including known non-PATH Docker Desktop locations.
|
||||
"""
|
||||
docker_exe = find_docker()
|
||||
if not docker_exe:
|
||||
logger.error(
|
||||
"Docker backend selected but no docker executable was found in PATH "
|
||||
"or known install locations. Install Docker Desktop and ensure the "
|
||||
"CLI is available."
|
||||
)
|
||||
raise RuntimeError(
|
||||
"Docker executable not found in PATH or known install locations. "
|
||||
"Install Docker and ensure the 'docker' command is available."
|
||||
)
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[docker_exe, "version"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
logger.error(
|
||||
"Docker backend selected but the resolved docker executable '%s' could "
|
||||
"not be executed.",
|
||||
docker_exe,
|
||||
exc_info=True,
|
||||
)
|
||||
raise RuntimeError(
|
||||
"Docker executable could not be executed. Check your Docker installation."
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.error(
|
||||
"Docker backend selected but '%s version' timed out. "
|
||||
"The Docker daemon may not be running.",
|
||||
docker_exe,
|
||||
exc_info=True,
|
||||
)
|
||||
raise RuntimeError(
|
||||
"Docker daemon is not responding. Ensure Docker is running and try again."
|
||||
)
|
||||
except Exception:
|
||||
logger.error(
|
||||
"Unexpected error while checking Docker availability.",
|
||||
exc_info=True,
|
||||
)
|
||||
raise
|
||||
else:
|
||||
if result.returncode != 0:
|
||||
logger.error(
|
||||
"Docker backend selected but '%s version' failed "
|
||||
"(exit code %d, stderr=%s)",
|
||||
docker_exe,
|
||||
result.returncode,
|
||||
result.stderr.strip(),
|
||||
)
|
||||
raise RuntimeError(
|
||||
"Docker command is available but 'docker version' failed. "
|
||||
"Check your Docker installation."
|
||||
)
|
||||
|
||||
|
||||
class DockerEnvironment(BaseEnvironment):
|
||||
"""Hardened Docker container execution with resource limits and persistence.
|
||||
|
||||
Security: all capabilities dropped, no privilege escalation, PID limits,
|
||||
size-limited tmpfs for scratch dirs. The container itself is the security
|
||||
boundary — the filesystem inside is writable so agents can install packages
|
||||
(pip, npm, apt) as needed. Writable workspace via tmpfs or bind mounts.
|
||||
|
||||
Persistence: when enabled, bind mounts preserve /workspace and /root
|
||||
across container restarts.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image: str,
|
||||
cwd: str = "/root",
|
||||
timeout: int = 60,
|
||||
cpu: float = 0,
|
||||
memory: int = 0,
|
||||
disk: int = 0,
|
||||
persistent_filesystem: bool = False,
|
||||
task_id: str = "default",
|
||||
volumes: list = None,
|
||||
forward_env: list[str] | None = None,
|
||||
env: dict | None = None,
|
||||
network: bool = True,
|
||||
host_cwd: str = None,
|
||||
auto_mount_cwd: bool = False,
|
||||
):
|
||||
if cwd == "~":
|
||||
cwd = "/root"
|
||||
super().__init__(cwd=cwd, timeout=timeout)
|
||||
self._persistent = persistent_filesystem
|
||||
self._task_id = task_id
|
||||
self._forward_env = _normalize_forward_env_names(forward_env)
|
||||
self._env = _normalize_env_dict(env)
|
||||
self._container_id: Optional[str] = None
|
||||
logger.info(f"DockerEnvironment volumes: {volumes}")
|
||||
# Ensure volumes is a list (config.yaml could be malformed)
|
||||
if volumes is not None and not isinstance(volumes, list):
|
||||
logger.warning(f"docker_volumes config is not a list: {volumes!r}")
|
||||
volumes = []
|
||||
|
||||
# Fail fast if Docker is not available.
|
||||
_ensure_docker_available()
|
||||
|
||||
# Build resource limit args
|
||||
resource_args = []
|
||||
if cpu > 0:
|
||||
resource_args.extend(["--cpus", str(cpu)])
|
||||
if memory > 0:
|
||||
resource_args.extend(["--memory", f"{memory}m"])
|
||||
if disk > 0 and sys.platform != "darwin":
|
||||
if self._storage_opt_supported():
|
||||
resource_args.extend(["--storage-opt", f"size={disk}m"])
|
||||
else:
|
||||
logger.warning(
|
||||
"Docker storage driver does not support per-container disk limits "
|
||||
"(requires overlay2 on XFS with pquota). Container will run without disk quota."
|
||||
)
|
||||
if not network:
|
||||
resource_args.append("--network=none")
|
||||
|
||||
# Persistent workspace via bind mounts from a configurable host directory
|
||||
# (TERMINAL_SANDBOX_DIR, default ~/.hermes/sandboxes/). Non-persistent
|
||||
# mode uses tmpfs (ephemeral, fast, gone on cleanup).
|
||||
from tools.environments.base import get_sandbox_dir
|
||||
|
||||
# User-configured volume mounts (from config.yaml docker_volumes)
|
||||
volume_args = []
|
||||
workspace_explicitly_mounted = False
|
||||
for vol in (volumes or []):
|
||||
if not isinstance(vol, str):
|
||||
logger.warning(f"Docker volume entry is not a string: {vol!r}")
|
||||
continue
|
||||
vol = vol.strip()
|
||||
if not vol:
|
||||
continue
|
||||
if ":" in vol:
|
||||
volume_args.extend(["-v", vol])
|
||||
if ":/workspace" in vol:
|
||||
workspace_explicitly_mounted = True
|
||||
else:
|
||||
logger.warning(f"Docker volume '{vol}' missing colon, skipping")
|
||||
|
||||
host_cwd_abs = os.path.abspath(os.path.expanduser(host_cwd)) if host_cwd else ""
|
||||
bind_host_cwd = (
|
||||
auto_mount_cwd
|
||||
and bool(host_cwd_abs)
|
||||
and os.path.isdir(host_cwd_abs)
|
||||
and not workspace_explicitly_mounted
|
||||
)
|
||||
if auto_mount_cwd and host_cwd and not os.path.isdir(host_cwd_abs):
|
||||
logger.debug(f"Skipping docker cwd mount: host_cwd is not a valid directory: {host_cwd}")
|
||||
|
||||
self._workspace_dir: Optional[str] = None
|
||||
self._home_dir: Optional[str] = None
|
||||
writable_args = []
|
||||
if self._persistent:
|
||||
sandbox = get_sandbox_dir() / "docker" / task_id
|
||||
self._home_dir = str(sandbox / "home")
|
||||
os.makedirs(self._home_dir, exist_ok=True)
|
||||
writable_args.extend([
|
||||
"-v", f"{self._home_dir}:/root",
|
||||
])
|
||||
if not bind_host_cwd and not workspace_explicitly_mounted:
|
||||
self._workspace_dir = str(sandbox / "workspace")
|
||||
os.makedirs(self._workspace_dir, exist_ok=True)
|
||||
writable_args.extend([
|
||||
"-v", f"{self._workspace_dir}:/workspace",
|
||||
])
|
||||
else:
|
||||
if not bind_host_cwd and not workspace_explicitly_mounted:
|
||||
writable_args.extend([
|
||||
"--tmpfs", "/workspace:rw,exec,size=10g",
|
||||
])
|
||||
writable_args.extend([
|
||||
"--tmpfs", "/home:rw,exec,size=1g",
|
||||
"--tmpfs", "/root:rw,exec,size=1g",
|
||||
])
|
||||
|
||||
if bind_host_cwd:
|
||||
logger.info(f"Mounting configured host cwd to /workspace: {host_cwd_abs}")
|
||||
volume_args = ["-v", f"{host_cwd_abs}:/workspace", *volume_args]
|
||||
elif workspace_explicitly_mounted:
|
||||
logger.debug("Skipping docker cwd mount: /workspace already mounted by user config")
|
||||
|
||||
# Mount credential files (OAuth tokens, etc.) declared by skills.
|
||||
# Read-only so the container can authenticate but not modify host creds.
|
||||
try:
|
||||
from tools.credential_files import (
|
||||
get_credential_file_mounts,
|
||||
get_skills_directory_mount,
|
||||
get_cache_directory_mounts,
|
||||
)
|
||||
|
||||
for mount_entry in get_credential_file_mounts():
|
||||
volume_args.extend([
|
||||
"-v",
|
||||
f"{mount_entry['host_path']}:{mount_entry['container_path']}:ro",
|
||||
])
|
||||
logger.info(
|
||||
"Docker: mounting credential %s -> %s",
|
||||
mount_entry["host_path"],
|
||||
mount_entry["container_path"],
|
||||
)
|
||||
|
||||
# Mount skill directories (local + external) so skill
|
||||
# scripts/templates are available inside the container.
|
||||
for skills_mount in get_skills_directory_mount():
|
||||
volume_args.extend([
|
||||
"-v",
|
||||
f"{skills_mount['host_path']}:{skills_mount['container_path']}:ro",
|
||||
])
|
||||
logger.info(
|
||||
"Docker: mounting skills dir %s -> %s",
|
||||
skills_mount["host_path"],
|
||||
skills_mount["container_path"],
|
||||
)
|
||||
|
||||
# Mount host-side cache directories (documents, images, audio,
|
||||
# screenshots) so the agent can access uploaded files and other
|
||||
# cached media from inside the container. Read-only — the
|
||||
# container reads these but the host gateway manages writes.
|
||||
for cache_mount in get_cache_directory_mounts():
|
||||
volume_args.extend([
|
||||
"-v",
|
||||
f"{cache_mount['host_path']}:{cache_mount['container_path']}:ro",
|
||||
])
|
||||
logger.info(
|
||||
"Docker: mounting cache dir %s -> %s",
|
||||
cache_mount["host_path"],
|
||||
cache_mount["container_path"],
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug("Docker: could not load credential file mounts: %s", e)
|
||||
|
||||
# Explicit environment variables (docker_env config) — set at container
|
||||
# creation so they're available to all processes (including entrypoint).
|
||||
env_args = []
|
||||
for key in sorted(self._env):
|
||||
env_args.extend(["-e", f"{key}={self._env[key]}"])
|
||||
|
||||
logger.info(f"Docker volume_args: {volume_args}")
|
||||
all_run_args = list(_SECURITY_ARGS) + writable_args + resource_args + volume_args + env_args
|
||||
logger.info(f"Docker run_args: {all_run_args}")
|
||||
|
||||
# Resolve the docker executable once so it works even when
|
||||
# /usr/local/bin is not in PATH (common on macOS gateway/service).
|
||||
self._docker_exe = find_docker() or "docker"
|
||||
|
||||
# Start the container directly via `docker run -d`.
|
||||
container_name = f"hermes-{uuid.uuid4().hex[:8]}"
|
||||
run_cmd = [
|
||||
self._docker_exe, "run", "-d",
|
||||
"--init", # tini/catatonit as PID 1 — reaps zombie children
|
||||
"--name", container_name,
|
||||
"-w", cwd,
|
||||
*all_run_args,
|
||||
image,
|
||||
"sleep", "infinity", # no fixed lifetime — idle reaper handles cleanup
|
||||
]
|
||||
logger.debug(f"Starting container: {' '.join(run_cmd)}")
|
||||
result = subprocess.run(
|
||||
run_cmd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120, # image pull may take a while
|
||||
check=True,
|
||||
)
|
||||
self._container_id = result.stdout.strip()
|
||||
logger.info(f"Started container {container_name} ({self._container_id[:12]})")
|
||||
|
||||
# Build the init-time env forwarding args (used only by init_session
|
||||
# to inject host env vars into the snapshot; subsequent commands get
|
||||
# them from the snapshot file).
|
||||
self._init_env_args = self._build_init_env_args()
|
||||
|
||||
# Initialize session snapshot inside the container
|
||||
self.init_session()
|
||||
|
||||
def _build_init_env_args(self) -> list[str]:
|
||||
"""Build -e KEY=VALUE args for injecting host env vars into init_session.
|
||||
|
||||
These are used once during init_session() so that export -p captures
|
||||
them into the snapshot. Subsequent execute() calls don't need -e flags.
|
||||
"""
|
||||
exec_env: dict[str, str] = dict(self._env)
|
||||
|
||||
explicit_forward_keys = set(self._forward_env)
|
||||
passthrough_keys: set[str] = set()
|
||||
try:
|
||||
from tools.env_passthrough import get_all_passthrough
|
||||
passthrough_keys = set(get_all_passthrough())
|
||||
except Exception:
|
||||
pass
|
||||
# Explicit docker_forward_env entries are an intentional opt-in and must
|
||||
# win over the generic Hermes secret blocklist. Only implicit passthrough
|
||||
# keys are filtered.
|
||||
forward_keys = explicit_forward_keys | (passthrough_keys - _HERMES_PROVIDER_ENV_BLOCKLIST)
|
||||
hermes_env = _load_hermes_env_vars() if forward_keys else {}
|
||||
for key in sorted(forward_keys):
|
||||
value = os.getenv(key)
|
||||
if value is None:
|
||||
value = hermes_env.get(key)
|
||||
if value is not None:
|
||||
exec_env[key] = value
|
||||
|
||||
args = []
|
||||
for key in sorted(exec_env):
|
||||
args.extend(["-e", f"{key}={exec_env[key]}"])
|
||||
return args
|
||||
|
||||
def _run_bash(self, cmd_string: str, *, login: bool = False,
|
||||
timeout: int = 120,
|
||||
stdin_data: str | None = None) -> subprocess.Popen:
|
||||
"""Spawn a bash process inside the Docker container."""
|
||||
assert self._container_id, "Container not started"
|
||||
cmd = [self._docker_exe, "exec"]
|
||||
if stdin_data is not None:
|
||||
cmd.append("-i")
|
||||
|
||||
# Only inject -e env args during init_session (login=True).
|
||||
# Subsequent commands get env vars from the snapshot.
|
||||
if login:
|
||||
cmd.extend(self._init_env_args)
|
||||
|
||||
cmd.extend([self._container_id])
|
||||
|
||||
if login:
|
||||
cmd.extend(["bash", "-l", "-c", cmd_string])
|
||||
else:
|
||||
cmd.extend(["bash", "-c", cmd_string])
|
||||
|
||||
return _popen_bash(cmd, stdin_data)
|
||||
|
||||
@staticmethod
|
||||
def _storage_opt_supported() -> bool:
|
||||
"""Check if Docker's storage driver supports --storage-opt size=.
|
||||
|
||||
Only overlay2 on XFS with pquota supports per-container disk quotas.
|
||||
Ubuntu (and most distros) default to ext4, where this flag errors out.
|
||||
"""
|
||||
global _storage_opt_ok
|
||||
if _storage_opt_ok is not None:
|
||||
return _storage_opt_ok
|
||||
try:
|
||||
docker = find_docker() or "docker"
|
||||
result = subprocess.run(
|
||||
[docker, "info", "--format", "{{.Driver}}"],
|
||||
capture_output=True, text=True, timeout=10,
|
||||
)
|
||||
driver = result.stdout.strip().lower()
|
||||
if driver != "overlay2":
|
||||
_storage_opt_ok = False
|
||||
return False
|
||||
# overlay2 only supports storage-opt on XFS with pquota.
|
||||
# Probe by attempting a dry-ish run — the fastest reliable check.
|
||||
probe = subprocess.run(
|
||||
[docker, "create", "--storage-opt", "size=1m", "hello-world"],
|
||||
capture_output=True, text=True, timeout=15,
|
||||
)
|
||||
if probe.returncode == 0:
|
||||
# Clean up the created container
|
||||
container_id = probe.stdout.strip()
|
||||
if container_id:
|
||||
subprocess.run([docker, "rm", container_id],
|
||||
capture_output=True, timeout=5)
|
||||
_storage_opt_ok = True
|
||||
else:
|
||||
_storage_opt_ok = False
|
||||
except Exception:
|
||||
_storage_opt_ok = False
|
||||
logger.debug("Docker --storage-opt support: %s", _storage_opt_ok)
|
||||
return _storage_opt_ok
|
||||
|
||||
def cleanup(self):
|
||||
"""Stop and remove the container. Bind-mount dirs persist if persistent=True."""
|
||||
if self._container_id:
|
||||
try:
|
||||
# Stop in background so cleanup doesn't block
|
||||
stop_cmd = (
|
||||
f"(timeout 60 {self._docker_exe} stop {self._container_id} || "
|
||||
f"{self._docker_exe} rm -f {self._container_id}) >/dev/null 2>&1 &"
|
||||
)
|
||||
subprocess.Popen(stop_cmd, shell=True)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to stop container %s: %s", self._container_id, e)
|
||||
|
||||
if not self._persistent:
|
||||
# Also schedule removal (stop only leaves it as stopped)
|
||||
try:
|
||||
subprocess.Popen(
|
||||
f"sleep 3 && {self._docker_exe} rm -f {self._container_id} >/dev/null 2>&1 &",
|
||||
shell=True,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
self._container_id = None
|
||||
|
||||
if not self._persistent:
|
||||
for d in (self._workspace_dir, self._home_dir):
|
||||
if d:
|
||||
shutil.rmtree(d, ignore_errors=True)
|
||||
@@ -0,0 +1,168 @@
|
||||
"""Shared file sync manager for remote execution backends.
|
||||
|
||||
Tracks local file changes via mtime+size, detects deletions, and
|
||||
syncs to remote environments transactionally. Used by SSH, Modal,
|
||||
and Daytona. Docker and Singularity use bind mounts (live host FS
|
||||
view) and don't need this.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shlex
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
from tools.environments.base import _file_mtime_key
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SYNC_INTERVAL_SECONDS = 5.0
|
||||
_FORCE_SYNC_ENV = "HERMES_FORCE_FILE_SYNC"
|
||||
|
||||
# Transport callbacks provided by each backend
|
||||
UploadFn = Callable[[str, str], None] # (host_path, remote_path) -> raises on failure
|
||||
BulkUploadFn = Callable[[list[tuple[str, str]]], None] # [(host_path, remote_path), ...] -> raises on failure
|
||||
DeleteFn = Callable[[list[str]], None] # (remote_paths) -> raises on failure
|
||||
GetFilesFn = Callable[[], list[tuple[str, str]]] # () -> [(host_path, remote_path), ...]
|
||||
|
||||
|
||||
def iter_sync_files(container_base: str = "/root/.hermes") -> list[tuple[str, str]]:
|
||||
"""Enumerate all files that should be synced to a remote environment.
|
||||
|
||||
Combines credentials, skills, and cache into a single flat list of
|
||||
(host_path, remote_path) pairs. Credential paths are remapped from
|
||||
the hardcoded /root/.hermes to *container_base* because the remote
|
||||
user's home may differ (e.g. /home/daytona, /home/user).
|
||||
"""
|
||||
# Late import: credential_files imports agent modules that create
|
||||
# circular dependencies if loaded at file_sync module level.
|
||||
from tools.credential_files import (
|
||||
get_credential_file_mounts,
|
||||
iter_cache_files,
|
||||
iter_skills_files,
|
||||
)
|
||||
|
||||
files: list[tuple[str, str]] = []
|
||||
for entry in get_credential_file_mounts():
|
||||
remote = entry["container_path"].replace(
|
||||
"/root/.hermes", container_base, 1
|
||||
)
|
||||
files.append((entry["host_path"], remote))
|
||||
for entry in iter_skills_files(container_base=container_base):
|
||||
files.append((entry["host_path"], entry["container_path"]))
|
||||
for entry in iter_cache_files(container_base=container_base):
|
||||
files.append((entry["host_path"], entry["container_path"]))
|
||||
return files
|
||||
|
||||
|
||||
def quoted_rm_command(remote_paths: list[str]) -> str:
|
||||
"""Build a shell ``rm -f`` command for a batch of remote paths."""
|
||||
return "rm -f " + " ".join(shlex.quote(p) for p in remote_paths)
|
||||
|
||||
|
||||
def quoted_mkdir_command(dirs: list[str]) -> str:
|
||||
"""Build a shell ``mkdir -p`` command for a batch of directories."""
|
||||
return "mkdir -p " + " ".join(shlex.quote(d) for d in dirs)
|
||||
|
||||
|
||||
def unique_parent_dirs(files: list[tuple[str, str]]) -> list[str]:
|
||||
"""Extract sorted unique parent directories from (host, remote) pairs."""
|
||||
return sorted({str(Path(remote).parent) for _, remote in files})
|
||||
|
||||
|
||||
class FileSyncManager:
|
||||
"""Tracks local file changes and syncs to a remote environment.
|
||||
|
||||
Backends instantiate this with transport callbacks (upload, delete)
|
||||
and a file-source callable. The manager handles mtime-based change
|
||||
detection, deletion tracking, rate limiting, and transactional state.
|
||||
|
||||
Not used by bind-mount backends (Docker, Singularity) — those get
|
||||
live host FS views and don't need file sync.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
get_files_fn: GetFilesFn,
|
||||
upload_fn: UploadFn,
|
||||
delete_fn: DeleteFn,
|
||||
sync_interval: float = _SYNC_INTERVAL_SECONDS,
|
||||
bulk_upload_fn: BulkUploadFn | None = None,
|
||||
):
|
||||
self._get_files_fn = get_files_fn
|
||||
self._upload_fn = upload_fn
|
||||
self._bulk_upload_fn = bulk_upload_fn
|
||||
self._delete_fn = delete_fn
|
||||
self._synced_files: dict[str, tuple[float, int]] = {} # remote_path -> (mtime, size)
|
||||
self._last_sync_time: float = 0.0 # monotonic; 0 ensures first sync runs
|
||||
self._sync_interval = sync_interval
|
||||
|
||||
def sync(self, *, force: bool = False) -> None:
|
||||
"""Run a sync cycle: upload changed files, delete removed files.
|
||||
|
||||
Rate-limited to once per ``sync_interval`` unless *force* is True
|
||||
or ``HERMES_FORCE_FILE_SYNC=1`` is set.
|
||||
|
||||
Transactional: state only committed if ALL operations succeed.
|
||||
On failure, state rolls back so the next cycle retries everything.
|
||||
"""
|
||||
if not force and not os.environ.get(_FORCE_SYNC_ENV):
|
||||
now = time.monotonic()
|
||||
if now - self._last_sync_time < self._sync_interval:
|
||||
return
|
||||
|
||||
current_files = self._get_files_fn()
|
||||
current_remote_paths = {remote for _, remote in current_files}
|
||||
|
||||
# --- Uploads: new or changed files ---
|
||||
to_upload: list[tuple[str, str]] = []
|
||||
new_files = dict(self._synced_files)
|
||||
for host_path, remote_path in current_files:
|
||||
file_key = _file_mtime_key(host_path)
|
||||
if file_key is None:
|
||||
continue
|
||||
if self._synced_files.get(remote_path) == file_key:
|
||||
continue
|
||||
to_upload.append((host_path, remote_path))
|
||||
new_files[remote_path] = file_key
|
||||
|
||||
# --- Deletes: synced paths no longer in current set ---
|
||||
to_delete = [p for p in self._synced_files if p not in current_remote_paths]
|
||||
|
||||
if not to_upload and not to_delete:
|
||||
self._last_sync_time = time.monotonic()
|
||||
return
|
||||
|
||||
# Snapshot for rollback (only when there's work to do)
|
||||
prev_files = dict(self._synced_files)
|
||||
|
||||
if to_upload:
|
||||
logger.debug("file_sync: uploading %d file(s)", len(to_upload))
|
||||
if to_delete:
|
||||
logger.debug("file_sync: deleting %d stale remote file(s)", len(to_delete))
|
||||
|
||||
try:
|
||||
if to_upload and self._bulk_upload_fn is not None:
|
||||
self._bulk_upload_fn(to_upload)
|
||||
logger.debug("file_sync: bulk-uploaded %d file(s)", len(to_upload))
|
||||
else:
|
||||
for host_path, remote_path in to_upload:
|
||||
self._upload_fn(host_path, remote_path)
|
||||
logger.debug("file_sync: uploaded %s -> %s", host_path, remote_path)
|
||||
|
||||
if to_delete:
|
||||
self._delete_fn(to_delete)
|
||||
logger.debug("file_sync: deleted %s", to_delete)
|
||||
|
||||
# --- Commit (all succeeded) ---
|
||||
for p in to_delete:
|
||||
new_files.pop(p, None)
|
||||
|
||||
self._synced_files = new_files
|
||||
self._last_sync_time = time.monotonic()
|
||||
|
||||
except Exception as exc:
|
||||
self._synced_files = prev_files
|
||||
self._last_sync_time = time.monotonic()
|
||||
logger.warning("file_sync: sync failed, rolled back state: %s", exc)
|
||||
@@ -0,0 +1,314 @@
|
||||
"""Local execution environment — spawn-per-call with session snapshot."""
|
||||
|
||||
import os
|
||||
import platform
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
from tools.environments.base import BaseEnvironment, _pipe_stdin
|
||||
|
||||
_IS_WINDOWS = platform.system() == "Windows"
|
||||
|
||||
|
||||
# Hermes-internal env vars that should NOT leak into terminal subprocesses.
|
||||
_HERMES_PROVIDER_ENV_FORCE_PREFIX = "_HERMES_FORCE_"
|
||||
|
||||
|
||||
def _build_provider_env_blocklist() -> frozenset:
|
||||
"""Derive the blocklist from provider, tool, and gateway config."""
|
||||
blocked: set[str] = set()
|
||||
|
||||
try:
|
||||
from hermes_cli.auth import PROVIDER_REGISTRY
|
||||
for pconfig in PROVIDER_REGISTRY.values():
|
||||
blocked.update(pconfig.api_key_env_vars)
|
||||
if pconfig.base_url_env_var:
|
||||
blocked.add(pconfig.base_url_env_var)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
from hermes_cli.config import OPTIONAL_ENV_VARS
|
||||
for name, metadata in OPTIONAL_ENV_VARS.items():
|
||||
category = metadata.get("category")
|
||||
if category in {"tool", "messaging"}:
|
||||
blocked.add(name)
|
||||
elif category == "setting" and metadata.get("password"):
|
||||
blocked.add(name)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
blocked.update({
|
||||
"OPENAI_BASE_URL",
|
||||
"OPENAI_API_KEY",
|
||||
"OPENAI_API_BASE",
|
||||
"OPENAI_ORG_ID",
|
||||
"OPENAI_ORGANIZATION",
|
||||
"OPENROUTER_API_KEY",
|
||||
"ANTHROPIC_BASE_URL",
|
||||
"ANTHROPIC_TOKEN",
|
||||
"CLAUDE_CODE_OAUTH_TOKEN",
|
||||
"LLM_MODEL",
|
||||
"GOOGLE_API_KEY",
|
||||
"DEEPSEEK_API_KEY",
|
||||
"MISTRAL_API_KEY",
|
||||
"GROQ_API_KEY",
|
||||
"TOGETHER_API_KEY",
|
||||
"PERPLEXITY_API_KEY",
|
||||
"COHERE_API_KEY",
|
||||
"FIREWORKS_API_KEY",
|
||||
"XAI_API_KEY",
|
||||
"HELICONE_API_KEY",
|
||||
"PARALLEL_API_KEY",
|
||||
"FIRECRAWL_API_KEY",
|
||||
"FIRECRAWL_API_URL",
|
||||
"TELEGRAM_HOME_CHANNEL",
|
||||
"TELEGRAM_HOME_CHANNEL_NAME",
|
||||
"DISCORD_HOME_CHANNEL",
|
||||
"DISCORD_HOME_CHANNEL_NAME",
|
||||
"DISCORD_REQUIRE_MENTION",
|
||||
"DISCORD_FREE_RESPONSE_CHANNELS",
|
||||
"DISCORD_AUTO_THREAD",
|
||||
"SLACK_HOME_CHANNEL",
|
||||
"SLACK_HOME_CHANNEL_NAME",
|
||||
"SLACK_ALLOWED_USERS",
|
||||
"WHATSAPP_ENABLED",
|
||||
"WHATSAPP_MODE",
|
||||
"WHATSAPP_ALLOWED_USERS",
|
||||
"SIGNAL_HTTP_URL",
|
||||
"SIGNAL_ACCOUNT",
|
||||
"SIGNAL_ALLOWED_USERS",
|
||||
"SIGNAL_GROUP_ALLOWED_USERS",
|
||||
"SIGNAL_HOME_CHANNEL",
|
||||
"SIGNAL_HOME_CHANNEL_NAME",
|
||||
"SIGNAL_IGNORE_STORIES",
|
||||
"HASS_TOKEN",
|
||||
"HASS_URL",
|
||||
"EMAIL_ADDRESS",
|
||||
"EMAIL_PASSWORD",
|
||||
"EMAIL_IMAP_HOST",
|
||||
"EMAIL_SMTP_HOST",
|
||||
"EMAIL_HOME_ADDRESS",
|
||||
"EMAIL_HOME_ADDRESS_NAME",
|
||||
"GATEWAY_ALLOWED_USERS",
|
||||
"GH_TOKEN",
|
||||
"GITHUB_APP_ID",
|
||||
"GITHUB_APP_PRIVATE_KEY_PATH",
|
||||
"GITHUB_APP_INSTALLATION_ID",
|
||||
"MODAL_TOKEN_ID",
|
||||
"MODAL_TOKEN_SECRET",
|
||||
"DAYTONA_API_KEY",
|
||||
})
|
||||
return frozenset(blocked)
|
||||
|
||||
|
||||
_HERMES_PROVIDER_ENV_BLOCKLIST = _build_provider_env_blocklist()
|
||||
|
||||
|
||||
def _sanitize_subprocess_env(base_env: dict | None, extra_env: dict | None = None) -> dict:
|
||||
"""Filter Hermes-managed secrets from a subprocess environment."""
|
||||
try:
|
||||
from tools.env_passthrough import is_env_passthrough as _is_passthrough
|
||||
except Exception:
|
||||
_is_passthrough = lambda _: False # noqa: E731
|
||||
|
||||
sanitized: dict[str, str] = {}
|
||||
|
||||
for key, value in (base_env or {}).items():
|
||||
if key.startswith(_HERMES_PROVIDER_ENV_FORCE_PREFIX):
|
||||
continue
|
||||
if key not in _HERMES_PROVIDER_ENV_BLOCKLIST or _is_passthrough(key):
|
||||
sanitized[key] = value
|
||||
|
||||
for key, value in (extra_env or {}).items():
|
||||
if key.startswith(_HERMES_PROVIDER_ENV_FORCE_PREFIX):
|
||||
real_key = key[len(_HERMES_PROVIDER_ENV_FORCE_PREFIX):]
|
||||
sanitized[real_key] = value
|
||||
elif key not in _HERMES_PROVIDER_ENV_BLOCKLIST or _is_passthrough(key):
|
||||
sanitized[key] = value
|
||||
|
||||
# Per-profile HOME isolation for background processes (same as _make_run_env).
|
||||
from hermes_constants import get_subprocess_home
|
||||
_profile_home = get_subprocess_home()
|
||||
if _profile_home:
|
||||
sanitized["HOME"] = _profile_home
|
||||
|
||||
return sanitized
|
||||
|
||||
|
||||
def _find_bash() -> str:
|
||||
"""Find bash for command execution."""
|
||||
if not _IS_WINDOWS:
|
||||
return (
|
||||
shutil.which("bash")
|
||||
or ("/usr/bin/bash" if os.path.isfile("/usr/bin/bash") else None)
|
||||
or ("/bin/bash" if os.path.isfile("/bin/bash") else None)
|
||||
or os.environ.get("SHELL")
|
||||
or "/bin/sh"
|
||||
)
|
||||
|
||||
custom = os.environ.get("HERMES_GIT_BASH_PATH")
|
||||
if custom and os.path.isfile(custom):
|
||||
return custom
|
||||
|
||||
found = shutil.which("bash")
|
||||
if found:
|
||||
return found
|
||||
|
||||
for candidate in (
|
||||
os.path.join(os.environ.get("ProgramFiles", r"C:\Program Files"), "Git", "bin", "bash.exe"),
|
||||
os.path.join(os.environ.get("ProgramFiles(x86)", r"C:\Program Files (x86)"), "Git", "bin", "bash.exe"),
|
||||
os.path.join(os.environ.get("LOCALAPPDATA", ""), "Programs", "Git", "bin", "bash.exe"),
|
||||
):
|
||||
if candidate and os.path.isfile(candidate):
|
||||
return candidate
|
||||
|
||||
raise RuntimeError(
|
||||
"Git Bash not found. Hermes Agent requires Git for Windows on Windows.\n"
|
||||
"Install it from: https://git-scm.com/download/win\n"
|
||||
"Or set HERMES_GIT_BASH_PATH to your bash.exe location."
|
||||
)
|
||||
|
||||
|
||||
# Backward compat — process_registry.py imports this name
|
||||
_find_shell = _find_bash
|
||||
|
||||
|
||||
# Standard PATH entries for environments with minimal PATH.
|
||||
_SANE_PATH = (
|
||||
"/opt/homebrew/bin:/opt/homebrew/sbin:"
|
||||
"/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||||
)
|
||||
|
||||
|
||||
def _make_run_env(env: dict) -> dict:
|
||||
"""Build a run environment with a sane PATH and provider-var stripping."""
|
||||
try:
|
||||
from tools.env_passthrough import is_env_passthrough as _is_passthrough
|
||||
except Exception:
|
||||
_is_passthrough = lambda _: False # noqa: E731
|
||||
|
||||
merged = dict(os.environ | env)
|
||||
run_env = {}
|
||||
for k, v in merged.items():
|
||||
if k.startswith(_HERMES_PROVIDER_ENV_FORCE_PREFIX):
|
||||
real_key = k[len(_HERMES_PROVIDER_ENV_FORCE_PREFIX):]
|
||||
run_env[real_key] = v
|
||||
elif k not in _HERMES_PROVIDER_ENV_BLOCKLIST or _is_passthrough(k):
|
||||
run_env[k] = v
|
||||
existing_path = run_env.get("PATH", "")
|
||||
if "/usr/bin" not in existing_path.split(":"):
|
||||
run_env["PATH"] = f"{existing_path}:{_SANE_PATH}" if existing_path else _SANE_PATH
|
||||
|
||||
# Per-profile HOME isolation: redirect system tool configs (git, ssh, gh,
|
||||
# npm …) into {HERMES_HOME}/home/ when that directory exists. Only the
|
||||
# subprocess sees the override — the Python process keeps the real HOME.
|
||||
from hermes_constants import get_subprocess_home
|
||||
_profile_home = get_subprocess_home()
|
||||
if _profile_home:
|
||||
run_env["HOME"] = _profile_home
|
||||
|
||||
return run_env
|
||||
|
||||
|
||||
class LocalEnvironment(BaseEnvironment):
|
||||
"""Run commands directly on the host machine.
|
||||
|
||||
Spawn-per-call: every execute() spawns a fresh bash process.
|
||||
Session snapshot preserves env vars across calls.
|
||||
CWD persists via file-based read after each command.
|
||||
"""
|
||||
|
||||
def __init__(self, cwd: str = "", timeout: int = 60, env: dict = None):
|
||||
super().__init__(cwd=cwd or os.getcwd(), timeout=timeout, env=env)
|
||||
self.init_session()
|
||||
|
||||
def get_temp_dir(self) -> str:
|
||||
"""Return a shell-safe writable temp dir for local execution.
|
||||
|
||||
Termux does not provide /tmp by default, but exposes a POSIX TMPDIR.
|
||||
Prefer POSIX-style env vars when available, keep using /tmp on regular
|
||||
Unix systems, and only fall back to tempfile.gettempdir() when it also
|
||||
resolves to a POSIX path.
|
||||
|
||||
Check the environment configured for this backend first so callers can
|
||||
override the temp root explicitly (for example via terminal.env or a
|
||||
custom TMPDIR), then fall back to the host process environment.
|
||||
"""
|
||||
for env_var in ("TMPDIR", "TMP", "TEMP"):
|
||||
candidate = self.env.get(env_var) or os.environ.get(env_var)
|
||||
if candidate and candidate.startswith("/"):
|
||||
return candidate.rstrip("/") or "/"
|
||||
|
||||
if os.path.isdir("/tmp") and os.access("/tmp", os.W_OK | os.X_OK):
|
||||
return "/tmp"
|
||||
|
||||
candidate = tempfile.gettempdir()
|
||||
if candidate.startswith("/"):
|
||||
return candidate.rstrip("/") or "/"
|
||||
|
||||
return "/tmp"
|
||||
|
||||
def _run_bash(self, cmd_string: str, *, login: bool = False,
|
||||
timeout: int = 120,
|
||||
stdin_data: str | None = None) -> subprocess.Popen:
|
||||
bash = _find_bash()
|
||||
args = [bash, "-l", "-c", cmd_string] if login else [bash, "-c", cmd_string]
|
||||
run_env = _make_run_env(self.env)
|
||||
|
||||
proc = subprocess.Popen(
|
||||
args,
|
||||
text=True,
|
||||
env=run_env,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
stdin=subprocess.PIPE if stdin_data is not None else subprocess.DEVNULL,
|
||||
preexec_fn=None if _IS_WINDOWS else os.setsid,
|
||||
)
|
||||
|
||||
if stdin_data is not None:
|
||||
_pipe_stdin(proc, stdin_data)
|
||||
|
||||
return proc
|
||||
|
||||
def _kill_process(self, proc):
|
||||
"""Kill the entire process group (all children)."""
|
||||
try:
|
||||
if _IS_WINDOWS:
|
||||
proc.terminate()
|
||||
else:
|
||||
pgid = os.getpgid(proc.pid)
|
||||
os.killpg(pgid, signal.SIGTERM)
|
||||
try:
|
||||
proc.wait(timeout=1.0)
|
||||
except subprocess.TimeoutExpired:
|
||||
os.killpg(pgid, signal.SIGKILL)
|
||||
except (ProcessLookupError, PermissionError):
|
||||
try:
|
||||
proc.kill()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _update_cwd(self, result: dict):
|
||||
"""Read CWD from temp file (local-only, no round-trip needed)."""
|
||||
try:
|
||||
cwd_path = open(self._cwd_file).read().strip()
|
||||
if cwd_path:
|
||||
self.cwd = cwd_path
|
||||
except (OSError, FileNotFoundError):
|
||||
pass
|
||||
|
||||
# Still strip the marker from output so it's not visible
|
||||
self._extract_cwd_from_output(result)
|
||||
|
||||
def cleanup(self):
|
||||
"""Clean up temp files."""
|
||||
for f in (self._snapshot_path, self._cwd_file):
|
||||
try:
|
||||
os.unlink(f)
|
||||
except OSError:
|
||||
pass
|
||||
@@ -0,0 +1,282 @@
|
||||
"""Managed Modal environment backed by tool-gateway."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import requests
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from tools.environments.modal_utils import (
|
||||
BaseModalExecutionEnvironment,
|
||||
ModalExecStart,
|
||||
PreparedModalExec,
|
||||
)
|
||||
from tools.managed_tool_gateway import resolve_managed_tool_gateway
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _request_timeout_env(name: str, default: float) -> float:
|
||||
try:
|
||||
value = float(os.getenv(name, str(default)))
|
||||
return value if value > 0 else default
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _ManagedModalExecHandle:
|
||||
exec_id: str
|
||||
|
||||
|
||||
class ManagedModalEnvironment(BaseModalExecutionEnvironment):
|
||||
"""Gateway-owned Modal sandbox with Hermes-compatible execute/cleanup."""
|
||||
|
||||
_CONNECT_TIMEOUT_SECONDS = _request_timeout_env("TERMINAL_MANAGED_MODAL_CONNECT_TIMEOUT_SECONDS", 1.0)
|
||||
_POLL_READ_TIMEOUT_SECONDS = _request_timeout_env("TERMINAL_MANAGED_MODAL_POLL_READ_TIMEOUT_SECONDS", 5.0)
|
||||
_CANCEL_READ_TIMEOUT_SECONDS = _request_timeout_env("TERMINAL_MANAGED_MODAL_CANCEL_READ_TIMEOUT_SECONDS", 5.0)
|
||||
_client_timeout_grace_seconds = 10.0
|
||||
_interrupt_output = "[Command interrupted - Modal sandbox exec cancelled]"
|
||||
_unexpected_error_prefix = "Managed Modal exec failed"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image: str,
|
||||
cwd: str = "/root",
|
||||
timeout: int = 60,
|
||||
modal_sandbox_kwargs: Optional[Dict[str, Any]] = None,
|
||||
persistent_filesystem: bool = True,
|
||||
task_id: str = "default",
|
||||
):
|
||||
super().__init__(cwd=cwd, timeout=timeout)
|
||||
|
||||
self._guard_unsupported_credential_passthrough()
|
||||
|
||||
gateway = resolve_managed_tool_gateway("modal")
|
||||
if gateway is None:
|
||||
raise ValueError("Managed Modal requires a configured tool gateway and Nous user token")
|
||||
|
||||
self._gateway_origin = gateway.gateway_origin.rstrip("/")
|
||||
self._nous_user_token = gateway.nous_user_token
|
||||
self._task_id = task_id
|
||||
self._persistent = persistent_filesystem
|
||||
self._image = image
|
||||
self._sandbox_kwargs = dict(modal_sandbox_kwargs or {})
|
||||
self._create_idempotency_key = str(uuid.uuid4())
|
||||
self._sandbox_id = self._create_sandbox()
|
||||
|
||||
def _start_modal_exec(self, prepared: PreparedModalExec) -> ModalExecStart:
|
||||
exec_id = str(uuid.uuid4())
|
||||
payload: Dict[str, Any] = {
|
||||
"execId": exec_id,
|
||||
"command": prepared.command,
|
||||
"cwd": prepared.cwd,
|
||||
"timeoutMs": int(prepared.timeout * 1000),
|
||||
}
|
||||
if prepared.stdin_data is not None:
|
||||
payload["stdinData"] = prepared.stdin_data
|
||||
|
||||
try:
|
||||
response = self._request(
|
||||
"POST",
|
||||
f"/v1/sandboxes/{self._sandbox_id}/execs",
|
||||
json=payload,
|
||||
timeout=10,
|
||||
)
|
||||
except Exception as exc:
|
||||
return ModalExecStart(
|
||||
immediate_result=self._error_result(f"Managed Modal exec failed: {exc}")
|
||||
)
|
||||
|
||||
if response.status_code >= 400:
|
||||
return ModalExecStart(
|
||||
immediate_result=self._error_result(
|
||||
self._format_error("Managed Modal exec failed", response)
|
||||
)
|
||||
)
|
||||
|
||||
body = response.json()
|
||||
status = body.get("status")
|
||||
if status in {"completed", "failed", "cancelled", "timeout"}:
|
||||
return ModalExecStart(
|
||||
immediate_result=self._result(
|
||||
body.get("output", ""),
|
||||
body.get("returncode", 1),
|
||||
)
|
||||
)
|
||||
|
||||
if body.get("execId") != exec_id:
|
||||
return ModalExecStart(
|
||||
immediate_result=self._error_result(
|
||||
"Managed Modal exec start did not return the expected exec id"
|
||||
)
|
||||
)
|
||||
|
||||
return ModalExecStart(handle=_ManagedModalExecHandle(exec_id=exec_id))
|
||||
|
||||
def _poll_modal_exec(self, handle: _ManagedModalExecHandle) -> dict | None:
|
||||
try:
|
||||
status_response = self._request(
|
||||
"GET",
|
||||
f"/v1/sandboxes/{self._sandbox_id}/execs/{handle.exec_id}",
|
||||
timeout=(self._CONNECT_TIMEOUT_SECONDS, self._POLL_READ_TIMEOUT_SECONDS),
|
||||
)
|
||||
except Exception as exc:
|
||||
return self._error_result(f"Managed Modal exec poll failed: {exc}")
|
||||
|
||||
if status_response.status_code == 404:
|
||||
return self._error_result("Managed Modal exec not found")
|
||||
|
||||
if status_response.status_code >= 400:
|
||||
return self._error_result(
|
||||
self._format_error("Managed Modal exec poll failed", status_response)
|
||||
)
|
||||
|
||||
status_body = status_response.json()
|
||||
status = status_body.get("status")
|
||||
if status in {"completed", "failed", "cancelled", "timeout"}:
|
||||
return self._result(
|
||||
status_body.get("output", ""),
|
||||
status_body.get("returncode", 1),
|
||||
)
|
||||
return None
|
||||
|
||||
def _cancel_modal_exec(self, handle: _ManagedModalExecHandle) -> None:
|
||||
self._cancel_exec(handle.exec_id)
|
||||
|
||||
def _timeout_result_for_modal(self, timeout: int) -> dict:
|
||||
return self._result(f"Managed Modal exec timed out after {timeout}s", 124)
|
||||
|
||||
def cleanup(self):
|
||||
if not getattr(self, "_sandbox_id", None):
|
||||
return
|
||||
|
||||
try:
|
||||
self._request(
|
||||
"POST",
|
||||
f"/v1/sandboxes/{self._sandbox_id}/terminate",
|
||||
json={
|
||||
"snapshotBeforeTerminate": self._persistent,
|
||||
},
|
||||
timeout=60,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("Managed Modal cleanup failed: %s", exc)
|
||||
finally:
|
||||
self._sandbox_id = None
|
||||
|
||||
def _create_sandbox(self) -> str:
|
||||
cpu = self._coerce_number(self._sandbox_kwargs.get("cpu"), 1)
|
||||
memory = self._coerce_number(
|
||||
self._sandbox_kwargs.get("memoryMiB", self._sandbox_kwargs.get("memory")),
|
||||
5120,
|
||||
)
|
||||
disk = self._coerce_number(
|
||||
self._sandbox_kwargs.get("ephemeral_disk", self._sandbox_kwargs.get("diskMiB")),
|
||||
None,
|
||||
)
|
||||
|
||||
create_payload = {
|
||||
"image": self._image,
|
||||
"cwd": self.cwd,
|
||||
"cpu": cpu,
|
||||
"memoryMiB": memory,
|
||||
"timeoutMs": 3_600_000,
|
||||
"idleTimeoutMs": max(300_000, int(self.timeout * 1000)),
|
||||
"persistentFilesystem": self._persistent,
|
||||
"logicalKey": self._task_id,
|
||||
}
|
||||
if disk is not None:
|
||||
create_payload["diskMiB"] = disk
|
||||
|
||||
response = self._request(
|
||||
"POST",
|
||||
"/v1/sandboxes",
|
||||
json=create_payload,
|
||||
timeout=60,
|
||||
extra_headers={
|
||||
"x-idempotency-key": self._create_idempotency_key,
|
||||
},
|
||||
)
|
||||
if response.status_code >= 400:
|
||||
raise RuntimeError(self._format_error("Managed Modal create failed", response))
|
||||
|
||||
body = response.json()
|
||||
sandbox_id = body.get("id")
|
||||
if not isinstance(sandbox_id, str) or not sandbox_id:
|
||||
raise RuntimeError("Managed Modal create did not return a sandbox id")
|
||||
return sandbox_id
|
||||
|
||||
def _guard_unsupported_credential_passthrough(self) -> None:
|
||||
"""Managed Modal does not sync or mount host credential files."""
|
||||
try:
|
||||
from tools.credential_files import get_credential_file_mounts
|
||||
except Exception:
|
||||
return
|
||||
|
||||
mounts = get_credential_file_mounts()
|
||||
if mounts:
|
||||
raise ValueError(
|
||||
"Managed Modal does not support host credential-file passthrough. "
|
||||
"Use TERMINAL_MODAL_MODE=direct when skills or config require "
|
||||
"credential files inside the sandbox."
|
||||
)
|
||||
|
||||
def _request(self, method: str, path: str, *,
|
||||
json: Dict[str, Any] | None = None,
|
||||
timeout: int = 30,
|
||||
extra_headers: Dict[str, str] | None = None) -> requests.Response:
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self._nous_user_token}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
if extra_headers:
|
||||
headers.update(extra_headers)
|
||||
|
||||
return requests.request(
|
||||
method,
|
||||
f"{self._gateway_origin}{path}",
|
||||
headers=headers,
|
||||
json=json,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
def _cancel_exec(self, exec_id: str) -> None:
|
||||
try:
|
||||
self._request(
|
||||
"POST",
|
||||
f"/v1/sandboxes/{self._sandbox_id}/execs/{exec_id}/cancel",
|
||||
timeout=(self._CONNECT_TIMEOUT_SECONDS, self._CANCEL_READ_TIMEOUT_SECONDS),
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("Managed Modal exec cancel failed: %s", exc)
|
||||
|
||||
@staticmethod
|
||||
def _coerce_number(value: Any, default: float) -> float:
|
||||
try:
|
||||
if value is None:
|
||||
return default
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
@staticmethod
|
||||
def _format_error(prefix: str, response: requests.Response) -> str:
|
||||
try:
|
||||
payload = response.json()
|
||||
if isinstance(payload, dict):
|
||||
message = payload.get("error") or payload.get("message") or payload.get("code")
|
||||
if isinstance(message, str) and message:
|
||||
return f"{prefix}: {message}"
|
||||
return f"{prefix}: {json.dumps(payload, ensure_ascii=False)}"
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
text = response.text.strip()
|
||||
if text:
|
||||
return f"{prefix}: {text}"
|
||||
return f"{prefix}: HTTP {response.status_code}"
|
||||
@@ -0,0 +1,434 @@
|
||||
"""Modal cloud execution environment using the native Modal SDK directly.
|
||||
|
||||
Uses ``Sandbox.create()`` + ``Sandbox.exec()`` instead of the older runtime
|
||||
wrapper, while preserving Hermes' persistent snapshot behavior across sessions.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import logging
|
||||
import shlex
|
||||
import tarfile
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
from hermes_constants import get_hermes_home
|
||||
from tools.environments.base import (
|
||||
BaseEnvironment,
|
||||
_ThreadedProcessHandle,
|
||||
_load_json_store,
|
||||
_save_json_store,
|
||||
)
|
||||
from tools.environments.file_sync import (
|
||||
FileSyncManager,
|
||||
iter_sync_files,
|
||||
quoted_mkdir_command,
|
||||
quoted_rm_command,
|
||||
unique_parent_dirs,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SNAPSHOT_STORE = get_hermes_home() / "modal_snapshots.json"
|
||||
_DIRECT_SNAPSHOT_NAMESPACE = "direct"
|
||||
|
||||
|
||||
def _load_snapshots() -> dict:
|
||||
return _load_json_store(_SNAPSHOT_STORE)
|
||||
|
||||
|
||||
def _save_snapshots(data: dict) -> None:
|
||||
_save_json_store(_SNAPSHOT_STORE, data)
|
||||
|
||||
|
||||
def _direct_snapshot_key(task_id: str) -> str:
|
||||
return f"{_DIRECT_SNAPSHOT_NAMESPACE}:{task_id}"
|
||||
|
||||
|
||||
def _get_snapshot_restore_candidate(task_id: str) -> tuple[str | None, bool]:
|
||||
snapshots = _load_snapshots()
|
||||
namespaced_key = _direct_snapshot_key(task_id)
|
||||
snapshot_id = snapshots.get(namespaced_key)
|
||||
if isinstance(snapshot_id, str) and snapshot_id:
|
||||
return snapshot_id, False
|
||||
legacy_snapshot_id = snapshots.get(task_id)
|
||||
if isinstance(legacy_snapshot_id, str) and legacy_snapshot_id:
|
||||
return legacy_snapshot_id, True
|
||||
return None, False
|
||||
|
||||
|
||||
def _store_direct_snapshot(task_id: str, snapshot_id: str) -> None:
|
||||
snapshots = _load_snapshots()
|
||||
snapshots[_direct_snapshot_key(task_id)] = snapshot_id
|
||||
snapshots.pop(task_id, None)
|
||||
_save_snapshots(snapshots)
|
||||
|
||||
|
||||
def _delete_direct_snapshot(task_id: str, snapshot_id: str | None = None) -> None:
|
||||
snapshots = _load_snapshots()
|
||||
updated = False
|
||||
for key in (_direct_snapshot_key(task_id), task_id):
|
||||
value = snapshots.get(key)
|
||||
if value is None:
|
||||
continue
|
||||
if snapshot_id is None or value == snapshot_id:
|
||||
snapshots.pop(key, None)
|
||||
updated = True
|
||||
if updated:
|
||||
_save_snapshots(snapshots)
|
||||
|
||||
|
||||
def _resolve_modal_image(image_spec: Any) -> Any:
|
||||
"""Convert registry references or snapshot ids into Modal image objects.
|
||||
|
||||
Includes add_python support for ubuntu/debian images (absorbed from PR 4511).
|
||||
"""
|
||||
import modal as _modal
|
||||
|
||||
if not isinstance(image_spec, str):
|
||||
return image_spec
|
||||
|
||||
if image_spec.startswith("im-"):
|
||||
return _modal.Image.from_id(image_spec)
|
||||
|
||||
# PR 4511: add python to ubuntu/debian images that don't have it
|
||||
lower = image_spec.lower()
|
||||
add_python = any(base in lower for base in ("ubuntu", "debian"))
|
||||
|
||||
setup_commands = [
|
||||
"RUN rm -rf /usr/local/lib/python*/site-packages/pip* 2>/dev/null; "
|
||||
"python -m ensurepip --upgrade --default-pip 2>/dev/null || true",
|
||||
]
|
||||
if add_python:
|
||||
setup_commands.insert(0,
|
||||
"RUN apt-get update -qq && apt-get install -y -qq python3 python3-venv > /dev/null 2>&1 || true"
|
||||
)
|
||||
|
||||
return _modal.Image.from_registry(
|
||||
image_spec,
|
||||
setup_dockerfile_commands=setup_commands,
|
||||
)
|
||||
|
||||
|
||||
class _AsyncWorker:
|
||||
"""Background thread with its own event loop for async-safe Modal calls."""
|
||||
|
||||
def __init__(self):
|
||||
self._loop: Optional[asyncio.AbstractEventLoop] = None
|
||||
self._thread: Optional[threading.Thread] = None
|
||||
self._started = threading.Event()
|
||||
|
||||
def start(self):
|
||||
self._thread = threading.Thread(target=self._run_loop, daemon=True)
|
||||
self._thread.start()
|
||||
self._started.wait(timeout=30)
|
||||
|
||||
def _run_loop(self):
|
||||
self._loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(self._loop)
|
||||
self._started.set()
|
||||
self._loop.run_forever()
|
||||
|
||||
def run_coroutine(self, coro, timeout=600):
|
||||
if self._loop is None or self._loop.is_closed():
|
||||
raise RuntimeError("AsyncWorker loop is not running")
|
||||
future = asyncio.run_coroutine_threadsafe(coro, self._loop)
|
||||
return future.result(timeout=timeout)
|
||||
|
||||
def stop(self):
|
||||
if self._loop and self._loop.is_running():
|
||||
self._loop.call_soon_threadsafe(self._loop.stop)
|
||||
if self._thread:
|
||||
self._thread.join(timeout=10)
|
||||
|
||||
|
||||
class ModalEnvironment(BaseEnvironment):
|
||||
"""Modal cloud execution via native Modal sandboxes.
|
||||
|
||||
Spawn-per-call via _ThreadedProcessHandle wrapping async SDK calls.
|
||||
cancel_fn wired to sandbox.terminate for interrupt support.
|
||||
"""
|
||||
|
||||
_stdin_mode = "heredoc"
|
||||
_snapshot_timeout = 60 # Modal cold starts can be slow
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image: str,
|
||||
cwd: str = "/root",
|
||||
timeout: int = 60,
|
||||
modal_sandbox_kwargs: Optional[dict[str, Any]] = None,
|
||||
persistent_filesystem: bool = True,
|
||||
task_id: str = "default",
|
||||
):
|
||||
super().__init__(cwd=cwd, timeout=timeout)
|
||||
|
||||
self._persistent = persistent_filesystem
|
||||
self._task_id = task_id
|
||||
self._sandbox = None
|
||||
self._app = None
|
||||
self._worker = _AsyncWorker()
|
||||
self._sync_manager: FileSyncManager | None = None # initialized after sandbox creation
|
||||
|
||||
sandbox_kwargs = dict(modal_sandbox_kwargs or {})
|
||||
|
||||
restored_snapshot_id = None
|
||||
restored_from_legacy_key = False
|
||||
if self._persistent:
|
||||
restored_snapshot_id, restored_from_legacy_key = _get_snapshot_restore_candidate(
|
||||
self._task_id
|
||||
)
|
||||
if restored_snapshot_id:
|
||||
logger.info("Modal: restoring from snapshot %s", restored_snapshot_id[:20])
|
||||
|
||||
import modal as _modal
|
||||
|
||||
cred_mounts = []
|
||||
try:
|
||||
from tools.credential_files import (
|
||||
get_credential_file_mounts,
|
||||
iter_skills_files,
|
||||
iter_cache_files,
|
||||
)
|
||||
|
||||
for mount_entry in get_credential_file_mounts():
|
||||
cred_mounts.append(
|
||||
_modal.Mount.from_local_file(
|
||||
mount_entry["host_path"],
|
||||
remote_path=mount_entry["container_path"],
|
||||
)
|
||||
)
|
||||
for entry in iter_skills_files():
|
||||
cred_mounts.append(
|
||||
_modal.Mount.from_local_file(
|
||||
entry["host_path"],
|
||||
remote_path=entry["container_path"],
|
||||
)
|
||||
)
|
||||
cache_files = iter_cache_files()
|
||||
for entry in cache_files:
|
||||
cred_mounts.append(
|
||||
_modal.Mount.from_local_file(
|
||||
entry["host_path"],
|
||||
remote_path=entry["container_path"],
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug("Modal: could not load credential file mounts: %s", e)
|
||||
|
||||
self._worker.start()
|
||||
|
||||
async def _create_sandbox(image_spec: Any):
|
||||
app = await _modal.App.lookup.aio("hermes-agent", create_if_missing=True)
|
||||
create_kwargs = dict(sandbox_kwargs)
|
||||
if cred_mounts:
|
||||
existing_mounts = list(create_kwargs.pop("mounts", []))
|
||||
existing_mounts.extend(cred_mounts)
|
||||
create_kwargs["mounts"] = existing_mounts
|
||||
sandbox = await _modal.Sandbox.create.aio(
|
||||
"sleep", "infinity",
|
||||
image=image_spec,
|
||||
app=app,
|
||||
timeout=int(create_kwargs.pop("timeout", 3600)),
|
||||
**create_kwargs,
|
||||
)
|
||||
return app, sandbox
|
||||
|
||||
try:
|
||||
target_image_spec = restored_snapshot_id or image
|
||||
try:
|
||||
effective_image = _resolve_modal_image(target_image_spec)
|
||||
self._app, self._sandbox = self._worker.run_coroutine(
|
||||
_create_sandbox(effective_image), timeout=300,
|
||||
)
|
||||
except Exception as exc:
|
||||
if not restored_snapshot_id:
|
||||
raise
|
||||
logger.warning(
|
||||
"Modal: failed to restore snapshot %s, retrying with base image: %s",
|
||||
restored_snapshot_id[:20], exc,
|
||||
)
|
||||
_delete_direct_snapshot(self._task_id, restored_snapshot_id)
|
||||
base_image = _resolve_modal_image(image)
|
||||
self._app, self._sandbox = self._worker.run_coroutine(
|
||||
_create_sandbox(base_image), timeout=300,
|
||||
)
|
||||
else:
|
||||
if restored_snapshot_id and restored_from_legacy_key:
|
||||
_store_direct_snapshot(self._task_id, restored_snapshot_id)
|
||||
except Exception:
|
||||
self._worker.stop()
|
||||
raise
|
||||
|
||||
logger.info("Modal: sandbox created (task=%s)", self._task_id)
|
||||
|
||||
self._sync_manager = FileSyncManager(
|
||||
get_files_fn=lambda: iter_sync_files("/root/.hermes"),
|
||||
upload_fn=self._modal_upload,
|
||||
delete_fn=self._modal_delete,
|
||||
bulk_upload_fn=self._modal_bulk_upload,
|
||||
)
|
||||
self._sync_manager.sync(force=True)
|
||||
self.init_session()
|
||||
|
||||
def _modal_upload(self, host_path: str, remote_path: str) -> None:
|
||||
"""Upload a single file via base64 piped through stdin."""
|
||||
content = Path(host_path).read_bytes()
|
||||
b64 = base64.b64encode(content).decode("ascii")
|
||||
container_dir = str(Path(remote_path).parent)
|
||||
cmd = (
|
||||
f"mkdir -p {shlex.quote(container_dir)} && "
|
||||
f"base64 -d > {shlex.quote(remote_path)}"
|
||||
)
|
||||
|
||||
async def _write():
|
||||
proc = await self._sandbox.exec.aio("bash", "-c", cmd)
|
||||
offset = 0
|
||||
chunk_size = self._STDIN_CHUNK_SIZE
|
||||
while offset < len(b64):
|
||||
proc.stdin.write(b64[offset:offset + chunk_size])
|
||||
await proc.stdin.drain.aio()
|
||||
offset += chunk_size
|
||||
proc.stdin.write_eof()
|
||||
await proc.stdin.drain.aio()
|
||||
await proc.wait.aio()
|
||||
|
||||
self._worker.run_coroutine(_write(), timeout=30)
|
||||
|
||||
# Modal SDK stdin buffer limit (legacy server path). The command-router
|
||||
# path allows 16 MB, but we must stay under the smaller 2 MB cap for
|
||||
# compatibility. Chunks are written below this threshold and flushed
|
||||
# individually via drain().
|
||||
_STDIN_CHUNK_SIZE = 1 * 1024 * 1024 # 1 MB — safe for both transport paths
|
||||
|
||||
def _modal_bulk_upload(self, files: list[tuple[str, str]]) -> None:
|
||||
"""Upload many files via tar archive piped through stdin.
|
||||
|
||||
Builds a gzipped tar archive in memory and streams it into a
|
||||
``base64 -d | tar xzf -`` pipeline via the process's stdin,
|
||||
avoiding the Modal SDK's 64 KB ``ARG_MAX_BYTES`` exec-arg limit.
|
||||
"""
|
||||
if not files:
|
||||
return
|
||||
|
||||
buf = io.BytesIO()
|
||||
with tarfile.open(fileobj=buf, mode="w:gz") as tar:
|
||||
for host_path, remote_path in files:
|
||||
tar.add(host_path, arcname=remote_path.lstrip("/"))
|
||||
payload = base64.b64encode(buf.getvalue()).decode("ascii")
|
||||
|
||||
parents = unique_parent_dirs(files)
|
||||
mkdir_part = quoted_mkdir_command(parents)
|
||||
cmd = f"{mkdir_part} && base64 -d | tar xzf - -C /"
|
||||
|
||||
async def _bulk():
|
||||
proc = await self._sandbox.exec.aio("bash", "-c", cmd)
|
||||
|
||||
# Stream payload through stdin in chunks to stay under the
|
||||
# SDK's per-write buffer limit (2 MB legacy / 16 MB router).
|
||||
offset = 0
|
||||
chunk_size = self._STDIN_CHUNK_SIZE
|
||||
while offset < len(payload):
|
||||
proc.stdin.write(payload[offset:offset + chunk_size])
|
||||
await proc.stdin.drain.aio()
|
||||
offset += chunk_size
|
||||
|
||||
proc.stdin.write_eof()
|
||||
await proc.stdin.drain.aio()
|
||||
|
||||
exit_code = await proc.wait.aio()
|
||||
if exit_code != 0:
|
||||
stderr_text = await proc.stderr.read.aio()
|
||||
raise RuntimeError(
|
||||
f"Modal bulk upload failed (exit {exit_code}): {stderr_text}"
|
||||
)
|
||||
|
||||
self._worker.run_coroutine(_bulk(), timeout=120)
|
||||
|
||||
def _modal_delete(self, remote_paths: list[str]) -> None:
|
||||
"""Batch-delete remote files via exec."""
|
||||
rm_cmd = quoted_rm_command(remote_paths)
|
||||
|
||||
async def _rm():
|
||||
proc = await self._sandbox.exec.aio("bash", "-c", rm_cmd)
|
||||
await proc.wait.aio()
|
||||
|
||||
self._worker.run_coroutine(_rm(), timeout=15)
|
||||
|
||||
def _before_execute(self) -> None:
|
||||
"""Sync files to sandbox via FileSyncManager (rate-limited internally)."""
|
||||
self._sync_manager.sync()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Execution
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _run_bash(self, cmd_string: str, *, login: bool = False,
|
||||
timeout: int = 120,
|
||||
stdin_data: str | None = None):
|
||||
"""Return a _ThreadedProcessHandle wrapping an async Modal sandbox exec."""
|
||||
sandbox = self._sandbox
|
||||
worker = self._worker
|
||||
|
||||
def cancel():
|
||||
worker.run_coroutine(sandbox.terminate.aio(), timeout=15)
|
||||
|
||||
def exec_fn() -> tuple[str, int]:
|
||||
async def _do():
|
||||
args = ["bash"]
|
||||
if login:
|
||||
args.extend(["-l", "-c", cmd_string])
|
||||
else:
|
||||
args.extend(["-c", cmd_string])
|
||||
process = await sandbox.exec.aio(*args, timeout=timeout)
|
||||
stdout = await process.stdout.read.aio()
|
||||
stderr = await process.stderr.read.aio()
|
||||
exit_code = await process.wait.aio()
|
||||
if isinstance(stdout, bytes):
|
||||
stdout = stdout.decode("utf-8", errors="replace")
|
||||
if isinstance(stderr, bytes):
|
||||
stderr = stderr.decode("utf-8", errors="replace")
|
||||
output = stdout
|
||||
if stderr:
|
||||
output = f"{stdout}\n{stderr}" if stdout else stderr
|
||||
return output, exit_code
|
||||
|
||||
return worker.run_coroutine(_do(), timeout=timeout + 30)
|
||||
|
||||
return _ThreadedProcessHandle(exec_fn, cancel_fn=cancel)
|
||||
|
||||
def cleanup(self):
|
||||
"""Snapshot the filesystem (if persistent) then stop the sandbox."""
|
||||
if self._sandbox is None:
|
||||
return
|
||||
|
||||
if self._persistent:
|
||||
try:
|
||||
async def _snapshot():
|
||||
img = await self._sandbox.snapshot_filesystem.aio()
|
||||
return img.object_id
|
||||
|
||||
try:
|
||||
snapshot_id = self._worker.run_coroutine(_snapshot(), timeout=60)
|
||||
except Exception:
|
||||
snapshot_id = None
|
||||
|
||||
if snapshot_id:
|
||||
_store_direct_snapshot(self._task_id, snapshot_id)
|
||||
logger.info(
|
||||
"Modal: saved filesystem snapshot %s for task %s",
|
||||
snapshot_id[:20], self._task_id,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Modal: filesystem snapshot failed: %s", e)
|
||||
|
||||
try:
|
||||
self._worker.run_coroutine(self._sandbox.terminate.aio(), timeout=15)
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
self._worker.stop()
|
||||
self._sandbox = None
|
||||
self._app = None
|
||||
@@ -0,0 +1,186 @@
|
||||
"""Shared Hermes-side execution flow for Modal transports.
|
||||
|
||||
This module deliberately stops at the Hermes boundary:
|
||||
- command preparation
|
||||
- cwd/timeout normalization
|
||||
- stdin/sudo shell wrapping
|
||||
- common result shape
|
||||
- interrupt/cancel polling
|
||||
|
||||
Direct Modal and managed Modal keep separate transport logic, persistence, and
|
||||
trust-boundary decisions in their own modules.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shlex
|
||||
import time
|
||||
import uuid
|
||||
from abc import abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from tools.environments.base import BaseEnvironment
|
||||
from tools.interrupt import is_interrupted
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PreparedModalExec:
|
||||
"""Normalized command data passed to a transport-specific exec runner."""
|
||||
|
||||
command: str
|
||||
cwd: str
|
||||
timeout: int
|
||||
stdin_data: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ModalExecStart:
|
||||
"""Transport response after starting an exec."""
|
||||
|
||||
handle: Any | None = None
|
||||
immediate_result: dict | None = None
|
||||
|
||||
|
||||
def wrap_modal_stdin_heredoc(command: str, stdin_data: str) -> str:
|
||||
"""Append stdin as a shell heredoc for transports without stdin piping."""
|
||||
marker = f"HERMES_EOF_{uuid.uuid4().hex[:8]}"
|
||||
while marker in stdin_data:
|
||||
marker = f"HERMES_EOF_{uuid.uuid4().hex[:8]}"
|
||||
return f"{command} << '{marker}'\n{stdin_data}\n{marker}"
|
||||
|
||||
|
||||
def wrap_modal_sudo_pipe(command: str, sudo_stdin: str) -> str:
|
||||
"""Feed sudo via a shell pipe for transports without direct stdin piping."""
|
||||
return f"printf '%s\\n' {shlex.quote(sudo_stdin.rstrip())} | {command}"
|
||||
|
||||
|
||||
class BaseModalExecutionEnvironment(BaseEnvironment):
|
||||
"""Execution flow for the *managed* Modal transport (gateway-owned sandbox).
|
||||
|
||||
This deliberately overrides :meth:`BaseEnvironment.execute` because the
|
||||
tool-gateway handles command preparation, CWD tracking, and env-snapshot
|
||||
management on the server side. The base class's ``_wrap_command`` /
|
||||
``_wait_for_process`` / snapshot machinery does not apply here — the
|
||||
gateway owns that responsibility. See ``ManagedModalEnvironment`` for the
|
||||
concrete subclass.
|
||||
"""
|
||||
|
||||
_stdin_mode = "payload"
|
||||
_poll_interval_seconds = 0.25
|
||||
_client_timeout_grace_seconds: float | None = None
|
||||
_interrupt_output = "[Command interrupted]"
|
||||
_unexpected_error_prefix = "Modal execution error"
|
||||
|
||||
def execute(
|
||||
self,
|
||||
command: str,
|
||||
cwd: str = "",
|
||||
*,
|
||||
timeout: int | None = None,
|
||||
stdin_data: str | None = None,
|
||||
) -> dict:
|
||||
self._before_execute()
|
||||
prepared = self._prepare_modal_exec(
|
||||
command,
|
||||
cwd=cwd,
|
||||
timeout=timeout,
|
||||
stdin_data=stdin_data,
|
||||
)
|
||||
|
||||
try:
|
||||
start = self._start_modal_exec(prepared)
|
||||
except Exception as exc:
|
||||
return self._error_result(f"{self._unexpected_error_prefix}: {exc}")
|
||||
|
||||
if start.immediate_result is not None:
|
||||
return start.immediate_result
|
||||
|
||||
if start.handle is None:
|
||||
return self._error_result(
|
||||
f"{self._unexpected_error_prefix}: transport did not return an exec handle"
|
||||
)
|
||||
|
||||
deadline = None
|
||||
if self._client_timeout_grace_seconds is not None:
|
||||
deadline = time.monotonic() + prepared.timeout + self._client_timeout_grace_seconds
|
||||
|
||||
while True:
|
||||
if is_interrupted():
|
||||
try:
|
||||
self._cancel_modal_exec(start.handle)
|
||||
except Exception:
|
||||
pass
|
||||
return self._result(self._interrupt_output, 130)
|
||||
|
||||
try:
|
||||
result = self._poll_modal_exec(start.handle)
|
||||
except Exception as exc:
|
||||
return self._error_result(f"{self._unexpected_error_prefix}: {exc}")
|
||||
|
||||
if result is not None:
|
||||
return result
|
||||
|
||||
if deadline is not None and time.monotonic() >= deadline:
|
||||
try:
|
||||
self._cancel_modal_exec(start.handle)
|
||||
except Exception:
|
||||
pass
|
||||
return self._timeout_result_for_modal(prepared.timeout)
|
||||
|
||||
time.sleep(self._poll_interval_seconds)
|
||||
|
||||
def _before_execute(self) -> None:
|
||||
"""Hook for backends that need pre-exec sync or validation."""
|
||||
pass
|
||||
|
||||
def _prepare_modal_exec(
|
||||
self,
|
||||
command: str,
|
||||
*,
|
||||
cwd: str = "",
|
||||
timeout: int | None = None,
|
||||
stdin_data: str | None = None,
|
||||
) -> PreparedModalExec:
|
||||
effective_cwd = cwd or self.cwd
|
||||
effective_timeout = timeout or self.timeout
|
||||
|
||||
exec_command = command
|
||||
exec_stdin = stdin_data if self._stdin_mode == "payload" else None
|
||||
if stdin_data is not None and self._stdin_mode == "heredoc":
|
||||
exec_command = wrap_modal_stdin_heredoc(exec_command, stdin_data)
|
||||
|
||||
exec_command, sudo_stdin = self._prepare_command(exec_command)
|
||||
if sudo_stdin is not None:
|
||||
exec_command = wrap_modal_sudo_pipe(exec_command, sudo_stdin)
|
||||
|
||||
return PreparedModalExec(
|
||||
command=exec_command,
|
||||
cwd=effective_cwd,
|
||||
timeout=effective_timeout,
|
||||
stdin_data=exec_stdin,
|
||||
)
|
||||
|
||||
def _result(self, output: str, returncode: int) -> dict:
|
||||
return {
|
||||
"output": output,
|
||||
"returncode": returncode,
|
||||
}
|
||||
|
||||
def _error_result(self, output: str) -> dict:
|
||||
return self._result(output, 1)
|
||||
|
||||
def _timeout_result_for_modal(self, timeout: int) -> dict:
|
||||
return self._result(f"Command timed out after {timeout}s", 124)
|
||||
|
||||
@abstractmethod
|
||||
def _start_modal_exec(self, prepared: PreparedModalExec) -> ModalExecStart:
|
||||
"""Begin a transport-specific exec."""
|
||||
|
||||
@abstractmethod
|
||||
def _poll_modal_exec(self, handle: Any) -> dict | None:
|
||||
"""Return a final result dict when complete, else ``None``."""
|
||||
|
||||
@abstractmethod
|
||||
def _cancel_modal_exec(self, handle: Any) -> None:
|
||||
"""Cancel or terminate the active transport exec."""
|
||||
@@ -0,0 +1,262 @@
|
||||
"""Singularity/Apptainer persistent container environment.
|
||||
|
||||
Security-hardened with --containall, --no-home, capability dropping.
|
||||
Supports configurable resource limits and optional filesystem persistence
|
||||
via writable overlay directories that survive across sessions.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import threading
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from hermes_constants import get_hermes_home
|
||||
from tools.environments.base import (
|
||||
BaseEnvironment,
|
||||
_load_json_store,
|
||||
_popen_bash,
|
||||
_save_json_store,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SNAPSHOT_STORE = get_hermes_home() / "singularity_snapshots.json"
|
||||
|
||||
|
||||
def _find_singularity_executable() -> str:
|
||||
"""Locate the apptainer or singularity CLI binary."""
|
||||
if shutil.which("apptainer"):
|
||||
return "apptainer"
|
||||
if shutil.which("singularity"):
|
||||
return "singularity"
|
||||
raise RuntimeError(
|
||||
"Neither 'apptainer' nor 'singularity' was found in PATH. "
|
||||
"Install Apptainer (https://apptainer.org/docs/admin/main/installation.html) "
|
||||
"or Singularity and ensure the CLI is available."
|
||||
)
|
||||
|
||||
|
||||
def _ensure_singularity_available() -> str:
|
||||
"""Preflight check: resolve the executable and verify it responds."""
|
||||
exe = _find_singularity_executable()
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[exe, "version"], capture_output=True, text=True, timeout=10,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
raise RuntimeError(
|
||||
f"Singularity backend selected but '{exe}' could not be executed."
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
raise RuntimeError(f"'{exe} version' timed out.")
|
||||
|
||||
if result.returncode != 0:
|
||||
stderr = result.stderr.strip()[:200]
|
||||
raise RuntimeError(f"'{exe} version' failed (exit code {result.returncode}): {stderr}")
|
||||
return exe
|
||||
|
||||
|
||||
def _load_snapshots() -> dict:
|
||||
return _load_json_store(_SNAPSHOT_STORE)
|
||||
|
||||
|
||||
def _save_snapshots(data: dict) -> None:
|
||||
_save_json_store(_SNAPSHOT_STORE, data)
|
||||
|
||||
|
||||
def _get_scratch_dir() -> Path:
|
||||
custom_scratch = os.getenv("TERMINAL_SCRATCH_DIR")
|
||||
if custom_scratch:
|
||||
scratch_path = Path(custom_scratch)
|
||||
scratch_path.mkdir(parents=True, exist_ok=True)
|
||||
return scratch_path
|
||||
|
||||
from tools.environments.base import get_sandbox_dir
|
||||
sandbox = get_sandbox_dir() / "singularity"
|
||||
|
||||
scratch = Path("/scratch")
|
||||
if scratch.exists() and os.access(scratch, os.W_OK):
|
||||
user_scratch = scratch / os.getenv("USER", "hermes") / "hermes-agent"
|
||||
user_scratch.mkdir(parents=True, exist_ok=True)
|
||||
logger.info("Using /scratch for sandboxes: %s", user_scratch)
|
||||
return user_scratch
|
||||
|
||||
sandbox.mkdir(parents=True, exist_ok=True)
|
||||
return sandbox
|
||||
|
||||
|
||||
def _get_apptainer_cache_dir() -> Path:
|
||||
cache_dir = os.getenv("APPTAINER_CACHEDIR")
|
||||
if cache_dir:
|
||||
cache_path = Path(cache_dir)
|
||||
cache_path.mkdir(parents=True, exist_ok=True)
|
||||
return cache_path
|
||||
scratch = _get_scratch_dir()
|
||||
cache_path = scratch / ".apptainer"
|
||||
cache_path.mkdir(parents=True, exist_ok=True)
|
||||
return cache_path
|
||||
|
||||
|
||||
_sif_build_lock = threading.Lock()
|
||||
|
||||
|
||||
def _get_or_build_sif(image: str, executable: str = "apptainer") -> str:
|
||||
if image.endswith('.sif') and Path(image).exists():
|
||||
return image
|
||||
if not image.startswith('docker://'):
|
||||
return image
|
||||
|
||||
image_name = image.replace('docker://', '').replace('/', '-').replace(':', '-')
|
||||
cache_dir = _get_apptainer_cache_dir()
|
||||
sif_path = cache_dir / f"{image_name}.sif"
|
||||
|
||||
if sif_path.exists():
|
||||
return str(sif_path)
|
||||
|
||||
with _sif_build_lock:
|
||||
if sif_path.exists():
|
||||
return str(sif_path)
|
||||
|
||||
logger.info("Building SIF image (one-time setup)...")
|
||||
logger.info(" Source: %s", image)
|
||||
logger.info(" Target: %s", sif_path)
|
||||
|
||||
tmp_dir = cache_dir / "tmp"
|
||||
tmp_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
env = os.environ.copy()
|
||||
env["APPTAINER_TMPDIR"] = str(tmp_dir)
|
||||
env["APPTAINER_CACHEDIR"] = str(cache_dir)
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[executable, "build", str(sif_path), image],
|
||||
capture_output=True, text=True, timeout=600, env=env,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
logger.warning("SIF build failed, falling back to docker:// URL")
|
||||
logger.warning(" Error: %s", result.stderr[:500])
|
||||
return image
|
||||
logger.info("SIF image built successfully")
|
||||
return str(sif_path)
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning("SIF build timed out, falling back to docker:// URL")
|
||||
if sif_path.exists():
|
||||
sif_path.unlink()
|
||||
return image
|
||||
except Exception as e:
|
||||
logger.warning("SIF build error: %s, falling back to docker:// URL", e)
|
||||
return image
|
||||
|
||||
|
||||
class SingularityEnvironment(BaseEnvironment):
|
||||
"""Hardened Singularity/Apptainer container with resource limits and persistence.
|
||||
|
||||
Spawn-per-call: every execute() spawns a fresh ``apptainer exec ... bash -c`` process.
|
||||
Session snapshot preserves env vars across calls.
|
||||
CWD persists via in-band stdout markers.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image: str,
|
||||
cwd: str = "~",
|
||||
timeout: int = 60,
|
||||
cpu: float = 0,
|
||||
memory: int = 0,
|
||||
disk: int = 0,
|
||||
persistent_filesystem: bool = False,
|
||||
task_id: str = "default",
|
||||
):
|
||||
super().__init__(cwd=cwd, timeout=timeout)
|
||||
self.executable = _ensure_singularity_available()
|
||||
self.image = _get_or_build_sif(image, self.executable)
|
||||
self.instance_id = f"hermes_{uuid.uuid4().hex[:12]}"
|
||||
self._instance_started = False
|
||||
self._persistent = persistent_filesystem
|
||||
self._task_id = task_id
|
||||
self._overlay_dir: Optional[Path] = None
|
||||
self._cpu = cpu
|
||||
self._memory = memory
|
||||
|
||||
if self._persistent:
|
||||
overlay_base = _get_scratch_dir() / "hermes-overlays"
|
||||
overlay_base.mkdir(parents=True, exist_ok=True)
|
||||
self._overlay_dir = overlay_base / f"overlay-{task_id}"
|
||||
self._overlay_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
self._start_instance()
|
||||
self.init_session()
|
||||
|
||||
def _start_instance(self):
|
||||
cmd = [self.executable, "instance", "start"]
|
||||
cmd.extend(["--containall", "--no-home"])
|
||||
|
||||
if self._persistent and self._overlay_dir:
|
||||
cmd.extend(["--overlay", str(self._overlay_dir)])
|
||||
else:
|
||||
cmd.append("--writable-tmpfs")
|
||||
|
||||
try:
|
||||
from tools.credential_files import get_credential_file_mounts, get_skills_directory_mount
|
||||
for mount_entry in get_credential_file_mounts():
|
||||
cmd.extend(["--bind", f"{mount_entry['host_path']}:{mount_entry['container_path']}:ro"])
|
||||
for skills_mount in get_skills_directory_mount():
|
||||
cmd.extend(["--bind", f"{skills_mount['host_path']}:{skills_mount['container_path']}:ro"])
|
||||
except Exception as e:
|
||||
logger.debug("Singularity: could not load credential/skills mounts: %s", e)
|
||||
|
||||
if self._memory > 0:
|
||||
cmd.extend(["--memory", f"{self._memory}M"])
|
||||
if self._cpu > 0:
|
||||
cmd.extend(["--cpus", str(self._cpu)])
|
||||
|
||||
cmd.extend([str(self.image), self.instance_id])
|
||||
|
||||
try:
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=120)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"Failed to start instance: {result.stderr}")
|
||||
self._instance_started = True
|
||||
logger.info("Singularity instance %s started (persistent=%s)",
|
||||
self.instance_id, self._persistent)
|
||||
except subprocess.TimeoutExpired:
|
||||
raise RuntimeError("Instance start timed out")
|
||||
|
||||
def _run_bash(self, cmd_string: str, *, login: bool = False,
|
||||
timeout: int = 120,
|
||||
stdin_data: str | None = None) -> subprocess.Popen:
|
||||
"""Spawn a bash process inside the Singularity instance."""
|
||||
if not self._instance_started:
|
||||
raise RuntimeError("Singularity instance not started")
|
||||
|
||||
cmd = [self.executable, "exec",
|
||||
f"instance://{self.instance_id}"]
|
||||
if login:
|
||||
cmd.extend(["bash", "-l", "-c", cmd_string])
|
||||
else:
|
||||
cmd.extend(["bash", "-c", cmd_string])
|
||||
|
||||
return _popen_bash(cmd, stdin_data)
|
||||
|
||||
def cleanup(self):
|
||||
"""Stop the instance. If persistent, the overlay dir survives."""
|
||||
if self._instance_started:
|
||||
try:
|
||||
subprocess.run(
|
||||
[self.executable, "instance", "stop", self.instance_id],
|
||||
capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
logger.info("Singularity instance %s stopped", self.instance_id)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to stop Singularity instance %s: %s", self.instance_id, e)
|
||||
self._instance_started = False
|
||||
|
||||
if self._persistent and self._overlay_dir:
|
||||
snapshots = _load_snapshots()
|
||||
snapshots[self._task_id] = str(self._overlay_dir)
|
||||
_save_snapshots(snapshots)
|
||||
@@ -0,0 +1,258 @@
|
||||
"""SSH remote execution environment with ControlMaster connection persistence."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from tools.environments.base import BaseEnvironment, _popen_bash
|
||||
from tools.environments.file_sync import (
|
||||
FileSyncManager,
|
||||
iter_sync_files,
|
||||
quoted_mkdir_command,
|
||||
quoted_rm_command,
|
||||
unique_parent_dirs,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _ensure_ssh_available() -> None:
|
||||
"""Fail fast with a clear error when the SSH client is unavailable."""
|
||||
if not shutil.which("ssh"):
|
||||
raise RuntimeError(
|
||||
"SSH is not installed or not in PATH. Install OpenSSH client: apt install openssh-client"
|
||||
)
|
||||
|
||||
|
||||
class SSHEnvironment(BaseEnvironment):
|
||||
"""Run commands on a remote machine over SSH.
|
||||
|
||||
Spawn-per-call: every execute() spawns a fresh ``ssh ... bash -c`` process.
|
||||
Session snapshot preserves env vars across calls.
|
||||
CWD persists via in-band stdout markers.
|
||||
Uses SSH ControlMaster for connection reuse.
|
||||
"""
|
||||
|
||||
def __init__(self, host: str, user: str, cwd: str = "~",
|
||||
timeout: int = 60, port: int = 22, key_path: str = ""):
|
||||
super().__init__(cwd=cwd, timeout=timeout)
|
||||
self.host = host
|
||||
self.user = user
|
||||
self.port = port
|
||||
self.key_path = key_path
|
||||
|
||||
self.control_dir = Path(tempfile.gettempdir()) / "hermes-ssh"
|
||||
self.control_dir.mkdir(parents=True, exist_ok=True)
|
||||
self.control_socket = self.control_dir / f"{user}@{host}:{port}.sock"
|
||||
_ensure_ssh_available()
|
||||
self._establish_connection()
|
||||
self._remote_home = self._detect_remote_home()
|
||||
|
||||
self._ensure_remote_dirs()
|
||||
self._sync_manager = FileSyncManager(
|
||||
get_files_fn=lambda: iter_sync_files(f"{self._remote_home}/.hermes"),
|
||||
upload_fn=self._scp_upload,
|
||||
delete_fn=self._ssh_delete,
|
||||
bulk_upload_fn=self._ssh_bulk_upload,
|
||||
)
|
||||
self._sync_manager.sync(force=True)
|
||||
|
||||
self.init_session()
|
||||
|
||||
def _build_ssh_command(self, extra_args: list | None = None) -> list:
|
||||
cmd = ["ssh"]
|
||||
cmd.extend(["-o", f"ControlPath={self.control_socket}"])
|
||||
cmd.extend(["-o", "ControlMaster=auto"])
|
||||
cmd.extend(["-o", "ControlPersist=300"])
|
||||
cmd.extend(["-o", "BatchMode=yes"])
|
||||
cmd.extend(["-o", "StrictHostKeyChecking=accept-new"])
|
||||
cmd.extend(["-o", "ConnectTimeout=10"])
|
||||
if self.port != 22:
|
||||
cmd.extend(["-p", str(self.port)])
|
||||
if self.key_path:
|
||||
cmd.extend(["-i", self.key_path])
|
||||
if extra_args:
|
||||
cmd.extend(extra_args)
|
||||
cmd.append(f"{self.user}@{self.host}")
|
||||
return cmd
|
||||
|
||||
def _establish_connection(self):
|
||||
cmd = self._build_ssh_command()
|
||||
cmd.append("echo 'SSH connection established'")
|
||||
try:
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=15)
|
||||
if result.returncode != 0:
|
||||
error_msg = result.stderr.strip() or result.stdout.strip()
|
||||
raise RuntimeError(f"SSH connection failed: {error_msg}")
|
||||
except subprocess.TimeoutExpired:
|
||||
raise RuntimeError(f"SSH connection to {self.user}@{self.host} timed out")
|
||||
|
||||
def _detect_remote_home(self) -> str:
|
||||
"""Detect the remote user's home directory."""
|
||||
try:
|
||||
cmd = self._build_ssh_command()
|
||||
cmd.append("echo $HOME")
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=10)
|
||||
home = result.stdout.strip()
|
||||
if home and result.returncode == 0:
|
||||
logger.debug("SSH: remote home = %s", home)
|
||||
return home
|
||||
except Exception:
|
||||
pass
|
||||
if self.user == "root":
|
||||
return "/root"
|
||||
return f"/home/{self.user}"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# File sync (via FileSyncManager)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _ensure_remote_dirs(self) -> None:
|
||||
"""Create base ~/.hermes directory tree on remote in one SSH call."""
|
||||
base = f"{self._remote_home}/.hermes"
|
||||
dirs = [base, f"{base}/skills", f"{base}/credentials", f"{base}/cache"]
|
||||
cmd = self._build_ssh_command()
|
||||
cmd.append(quoted_mkdir_command(dirs))
|
||||
subprocess.run(cmd, capture_output=True, text=True, timeout=10)
|
||||
|
||||
# _get_sync_files provided via iter_sync_files in FileSyncManager init
|
||||
|
||||
def _scp_upload(self, host_path: str, remote_path: str) -> None:
|
||||
"""Upload a single file via scp over ControlMaster."""
|
||||
parent = str(Path(remote_path).parent)
|
||||
mkdir_cmd = self._build_ssh_command()
|
||||
mkdir_cmd.append(f"mkdir -p {shlex.quote(parent)}")
|
||||
subprocess.run(mkdir_cmd, capture_output=True, text=True, timeout=10)
|
||||
|
||||
scp_cmd = ["scp", "-o", f"ControlPath={self.control_socket}"]
|
||||
if self.port != 22:
|
||||
scp_cmd.extend(["-P", str(self.port)])
|
||||
if self.key_path:
|
||||
scp_cmd.extend(["-i", self.key_path])
|
||||
scp_cmd.extend([host_path, f"{self.user}@{self.host}:{remote_path}"])
|
||||
result = subprocess.run(scp_cmd, capture_output=True, text=True, timeout=30)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"scp failed: {result.stderr.strip()}")
|
||||
|
||||
def _ssh_bulk_upload(self, files: list[tuple[str, str]]) -> None:
|
||||
"""Upload many files in a single tar-over-SSH stream.
|
||||
|
||||
Pipes ``tar c`` on the local side through an SSH connection to
|
||||
``tar x`` on the remote, transferring all files in one TCP stream
|
||||
instead of spawning a subprocess per file. Directory creation is
|
||||
batched into a single ``mkdir -p`` call beforehand.
|
||||
|
||||
Typical improvement: ~580 files goes from O(N) scp round-trips
|
||||
to a single streaming transfer.
|
||||
"""
|
||||
if not files:
|
||||
return
|
||||
|
||||
parents = unique_parent_dirs(files)
|
||||
if parents:
|
||||
cmd = self._build_ssh_command()
|
||||
cmd.append(quoted_mkdir_command(parents))
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"remote mkdir failed: {result.stderr.strip()}")
|
||||
|
||||
# Symlink staging avoids fragile GNU tar --transform rules.
|
||||
with tempfile.TemporaryDirectory(prefix="hermes-ssh-bulk-") as staging:
|
||||
for host_path, remote_path in files:
|
||||
staged = os.path.join(staging, remote_path.lstrip("/"))
|
||||
os.makedirs(os.path.dirname(staged), exist_ok=True)
|
||||
os.symlink(os.path.abspath(host_path), staged)
|
||||
|
||||
tar_cmd = ["tar", "-chf", "-", "-C", staging, "."]
|
||||
ssh_cmd = self._build_ssh_command()
|
||||
ssh_cmd.append("tar xf - -C /")
|
||||
|
||||
tar_proc = subprocess.Popen(
|
||||
tar_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE
|
||||
)
|
||||
try:
|
||||
ssh_proc = subprocess.Popen(
|
||||
ssh_cmd, stdin=tar_proc.stdout, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
except Exception:
|
||||
tar_proc.kill()
|
||||
tar_proc.wait()
|
||||
raise
|
||||
|
||||
# Allow tar_proc to receive SIGPIPE if ssh_proc exits early
|
||||
tar_proc.stdout.close()
|
||||
|
||||
try:
|
||||
_, ssh_stderr = ssh_proc.communicate(timeout=120)
|
||||
# Use communicate() instead of wait() to drain stderr and
|
||||
# avoid deadlock if tar produces more than PIPE_BUF of errors.
|
||||
tar_stderr_raw = b""
|
||||
if tar_proc.poll() is None:
|
||||
_, tar_stderr_raw = tar_proc.communicate(timeout=10)
|
||||
else:
|
||||
tar_stderr_raw = tar_proc.stderr.read() if tar_proc.stderr else b""
|
||||
except subprocess.TimeoutExpired:
|
||||
tar_proc.kill()
|
||||
ssh_proc.kill()
|
||||
tar_proc.wait()
|
||||
ssh_proc.wait()
|
||||
raise RuntimeError("SSH bulk upload timed out")
|
||||
|
||||
if tar_proc.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"tar create failed (rc={tar_proc.returncode}): "
|
||||
f"{tar_stderr_raw.decode(errors='replace').strip()}"
|
||||
)
|
||||
if ssh_proc.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"tar extract over SSH failed (rc={ssh_proc.returncode}): "
|
||||
f"{ssh_stderr.decode(errors='replace').strip()}"
|
||||
)
|
||||
|
||||
logger.debug("SSH: bulk-uploaded %d file(s) via tar pipe", len(files))
|
||||
|
||||
def _ssh_delete(self, remote_paths: list[str]) -> None:
|
||||
"""Batch-delete remote files in one SSH call."""
|
||||
cmd = self._build_ssh_command()
|
||||
cmd.append(quoted_rm_command(remote_paths))
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=10)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"remote rm failed: {result.stderr.strip()}")
|
||||
|
||||
def _before_execute(self) -> None:
|
||||
"""Sync files to remote via FileSyncManager (rate-limited internally)."""
|
||||
self._sync_manager.sync()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Execution
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _run_bash(self, cmd_string: str, *, login: bool = False,
|
||||
timeout: int = 120,
|
||||
stdin_data: str | None = None) -> subprocess.Popen:
|
||||
"""Spawn an SSH process that runs bash on the remote host."""
|
||||
cmd = self._build_ssh_command()
|
||||
if login:
|
||||
cmd.extend(["bash", "-l", "-c", shlex.quote(cmd_string)])
|
||||
else:
|
||||
cmd.extend(["bash", "-c", shlex.quote(cmd_string)])
|
||||
|
||||
return _popen_bash(cmd, stdin_data)
|
||||
|
||||
def cleanup(self):
|
||||
if self.control_socket.exists():
|
||||
try:
|
||||
cmd = ["ssh", "-o", f"ControlPath={self.control_socket}",
|
||||
"-O", "exit", f"{self.user}@{self.host}"]
|
||||
subprocess.run(cmd, capture_output=True, timeout=5)
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
pass
|
||||
try:
|
||||
self.control_socket.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
Reference in New Issue
Block a user