init: MindOS CLI 本地执行体（从 mindOSv2/mindos-cli 独立）
- 独立 pyproject.toml（pip install -e .） - vendor_hermes.sh 已改为显式路径模式（不再依赖相对目录） - 包含 hermes vendor 快照
This commit is contained in:
@@ -0,0 +1,25 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Tools package namespace.
|
||||
|
||||
Keep package import side effects minimal. Importing ``tools`` should not
|
||||
eagerly import the full tool stack, because several subsystems load tools while
|
||||
``hermes_cli.config`` is still initializing.
|
||||
|
||||
Callers should import concrete submodules directly, for example:
|
||||
|
||||
import tools.web_tools
|
||||
from tools import browser_tool
|
||||
|
||||
Python will resolve those submodules via the package path without needing them
|
||||
to be re-exported here.
|
||||
"""
|
||||
|
||||
|
||||
def check_file_requirements():
|
||||
"""File tools only require terminal backend availability."""
|
||||
from .terminal_tool import check_terminal_requirements
|
||||
|
||||
return check_terminal_requirements()
|
||||
|
||||
|
||||
__all__ = ["check_file_requirements"]
|
||||
@@ -0,0 +1,59 @@
|
||||
"""
|
||||
用户配置存取（per-userId，基于 wiki 物理目录）
|
||||
|
||||
所有连接器凭据通过 user_config_read / user_config_write 进行读写，
|
||||
存储路径：{MINDOS_WIKI_DIR}/{userId}/.config/{connector}.json
|
||||
|
||||
设计原则：
|
||||
- 与 wiki/{userId}/ 物理目录隔离模式一致
|
||||
- .config/ 前缀防止 Agent 的 search_files 扫到凭据
|
||||
- 一个用户一个目录，导出/删除用户数据只需操作 wiki/{userId}/
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _wiki_root() -> Path:
|
||||
"""返回知识花园根目录"""
|
||||
return Path(os.getenv("MINDOS_WIKI_DIR", os.path.expanduser("~/.hermes/wiki")))
|
||||
|
||||
|
||||
def user_config_path(user_id: str, connector: str) -> Path:
|
||||
"""返回 wiki/{userId}/.config/{connector}.json 的路径"""
|
||||
return _wiki_root() / user_id / ".config" / f"{connector}.json"
|
||||
|
||||
|
||||
def user_config_read(user_id: str, connector: str) -> Dict[str, Any]:
|
||||
"""读取指定用户的连接器配置，不存在返回空 dict"""
|
||||
p = user_config_path(user_id, connector)
|
||||
try:
|
||||
if p.exists():
|
||||
return json.loads(p.read_text(encoding="utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning("[UserConfig] read failed %s/%s: %s", user_id[:8], connector, e)
|
||||
return {}
|
||||
|
||||
|
||||
def user_config_write(user_id: str, connector: str, data: Dict[str, Any]) -> None:
|
||||
"""写入指定用户的连接器配置，自动创建目录"""
|
||||
p = user_config_path(user_id, connector)
|
||||
try:
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
except Exception as e:
|
||||
logger.warning("[UserConfig] write failed %s/%s: %s", user_id[:8], connector, e)
|
||||
|
||||
|
||||
def user_config_delete(user_id: str, connector: str) -> None:
|
||||
"""删除指定用户的连接器配置"""
|
||||
p = user_config_path(user_id, connector)
|
||||
try:
|
||||
p.unlink(missing_ok=True)
|
||||
except Exception as e:
|
||||
logger.warning("[UserConfig] delete failed %s/%s: %s", user_id[:8], connector, e)
|
||||
@@ -0,0 +1,44 @@
|
||||
"""Strip ANSI escape sequences from subprocess output.
|
||||
|
||||
Used by terminal_tool, code_execution_tool, and process_registry to clean
|
||||
command output before returning it to the model. This prevents ANSI codes
|
||||
from entering the model's context — which is the root cause of models
|
||||
copying escape sequences into file writes.
|
||||
|
||||
Covers the full ECMA-48 spec: CSI (including private-mode ``?`` prefix,
|
||||
colon-separated params, intermediate bytes), OSC (BEL and ST terminators),
|
||||
DCS/SOS/PM/APC string sequences, nF multi-byte escapes, Fp/Fe/Fs
|
||||
single-byte escapes, and 8-bit C1 control characters.
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
_ANSI_ESCAPE_RE = re.compile(
|
||||
r"\x1b"
|
||||
r"(?:"
|
||||
r"\[[\x30-\x3f]*[\x20-\x2f]*[\x40-\x7e]" # CSI sequence
|
||||
r"|\][\s\S]*?(?:\x07|\x1b\\)" # OSC (BEL or ST terminator)
|
||||
r"|[PX^_][\s\S]*?(?:\x1b\\)" # DCS/SOS/PM/APC strings
|
||||
r"|[\x20-\x2f]+[\x30-\x7e]" # nF escape sequences
|
||||
r"|[\x30-\x7e]" # Fp/Fe/Fs single-byte
|
||||
r")"
|
||||
r"|\x9b[\x30-\x3f]*[\x20-\x2f]*[\x40-\x7e]" # 8-bit CSI
|
||||
r"|\x9d[\s\S]*?(?:\x07|\x9c)" # 8-bit OSC
|
||||
r"|[\x80-\x9f]", # Other 8-bit C1 controls
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
# Fast-path check — skip full regex when no escape-like bytes are present.
|
||||
_HAS_ESCAPE = re.compile(r"[\x1b\x80-\x9f]")
|
||||
|
||||
|
||||
def strip_ansi(text: str) -> str:
|
||||
"""Remove ANSI escape sequences from text.
|
||||
|
||||
Returns the input unchanged (fast path) when no ESC or C1 bytes are
|
||||
present. Safe to call on any string — clean text passes through
|
||||
with negligible overhead.
|
||||
"""
|
||||
if not text or not _HAS_ESCAPE.search(text):
|
||||
return text
|
||||
return _ANSI_ESCAPE_RE.sub("", text)
|
||||
@@ -0,0 +1,921 @@
|
||||
"""Dangerous command approval -- detection, prompting, and per-session state.
|
||||
|
||||
This module is the single source of truth for the dangerous command system:
|
||||
- Pattern detection (DANGEROUS_PATTERNS, detect_dangerous_command)
|
||||
- Per-session approval state (thread-safe, keyed by session_key)
|
||||
- Approval prompting (CLI interactive + gateway async)
|
||||
- Smart approval via auxiliary LLM (auto-approve low-risk commands)
|
||||
- Permanent allowlist persistence (config.yaml)
|
||||
"""
|
||||
|
||||
import contextvars
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import unicodedata
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Per-thread/per-task gateway session identity.
|
||||
# Gateway runs agent turns concurrently in executor threads, so reading a
|
||||
# process-global env var for session identity is racy. Keep env fallback for
|
||||
# legacy single-threaded callers, but prefer the context-local value when set.
|
||||
_approval_session_key: contextvars.ContextVar[str] = contextvars.ContextVar(
|
||||
"approval_session_key",
|
||||
default="",
|
||||
)
|
||||
|
||||
|
||||
def set_current_session_key(session_key: str) -> contextvars.Token[str]:
|
||||
"""Bind the active approval session key to the current context."""
|
||||
return _approval_session_key.set(session_key or "")
|
||||
|
||||
|
||||
def reset_current_session_key(token: contextvars.Token[str]) -> None:
|
||||
"""Restore the prior approval session key context."""
|
||||
_approval_session_key.reset(token)
|
||||
|
||||
|
||||
def get_current_session_key(default: str = "default") -> str:
|
||||
"""Return the active session key, preferring context-local state.
|
||||
|
||||
Resolution order:
|
||||
1. approval-specific contextvars (set by gateway before agent.run)
|
||||
2. session_context contextvars (set by _set_session_env)
|
||||
3. os.environ fallback (CLI, cron, tests)
|
||||
"""
|
||||
session_key = _approval_session_key.get()
|
||||
if session_key:
|
||||
return session_key
|
||||
from gateway.session_context import get_session_env
|
||||
return get_session_env("HERMES_SESSION_KEY", default)
|
||||
|
||||
# Sensitive write targets that should trigger approval even when referenced
|
||||
# via shell expansions like $HOME or $HERMES_HOME.
|
||||
_SSH_SENSITIVE_PATH = r'(?:~|\$home|\$\{home\})/\.ssh(?:/|$)'
|
||||
_HERMES_ENV_PATH = (
|
||||
r'(?:~\/\.hermes/|'
|
||||
r'(?:\$home|\$\{home\})/\.hermes/|'
|
||||
r'(?:\$hermes_home|\$\{hermes_home\})/)'
|
||||
r'\.env\b'
|
||||
)
|
||||
_SENSITIVE_WRITE_TARGET = (
|
||||
r'(?:/etc/|/dev/sd|'
|
||||
rf'{_SSH_SENSITIVE_PATH}|'
|
||||
rf'{_HERMES_ENV_PATH})'
|
||||
)
|
||||
|
||||
# =========================================================================
|
||||
# Dangerous command patterns
|
||||
# =========================================================================
|
||||
|
||||
DANGEROUS_PATTERNS = [
|
||||
(r'\brm\s+(-[^\s]*\s+)*/', "delete in root path"),
|
||||
(r'\brm\s+-[^\s]*r', "recursive delete"),
|
||||
(r'\brm\s+--recursive\b', "recursive delete (long flag)"),
|
||||
(r'\bchmod\s+(-[^\s]*\s+)*(777|666|o\+[rwx]*w|a\+[rwx]*w)\b', "world/other-writable permissions"),
|
||||
(r'\bchmod\s+--recursive\b.*(777|666|o\+[rwx]*w|a\+[rwx]*w)', "recursive world/other-writable (long flag)"),
|
||||
(r'\bchown\s+(-[^\s]*)?R\s+root', "recursive chown to root"),
|
||||
(r'\bchown\s+--recursive\b.*root', "recursive chown to root (long flag)"),
|
||||
(r'\bmkfs\b', "format filesystem"),
|
||||
(r'\bdd\s+.*if=', "disk copy"),
|
||||
(r'>\s*/dev/sd', "write to block device"),
|
||||
(r'\bDROP\s+(TABLE|DATABASE)\b', "SQL DROP"),
|
||||
(r'\bDELETE\s+FROM\b(?!.*\bWHERE\b)', "SQL DELETE without WHERE"),
|
||||
(r'\bTRUNCATE\s+(TABLE)?\s*\w', "SQL TRUNCATE"),
|
||||
(r'>\s*/etc/', "overwrite system config"),
|
||||
(r'\bsystemctl\s+(stop|disable|mask)\b', "stop/disable system service"),
|
||||
(r'\bkill\s+-9\s+-1\b', "kill all processes"),
|
||||
(r'\bpkill\s+-9\b', "force kill processes"),
|
||||
(r':\(\)\s*\{\s*:\s*\|\s*:\s*&\s*\}\s*;\s*:', "fork bomb"),
|
||||
# Any shell invocation via -c or combined flags like -lc, -ic, etc.
|
||||
(r'\b(bash|sh|zsh|ksh)\s+-[^\s]*c(\s+|$)', "shell command via -c/-lc flag"),
|
||||
(r'\b(python[23]?|perl|ruby|node)\s+-[ec]\s+', "script execution via -e/-c flag"),
|
||||
(r'\b(curl|wget)\b.*\|\s*(ba)?sh\b', "pipe remote content to shell"),
|
||||
(r'\b(bash|sh|zsh|ksh)\s+<\s*<?\s*\(\s*(curl|wget)\b', "execute remote script via process substitution"),
|
||||
(rf'\btee\b.*["\']?{_SENSITIVE_WRITE_TARGET}', "overwrite system file via tee"),
|
||||
(rf'>>?\s*["\']?{_SENSITIVE_WRITE_TARGET}', "overwrite system file via redirection"),
|
||||
(r'\bxargs\s+.*\brm\b', "xargs with rm"),
|
||||
(r'\bfind\b.*-exec\s+(/\S*/)?rm\b', "find -exec rm"),
|
||||
(r'\bfind\b.*-delete\b', "find -delete"),
|
||||
# Gateway protection: never start gateway outside systemd management
|
||||
(r'gateway\s+run\b.*(&\s*$|&\s*;|\bdisown\b|\bsetsid\b)', "start gateway outside systemd (use 'systemctl --user restart hermes-gateway')"),
|
||||
(r'\bnohup\b.*gateway\s+run\b', "start gateway outside systemd (use 'systemctl --user restart hermes-gateway')"),
|
||||
# Self-termination protection: prevent agent from killing its own process
|
||||
(r'\b(pkill|killall)\b.*\b(hermes|gateway|cli\.py)\b', "kill hermes/gateway process (self-termination)"),
|
||||
# Self-termination via kill + command substitution (pgrep/pidof).
|
||||
# The name-based pattern above catches `pkill hermes` but not
|
||||
# `kill -9 $(pgrep -f hermes)` because the substitution is opaque
|
||||
# to regex at detection time. Catch the structural pattern instead.
|
||||
(r'\bkill\b.*\$\(\s*pgrep\b', "kill process via pgrep expansion (self-termination)"),
|
||||
(r'\bkill\b.*`\s*pgrep\b', "kill process via backtick pgrep expansion (self-termination)"),
|
||||
# File copy/move/edit into sensitive system paths
|
||||
(r'\b(cp|mv|install)\b.*\s/etc/', "copy/move file into /etc/"),
|
||||
(r'\bsed\s+-[^\s]*i.*\s/etc/', "in-place edit of system config"),
|
||||
(r'\bsed\s+--in-place\b.*\s/etc/', "in-place edit of system config (long flag)"),
|
||||
# Script execution via heredoc — bypasses the -e/-c flag patterns above.
|
||||
# `python3 << 'EOF'` feeds arbitrary code via stdin without -c/-e flags.
|
||||
(r'\b(python[23]?|perl|ruby|node)\s+<<', "script execution via heredoc"),
|
||||
# Git destructive operations that can lose uncommitted work or rewrite
|
||||
# shared history. Not captured by rm/chmod/etc patterns.
|
||||
(r'\bgit\s+reset\s+--hard\b', "git reset --hard (destroys uncommitted changes)"),
|
||||
(r'\bgit\s+push\b.*--force\b', "git force push (rewrites remote history)"),
|
||||
(r'\bgit\s+push\b.*-f\b', "git force push short flag (rewrites remote history)"),
|
||||
(r'\bgit\s+clean\s+-[^\s]*f', "git clean with force (deletes untracked files)"),
|
||||
(r'\bgit\s+branch\s+-D\b', "git branch force delete"),
|
||||
# Script execution after chmod +x — catches the two-step pattern where
|
||||
# a script is first made executable then immediately run. The script
|
||||
# content may contain dangerous commands that individual patterns miss.
|
||||
(r'\bchmod\s+\+x\b.*[;&|]+\s*\./', "chmod +x followed by immediate execution"),
|
||||
]
|
||||
|
||||
|
||||
def _legacy_pattern_key(pattern: str) -> str:
|
||||
"""Reproduce the old regex-derived approval key for backwards compatibility."""
|
||||
return pattern.split(r'\b')[1] if r'\b' in pattern else pattern[:20]
|
||||
|
||||
|
||||
_PATTERN_KEY_ALIASES: dict[str, set[str]] = {}
|
||||
for _pattern, _description in DANGEROUS_PATTERNS:
|
||||
_legacy_key = _legacy_pattern_key(_pattern)
|
||||
_canonical_key = _description
|
||||
_PATTERN_KEY_ALIASES.setdefault(_canonical_key, set()).update({_canonical_key, _legacy_key})
|
||||
_PATTERN_KEY_ALIASES.setdefault(_legacy_key, set()).update({_legacy_key, _canonical_key})
|
||||
|
||||
|
||||
def _approval_key_aliases(pattern_key: str) -> set[str]:
|
||||
"""Return all approval keys that should match this pattern.
|
||||
|
||||
New approvals use the human-readable description string, but older
|
||||
command_allowlist entries and session approvals may still contain the
|
||||
historical regex-derived key.
|
||||
"""
|
||||
return _PATTERN_KEY_ALIASES.get(pattern_key, {pattern_key})
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Detection
|
||||
# =========================================================================
|
||||
|
||||
def _normalize_command_for_detection(command: str) -> str:
|
||||
"""Normalize a command string before dangerous-pattern matching.
|
||||
|
||||
Strips ANSI escape sequences (full ECMA-48 via tools.ansi_strip),
|
||||
null bytes, and normalizes Unicode fullwidth characters so that
|
||||
obfuscation techniques cannot bypass the pattern-based detection.
|
||||
"""
|
||||
from tools.ansi_strip import strip_ansi
|
||||
|
||||
# Strip all ANSI escape sequences (CSI, OSC, DCS, 8-bit C1, etc.)
|
||||
command = strip_ansi(command)
|
||||
# Strip null bytes
|
||||
command = command.replace('\x00', '')
|
||||
# Normalize Unicode (fullwidth Latin, halfwidth Katakana, etc.)
|
||||
command = unicodedata.normalize('NFKC', command)
|
||||
return command
|
||||
|
||||
|
||||
def detect_dangerous_command(command: str) -> tuple:
|
||||
"""Check if a command matches any dangerous patterns.
|
||||
|
||||
Returns:
|
||||
(is_dangerous, pattern_key, description) or (False, None, None)
|
||||
"""
|
||||
command_lower = _normalize_command_for_detection(command).lower()
|
||||
for pattern, description in DANGEROUS_PATTERNS:
|
||||
if re.search(pattern, command_lower, re.IGNORECASE | re.DOTALL):
|
||||
pattern_key = description
|
||||
return (True, pattern_key, description)
|
||||
return (False, None, None)
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Per-session approval state (thread-safe)
|
||||
# =========================================================================
|
||||
|
||||
_lock = threading.Lock()
|
||||
_pending: dict[str, dict] = {}
|
||||
_session_approved: dict[str, set] = {}
|
||||
_session_yolo: set[str] = set()
|
||||
_permanent_approved: set = set()
|
||||
|
||||
# =========================================================================
|
||||
# Blocking gateway approval (mirrors CLI's synchronous input() flow)
|
||||
# =========================================================================
|
||||
# Per-session QUEUE of pending approvals. Multiple threads (parallel
|
||||
# subagents, execute_code RPC handlers) can block concurrently — each gets
|
||||
# its own threading.Event. /approve resolves the oldest, /approve all
|
||||
# resolves every pending approval in the session.
|
||||
|
||||
|
||||
class _ApprovalEntry:
|
||||
"""One pending dangerous-command approval inside a gateway session."""
|
||||
__slots__ = ("event", "data", "result")
|
||||
|
||||
def __init__(self, data: dict):
|
||||
self.event = threading.Event()
|
||||
self.data = data # command, description, pattern_keys, …
|
||||
self.result: Optional[str] = None # "once"|"session"|"always"|"deny"
|
||||
|
||||
|
||||
_gateway_queues: dict[str, list] = {} # session_key → [_ApprovalEntry, …]
|
||||
_gateway_notify_cbs: dict[str, object] = {} # session_key → callable(approval_data)
|
||||
|
||||
|
||||
def register_gateway_notify(session_key: str, cb) -> None:
|
||||
"""Register a per-session callback for sending approval requests to the user.
|
||||
|
||||
The callback signature is ``cb(approval_data: dict) -> None`` where
|
||||
*approval_data* contains ``command``, ``description``, and
|
||||
``pattern_keys``. The callback bridges sync→async (runs in the agent
|
||||
thread, must schedule the actual send on the event loop).
|
||||
"""
|
||||
with _lock:
|
||||
_gateway_notify_cbs[session_key] = cb
|
||||
|
||||
|
||||
def unregister_gateway_notify(session_key: str) -> None:
|
||||
"""Unregister the per-session gateway approval callback.
|
||||
|
||||
Signals ALL blocked threads for this session so they don't hang forever
|
||||
(e.g. when the agent run finishes or is interrupted).
|
||||
"""
|
||||
with _lock:
|
||||
_gateway_notify_cbs.pop(session_key, None)
|
||||
entries = _gateway_queues.pop(session_key, [])
|
||||
for entry in entries:
|
||||
entry.event.set()
|
||||
|
||||
|
||||
def resolve_gateway_approval(session_key: str, choice: str,
|
||||
resolve_all: bool = False) -> int:
|
||||
"""Called by the gateway's /approve or /deny handler to unblock
|
||||
waiting agent thread(s).
|
||||
|
||||
When *resolve_all* is True every pending approval in the session is
|
||||
resolved at once (``/approve all``). Otherwise only the oldest one
|
||||
is resolved (FIFO).
|
||||
|
||||
Returns the number of approvals resolved (0 means nothing was pending).
|
||||
"""
|
||||
with _lock:
|
||||
queue = _gateway_queues.get(session_key)
|
||||
if not queue:
|
||||
return 0
|
||||
if resolve_all:
|
||||
targets = list(queue)
|
||||
queue.clear()
|
||||
else:
|
||||
targets = [queue.pop(0)]
|
||||
if not queue:
|
||||
_gateway_queues.pop(session_key, None)
|
||||
|
||||
for entry in targets:
|
||||
entry.result = choice
|
||||
entry.event.set()
|
||||
return len(targets)
|
||||
|
||||
|
||||
def has_blocking_approval(session_key: str) -> bool:
|
||||
"""Check if a session has one or more blocking gateway approvals waiting."""
|
||||
with _lock:
|
||||
return bool(_gateway_queues.get(session_key))
|
||||
|
||||
|
||||
def submit_pending(session_key: str, approval: dict):
|
||||
"""Store a pending approval request for a session."""
|
||||
with _lock:
|
||||
_pending[session_key] = approval
|
||||
|
||||
|
||||
def approve_session(session_key: str, pattern_key: str):
|
||||
"""Approve a pattern for this session only."""
|
||||
with _lock:
|
||||
_session_approved.setdefault(session_key, set()).add(pattern_key)
|
||||
|
||||
|
||||
def enable_session_yolo(session_key: str) -> None:
|
||||
"""Enable YOLO bypass for a single session key."""
|
||||
if not session_key:
|
||||
return
|
||||
with _lock:
|
||||
_session_yolo.add(session_key)
|
||||
|
||||
|
||||
def disable_session_yolo(session_key: str) -> None:
|
||||
"""Disable YOLO bypass for a single session key."""
|
||||
if not session_key:
|
||||
return
|
||||
with _lock:
|
||||
_session_yolo.discard(session_key)
|
||||
|
||||
|
||||
def clear_session(session_key: str) -> None:
|
||||
"""Remove all approval and yolo state for a given session."""
|
||||
if not session_key:
|
||||
return
|
||||
with _lock:
|
||||
_session_approved.pop(session_key, None)
|
||||
_session_yolo.discard(session_key)
|
||||
_pending.pop(session_key, None)
|
||||
_gateway_queues.pop(session_key, None)
|
||||
|
||||
|
||||
def is_session_yolo_enabled(session_key: str) -> bool:
|
||||
"""Return True when YOLO bypass is enabled for a specific session."""
|
||||
if not session_key:
|
||||
return False
|
||||
with _lock:
|
||||
return session_key in _session_yolo
|
||||
|
||||
|
||||
def is_current_session_yolo_enabled() -> bool:
|
||||
"""Return True when the active approval session has YOLO bypass enabled."""
|
||||
return is_session_yolo_enabled(get_current_session_key(default=""))
|
||||
|
||||
|
||||
def is_approved(session_key: str, pattern_key: str) -> bool:
|
||||
"""Check if a pattern is approved (session-scoped or permanent).
|
||||
|
||||
Accept both the current canonical key and the legacy regex-derived key so
|
||||
existing command_allowlist entries continue to work after key migrations.
|
||||
"""
|
||||
aliases = _approval_key_aliases(pattern_key)
|
||||
with _lock:
|
||||
if any(alias in _permanent_approved for alias in aliases):
|
||||
return True
|
||||
session_approvals = _session_approved.get(session_key, set())
|
||||
return any(alias in session_approvals for alias in aliases)
|
||||
|
||||
|
||||
def approve_permanent(pattern_key: str):
|
||||
"""Add a pattern to the permanent allowlist."""
|
||||
with _lock:
|
||||
_permanent_approved.add(pattern_key)
|
||||
|
||||
|
||||
def load_permanent(patterns: set):
|
||||
"""Bulk-load permanent allowlist entries from config."""
|
||||
with _lock:
|
||||
_permanent_approved.update(patterns)
|
||||
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Config persistence for permanent allowlist
|
||||
# =========================================================================
|
||||
|
||||
def load_permanent_allowlist() -> set:
|
||||
"""Load permanently allowed command patterns from config.
|
||||
|
||||
Also syncs them into the approval module so is_approved() works for
|
||||
patterns added via 'always' in a previous session.
|
||||
"""
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
config = load_config()
|
||||
patterns = set(config.get("command_allowlist", []) or [])
|
||||
if patterns:
|
||||
load_permanent(patterns)
|
||||
return patterns
|
||||
except Exception as e:
|
||||
logger.warning("Failed to load permanent allowlist: %s", e)
|
||||
return set()
|
||||
|
||||
|
||||
def save_permanent_allowlist(patterns: set):
|
||||
"""Save permanently allowed command patterns to config."""
|
||||
try:
|
||||
from hermes_cli.config import load_config, save_config
|
||||
config = load_config()
|
||||
config["command_allowlist"] = list(patterns)
|
||||
save_config(config)
|
||||
except Exception as e:
|
||||
logger.warning("Could not save allowlist: %s", e)
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Approval prompting + orchestration
|
||||
# =========================================================================
|
||||
|
||||
def prompt_dangerous_approval(command: str, description: str,
|
||||
timeout_seconds: int | None = None,
|
||||
allow_permanent: bool = True,
|
||||
approval_callback=None) -> str:
|
||||
"""Prompt the user to approve a dangerous command (CLI only).
|
||||
|
||||
Args:
|
||||
allow_permanent: When False, hide the [a]lways option (used when
|
||||
tirith warnings are present, since broad permanent allowlisting
|
||||
is inappropriate for content-level security findings).
|
||||
approval_callback: Optional callback registered by the CLI for
|
||||
prompt_toolkit integration. Signature:
|
||||
(command, description, *, allow_permanent=True) -> str.
|
||||
|
||||
Returns: 'once', 'session', 'always', or 'deny'
|
||||
"""
|
||||
if timeout_seconds is None:
|
||||
timeout_seconds = _get_approval_timeout()
|
||||
|
||||
if approval_callback is not None:
|
||||
try:
|
||||
return approval_callback(command, description,
|
||||
allow_permanent=allow_permanent)
|
||||
except Exception as e:
|
||||
logger.error("Approval callback failed: %s", e, exc_info=True)
|
||||
return "deny"
|
||||
|
||||
os.environ["HERMES_SPINNER_PAUSE"] = "1"
|
||||
try:
|
||||
while True:
|
||||
print()
|
||||
print(f" ⚠️ DANGEROUS COMMAND: {description}")
|
||||
print(f" {command}")
|
||||
print()
|
||||
if allow_permanent:
|
||||
print(" [o]nce | [s]ession | [a]lways | [d]eny")
|
||||
else:
|
||||
print(" [o]nce | [s]ession | [d]eny")
|
||||
print()
|
||||
sys.stdout.flush()
|
||||
|
||||
result = {"choice": ""}
|
||||
|
||||
def get_input():
|
||||
try:
|
||||
prompt = " Choice [o/s/a/D]: " if allow_permanent else " Choice [o/s/D]: "
|
||||
result["choice"] = input(prompt).strip().lower()
|
||||
except (EOFError, OSError):
|
||||
result["choice"] = ""
|
||||
|
||||
thread = threading.Thread(target=get_input, daemon=True)
|
||||
thread.start()
|
||||
thread.join(timeout=timeout_seconds)
|
||||
|
||||
if thread.is_alive():
|
||||
print("\n ⏱ Timeout - denying command")
|
||||
return "deny"
|
||||
|
||||
choice = result["choice"]
|
||||
if choice in ('o', 'once'):
|
||||
print(" ✓ Allowed once")
|
||||
return "once"
|
||||
elif choice in ('s', 'session'):
|
||||
print(" ✓ Allowed for this session")
|
||||
return "session"
|
||||
elif choice in ('a', 'always'):
|
||||
if not allow_permanent:
|
||||
print(" ✓ Allowed for this session")
|
||||
return "session"
|
||||
print(" ✓ Added to permanent allowlist")
|
||||
return "always"
|
||||
else:
|
||||
print(" ✗ Denied")
|
||||
return "deny"
|
||||
|
||||
except (EOFError, KeyboardInterrupt):
|
||||
print("\n ✗ Cancelled")
|
||||
return "deny"
|
||||
finally:
|
||||
if "HERMES_SPINNER_PAUSE" in os.environ:
|
||||
del os.environ["HERMES_SPINNER_PAUSE"]
|
||||
print()
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def _normalize_approval_mode(mode) -> str:
|
||||
"""Normalize approval mode values loaded from YAML/config.
|
||||
|
||||
YAML 1.1 treats bare words like `off` as booleans, so a config entry like
|
||||
`approvals:\n mode: off` is parsed as False unless quoted. Treat that as the
|
||||
intended string mode instead of falling back to manual approvals.
|
||||
"""
|
||||
if isinstance(mode, bool):
|
||||
return "off" if mode is False else "manual"
|
||||
if isinstance(mode, str):
|
||||
normalized = mode.strip().lower()
|
||||
return normalized or "manual"
|
||||
return "manual"
|
||||
|
||||
|
||||
def _get_approval_config() -> dict:
|
||||
"""Read the approvals config block. Returns a dict with 'mode', 'timeout', etc."""
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
config = load_config()
|
||||
return config.get("approvals", {}) or {}
|
||||
except Exception as e:
|
||||
logger.warning("Failed to load approval config: %s", e)
|
||||
return {}
|
||||
|
||||
|
||||
def _get_approval_mode() -> str:
|
||||
"""Read the approval mode from config. Returns 'manual', 'smart', or 'off'."""
|
||||
mode = _get_approval_config().get("mode", "manual")
|
||||
return _normalize_approval_mode(mode)
|
||||
|
||||
|
||||
def _get_approval_timeout() -> int:
|
||||
"""Read the approval timeout from config. Defaults to 60 seconds."""
|
||||
try:
|
||||
return int(_get_approval_config().get("timeout", 60))
|
||||
except (ValueError, TypeError):
|
||||
return 60
|
||||
|
||||
|
||||
def _smart_approve(command: str, description: str) -> str:
|
||||
"""Use the auxiliary LLM to assess risk and decide approval.
|
||||
|
||||
Returns 'approve' if the LLM determines the command is safe,
|
||||
'deny' if genuinely dangerous, or 'escalate' if uncertain.
|
||||
|
||||
Inspired by OpenAI Codex's Smart Approvals guardian subagent
|
||||
(openai/codex#13860).
|
||||
"""
|
||||
try:
|
||||
from agent.auxiliary_client import get_text_auxiliary_client, auxiliary_max_tokens_param
|
||||
|
||||
client, model = get_text_auxiliary_client(task="approval")
|
||||
if not client or not model:
|
||||
logger.debug("Smart approvals: no aux client available, escalating")
|
||||
return "escalate"
|
||||
|
||||
prompt = f"""You are a security reviewer for an AI coding agent. A terminal command was flagged by pattern matching as potentially dangerous.
|
||||
|
||||
Command: {command}
|
||||
Flagged reason: {description}
|
||||
|
||||
Assess the ACTUAL risk of this command. Many flagged commands are false positives — for example, `python -c "print('hello')"` is flagged as "script execution via -c flag" but is completely harmless.
|
||||
|
||||
Rules:
|
||||
- APPROVE if the command is clearly safe (benign script execution, safe file operations, development tools, package installs, git operations, etc.)
|
||||
- DENY if the command could genuinely damage the system (recursive delete of important paths, overwriting system files, fork bombs, wiping disks, dropping databases, etc.)
|
||||
- ESCALATE if you're uncertain
|
||||
|
||||
Respond with exactly one word: APPROVE, DENY, or ESCALATE"""
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
**auxiliary_max_tokens_param(16),
|
||||
temperature=0,
|
||||
)
|
||||
|
||||
answer = (response.choices[0].message.content or "").strip().upper()
|
||||
|
||||
if "APPROVE" in answer:
|
||||
return "approve"
|
||||
elif "DENY" in answer:
|
||||
return "deny"
|
||||
else:
|
||||
return "escalate"
|
||||
|
||||
except Exception as e:
|
||||
logger.debug("Smart approvals: LLM call failed (%s), escalating", e)
|
||||
return "escalate"
|
||||
|
||||
|
||||
def check_dangerous_command(command: str, env_type: str,
|
||||
approval_callback=None) -> dict:
|
||||
"""Check if a command is dangerous and handle approval.
|
||||
|
||||
This is the main entry point called by terminal_tool before executing
|
||||
any command. It orchestrates detection, session checks, and prompting.
|
||||
|
||||
Args:
|
||||
command: The shell command to check.
|
||||
env_type: Terminal backend type ('local', 'ssh', 'docker', etc.).
|
||||
approval_callback: Optional CLI callback for interactive prompts.
|
||||
|
||||
Returns:
|
||||
{"approved": True/False, "message": str or None, ...}
|
||||
"""
|
||||
if env_type in ("docker", "singularity", "modal", "daytona"):
|
||||
return {"approved": True, "message": None}
|
||||
|
||||
# --yolo: bypass all approval prompts. Gateway /yolo is session-scoped;
|
||||
# CLI --yolo remains process-scoped via the env var for local use.
|
||||
if os.getenv("HERMES_YOLO_MODE") or is_current_session_yolo_enabled():
|
||||
return {"approved": True, "message": None}
|
||||
|
||||
is_dangerous, pattern_key, description = detect_dangerous_command(command)
|
||||
if not is_dangerous:
|
||||
return {"approved": True, "message": None}
|
||||
|
||||
session_key = get_current_session_key()
|
||||
if is_approved(session_key, pattern_key):
|
||||
return {"approved": True, "message": None}
|
||||
|
||||
is_cli = os.getenv("HERMES_INTERACTIVE")
|
||||
is_gateway = os.getenv("HERMES_GATEWAY_SESSION")
|
||||
|
||||
if not is_cli and not is_gateway:
|
||||
return {"approved": True, "message": None}
|
||||
|
||||
if is_gateway or os.getenv("HERMES_EXEC_ASK"):
|
||||
submit_pending(session_key, {
|
||||
"command": command,
|
||||
"pattern_key": pattern_key,
|
||||
"description": description,
|
||||
})
|
||||
return {
|
||||
"approved": False,
|
||||
"pattern_key": pattern_key,
|
||||
"status": "approval_required",
|
||||
"command": command,
|
||||
"description": description,
|
||||
"message": (
|
||||
f"⚠️ This command is potentially dangerous ({description}). "
|
||||
f"Asking the user for approval.\n\n**Command:**\n```\n{command}\n```"
|
||||
),
|
||||
}
|
||||
|
||||
choice = prompt_dangerous_approval(command, description,
|
||||
approval_callback=approval_callback)
|
||||
|
||||
if choice == "deny":
|
||||
return {
|
||||
"approved": False,
|
||||
"message": f"BLOCKED: User denied this potentially dangerous command (matched '{description}' pattern). Do NOT retry this command - the user has explicitly rejected it.",
|
||||
"pattern_key": pattern_key,
|
||||
"description": description,
|
||||
}
|
||||
|
||||
if choice == "session":
|
||||
approve_session(session_key, pattern_key)
|
||||
elif choice == "always":
|
||||
approve_session(session_key, pattern_key)
|
||||
approve_permanent(pattern_key)
|
||||
save_permanent_allowlist(_permanent_approved)
|
||||
|
||||
return {"approved": True, "message": None}
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Combined pre-exec guard (tirith + dangerous command detection)
|
||||
# =========================================================================
|
||||
|
||||
def _format_tirith_description(tirith_result: dict) -> str:
|
||||
"""Build a human-readable description from tirith findings.
|
||||
|
||||
Includes severity, title, and description for each finding so users
|
||||
can make an informed approval decision.
|
||||
"""
|
||||
findings = tirith_result.get("findings") or []
|
||||
if not findings:
|
||||
summary = tirith_result.get("summary") or "security issue detected"
|
||||
return f"Security scan: {summary}"
|
||||
|
||||
parts = []
|
||||
for f in findings:
|
||||
severity = f.get("severity", "")
|
||||
title = f.get("title", "")
|
||||
desc = f.get("description", "")
|
||||
if title and desc:
|
||||
parts.append(f"[{severity}] {title}: {desc}" if severity else f"{title}: {desc}")
|
||||
elif title:
|
||||
parts.append(f"[{severity}] {title}" if severity else title)
|
||||
if not parts:
|
||||
summary = tirith_result.get("summary") or "security issue detected"
|
||||
return f"Security scan: {summary}"
|
||||
|
||||
return "Security scan — " + "; ".join(parts)
|
||||
|
||||
|
||||
def check_all_command_guards(command: str, env_type: str,
|
||||
approval_callback=None) -> dict:
|
||||
"""Run all pre-exec security checks and return a single approval decision.
|
||||
|
||||
Gathers findings from tirith and dangerous-command detection, then
|
||||
presents them as a single combined approval request. This prevents
|
||||
a gateway force=True replay from bypassing one check when only the
|
||||
other was shown to the user.
|
||||
"""
|
||||
# Skip containers for both checks
|
||||
if env_type in ("docker", "singularity", "modal", "daytona"):
|
||||
return {"approved": True, "message": None}
|
||||
|
||||
# --yolo or approvals.mode=off: bypass all approval prompts.
|
||||
# Gateway /yolo is session-scoped; CLI --yolo remains process-scoped.
|
||||
approval_mode = _get_approval_mode()
|
||||
if os.getenv("HERMES_YOLO_MODE") or is_current_session_yolo_enabled() or approval_mode == "off":
|
||||
return {"approved": True, "message": None}
|
||||
|
||||
is_cli = os.getenv("HERMES_INTERACTIVE")
|
||||
is_gateway = os.getenv("HERMES_GATEWAY_SESSION")
|
||||
is_ask = os.getenv("HERMES_EXEC_ASK")
|
||||
|
||||
# Preserve the existing non-interactive behavior: outside CLI/gateway/ask
|
||||
# flows, we do not block on approvals and we skip external guard work.
|
||||
if not is_cli and not is_gateway and not is_ask:
|
||||
return {"approved": True, "message": None}
|
||||
|
||||
# --- Phase 1: Gather findings from both checks ---
|
||||
|
||||
# Tirith check — wrapper guarantees no raise for expected failures.
|
||||
# Only catch ImportError (module not installed).
|
||||
tirith_result = {"action": "allow", "findings": [], "summary": ""}
|
||||
try:
|
||||
from tools.tirith_security import check_command_security
|
||||
tirith_result = check_command_security(command)
|
||||
except ImportError:
|
||||
pass # tirith module not installed — allow
|
||||
|
||||
# Dangerous command check (detection only, no approval)
|
||||
is_dangerous, pattern_key, description = detect_dangerous_command(command)
|
||||
|
||||
# --- Phase 2: Decide ---
|
||||
|
||||
# Collect warnings that need approval
|
||||
warnings = [] # list of (pattern_key, description, is_tirith)
|
||||
|
||||
session_key = get_current_session_key()
|
||||
|
||||
# Tirith block/warn → approvable warning with rich findings.
|
||||
# Previously, tirith "block" was a hard block with no approval prompt.
|
||||
# Now both block and warn go through the approval flow so users can
|
||||
# inspect the explanation and approve if they understand the risk.
|
||||
if tirith_result["action"] in ("block", "warn"):
|
||||
findings = tirith_result.get("findings") or []
|
||||
rule_id = findings[0].get("rule_id", "unknown") if findings else "unknown"
|
||||
tirith_key = f"tirith:{rule_id}"
|
||||
tirith_desc = _format_tirith_description(tirith_result)
|
||||
if not is_approved(session_key, tirith_key):
|
||||
warnings.append((tirith_key, tirith_desc, True))
|
||||
|
||||
if is_dangerous:
|
||||
if not is_approved(session_key, pattern_key):
|
||||
warnings.append((pattern_key, description, False))
|
||||
|
||||
# Nothing to warn about
|
||||
if not warnings:
|
||||
return {"approved": True, "message": None}
|
||||
|
||||
# --- Phase 2.5: Smart approval (auxiliary LLM risk assessment) ---
|
||||
# When approvals.mode=smart, ask the aux LLM before prompting the user.
|
||||
# Inspired by OpenAI Codex's Smart Approvals guardian subagent
|
||||
# (openai/codex#13860).
|
||||
if approval_mode == "smart":
|
||||
combined_desc_for_llm = "; ".join(desc for _, desc, _ in warnings)
|
||||
verdict = _smart_approve(command, combined_desc_for_llm)
|
||||
if verdict == "approve":
|
||||
# Auto-approve and grant session-level approval for these patterns
|
||||
for key, _, _ in warnings:
|
||||
approve_session(session_key, key)
|
||||
logger.debug("Smart approval: auto-approved '%s' (%s)",
|
||||
command[:60], combined_desc_for_llm)
|
||||
return {"approved": True, "message": None,
|
||||
"smart_approved": True,
|
||||
"description": combined_desc_for_llm}
|
||||
elif verdict == "deny":
|
||||
combined_desc_for_llm = "; ".join(desc for _, desc, _ in warnings)
|
||||
return {
|
||||
"approved": False,
|
||||
"message": f"BLOCKED by smart approval: {combined_desc_for_llm}. "
|
||||
"The command was assessed as genuinely dangerous. Do NOT retry.",
|
||||
"smart_denied": True,
|
||||
}
|
||||
# verdict == "escalate" → fall through to manual prompt
|
||||
|
||||
# --- Phase 3: Approval ---
|
||||
|
||||
# Combine descriptions for a single approval prompt
|
||||
combined_desc = "; ".join(desc for _, desc, _ in warnings)
|
||||
primary_key = warnings[0][0]
|
||||
all_keys = [key for key, _, _ in warnings]
|
||||
has_tirith = any(is_t for _, _, is_t in warnings)
|
||||
|
||||
# Gateway/async approval — block the agent thread until the user
|
||||
# responds with /approve or /deny, mirroring the CLI's synchronous
|
||||
# input() flow. The agent never sees "approval_required"; it either
|
||||
# gets the command output (approved) or a definitive "BLOCKED" message.
|
||||
if is_gateway or is_ask:
|
||||
notify_cb = None
|
||||
with _lock:
|
||||
notify_cb = _gateway_notify_cbs.get(session_key)
|
||||
|
||||
if notify_cb is not None:
|
||||
# --- Blocking gateway approval (queue-based) ---
|
||||
# Each call gets its own _ApprovalEntry so parallel subagents
|
||||
# and execute_code threads can block concurrently.
|
||||
approval_data = {
|
||||
"command": command,
|
||||
"pattern_key": primary_key,
|
||||
"pattern_keys": all_keys,
|
||||
"description": combined_desc,
|
||||
}
|
||||
entry = _ApprovalEntry(approval_data)
|
||||
with _lock:
|
||||
_gateway_queues.setdefault(session_key, []).append(entry)
|
||||
|
||||
# Notify the user (bridges sync agent thread → async gateway)
|
||||
try:
|
||||
notify_cb(approval_data)
|
||||
except Exception as exc:
|
||||
logger.warning("Gateway approval notify failed: %s", exc)
|
||||
with _lock:
|
||||
queue = _gateway_queues.get(session_key, [])
|
||||
if entry in queue:
|
||||
queue.remove(entry)
|
||||
if not queue:
|
||||
_gateway_queues.pop(session_key, None)
|
||||
return {
|
||||
"approved": False,
|
||||
"message": "BLOCKED: Failed to send approval request to user. Do NOT retry.",
|
||||
"pattern_key": primary_key,
|
||||
"description": combined_desc,
|
||||
}
|
||||
|
||||
# Block until the user responds or timeout (default 5 min)
|
||||
timeout = _get_approval_config().get("gateway_timeout", 300)
|
||||
try:
|
||||
timeout = int(timeout)
|
||||
except (ValueError, TypeError):
|
||||
timeout = 300
|
||||
resolved = entry.event.wait(timeout=timeout)
|
||||
|
||||
# Clean up this entry from the queue
|
||||
with _lock:
|
||||
queue = _gateway_queues.get(session_key, [])
|
||||
if entry in queue:
|
||||
queue.remove(entry)
|
||||
if not queue:
|
||||
_gateway_queues.pop(session_key, None)
|
||||
|
||||
choice = entry.result
|
||||
if not resolved or choice is None or choice == "deny":
|
||||
reason = "timed out" if not resolved else "denied by user"
|
||||
return {
|
||||
"approved": False,
|
||||
"message": f"BLOCKED: Command {reason}. Do NOT retry this command.",
|
||||
"pattern_key": primary_key,
|
||||
"description": combined_desc,
|
||||
}
|
||||
|
||||
# User approved — persist based on scope (same logic as CLI)
|
||||
for key, _, is_tirith in warnings:
|
||||
if choice == "session" or (choice == "always" and is_tirith):
|
||||
approve_session(session_key, key)
|
||||
elif choice == "always":
|
||||
approve_session(session_key, key)
|
||||
approve_permanent(key)
|
||||
save_permanent_allowlist(_permanent_approved)
|
||||
# choice == "once": no persistence — command allowed this
|
||||
# single time only, matching the CLI's behavior.
|
||||
|
||||
return {"approved": True, "message": None,
|
||||
"user_approved": True, "description": combined_desc}
|
||||
|
||||
# Fallback: no gateway callback registered (e.g. cron, batch).
|
||||
# Return approval_required for backward compat.
|
||||
submit_pending(session_key, {
|
||||
"command": command,
|
||||
"pattern_key": primary_key,
|
||||
"pattern_keys": all_keys,
|
||||
"description": combined_desc,
|
||||
})
|
||||
return {
|
||||
"approved": False,
|
||||
"pattern_key": primary_key,
|
||||
"status": "approval_required",
|
||||
"command": command,
|
||||
"description": combined_desc,
|
||||
"message": (
|
||||
f"⚠️ {combined_desc}. Asking the user for approval.\n\n**Command:**\n```\n{command}\n```"
|
||||
),
|
||||
}
|
||||
|
||||
# CLI interactive: single combined prompt
|
||||
# Hide [a]lways when any tirith warning is present
|
||||
choice = prompt_dangerous_approval(command, combined_desc,
|
||||
allow_permanent=not has_tirith,
|
||||
approval_callback=approval_callback)
|
||||
|
||||
if choice == "deny":
|
||||
return {
|
||||
"approved": False,
|
||||
"message": "BLOCKED: User denied. Do NOT retry.",
|
||||
"pattern_key": primary_key,
|
||||
"description": combined_desc,
|
||||
}
|
||||
|
||||
# Persist approval for each warning individually
|
||||
for key, _, is_tirith in warnings:
|
||||
if choice == "session" or (choice == "always" and is_tirith):
|
||||
# tirith: session only (no permanent broad allowlisting)
|
||||
approve_session(session_key, key)
|
||||
elif choice == "always":
|
||||
# dangerous patterns: permanent allowed
|
||||
approve_session(session_key, key)
|
||||
approve_permanent(key)
|
||||
save_permanent_allowlist(_permanent_approved)
|
||||
|
||||
return {"approved": True, "message": None,
|
||||
"user_approved": True, "description": combined_desc}
|
||||
|
||||
|
||||
# Load permanent allowlist from config on module import
|
||||
load_permanent_allowlist()
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Binary file extensions to skip for text-based operations.
|
||||
|
||||
These files can't be meaningfully compared as text and are often large.
|
||||
Ported from free-code src/constants/files.ts.
|
||||
"""
|
||||
|
||||
BINARY_EXTENSIONS = frozenset({
|
||||
# Images
|
||||
".png", ".jpg", ".jpeg", ".gif", ".bmp", ".ico", ".webp", ".tiff", ".tif",
|
||||
# Videos
|
||||
".mp4", ".mov", ".avi", ".mkv", ".webm", ".wmv", ".flv", ".m4v", ".mpeg", ".mpg",
|
||||
# Audio
|
||||
".mp3", ".wav", ".ogg", ".flac", ".aac", ".m4a", ".wma", ".aiff", ".opus",
|
||||
# Archives
|
||||
".zip", ".tar", ".gz", ".bz2", ".7z", ".rar", ".xz", ".z", ".tgz", ".iso",
|
||||
# Executables/binaries
|
||||
".exe", ".dll", ".so", ".dylib", ".bin", ".o", ".a", ".obj", ".lib",
|
||||
".app", ".msi", ".deb", ".rpm",
|
||||
# Documents (exclude .pdf — text-based, agents may want to inspect)
|
||||
".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx",
|
||||
".odt", ".ods", ".odp",
|
||||
# Fonts
|
||||
".ttf", ".otf", ".woff", ".woff2", ".eot",
|
||||
# Bytecode / VM artifacts
|
||||
".pyc", ".pyo", ".class", ".jar", ".war", ".ear", ".node", ".wasm", ".rlib",
|
||||
# Database files
|
||||
".sqlite", ".sqlite3", ".db", ".mdb", ".idx",
|
||||
# Design / 3D
|
||||
".psd", ".ai", ".eps", ".sketch", ".fig", ".xd", ".blend", ".3ds", ".max",
|
||||
# Flash
|
||||
".swf", ".fla",
|
||||
# Lock/profiling data
|
||||
".lockb", ".dat", ".data",
|
||||
})
|
||||
|
||||
|
||||
def has_binary_extension(path: str) -> bool:
|
||||
"""Check if a file path has a binary extension. Pure string check, no I/O."""
|
||||
dot = path.rfind(".")
|
||||
if dot == -1:
|
||||
return False
|
||||
return path[dot:].lower() in BINARY_EXTENSIONS
|
||||
@@ -0,0 +1,592 @@
|
||||
"""Camofox browser backend — local anti-detection browser via REST API.
|
||||
|
||||
Camofox-browser is a self-hosted Node.js server wrapping Camoufox (Firefox
|
||||
fork with C++ fingerprint spoofing). It exposes a REST API that maps 1:1
|
||||
to our browser tool interface: accessibility snapshots with element refs,
|
||||
click/type/scroll by ref, screenshots, etc.
|
||||
|
||||
When ``CAMOFOX_URL`` is set (e.g. ``http://localhost:9377``), the browser
|
||||
tools route through this module instead of the ``agent-browser`` CLI.
|
||||
|
||||
Setup::
|
||||
|
||||
# Option 1: npm
|
||||
git clone https://github.com/jo-inc/camofox-browser && cd camofox-browser
|
||||
npm install && npm start # downloads Camoufox (~300MB) on first run
|
||||
|
||||
# Option 2: Docker
|
||||
docker run -p 9377:9377 -e CAMOFOX_PORT=9377 jo-inc/camofox-browser
|
||||
|
||||
Then set ``CAMOFOX_URL=http://localhost:9377`` in ``~/.hermes/.env``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import uuid
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
import requests
|
||||
|
||||
from hermes_cli.config import load_config
|
||||
from tools.browser_camofox_state import get_camofox_identity
|
||||
from tools.registry import tool_error
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Configuration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_DEFAULT_TIMEOUT = 30 # seconds per HTTP request
|
||||
_SNAPSHOT_MAX_CHARS = 80_000 # camofox paginates at this limit
|
||||
_vnc_url: Optional[str] = None # cached from /health response
|
||||
_vnc_url_checked = False # only probe once per process
|
||||
|
||||
|
||||
def get_camofox_url() -> str:
|
||||
"""Return the configured Camofox server URL, or empty string."""
|
||||
return os.getenv("CAMOFOX_URL", "").rstrip("/")
|
||||
|
||||
|
||||
def is_camofox_mode() -> bool:
|
||||
"""True when Camofox backend is configured."""
|
||||
return bool(get_camofox_url())
|
||||
|
||||
|
||||
def check_camofox_available() -> bool:
|
||||
"""Verify the Camofox server is reachable."""
|
||||
global _vnc_url, _vnc_url_checked
|
||||
url = get_camofox_url()
|
||||
if not url:
|
||||
return False
|
||||
try:
|
||||
resp = requests.get(f"{url}/health", timeout=5)
|
||||
if resp.status_code == 200 and not _vnc_url_checked:
|
||||
try:
|
||||
data = resp.json()
|
||||
vnc_port = data.get("vncPort")
|
||||
if isinstance(vnc_port, int) and 1 <= vnc_port <= 65535:
|
||||
from urllib.parse import urlparse
|
||||
parsed = urlparse(url)
|
||||
host = parsed.hostname or "localhost"
|
||||
_vnc_url = f"http://{host}:{vnc_port}"
|
||||
except (ValueError, KeyError):
|
||||
pass
|
||||
_vnc_url_checked = True
|
||||
return resp.status_code == 200
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def get_vnc_url() -> Optional[str]:
|
||||
"""Return the VNC URL if the Camofox server exposes one, or None."""
|
||||
if not _vnc_url_checked:
|
||||
check_camofox_available()
|
||||
return _vnc_url
|
||||
|
||||
|
||||
def _managed_persistence_enabled() -> bool:
|
||||
"""Return whether Hermes-managed persistence is enabled for Camofox.
|
||||
|
||||
When enabled, sessions use a stable profile-scoped userId so the
|
||||
Camofox server can map it to a persistent browser profile directory.
|
||||
When disabled (default), each session gets a random userId (ephemeral).
|
||||
|
||||
Controlled by ``browser.camofox.managed_persistence`` in config.yaml.
|
||||
"""
|
||||
try:
|
||||
camofox_cfg = load_config().get("browser", {}).get("camofox", {})
|
||||
except Exception as exc:
|
||||
logger.warning("managed_persistence check failed, defaulting to disabled: %s", exc)
|
||||
return False
|
||||
return bool(camofox_cfg.get("managed_persistence"))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Session management
|
||||
# ---------------------------------------------------------------------------
|
||||
# Maps task_id -> {"user_id": str, "tab_id": str|None}
|
||||
_sessions: Dict[str, Dict[str, Any]] = {}
|
||||
_sessions_lock = threading.Lock()
|
||||
|
||||
|
||||
def _get_session(task_id: Optional[str]) -> Dict[str, Any]:
|
||||
"""Get or create a camofox session for the given task.
|
||||
|
||||
When managed persistence is enabled, uses a deterministic userId
|
||||
derived from the Hermes profile so the Camofox server can map it
|
||||
to the same persistent browser profile across restarts.
|
||||
"""
|
||||
task_id = task_id or "default"
|
||||
with _sessions_lock:
|
||||
if task_id in _sessions:
|
||||
return _sessions[task_id]
|
||||
if _managed_persistence_enabled():
|
||||
identity = get_camofox_identity(task_id)
|
||||
session = {
|
||||
"user_id": identity["user_id"],
|
||||
"tab_id": None,
|
||||
"session_key": identity["session_key"],
|
||||
"managed": True,
|
||||
}
|
||||
else:
|
||||
session = {
|
||||
"user_id": f"hermes_{uuid.uuid4().hex[:10]}",
|
||||
"tab_id": None,
|
||||
"session_key": f"task_{task_id[:16]}",
|
||||
"managed": False,
|
||||
}
|
||||
_sessions[task_id] = session
|
||||
return session
|
||||
|
||||
|
||||
def _ensure_tab(task_id: Optional[str], url: str = "about:blank") -> Dict[str, Any]:
|
||||
"""Ensure a tab exists for the session, creating one if needed."""
|
||||
session = _get_session(task_id)
|
||||
if session["tab_id"]:
|
||||
return session
|
||||
base = get_camofox_url()
|
||||
resp = requests.post(
|
||||
f"{base}/tabs",
|
||||
json={
|
||||
"userId": session["user_id"],
|
||||
"sessionKey": session["session_key"],
|
||||
"url": url,
|
||||
},
|
||||
timeout=_DEFAULT_TIMEOUT,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
session["tab_id"] = data.get("tabId")
|
||||
return session
|
||||
|
||||
|
||||
def _drop_session(task_id: Optional[str]) -> Optional[Dict[str, Any]]:
|
||||
"""Remove and return session info."""
|
||||
task_id = task_id or "default"
|
||||
with _sessions_lock:
|
||||
return _sessions.pop(task_id, None)
|
||||
|
||||
|
||||
def camofox_soft_cleanup(task_id: Optional[str] = None) -> bool:
|
||||
"""Release the in-memory session without destroying the server-side context.
|
||||
|
||||
When managed persistence is enabled the browser profile (and its cookies)
|
||||
must survive across agent tasks. This helper drops only the local tracking
|
||||
entry and returns ``True``. When managed persistence is *not* enabled it
|
||||
does nothing and returns ``False`` so the caller can fall back to
|
||||
:func:`camofox_close`.
|
||||
"""
|
||||
if _managed_persistence_enabled():
|
||||
_drop_session(task_id)
|
||||
logger.debug("Camofox soft cleanup for task %s (managed persistence)", task_id)
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HTTP helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _post(path: str, body: dict, timeout: int = _DEFAULT_TIMEOUT) -> dict:
|
||||
"""POST JSON to camofox and return parsed response."""
|
||||
url = f"{get_camofox_url()}{path}"
|
||||
resp = requests.post(url, json=body, timeout=timeout)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
|
||||
def _get(path: str, params: dict = None, timeout: int = _DEFAULT_TIMEOUT) -> dict:
|
||||
"""GET from camofox and return parsed response."""
|
||||
url = f"{get_camofox_url()}{path}"
|
||||
resp = requests.get(url, params=params, timeout=timeout)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
|
||||
def _get_raw(path: str, params: dict = None, timeout: int = _DEFAULT_TIMEOUT) -> requests.Response:
|
||||
"""GET from camofox and return raw response (for binary data)."""
|
||||
url = f"{get_camofox_url()}{path}"
|
||||
resp = requests.get(url, params=params, timeout=timeout)
|
||||
resp.raise_for_status()
|
||||
return resp
|
||||
|
||||
|
||||
def _delete(path: str, body: dict = None, timeout: int = _DEFAULT_TIMEOUT) -> dict:
|
||||
"""DELETE to camofox and return parsed response."""
|
||||
url = f"{get_camofox_url()}{path}"
|
||||
resp = requests.delete(url, json=body, timeout=timeout)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool implementations
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def camofox_navigate(url: str, task_id: Optional[str] = None) -> str:
|
||||
"""Navigate to a URL via Camofox."""
|
||||
try:
|
||||
session = _get_session(task_id)
|
||||
if not session["tab_id"]:
|
||||
# Create tab with the target URL directly
|
||||
session = _ensure_tab(task_id, url)
|
||||
data = {"ok": True, "url": url}
|
||||
else:
|
||||
# Navigate existing tab
|
||||
data = _post(
|
||||
f"/tabs/{session['tab_id']}/navigate",
|
||||
{"userId": session["user_id"], "url": url},
|
||||
timeout=60,
|
||||
)
|
||||
result = {
|
||||
"success": True,
|
||||
"url": data.get("url", url),
|
||||
"title": data.get("title", ""),
|
||||
}
|
||||
vnc = get_vnc_url()
|
||||
if vnc:
|
||||
result["vnc_url"] = vnc
|
||||
result["vnc_hint"] = (
|
||||
"Browser is visible via VNC. "
|
||||
"Share this link with the user so they can watch the browser live."
|
||||
)
|
||||
|
||||
# Auto-take a compact snapshot so the model can act immediately
|
||||
try:
|
||||
snap_data = _get(
|
||||
f"/tabs/{session['tab_id']}/snapshot",
|
||||
params={"userId": session["user_id"]},
|
||||
)
|
||||
snapshot_text = snap_data.get("snapshot", "")
|
||||
from tools.browser_tool import (
|
||||
SNAPSHOT_SUMMARIZE_THRESHOLD,
|
||||
_truncate_snapshot,
|
||||
)
|
||||
if len(snapshot_text) > SNAPSHOT_SUMMARIZE_THRESHOLD:
|
||||
snapshot_text = _truncate_snapshot(snapshot_text)
|
||||
result["snapshot"] = snapshot_text
|
||||
result["element_count"] = snap_data.get("refsCount", 0)
|
||||
except Exception:
|
||||
pass # Navigation succeeded; snapshot is a bonus
|
||||
|
||||
return json.dumps(result)
|
||||
except requests.HTTPError as e:
|
||||
return tool_error(f"Navigation failed: {e}", success=False)
|
||||
except requests.ConnectionError:
|
||||
return json.dumps({
|
||||
"success": False,
|
||||
"error": f"Cannot connect to Camofox at {get_camofox_url()}. "
|
||||
"Is the server running? Start with: npm start (in camofox-browser dir) "
|
||||
"or: docker run -p 9377:9377 -e CAMOFOX_PORT=9377 jo-inc/camofox-browser",
|
||||
})
|
||||
except Exception as e:
|
||||
return tool_error(str(e), success=False)
|
||||
|
||||
|
||||
def camofox_snapshot(full: bool = False, task_id: Optional[str] = None,
|
||||
user_task: Optional[str] = None) -> str:
|
||||
"""Get accessibility tree snapshot from Camofox."""
|
||||
try:
|
||||
session = _get_session(task_id)
|
||||
if not session["tab_id"]:
|
||||
return tool_error("No browser session. Call browser_navigate first.", success=False)
|
||||
|
||||
data = _get(
|
||||
f"/tabs/{session['tab_id']}/snapshot",
|
||||
params={"userId": session["user_id"]},
|
||||
)
|
||||
|
||||
snapshot = data.get("snapshot", "")
|
||||
refs_count = data.get("refsCount", 0)
|
||||
|
||||
# Apply same summarization logic as the main browser tool
|
||||
from tools.browser_tool import (
|
||||
SNAPSHOT_SUMMARIZE_THRESHOLD,
|
||||
_extract_relevant_content,
|
||||
_truncate_snapshot,
|
||||
)
|
||||
|
||||
if len(snapshot) > SNAPSHOT_SUMMARIZE_THRESHOLD:
|
||||
if user_task:
|
||||
snapshot = _extract_relevant_content(snapshot, user_task)
|
||||
else:
|
||||
snapshot = _truncate_snapshot(snapshot)
|
||||
|
||||
return json.dumps({
|
||||
"success": True,
|
||||
"snapshot": snapshot,
|
||||
"element_count": refs_count,
|
||||
})
|
||||
except Exception as e:
|
||||
return tool_error(str(e), success=False)
|
||||
|
||||
|
||||
def camofox_click(ref: str, task_id: Optional[str] = None) -> str:
|
||||
"""Click an element by ref via Camofox."""
|
||||
try:
|
||||
session = _get_session(task_id)
|
||||
if not session["tab_id"]:
|
||||
return tool_error("No browser session. Call browser_navigate first.", success=False)
|
||||
|
||||
# Strip @ prefix if present (our tool convention)
|
||||
clean_ref = ref.lstrip("@")
|
||||
|
||||
data = _post(
|
||||
f"/tabs/{session['tab_id']}/click",
|
||||
{"userId": session["user_id"], "ref": clean_ref},
|
||||
)
|
||||
return json.dumps({
|
||||
"success": True,
|
||||
"clicked": clean_ref,
|
||||
"url": data.get("url", ""),
|
||||
})
|
||||
except Exception as e:
|
||||
return tool_error(str(e), success=False)
|
||||
|
||||
|
||||
def camofox_type(ref: str, text: str, task_id: Optional[str] = None) -> str:
|
||||
"""Type text into an element by ref via Camofox."""
|
||||
try:
|
||||
session = _get_session(task_id)
|
||||
if not session["tab_id"]:
|
||||
return tool_error("No browser session. Call browser_navigate first.", success=False)
|
||||
|
||||
clean_ref = ref.lstrip("@")
|
||||
|
||||
_post(
|
||||
f"/tabs/{session['tab_id']}/type",
|
||||
{"userId": session["user_id"], "ref": clean_ref, "text": text},
|
||||
)
|
||||
return json.dumps({
|
||||
"success": True,
|
||||
"typed": text,
|
||||
"element": clean_ref,
|
||||
})
|
||||
except Exception as e:
|
||||
return tool_error(str(e), success=False)
|
||||
|
||||
|
||||
def camofox_scroll(direction: str, task_id: Optional[str] = None) -> str:
|
||||
"""Scroll the page via Camofox."""
|
||||
try:
|
||||
session = _get_session(task_id)
|
||||
if not session["tab_id"]:
|
||||
return tool_error("No browser session. Call browser_navigate first.", success=False)
|
||||
|
||||
_post(
|
||||
f"/tabs/{session['tab_id']}/scroll",
|
||||
{"userId": session["user_id"], "direction": direction},
|
||||
)
|
||||
return json.dumps({"success": True, "scrolled": direction})
|
||||
except Exception as e:
|
||||
return tool_error(str(e), success=False)
|
||||
|
||||
|
||||
def camofox_back(task_id: Optional[str] = None) -> str:
|
||||
"""Navigate back via Camofox."""
|
||||
try:
|
||||
session = _get_session(task_id)
|
||||
if not session["tab_id"]:
|
||||
return tool_error("No browser session. Call browser_navigate first.", success=False)
|
||||
|
||||
data = _post(
|
||||
f"/tabs/{session['tab_id']}/back",
|
||||
{"userId": session["user_id"]},
|
||||
)
|
||||
return json.dumps({"success": True, "url": data.get("url", "")})
|
||||
except Exception as e:
|
||||
return tool_error(str(e), success=False)
|
||||
|
||||
|
||||
def camofox_press(key: str, task_id: Optional[str] = None) -> str:
|
||||
"""Press a keyboard key via Camofox."""
|
||||
try:
|
||||
session = _get_session(task_id)
|
||||
if not session["tab_id"]:
|
||||
return tool_error("No browser session. Call browser_navigate first.", success=False)
|
||||
|
||||
_post(
|
||||
f"/tabs/{session['tab_id']}/press",
|
||||
{"userId": session["user_id"], "key": key},
|
||||
)
|
||||
return json.dumps({"success": True, "pressed": key})
|
||||
except Exception as e:
|
||||
return tool_error(str(e), success=False)
|
||||
|
||||
|
||||
def camofox_close(task_id: Optional[str] = None) -> str:
|
||||
"""Close the browser session via Camofox."""
|
||||
try:
|
||||
session = _drop_session(task_id)
|
||||
if not session:
|
||||
return json.dumps({"success": True, "closed": True})
|
||||
|
||||
_delete(
|
||||
f"/sessions/{session['user_id']}",
|
||||
)
|
||||
return json.dumps({"success": True, "closed": True})
|
||||
except Exception as e:
|
||||
return json.dumps({"success": True, "closed": True, "warning": str(e)})
|
||||
|
||||
|
||||
def camofox_get_images(task_id: Optional[str] = None) -> str:
|
||||
"""Get images on the current page via Camofox.
|
||||
|
||||
Extracts image information from the accessibility tree snapshot,
|
||||
since Camofox does not expose a dedicated /images endpoint.
|
||||
"""
|
||||
try:
|
||||
session = _get_session(task_id)
|
||||
if not session["tab_id"]:
|
||||
return tool_error("No browser session. Call browser_navigate first.", success=False)
|
||||
|
||||
import re
|
||||
|
||||
data = _get(
|
||||
f"/tabs/{session['tab_id']}/snapshot",
|
||||
params={"userId": session["user_id"]},
|
||||
)
|
||||
snapshot = data.get("snapshot", "")
|
||||
|
||||
# Parse img elements from the accessibility tree.
|
||||
# Format: img "alt text" or img "alt text" [eN]
|
||||
# URLs appear on /url: lines following img entries
|
||||
images = []
|
||||
lines = snapshot.split("\n")
|
||||
for i, line in enumerate(lines):
|
||||
stripped = line.strip()
|
||||
if stripped.startswith(("- img ", "img ")):
|
||||
alt_match = re.search(r'img\s+"([^"]*)"', stripped)
|
||||
alt = alt_match.group(1) if alt_match else ""
|
||||
# Look for URL on the next line
|
||||
src = ""
|
||||
if i + 1 < len(lines):
|
||||
url_match = re.search(r'/url:\s*(\S+)', lines[i + 1].strip())
|
||||
if url_match:
|
||||
src = url_match.group(1)
|
||||
if alt or src:
|
||||
images.append({"src": src, "alt": alt})
|
||||
|
||||
return json.dumps({
|
||||
"success": True,
|
||||
"images": images,
|
||||
"count": len(images),
|
||||
})
|
||||
except Exception as e:
|
||||
return tool_error(str(e), success=False)
|
||||
|
||||
|
||||
def camofox_vision(question: str, annotate: bool = False,
|
||||
task_id: Optional[str] = None) -> str:
|
||||
"""Take a screenshot and analyze it with vision AI via Camofox."""
|
||||
try:
|
||||
session = _get_session(task_id)
|
||||
if not session["tab_id"]:
|
||||
return tool_error("No browser session. Call browser_navigate first.", success=False)
|
||||
|
||||
# Get screenshot as binary PNG
|
||||
resp = _get_raw(
|
||||
f"/tabs/{session['tab_id']}/screenshot",
|
||||
params={"userId": session["user_id"]},
|
||||
)
|
||||
|
||||
# Save screenshot to cache
|
||||
from hermes_constants import get_hermes_home
|
||||
screenshots_dir = get_hermes_home() / "browser_screenshots"
|
||||
screenshots_dir.mkdir(parents=True, exist_ok=True)
|
||||
screenshot_path = str(screenshots_dir / f"browser_screenshot_{uuid.uuid4().hex[:8]}.png")
|
||||
|
||||
with open(screenshot_path, "wb") as f:
|
||||
f.write(resp.content)
|
||||
|
||||
# Encode for vision LLM
|
||||
img_b64 = base64.b64encode(resp.content).decode("utf-8")
|
||||
|
||||
# Also get annotated snapshot if requested
|
||||
annotation_context = ""
|
||||
if annotate:
|
||||
try:
|
||||
snap_data = _get(
|
||||
f"/tabs/{session['tab_id']}/snapshot",
|
||||
params={"userId": session["user_id"]},
|
||||
)
|
||||
annotation_context = f"\n\nAccessibility tree (element refs for interaction):\n{snap_data.get('snapshot', '')[:3000]}"
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Redact secrets from annotation context before sending to vision LLM.
|
||||
# The screenshot image itself cannot be redacted, but at least the
|
||||
# text-based accessibility tree snippet won't leak secret values.
|
||||
from agent.redact import redact_sensitive_text
|
||||
annotation_context = redact_sensitive_text(annotation_context)
|
||||
|
||||
# Send to vision LLM
|
||||
from agent.auxiliary_client import call_llm
|
||||
|
||||
vision_prompt = (
|
||||
f"Analyze this browser screenshot and answer: {question}"
|
||||
f"{annotation_context}"
|
||||
)
|
||||
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
_cfg = load_config()
|
||||
_vision_timeout = int(_cfg.get("auxiliary", {}).get("vision", {}).get("timeout", 120))
|
||||
except Exception:
|
||||
_vision_timeout = 120
|
||||
|
||||
response = call_llm(
|
||||
messages=[{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": vision_prompt},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": f"data:image/png;base64,{img_b64}",
|
||||
},
|
||||
},
|
||||
],
|
||||
}],
|
||||
task="vision",
|
||||
timeout=_vision_timeout,
|
||||
)
|
||||
analysis = (response.choices[0].message.content or "").strip() if response.choices else ""
|
||||
|
||||
# Redact secrets the vision LLM may have read from the screenshot.
|
||||
from agent.redact import redact_sensitive_text
|
||||
analysis = redact_sensitive_text(analysis)
|
||||
|
||||
return json.dumps({
|
||||
"success": True,
|
||||
"analysis": analysis,
|
||||
"screenshot_path": screenshot_path,
|
||||
})
|
||||
except Exception as e:
|
||||
return tool_error(str(e), success=False)
|
||||
|
||||
|
||||
def camofox_console(clear: bool = False, task_id: Optional[str] = None) -> str:
|
||||
"""Get console output — limited support in Camofox.
|
||||
|
||||
Camofox does not expose browser console logs via its REST API.
|
||||
Returns an empty result with a note.
|
||||
"""
|
||||
return json.dumps({
|
||||
"success": True,
|
||||
"console_messages": [],
|
||||
"js_errors": [],
|
||||
"total_messages": 0,
|
||||
"total_errors": 0,
|
||||
"note": "Console log capture is not available with the Camofox backend. "
|
||||
"Use browser_snapshot or browser_vision to inspect page state.",
|
||||
})
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Hermes-managed Camofox state helpers.
|
||||
|
||||
Provides profile-scoped identity and state directory paths for Camofox
|
||||
persistent browser profiles. When managed persistence is enabled, Hermes
|
||||
sends a deterministic userId derived from the active profile so that
|
||||
Camofox can map it to the same persistent browser profile directory
|
||||
across restarts.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Dict, Optional
|
||||
|
||||
from hermes_constants import get_hermes_home
|
||||
|
||||
CAMOFOX_STATE_DIR_NAME = "browser_auth"
|
||||
CAMOFOX_STATE_SUBDIR = "camofox"
|
||||
|
||||
|
||||
def get_camofox_state_dir() -> Path:
|
||||
"""Return the profile-scoped root directory for Camofox persistence."""
|
||||
return get_hermes_home() / CAMOFOX_STATE_DIR_NAME / CAMOFOX_STATE_SUBDIR
|
||||
|
||||
|
||||
def get_camofox_identity(task_id: Optional[str] = None) -> Dict[str, str]:
|
||||
"""Return the stable Hermes-managed Camofox identity for this profile.
|
||||
|
||||
The user identity is profile-scoped (same Hermes profile = same userId).
|
||||
The session key is scoped to the logical browser task so newly created
|
||||
tabs within the same profile reuse the same identity contract.
|
||||
"""
|
||||
scope_root = str(get_camofox_state_dir())
|
||||
logical_scope = task_id or "default"
|
||||
user_digest = uuid.uuid5(
|
||||
uuid.NAMESPACE_URL,
|
||||
f"camofox-user:{scope_root}",
|
||||
).hex[:10]
|
||||
session_digest = uuid.uuid5(
|
||||
uuid.NAMESPACE_URL,
|
||||
f"camofox-session:{scope_root}:{logical_scope}",
|
||||
).hex[:16]
|
||||
return {
|
||||
"user_id": f"hermes_{user_digest}",
|
||||
"session_key": f"task_{session_digest}",
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
"""Cloud browser provider abstraction.
|
||||
|
||||
Import the ABC so callers can do::
|
||||
|
||||
from tools.browser_providers import CloudBrowserProvider
|
||||
"""
|
||||
|
||||
from tools.browser_providers.base import CloudBrowserProvider
|
||||
|
||||
__all__ = ["CloudBrowserProvider"]
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Abstract base class for cloud browser providers."""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Dict
|
||||
|
||||
|
||||
class CloudBrowserProvider(ABC):
|
||||
"""Interface for cloud browser backends (Browserbase, Steel, etc.).
|
||||
|
||||
Implementations live in sibling modules and are registered in
|
||||
``browser_tool._PROVIDER_REGISTRY``. The user selects a provider via
|
||||
``hermes setup`` / ``hermes tools``; the choice is persisted as
|
||||
``config["browser"]["cloud_provider"]``.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def provider_name(self) -> str:
|
||||
"""Short, human-readable name shown in logs and diagnostics."""
|
||||
|
||||
@abstractmethod
|
||||
def is_configured(self) -> bool:
|
||||
"""Return True when all required env vars / credentials are present.
|
||||
|
||||
Called at tool-registration time (``check_browser_requirements``) to
|
||||
gate availability. Must be cheap — no network calls.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def create_session(self, task_id: str) -> Dict[str, object]:
|
||||
"""Create a cloud browser session and return session metadata.
|
||||
|
||||
Must return a dict with at least::
|
||||
|
||||
{
|
||||
"session_name": str, # unique name for agent-browser --session
|
||||
"bb_session_id": str, # provider session ID (for close/cleanup)
|
||||
"cdp_url": str, # CDP websocket URL
|
||||
"features": dict, # feature flags that were enabled
|
||||
}
|
||||
|
||||
``bb_session_id`` is a legacy key name kept for backward compat with
|
||||
the rest of browser_tool.py — it holds the provider's session ID
|
||||
regardless of which provider is in use.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def close_session(self, session_id: str) -> bool:
|
||||
"""Release / terminate a cloud session by its provider session ID.
|
||||
|
||||
Returns True on success, False on failure. Should not raise.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def emergency_cleanup(self, session_id: str) -> None:
|
||||
"""Best-effort session teardown during process exit.
|
||||
|
||||
Called from atexit / signal handlers. Must tolerate missing
|
||||
credentials, network errors, etc. — log and move on.
|
||||
"""
|
||||
@@ -0,0 +1,215 @@
|
||||
"""Browser Use cloud browser provider."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import uuid
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
import requests
|
||||
|
||||
from tools.browser_providers.base import CloudBrowserProvider
|
||||
from tools.managed_tool_gateway import resolve_managed_tool_gateway
|
||||
from tools.tool_backend_helpers import managed_nous_tools_enabled
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_pending_create_keys: Dict[str, str] = {}
|
||||
_pending_create_keys_lock = threading.Lock()
|
||||
|
||||
_BASE_URL = "https://api.browser-use.com/api/v3"
|
||||
_DEFAULT_MANAGED_TIMEOUT_MINUTES = 5
|
||||
_DEFAULT_MANAGED_PROXY_COUNTRY_CODE = "us"
|
||||
|
||||
|
||||
def _get_or_create_pending_create_key(task_id: str) -> str:
|
||||
with _pending_create_keys_lock:
|
||||
existing = _pending_create_keys.get(task_id)
|
||||
if existing:
|
||||
return existing
|
||||
|
||||
created = f"browser-use-session-create:{uuid.uuid4().hex}"
|
||||
_pending_create_keys[task_id] = created
|
||||
return created
|
||||
|
||||
|
||||
def _clear_pending_create_key(task_id: str) -> None:
|
||||
with _pending_create_keys_lock:
|
||||
_pending_create_keys.pop(task_id, None)
|
||||
|
||||
|
||||
def _should_preserve_pending_create_key(response: requests.Response) -> bool:
|
||||
if response.status_code >= 500:
|
||||
return True
|
||||
|
||||
if response.status_code != 409:
|
||||
return False
|
||||
|
||||
try:
|
||||
payload = response.json()
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
if not isinstance(payload, dict):
|
||||
return False
|
||||
|
||||
error = payload.get("error")
|
||||
if not isinstance(error, dict):
|
||||
return False
|
||||
|
||||
message = str(error.get("message") or "").lower()
|
||||
return "already in progress" in message
|
||||
|
||||
|
||||
class BrowserUseProvider(CloudBrowserProvider):
|
||||
"""Browser Use (https://browser-use.com) cloud browser backend."""
|
||||
|
||||
def provider_name(self) -> str:
|
||||
return "Browser Use"
|
||||
|
||||
def is_configured(self) -> bool:
|
||||
return self._get_config_or_none() is not None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Config resolution (direct API key OR managed Nous gateway)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _get_config_or_none(self) -> Optional[Dict[str, Any]]:
|
||||
api_key = os.environ.get("BROWSER_USE_API_KEY")
|
||||
if api_key:
|
||||
return {
|
||||
"api_key": api_key,
|
||||
"base_url": _BASE_URL,
|
||||
"managed_mode": False,
|
||||
}
|
||||
|
||||
managed = resolve_managed_tool_gateway("browser-use")
|
||||
if managed is None:
|
||||
return None
|
||||
|
||||
return {
|
||||
"api_key": managed.nous_user_token,
|
||||
"base_url": managed.gateway_origin.rstrip("/"),
|
||||
"managed_mode": True,
|
||||
}
|
||||
|
||||
def _get_config(self) -> Dict[str, Any]:
|
||||
config = self._get_config_or_none()
|
||||
if config is None:
|
||||
message = (
|
||||
"Browser Use requires a direct BROWSER_USE_API_KEY credential."
|
||||
)
|
||||
if managed_nous_tools_enabled():
|
||||
message = (
|
||||
"Browser Use requires either a direct BROWSER_USE_API_KEY "
|
||||
"credential or a managed Browser Use gateway configuration."
|
||||
)
|
||||
raise ValueError(message)
|
||||
return config
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Session lifecycle
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _headers(self, config: Dict[str, Any]) -> Dict[str, str]:
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"X-Browser-Use-API-Key": config["api_key"],
|
||||
}
|
||||
return headers
|
||||
|
||||
def create_session(self, task_id: str) -> Dict[str, object]:
|
||||
config = self._get_config()
|
||||
managed_mode = bool(config.get("managed_mode"))
|
||||
|
||||
headers = self._headers(config)
|
||||
if managed_mode:
|
||||
headers["X-Idempotency-Key"] = _get_or_create_pending_create_key(task_id)
|
||||
|
||||
# Keep gateway-backed sessions short so billing authorization does not
|
||||
# default to a long Browser-Use timeout when Hermes only needs a task-
|
||||
# scoped ephemeral browser.
|
||||
payload = (
|
||||
{
|
||||
"timeout": _DEFAULT_MANAGED_TIMEOUT_MINUTES,
|
||||
"proxyCountryCode": _DEFAULT_MANAGED_PROXY_COUNTRY_CODE,
|
||||
}
|
||||
if managed_mode
|
||||
else {}
|
||||
)
|
||||
|
||||
response = requests.post(
|
||||
f"{config['base_url']}/browsers",
|
||||
headers=headers,
|
||||
json=payload,
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
if not response.ok:
|
||||
if managed_mode and not _should_preserve_pending_create_key(response):
|
||||
_clear_pending_create_key(task_id)
|
||||
raise RuntimeError(
|
||||
f"Failed to create Browser Use session: "
|
||||
f"{response.status_code} {response.text}"
|
||||
)
|
||||
|
||||
session_data = response.json()
|
||||
if managed_mode:
|
||||
_clear_pending_create_key(task_id)
|
||||
session_name = f"hermes_{task_id}_{uuid.uuid4().hex[:8]}"
|
||||
external_call_id = response.headers.get("x-external-call-id") if managed_mode else None
|
||||
|
||||
logger.info("Created Browser Use session %s", session_name)
|
||||
|
||||
cdp_url = session_data.get("cdpUrl") or session_data.get("connectUrl") or ""
|
||||
|
||||
return {
|
||||
"session_name": session_name,
|
||||
"bb_session_id": session_data["id"],
|
||||
"cdp_url": cdp_url,
|
||||
"features": {"browser_use": True},
|
||||
"external_call_id": external_call_id,
|
||||
}
|
||||
|
||||
def close_session(self, session_id: str) -> bool:
|
||||
try:
|
||||
config = self._get_config()
|
||||
except ValueError:
|
||||
logger.warning("Cannot close Browser Use session %s — missing credentials", session_id)
|
||||
return False
|
||||
|
||||
try:
|
||||
response = requests.patch(
|
||||
f"{config['base_url']}/browsers/{session_id}",
|
||||
headers=self._headers(config),
|
||||
json={"action": "stop"},
|
||||
timeout=10,
|
||||
)
|
||||
if response.status_code in (200, 201, 204):
|
||||
logger.debug("Successfully closed Browser Use session %s", session_id)
|
||||
return True
|
||||
else:
|
||||
logger.warning(
|
||||
"Failed to close Browser Use session %s: HTTP %s - %s",
|
||||
session_id,
|
||||
response.status_code,
|
||||
response.text[:200],
|
||||
)
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error("Exception closing Browser Use session %s: %s", session_id, e)
|
||||
return False
|
||||
|
||||
def emergency_cleanup(self, session_id: str) -> None:
|
||||
config = self._get_config_or_none()
|
||||
if config is None:
|
||||
logger.warning("Cannot emergency-cleanup Browser Use session %s — missing credentials", session_id)
|
||||
return
|
||||
try:
|
||||
requests.patch(
|
||||
f"{config['base_url']}/browsers/{session_id}",
|
||||
headers=self._headers(config),
|
||||
json={"action": "stop"},
|
||||
timeout=5,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug("Emergency cleanup failed for Browser Use session %s: %s", session_id, e)
|
||||
@@ -0,0 +1,217 @@
|
||||
"""Browserbase cloud browser provider (direct credentials only)."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import uuid
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
import requests
|
||||
|
||||
from tools.browser_providers.base import CloudBrowserProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BrowserbaseProvider(CloudBrowserProvider):
|
||||
"""Browserbase (https://browserbase.com) cloud browser backend.
|
||||
|
||||
This provider requires direct BROWSERBASE_API_KEY and BROWSERBASE_PROJECT_ID
|
||||
credentials. Managed Nous gateway support has been removed — the Nous
|
||||
subscription now routes through Browser Use instead.
|
||||
"""
|
||||
|
||||
def provider_name(self) -> str:
|
||||
return "Browserbase"
|
||||
|
||||
def is_configured(self) -> bool:
|
||||
return self._get_config_or_none() is not None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Session lifecycle
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _get_config_or_none(self) -> Optional[Dict[str, Any]]:
|
||||
api_key = os.environ.get("BROWSERBASE_API_KEY")
|
||||
project_id = os.environ.get("BROWSERBASE_PROJECT_ID")
|
||||
if api_key and project_id:
|
||||
return {
|
||||
"api_key": api_key,
|
||||
"project_id": project_id,
|
||||
"base_url": os.environ.get("BROWSERBASE_BASE_URL", "https://api.browserbase.com").rstrip("/"),
|
||||
}
|
||||
return None
|
||||
|
||||
def _get_config(self) -> Dict[str, Any]:
|
||||
config = self._get_config_or_none()
|
||||
if config is None:
|
||||
raise ValueError(
|
||||
"Browserbase requires BROWSERBASE_API_KEY and BROWSERBASE_PROJECT_ID "
|
||||
"environment variables."
|
||||
)
|
||||
return config
|
||||
|
||||
def create_session(self, task_id: str) -> Dict[str, object]:
|
||||
config = self._get_config()
|
||||
|
||||
# Optional env-var knobs
|
||||
enable_proxies = os.environ.get("BROWSERBASE_PROXIES", "true").lower() != "false"
|
||||
enable_advanced_stealth = os.environ.get("BROWSERBASE_ADVANCED_STEALTH", "false").lower() == "true"
|
||||
enable_keep_alive = os.environ.get("BROWSERBASE_KEEP_ALIVE", "true").lower() != "false"
|
||||
custom_timeout_ms = os.environ.get("BROWSERBASE_SESSION_TIMEOUT")
|
||||
|
||||
features_enabled = {
|
||||
"basic_stealth": True,
|
||||
"proxies": False,
|
||||
"advanced_stealth": False,
|
||||
"keep_alive": False,
|
||||
"custom_timeout": False,
|
||||
}
|
||||
|
||||
session_config: Dict[str, object] = {"projectId": config["project_id"]}
|
||||
|
||||
if enable_keep_alive:
|
||||
session_config["keepAlive"] = True
|
||||
|
||||
if custom_timeout_ms:
|
||||
try:
|
||||
timeout_val = int(custom_timeout_ms)
|
||||
if timeout_val > 0:
|
||||
session_config["timeout"] = timeout_val
|
||||
except ValueError:
|
||||
logger.warning("Invalid BROWSERBASE_SESSION_TIMEOUT value: %s", custom_timeout_ms)
|
||||
|
||||
if enable_proxies:
|
||||
session_config["proxies"] = True
|
||||
|
||||
if enable_advanced_stealth:
|
||||
session_config["browserSettings"] = {"advancedStealth": True}
|
||||
|
||||
# --- Create session via API ---
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"X-BB-API-Key": config["api_key"],
|
||||
}
|
||||
|
||||
response = requests.post(
|
||||
f"{config['base_url']}/v1/sessions",
|
||||
headers=headers,
|
||||
json=session_config,
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
proxies_fallback = False
|
||||
keepalive_fallback = False
|
||||
|
||||
# Handle 402 — paid features unavailable
|
||||
if response.status_code == 402:
|
||||
if enable_keep_alive:
|
||||
keepalive_fallback = True
|
||||
logger.warning(
|
||||
"keepAlive may require paid plan (402), retrying without it. "
|
||||
"Sessions may timeout during long operations."
|
||||
)
|
||||
session_config.pop("keepAlive", None)
|
||||
response = requests.post(
|
||||
f"{config['base_url']}/v1/sessions",
|
||||
headers=headers,
|
||||
json=session_config,
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
if response.status_code == 402 and enable_proxies:
|
||||
proxies_fallback = True
|
||||
logger.warning(
|
||||
"Proxies unavailable (402), retrying without proxies. "
|
||||
"Bot detection may be less effective."
|
||||
)
|
||||
session_config.pop("proxies", None)
|
||||
response = requests.post(
|
||||
f"{config['base_url']}/v1/sessions",
|
||||
headers=headers,
|
||||
json=session_config,
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
if not response.ok:
|
||||
raise RuntimeError(
|
||||
f"Failed to create Browserbase session: "
|
||||
f"{response.status_code} {response.text}"
|
||||
)
|
||||
|
||||
session_data = response.json()
|
||||
session_name = f"hermes_{task_id}_{uuid.uuid4().hex[:8]}"
|
||||
|
||||
if enable_proxies and not proxies_fallback:
|
||||
features_enabled["proxies"] = True
|
||||
if enable_advanced_stealth:
|
||||
features_enabled["advanced_stealth"] = True
|
||||
if enable_keep_alive and not keepalive_fallback:
|
||||
features_enabled["keep_alive"] = True
|
||||
if custom_timeout_ms and "timeout" in session_config:
|
||||
features_enabled["custom_timeout"] = True
|
||||
|
||||
feature_str = ", ".join(k for k, v in features_enabled.items() if v)
|
||||
logger.info("Created Browserbase session %s with features: %s", session_name, feature_str)
|
||||
|
||||
return {
|
||||
"session_name": session_name,
|
||||
"bb_session_id": session_data["id"],
|
||||
"cdp_url": session_data["connectUrl"],
|
||||
"features": features_enabled,
|
||||
}
|
||||
|
||||
def close_session(self, session_id: str) -> bool:
|
||||
try:
|
||||
config = self._get_config()
|
||||
except ValueError:
|
||||
logger.warning("Cannot close Browserbase session %s — missing credentials", session_id)
|
||||
return False
|
||||
|
||||
try:
|
||||
response = requests.post(
|
||||
f"{config['base_url']}/v1/sessions/{session_id}",
|
||||
headers={
|
||||
"X-BB-API-Key": config["api_key"],
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
json={
|
||||
"projectId": config["project_id"],
|
||||
"status": "REQUEST_RELEASE",
|
||||
},
|
||||
timeout=10,
|
||||
)
|
||||
if response.status_code in (200, 201, 204):
|
||||
logger.debug("Successfully closed Browserbase session %s", session_id)
|
||||
return True
|
||||
else:
|
||||
logger.warning(
|
||||
"Failed to close session %s: HTTP %s - %s",
|
||||
session_id,
|
||||
response.status_code,
|
||||
response.text[:200],
|
||||
)
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error("Exception closing Browserbase session %s: %s", session_id, e)
|
||||
return False
|
||||
|
||||
def emergency_cleanup(self, session_id: str) -> None:
|
||||
config = self._get_config_or_none()
|
||||
if config is None:
|
||||
logger.warning("Cannot emergency-cleanup Browserbase session %s — missing credentials", session_id)
|
||||
return
|
||||
try:
|
||||
requests.post(
|
||||
f"{config['base_url']}/v1/sessions/{session_id}",
|
||||
headers={
|
||||
"X-BB-API-Key": config["api_key"],
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
json={
|
||||
"projectId": config["project_id"],
|
||||
"status": "REQUEST_RELEASE",
|
||||
},
|
||||
timeout=5,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug("Emergency cleanup failed for Browserbase session %s: %s", session_id, e)
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Firecrawl cloud browser provider."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import uuid
|
||||
from typing import Dict
|
||||
|
||||
import requests
|
||||
|
||||
from tools.browser_providers.base import CloudBrowserProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_BASE_URL = "https://api.firecrawl.dev"
|
||||
|
||||
|
||||
class FirecrawlProvider(CloudBrowserProvider):
|
||||
"""Firecrawl (https://firecrawl.dev) cloud browser backend."""
|
||||
|
||||
def provider_name(self) -> str:
|
||||
return "Firecrawl"
|
||||
|
||||
def is_configured(self) -> bool:
|
||||
return bool(os.environ.get("FIRECRAWL_API_KEY"))
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Session lifecycle
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _api_url(self) -> str:
|
||||
return os.environ.get("FIRECRAWL_API_URL", _BASE_URL)
|
||||
|
||||
def _headers(self) -> Dict[str, str]:
|
||||
api_key = os.environ.get("FIRECRAWL_API_KEY")
|
||||
if not api_key:
|
||||
raise ValueError(
|
||||
"FIRECRAWL_API_KEY environment variable is required. "
|
||||
"Get your key at https://firecrawl.dev"
|
||||
)
|
||||
return {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
}
|
||||
|
||||
def create_session(self, task_id: str) -> Dict[str, object]:
|
||||
ttl = int(os.environ.get("FIRECRAWL_BROWSER_TTL", "300"))
|
||||
|
||||
body: Dict[str, object] = {"ttl": ttl}
|
||||
|
||||
response = requests.post(
|
||||
f"{self._api_url()}/v2/browser",
|
||||
headers=self._headers(),
|
||||
json=body,
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
if not response.ok:
|
||||
raise RuntimeError(
|
||||
f"Failed to create Firecrawl browser session: "
|
||||
f"{response.status_code} {response.text}"
|
||||
)
|
||||
|
||||
data = response.json()
|
||||
session_name = f"hermes_{task_id}_{uuid.uuid4().hex[:8]}"
|
||||
|
||||
logger.info("Created Firecrawl browser session %s", session_name)
|
||||
|
||||
return {
|
||||
"session_name": session_name,
|
||||
"bb_session_id": data["id"],
|
||||
"cdp_url": data["cdpUrl"],
|
||||
"features": {"firecrawl": True},
|
||||
}
|
||||
|
||||
def close_session(self, session_id: str) -> bool:
|
||||
try:
|
||||
response = requests.delete(
|
||||
f"{self._api_url()}/v2/browser/{session_id}",
|
||||
headers=self._headers(),
|
||||
timeout=10,
|
||||
)
|
||||
if response.status_code in (200, 201, 204):
|
||||
logger.debug("Successfully closed Firecrawl session %s", session_id)
|
||||
return True
|
||||
else:
|
||||
logger.warning(
|
||||
"Failed to close Firecrawl session %s: HTTP %s - %s",
|
||||
session_id,
|
||||
response.status_code,
|
||||
response.text[:200],
|
||||
)
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error("Exception closing Firecrawl session %s: %s", session_id, e)
|
||||
return False
|
||||
|
||||
def emergency_cleanup(self, session_id: str) -> None:
|
||||
try:
|
||||
requests.delete(
|
||||
f"{self._api_url()}/v2/browser/{session_id}",
|
||||
headers=self._headers(),
|
||||
timeout=5,
|
||||
)
|
||||
except ValueError:
|
||||
logger.warning("Cannot emergency-cleanup Firecrawl session %s — missing credentials", session_id)
|
||||
except Exception as e:
|
||||
logger.debug("Emergency cleanup failed for Firecrawl session %s: %s", session_id, e)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,52 @@
|
||||
"""Configurable budget constants for tool result persistence.
|
||||
|
||||
Overridable at the RL environment level via HermesAgentEnvConfig fields.
|
||||
Per-tool resolution: pinned > config overrides > registry > default.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict
|
||||
|
||||
# Tools whose thresholds must never be overridden.
|
||||
# read_file=inf prevents infinite persist->read->persist loops.
|
||||
PINNED_THRESHOLDS: Dict[str, float] = {
|
||||
"read_file": float("inf"),
|
||||
}
|
||||
|
||||
# Defaults matching the current hardcoded values in tool_result_storage.py.
|
||||
# Kept here as the single source of truth; tool_result_storage.py imports these.
|
||||
DEFAULT_RESULT_SIZE_CHARS: int = 100_000
|
||||
DEFAULT_TURN_BUDGET_CHARS: int = 200_000
|
||||
DEFAULT_PREVIEW_SIZE_CHARS: int = 1_500
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BudgetConfig:
|
||||
"""Immutable budget constants for the 3-layer tool result persistence system.
|
||||
|
||||
Layer 2 (per-result): resolve_threshold(tool_name) -> threshold in chars.
|
||||
Layer 3 (per-turn): turn_budget -> aggregate char budget across all tool
|
||||
results in a single assistant turn.
|
||||
Preview: preview_size -> inline snippet size after persistence.
|
||||
"""
|
||||
|
||||
default_result_size: int = DEFAULT_RESULT_SIZE_CHARS
|
||||
turn_budget: int = DEFAULT_TURN_BUDGET_CHARS
|
||||
preview_size: int = DEFAULT_PREVIEW_SIZE_CHARS
|
||||
tool_overrides: Dict[str, int] = field(default_factory=dict)
|
||||
|
||||
def resolve_threshold(self, tool_name: str) -> int | float:
|
||||
"""Resolve the persistence threshold for a tool.
|
||||
|
||||
Priority: pinned -> tool_overrides -> registry per-tool -> default.
|
||||
"""
|
||||
if tool_name in PINNED_THRESHOLDS:
|
||||
return PINNED_THRESHOLDS[tool_name]
|
||||
if tool_name in self.tool_overrides:
|
||||
return self.tool_overrides[tool_name]
|
||||
from tools.registry import registry
|
||||
return registry.get_max_result_size(tool_name, default=self.default_result_size)
|
||||
|
||||
|
||||
# Default config -- matches current hardcoded behavior exactly.
|
||||
DEFAULT_BUDGET = BudgetConfig()
|
||||
@@ -0,0 +1,623 @@
|
||||
"""
|
||||
Checkpoint Manager — Transparent filesystem snapshots via shadow git repos.
|
||||
|
||||
Creates automatic snapshots of working directories before file-mutating
|
||||
operations (write_file, patch), triggered once per conversation turn.
|
||||
Provides rollback to any previous checkpoint.
|
||||
|
||||
This is NOT a tool — the LLM never sees it. It's transparent infrastructure
|
||||
controlled by the ``checkpoints`` config flag or ``--checkpoints`` CLI flag.
|
||||
|
||||
Architecture:
|
||||
~/.hermes/checkpoints/{sha256(abs_dir)[:16]}/ — shadow git repo
|
||||
HEAD, refs/, objects/ — standard git internals
|
||||
HERMES_WORKDIR — original dir path
|
||||
info/exclude — default excludes
|
||||
|
||||
The shadow repo uses GIT_DIR + GIT_WORK_TREE so no git state leaks
|
||||
into the user's project directory.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from hermes_constants import get_hermes_home
|
||||
from typing import Dict, List, Optional, Set
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
CHECKPOINT_BASE = get_hermes_home() / "checkpoints"
|
||||
|
||||
DEFAULT_EXCLUDES = [
|
||||
"node_modules/",
|
||||
"dist/",
|
||||
"build/",
|
||||
".env",
|
||||
".env.*",
|
||||
".env.local",
|
||||
".env.*.local",
|
||||
"__pycache__/",
|
||||
"*.pyc",
|
||||
"*.pyo",
|
||||
".DS_Store",
|
||||
"*.log",
|
||||
".cache/",
|
||||
".next/",
|
||||
".nuxt/",
|
||||
"coverage/",
|
||||
".pytest_cache/",
|
||||
".venv/",
|
||||
"venv/",
|
||||
".git/",
|
||||
]
|
||||
|
||||
# Git subprocess timeout (seconds).
|
||||
_GIT_TIMEOUT: int = max(10, min(60, int(os.getenv("HERMES_CHECKPOINT_TIMEOUT", "30"))))
|
||||
|
||||
# Max files to snapshot — skip huge directories to avoid slowdowns.
|
||||
_MAX_FILES = 50_000
|
||||
|
||||
# Valid git commit hash pattern: 4–40 hex chars (short or full SHA-1/SHA-256).
|
||||
_COMMIT_HASH_RE = re.compile(r'^[0-9a-fA-F]{4,64}$')
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Input validation helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _validate_commit_hash(commit_hash: str) -> Optional[str]:
|
||||
"""Validate a commit hash to prevent git argument injection.
|
||||
|
||||
Returns an error string if invalid, None if valid.
|
||||
Values starting with '-' would be interpreted as git flags
|
||||
(e.g., '--patch', '-p') instead of revision specifiers.
|
||||
"""
|
||||
if not commit_hash or not commit_hash.strip():
|
||||
return "Empty commit hash"
|
||||
if commit_hash.startswith("-"):
|
||||
return f"Invalid commit hash (must not start with '-'): {commit_hash!r}"
|
||||
if not _COMMIT_HASH_RE.match(commit_hash):
|
||||
return f"Invalid commit hash (expected 4-64 hex characters): {commit_hash!r}"
|
||||
return None
|
||||
|
||||
|
||||
def _validate_file_path(file_path: str, working_dir: str) -> Optional[str]:
|
||||
"""Validate a file path to prevent path traversal outside the working directory.
|
||||
|
||||
Returns an error string if invalid, None if valid.
|
||||
"""
|
||||
if not file_path or not file_path.strip():
|
||||
return "Empty file path"
|
||||
# Reject absolute paths — restore targets must be relative to the workdir
|
||||
if os.path.isabs(file_path):
|
||||
return f"File path must be relative, got absolute path: {file_path!r}"
|
||||
# Resolve and check containment within working_dir
|
||||
abs_workdir = _normalize_path(working_dir)
|
||||
resolved = (abs_workdir / file_path).resolve()
|
||||
try:
|
||||
resolved.relative_to(abs_workdir)
|
||||
except ValueError:
|
||||
return f"File path escapes the working directory via traversal: {file_path!r}"
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shadow repo helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _normalize_path(path_value: str) -> Path:
|
||||
"""Return a canonical absolute path for checkpoint operations."""
|
||||
return Path(path_value).expanduser().resolve()
|
||||
|
||||
|
||||
def _shadow_repo_path(working_dir: str) -> Path:
|
||||
"""Deterministic shadow repo path: sha256(abs_path)[:16]."""
|
||||
abs_path = str(_normalize_path(working_dir))
|
||||
dir_hash = hashlib.sha256(abs_path.encode()).hexdigest()[:16]
|
||||
return CHECKPOINT_BASE / dir_hash
|
||||
|
||||
|
||||
def _git_env(shadow_repo: Path, working_dir: str) -> dict:
|
||||
"""Build env dict that redirects git to the shadow repo."""
|
||||
normalized_working_dir = _normalize_path(working_dir)
|
||||
env = os.environ.copy()
|
||||
env["GIT_DIR"] = str(shadow_repo)
|
||||
env["GIT_WORK_TREE"] = str(normalized_working_dir)
|
||||
env.pop("GIT_INDEX_FILE", None)
|
||||
env.pop("GIT_NAMESPACE", None)
|
||||
env.pop("GIT_ALTERNATE_OBJECT_DIRECTORIES", None)
|
||||
return env
|
||||
|
||||
|
||||
def _run_git(
|
||||
args: List[str],
|
||||
shadow_repo: Path,
|
||||
working_dir: str,
|
||||
timeout: int = _GIT_TIMEOUT,
|
||||
allowed_returncodes: Optional[Set[int]] = None,
|
||||
) -> tuple:
|
||||
"""Run a git command against the shadow repo. Returns (ok, stdout, stderr).
|
||||
|
||||
``allowed_returncodes`` suppresses error logging for known/expected non-zero
|
||||
exits while preserving the normal ``ok = (returncode == 0)`` contract.
|
||||
Example: ``git diff --cached --quiet`` returns 1 when changes exist.
|
||||
"""
|
||||
normalized_working_dir = _normalize_path(working_dir)
|
||||
if not normalized_working_dir.exists():
|
||||
msg = f"working directory not found: {normalized_working_dir}"
|
||||
logger.error("Git command skipped: %s (%s)", " ".join(["git"] + list(args)), msg)
|
||||
return False, "", msg
|
||||
if not normalized_working_dir.is_dir():
|
||||
msg = f"working directory is not a directory: {normalized_working_dir}"
|
||||
logger.error("Git command skipped: %s (%s)", " ".join(["git"] + list(args)), msg)
|
||||
return False, "", msg
|
||||
|
||||
env = _git_env(shadow_repo, str(normalized_working_dir))
|
||||
cmd = ["git"] + list(args)
|
||||
allowed_returncodes = allowed_returncodes or set()
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
env=env,
|
||||
cwd=str(normalized_working_dir),
|
||||
)
|
||||
ok = result.returncode == 0
|
||||
stdout = result.stdout.strip()
|
||||
stderr = result.stderr.strip()
|
||||
if not ok and result.returncode not in allowed_returncodes:
|
||||
logger.error(
|
||||
"Git command failed: %s (rc=%d) stderr=%s",
|
||||
" ".join(cmd), result.returncode, stderr,
|
||||
)
|
||||
return ok, stdout, stderr
|
||||
except subprocess.TimeoutExpired:
|
||||
msg = f"git timed out after {timeout}s: {' '.join(cmd)}"
|
||||
logger.error(msg, exc_info=True)
|
||||
return False, "", msg
|
||||
except FileNotFoundError as exc:
|
||||
missing_target = getattr(exc, "filename", None)
|
||||
if missing_target == "git":
|
||||
logger.error("Git executable not found: %s", " ".join(cmd), exc_info=True)
|
||||
return False, "", "git not found"
|
||||
msg = f"working directory not found: {normalized_working_dir}"
|
||||
logger.error("Git command failed before execution: %s (%s)", " ".join(cmd), msg, exc_info=True)
|
||||
return False, "", msg
|
||||
except Exception as exc:
|
||||
logger.error("Unexpected git error running %s: %s", " ".join(cmd), exc, exc_info=True)
|
||||
return False, "", str(exc)
|
||||
|
||||
|
||||
def _init_shadow_repo(shadow_repo: Path, working_dir: str) -> Optional[str]:
|
||||
"""Initialise shadow repo if needed. Returns error string or None."""
|
||||
if (shadow_repo / "HEAD").exists():
|
||||
return None
|
||||
|
||||
shadow_repo.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
ok, _, err = _run_git(["init"], shadow_repo, working_dir)
|
||||
if not ok:
|
||||
return f"Shadow repo init failed: {err}"
|
||||
|
||||
_run_git(["config", "user.email", "hermes@local"], shadow_repo, working_dir)
|
||||
_run_git(["config", "user.name", "Hermes Checkpoint"], shadow_repo, working_dir)
|
||||
|
||||
info_dir = shadow_repo / "info"
|
||||
info_dir.mkdir(exist_ok=True)
|
||||
(info_dir / "exclude").write_text(
|
||||
"\n".join(DEFAULT_EXCLUDES) + "\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
(shadow_repo / "HERMES_WORKDIR").write_text(
|
||||
str(_normalize_path(working_dir)) + "\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
logger.debug("Initialised checkpoint repo at %s for %s", shadow_repo, working_dir)
|
||||
return None
|
||||
|
||||
|
||||
def _dir_file_count(path: str) -> int:
|
||||
"""Quick file count estimate (stops early if over _MAX_FILES)."""
|
||||
count = 0
|
||||
try:
|
||||
for _ in Path(path).rglob("*"):
|
||||
count += 1
|
||||
if count > _MAX_FILES:
|
||||
return count
|
||||
except (PermissionError, OSError):
|
||||
pass
|
||||
return count
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CheckpointManager
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class CheckpointManager:
|
||||
"""Manages automatic filesystem checkpoints.
|
||||
|
||||
Designed to be owned by AIAgent. Call ``new_turn()`` at the start of
|
||||
each conversation turn and ``ensure_checkpoint(dir, reason)`` before
|
||||
any file-mutating tool call. The manager deduplicates so at most one
|
||||
snapshot is taken per directory per turn.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
enabled : bool
|
||||
Master switch (from config / CLI flag).
|
||||
max_snapshots : int
|
||||
Keep at most this many checkpoints per directory.
|
||||
"""
|
||||
|
||||
def __init__(self, enabled: bool = False, max_snapshots: int = 50):
|
||||
self.enabled = enabled
|
||||
self.max_snapshots = max_snapshots
|
||||
self._checkpointed_dirs: Set[str] = set()
|
||||
self._git_available: Optional[bool] = None # lazy probe
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Turn lifecycle
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def new_turn(self) -> None:
|
||||
"""Reset per-turn dedup. Call at the start of each agent iteration."""
|
||||
self._checkpointed_dirs.clear()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Public API
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def ensure_checkpoint(self, working_dir: str, reason: str = "auto") -> bool:
|
||||
"""Take a checkpoint if enabled and not already done this turn.
|
||||
|
||||
Returns True if a checkpoint was taken, False otherwise.
|
||||
Never raises — all errors are silently logged.
|
||||
"""
|
||||
if not self.enabled:
|
||||
return False
|
||||
|
||||
# Lazy git probe
|
||||
if self._git_available is None:
|
||||
self._git_available = shutil.which("git") is not None
|
||||
if not self._git_available:
|
||||
logger.debug("Checkpoints disabled: git not found")
|
||||
if not self._git_available:
|
||||
return False
|
||||
|
||||
abs_dir = str(_normalize_path(working_dir))
|
||||
|
||||
# Skip root, home, and other overly broad directories
|
||||
if abs_dir in ("/", str(Path.home())):
|
||||
logger.debug("Checkpoint skipped: directory too broad (%s)", abs_dir)
|
||||
return False
|
||||
|
||||
# Already checkpointed this turn?
|
||||
if abs_dir in self._checkpointed_dirs:
|
||||
return False
|
||||
|
||||
self._checkpointed_dirs.add(abs_dir)
|
||||
|
||||
try:
|
||||
return self._take(abs_dir, reason)
|
||||
except Exception as e:
|
||||
logger.debug("Checkpoint failed (non-fatal): %s", e)
|
||||
return False
|
||||
|
||||
def list_checkpoints(self, working_dir: str) -> List[Dict]:
|
||||
"""List available checkpoints for a directory.
|
||||
|
||||
Returns a list of dicts with keys: hash, short_hash, timestamp, reason,
|
||||
files_changed, insertions, deletions. Most recent first.
|
||||
"""
|
||||
abs_dir = str(_normalize_path(working_dir))
|
||||
shadow = _shadow_repo_path(abs_dir)
|
||||
|
||||
if not (shadow / "HEAD").exists():
|
||||
return []
|
||||
|
||||
ok, stdout, _ = _run_git(
|
||||
["log", "--format=%H|%h|%aI|%s", "-n", str(self.max_snapshots)],
|
||||
shadow, abs_dir,
|
||||
)
|
||||
|
||||
if not ok or not stdout:
|
||||
return []
|
||||
|
||||
results = []
|
||||
for line in stdout.splitlines():
|
||||
parts = line.split("|", 3)
|
||||
if len(parts) == 4:
|
||||
entry = {
|
||||
"hash": parts[0],
|
||||
"short_hash": parts[1],
|
||||
"timestamp": parts[2],
|
||||
"reason": parts[3],
|
||||
"files_changed": 0,
|
||||
"insertions": 0,
|
||||
"deletions": 0,
|
||||
}
|
||||
# Get diffstat for this commit
|
||||
stat_ok, stat_out, _ = _run_git(
|
||||
["diff", "--shortstat", f"{parts[0]}~1", parts[0]],
|
||||
shadow, abs_dir,
|
||||
allowed_returncodes={128, 129}, # first commit has no parent
|
||||
)
|
||||
if stat_ok and stat_out:
|
||||
self._parse_shortstat(stat_out, entry)
|
||||
results.append(entry)
|
||||
return results
|
||||
|
||||
@staticmethod
|
||||
def _parse_shortstat(stat_line: str, entry: Dict) -> None:
|
||||
"""Parse git --shortstat output into entry dict."""
|
||||
import re
|
||||
m = re.search(r'(\d+) file', stat_line)
|
||||
if m:
|
||||
entry["files_changed"] = int(m.group(1))
|
||||
m = re.search(r'(\d+) insertion', stat_line)
|
||||
if m:
|
||||
entry["insertions"] = int(m.group(1))
|
||||
m = re.search(r'(\d+) deletion', stat_line)
|
||||
if m:
|
||||
entry["deletions"] = int(m.group(1))
|
||||
|
||||
def diff(self, working_dir: str, commit_hash: str) -> Dict:
|
||||
"""Show diff between a checkpoint and the current working tree.
|
||||
|
||||
Returns dict with success, diff text, and stat summary.
|
||||
"""
|
||||
# Validate commit_hash to prevent git argument injection
|
||||
hash_err = _validate_commit_hash(commit_hash)
|
||||
if hash_err:
|
||||
return {"success": False, "error": hash_err}
|
||||
|
||||
abs_dir = str(_normalize_path(working_dir))
|
||||
shadow = _shadow_repo_path(abs_dir)
|
||||
|
||||
if not (shadow / "HEAD").exists():
|
||||
return {"success": False, "error": "No checkpoints exist for this directory"}
|
||||
|
||||
# Verify the commit exists
|
||||
ok, _, err = _run_git(
|
||||
["cat-file", "-t", commit_hash], shadow, abs_dir,
|
||||
)
|
||||
if not ok:
|
||||
return {"success": False, "error": f"Checkpoint '{commit_hash}' not found"}
|
||||
|
||||
# Stage current state to compare against checkpoint
|
||||
_run_git(["add", "-A"], shadow, abs_dir, timeout=_GIT_TIMEOUT * 2)
|
||||
|
||||
# Get stat summary: checkpoint vs current working tree
|
||||
ok_stat, stat_out, _ = _run_git(
|
||||
["diff", "--stat", commit_hash, "--cached"],
|
||||
shadow, abs_dir,
|
||||
)
|
||||
|
||||
# Get actual diff (limited to avoid terminal flood)
|
||||
ok_diff, diff_out, _ = _run_git(
|
||||
["diff", commit_hash, "--cached", "--no-color"],
|
||||
shadow, abs_dir,
|
||||
)
|
||||
|
||||
# Unstage to avoid polluting the shadow repo index
|
||||
_run_git(["reset", "HEAD", "--quiet"], shadow, abs_dir)
|
||||
|
||||
if not ok_stat and not ok_diff:
|
||||
return {"success": False, "error": "Could not generate diff"}
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"stat": stat_out if ok_stat else "",
|
||||
"diff": diff_out if ok_diff else "",
|
||||
}
|
||||
|
||||
def restore(self, working_dir: str, commit_hash: str, file_path: str = None) -> Dict:
|
||||
"""Restore files to a checkpoint state.
|
||||
|
||||
Uses ``git checkout <hash> -- .`` (or a specific file) which restores
|
||||
tracked files without moving HEAD — safe and reversible.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
file_path : str, optional
|
||||
If provided, restore only this file instead of the entire directory.
|
||||
|
||||
Returns dict with success/error info.
|
||||
"""
|
||||
# Validate commit_hash to prevent git argument injection
|
||||
hash_err = _validate_commit_hash(commit_hash)
|
||||
if hash_err:
|
||||
return {"success": False, "error": hash_err}
|
||||
|
||||
abs_dir = str(_normalize_path(working_dir))
|
||||
|
||||
# Validate file_path to prevent path traversal outside the working dir
|
||||
if file_path:
|
||||
path_err = _validate_file_path(file_path, abs_dir)
|
||||
if path_err:
|
||||
return {"success": False, "error": path_err}
|
||||
|
||||
shadow = _shadow_repo_path(abs_dir)
|
||||
|
||||
if not (shadow / "HEAD").exists():
|
||||
return {"success": False, "error": "No checkpoints exist for this directory"}
|
||||
|
||||
# Verify the commit exists
|
||||
ok, _, err = _run_git(
|
||||
["cat-file", "-t", commit_hash], shadow, abs_dir,
|
||||
)
|
||||
if not ok:
|
||||
return {"success": False, "error": f"Checkpoint '{commit_hash}' not found", "debug": err or None}
|
||||
|
||||
# Take a checkpoint of current state before restoring (so you can undo the undo)
|
||||
self._take(abs_dir, f"pre-rollback snapshot (restoring to {commit_hash[:8]})")
|
||||
|
||||
# Restore — full directory or single file
|
||||
restore_target = file_path if file_path else "."
|
||||
ok, stdout, err = _run_git(
|
||||
["checkout", commit_hash, "--", restore_target],
|
||||
shadow, abs_dir, timeout=_GIT_TIMEOUT * 2,
|
||||
)
|
||||
|
||||
if not ok:
|
||||
return {"success": False, "error": f"Restore failed: {err}", "debug": err or None}
|
||||
|
||||
# Get info about what was restored
|
||||
ok2, reason_out, _ = _run_git(
|
||||
["log", "--format=%s", "-1", commit_hash], shadow, abs_dir,
|
||||
)
|
||||
reason = reason_out if ok2 else "unknown"
|
||||
|
||||
result = {
|
||||
"success": True,
|
||||
"restored_to": commit_hash[:8],
|
||||
"reason": reason,
|
||||
"directory": abs_dir,
|
||||
}
|
||||
if file_path:
|
||||
result["file"] = file_path
|
||||
return result
|
||||
|
||||
def get_working_dir_for_path(self, file_path: str) -> str:
|
||||
"""Resolve a file path to its working directory for checkpointing.
|
||||
|
||||
Walks up from the file's parent to find a reasonable project root
|
||||
(directory containing .git, pyproject.toml, package.json, etc.).
|
||||
Falls back to the file's parent directory.
|
||||
"""
|
||||
path = _normalize_path(file_path)
|
||||
if path.is_dir():
|
||||
candidate = path
|
||||
else:
|
||||
candidate = path.parent
|
||||
|
||||
# Walk up looking for project root markers
|
||||
markers = {".git", "pyproject.toml", "package.json", "Cargo.toml",
|
||||
"go.mod", "Makefile", "pom.xml", ".hg", "Gemfile"}
|
||||
check = candidate
|
||||
while check != check.parent:
|
||||
if any((check / m).exists() for m in markers):
|
||||
return str(check)
|
||||
check = check.parent
|
||||
|
||||
# No project root found — use the file's parent
|
||||
return str(candidate)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internal
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _take(self, working_dir: str, reason: str) -> bool:
|
||||
"""Take a snapshot. Returns True on success."""
|
||||
shadow = _shadow_repo_path(working_dir)
|
||||
|
||||
# Init if needed
|
||||
err = _init_shadow_repo(shadow, working_dir)
|
||||
if err:
|
||||
logger.debug("Checkpoint init failed: %s", err)
|
||||
return False
|
||||
|
||||
# Quick size guard — don't try to snapshot enormous directories
|
||||
if _dir_file_count(working_dir) > _MAX_FILES:
|
||||
logger.debug("Checkpoint skipped: >%d files in %s", _MAX_FILES, working_dir)
|
||||
return False
|
||||
|
||||
# Stage everything
|
||||
ok, _, err = _run_git(
|
||||
["add", "-A"], shadow, working_dir, timeout=_GIT_TIMEOUT * 2,
|
||||
)
|
||||
if not ok:
|
||||
logger.debug("Checkpoint git-add failed: %s", err)
|
||||
return False
|
||||
|
||||
# Check if there's anything to commit
|
||||
ok_diff, diff_out, _ = _run_git(
|
||||
["diff", "--cached", "--quiet"],
|
||||
shadow,
|
||||
working_dir,
|
||||
allowed_returncodes={1},
|
||||
)
|
||||
if ok_diff:
|
||||
# No changes to commit
|
||||
logger.debug("Checkpoint skipped: no changes in %s", working_dir)
|
||||
return False
|
||||
|
||||
# Commit
|
||||
ok, _, err = _run_git(
|
||||
["commit", "-m", reason, "--allow-empty-message"],
|
||||
shadow, working_dir, timeout=_GIT_TIMEOUT * 2,
|
||||
)
|
||||
if not ok:
|
||||
logger.debug("Checkpoint commit failed: %s", err)
|
||||
return False
|
||||
|
||||
logger.debug("Checkpoint taken in %s: %s", working_dir, reason)
|
||||
|
||||
# Prune old snapshots
|
||||
self._prune(shadow, working_dir)
|
||||
|
||||
return True
|
||||
|
||||
def _prune(self, shadow_repo: Path, working_dir: str) -> None:
|
||||
"""Keep only the last max_snapshots commits via orphan reset."""
|
||||
ok, stdout, _ = _run_git(
|
||||
["rev-list", "--count", "HEAD"], shadow_repo, working_dir,
|
||||
)
|
||||
if not ok:
|
||||
return
|
||||
|
||||
try:
|
||||
count = int(stdout)
|
||||
except ValueError:
|
||||
return
|
||||
|
||||
if count <= self.max_snapshots:
|
||||
return
|
||||
|
||||
# For simplicity, we don't actually prune — git's pack mechanism
|
||||
# handles this efficiently, and the objects are small. The log
|
||||
# listing is already limited by max_snapshots.
|
||||
# Full pruning would require rebase --onto or filter-branch which
|
||||
# is fragile for a background feature. We just limit the log view.
|
||||
logger.debug("Checkpoint repo has %d commits (limit %d)", count, self.max_snapshots)
|
||||
|
||||
|
||||
def format_checkpoint_list(checkpoints: List[Dict], directory: str) -> str:
|
||||
"""Format checkpoint list for display to user."""
|
||||
if not checkpoints:
|
||||
return f"No checkpoints found for {directory}"
|
||||
|
||||
lines = [f"📸 Checkpoints for {directory}:\n"]
|
||||
for i, cp in enumerate(checkpoints, 1):
|
||||
# Parse ISO timestamp to something readable
|
||||
ts = cp["timestamp"]
|
||||
if "T" in ts:
|
||||
ts = ts.split("T")[1].split("+")[0].split("-")[0][:5] # HH:MM
|
||||
date = cp["timestamp"].split("T")[0]
|
||||
ts = f"{date} {ts}"
|
||||
|
||||
# Build change summary
|
||||
files = cp.get("files_changed", 0)
|
||||
ins = cp.get("insertions", 0)
|
||||
dele = cp.get("deletions", 0)
|
||||
if files:
|
||||
stat = f" ({files} file{'s' if files != 1 else ''}, +{ins}/-{dele})"
|
||||
else:
|
||||
stat = ""
|
||||
|
||||
lines.append(f" {i}. {cp['short_hash']} {ts} {cp['reason']}{stat}")
|
||||
|
||||
lines.append("\n /rollback <N> restore to checkpoint N")
|
||||
lines.append(" /rollback diff <N> preview changes since checkpoint N")
|
||||
lines.append(" /rollback <N> <file> restore a single file from checkpoint N")
|
||||
return "\n".join(lines)
|
||||
@@ -0,0 +1,141 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Clarify Tool Module - Interactive Clarifying Questions
|
||||
|
||||
Allows the agent to present structured multiple-choice questions or open-ended
|
||||
prompts to the user. In CLI mode, choices are navigable with arrow keys. On
|
||||
messaging platforms, choices are rendered as a numbered list.
|
||||
|
||||
The actual user-interaction logic lives in the platform layer (cli.py for CLI,
|
||||
gateway/run.py for messaging). This module defines the schema, validation, and
|
||||
a thin dispatcher that delegates to a platform-provided callback.
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import List, Optional, Callable
|
||||
|
||||
|
||||
# Maximum number of predefined choices the agent can offer.
|
||||
# A 5th "Other (type your answer)" option is always appended by the UI.
|
||||
MAX_CHOICES = 4
|
||||
|
||||
|
||||
def clarify_tool(
|
||||
question: str,
|
||||
choices: Optional[List[str]] = None,
|
||||
callback: Optional[Callable] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Ask the user a question, optionally with multiple-choice options.
|
||||
|
||||
Args:
|
||||
question: The question text to present.
|
||||
choices: Up to 4 predefined answer choices. When omitted the
|
||||
question is purely open-ended.
|
||||
callback: Platform-provided function that handles the actual UI
|
||||
interaction. Signature: callback(question, choices) -> str.
|
||||
Injected by the agent runner (cli.py / gateway).
|
||||
|
||||
Returns:
|
||||
JSON string with the user's response.
|
||||
"""
|
||||
if not question or not question.strip():
|
||||
return tool_error("Question text is required.")
|
||||
|
||||
question = question.strip()
|
||||
|
||||
# Validate and trim choices
|
||||
if choices is not None:
|
||||
if not isinstance(choices, list):
|
||||
return tool_error("choices must be a list of strings.")
|
||||
choices = [str(c).strip() for c in choices if str(c).strip()]
|
||||
if len(choices) > MAX_CHOICES:
|
||||
choices = choices[:MAX_CHOICES]
|
||||
if not choices:
|
||||
choices = None # empty list → open-ended
|
||||
|
||||
if callback is None:
|
||||
return json.dumps(
|
||||
{"error": "Clarify tool is not available in this execution context."},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
try:
|
||||
user_response = callback(question, choices)
|
||||
except Exception as exc:
|
||||
return json.dumps(
|
||||
{"error": f"Failed to get user input: {exc}"},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
return json.dumps({
|
||||
"question": question,
|
||||
"choices_offered": choices,
|
||||
"user_response": str(user_response).strip(),
|
||||
}, ensure_ascii=False)
|
||||
|
||||
|
||||
def check_clarify_requirements() -> bool:
|
||||
"""Clarify tool has no external requirements -- always available."""
|
||||
return True
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# OpenAI Function-Calling Schema
|
||||
# =============================================================================
|
||||
|
||||
CLARIFY_SCHEMA = {
|
||||
"name": "clarify",
|
||||
"description": (
|
||||
"Ask the user a question when you need clarification, feedback, or a "
|
||||
"decision before proceeding. Supports two modes:\n\n"
|
||||
"1. **Multiple choice** — provide up to 4 choices. The user picks one "
|
||||
"or types their own answer via a 5th 'Other' option.\n"
|
||||
"2. **Open-ended** — omit choices entirely. The user types a free-form "
|
||||
"response.\n\n"
|
||||
"Use this tool when:\n"
|
||||
"- The task is ambiguous and you need the user to choose an approach\n"
|
||||
"- You want post-task feedback ('How did that work out?')\n"
|
||||
"- You want to offer to save a skill or update memory\n"
|
||||
"- A decision has meaningful trade-offs the user should weigh in on\n\n"
|
||||
"Do NOT use this tool for simple yes/no confirmation of dangerous "
|
||||
"commands (the terminal tool handles that). Prefer making a reasonable "
|
||||
"default choice yourself when the decision is low-stakes."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"question": {
|
||||
"type": "string",
|
||||
"description": "The question to present to the user.",
|
||||
},
|
||||
"choices": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"maxItems": MAX_CHOICES,
|
||||
"description": (
|
||||
"Up to 4 answer choices. Omit this parameter entirely to "
|
||||
"ask an open-ended question. When provided, the UI "
|
||||
"automatically appends an 'Other (type your answer)' option."
|
||||
),
|
||||
},
|
||||
},
|
||||
"required": ["question"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# --- Registry ---
|
||||
from tools.registry import registry, tool_error
|
||||
|
||||
registry.register(
|
||||
name="clarify",
|
||||
toolset="clarify",
|
||||
schema=CLARIFY_SCHEMA,
|
||||
handler=lambda args, **kw: clarify_tool(
|
||||
question=args.get("question", ""),
|
||||
choices=args.get("choices"),
|
||||
callback=kw.get("callback")),
|
||||
check_fn=check_clarify_requirements,
|
||||
emoji="❓",
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,407 @@
|
||||
"""File passthrough registry for remote terminal backends.
|
||||
|
||||
Remote backends (Docker, Modal, SSH) create sandboxes with no host files.
|
||||
This module ensures that credential files, skill directories, and host-side
|
||||
cache directories (documents, images, audio, screenshots) are mounted or
|
||||
synced into those sandboxes so the agent can access them.
|
||||
|
||||
**Credentials and skills** — session-scoped registry fed by skill declarations
|
||||
(``required_credential_files``) and user config (``terminal.credential_files``).
|
||||
|
||||
**Cache directories** — gateway-cached uploads, browser screenshots, TTS
|
||||
audio, and processed images. Mounted read-only so the remote terminal can
|
||||
reference files the host side created (e.g. ``unzip`` an uploaded archive).
|
||||
|
||||
Remote backends call :func:`get_credential_file_mounts`,
|
||||
:func:`get_skills_directory_mount` / :func:`iter_skills_files`, and
|
||||
:func:`get_cache_directory_mounts` / :func:`iter_cache_files` at sandbox
|
||||
creation time and before each command (for resync on Modal).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from contextvars import ContextVar
|
||||
from pathlib import Path
|
||||
from typing import Dict, List
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Session-scoped list of credential files to mount.
|
||||
# Backed by ContextVar to prevent cross-session data bleed in the gateway pipeline.
|
||||
_registered_files_var: ContextVar[Dict[str, str]] = ContextVar("_registered_files")
|
||||
|
||||
|
||||
def _get_registered() -> Dict[str, str]:
|
||||
"""Get or create the registered credential files dict for the current context/session."""
|
||||
try:
|
||||
return _registered_files_var.get()
|
||||
except LookupError:
|
||||
val: Dict[str, str] = {}
|
||||
_registered_files_var.set(val)
|
||||
return val
|
||||
|
||||
|
||||
# Cache for config-based file list (loaded once per process).
|
||||
_config_files: List[Dict[str, str]] | None = None
|
||||
|
||||
|
||||
def _resolve_hermes_home() -> Path:
|
||||
from hermes_constants import get_hermes_home
|
||||
return get_hermes_home()
|
||||
|
||||
|
||||
def register_credential_file(
|
||||
relative_path: str,
|
||||
container_base: str = "/root/.hermes",
|
||||
) -> bool:
|
||||
"""Register a credential file for mounting into remote sandboxes.
|
||||
|
||||
*relative_path* is relative to ``HERMES_HOME`` (e.g. ``google_token.json``).
|
||||
Returns True if the file exists on the host and was registered.
|
||||
|
||||
Security: rejects absolute paths and path traversal sequences (``..``).
|
||||
The resolved host path must remain inside HERMES_HOME so that a malicious
|
||||
skill cannot declare ``required_credential_files: ['../../.ssh/id_rsa']``
|
||||
and exfiltrate sensitive host files into a container sandbox.
|
||||
"""
|
||||
hermes_home = _resolve_hermes_home()
|
||||
|
||||
# Reject absolute paths — they bypass the HERMES_HOME sandbox entirely.
|
||||
if os.path.isabs(relative_path):
|
||||
logger.warning(
|
||||
"credential_files: rejected absolute path %r (must be relative to HERMES_HOME)",
|
||||
relative_path,
|
||||
)
|
||||
return False
|
||||
|
||||
host_path = hermes_home / relative_path
|
||||
|
||||
# Resolve symlinks and normalise ``..`` before the containment check so
|
||||
# that traversal like ``../. ssh/id_rsa`` cannot escape HERMES_HOME.
|
||||
from tools.path_security import validate_within_dir
|
||||
|
||||
containment_error = validate_within_dir(host_path, hermes_home)
|
||||
if containment_error:
|
||||
logger.warning(
|
||||
"credential_files: rejected path traversal %r (%s)",
|
||||
relative_path,
|
||||
containment_error,
|
||||
)
|
||||
return False
|
||||
|
||||
resolved = host_path.resolve()
|
||||
if not resolved.is_file():
|
||||
logger.debug("credential_files: skipping %s (not found)", resolved)
|
||||
return False
|
||||
|
||||
container_path = f"{container_base.rstrip('/')}/{relative_path}"
|
||||
_get_registered()[container_path] = str(resolved)
|
||||
logger.debug("credential_files: registered %s -> %s", resolved, container_path)
|
||||
return True
|
||||
|
||||
|
||||
def register_credential_files(
|
||||
entries: list,
|
||||
container_base: str = "/root/.hermes",
|
||||
) -> List[str]:
|
||||
"""Register multiple credential files from skill frontmatter entries.
|
||||
|
||||
Each entry is either a string (relative path) or a dict with a ``path``
|
||||
key. Returns the list of relative paths that were NOT found on the host
|
||||
(i.e. missing files).
|
||||
"""
|
||||
missing = []
|
||||
for entry in entries:
|
||||
if isinstance(entry, str):
|
||||
rel_path = entry.strip()
|
||||
elif isinstance(entry, dict):
|
||||
rel_path = (entry.get("path") or entry.get("name") or "").strip()
|
||||
else:
|
||||
continue
|
||||
if not rel_path:
|
||||
continue
|
||||
if not register_credential_file(rel_path, container_base):
|
||||
missing.append(rel_path)
|
||||
return missing
|
||||
|
||||
|
||||
def _load_config_files() -> List[Dict[str, str]]:
|
||||
"""Load ``terminal.credential_files`` from config.yaml (cached)."""
|
||||
global _config_files
|
||||
if _config_files is not None:
|
||||
return _config_files
|
||||
|
||||
result: List[Dict[str, str]] = []
|
||||
try:
|
||||
from hermes_cli.config import read_raw_config
|
||||
hermes_home = _resolve_hermes_home()
|
||||
cfg = read_raw_config()
|
||||
cred_files = cfg.get("terminal", {}).get("credential_files")
|
||||
if isinstance(cred_files, list):
|
||||
from tools.path_security import validate_within_dir
|
||||
|
||||
for item in cred_files:
|
||||
if isinstance(item, str) and item.strip():
|
||||
rel = item.strip()
|
||||
if os.path.isabs(rel):
|
||||
logger.warning(
|
||||
"credential_files: rejected absolute config path %r", rel,
|
||||
)
|
||||
continue
|
||||
host_path = hermes_home / rel
|
||||
containment_error = validate_within_dir(host_path, hermes_home)
|
||||
if containment_error:
|
||||
logger.warning(
|
||||
"credential_files: rejected config path traversal %r (%s)",
|
||||
rel, containment_error,
|
||||
)
|
||||
continue
|
||||
resolved_path = host_path.resolve()
|
||||
if resolved_path.is_file():
|
||||
container_path = f"/root/.hermes/{rel}"
|
||||
result.append({
|
||||
"host_path": str(resolved_path),
|
||||
"container_path": container_path,
|
||||
})
|
||||
except Exception as e:
|
||||
logger.warning("Could not read terminal.credential_files from config: %s", e)
|
||||
|
||||
_config_files = result
|
||||
return _config_files
|
||||
|
||||
|
||||
def get_credential_file_mounts() -> List[Dict[str, str]]:
|
||||
"""Return all credential files that should be mounted into remote sandboxes.
|
||||
|
||||
Each item has ``host_path`` and ``container_path`` keys.
|
||||
Combines skill-registered files and user config.
|
||||
"""
|
||||
mounts: Dict[str, str] = {}
|
||||
|
||||
# Skill-registered files
|
||||
for container_path, host_path in _get_registered().items():
|
||||
# Re-check existence (file may have been deleted since registration)
|
||||
if Path(host_path).is_file():
|
||||
mounts[container_path] = host_path
|
||||
|
||||
# Config-based files
|
||||
for entry in _load_config_files():
|
||||
cp = entry["container_path"]
|
||||
if cp not in mounts and Path(entry["host_path"]).is_file():
|
||||
mounts[cp] = entry["host_path"]
|
||||
|
||||
return [
|
||||
{"host_path": hp, "container_path": cp}
|
||||
for cp, hp in mounts.items()
|
||||
]
|
||||
|
||||
|
||||
def get_skills_directory_mount(
|
||||
container_base: str = "/root/.hermes",
|
||||
) -> list[Dict[str, str]]:
|
||||
"""Return mount info for all skill directories (local + external).
|
||||
|
||||
Skills may include ``scripts/``, ``templates/``, and ``references/``
|
||||
subdirectories that the agent needs to execute inside remote sandboxes.
|
||||
|
||||
**Security:** Bind mounts follow symlinks, so a malicious symlink inside
|
||||
the skills tree could expose arbitrary host files to the container. When
|
||||
symlinks are detected, this function creates a sanitized copy (regular
|
||||
files only) in a temp directory and returns that path instead. When no
|
||||
symlinks are present (the common case), the original directory is returned
|
||||
directly with zero overhead.
|
||||
|
||||
Returns a list of dicts with ``host_path`` and ``container_path`` keys.
|
||||
The local skills dir mounts at ``<container_base>/skills``, external dirs
|
||||
at ``<container_base>/external_skills/<index>``.
|
||||
"""
|
||||
mounts = []
|
||||
hermes_home = _resolve_hermes_home()
|
||||
skills_dir = hermes_home / "skills"
|
||||
if skills_dir.is_dir():
|
||||
host_path = _safe_skills_path(skills_dir)
|
||||
mounts.append({
|
||||
"host_path": host_path,
|
||||
"container_path": f"{container_base.rstrip('/')}/skills",
|
||||
})
|
||||
|
||||
# Mount external skill dirs
|
||||
try:
|
||||
from agent.skill_utils import get_external_skills_dirs
|
||||
for idx, ext_dir in enumerate(get_external_skills_dirs()):
|
||||
if ext_dir.is_dir():
|
||||
host_path = _safe_skills_path(ext_dir)
|
||||
mounts.append({
|
||||
"host_path": host_path,
|
||||
"container_path": f"{container_base.rstrip('/')}/external_skills/{idx}",
|
||||
})
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
return mounts
|
||||
|
||||
|
||||
_safe_skills_tempdir: Path | None = None
|
||||
|
||||
|
||||
def _safe_skills_path(skills_dir: Path) -> str:
|
||||
"""Return *skills_dir* if symlink-free, else a sanitized temp copy."""
|
||||
global _safe_skills_tempdir
|
||||
|
||||
symlinks = [p for p in skills_dir.rglob("*") if p.is_symlink()]
|
||||
if not symlinks:
|
||||
return str(skills_dir)
|
||||
|
||||
for link in symlinks:
|
||||
logger.warning("credential_files: skipping symlink in skills dir: %s -> %s",
|
||||
link, os.readlink(link))
|
||||
|
||||
import atexit
|
||||
import shutil
|
||||
import tempfile
|
||||
|
||||
# Reuse the same temp dir across calls to avoid accumulation.
|
||||
if _safe_skills_tempdir and _safe_skills_tempdir.is_dir():
|
||||
shutil.rmtree(_safe_skills_tempdir, ignore_errors=True)
|
||||
|
||||
safe_dir = Path(tempfile.mkdtemp(prefix="hermes-skills-safe-"))
|
||||
_safe_skills_tempdir = safe_dir
|
||||
|
||||
for item in skills_dir.rglob("*"):
|
||||
if item.is_symlink():
|
||||
continue
|
||||
rel = item.relative_to(skills_dir)
|
||||
target = safe_dir / rel
|
||||
if item.is_dir():
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
elif item.is_file():
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(str(item), str(target))
|
||||
|
||||
def _cleanup():
|
||||
if safe_dir.is_dir():
|
||||
shutil.rmtree(safe_dir, ignore_errors=True)
|
||||
|
||||
atexit.register(_cleanup)
|
||||
logger.info("credential_files: created symlink-safe skills copy at %s", safe_dir)
|
||||
return str(safe_dir)
|
||||
|
||||
|
||||
def iter_skills_files(
|
||||
container_base: str = "/root/.hermes",
|
||||
) -> List[Dict[str, str]]:
|
||||
"""Yield individual (host_path, container_path) entries for skills files.
|
||||
|
||||
Includes both the local skills dir and any external dirs configured via
|
||||
skills.external_dirs. Skips symlinks entirely. Preferred for backends
|
||||
that upload files individually (Daytona, Modal) rather than mounting a
|
||||
directory.
|
||||
"""
|
||||
result: List[Dict[str, str]] = []
|
||||
|
||||
hermes_home = _resolve_hermes_home()
|
||||
skills_dir = hermes_home / "skills"
|
||||
if skills_dir.is_dir():
|
||||
container_root = f"{container_base.rstrip('/')}/skills"
|
||||
for item in skills_dir.rglob("*"):
|
||||
if item.is_symlink() or not item.is_file():
|
||||
continue
|
||||
rel = item.relative_to(skills_dir)
|
||||
result.append({
|
||||
"host_path": str(item),
|
||||
"container_path": f"{container_root}/{rel}",
|
||||
})
|
||||
|
||||
# Include external skill dirs
|
||||
try:
|
||||
from agent.skill_utils import get_external_skills_dirs
|
||||
for idx, ext_dir in enumerate(get_external_skills_dirs()):
|
||||
if not ext_dir.is_dir():
|
||||
continue
|
||||
container_root = f"{container_base.rstrip('/')}/external_skills/{idx}"
|
||||
for item in ext_dir.rglob("*"):
|
||||
if item.is_symlink() or not item.is_file():
|
||||
continue
|
||||
rel = item.relative_to(ext_dir)
|
||||
result.append({
|
||||
"host_path": str(item),
|
||||
"container_path": f"{container_root}/{rel}",
|
||||
})
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cache directory mounts (documents, images, audio, screenshots)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# The four cache subdirectories that should be mirrored into remote backends.
|
||||
# Each tuple is (new_subpath, old_name) matching hermes_constants.get_hermes_dir().
|
||||
_CACHE_DIRS: list[tuple[str, str]] = [
|
||||
("cache/documents", "document_cache"),
|
||||
("cache/images", "image_cache"),
|
||||
("cache/audio", "audio_cache"),
|
||||
("cache/screenshots", "browser_screenshots"),
|
||||
]
|
||||
|
||||
|
||||
def get_cache_directory_mounts(
|
||||
container_base: str = "/root/.hermes",
|
||||
) -> List[Dict[str, str]]:
|
||||
"""Return mount entries for each cache directory that exists on disk.
|
||||
|
||||
Used by Docker to create bind mounts. Each entry has ``host_path`` and
|
||||
``container_path`` keys. The host path is resolved via
|
||||
``get_hermes_dir()`` for backward compatibility with old directory layouts.
|
||||
"""
|
||||
from hermes_constants import get_hermes_dir
|
||||
|
||||
mounts: List[Dict[str, str]] = []
|
||||
for new_subpath, old_name in _CACHE_DIRS:
|
||||
host_dir = get_hermes_dir(new_subpath, old_name)
|
||||
if host_dir.is_dir():
|
||||
# Always map to the *new* container layout regardless of host layout.
|
||||
container_path = f"{container_base.rstrip('/')}/{new_subpath}"
|
||||
mounts.append({
|
||||
"host_path": str(host_dir),
|
||||
"container_path": container_path,
|
||||
})
|
||||
return mounts
|
||||
|
||||
|
||||
def iter_cache_files(
|
||||
container_base: str = "/root/.hermes",
|
||||
) -> List[Dict[str, str]]:
|
||||
"""Return individual (host_path, container_path) entries for cache files.
|
||||
|
||||
Used by Modal to upload files individually and resync before each command.
|
||||
Skips symlinks. The container paths use the new ``cache/<subdir>`` layout.
|
||||
"""
|
||||
from hermes_constants import get_hermes_dir
|
||||
|
||||
result: List[Dict[str, str]] = []
|
||||
for new_subpath, old_name in _CACHE_DIRS:
|
||||
host_dir = get_hermes_dir(new_subpath, old_name)
|
||||
if not host_dir.is_dir():
|
||||
continue
|
||||
container_root = f"{container_base.rstrip('/')}/{new_subpath}"
|
||||
for item in host_dir.rglob("*"):
|
||||
if item.is_symlink() or not item.is_file():
|
||||
continue
|
||||
rel = item.relative_to(host_dir)
|
||||
result.append({
|
||||
"host_path": str(item),
|
||||
"container_path": f"{container_root}/{rel}",
|
||||
})
|
||||
return result
|
||||
|
||||
|
||||
def clear_credential_files() -> None:
|
||||
"""Reset the skill-scoped registry (e.g. on session reset)."""
|
||||
_get_registered().clear()
|
||||
|
||||
|
||||
@@ -0,0 +1,506 @@
|
||||
"""
|
||||
Cron job management tools for Hermes Agent.
|
||||
|
||||
Expose a single compressed action-oriented tool to avoid schema/context bloat.
|
||||
Compatibility wrappers remain for direct Python callers and legacy tests.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Import from cron module (will be available when properly installed)
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from cron.jobs import (
|
||||
create_job,
|
||||
get_job,
|
||||
list_jobs,
|
||||
parse_schedule,
|
||||
pause_job,
|
||||
remove_job,
|
||||
resume_job,
|
||||
trigger_job,
|
||||
update_job,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cron prompt scanning — critical-severity patterns only, since cron prompts
|
||||
# run in fresh sessions with full tool access.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_CRON_THREAT_PATTERNS = [
|
||||
(r'ignore\s+(?:\w+\s+)*(?:previous|all|above|prior)\s+(?:\w+\s+)*instructions', "prompt_injection"),
|
||||
(r'do\s+not\s+tell\s+the\s+user', "deception_hide"),
|
||||
(r'system\s+prompt\s+override', "sys_prompt_override"),
|
||||
(r'disregard\s+(your|all|any)\s+(instructions|rules|guidelines)', "disregard_rules"),
|
||||
(r'curl\s+[^\n]*\$\{?\w*(KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|API)', "exfil_curl"),
|
||||
(r'wget\s+[^\n]*\$\{?\w*(KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|API)', "exfil_wget"),
|
||||
(r'cat\s+[^\n]*(\.env|credentials|\.netrc|\.pgpass)', "read_secrets"),
|
||||
(r'authorized_keys', "ssh_backdoor"),
|
||||
(r'/etc/sudoers|visudo', "sudoers_mod"),
|
||||
(r'rm\s+-rf\s+/', "destructive_root_rm"),
|
||||
]
|
||||
|
||||
_CRON_INVISIBLE_CHARS = {
|
||||
'\u200b', '\u200c', '\u200d', '\u2060', '\ufeff',
|
||||
'\u202a', '\u202b', '\u202c', '\u202d', '\u202e',
|
||||
}
|
||||
|
||||
|
||||
def _scan_cron_prompt(prompt: str) -> str:
|
||||
"""Scan a cron prompt for critical threats. Returns error string if blocked, else empty."""
|
||||
for char in _CRON_INVISIBLE_CHARS:
|
||||
if char in prompt:
|
||||
return f"Blocked: prompt contains invisible unicode U+{ord(char):04X} (possible injection)."
|
||||
for pattern, pid in _CRON_THREAT_PATTERNS:
|
||||
if re.search(pattern, prompt, re.IGNORECASE):
|
||||
return f"Blocked: prompt matches threat pattern '{pid}'. Cron prompts must not contain injection or exfiltration payloads."
|
||||
return ""
|
||||
|
||||
|
||||
def _origin_from_env() -> Optional[Dict[str, str]]:
|
||||
from gateway.session_context import get_session_env
|
||||
origin_platform = get_session_env("HERMES_SESSION_PLATFORM")
|
||||
origin_chat_id = get_session_env("HERMES_SESSION_CHAT_ID")
|
||||
if origin_platform and origin_chat_id:
|
||||
thread_id = get_session_env("HERMES_SESSION_THREAD_ID") or None
|
||||
if thread_id:
|
||||
logger.debug(
|
||||
"Cron origin captured thread_id=%s for %s:%s",
|
||||
thread_id, origin_platform, origin_chat_id,
|
||||
)
|
||||
return {
|
||||
"platform": origin_platform,
|
||||
"chat_id": origin_chat_id,
|
||||
"chat_name": get_session_env("HERMES_SESSION_CHAT_NAME") or None,
|
||||
"thread_id": thread_id,
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def _repeat_display(job: Dict[str, Any]) -> str:
|
||||
times = (job.get("repeat") or {}).get("times")
|
||||
completed = (job.get("repeat") or {}).get("completed", 0)
|
||||
if times is None:
|
||||
return "forever"
|
||||
if times == 1:
|
||||
return "once" if completed == 0 else "1/1"
|
||||
return f"{completed}/{times}" if completed else f"{times} times"
|
||||
|
||||
|
||||
def _canonical_skills(skill: Optional[str] = None, skills: Optional[Any] = None) -> List[str]:
|
||||
if skills is None:
|
||||
raw_items = [skill] if skill else []
|
||||
elif isinstance(skills, str):
|
||||
raw_items = [skills]
|
||||
else:
|
||||
raw_items = list(skills)
|
||||
|
||||
normalized: List[str] = []
|
||||
for item in raw_items:
|
||||
text = str(item or "").strip()
|
||||
if text and text not in normalized:
|
||||
normalized.append(text)
|
||||
return normalized
|
||||
|
||||
|
||||
|
||||
|
||||
def _resolve_model_override(model_obj: Optional[Dict[str, Any]]) -> tuple:
|
||||
"""Resolve a model override object into (provider, model) for job storage.
|
||||
|
||||
If provider is omitted, pins the current main provider from config so the
|
||||
job doesn't drift when the user later changes their default via hermes model.
|
||||
|
||||
Returns (provider_str_or_none, model_str_or_none).
|
||||
"""
|
||||
if not model_obj or not isinstance(model_obj, dict):
|
||||
return (None, None)
|
||||
model_name = (model_obj.get("model") or "").strip() or None
|
||||
provider_name = (model_obj.get("provider") or "").strip() or None
|
||||
if model_name and not provider_name:
|
||||
# Pin to the current main provider so the job is stable
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
cfg = load_config()
|
||||
model_cfg = cfg.get("model", {})
|
||||
if isinstance(model_cfg, dict):
|
||||
provider_name = model_cfg.get("provider") or None
|
||||
except Exception:
|
||||
pass # Best-effort; provider stays None
|
||||
return (provider_name, model_name)
|
||||
|
||||
|
||||
def _normalize_optional_job_value(value: Optional[Any], *, strip_trailing_slash: bool = False) -> Optional[str]:
|
||||
if value is None:
|
||||
return None
|
||||
text = str(value).strip()
|
||||
if strip_trailing_slash:
|
||||
text = text.rstrip("/")
|
||||
return text or None
|
||||
|
||||
|
||||
def _validate_cron_script_path(script: Optional[str]) -> Optional[str]:
|
||||
"""Validate a cron job script path at the API boundary.
|
||||
|
||||
Scripts must be relative paths that resolve within HERMES_HOME/scripts/.
|
||||
Absolute paths and ~ expansion are rejected to prevent arbitrary script
|
||||
execution via prompt injection.
|
||||
|
||||
Returns an error string if blocked, else None (valid).
|
||||
"""
|
||||
if not script or not script.strip():
|
||||
return None # empty/None = clearing the field, always OK
|
||||
|
||||
from hermes_constants import get_hermes_home
|
||||
|
||||
raw = script.strip()
|
||||
|
||||
# Reject absolute paths and ~ expansion at the API boundary.
|
||||
# Only relative paths within ~/.hermes/scripts/ are allowed.
|
||||
if raw.startswith(("/", "~")) or (len(raw) >= 2 and raw[1] == ":"):
|
||||
return (
|
||||
f"Script path must be relative to ~/.hermes/scripts/. "
|
||||
f"Got absolute or home-relative path: {raw!r}. "
|
||||
f"Place scripts in ~/.hermes/scripts/ and use just the filename."
|
||||
)
|
||||
|
||||
# Validate containment after resolution
|
||||
from tools.path_security import validate_within_dir
|
||||
|
||||
scripts_dir = get_hermes_home() / "scripts"
|
||||
scripts_dir.mkdir(parents=True, exist_ok=True)
|
||||
containment_error = validate_within_dir(scripts_dir / raw, scripts_dir)
|
||||
if containment_error:
|
||||
return (
|
||||
f"Script path escapes the scripts directory via traversal: {raw!r}"
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _format_job(job: Dict[str, Any]) -> Dict[str, Any]:
|
||||
prompt = job.get("prompt", "")
|
||||
skills = _canonical_skills(job.get("skill"), job.get("skills"))
|
||||
result = {
|
||||
"job_id": job["id"],
|
||||
"name": job["name"],
|
||||
"skill": skills[0] if skills else None,
|
||||
"skills": skills,
|
||||
"prompt_preview": prompt[:100] + "..." if len(prompt) > 100 else prompt,
|
||||
"model": job.get("model"),
|
||||
"provider": job.get("provider"),
|
||||
"base_url": job.get("base_url"),
|
||||
"schedule": job.get("schedule_display"),
|
||||
"repeat": _repeat_display(job),
|
||||
"deliver": job.get("deliver", "local"),
|
||||
"next_run_at": job.get("next_run_at"),
|
||||
"last_run_at": job.get("last_run_at"),
|
||||
"last_status": job.get("last_status"),
|
||||
"last_delivery_error": job.get("last_delivery_error"),
|
||||
"enabled": job.get("enabled", True),
|
||||
"state": job.get("state", "scheduled" if job.get("enabled", True) else "paused"),
|
||||
"paused_at": job.get("paused_at"),
|
||||
"paused_reason": job.get("paused_reason"),
|
||||
}
|
||||
if job.get("script"):
|
||||
result["script"] = job["script"]
|
||||
return result
|
||||
|
||||
|
||||
def cronjob(
|
||||
action: str,
|
||||
job_id: Optional[str] = None,
|
||||
prompt: Optional[str] = None,
|
||||
schedule: Optional[str] = None,
|
||||
name: Optional[str] = None,
|
||||
repeat: Optional[int] = None,
|
||||
deliver: Optional[str] = None,
|
||||
include_disabled: bool = False,
|
||||
skill: Optional[str] = None,
|
||||
skills: Optional[List[str]] = None,
|
||||
model: Optional[str] = None,
|
||||
provider: Optional[str] = None,
|
||||
base_url: Optional[str] = None,
|
||||
reason: Optional[str] = None,
|
||||
script: Optional[str] = None,
|
||||
task_id: str = None,
|
||||
) -> str:
|
||||
"""Unified cron job management tool."""
|
||||
del task_id # unused but kept for handler signature compatibility
|
||||
|
||||
try:
|
||||
normalized = (action or "").strip().lower()
|
||||
|
||||
if normalized == "create":
|
||||
if not schedule:
|
||||
return tool_error("schedule is required for create", success=False)
|
||||
canonical_skills = _canonical_skills(skill, skills)
|
||||
if not prompt and not canonical_skills:
|
||||
return tool_error("create requires either prompt or at least one skill", success=False)
|
||||
if prompt:
|
||||
scan_error = _scan_cron_prompt(prompt)
|
||||
if scan_error:
|
||||
return tool_error(scan_error, success=False)
|
||||
|
||||
# Validate script path before storing
|
||||
if script:
|
||||
script_error = _validate_cron_script_path(script)
|
||||
if script_error:
|
||||
return tool_error(script_error, success=False)
|
||||
|
||||
job = create_job(
|
||||
prompt=prompt or "",
|
||||
schedule=schedule,
|
||||
name=name,
|
||||
repeat=repeat,
|
||||
deliver=deliver,
|
||||
origin=_origin_from_env(),
|
||||
skills=canonical_skills,
|
||||
model=_normalize_optional_job_value(model),
|
||||
provider=_normalize_optional_job_value(provider),
|
||||
base_url=_normalize_optional_job_value(base_url, strip_trailing_slash=True),
|
||||
script=_normalize_optional_job_value(script),
|
||||
)
|
||||
return json.dumps(
|
||||
{
|
||||
"success": True,
|
||||
"job_id": job["id"],
|
||||
"name": job["name"],
|
||||
"skill": job.get("skill"),
|
||||
"skills": job.get("skills", []),
|
||||
"schedule": job["schedule_display"],
|
||||
"repeat": _repeat_display(job),
|
||||
"deliver": job.get("deliver", "local"),
|
||||
"next_run_at": job["next_run_at"],
|
||||
"job": _format_job(job),
|
||||
"message": f"Cron job '{job['name']}' created.",
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
|
||||
if normalized == "list":
|
||||
jobs = [_format_job(job) for job in list_jobs(include_disabled=include_disabled)]
|
||||
return json.dumps({"success": True, "count": len(jobs), "jobs": jobs}, indent=2)
|
||||
|
||||
if not job_id:
|
||||
return tool_error(f"job_id is required for action '{normalized}'", success=False)
|
||||
|
||||
job = get_job(job_id)
|
||||
if not job:
|
||||
return json.dumps(
|
||||
{"success": False, "error": f"Job with ID '{job_id}' not found. Use cronjob(action='list') to inspect jobs."},
|
||||
indent=2,
|
||||
)
|
||||
|
||||
if normalized == "remove":
|
||||
removed = remove_job(job_id)
|
||||
if not removed:
|
||||
return tool_error(f"Failed to remove job '{job_id}'", success=False)
|
||||
return json.dumps(
|
||||
{
|
||||
"success": True,
|
||||
"message": f"Cron job '{job['name']}' removed.",
|
||||
"removed_job": {
|
||||
"id": job_id,
|
||||
"name": job["name"],
|
||||
"schedule": job.get("schedule_display"),
|
||||
},
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
|
||||
if normalized == "pause":
|
||||
updated = pause_job(job_id, reason=reason)
|
||||
return json.dumps({"success": True, "job": _format_job(updated)}, indent=2)
|
||||
|
||||
if normalized == "resume":
|
||||
updated = resume_job(job_id)
|
||||
return json.dumps({"success": True, "job": _format_job(updated)}, indent=2)
|
||||
|
||||
if normalized in {"run", "run_now", "trigger"}:
|
||||
updated = trigger_job(job_id)
|
||||
return json.dumps({"success": True, "job": _format_job(updated)}, indent=2)
|
||||
|
||||
if normalized == "update":
|
||||
updates: Dict[str, Any] = {}
|
||||
if prompt is not None:
|
||||
scan_error = _scan_cron_prompt(prompt)
|
||||
if scan_error:
|
||||
return tool_error(scan_error, success=False)
|
||||
updates["prompt"] = prompt
|
||||
if name is not None:
|
||||
updates["name"] = name
|
||||
if deliver is not None:
|
||||
updates["deliver"] = deliver
|
||||
if skills is not None or skill is not None:
|
||||
canonical_skills = _canonical_skills(skill, skills)
|
||||
updates["skills"] = canonical_skills
|
||||
updates["skill"] = canonical_skills[0] if canonical_skills else None
|
||||
if model is not None:
|
||||
updates["model"] = _normalize_optional_job_value(model)
|
||||
if provider is not None:
|
||||
updates["provider"] = _normalize_optional_job_value(provider)
|
||||
if base_url is not None:
|
||||
updates["base_url"] = _normalize_optional_job_value(base_url, strip_trailing_slash=True)
|
||||
if script is not None:
|
||||
# Pass empty string to clear an existing script
|
||||
if script:
|
||||
script_error = _validate_cron_script_path(script)
|
||||
if script_error:
|
||||
return tool_error(script_error, success=False)
|
||||
updates["script"] = _normalize_optional_job_value(script) if script else None
|
||||
if repeat is not None:
|
||||
# Normalize: treat 0 or negative as None (infinite)
|
||||
normalized_repeat = None if repeat <= 0 else repeat
|
||||
repeat_state = dict(job.get("repeat") or {})
|
||||
repeat_state["times"] = normalized_repeat
|
||||
updates["repeat"] = repeat_state
|
||||
if schedule is not None:
|
||||
parsed_schedule = parse_schedule(schedule)
|
||||
updates["schedule"] = parsed_schedule
|
||||
updates["schedule_display"] = parsed_schedule.get("display", schedule)
|
||||
if job.get("state") != "paused":
|
||||
updates["state"] = "scheduled"
|
||||
updates["enabled"] = True
|
||||
if not updates:
|
||||
return tool_error("No updates provided.", success=False)
|
||||
updated = update_job(job_id, updates)
|
||||
return json.dumps({"success": True, "job": _format_job(updated)}, indent=2)
|
||||
|
||||
return tool_error(f"Unknown cron action '{action}'", success=False)
|
||||
|
||||
except Exception as e:
|
||||
return tool_error(str(e), success=False)
|
||||
|
||||
|
||||
|
||||
CRONJOB_SCHEMA = {
|
||||
"name": "cronjob",
|
||||
"description": """Manage scheduled cron jobs with a single compressed tool.
|
||||
|
||||
Use action='create' to schedule a new job from a prompt or one or more skills.
|
||||
Use action='list' to inspect jobs.
|
||||
Use action='update', 'pause', 'resume', 'remove', or 'run' to manage an existing job.
|
||||
|
||||
Jobs run in a fresh session with no current-chat context, so prompts must be self-contained.
|
||||
If skills are provided on create, the future cron run loads those skills in order, then follows the prompt as the task instruction.
|
||||
On update, passing skills=[] clears attached skills.
|
||||
|
||||
NOTE: The agent's final response is auto-delivered to the target. Put the primary
|
||||
user-facing content in the final response. Cron jobs run autonomously with no user
|
||||
present — they cannot ask questions or request clarification.
|
||||
|
||||
Important safety rule: cron-run sessions should not recursively schedule more cron jobs.""",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"action": {
|
||||
"type": "string",
|
||||
"description": "One of: create, list, update, pause, resume, remove, run"
|
||||
},
|
||||
"job_id": {
|
||||
"type": "string",
|
||||
"description": "Required for update/pause/resume/remove/run"
|
||||
},
|
||||
"prompt": {
|
||||
"type": "string",
|
||||
"description": "For create: the full self-contained prompt. If skills are also provided, this becomes the task instruction paired with those skills."
|
||||
},
|
||||
"schedule": {
|
||||
"type": "string",
|
||||
"description": "For create/update: '30m', 'every 2h', '0 9 * * *', or ISO timestamp"
|
||||
},
|
||||
"name": {
|
||||
"type": "string",
|
||||
"description": "Optional human-friendly name"
|
||||
},
|
||||
"repeat": {
|
||||
"type": "integer",
|
||||
"description": "Optional repeat count. Omit for defaults (once for one-shot, forever for recurring)."
|
||||
},
|
||||
"deliver": {
|
||||
"type": "string",
|
||||
"description": "Omit this parameter to auto-deliver back to the current chat and topic (recommended). Auto-detection preserves thread/topic context. Only set explicitly when the user asks to deliver somewhere OTHER than the current conversation. Values: 'origin' (same as omitting), 'local' (no delivery, save only), or platform:chat_id:thread_id for a specific destination. Examples: 'telegram:-1001234567890:17585', 'discord:#engineering', 'sms:+15551234567'. WARNING: 'platform:chat_id' without :thread_id loses topic targeting."
|
||||
},
|
||||
"skills": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Optional ordered list of skill names to load before executing the cron prompt. On update, pass an empty array to clear attached skills."
|
||||
},
|
||||
"model": {
|
||||
"type": "object",
|
||||
"description": "Optional per-job model override. If provider is omitted, the current main provider is pinned at creation time so the job stays stable.",
|
||||
"properties": {
|
||||
"provider": {
|
||||
"type": "string",
|
||||
"description": "Provider name (e.g. 'openrouter', 'anthropic'). Omit to use and pin the current provider."
|
||||
},
|
||||
"model": {
|
||||
"type": "string",
|
||||
"description": "Model name (e.g. 'anthropic/claude-sonnet-4', 'claude-sonnet-4')"
|
||||
}
|
||||
},
|
||||
"required": ["model"]
|
||||
},
|
||||
"script": {
|
||||
"type": "string",
|
||||
"description": "Optional path to a Python script that runs before each cron job execution. Its stdout is injected into the prompt as context. Use for data collection and change detection. Relative paths resolve under ~/.hermes/scripts/. On update, pass empty string to clear."
|
||||
},
|
||||
},
|
||||
"required": ["action"]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def check_cronjob_requirements() -> bool:
|
||||
"""
|
||||
Check if cronjob tools can be used.
|
||||
|
||||
Available in interactive CLI mode and gateway/messaging platforms.
|
||||
The cron system is internal (JSON file-based scheduler ticked by the gateway),
|
||||
so no external crontab executable is required.
|
||||
"""
|
||||
return bool(
|
||||
os.getenv("HERMES_INTERACTIVE")
|
||||
or os.getenv("HERMES_GATEWAY_SESSION")
|
||||
or os.getenv("HERMES_EXEC_ASK")
|
||||
)
|
||||
|
||||
|
||||
# --- Registry ---
|
||||
from tools.registry import registry, tool_error
|
||||
|
||||
registry.register(
|
||||
name="cronjob",
|
||||
toolset="cronjob",
|
||||
schema=CRONJOB_SCHEMA,
|
||||
handler=lambda args, **kw: (lambda _mo=_resolve_model_override(args.get("model")): cronjob(
|
||||
action=args.get("action", ""),
|
||||
job_id=args.get("job_id"),
|
||||
prompt=args.get("prompt"),
|
||||
schedule=args.get("schedule"),
|
||||
name=args.get("name"),
|
||||
repeat=args.get("repeat"),
|
||||
deliver=args.get("deliver"),
|
||||
include_disabled=args.get("include_disabled", True),
|
||||
skill=args.get("skill"),
|
||||
skills=args.get("skills"),
|
||||
model=_mo[1],
|
||||
provider=_mo[0] or args.get("provider"),
|
||||
base_url=args.get("base_url"),
|
||||
reason=args.get("reason"),
|
||||
script=args.get("script"),
|
||||
task_id=kw.get("task_id"),
|
||||
))(),
|
||||
check_fn=check_cronjob_requirements,
|
||||
emoji="⏰",
|
||||
)
|
||||
@@ -0,0 +1,105 @@
|
||||
"""Shared debug session infrastructure for Hermes tools.
|
||||
|
||||
Replaces the identical DEBUG_MODE / _log_debug_call / _save_debug_log /
|
||||
get_debug_session_info boilerplate previously duplicated across web_tools,
|
||||
vision_tools, mixture_of_agents_tool, and image_generation_tool.
|
||||
|
||||
Usage in a tool module:
|
||||
|
||||
from tools.debug_helpers import DebugSession
|
||||
|
||||
_debug = DebugSession("web_tools", env_var="WEB_TOOLS_DEBUG")
|
||||
|
||||
# Log a call (no-op when debug mode is off)
|
||||
_debug.log_call("web_search", {"query": q, "results": len(r)})
|
||||
|
||||
# Save the debug log (no-op when debug mode is off)
|
||||
_debug.save()
|
||||
|
||||
# Expose debug info to external callers
|
||||
def get_debug_session_info():
|
||||
return _debug.get_session_info()
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import uuid
|
||||
from typing import Any, Dict
|
||||
|
||||
from hermes_constants import get_hermes_home
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DebugSession:
|
||||
"""Per-tool debug session that records tool calls to a JSON log file.
|
||||
|
||||
Activated by a tool-specific environment variable (e.g. WEB_TOOLS_DEBUG=true).
|
||||
When disabled, all methods are cheap no-ops.
|
||||
"""
|
||||
|
||||
def __init__(self, tool_name: str, *, env_var: str) -> None:
|
||||
self.tool_name = tool_name
|
||||
self.enabled = os.getenv(env_var, "false").lower() == "true"
|
||||
self.session_id = str(uuid.uuid4()) if self.enabled else ""
|
||||
self.log_dir = get_hermes_home() / "logs"
|
||||
self._calls: list[Dict[str, Any]] = []
|
||||
self._start_time = datetime.datetime.now().isoformat() if self.enabled else ""
|
||||
|
||||
if self.enabled:
|
||||
self.log_dir.mkdir(parents=True, exist_ok=True)
|
||||
logger.debug("%s debug mode enabled - Session ID: %s",
|
||||
tool_name, self.session_id)
|
||||
|
||||
@property
|
||||
def active(self) -> bool:
|
||||
return self.enabled
|
||||
|
||||
def log_call(self, call_name: str, call_data: Dict[str, Any]) -> None:
|
||||
"""Append a tool-call entry to the in-memory log."""
|
||||
if not self.enabled:
|
||||
return
|
||||
self._calls.append({
|
||||
"timestamp": datetime.datetime.now().isoformat(),
|
||||
"tool_name": call_name,
|
||||
**call_data,
|
||||
})
|
||||
|
||||
def save(self) -> None:
|
||||
"""Flush the in-memory log to a JSON file in the logs directory."""
|
||||
if not self.enabled:
|
||||
return
|
||||
try:
|
||||
filename = f"{self.tool_name}_debug_{self.session_id}.json"
|
||||
filepath = self.log_dir / filename
|
||||
payload = {
|
||||
"session_id": self.session_id,
|
||||
"start_time": self._start_time,
|
||||
"end_time": datetime.datetime.now().isoformat(),
|
||||
"debug_enabled": True,
|
||||
"total_calls": len(self._calls),
|
||||
"tool_calls": self._calls,
|
||||
}
|
||||
with open(filepath, "w", encoding="utf-8") as f:
|
||||
json.dump(payload, f, indent=2, ensure_ascii=False)
|
||||
logger.debug("%s debug log saved: %s", self.tool_name, filepath)
|
||||
except Exception as e:
|
||||
logger.error("Error saving %s debug log: %s", self.tool_name, e)
|
||||
|
||||
def get_session_info(self) -> Dict[str, Any]:
|
||||
"""Return a summary dict suitable for returning from get_debug_session_info()."""
|
||||
if not self.enabled:
|
||||
return {
|
||||
"enabled": False,
|
||||
"session_id": None,
|
||||
"log_path": None,
|
||||
"total_calls": 0,
|
||||
}
|
||||
return {
|
||||
"enabled": True,
|
||||
"session_id": self.session_id,
|
||||
"log_path": str(self.log_dir / f"{self.tool_name}_debug_{self.session_id}.json"),
|
||||
"total_calls": len(self._calls),
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,101 @@
|
||||
"""Environment variable passthrough registry.
|
||||
|
||||
Skills that declare ``required_environment_variables`` in their frontmatter
|
||||
need those vars available in sandboxed execution environments (execute_code,
|
||||
terminal). By default both sandboxes strip secrets from the child process
|
||||
environment for security. This module provides a session-scoped allowlist
|
||||
so skill-declared vars (and user-configured overrides) pass through.
|
||||
|
||||
Two sources feed the allowlist:
|
||||
|
||||
1. **Skill declarations** — when a skill is loaded via ``skill_view``, its
|
||||
``required_environment_variables`` are registered here automatically.
|
||||
2. **User config** — ``terminal.env_passthrough`` in config.yaml lets users
|
||||
explicitly allowlist vars for non-skill use cases.
|
||||
|
||||
Both ``code_execution_tool.py`` and ``tools/environments/local.py`` consult
|
||||
:func:`is_env_passthrough` before stripping a variable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from contextvars import ContextVar
|
||||
from typing import Iterable
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Session-scoped set of env var names that should pass through to sandboxes.
|
||||
# Backed by ContextVar to prevent cross-session data bleed in the gateway pipeline.
|
||||
_allowed_env_vars_var: ContextVar[set[str]] = ContextVar("_allowed_env_vars")
|
||||
|
||||
|
||||
def _get_allowed() -> set[str]:
|
||||
"""Get or create the allowed env vars set for the current context/session."""
|
||||
try:
|
||||
return _allowed_env_vars_var.get()
|
||||
except LookupError:
|
||||
val: set[str] = set()
|
||||
_allowed_env_vars_var.set(val)
|
||||
return val
|
||||
|
||||
|
||||
# Cache for the config-based allowlist (loaded once per process).
|
||||
_config_passthrough: frozenset[str] | None = None
|
||||
|
||||
|
||||
def register_env_passthrough(var_names: Iterable[str]) -> None:
|
||||
"""Register environment variable names as allowed in sandboxed environments.
|
||||
|
||||
Typically called when a skill declares ``required_environment_variables``.
|
||||
"""
|
||||
for name in var_names:
|
||||
name = name.strip()
|
||||
if name:
|
||||
_get_allowed().add(name)
|
||||
logger.debug("env passthrough: registered %s", name)
|
||||
|
||||
|
||||
def _load_config_passthrough() -> frozenset[str]:
|
||||
"""Load ``tools.env_passthrough`` from config.yaml (cached)."""
|
||||
global _config_passthrough
|
||||
if _config_passthrough is not None:
|
||||
return _config_passthrough
|
||||
|
||||
result: set[str] = set()
|
||||
try:
|
||||
from hermes_cli.config import read_raw_config
|
||||
cfg = read_raw_config()
|
||||
passthrough = cfg.get("terminal", {}).get("env_passthrough")
|
||||
if isinstance(passthrough, list):
|
||||
for item in passthrough:
|
||||
if isinstance(item, str) and item.strip():
|
||||
result.add(item.strip())
|
||||
except Exception as e:
|
||||
logger.debug("Could not read tools.env_passthrough from config: %s", e)
|
||||
|
||||
_config_passthrough = frozenset(result)
|
||||
return _config_passthrough
|
||||
|
||||
|
||||
def is_env_passthrough(var_name: str) -> bool:
|
||||
"""Check whether *var_name* is allowed to pass through to sandboxes.
|
||||
|
||||
Returns ``True`` if the variable was registered by a skill or listed in
|
||||
the user's ``tools.env_passthrough`` config.
|
||||
"""
|
||||
if var_name in _get_allowed():
|
||||
return True
|
||||
return var_name in _load_config_passthrough()
|
||||
|
||||
|
||||
def get_all_passthrough() -> frozenset[str]:
|
||||
"""Return the union of skill-registered and config-based passthrough vars."""
|
||||
return frozenset(_get_allowed()) | _load_config_passthrough()
|
||||
|
||||
|
||||
def clear_env_passthrough() -> None:
|
||||
"""Reset the skill-scoped allowlist (e.g. on session reset)."""
|
||||
_get_allowed().clear()
|
||||
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
"""Hermes execution environment backends.
|
||||
|
||||
Each backend provides the same interface (BaseEnvironment ABC) for running
|
||||
shell commands in a specific execution context: local, Docker, Singularity,
|
||||
SSH, Modal, or Daytona.
|
||||
|
||||
The terminal_tool.py factory (_create_environment) selects the backend
|
||||
based on the TERMINAL_ENV configuration.
|
||||
"""
|
||||
|
||||
from tools.environments.base import BaseEnvironment
|
||||
|
||||
__all__ = ["BaseEnvironment"]
|
||||
@@ -0,0 +1,579 @@
|
||||
"""Base class for all Hermes execution environment backends.
|
||||
|
||||
Unified spawn-per-call model: every command spawns a fresh ``bash -c`` process.
|
||||
A session snapshot (env vars, functions, aliases) is captured once at init and
|
||||
re-sourced before each command. CWD persists via in-band stdout markers (remote)
|
||||
or a temp file (local).
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from abc import ABC, abstractmethod
|
||||
from pathlib import Path
|
||||
from typing import IO, Callable, Protocol
|
||||
|
||||
from hermes_constants import get_hermes_home
|
||||
from tools.interrupt import is_interrupted
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Thread-local activity callback. The agent sets this before a tool call so
|
||||
# long-running _wait_for_process loops can report liveness to the gateway.
|
||||
_activity_callback_local = threading.local()
|
||||
|
||||
|
||||
def set_activity_callback(cb: Callable[[str], None] | None) -> None:
|
||||
"""Register a callback that _wait_for_process fires periodically."""
|
||||
_activity_callback_local.callback = cb
|
||||
|
||||
|
||||
def _get_activity_callback() -> Callable[[str], None] | None:
|
||||
return getattr(_activity_callback_local, "callback", None)
|
||||
|
||||
|
||||
def get_sandbox_dir() -> Path:
|
||||
"""Return the host-side root for all sandbox storage (Docker workspaces,
|
||||
Singularity overlays/SIF cache, etc.).
|
||||
|
||||
Configurable via TERMINAL_SANDBOX_DIR. Defaults to {HERMES_HOME}/sandboxes/.
|
||||
"""
|
||||
custom = os.getenv("TERMINAL_SANDBOX_DIR")
|
||||
if custom:
|
||||
p = Path(custom)
|
||||
else:
|
||||
p = get_hermes_home() / "sandboxes"
|
||||
p.mkdir(parents=True, exist_ok=True)
|
||||
return p
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shared constants and utilities
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _pipe_stdin(proc: subprocess.Popen, data: str) -> None:
|
||||
"""Write *data* to proc.stdin on a daemon thread to avoid pipe-buffer deadlocks."""
|
||||
|
||||
def _write():
|
||||
try:
|
||||
proc.stdin.write(data)
|
||||
proc.stdin.close()
|
||||
except (BrokenPipeError, OSError):
|
||||
pass
|
||||
|
||||
threading.Thread(target=_write, daemon=True).start()
|
||||
|
||||
|
||||
def _popen_bash(
|
||||
cmd: list[str], stdin_data: str | None = None, **kwargs
|
||||
) -> subprocess.Popen:
|
||||
"""Spawn a subprocess with standard stdout/stderr/stdin setup.
|
||||
|
||||
If *stdin_data* is provided, writes it asynchronously via :func:`_pipe_stdin`.
|
||||
Backends with special Popen needs (e.g. local's ``preexec_fn``) can bypass
|
||||
this and call :func:`_pipe_stdin` directly.
|
||||
"""
|
||||
proc = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
stdin=subprocess.PIPE if stdin_data is not None else subprocess.DEVNULL,
|
||||
text=True,
|
||||
**kwargs,
|
||||
)
|
||||
if stdin_data is not None:
|
||||
_pipe_stdin(proc, stdin_data)
|
||||
return proc
|
||||
|
||||
|
||||
def _load_json_store(path: Path) -> dict:
|
||||
"""Load a JSON file as a dict, returning ``{}`` on any error."""
|
||||
if path.exists():
|
||||
try:
|
||||
return json.loads(path.read_text())
|
||||
except Exception:
|
||||
pass
|
||||
return {}
|
||||
|
||||
|
||||
def _save_json_store(path: Path, data: dict) -> None:
|
||||
"""Write *data* as pretty-printed JSON to *path*."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, indent=2))
|
||||
|
||||
|
||||
def _file_mtime_key(host_path: str) -> tuple[float, int] | None:
|
||||
"""Return ``(mtime, size)`` for cache comparison, or ``None`` if unreadable."""
|
||||
try:
|
||||
st = Path(host_path).stat()
|
||||
return (st.st_mtime, st.st_size)
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ProcessHandle protocol
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ProcessHandle(Protocol):
|
||||
"""Duck type that every backend's _run_bash() must return.
|
||||
|
||||
subprocess.Popen satisfies this natively. SDK backends (Modal, Daytona)
|
||||
return _ThreadedProcessHandle which adapts their blocking calls.
|
||||
"""
|
||||
|
||||
def poll(self) -> int | None: ...
|
||||
def kill(self) -> None: ...
|
||||
def wait(self, timeout: float | None = None) -> int: ...
|
||||
|
||||
@property
|
||||
def stdout(self) -> IO[str] | None: ...
|
||||
|
||||
@property
|
||||
def returncode(self) -> int | None: ...
|
||||
|
||||
|
||||
class _ThreadedProcessHandle:
|
||||
"""Adapter for SDK backends (Modal, Daytona) that have no real subprocess.
|
||||
|
||||
Wraps a blocking ``exec_fn() -> (output_str, exit_code)`` in a background
|
||||
thread and exposes a ProcessHandle-compatible interface. An optional
|
||||
``cancel_fn`` is invoked on ``kill()`` for backend-specific cancellation
|
||||
(e.g. Modal sandbox.terminate, Daytona sandbox.stop).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
exec_fn: Callable[[], tuple[str, int]],
|
||||
cancel_fn: Callable[[], None] | None = None,
|
||||
):
|
||||
self._cancel_fn = cancel_fn
|
||||
self._done = threading.Event()
|
||||
self._returncode: int | None = None
|
||||
self._error: Exception | None = None
|
||||
|
||||
# Pipe for stdout — drain thread in _wait_for_process reads the read end.
|
||||
read_fd, write_fd = os.pipe()
|
||||
self._stdout = os.fdopen(read_fd, "r", encoding="utf-8", errors="replace")
|
||||
self._write_fd = write_fd
|
||||
|
||||
def _worker():
|
||||
try:
|
||||
output, exit_code = exec_fn()
|
||||
self._returncode = exit_code
|
||||
# Write output into the pipe so drain thread picks it up.
|
||||
try:
|
||||
os.write(self._write_fd, output.encode("utf-8", errors="replace"))
|
||||
except OSError:
|
||||
pass
|
||||
except Exception as exc:
|
||||
self._error = exc
|
||||
self._returncode = 1
|
||||
finally:
|
||||
try:
|
||||
os.close(self._write_fd)
|
||||
except OSError:
|
||||
pass
|
||||
self._done.set()
|
||||
|
||||
t = threading.Thread(target=_worker, daemon=True)
|
||||
t.start()
|
||||
|
||||
@property
|
||||
def stdout(self):
|
||||
return self._stdout
|
||||
|
||||
@property
|
||||
def returncode(self) -> int | None:
|
||||
return self._returncode
|
||||
|
||||
def poll(self) -> int | None:
|
||||
return self._returncode if self._done.is_set() else None
|
||||
|
||||
def kill(self):
|
||||
if self._cancel_fn:
|
||||
try:
|
||||
self._cancel_fn()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def wait(self, timeout: float | None = None) -> int:
|
||||
self._done.wait(timeout=timeout)
|
||||
return self._returncode
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CWD marker for remote backends
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _cwd_marker(session_id: str) -> str:
|
||||
return f"__HERMES_CWD_{session_id}__"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# BaseEnvironment
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class BaseEnvironment(ABC):
|
||||
"""Common interface and unified execution flow for all Hermes backends.
|
||||
|
||||
Subclasses implement ``_run_bash()`` and ``cleanup()``. The base class
|
||||
provides ``execute()`` with session snapshot sourcing, CWD tracking,
|
||||
interrupt handling, and timeout enforcement.
|
||||
"""
|
||||
|
||||
# Subclasses that embed stdin as a heredoc (Modal, Daytona) set this.
|
||||
_stdin_mode: str = "pipe" # "pipe" or "heredoc"
|
||||
|
||||
# Snapshot creation timeout (override for slow cold-starts).
|
||||
_snapshot_timeout: int = 30
|
||||
|
||||
def get_temp_dir(self) -> str:
|
||||
"""Return the backend temp directory used for session artifacts.
|
||||
|
||||
Most sandboxed backends use ``/tmp`` inside the target environment.
|
||||
LocalEnvironment overrides this on platforms like Termux where ``/tmp``
|
||||
may be missing and ``TMPDIR`` is the portable writable location.
|
||||
"""
|
||||
return "/tmp"
|
||||
|
||||
def __init__(self, cwd: str, timeout: int, env: dict = None):
|
||||
self.cwd = cwd
|
||||
self.timeout = timeout
|
||||
self.env = env or {}
|
||||
|
||||
self._session_id = uuid.uuid4().hex[:12]
|
||||
temp_dir = self.get_temp_dir().rstrip("/") or "/"
|
||||
self._snapshot_path = f"{temp_dir}/hermes-snap-{self._session_id}.sh"
|
||||
self._cwd_file = f"{temp_dir}/hermes-cwd-{self._session_id}.txt"
|
||||
self._cwd_marker = _cwd_marker(self._session_id)
|
||||
self._snapshot_ready = False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Abstract methods
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _run_bash(
|
||||
self,
|
||||
cmd_string: str,
|
||||
*,
|
||||
login: bool = False,
|
||||
timeout: int = 120,
|
||||
stdin_data: str | None = None,
|
||||
) -> ProcessHandle:
|
||||
"""Spawn a bash process to run *cmd_string*.
|
||||
|
||||
Returns a ProcessHandle (subprocess.Popen or _ThreadedProcessHandle).
|
||||
Must be overridden by every backend.
|
||||
"""
|
||||
raise NotImplementedError(f"{type(self).__name__} must implement _run_bash()")
|
||||
|
||||
@abstractmethod
|
||||
def cleanup(self):
|
||||
"""Release backend resources (container, instance, connection)."""
|
||||
...
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Session snapshot (init_session)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def init_session(self):
|
||||
"""Capture login shell environment into a snapshot file.
|
||||
|
||||
Called once after backend construction. On success, sets
|
||||
``_snapshot_ready = True`` so subsequent commands source the snapshot
|
||||
instead of running with ``bash -l``.
|
||||
"""
|
||||
# Full capture: env vars, functions (filtered), aliases, shell options.
|
||||
bootstrap = (
|
||||
f"export -p > {self._snapshot_path}\n"
|
||||
f"declare -f | grep -vE '^_[^_]' >> {self._snapshot_path}\n"
|
||||
f"alias -p >> {self._snapshot_path}\n"
|
||||
f"echo 'shopt -s expand_aliases' >> {self._snapshot_path}\n"
|
||||
f"echo 'set +e' >> {self._snapshot_path}\n"
|
||||
f"echo 'set +u' >> {self._snapshot_path}\n"
|
||||
f"pwd -P > {self._cwd_file} 2>/dev/null || true\n"
|
||||
f"printf '\\n{self._cwd_marker}%s{self._cwd_marker}\\n' \"$(pwd -P)\"\n"
|
||||
)
|
||||
try:
|
||||
proc = self._run_bash(bootstrap, login=True, timeout=self._snapshot_timeout)
|
||||
result = self._wait_for_process(proc, timeout=self._snapshot_timeout)
|
||||
self._snapshot_ready = True
|
||||
self._update_cwd(result)
|
||||
logger.info(
|
||||
"Session snapshot created (session=%s, cwd=%s)",
|
||||
self._session_id,
|
||||
self.cwd,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"init_session failed (session=%s): %s — "
|
||||
"falling back to bash -l per command",
|
||||
self._session_id,
|
||||
exc,
|
||||
)
|
||||
self._snapshot_ready = False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Command wrapping
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _wrap_command(self, command: str, cwd: str) -> str:
|
||||
"""Build the full bash script that sources snapshot, cd's, runs command,
|
||||
re-dumps env vars, and emits CWD markers."""
|
||||
escaped = command.replace("'", "'\\''")
|
||||
|
||||
parts = []
|
||||
|
||||
# Source snapshot (env vars from previous commands)
|
||||
if self._snapshot_ready:
|
||||
parts.append(f"source {self._snapshot_path} 2>/dev/null || true")
|
||||
|
||||
# cd to working directory — let bash expand ~ natively
|
||||
quoted_cwd = (
|
||||
shlex.quote(cwd) if cwd != "~" and not cwd.startswith("~/") else cwd
|
||||
)
|
||||
parts.append(f"cd {quoted_cwd} || exit 126")
|
||||
|
||||
# Run the actual command
|
||||
parts.append(f"eval '{escaped}'")
|
||||
parts.append("__hermes_ec=$?")
|
||||
|
||||
# Re-dump env vars to snapshot (last-writer-wins for concurrent calls)
|
||||
if self._snapshot_ready:
|
||||
parts.append(f"export -p > {self._snapshot_path} 2>/dev/null || true")
|
||||
|
||||
# Write CWD to file (local reads this) and stdout marker (remote parses this)
|
||||
parts.append(f"pwd -P > {self._cwd_file} 2>/dev/null || true")
|
||||
# Use a distinct line for the marker. The leading \n ensures
|
||||
# the marker starts on its own line even if the command doesn't
|
||||
# end with a newline (e.g. printf 'exact'). We'll strip this
|
||||
# injected newline in _extract_cwd_from_output.
|
||||
parts.append(
|
||||
f"printf '\\n{self._cwd_marker}%s{self._cwd_marker}\\n' \"$(pwd -P)\""
|
||||
)
|
||||
parts.append("exit $__hermes_ec")
|
||||
|
||||
return "\n".join(parts)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Stdin heredoc embedding (for SDK backends)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _embed_stdin_heredoc(command: str, stdin_data: str) -> str:
|
||||
"""Append stdin_data as a shell heredoc to the command string."""
|
||||
delimiter = f"HERMES_STDIN_{uuid.uuid4().hex[:12]}"
|
||||
return f"{command} << '{delimiter}'\n{stdin_data}\n{delimiter}"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Process lifecycle
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _wait_for_process(self, proc: ProcessHandle, timeout: int = 120) -> dict:
|
||||
"""Poll-based wait with interrupt checking and stdout draining.
|
||||
|
||||
Shared across all backends — not overridden.
|
||||
|
||||
Fires the ``activity_callback`` (if set on this instance) every 10s
|
||||
while the process is running so the gateway's inactivity timeout
|
||||
doesn't kill long-running commands.
|
||||
"""
|
||||
output_chunks: list[str] = []
|
||||
|
||||
def _drain():
|
||||
try:
|
||||
for line in proc.stdout:
|
||||
output_chunks.append(line)
|
||||
except UnicodeDecodeError:
|
||||
output_chunks.clear()
|
||||
output_chunks.append(
|
||||
"[binary output detected — raw bytes not displayable]"
|
||||
)
|
||||
except (ValueError, OSError):
|
||||
pass
|
||||
|
||||
drain_thread = threading.Thread(target=_drain, daemon=True)
|
||||
drain_thread.start()
|
||||
deadline = time.monotonic() + timeout
|
||||
_last_activity_touch = time.monotonic()
|
||||
_ACTIVITY_INTERVAL = 10.0 # seconds between activity touches
|
||||
|
||||
while proc.poll() is None:
|
||||
if is_interrupted():
|
||||
self._kill_process(proc)
|
||||
drain_thread.join(timeout=2)
|
||||
return {
|
||||
"output": "".join(output_chunks) + "\n[Command interrupted]",
|
||||
"returncode": 130,
|
||||
}
|
||||
if time.monotonic() > deadline:
|
||||
self._kill_process(proc)
|
||||
drain_thread.join(timeout=2)
|
||||
partial = "".join(output_chunks)
|
||||
timeout_msg = f"\n[Command timed out after {timeout}s]"
|
||||
return {
|
||||
"output": partial + timeout_msg
|
||||
if partial
|
||||
else timeout_msg.lstrip(),
|
||||
"returncode": 124,
|
||||
}
|
||||
# Periodic activity touch so the gateway knows we're alive
|
||||
_now = time.monotonic()
|
||||
if _now - _last_activity_touch >= _ACTIVITY_INTERVAL:
|
||||
_last_activity_touch = _now
|
||||
_cb = _get_activity_callback()
|
||||
if _cb:
|
||||
try:
|
||||
_elapsed = int(_now - (deadline - timeout))
|
||||
_cb(f"terminal command running ({_elapsed}s elapsed)")
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(0.2)
|
||||
|
||||
drain_thread.join(timeout=5)
|
||||
|
||||
try:
|
||||
proc.stdout.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return {"output": "".join(output_chunks), "returncode": proc.returncode}
|
||||
|
||||
def _kill_process(self, proc: ProcessHandle):
|
||||
"""Terminate a process. Subclasses may override for process-group kill."""
|
||||
try:
|
||||
proc.kill()
|
||||
except (ProcessLookupError, PermissionError, OSError):
|
||||
pass
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# CWD extraction
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _update_cwd(self, result: dict):
|
||||
"""Extract CWD from command output. Override for local file-based read."""
|
||||
self._extract_cwd_from_output(result)
|
||||
|
||||
def _extract_cwd_from_output(self, result: dict):
|
||||
"""Parse the __HERMES_CWD_{session}__ marker from stdout output.
|
||||
|
||||
Updates self.cwd and strips the marker from result["output"].
|
||||
Used by remote backends (Docker, SSH, Modal, Daytona, Singularity).
|
||||
"""
|
||||
output = result.get("output", "")
|
||||
marker = self._cwd_marker
|
||||
last = output.rfind(marker)
|
||||
if last == -1:
|
||||
return
|
||||
|
||||
# Find the opening marker before this closing one
|
||||
search_start = max(0, last - 4096) # CWD path won't be >4KB
|
||||
first = output.rfind(marker, search_start, last)
|
||||
if first == -1 or first == last:
|
||||
return
|
||||
|
||||
cwd_path = output[first + len(marker) : last].strip()
|
||||
if cwd_path:
|
||||
self.cwd = cwd_path
|
||||
|
||||
# Strip the marker line AND the \n we injected before it.
|
||||
# The wrapper emits: printf '\n__MARKER__%s__MARKER__\n'
|
||||
# So the output looks like: <cmd output>\n__MARKER__path__MARKER__\n
|
||||
# We want to remove everything from the injected \n onwards.
|
||||
line_start = output.rfind("\n", 0, first)
|
||||
if line_start == -1:
|
||||
line_start = first
|
||||
line_end = output.find("\n", last + len(marker))
|
||||
line_end = line_end + 1 if line_end != -1 else len(output)
|
||||
|
||||
result["output"] = output[:line_start] + output[line_end:]
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Hooks
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _before_execute(self) -> None:
|
||||
"""Hook called before each command execution.
|
||||
|
||||
Remote backends (SSH, Modal, Daytona) override this to trigger
|
||||
their FileSyncManager. Bind-mount backends (Docker, Singularity)
|
||||
and Local don't need file sync — the host filesystem is directly
|
||||
visible inside the container/process.
|
||||
"""
|
||||
pass
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Unified execute()
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def execute(
|
||||
self,
|
||||
command: str,
|
||||
cwd: str = "",
|
||||
*,
|
||||
timeout: int | None = None,
|
||||
stdin_data: str | None = None,
|
||||
) -> dict:
|
||||
"""Execute a command, return {"output": str, "returncode": int}."""
|
||||
self._before_execute()
|
||||
|
||||
exec_command, sudo_stdin = self._prepare_command(command)
|
||||
effective_timeout = timeout or self.timeout
|
||||
effective_cwd = cwd or self.cwd
|
||||
|
||||
# Merge sudo stdin with caller stdin
|
||||
if sudo_stdin is not None and stdin_data is not None:
|
||||
effective_stdin = sudo_stdin + stdin_data
|
||||
elif sudo_stdin is not None:
|
||||
effective_stdin = sudo_stdin
|
||||
else:
|
||||
effective_stdin = stdin_data
|
||||
|
||||
# Embed stdin as heredoc for backends that need it
|
||||
if effective_stdin and self._stdin_mode == "heredoc":
|
||||
exec_command = self._embed_stdin_heredoc(exec_command, effective_stdin)
|
||||
effective_stdin = None
|
||||
|
||||
wrapped = self._wrap_command(exec_command, effective_cwd)
|
||||
|
||||
# Use login shell if snapshot failed (so user's profile still loads)
|
||||
login = not self._snapshot_ready
|
||||
|
||||
proc = self._run_bash(
|
||||
wrapped, login=login, timeout=effective_timeout, stdin_data=effective_stdin
|
||||
)
|
||||
result = self._wait_for_process(proc, timeout=effective_timeout)
|
||||
self._update_cwd(result)
|
||||
|
||||
return result
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Shared helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def stop(self):
|
||||
"""Alias for cleanup (compat with older callers)."""
|
||||
self.cleanup()
|
||||
|
||||
def __del__(self):
|
||||
try:
|
||||
self.cleanup()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _prepare_command(self, command: str) -> tuple[str, str | None]:
|
||||
"""Transform sudo commands if SUDO_PASSWORD is available."""
|
||||
from tools.terminal_tool import _transform_sudo_command
|
||||
|
||||
return _transform_sudo_command(command)
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
"""Daytona cloud execution environment.
|
||||
|
||||
Uses the Daytona Python SDK to run commands in cloud sandboxes.
|
||||
Supports persistent sandboxes: when enabled, sandboxes are stopped on cleanup
|
||||
and resumed on next creation, preserving the filesystem across sessions.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import math
|
||||
import shlex
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
from tools.environments.base import (
|
||||
BaseEnvironment,
|
||||
_ThreadedProcessHandle,
|
||||
)
|
||||
from tools.environments.file_sync import (
|
||||
FileSyncManager,
|
||||
iter_sync_files,
|
||||
quoted_mkdir_command,
|
||||
quoted_rm_command,
|
||||
unique_parent_dirs,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DaytonaEnvironment(BaseEnvironment):
|
||||
"""Daytona cloud sandbox execution backend.
|
||||
|
||||
Spawn-per-call via _ThreadedProcessHandle wrapping blocking SDK calls.
|
||||
cancel_fn wired to sandbox.stop() for interrupt support.
|
||||
Shell timeout wrapper preserved (SDK timeout unreliable).
|
||||
"""
|
||||
|
||||
_stdin_mode = "heredoc"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image: str,
|
||||
cwd: str = "/home/daytona",
|
||||
timeout: int = 60,
|
||||
cpu: int = 1,
|
||||
memory: int = 5120,
|
||||
disk: int = 10240,
|
||||
persistent_filesystem: bool = True,
|
||||
task_id: str = "default",
|
||||
):
|
||||
requested_cwd = cwd
|
||||
super().__init__(cwd=cwd, timeout=timeout)
|
||||
|
||||
from daytona import (
|
||||
Daytona,
|
||||
CreateSandboxFromImageParams,
|
||||
DaytonaError,
|
||||
Resources,
|
||||
SandboxState,
|
||||
)
|
||||
|
||||
self._persistent = persistent_filesystem
|
||||
self._task_id = task_id
|
||||
self._SandboxState = SandboxState
|
||||
self._daytona = Daytona()
|
||||
self._sandbox = None
|
||||
self._lock = threading.Lock()
|
||||
|
||||
memory_gib = max(1, math.ceil(memory / 1024))
|
||||
disk_gib = max(1, math.ceil(disk / 1024))
|
||||
if disk_gib > 10:
|
||||
logger.warning(
|
||||
"Daytona: requested disk (%dGB) exceeds platform limit (10GB). "
|
||||
"Capping to 10GB.", disk_gib,
|
||||
)
|
||||
disk_gib = 10
|
||||
resources = Resources(cpu=cpu, memory=memory_gib, disk=disk_gib)
|
||||
|
||||
labels = {"hermes_task_id": task_id}
|
||||
sandbox_name = f"hermes-{task_id}"
|
||||
|
||||
if self._persistent:
|
||||
try:
|
||||
self._sandbox = self._daytona.get(sandbox_name)
|
||||
self._sandbox.start()
|
||||
logger.info("Daytona: resumed sandbox %s for task %s",
|
||||
self._sandbox.id, task_id)
|
||||
except DaytonaError:
|
||||
self._sandbox = None
|
||||
except Exception as e:
|
||||
logger.warning("Daytona: failed to resume sandbox for task %s: %s",
|
||||
task_id, e)
|
||||
self._sandbox = None
|
||||
|
||||
if self._sandbox is None:
|
||||
try:
|
||||
page = self._daytona.list(labels=labels, page=1, limit=1)
|
||||
if page.items:
|
||||
self._sandbox = page.items[0]
|
||||
self._sandbox.start()
|
||||
logger.info("Daytona: resumed legacy sandbox %s for task %s",
|
||||
self._sandbox.id, task_id)
|
||||
except Exception as e:
|
||||
logger.debug("Daytona: no legacy sandbox found for task %s: %s",
|
||||
task_id, e)
|
||||
self._sandbox = None
|
||||
|
||||
if self._sandbox is None:
|
||||
self._sandbox = self._daytona.create(
|
||||
CreateSandboxFromImageParams(
|
||||
image=image,
|
||||
name=sandbox_name,
|
||||
labels=labels,
|
||||
auto_stop_interval=0,
|
||||
resources=resources,
|
||||
)
|
||||
)
|
||||
logger.info("Daytona: created sandbox %s for task %s",
|
||||
self._sandbox.id, task_id)
|
||||
|
||||
# Detect remote home dir
|
||||
self._remote_home = "/root"
|
||||
try:
|
||||
home = self._sandbox.process.exec("echo $HOME").result.strip()
|
||||
if home:
|
||||
self._remote_home = home
|
||||
if requested_cwd in ("~", "/home/daytona"):
|
||||
self.cwd = home
|
||||
except Exception:
|
||||
pass
|
||||
logger.info("Daytona: resolved home to %s, cwd to %s", self._remote_home, self.cwd)
|
||||
|
||||
self._sync_manager = FileSyncManager(
|
||||
get_files_fn=lambda: iter_sync_files(f"{self._remote_home}/.hermes"),
|
||||
upload_fn=self._daytona_upload,
|
||||
delete_fn=self._daytona_delete,
|
||||
bulk_upload_fn=self._daytona_bulk_upload,
|
||||
)
|
||||
self._sync_manager.sync(force=True)
|
||||
self.init_session()
|
||||
|
||||
def _daytona_upload(self, host_path: str, remote_path: str) -> None:
|
||||
"""Upload a single file via Daytona SDK."""
|
||||
parent = str(Path(remote_path).parent)
|
||||
self._sandbox.process.exec(f"mkdir -p {parent}")
|
||||
self._sandbox.fs.upload_file(host_path, remote_path)
|
||||
|
||||
def _daytona_bulk_upload(self, files: list[tuple[str, str]]) -> None:
|
||||
"""Upload many files in a single HTTP call via Daytona SDK.
|
||||
|
||||
Uses ``sandbox.fs.upload_files()`` which batches all files into one
|
||||
multipart POST, avoiding per-file TLS/HTTP overhead (~580 files
|
||||
goes from ~5 min to <2 s).
|
||||
"""
|
||||
from daytona.common.filesystem import FileUpload
|
||||
|
||||
if not files:
|
||||
return
|
||||
|
||||
parents = unique_parent_dirs(files)
|
||||
if parents:
|
||||
self._sandbox.process.exec(quoted_mkdir_command(parents))
|
||||
|
||||
uploads = [
|
||||
FileUpload(source=host_path, destination=remote_path)
|
||||
for host_path, remote_path in files
|
||||
]
|
||||
self._sandbox.fs.upload_files(uploads)
|
||||
|
||||
def _daytona_delete(self, remote_paths: list[str]) -> None:
|
||||
"""Batch-delete remote files via SDK exec."""
|
||||
self._sandbox.process.exec(quoted_rm_command(remote_paths))
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Sandbox lifecycle
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _ensure_sandbox_ready(self) -> None:
|
||||
"""Restart sandbox if it was stopped (e.g., by a previous interrupt)."""
|
||||
self._sandbox.refresh_data()
|
||||
if self._sandbox.state in (self._SandboxState.STOPPED, self._SandboxState.ARCHIVED):
|
||||
self._sandbox.start()
|
||||
logger.info("Daytona: restarted sandbox %s", self._sandbox.id)
|
||||
|
||||
def _before_execute(self) -> None:
|
||||
"""Ensure sandbox is ready, then sync files via FileSyncManager."""
|
||||
with self._lock:
|
||||
self._ensure_sandbox_ready()
|
||||
self._sync_manager.sync()
|
||||
|
||||
def _run_bash(self, cmd_string: str, *, login: bool = False,
|
||||
timeout: int = 120,
|
||||
stdin_data: str | None = None):
|
||||
"""Return a _ThreadedProcessHandle wrapping a blocking Daytona SDK call."""
|
||||
sandbox = self._sandbox
|
||||
lock = self._lock
|
||||
|
||||
def cancel():
|
||||
with lock:
|
||||
try:
|
||||
sandbox.stop()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if login:
|
||||
shell_cmd = f"bash -l -c {shlex.quote(cmd_string)}"
|
||||
else:
|
||||
shell_cmd = f"bash -c {shlex.quote(cmd_string)}"
|
||||
|
||||
def exec_fn() -> tuple[str, int]:
|
||||
response = sandbox.process.exec(shell_cmd, timeout=timeout)
|
||||
return (response.result or "", response.exit_code)
|
||||
|
||||
return _ThreadedProcessHandle(exec_fn, cancel_fn=cancel)
|
||||
|
||||
def cleanup(self):
|
||||
with self._lock:
|
||||
if self._sandbox is None:
|
||||
return
|
||||
try:
|
||||
if self._persistent:
|
||||
self._sandbox.stop()
|
||||
logger.info("Daytona: stopped sandbox %s (filesystem preserved)",
|
||||
self._sandbox.id)
|
||||
else:
|
||||
self._daytona.delete(self._sandbox)
|
||||
logger.info("Daytona: deleted sandbox %s", self._sandbox.id)
|
||||
except Exception as e:
|
||||
logger.warning("Daytona: cleanup failed: %s", e)
|
||||
self._sandbox = None
|
||||
@@ -0,0 +1,560 @@
|
||||
"""Docker execution environment for sandboxed command execution.
|
||||
|
||||
Security hardened (cap-drop ALL, no-new-privileges, PID limits),
|
||||
configurable resource limits (CPU, memory, disk), and optional filesystem
|
||||
persistence via bind mounts.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import uuid
|
||||
from typing import Optional
|
||||
|
||||
from tools.environments.base import BaseEnvironment, _popen_bash
|
||||
from tools.environments.local import _HERMES_PROVIDER_ENV_BLOCKLIST
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Common Docker Desktop install paths checked when 'docker' is not in PATH.
|
||||
# macOS Intel: /usr/local/bin, macOS Apple Silicon (Homebrew): /opt/homebrew/bin,
|
||||
# Docker Desktop app bundle: /Applications/Docker.app/Contents/Resources/bin
|
||||
_DOCKER_SEARCH_PATHS = [
|
||||
"/usr/local/bin/docker",
|
||||
"/opt/homebrew/bin/docker",
|
||||
"/Applications/Docker.app/Contents/Resources/bin/docker",
|
||||
]
|
||||
|
||||
_docker_executable: Optional[str] = None # resolved once, cached
|
||||
_ENV_VAR_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
||||
|
||||
|
||||
def _normalize_forward_env_names(forward_env: list[str] | None) -> list[str]:
|
||||
"""Return a deduplicated list of valid environment variable names."""
|
||||
normalized: list[str] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
for item in forward_env or []:
|
||||
if not isinstance(item, str):
|
||||
logger.warning("Ignoring non-string docker_forward_env entry: %r", item)
|
||||
continue
|
||||
|
||||
key = item.strip()
|
||||
if not key:
|
||||
continue
|
||||
if not _ENV_VAR_NAME_RE.match(key):
|
||||
logger.warning("Ignoring invalid docker_forward_env entry: %r", item)
|
||||
continue
|
||||
if key in seen:
|
||||
continue
|
||||
|
||||
seen.add(key)
|
||||
normalized.append(key)
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
def _normalize_env_dict(env: dict | None) -> dict[str, str]:
|
||||
"""Validate and normalize a docker_env dict to {str: str}.
|
||||
|
||||
Filters out entries with invalid variable names or non-string values.
|
||||
"""
|
||||
if not env:
|
||||
return {}
|
||||
if not isinstance(env, dict):
|
||||
logger.warning("docker_env is not a dict: %r", env)
|
||||
return {}
|
||||
|
||||
normalized: dict[str, str] = {}
|
||||
for key, value in env.items():
|
||||
if not isinstance(key, str) or not _ENV_VAR_NAME_RE.match(key.strip()):
|
||||
logger.warning("Ignoring invalid docker_env key: %r", key)
|
||||
continue
|
||||
key = key.strip()
|
||||
if not isinstance(value, str):
|
||||
# Coerce simple scalar types (int, bool, float) to string;
|
||||
# reject complex types.
|
||||
if isinstance(value, (int, float, bool)):
|
||||
value = str(value)
|
||||
else:
|
||||
logger.warning("Ignoring non-string docker_env value for %r: %r", key, value)
|
||||
continue
|
||||
normalized[key] = value
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
def _load_hermes_env_vars() -> dict[str, str]:
|
||||
"""Load ~/.hermes/.env values without failing Docker command execution."""
|
||||
try:
|
||||
from hermes_cli.config import load_env
|
||||
|
||||
return load_env() or {}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def find_docker() -> Optional[str]:
|
||||
"""Locate the docker CLI binary.
|
||||
|
||||
Checks ``shutil.which`` first (respects PATH), then probes well-known
|
||||
install locations on macOS where Docker Desktop may not be in PATH
|
||||
(e.g. when running as a gateway service via launchd).
|
||||
|
||||
Returns the absolute path, or ``None`` if docker cannot be found.
|
||||
"""
|
||||
global _docker_executable
|
||||
if _docker_executable is not None:
|
||||
return _docker_executable
|
||||
|
||||
found = shutil.which("docker")
|
||||
if found:
|
||||
_docker_executable = found
|
||||
return found
|
||||
|
||||
for path in _DOCKER_SEARCH_PATHS:
|
||||
if os.path.isfile(path) and os.access(path, os.X_OK):
|
||||
_docker_executable = path
|
||||
logger.info("Found docker at non-PATH location: %s", path)
|
||||
return path
|
||||
|
||||
return None
|
||||
|
||||
|
||||
# Security flags applied to every container.
|
||||
# The container itself is the security boundary (isolated from host).
|
||||
# We drop all capabilities then add back the minimum needed:
|
||||
# DAC_OVERRIDE - root can write to bind-mounted dirs owned by host user
|
||||
# CHOWN/FOWNER - package managers (pip, npm, apt) need to set file ownership
|
||||
# Block privilege escalation and limit PIDs.
|
||||
# /tmp is size-limited and nosuid but allows exec (needed by pip/npm builds).
|
||||
_SECURITY_ARGS = [
|
||||
"--cap-drop", "ALL",
|
||||
"--cap-add", "DAC_OVERRIDE",
|
||||
"--cap-add", "CHOWN",
|
||||
"--cap-add", "FOWNER",
|
||||
"--security-opt", "no-new-privileges",
|
||||
"--pids-limit", "256",
|
||||
"--tmpfs", "/tmp:rw,nosuid,size=512m",
|
||||
"--tmpfs", "/var/tmp:rw,noexec,nosuid,size=256m",
|
||||
"--tmpfs", "/run:rw,noexec,nosuid,size=64m",
|
||||
]
|
||||
|
||||
|
||||
_storage_opt_ok: Optional[bool] = None # cached result across instances
|
||||
|
||||
|
||||
def _ensure_docker_available() -> None:
|
||||
"""Best-effort check that the docker CLI is available before use.
|
||||
|
||||
Reuses ``find_docker()`` so this preflight stays consistent with the rest of
|
||||
the Docker backend, including known non-PATH Docker Desktop locations.
|
||||
"""
|
||||
docker_exe = find_docker()
|
||||
if not docker_exe:
|
||||
logger.error(
|
||||
"Docker backend selected but no docker executable was found in PATH "
|
||||
"or known install locations. Install Docker Desktop and ensure the "
|
||||
"CLI is available."
|
||||
)
|
||||
raise RuntimeError(
|
||||
"Docker executable not found in PATH or known install locations. "
|
||||
"Install Docker and ensure the 'docker' command is available."
|
||||
)
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[docker_exe, "version"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
logger.error(
|
||||
"Docker backend selected but the resolved docker executable '%s' could "
|
||||
"not be executed.",
|
||||
docker_exe,
|
||||
exc_info=True,
|
||||
)
|
||||
raise RuntimeError(
|
||||
"Docker executable could not be executed. Check your Docker installation."
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.error(
|
||||
"Docker backend selected but '%s version' timed out. "
|
||||
"The Docker daemon may not be running.",
|
||||
docker_exe,
|
||||
exc_info=True,
|
||||
)
|
||||
raise RuntimeError(
|
||||
"Docker daemon is not responding. Ensure Docker is running and try again."
|
||||
)
|
||||
except Exception:
|
||||
logger.error(
|
||||
"Unexpected error while checking Docker availability.",
|
||||
exc_info=True,
|
||||
)
|
||||
raise
|
||||
else:
|
||||
if result.returncode != 0:
|
||||
logger.error(
|
||||
"Docker backend selected but '%s version' failed "
|
||||
"(exit code %d, stderr=%s)",
|
||||
docker_exe,
|
||||
result.returncode,
|
||||
result.stderr.strip(),
|
||||
)
|
||||
raise RuntimeError(
|
||||
"Docker command is available but 'docker version' failed. "
|
||||
"Check your Docker installation."
|
||||
)
|
||||
|
||||
|
||||
class DockerEnvironment(BaseEnvironment):
|
||||
"""Hardened Docker container execution with resource limits and persistence.
|
||||
|
||||
Security: all capabilities dropped, no privilege escalation, PID limits,
|
||||
size-limited tmpfs for scratch dirs. The container itself is the security
|
||||
boundary — the filesystem inside is writable so agents can install packages
|
||||
(pip, npm, apt) as needed. Writable workspace via tmpfs or bind mounts.
|
||||
|
||||
Persistence: when enabled, bind mounts preserve /workspace and /root
|
||||
across container restarts.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image: str,
|
||||
cwd: str = "/root",
|
||||
timeout: int = 60,
|
||||
cpu: float = 0,
|
||||
memory: int = 0,
|
||||
disk: int = 0,
|
||||
persistent_filesystem: bool = False,
|
||||
task_id: str = "default",
|
||||
volumes: list = None,
|
||||
forward_env: list[str] | None = None,
|
||||
env: dict | None = None,
|
||||
network: bool = True,
|
||||
host_cwd: str = None,
|
||||
auto_mount_cwd: bool = False,
|
||||
):
|
||||
if cwd == "~":
|
||||
cwd = "/root"
|
||||
super().__init__(cwd=cwd, timeout=timeout)
|
||||
self._persistent = persistent_filesystem
|
||||
self._task_id = task_id
|
||||
self._forward_env = _normalize_forward_env_names(forward_env)
|
||||
self._env = _normalize_env_dict(env)
|
||||
self._container_id: Optional[str] = None
|
||||
logger.info(f"DockerEnvironment volumes: {volumes}")
|
||||
# Ensure volumes is a list (config.yaml could be malformed)
|
||||
if volumes is not None and not isinstance(volumes, list):
|
||||
logger.warning(f"docker_volumes config is not a list: {volumes!r}")
|
||||
volumes = []
|
||||
|
||||
# Fail fast if Docker is not available.
|
||||
_ensure_docker_available()
|
||||
|
||||
# Build resource limit args
|
||||
resource_args = []
|
||||
if cpu > 0:
|
||||
resource_args.extend(["--cpus", str(cpu)])
|
||||
if memory > 0:
|
||||
resource_args.extend(["--memory", f"{memory}m"])
|
||||
if disk > 0 and sys.platform != "darwin":
|
||||
if self._storage_opt_supported():
|
||||
resource_args.extend(["--storage-opt", f"size={disk}m"])
|
||||
else:
|
||||
logger.warning(
|
||||
"Docker storage driver does not support per-container disk limits "
|
||||
"(requires overlay2 on XFS with pquota). Container will run without disk quota."
|
||||
)
|
||||
if not network:
|
||||
resource_args.append("--network=none")
|
||||
|
||||
# Persistent workspace via bind mounts from a configurable host directory
|
||||
# (TERMINAL_SANDBOX_DIR, default ~/.hermes/sandboxes/). Non-persistent
|
||||
# mode uses tmpfs (ephemeral, fast, gone on cleanup).
|
||||
from tools.environments.base import get_sandbox_dir
|
||||
|
||||
# User-configured volume mounts (from config.yaml docker_volumes)
|
||||
volume_args = []
|
||||
workspace_explicitly_mounted = False
|
||||
for vol in (volumes or []):
|
||||
if not isinstance(vol, str):
|
||||
logger.warning(f"Docker volume entry is not a string: {vol!r}")
|
||||
continue
|
||||
vol = vol.strip()
|
||||
if not vol:
|
||||
continue
|
||||
if ":" in vol:
|
||||
volume_args.extend(["-v", vol])
|
||||
if ":/workspace" in vol:
|
||||
workspace_explicitly_mounted = True
|
||||
else:
|
||||
logger.warning(f"Docker volume '{vol}' missing colon, skipping")
|
||||
|
||||
host_cwd_abs = os.path.abspath(os.path.expanduser(host_cwd)) if host_cwd else ""
|
||||
bind_host_cwd = (
|
||||
auto_mount_cwd
|
||||
and bool(host_cwd_abs)
|
||||
and os.path.isdir(host_cwd_abs)
|
||||
and not workspace_explicitly_mounted
|
||||
)
|
||||
if auto_mount_cwd and host_cwd and not os.path.isdir(host_cwd_abs):
|
||||
logger.debug(f"Skipping docker cwd mount: host_cwd is not a valid directory: {host_cwd}")
|
||||
|
||||
self._workspace_dir: Optional[str] = None
|
||||
self._home_dir: Optional[str] = None
|
||||
writable_args = []
|
||||
if self._persistent:
|
||||
sandbox = get_sandbox_dir() / "docker" / task_id
|
||||
self._home_dir = str(sandbox / "home")
|
||||
os.makedirs(self._home_dir, exist_ok=True)
|
||||
writable_args.extend([
|
||||
"-v", f"{self._home_dir}:/root",
|
||||
])
|
||||
if not bind_host_cwd and not workspace_explicitly_mounted:
|
||||
self._workspace_dir = str(sandbox / "workspace")
|
||||
os.makedirs(self._workspace_dir, exist_ok=True)
|
||||
writable_args.extend([
|
||||
"-v", f"{self._workspace_dir}:/workspace",
|
||||
])
|
||||
else:
|
||||
if not bind_host_cwd and not workspace_explicitly_mounted:
|
||||
writable_args.extend([
|
||||
"--tmpfs", "/workspace:rw,exec,size=10g",
|
||||
])
|
||||
writable_args.extend([
|
||||
"--tmpfs", "/home:rw,exec,size=1g",
|
||||
"--tmpfs", "/root:rw,exec,size=1g",
|
||||
])
|
||||
|
||||
if bind_host_cwd:
|
||||
logger.info(f"Mounting configured host cwd to /workspace: {host_cwd_abs}")
|
||||
volume_args = ["-v", f"{host_cwd_abs}:/workspace", *volume_args]
|
||||
elif workspace_explicitly_mounted:
|
||||
logger.debug("Skipping docker cwd mount: /workspace already mounted by user config")
|
||||
|
||||
# Mount credential files (OAuth tokens, etc.) declared by skills.
|
||||
# Read-only so the container can authenticate but not modify host creds.
|
||||
try:
|
||||
from tools.credential_files import (
|
||||
get_credential_file_mounts,
|
||||
get_skills_directory_mount,
|
||||
get_cache_directory_mounts,
|
||||
)
|
||||
|
||||
for mount_entry in get_credential_file_mounts():
|
||||
volume_args.extend([
|
||||
"-v",
|
||||
f"{mount_entry['host_path']}:{mount_entry['container_path']}:ro",
|
||||
])
|
||||
logger.info(
|
||||
"Docker: mounting credential %s -> %s",
|
||||
mount_entry["host_path"],
|
||||
mount_entry["container_path"],
|
||||
)
|
||||
|
||||
# Mount skill directories (local + external) so skill
|
||||
# scripts/templates are available inside the container.
|
||||
for skills_mount in get_skills_directory_mount():
|
||||
volume_args.extend([
|
||||
"-v",
|
||||
f"{skills_mount['host_path']}:{skills_mount['container_path']}:ro",
|
||||
])
|
||||
logger.info(
|
||||
"Docker: mounting skills dir %s -> %s",
|
||||
skills_mount["host_path"],
|
||||
skills_mount["container_path"],
|
||||
)
|
||||
|
||||
# Mount host-side cache directories (documents, images, audio,
|
||||
# screenshots) so the agent can access uploaded files and other
|
||||
# cached media from inside the container. Read-only — the
|
||||
# container reads these but the host gateway manages writes.
|
||||
for cache_mount in get_cache_directory_mounts():
|
||||
volume_args.extend([
|
||||
"-v",
|
||||
f"{cache_mount['host_path']}:{cache_mount['container_path']}:ro",
|
||||
])
|
||||
logger.info(
|
||||
"Docker: mounting cache dir %s -> %s",
|
||||
cache_mount["host_path"],
|
||||
cache_mount["container_path"],
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug("Docker: could not load credential file mounts: %s", e)
|
||||
|
||||
# Explicit environment variables (docker_env config) — set at container
|
||||
# creation so they're available to all processes (including entrypoint).
|
||||
env_args = []
|
||||
for key in sorted(self._env):
|
||||
env_args.extend(["-e", f"{key}={self._env[key]}"])
|
||||
|
||||
logger.info(f"Docker volume_args: {volume_args}")
|
||||
all_run_args = list(_SECURITY_ARGS) + writable_args + resource_args + volume_args + env_args
|
||||
logger.info(f"Docker run_args: {all_run_args}")
|
||||
|
||||
# Resolve the docker executable once so it works even when
|
||||
# /usr/local/bin is not in PATH (common on macOS gateway/service).
|
||||
self._docker_exe = find_docker() or "docker"
|
||||
|
||||
# Start the container directly via `docker run -d`.
|
||||
container_name = f"hermes-{uuid.uuid4().hex[:8]}"
|
||||
run_cmd = [
|
||||
self._docker_exe, "run", "-d",
|
||||
"--init", # tini/catatonit as PID 1 — reaps zombie children
|
||||
"--name", container_name,
|
||||
"-w", cwd,
|
||||
*all_run_args,
|
||||
image,
|
||||
"sleep", "infinity", # no fixed lifetime — idle reaper handles cleanup
|
||||
]
|
||||
logger.debug(f"Starting container: {' '.join(run_cmd)}")
|
||||
result = subprocess.run(
|
||||
run_cmd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120, # image pull may take a while
|
||||
check=True,
|
||||
)
|
||||
self._container_id = result.stdout.strip()
|
||||
logger.info(f"Started container {container_name} ({self._container_id[:12]})")
|
||||
|
||||
# Build the init-time env forwarding args (used only by init_session
|
||||
# to inject host env vars into the snapshot; subsequent commands get
|
||||
# them from the snapshot file).
|
||||
self._init_env_args = self._build_init_env_args()
|
||||
|
||||
# Initialize session snapshot inside the container
|
||||
self.init_session()
|
||||
|
||||
def _build_init_env_args(self) -> list[str]:
|
||||
"""Build -e KEY=VALUE args for injecting host env vars into init_session.
|
||||
|
||||
These are used once during init_session() so that export -p captures
|
||||
them into the snapshot. Subsequent execute() calls don't need -e flags.
|
||||
"""
|
||||
exec_env: dict[str, str] = dict(self._env)
|
||||
|
||||
explicit_forward_keys = set(self._forward_env)
|
||||
passthrough_keys: set[str] = set()
|
||||
try:
|
||||
from tools.env_passthrough import get_all_passthrough
|
||||
passthrough_keys = set(get_all_passthrough())
|
||||
except Exception:
|
||||
pass
|
||||
# Explicit docker_forward_env entries are an intentional opt-in and must
|
||||
# win over the generic Hermes secret blocklist. Only implicit passthrough
|
||||
# keys are filtered.
|
||||
forward_keys = explicit_forward_keys | (passthrough_keys - _HERMES_PROVIDER_ENV_BLOCKLIST)
|
||||
hermes_env = _load_hermes_env_vars() if forward_keys else {}
|
||||
for key in sorted(forward_keys):
|
||||
value = os.getenv(key)
|
||||
if value is None:
|
||||
value = hermes_env.get(key)
|
||||
if value is not None:
|
||||
exec_env[key] = value
|
||||
|
||||
args = []
|
||||
for key in sorted(exec_env):
|
||||
args.extend(["-e", f"{key}={exec_env[key]}"])
|
||||
return args
|
||||
|
||||
def _run_bash(self, cmd_string: str, *, login: bool = False,
|
||||
timeout: int = 120,
|
||||
stdin_data: str | None = None) -> subprocess.Popen:
|
||||
"""Spawn a bash process inside the Docker container."""
|
||||
assert self._container_id, "Container not started"
|
||||
cmd = [self._docker_exe, "exec"]
|
||||
if stdin_data is not None:
|
||||
cmd.append("-i")
|
||||
|
||||
# Only inject -e env args during init_session (login=True).
|
||||
# Subsequent commands get env vars from the snapshot.
|
||||
if login:
|
||||
cmd.extend(self._init_env_args)
|
||||
|
||||
cmd.extend([self._container_id])
|
||||
|
||||
if login:
|
||||
cmd.extend(["bash", "-l", "-c", cmd_string])
|
||||
else:
|
||||
cmd.extend(["bash", "-c", cmd_string])
|
||||
|
||||
return _popen_bash(cmd, stdin_data)
|
||||
|
||||
@staticmethod
|
||||
def _storage_opt_supported() -> bool:
|
||||
"""Check if Docker's storage driver supports --storage-opt size=.
|
||||
|
||||
Only overlay2 on XFS with pquota supports per-container disk quotas.
|
||||
Ubuntu (and most distros) default to ext4, where this flag errors out.
|
||||
"""
|
||||
global _storage_opt_ok
|
||||
if _storage_opt_ok is not None:
|
||||
return _storage_opt_ok
|
||||
try:
|
||||
docker = find_docker() or "docker"
|
||||
result = subprocess.run(
|
||||
[docker, "info", "--format", "{{.Driver}}"],
|
||||
capture_output=True, text=True, timeout=10,
|
||||
)
|
||||
driver = result.stdout.strip().lower()
|
||||
if driver != "overlay2":
|
||||
_storage_opt_ok = False
|
||||
return False
|
||||
# overlay2 only supports storage-opt on XFS with pquota.
|
||||
# Probe by attempting a dry-ish run — the fastest reliable check.
|
||||
probe = subprocess.run(
|
||||
[docker, "create", "--storage-opt", "size=1m", "hello-world"],
|
||||
capture_output=True, text=True, timeout=15,
|
||||
)
|
||||
if probe.returncode == 0:
|
||||
# Clean up the created container
|
||||
container_id = probe.stdout.strip()
|
||||
if container_id:
|
||||
subprocess.run([docker, "rm", container_id],
|
||||
capture_output=True, timeout=5)
|
||||
_storage_opt_ok = True
|
||||
else:
|
||||
_storage_opt_ok = False
|
||||
except Exception:
|
||||
_storage_opt_ok = False
|
||||
logger.debug("Docker --storage-opt support: %s", _storage_opt_ok)
|
||||
return _storage_opt_ok
|
||||
|
||||
def cleanup(self):
|
||||
"""Stop and remove the container. Bind-mount dirs persist if persistent=True."""
|
||||
if self._container_id:
|
||||
try:
|
||||
# Stop in background so cleanup doesn't block
|
||||
stop_cmd = (
|
||||
f"(timeout 60 {self._docker_exe} stop {self._container_id} || "
|
||||
f"{self._docker_exe} rm -f {self._container_id}) >/dev/null 2>&1 &"
|
||||
)
|
||||
subprocess.Popen(stop_cmd, shell=True)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to stop container %s: %s", self._container_id, e)
|
||||
|
||||
if not self._persistent:
|
||||
# Also schedule removal (stop only leaves it as stopped)
|
||||
try:
|
||||
subprocess.Popen(
|
||||
f"sleep 3 && {self._docker_exe} rm -f {self._container_id} >/dev/null 2>&1 &",
|
||||
shell=True,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
self._container_id = None
|
||||
|
||||
if not self._persistent:
|
||||
for d in (self._workspace_dir, self._home_dir):
|
||||
if d:
|
||||
shutil.rmtree(d, ignore_errors=True)
|
||||
@@ -0,0 +1,168 @@
|
||||
"""Shared file sync manager for remote execution backends.
|
||||
|
||||
Tracks local file changes via mtime+size, detects deletions, and
|
||||
syncs to remote environments transactionally. Used by SSH, Modal,
|
||||
and Daytona. Docker and Singularity use bind mounts (live host FS
|
||||
view) and don't need this.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shlex
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
from tools.environments.base import _file_mtime_key
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SYNC_INTERVAL_SECONDS = 5.0
|
||||
_FORCE_SYNC_ENV = "HERMES_FORCE_FILE_SYNC"
|
||||
|
||||
# Transport callbacks provided by each backend
|
||||
UploadFn = Callable[[str, str], None] # (host_path, remote_path) -> raises on failure
|
||||
BulkUploadFn = Callable[[list[tuple[str, str]]], None] # [(host_path, remote_path), ...] -> raises on failure
|
||||
DeleteFn = Callable[[list[str]], None] # (remote_paths) -> raises on failure
|
||||
GetFilesFn = Callable[[], list[tuple[str, str]]] # () -> [(host_path, remote_path), ...]
|
||||
|
||||
|
||||
def iter_sync_files(container_base: str = "/root/.hermes") -> list[tuple[str, str]]:
|
||||
"""Enumerate all files that should be synced to a remote environment.
|
||||
|
||||
Combines credentials, skills, and cache into a single flat list of
|
||||
(host_path, remote_path) pairs. Credential paths are remapped from
|
||||
the hardcoded /root/.hermes to *container_base* because the remote
|
||||
user's home may differ (e.g. /home/daytona, /home/user).
|
||||
"""
|
||||
# Late import: credential_files imports agent modules that create
|
||||
# circular dependencies if loaded at file_sync module level.
|
||||
from tools.credential_files import (
|
||||
get_credential_file_mounts,
|
||||
iter_cache_files,
|
||||
iter_skills_files,
|
||||
)
|
||||
|
||||
files: list[tuple[str, str]] = []
|
||||
for entry in get_credential_file_mounts():
|
||||
remote = entry["container_path"].replace(
|
||||
"/root/.hermes", container_base, 1
|
||||
)
|
||||
files.append((entry["host_path"], remote))
|
||||
for entry in iter_skills_files(container_base=container_base):
|
||||
files.append((entry["host_path"], entry["container_path"]))
|
||||
for entry in iter_cache_files(container_base=container_base):
|
||||
files.append((entry["host_path"], entry["container_path"]))
|
||||
return files
|
||||
|
||||
|
||||
def quoted_rm_command(remote_paths: list[str]) -> str:
|
||||
"""Build a shell ``rm -f`` command for a batch of remote paths."""
|
||||
return "rm -f " + " ".join(shlex.quote(p) for p in remote_paths)
|
||||
|
||||
|
||||
def quoted_mkdir_command(dirs: list[str]) -> str:
|
||||
"""Build a shell ``mkdir -p`` command for a batch of directories."""
|
||||
return "mkdir -p " + " ".join(shlex.quote(d) for d in dirs)
|
||||
|
||||
|
||||
def unique_parent_dirs(files: list[tuple[str, str]]) -> list[str]:
|
||||
"""Extract sorted unique parent directories from (host, remote) pairs."""
|
||||
return sorted({str(Path(remote).parent) for _, remote in files})
|
||||
|
||||
|
||||
class FileSyncManager:
|
||||
"""Tracks local file changes and syncs to a remote environment.
|
||||
|
||||
Backends instantiate this with transport callbacks (upload, delete)
|
||||
and a file-source callable. The manager handles mtime-based change
|
||||
detection, deletion tracking, rate limiting, and transactional state.
|
||||
|
||||
Not used by bind-mount backends (Docker, Singularity) — those get
|
||||
live host FS views and don't need file sync.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
get_files_fn: GetFilesFn,
|
||||
upload_fn: UploadFn,
|
||||
delete_fn: DeleteFn,
|
||||
sync_interval: float = _SYNC_INTERVAL_SECONDS,
|
||||
bulk_upload_fn: BulkUploadFn | None = None,
|
||||
):
|
||||
self._get_files_fn = get_files_fn
|
||||
self._upload_fn = upload_fn
|
||||
self._bulk_upload_fn = bulk_upload_fn
|
||||
self._delete_fn = delete_fn
|
||||
self._synced_files: dict[str, tuple[float, int]] = {} # remote_path -> (mtime, size)
|
||||
self._last_sync_time: float = 0.0 # monotonic; 0 ensures first sync runs
|
||||
self._sync_interval = sync_interval
|
||||
|
||||
def sync(self, *, force: bool = False) -> None:
|
||||
"""Run a sync cycle: upload changed files, delete removed files.
|
||||
|
||||
Rate-limited to once per ``sync_interval`` unless *force* is True
|
||||
or ``HERMES_FORCE_FILE_SYNC=1`` is set.
|
||||
|
||||
Transactional: state only committed if ALL operations succeed.
|
||||
On failure, state rolls back so the next cycle retries everything.
|
||||
"""
|
||||
if not force and not os.environ.get(_FORCE_SYNC_ENV):
|
||||
now = time.monotonic()
|
||||
if now - self._last_sync_time < self._sync_interval:
|
||||
return
|
||||
|
||||
current_files = self._get_files_fn()
|
||||
current_remote_paths = {remote for _, remote in current_files}
|
||||
|
||||
# --- Uploads: new or changed files ---
|
||||
to_upload: list[tuple[str, str]] = []
|
||||
new_files = dict(self._synced_files)
|
||||
for host_path, remote_path in current_files:
|
||||
file_key = _file_mtime_key(host_path)
|
||||
if file_key is None:
|
||||
continue
|
||||
if self._synced_files.get(remote_path) == file_key:
|
||||
continue
|
||||
to_upload.append((host_path, remote_path))
|
||||
new_files[remote_path] = file_key
|
||||
|
||||
# --- Deletes: synced paths no longer in current set ---
|
||||
to_delete = [p for p in self._synced_files if p not in current_remote_paths]
|
||||
|
||||
if not to_upload and not to_delete:
|
||||
self._last_sync_time = time.monotonic()
|
||||
return
|
||||
|
||||
# Snapshot for rollback (only when there's work to do)
|
||||
prev_files = dict(self._synced_files)
|
||||
|
||||
if to_upload:
|
||||
logger.debug("file_sync: uploading %d file(s)", len(to_upload))
|
||||
if to_delete:
|
||||
logger.debug("file_sync: deleting %d stale remote file(s)", len(to_delete))
|
||||
|
||||
try:
|
||||
if to_upload and self._bulk_upload_fn is not None:
|
||||
self._bulk_upload_fn(to_upload)
|
||||
logger.debug("file_sync: bulk-uploaded %d file(s)", len(to_upload))
|
||||
else:
|
||||
for host_path, remote_path in to_upload:
|
||||
self._upload_fn(host_path, remote_path)
|
||||
logger.debug("file_sync: uploaded %s -> %s", host_path, remote_path)
|
||||
|
||||
if to_delete:
|
||||
self._delete_fn(to_delete)
|
||||
logger.debug("file_sync: deleted %s", to_delete)
|
||||
|
||||
# --- Commit (all succeeded) ---
|
||||
for p in to_delete:
|
||||
new_files.pop(p, None)
|
||||
|
||||
self._synced_files = new_files
|
||||
self._last_sync_time = time.monotonic()
|
||||
|
||||
except Exception as exc:
|
||||
self._synced_files = prev_files
|
||||
self._last_sync_time = time.monotonic()
|
||||
logger.warning("file_sync: sync failed, rolled back state: %s", exc)
|
||||
@@ -0,0 +1,314 @@
|
||||
"""Local execution environment — spawn-per-call with session snapshot."""
|
||||
|
||||
import os
|
||||
import platform
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
from tools.environments.base import BaseEnvironment, _pipe_stdin
|
||||
|
||||
_IS_WINDOWS = platform.system() == "Windows"
|
||||
|
||||
|
||||
# Hermes-internal env vars that should NOT leak into terminal subprocesses.
|
||||
_HERMES_PROVIDER_ENV_FORCE_PREFIX = "_HERMES_FORCE_"
|
||||
|
||||
|
||||
def _build_provider_env_blocklist() -> frozenset:
|
||||
"""Derive the blocklist from provider, tool, and gateway config."""
|
||||
blocked: set[str] = set()
|
||||
|
||||
try:
|
||||
from hermes_cli.auth import PROVIDER_REGISTRY
|
||||
for pconfig in PROVIDER_REGISTRY.values():
|
||||
blocked.update(pconfig.api_key_env_vars)
|
||||
if pconfig.base_url_env_var:
|
||||
blocked.add(pconfig.base_url_env_var)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
from hermes_cli.config import OPTIONAL_ENV_VARS
|
||||
for name, metadata in OPTIONAL_ENV_VARS.items():
|
||||
category = metadata.get("category")
|
||||
if category in {"tool", "messaging"}:
|
||||
blocked.add(name)
|
||||
elif category == "setting" and metadata.get("password"):
|
||||
blocked.add(name)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
blocked.update({
|
||||
"OPENAI_BASE_URL",
|
||||
"OPENAI_API_KEY",
|
||||
"OPENAI_API_BASE",
|
||||
"OPENAI_ORG_ID",
|
||||
"OPENAI_ORGANIZATION",
|
||||
"OPENROUTER_API_KEY",
|
||||
"ANTHROPIC_BASE_URL",
|
||||
"ANTHROPIC_TOKEN",
|
||||
"CLAUDE_CODE_OAUTH_TOKEN",
|
||||
"LLM_MODEL",
|
||||
"GOOGLE_API_KEY",
|
||||
"DEEPSEEK_API_KEY",
|
||||
"MISTRAL_API_KEY",
|
||||
"GROQ_API_KEY",
|
||||
"TOGETHER_API_KEY",
|
||||
"PERPLEXITY_API_KEY",
|
||||
"COHERE_API_KEY",
|
||||
"FIREWORKS_API_KEY",
|
||||
"XAI_API_KEY",
|
||||
"HELICONE_API_KEY",
|
||||
"PARALLEL_API_KEY",
|
||||
"FIRECRAWL_API_KEY",
|
||||
"FIRECRAWL_API_URL",
|
||||
"TELEGRAM_HOME_CHANNEL",
|
||||
"TELEGRAM_HOME_CHANNEL_NAME",
|
||||
"DISCORD_HOME_CHANNEL",
|
||||
"DISCORD_HOME_CHANNEL_NAME",
|
||||
"DISCORD_REQUIRE_MENTION",
|
||||
"DISCORD_FREE_RESPONSE_CHANNELS",
|
||||
"DISCORD_AUTO_THREAD",
|
||||
"SLACK_HOME_CHANNEL",
|
||||
"SLACK_HOME_CHANNEL_NAME",
|
||||
"SLACK_ALLOWED_USERS",
|
||||
"WHATSAPP_ENABLED",
|
||||
"WHATSAPP_MODE",
|
||||
"WHATSAPP_ALLOWED_USERS",
|
||||
"SIGNAL_HTTP_URL",
|
||||
"SIGNAL_ACCOUNT",
|
||||
"SIGNAL_ALLOWED_USERS",
|
||||
"SIGNAL_GROUP_ALLOWED_USERS",
|
||||
"SIGNAL_HOME_CHANNEL",
|
||||
"SIGNAL_HOME_CHANNEL_NAME",
|
||||
"SIGNAL_IGNORE_STORIES",
|
||||
"HASS_TOKEN",
|
||||
"HASS_URL",
|
||||
"EMAIL_ADDRESS",
|
||||
"EMAIL_PASSWORD",
|
||||
"EMAIL_IMAP_HOST",
|
||||
"EMAIL_SMTP_HOST",
|
||||
"EMAIL_HOME_ADDRESS",
|
||||
"EMAIL_HOME_ADDRESS_NAME",
|
||||
"GATEWAY_ALLOWED_USERS",
|
||||
"GH_TOKEN",
|
||||
"GITHUB_APP_ID",
|
||||
"GITHUB_APP_PRIVATE_KEY_PATH",
|
||||
"GITHUB_APP_INSTALLATION_ID",
|
||||
"MODAL_TOKEN_ID",
|
||||
"MODAL_TOKEN_SECRET",
|
||||
"DAYTONA_API_KEY",
|
||||
})
|
||||
return frozenset(blocked)
|
||||
|
||||
|
||||
_HERMES_PROVIDER_ENV_BLOCKLIST = _build_provider_env_blocklist()
|
||||
|
||||
|
||||
def _sanitize_subprocess_env(base_env: dict | None, extra_env: dict | None = None) -> dict:
|
||||
"""Filter Hermes-managed secrets from a subprocess environment."""
|
||||
try:
|
||||
from tools.env_passthrough import is_env_passthrough as _is_passthrough
|
||||
except Exception:
|
||||
_is_passthrough = lambda _: False # noqa: E731
|
||||
|
||||
sanitized: dict[str, str] = {}
|
||||
|
||||
for key, value in (base_env or {}).items():
|
||||
if key.startswith(_HERMES_PROVIDER_ENV_FORCE_PREFIX):
|
||||
continue
|
||||
if key not in _HERMES_PROVIDER_ENV_BLOCKLIST or _is_passthrough(key):
|
||||
sanitized[key] = value
|
||||
|
||||
for key, value in (extra_env or {}).items():
|
||||
if key.startswith(_HERMES_PROVIDER_ENV_FORCE_PREFIX):
|
||||
real_key = key[len(_HERMES_PROVIDER_ENV_FORCE_PREFIX):]
|
||||
sanitized[real_key] = value
|
||||
elif key not in _HERMES_PROVIDER_ENV_BLOCKLIST or _is_passthrough(key):
|
||||
sanitized[key] = value
|
||||
|
||||
# Per-profile HOME isolation for background processes (same as _make_run_env).
|
||||
from hermes_constants import get_subprocess_home
|
||||
_profile_home = get_subprocess_home()
|
||||
if _profile_home:
|
||||
sanitized["HOME"] = _profile_home
|
||||
|
||||
return sanitized
|
||||
|
||||
|
||||
def _find_bash() -> str:
|
||||
"""Find bash for command execution."""
|
||||
if not _IS_WINDOWS:
|
||||
return (
|
||||
shutil.which("bash")
|
||||
or ("/usr/bin/bash" if os.path.isfile("/usr/bin/bash") else None)
|
||||
or ("/bin/bash" if os.path.isfile("/bin/bash") else None)
|
||||
or os.environ.get("SHELL")
|
||||
or "/bin/sh"
|
||||
)
|
||||
|
||||
custom = os.environ.get("HERMES_GIT_BASH_PATH")
|
||||
if custom and os.path.isfile(custom):
|
||||
return custom
|
||||
|
||||
found = shutil.which("bash")
|
||||
if found:
|
||||
return found
|
||||
|
||||
for candidate in (
|
||||
os.path.join(os.environ.get("ProgramFiles", r"C:\Program Files"), "Git", "bin", "bash.exe"),
|
||||
os.path.join(os.environ.get("ProgramFiles(x86)", r"C:\Program Files (x86)"), "Git", "bin", "bash.exe"),
|
||||
os.path.join(os.environ.get("LOCALAPPDATA", ""), "Programs", "Git", "bin", "bash.exe"),
|
||||
):
|
||||
if candidate and os.path.isfile(candidate):
|
||||
return candidate
|
||||
|
||||
raise RuntimeError(
|
||||
"Git Bash not found. Hermes Agent requires Git for Windows on Windows.\n"
|
||||
"Install it from: https://git-scm.com/download/win\n"
|
||||
"Or set HERMES_GIT_BASH_PATH to your bash.exe location."
|
||||
)
|
||||
|
||||
|
||||
# Backward compat — process_registry.py imports this name
|
||||
_find_shell = _find_bash
|
||||
|
||||
|
||||
# Standard PATH entries for environments with minimal PATH.
|
||||
_SANE_PATH = (
|
||||
"/opt/homebrew/bin:/opt/homebrew/sbin:"
|
||||
"/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||||
)
|
||||
|
||||
|
||||
def _make_run_env(env: dict) -> dict:
|
||||
"""Build a run environment with a sane PATH and provider-var stripping."""
|
||||
try:
|
||||
from tools.env_passthrough import is_env_passthrough as _is_passthrough
|
||||
except Exception:
|
||||
_is_passthrough = lambda _: False # noqa: E731
|
||||
|
||||
merged = dict(os.environ | env)
|
||||
run_env = {}
|
||||
for k, v in merged.items():
|
||||
if k.startswith(_HERMES_PROVIDER_ENV_FORCE_PREFIX):
|
||||
real_key = k[len(_HERMES_PROVIDER_ENV_FORCE_PREFIX):]
|
||||
run_env[real_key] = v
|
||||
elif k not in _HERMES_PROVIDER_ENV_BLOCKLIST or _is_passthrough(k):
|
||||
run_env[k] = v
|
||||
existing_path = run_env.get("PATH", "")
|
||||
if "/usr/bin" not in existing_path.split(":"):
|
||||
run_env["PATH"] = f"{existing_path}:{_SANE_PATH}" if existing_path else _SANE_PATH
|
||||
|
||||
# Per-profile HOME isolation: redirect system tool configs (git, ssh, gh,
|
||||
# npm …) into {HERMES_HOME}/home/ when that directory exists. Only the
|
||||
# subprocess sees the override — the Python process keeps the real HOME.
|
||||
from hermes_constants import get_subprocess_home
|
||||
_profile_home = get_subprocess_home()
|
||||
if _profile_home:
|
||||
run_env["HOME"] = _profile_home
|
||||
|
||||
return run_env
|
||||
|
||||
|
||||
class LocalEnvironment(BaseEnvironment):
|
||||
"""Run commands directly on the host machine.
|
||||
|
||||
Spawn-per-call: every execute() spawns a fresh bash process.
|
||||
Session snapshot preserves env vars across calls.
|
||||
CWD persists via file-based read after each command.
|
||||
"""
|
||||
|
||||
def __init__(self, cwd: str = "", timeout: int = 60, env: dict = None):
|
||||
super().__init__(cwd=cwd or os.getcwd(), timeout=timeout, env=env)
|
||||
self.init_session()
|
||||
|
||||
def get_temp_dir(self) -> str:
|
||||
"""Return a shell-safe writable temp dir for local execution.
|
||||
|
||||
Termux does not provide /tmp by default, but exposes a POSIX TMPDIR.
|
||||
Prefer POSIX-style env vars when available, keep using /tmp on regular
|
||||
Unix systems, and only fall back to tempfile.gettempdir() when it also
|
||||
resolves to a POSIX path.
|
||||
|
||||
Check the environment configured for this backend first so callers can
|
||||
override the temp root explicitly (for example via terminal.env or a
|
||||
custom TMPDIR), then fall back to the host process environment.
|
||||
"""
|
||||
for env_var in ("TMPDIR", "TMP", "TEMP"):
|
||||
candidate = self.env.get(env_var) or os.environ.get(env_var)
|
||||
if candidate and candidate.startswith("/"):
|
||||
return candidate.rstrip("/") or "/"
|
||||
|
||||
if os.path.isdir("/tmp") and os.access("/tmp", os.W_OK | os.X_OK):
|
||||
return "/tmp"
|
||||
|
||||
candidate = tempfile.gettempdir()
|
||||
if candidate.startswith("/"):
|
||||
return candidate.rstrip("/") or "/"
|
||||
|
||||
return "/tmp"
|
||||
|
||||
def _run_bash(self, cmd_string: str, *, login: bool = False,
|
||||
timeout: int = 120,
|
||||
stdin_data: str | None = None) -> subprocess.Popen:
|
||||
bash = _find_bash()
|
||||
args = [bash, "-l", "-c", cmd_string] if login else [bash, "-c", cmd_string]
|
||||
run_env = _make_run_env(self.env)
|
||||
|
||||
proc = subprocess.Popen(
|
||||
args,
|
||||
text=True,
|
||||
env=run_env,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
stdin=subprocess.PIPE if stdin_data is not None else subprocess.DEVNULL,
|
||||
preexec_fn=None if _IS_WINDOWS else os.setsid,
|
||||
)
|
||||
|
||||
if stdin_data is not None:
|
||||
_pipe_stdin(proc, stdin_data)
|
||||
|
||||
return proc
|
||||
|
||||
def _kill_process(self, proc):
|
||||
"""Kill the entire process group (all children)."""
|
||||
try:
|
||||
if _IS_WINDOWS:
|
||||
proc.terminate()
|
||||
else:
|
||||
pgid = os.getpgid(proc.pid)
|
||||
os.killpg(pgid, signal.SIGTERM)
|
||||
try:
|
||||
proc.wait(timeout=1.0)
|
||||
except subprocess.TimeoutExpired:
|
||||
os.killpg(pgid, signal.SIGKILL)
|
||||
except (ProcessLookupError, PermissionError):
|
||||
try:
|
||||
proc.kill()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _update_cwd(self, result: dict):
|
||||
"""Read CWD from temp file (local-only, no round-trip needed)."""
|
||||
try:
|
||||
cwd_path = open(self._cwd_file).read().strip()
|
||||
if cwd_path:
|
||||
self.cwd = cwd_path
|
||||
except (OSError, FileNotFoundError):
|
||||
pass
|
||||
|
||||
# Still strip the marker from output so it's not visible
|
||||
self._extract_cwd_from_output(result)
|
||||
|
||||
def cleanup(self):
|
||||
"""Clean up temp files."""
|
||||
for f in (self._snapshot_path, self._cwd_file):
|
||||
try:
|
||||
os.unlink(f)
|
||||
except OSError:
|
||||
pass
|
||||
@@ -0,0 +1,282 @@
|
||||
"""Managed Modal environment backed by tool-gateway."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import requests
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from tools.environments.modal_utils import (
|
||||
BaseModalExecutionEnvironment,
|
||||
ModalExecStart,
|
||||
PreparedModalExec,
|
||||
)
|
||||
from tools.managed_tool_gateway import resolve_managed_tool_gateway
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _request_timeout_env(name: str, default: float) -> float:
|
||||
try:
|
||||
value = float(os.getenv(name, str(default)))
|
||||
return value if value > 0 else default
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _ManagedModalExecHandle:
|
||||
exec_id: str
|
||||
|
||||
|
||||
class ManagedModalEnvironment(BaseModalExecutionEnvironment):
|
||||
"""Gateway-owned Modal sandbox with Hermes-compatible execute/cleanup."""
|
||||
|
||||
_CONNECT_TIMEOUT_SECONDS = _request_timeout_env("TERMINAL_MANAGED_MODAL_CONNECT_TIMEOUT_SECONDS", 1.0)
|
||||
_POLL_READ_TIMEOUT_SECONDS = _request_timeout_env("TERMINAL_MANAGED_MODAL_POLL_READ_TIMEOUT_SECONDS", 5.0)
|
||||
_CANCEL_READ_TIMEOUT_SECONDS = _request_timeout_env("TERMINAL_MANAGED_MODAL_CANCEL_READ_TIMEOUT_SECONDS", 5.0)
|
||||
_client_timeout_grace_seconds = 10.0
|
||||
_interrupt_output = "[Command interrupted - Modal sandbox exec cancelled]"
|
||||
_unexpected_error_prefix = "Managed Modal exec failed"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image: str,
|
||||
cwd: str = "/root",
|
||||
timeout: int = 60,
|
||||
modal_sandbox_kwargs: Optional[Dict[str, Any]] = None,
|
||||
persistent_filesystem: bool = True,
|
||||
task_id: str = "default",
|
||||
):
|
||||
super().__init__(cwd=cwd, timeout=timeout)
|
||||
|
||||
self._guard_unsupported_credential_passthrough()
|
||||
|
||||
gateway = resolve_managed_tool_gateway("modal")
|
||||
if gateway is None:
|
||||
raise ValueError("Managed Modal requires a configured tool gateway and Nous user token")
|
||||
|
||||
self._gateway_origin = gateway.gateway_origin.rstrip("/")
|
||||
self._nous_user_token = gateway.nous_user_token
|
||||
self._task_id = task_id
|
||||
self._persistent = persistent_filesystem
|
||||
self._image = image
|
||||
self._sandbox_kwargs = dict(modal_sandbox_kwargs or {})
|
||||
self._create_idempotency_key = str(uuid.uuid4())
|
||||
self._sandbox_id = self._create_sandbox()
|
||||
|
||||
def _start_modal_exec(self, prepared: PreparedModalExec) -> ModalExecStart:
|
||||
exec_id = str(uuid.uuid4())
|
||||
payload: Dict[str, Any] = {
|
||||
"execId": exec_id,
|
||||
"command": prepared.command,
|
||||
"cwd": prepared.cwd,
|
||||
"timeoutMs": int(prepared.timeout * 1000),
|
||||
}
|
||||
if prepared.stdin_data is not None:
|
||||
payload["stdinData"] = prepared.stdin_data
|
||||
|
||||
try:
|
||||
response = self._request(
|
||||
"POST",
|
||||
f"/v1/sandboxes/{self._sandbox_id}/execs",
|
||||
json=payload,
|
||||
timeout=10,
|
||||
)
|
||||
except Exception as exc:
|
||||
return ModalExecStart(
|
||||
immediate_result=self._error_result(f"Managed Modal exec failed: {exc}")
|
||||
)
|
||||
|
||||
if response.status_code >= 400:
|
||||
return ModalExecStart(
|
||||
immediate_result=self._error_result(
|
||||
self._format_error("Managed Modal exec failed", response)
|
||||
)
|
||||
)
|
||||
|
||||
body = response.json()
|
||||
status = body.get("status")
|
||||
if status in {"completed", "failed", "cancelled", "timeout"}:
|
||||
return ModalExecStart(
|
||||
immediate_result=self._result(
|
||||
body.get("output", ""),
|
||||
body.get("returncode", 1),
|
||||
)
|
||||
)
|
||||
|
||||
if body.get("execId") != exec_id:
|
||||
return ModalExecStart(
|
||||
immediate_result=self._error_result(
|
||||
"Managed Modal exec start did not return the expected exec id"
|
||||
)
|
||||
)
|
||||
|
||||
return ModalExecStart(handle=_ManagedModalExecHandle(exec_id=exec_id))
|
||||
|
||||
def _poll_modal_exec(self, handle: _ManagedModalExecHandle) -> dict | None:
|
||||
try:
|
||||
status_response = self._request(
|
||||
"GET",
|
||||
f"/v1/sandboxes/{self._sandbox_id}/execs/{handle.exec_id}",
|
||||
timeout=(self._CONNECT_TIMEOUT_SECONDS, self._POLL_READ_TIMEOUT_SECONDS),
|
||||
)
|
||||
except Exception as exc:
|
||||
return self._error_result(f"Managed Modal exec poll failed: {exc}")
|
||||
|
||||
if status_response.status_code == 404:
|
||||
return self._error_result("Managed Modal exec not found")
|
||||
|
||||
if status_response.status_code >= 400:
|
||||
return self._error_result(
|
||||
self._format_error("Managed Modal exec poll failed", status_response)
|
||||
)
|
||||
|
||||
status_body = status_response.json()
|
||||
status = status_body.get("status")
|
||||
if status in {"completed", "failed", "cancelled", "timeout"}:
|
||||
return self._result(
|
||||
status_body.get("output", ""),
|
||||
status_body.get("returncode", 1),
|
||||
)
|
||||
return None
|
||||
|
||||
def _cancel_modal_exec(self, handle: _ManagedModalExecHandle) -> None:
|
||||
self._cancel_exec(handle.exec_id)
|
||||
|
||||
def _timeout_result_for_modal(self, timeout: int) -> dict:
|
||||
return self._result(f"Managed Modal exec timed out after {timeout}s", 124)
|
||||
|
||||
def cleanup(self):
|
||||
if not getattr(self, "_sandbox_id", None):
|
||||
return
|
||||
|
||||
try:
|
||||
self._request(
|
||||
"POST",
|
||||
f"/v1/sandboxes/{self._sandbox_id}/terminate",
|
||||
json={
|
||||
"snapshotBeforeTerminate": self._persistent,
|
||||
},
|
||||
timeout=60,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("Managed Modal cleanup failed: %s", exc)
|
||||
finally:
|
||||
self._sandbox_id = None
|
||||
|
||||
def _create_sandbox(self) -> str:
|
||||
cpu = self._coerce_number(self._sandbox_kwargs.get("cpu"), 1)
|
||||
memory = self._coerce_number(
|
||||
self._sandbox_kwargs.get("memoryMiB", self._sandbox_kwargs.get("memory")),
|
||||
5120,
|
||||
)
|
||||
disk = self._coerce_number(
|
||||
self._sandbox_kwargs.get("ephemeral_disk", self._sandbox_kwargs.get("diskMiB")),
|
||||
None,
|
||||
)
|
||||
|
||||
create_payload = {
|
||||
"image": self._image,
|
||||
"cwd": self.cwd,
|
||||
"cpu": cpu,
|
||||
"memoryMiB": memory,
|
||||
"timeoutMs": 3_600_000,
|
||||
"idleTimeoutMs": max(300_000, int(self.timeout * 1000)),
|
||||
"persistentFilesystem": self._persistent,
|
||||
"logicalKey": self._task_id,
|
||||
}
|
||||
if disk is not None:
|
||||
create_payload["diskMiB"] = disk
|
||||
|
||||
response = self._request(
|
||||
"POST",
|
||||
"/v1/sandboxes",
|
||||
json=create_payload,
|
||||
timeout=60,
|
||||
extra_headers={
|
||||
"x-idempotency-key": self._create_idempotency_key,
|
||||
},
|
||||
)
|
||||
if response.status_code >= 400:
|
||||
raise RuntimeError(self._format_error("Managed Modal create failed", response))
|
||||
|
||||
body = response.json()
|
||||
sandbox_id = body.get("id")
|
||||
if not isinstance(sandbox_id, str) or not sandbox_id:
|
||||
raise RuntimeError("Managed Modal create did not return a sandbox id")
|
||||
return sandbox_id
|
||||
|
||||
def _guard_unsupported_credential_passthrough(self) -> None:
|
||||
"""Managed Modal does not sync or mount host credential files."""
|
||||
try:
|
||||
from tools.credential_files import get_credential_file_mounts
|
||||
except Exception:
|
||||
return
|
||||
|
||||
mounts = get_credential_file_mounts()
|
||||
if mounts:
|
||||
raise ValueError(
|
||||
"Managed Modal does not support host credential-file passthrough. "
|
||||
"Use TERMINAL_MODAL_MODE=direct when skills or config require "
|
||||
"credential files inside the sandbox."
|
||||
)
|
||||
|
||||
def _request(self, method: str, path: str, *,
|
||||
json: Dict[str, Any] | None = None,
|
||||
timeout: int = 30,
|
||||
extra_headers: Dict[str, str] | None = None) -> requests.Response:
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self._nous_user_token}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
if extra_headers:
|
||||
headers.update(extra_headers)
|
||||
|
||||
return requests.request(
|
||||
method,
|
||||
f"{self._gateway_origin}{path}",
|
||||
headers=headers,
|
||||
json=json,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
def _cancel_exec(self, exec_id: str) -> None:
|
||||
try:
|
||||
self._request(
|
||||
"POST",
|
||||
f"/v1/sandboxes/{self._sandbox_id}/execs/{exec_id}/cancel",
|
||||
timeout=(self._CONNECT_TIMEOUT_SECONDS, self._CANCEL_READ_TIMEOUT_SECONDS),
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("Managed Modal exec cancel failed: %s", exc)
|
||||
|
||||
@staticmethod
|
||||
def _coerce_number(value: Any, default: float) -> float:
|
||||
try:
|
||||
if value is None:
|
||||
return default
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
@staticmethod
|
||||
def _format_error(prefix: str, response: requests.Response) -> str:
|
||||
try:
|
||||
payload = response.json()
|
||||
if isinstance(payload, dict):
|
||||
message = payload.get("error") or payload.get("message") or payload.get("code")
|
||||
if isinstance(message, str) and message:
|
||||
return f"{prefix}: {message}"
|
||||
return f"{prefix}: {json.dumps(payload, ensure_ascii=False)}"
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
text = response.text.strip()
|
||||
if text:
|
||||
return f"{prefix}: {text}"
|
||||
return f"{prefix}: HTTP {response.status_code}"
|
||||
@@ -0,0 +1,434 @@
|
||||
"""Modal cloud execution environment using the native Modal SDK directly.
|
||||
|
||||
Uses ``Sandbox.create()`` + ``Sandbox.exec()`` instead of the older runtime
|
||||
wrapper, while preserving Hermes' persistent snapshot behavior across sessions.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import logging
|
||||
import shlex
|
||||
import tarfile
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
from hermes_constants import get_hermes_home
|
||||
from tools.environments.base import (
|
||||
BaseEnvironment,
|
||||
_ThreadedProcessHandle,
|
||||
_load_json_store,
|
||||
_save_json_store,
|
||||
)
|
||||
from tools.environments.file_sync import (
|
||||
FileSyncManager,
|
||||
iter_sync_files,
|
||||
quoted_mkdir_command,
|
||||
quoted_rm_command,
|
||||
unique_parent_dirs,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SNAPSHOT_STORE = get_hermes_home() / "modal_snapshots.json"
|
||||
_DIRECT_SNAPSHOT_NAMESPACE = "direct"
|
||||
|
||||
|
||||
def _load_snapshots() -> dict:
|
||||
return _load_json_store(_SNAPSHOT_STORE)
|
||||
|
||||
|
||||
def _save_snapshots(data: dict) -> None:
|
||||
_save_json_store(_SNAPSHOT_STORE, data)
|
||||
|
||||
|
||||
def _direct_snapshot_key(task_id: str) -> str:
|
||||
return f"{_DIRECT_SNAPSHOT_NAMESPACE}:{task_id}"
|
||||
|
||||
|
||||
def _get_snapshot_restore_candidate(task_id: str) -> tuple[str | None, bool]:
|
||||
snapshots = _load_snapshots()
|
||||
namespaced_key = _direct_snapshot_key(task_id)
|
||||
snapshot_id = snapshots.get(namespaced_key)
|
||||
if isinstance(snapshot_id, str) and snapshot_id:
|
||||
return snapshot_id, False
|
||||
legacy_snapshot_id = snapshots.get(task_id)
|
||||
if isinstance(legacy_snapshot_id, str) and legacy_snapshot_id:
|
||||
return legacy_snapshot_id, True
|
||||
return None, False
|
||||
|
||||
|
||||
def _store_direct_snapshot(task_id: str, snapshot_id: str) -> None:
|
||||
snapshots = _load_snapshots()
|
||||
snapshots[_direct_snapshot_key(task_id)] = snapshot_id
|
||||
snapshots.pop(task_id, None)
|
||||
_save_snapshots(snapshots)
|
||||
|
||||
|
||||
def _delete_direct_snapshot(task_id: str, snapshot_id: str | None = None) -> None:
|
||||
snapshots = _load_snapshots()
|
||||
updated = False
|
||||
for key in (_direct_snapshot_key(task_id), task_id):
|
||||
value = snapshots.get(key)
|
||||
if value is None:
|
||||
continue
|
||||
if snapshot_id is None or value == snapshot_id:
|
||||
snapshots.pop(key, None)
|
||||
updated = True
|
||||
if updated:
|
||||
_save_snapshots(snapshots)
|
||||
|
||||
|
||||
def _resolve_modal_image(image_spec: Any) -> Any:
|
||||
"""Convert registry references or snapshot ids into Modal image objects.
|
||||
|
||||
Includes add_python support for ubuntu/debian images (absorbed from PR 4511).
|
||||
"""
|
||||
import modal as _modal
|
||||
|
||||
if not isinstance(image_spec, str):
|
||||
return image_spec
|
||||
|
||||
if image_spec.startswith("im-"):
|
||||
return _modal.Image.from_id(image_spec)
|
||||
|
||||
# PR 4511: add python to ubuntu/debian images that don't have it
|
||||
lower = image_spec.lower()
|
||||
add_python = any(base in lower for base in ("ubuntu", "debian"))
|
||||
|
||||
setup_commands = [
|
||||
"RUN rm -rf /usr/local/lib/python*/site-packages/pip* 2>/dev/null; "
|
||||
"python -m ensurepip --upgrade --default-pip 2>/dev/null || true",
|
||||
]
|
||||
if add_python:
|
||||
setup_commands.insert(0,
|
||||
"RUN apt-get update -qq && apt-get install -y -qq python3 python3-venv > /dev/null 2>&1 || true"
|
||||
)
|
||||
|
||||
return _modal.Image.from_registry(
|
||||
image_spec,
|
||||
setup_dockerfile_commands=setup_commands,
|
||||
)
|
||||
|
||||
|
||||
class _AsyncWorker:
|
||||
"""Background thread with its own event loop for async-safe Modal calls."""
|
||||
|
||||
def __init__(self):
|
||||
self._loop: Optional[asyncio.AbstractEventLoop] = None
|
||||
self._thread: Optional[threading.Thread] = None
|
||||
self._started = threading.Event()
|
||||
|
||||
def start(self):
|
||||
self._thread = threading.Thread(target=self._run_loop, daemon=True)
|
||||
self._thread.start()
|
||||
self._started.wait(timeout=30)
|
||||
|
||||
def _run_loop(self):
|
||||
self._loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(self._loop)
|
||||
self._started.set()
|
||||
self._loop.run_forever()
|
||||
|
||||
def run_coroutine(self, coro, timeout=600):
|
||||
if self._loop is None or self._loop.is_closed():
|
||||
raise RuntimeError("AsyncWorker loop is not running")
|
||||
future = asyncio.run_coroutine_threadsafe(coro, self._loop)
|
||||
return future.result(timeout=timeout)
|
||||
|
||||
def stop(self):
|
||||
if self._loop and self._loop.is_running():
|
||||
self._loop.call_soon_threadsafe(self._loop.stop)
|
||||
if self._thread:
|
||||
self._thread.join(timeout=10)
|
||||
|
||||
|
||||
class ModalEnvironment(BaseEnvironment):
|
||||
"""Modal cloud execution via native Modal sandboxes.
|
||||
|
||||
Spawn-per-call via _ThreadedProcessHandle wrapping async SDK calls.
|
||||
cancel_fn wired to sandbox.terminate for interrupt support.
|
||||
"""
|
||||
|
||||
_stdin_mode = "heredoc"
|
||||
_snapshot_timeout = 60 # Modal cold starts can be slow
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image: str,
|
||||
cwd: str = "/root",
|
||||
timeout: int = 60,
|
||||
modal_sandbox_kwargs: Optional[dict[str, Any]] = None,
|
||||
persistent_filesystem: bool = True,
|
||||
task_id: str = "default",
|
||||
):
|
||||
super().__init__(cwd=cwd, timeout=timeout)
|
||||
|
||||
self._persistent = persistent_filesystem
|
||||
self._task_id = task_id
|
||||
self._sandbox = None
|
||||
self._app = None
|
||||
self._worker = _AsyncWorker()
|
||||
self._sync_manager: FileSyncManager | None = None # initialized after sandbox creation
|
||||
|
||||
sandbox_kwargs = dict(modal_sandbox_kwargs or {})
|
||||
|
||||
restored_snapshot_id = None
|
||||
restored_from_legacy_key = False
|
||||
if self._persistent:
|
||||
restored_snapshot_id, restored_from_legacy_key = _get_snapshot_restore_candidate(
|
||||
self._task_id
|
||||
)
|
||||
if restored_snapshot_id:
|
||||
logger.info("Modal: restoring from snapshot %s", restored_snapshot_id[:20])
|
||||
|
||||
import modal as _modal
|
||||
|
||||
cred_mounts = []
|
||||
try:
|
||||
from tools.credential_files import (
|
||||
get_credential_file_mounts,
|
||||
iter_skills_files,
|
||||
iter_cache_files,
|
||||
)
|
||||
|
||||
for mount_entry in get_credential_file_mounts():
|
||||
cred_mounts.append(
|
||||
_modal.Mount.from_local_file(
|
||||
mount_entry["host_path"],
|
||||
remote_path=mount_entry["container_path"],
|
||||
)
|
||||
)
|
||||
for entry in iter_skills_files():
|
||||
cred_mounts.append(
|
||||
_modal.Mount.from_local_file(
|
||||
entry["host_path"],
|
||||
remote_path=entry["container_path"],
|
||||
)
|
||||
)
|
||||
cache_files = iter_cache_files()
|
||||
for entry in cache_files:
|
||||
cred_mounts.append(
|
||||
_modal.Mount.from_local_file(
|
||||
entry["host_path"],
|
||||
remote_path=entry["container_path"],
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug("Modal: could not load credential file mounts: %s", e)
|
||||
|
||||
self._worker.start()
|
||||
|
||||
async def _create_sandbox(image_spec: Any):
|
||||
app = await _modal.App.lookup.aio("hermes-agent", create_if_missing=True)
|
||||
create_kwargs = dict(sandbox_kwargs)
|
||||
if cred_mounts:
|
||||
existing_mounts = list(create_kwargs.pop("mounts", []))
|
||||
existing_mounts.extend(cred_mounts)
|
||||
create_kwargs["mounts"] = existing_mounts
|
||||
sandbox = await _modal.Sandbox.create.aio(
|
||||
"sleep", "infinity",
|
||||
image=image_spec,
|
||||
app=app,
|
||||
timeout=int(create_kwargs.pop("timeout", 3600)),
|
||||
**create_kwargs,
|
||||
)
|
||||
return app, sandbox
|
||||
|
||||
try:
|
||||
target_image_spec = restored_snapshot_id or image
|
||||
try:
|
||||
effective_image = _resolve_modal_image(target_image_spec)
|
||||
self._app, self._sandbox = self._worker.run_coroutine(
|
||||
_create_sandbox(effective_image), timeout=300,
|
||||
)
|
||||
except Exception as exc:
|
||||
if not restored_snapshot_id:
|
||||
raise
|
||||
logger.warning(
|
||||
"Modal: failed to restore snapshot %s, retrying with base image: %s",
|
||||
restored_snapshot_id[:20], exc,
|
||||
)
|
||||
_delete_direct_snapshot(self._task_id, restored_snapshot_id)
|
||||
base_image = _resolve_modal_image(image)
|
||||
self._app, self._sandbox = self._worker.run_coroutine(
|
||||
_create_sandbox(base_image), timeout=300,
|
||||
)
|
||||
else:
|
||||
if restored_snapshot_id and restored_from_legacy_key:
|
||||
_store_direct_snapshot(self._task_id, restored_snapshot_id)
|
||||
except Exception:
|
||||
self._worker.stop()
|
||||
raise
|
||||
|
||||
logger.info("Modal: sandbox created (task=%s)", self._task_id)
|
||||
|
||||
self._sync_manager = FileSyncManager(
|
||||
get_files_fn=lambda: iter_sync_files("/root/.hermes"),
|
||||
upload_fn=self._modal_upload,
|
||||
delete_fn=self._modal_delete,
|
||||
bulk_upload_fn=self._modal_bulk_upload,
|
||||
)
|
||||
self._sync_manager.sync(force=True)
|
||||
self.init_session()
|
||||
|
||||
def _modal_upload(self, host_path: str, remote_path: str) -> None:
|
||||
"""Upload a single file via base64 piped through stdin."""
|
||||
content = Path(host_path).read_bytes()
|
||||
b64 = base64.b64encode(content).decode("ascii")
|
||||
container_dir = str(Path(remote_path).parent)
|
||||
cmd = (
|
||||
f"mkdir -p {shlex.quote(container_dir)} && "
|
||||
f"base64 -d > {shlex.quote(remote_path)}"
|
||||
)
|
||||
|
||||
async def _write():
|
||||
proc = await self._sandbox.exec.aio("bash", "-c", cmd)
|
||||
offset = 0
|
||||
chunk_size = self._STDIN_CHUNK_SIZE
|
||||
while offset < len(b64):
|
||||
proc.stdin.write(b64[offset:offset + chunk_size])
|
||||
await proc.stdin.drain.aio()
|
||||
offset += chunk_size
|
||||
proc.stdin.write_eof()
|
||||
await proc.stdin.drain.aio()
|
||||
await proc.wait.aio()
|
||||
|
||||
self._worker.run_coroutine(_write(), timeout=30)
|
||||
|
||||
# Modal SDK stdin buffer limit (legacy server path). The command-router
|
||||
# path allows 16 MB, but we must stay under the smaller 2 MB cap for
|
||||
# compatibility. Chunks are written below this threshold and flushed
|
||||
# individually via drain().
|
||||
_STDIN_CHUNK_SIZE = 1 * 1024 * 1024 # 1 MB — safe for both transport paths
|
||||
|
||||
def _modal_bulk_upload(self, files: list[tuple[str, str]]) -> None:
|
||||
"""Upload many files via tar archive piped through stdin.
|
||||
|
||||
Builds a gzipped tar archive in memory and streams it into a
|
||||
``base64 -d | tar xzf -`` pipeline via the process's stdin,
|
||||
avoiding the Modal SDK's 64 KB ``ARG_MAX_BYTES`` exec-arg limit.
|
||||
"""
|
||||
if not files:
|
||||
return
|
||||
|
||||
buf = io.BytesIO()
|
||||
with tarfile.open(fileobj=buf, mode="w:gz") as tar:
|
||||
for host_path, remote_path in files:
|
||||
tar.add(host_path, arcname=remote_path.lstrip("/"))
|
||||
payload = base64.b64encode(buf.getvalue()).decode("ascii")
|
||||
|
||||
parents = unique_parent_dirs(files)
|
||||
mkdir_part = quoted_mkdir_command(parents)
|
||||
cmd = f"{mkdir_part} && base64 -d | tar xzf - -C /"
|
||||
|
||||
async def _bulk():
|
||||
proc = await self._sandbox.exec.aio("bash", "-c", cmd)
|
||||
|
||||
# Stream payload through stdin in chunks to stay under the
|
||||
# SDK's per-write buffer limit (2 MB legacy / 16 MB router).
|
||||
offset = 0
|
||||
chunk_size = self._STDIN_CHUNK_SIZE
|
||||
while offset < len(payload):
|
||||
proc.stdin.write(payload[offset:offset + chunk_size])
|
||||
await proc.stdin.drain.aio()
|
||||
offset += chunk_size
|
||||
|
||||
proc.stdin.write_eof()
|
||||
await proc.stdin.drain.aio()
|
||||
|
||||
exit_code = await proc.wait.aio()
|
||||
if exit_code != 0:
|
||||
stderr_text = await proc.stderr.read.aio()
|
||||
raise RuntimeError(
|
||||
f"Modal bulk upload failed (exit {exit_code}): {stderr_text}"
|
||||
)
|
||||
|
||||
self._worker.run_coroutine(_bulk(), timeout=120)
|
||||
|
||||
def _modal_delete(self, remote_paths: list[str]) -> None:
|
||||
"""Batch-delete remote files via exec."""
|
||||
rm_cmd = quoted_rm_command(remote_paths)
|
||||
|
||||
async def _rm():
|
||||
proc = await self._sandbox.exec.aio("bash", "-c", rm_cmd)
|
||||
await proc.wait.aio()
|
||||
|
||||
self._worker.run_coroutine(_rm(), timeout=15)
|
||||
|
||||
def _before_execute(self) -> None:
|
||||
"""Sync files to sandbox via FileSyncManager (rate-limited internally)."""
|
||||
self._sync_manager.sync()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Execution
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _run_bash(self, cmd_string: str, *, login: bool = False,
|
||||
timeout: int = 120,
|
||||
stdin_data: str | None = None):
|
||||
"""Return a _ThreadedProcessHandle wrapping an async Modal sandbox exec."""
|
||||
sandbox = self._sandbox
|
||||
worker = self._worker
|
||||
|
||||
def cancel():
|
||||
worker.run_coroutine(sandbox.terminate.aio(), timeout=15)
|
||||
|
||||
def exec_fn() -> tuple[str, int]:
|
||||
async def _do():
|
||||
args = ["bash"]
|
||||
if login:
|
||||
args.extend(["-l", "-c", cmd_string])
|
||||
else:
|
||||
args.extend(["-c", cmd_string])
|
||||
process = await sandbox.exec.aio(*args, timeout=timeout)
|
||||
stdout = await process.stdout.read.aio()
|
||||
stderr = await process.stderr.read.aio()
|
||||
exit_code = await process.wait.aio()
|
||||
if isinstance(stdout, bytes):
|
||||
stdout = stdout.decode("utf-8", errors="replace")
|
||||
if isinstance(stderr, bytes):
|
||||
stderr = stderr.decode("utf-8", errors="replace")
|
||||
output = stdout
|
||||
if stderr:
|
||||
output = f"{stdout}\n{stderr}" if stdout else stderr
|
||||
return output, exit_code
|
||||
|
||||
return worker.run_coroutine(_do(), timeout=timeout + 30)
|
||||
|
||||
return _ThreadedProcessHandle(exec_fn, cancel_fn=cancel)
|
||||
|
||||
def cleanup(self):
|
||||
"""Snapshot the filesystem (if persistent) then stop the sandbox."""
|
||||
if self._sandbox is None:
|
||||
return
|
||||
|
||||
if self._persistent:
|
||||
try:
|
||||
async def _snapshot():
|
||||
img = await self._sandbox.snapshot_filesystem.aio()
|
||||
return img.object_id
|
||||
|
||||
try:
|
||||
snapshot_id = self._worker.run_coroutine(_snapshot(), timeout=60)
|
||||
except Exception:
|
||||
snapshot_id = None
|
||||
|
||||
if snapshot_id:
|
||||
_store_direct_snapshot(self._task_id, snapshot_id)
|
||||
logger.info(
|
||||
"Modal: saved filesystem snapshot %s for task %s",
|
||||
snapshot_id[:20], self._task_id,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Modal: filesystem snapshot failed: %s", e)
|
||||
|
||||
try:
|
||||
self._worker.run_coroutine(self._sandbox.terminate.aio(), timeout=15)
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
self._worker.stop()
|
||||
self._sandbox = None
|
||||
self._app = None
|
||||
@@ -0,0 +1,186 @@
|
||||
"""Shared Hermes-side execution flow for Modal transports.
|
||||
|
||||
This module deliberately stops at the Hermes boundary:
|
||||
- command preparation
|
||||
- cwd/timeout normalization
|
||||
- stdin/sudo shell wrapping
|
||||
- common result shape
|
||||
- interrupt/cancel polling
|
||||
|
||||
Direct Modal and managed Modal keep separate transport logic, persistence, and
|
||||
trust-boundary decisions in their own modules.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shlex
|
||||
import time
|
||||
import uuid
|
||||
from abc import abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from tools.environments.base import BaseEnvironment
|
||||
from tools.interrupt import is_interrupted
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PreparedModalExec:
|
||||
"""Normalized command data passed to a transport-specific exec runner."""
|
||||
|
||||
command: str
|
||||
cwd: str
|
||||
timeout: int
|
||||
stdin_data: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ModalExecStart:
|
||||
"""Transport response after starting an exec."""
|
||||
|
||||
handle: Any | None = None
|
||||
immediate_result: dict | None = None
|
||||
|
||||
|
||||
def wrap_modal_stdin_heredoc(command: str, stdin_data: str) -> str:
|
||||
"""Append stdin as a shell heredoc for transports without stdin piping."""
|
||||
marker = f"HERMES_EOF_{uuid.uuid4().hex[:8]}"
|
||||
while marker in stdin_data:
|
||||
marker = f"HERMES_EOF_{uuid.uuid4().hex[:8]}"
|
||||
return f"{command} << '{marker}'\n{stdin_data}\n{marker}"
|
||||
|
||||
|
||||
def wrap_modal_sudo_pipe(command: str, sudo_stdin: str) -> str:
|
||||
"""Feed sudo via a shell pipe for transports without direct stdin piping."""
|
||||
return f"printf '%s\\n' {shlex.quote(sudo_stdin.rstrip())} | {command}"
|
||||
|
||||
|
||||
class BaseModalExecutionEnvironment(BaseEnvironment):
|
||||
"""Execution flow for the *managed* Modal transport (gateway-owned sandbox).
|
||||
|
||||
This deliberately overrides :meth:`BaseEnvironment.execute` because the
|
||||
tool-gateway handles command preparation, CWD tracking, and env-snapshot
|
||||
management on the server side. The base class's ``_wrap_command`` /
|
||||
``_wait_for_process`` / snapshot machinery does not apply here — the
|
||||
gateway owns that responsibility. See ``ManagedModalEnvironment`` for the
|
||||
concrete subclass.
|
||||
"""
|
||||
|
||||
_stdin_mode = "payload"
|
||||
_poll_interval_seconds = 0.25
|
||||
_client_timeout_grace_seconds: float | None = None
|
||||
_interrupt_output = "[Command interrupted]"
|
||||
_unexpected_error_prefix = "Modal execution error"
|
||||
|
||||
def execute(
|
||||
self,
|
||||
command: str,
|
||||
cwd: str = "",
|
||||
*,
|
||||
timeout: int | None = None,
|
||||
stdin_data: str | None = None,
|
||||
) -> dict:
|
||||
self._before_execute()
|
||||
prepared = self._prepare_modal_exec(
|
||||
command,
|
||||
cwd=cwd,
|
||||
timeout=timeout,
|
||||
stdin_data=stdin_data,
|
||||
)
|
||||
|
||||
try:
|
||||
start = self._start_modal_exec(prepared)
|
||||
except Exception as exc:
|
||||
return self._error_result(f"{self._unexpected_error_prefix}: {exc}")
|
||||
|
||||
if start.immediate_result is not None:
|
||||
return start.immediate_result
|
||||
|
||||
if start.handle is None:
|
||||
return self._error_result(
|
||||
f"{self._unexpected_error_prefix}: transport did not return an exec handle"
|
||||
)
|
||||
|
||||
deadline = None
|
||||
if self._client_timeout_grace_seconds is not None:
|
||||
deadline = time.monotonic() + prepared.timeout + self._client_timeout_grace_seconds
|
||||
|
||||
while True:
|
||||
if is_interrupted():
|
||||
try:
|
||||
self._cancel_modal_exec(start.handle)
|
||||
except Exception:
|
||||
pass
|
||||
return self._result(self._interrupt_output, 130)
|
||||
|
||||
try:
|
||||
result = self._poll_modal_exec(start.handle)
|
||||
except Exception as exc:
|
||||
return self._error_result(f"{self._unexpected_error_prefix}: {exc}")
|
||||
|
||||
if result is not None:
|
||||
return result
|
||||
|
||||
if deadline is not None and time.monotonic() >= deadline:
|
||||
try:
|
||||
self._cancel_modal_exec(start.handle)
|
||||
except Exception:
|
||||
pass
|
||||
return self._timeout_result_for_modal(prepared.timeout)
|
||||
|
||||
time.sleep(self._poll_interval_seconds)
|
||||
|
||||
def _before_execute(self) -> None:
|
||||
"""Hook for backends that need pre-exec sync or validation."""
|
||||
pass
|
||||
|
||||
def _prepare_modal_exec(
|
||||
self,
|
||||
command: str,
|
||||
*,
|
||||
cwd: str = "",
|
||||
timeout: int | None = None,
|
||||
stdin_data: str | None = None,
|
||||
) -> PreparedModalExec:
|
||||
effective_cwd = cwd or self.cwd
|
||||
effective_timeout = timeout or self.timeout
|
||||
|
||||
exec_command = command
|
||||
exec_stdin = stdin_data if self._stdin_mode == "payload" else None
|
||||
if stdin_data is not None and self._stdin_mode == "heredoc":
|
||||
exec_command = wrap_modal_stdin_heredoc(exec_command, stdin_data)
|
||||
|
||||
exec_command, sudo_stdin = self._prepare_command(exec_command)
|
||||
if sudo_stdin is not None:
|
||||
exec_command = wrap_modal_sudo_pipe(exec_command, sudo_stdin)
|
||||
|
||||
return PreparedModalExec(
|
||||
command=exec_command,
|
||||
cwd=effective_cwd,
|
||||
timeout=effective_timeout,
|
||||
stdin_data=exec_stdin,
|
||||
)
|
||||
|
||||
def _result(self, output: str, returncode: int) -> dict:
|
||||
return {
|
||||
"output": output,
|
||||
"returncode": returncode,
|
||||
}
|
||||
|
||||
def _error_result(self, output: str) -> dict:
|
||||
return self._result(output, 1)
|
||||
|
||||
def _timeout_result_for_modal(self, timeout: int) -> dict:
|
||||
return self._result(f"Command timed out after {timeout}s", 124)
|
||||
|
||||
@abstractmethod
|
||||
def _start_modal_exec(self, prepared: PreparedModalExec) -> ModalExecStart:
|
||||
"""Begin a transport-specific exec."""
|
||||
|
||||
@abstractmethod
|
||||
def _poll_modal_exec(self, handle: Any) -> dict | None:
|
||||
"""Return a final result dict when complete, else ``None``."""
|
||||
|
||||
@abstractmethod
|
||||
def _cancel_modal_exec(self, handle: Any) -> None:
|
||||
"""Cancel or terminate the active transport exec."""
|
||||
@@ -0,0 +1,262 @@
|
||||
"""Singularity/Apptainer persistent container environment.
|
||||
|
||||
Security-hardened with --containall, --no-home, capability dropping.
|
||||
Supports configurable resource limits and optional filesystem persistence
|
||||
via writable overlay directories that survive across sessions.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import threading
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from hermes_constants import get_hermes_home
|
||||
from tools.environments.base import (
|
||||
BaseEnvironment,
|
||||
_load_json_store,
|
||||
_popen_bash,
|
||||
_save_json_store,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SNAPSHOT_STORE = get_hermes_home() / "singularity_snapshots.json"
|
||||
|
||||
|
||||
def _find_singularity_executable() -> str:
|
||||
"""Locate the apptainer or singularity CLI binary."""
|
||||
if shutil.which("apptainer"):
|
||||
return "apptainer"
|
||||
if shutil.which("singularity"):
|
||||
return "singularity"
|
||||
raise RuntimeError(
|
||||
"Neither 'apptainer' nor 'singularity' was found in PATH. "
|
||||
"Install Apptainer (https://apptainer.org/docs/admin/main/installation.html) "
|
||||
"or Singularity and ensure the CLI is available."
|
||||
)
|
||||
|
||||
|
||||
def _ensure_singularity_available() -> str:
|
||||
"""Preflight check: resolve the executable and verify it responds."""
|
||||
exe = _find_singularity_executable()
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[exe, "version"], capture_output=True, text=True, timeout=10,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
raise RuntimeError(
|
||||
f"Singularity backend selected but '{exe}' could not be executed."
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
raise RuntimeError(f"'{exe} version' timed out.")
|
||||
|
||||
if result.returncode != 0:
|
||||
stderr = result.stderr.strip()[:200]
|
||||
raise RuntimeError(f"'{exe} version' failed (exit code {result.returncode}): {stderr}")
|
||||
return exe
|
||||
|
||||
|
||||
def _load_snapshots() -> dict:
|
||||
return _load_json_store(_SNAPSHOT_STORE)
|
||||
|
||||
|
||||
def _save_snapshots(data: dict) -> None:
|
||||
_save_json_store(_SNAPSHOT_STORE, data)
|
||||
|
||||
|
||||
def _get_scratch_dir() -> Path:
|
||||
custom_scratch = os.getenv("TERMINAL_SCRATCH_DIR")
|
||||
if custom_scratch:
|
||||
scratch_path = Path(custom_scratch)
|
||||
scratch_path.mkdir(parents=True, exist_ok=True)
|
||||
return scratch_path
|
||||
|
||||
from tools.environments.base import get_sandbox_dir
|
||||
sandbox = get_sandbox_dir() / "singularity"
|
||||
|
||||
scratch = Path("/scratch")
|
||||
if scratch.exists() and os.access(scratch, os.W_OK):
|
||||
user_scratch = scratch / os.getenv("USER", "hermes") / "hermes-agent"
|
||||
user_scratch.mkdir(parents=True, exist_ok=True)
|
||||
logger.info("Using /scratch for sandboxes: %s", user_scratch)
|
||||
return user_scratch
|
||||
|
||||
sandbox.mkdir(parents=True, exist_ok=True)
|
||||
return sandbox
|
||||
|
||||
|
||||
def _get_apptainer_cache_dir() -> Path:
|
||||
cache_dir = os.getenv("APPTAINER_CACHEDIR")
|
||||
if cache_dir:
|
||||
cache_path = Path(cache_dir)
|
||||
cache_path.mkdir(parents=True, exist_ok=True)
|
||||
return cache_path
|
||||
scratch = _get_scratch_dir()
|
||||
cache_path = scratch / ".apptainer"
|
||||
cache_path.mkdir(parents=True, exist_ok=True)
|
||||
return cache_path
|
||||
|
||||
|
||||
_sif_build_lock = threading.Lock()
|
||||
|
||||
|
||||
def _get_or_build_sif(image: str, executable: str = "apptainer") -> str:
|
||||
if image.endswith('.sif') and Path(image).exists():
|
||||
return image
|
||||
if not image.startswith('docker://'):
|
||||
return image
|
||||
|
||||
image_name = image.replace('docker://', '').replace('/', '-').replace(':', '-')
|
||||
cache_dir = _get_apptainer_cache_dir()
|
||||
sif_path = cache_dir / f"{image_name}.sif"
|
||||
|
||||
if sif_path.exists():
|
||||
return str(sif_path)
|
||||
|
||||
with _sif_build_lock:
|
||||
if sif_path.exists():
|
||||
return str(sif_path)
|
||||
|
||||
logger.info("Building SIF image (one-time setup)...")
|
||||
logger.info(" Source: %s", image)
|
||||
logger.info(" Target: %s", sif_path)
|
||||
|
||||
tmp_dir = cache_dir / "tmp"
|
||||
tmp_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
env = os.environ.copy()
|
||||
env["APPTAINER_TMPDIR"] = str(tmp_dir)
|
||||
env["APPTAINER_CACHEDIR"] = str(cache_dir)
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[executable, "build", str(sif_path), image],
|
||||
capture_output=True, text=True, timeout=600, env=env,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
logger.warning("SIF build failed, falling back to docker:// URL")
|
||||
logger.warning(" Error: %s", result.stderr[:500])
|
||||
return image
|
||||
logger.info("SIF image built successfully")
|
||||
return str(sif_path)
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning("SIF build timed out, falling back to docker:// URL")
|
||||
if sif_path.exists():
|
||||
sif_path.unlink()
|
||||
return image
|
||||
except Exception as e:
|
||||
logger.warning("SIF build error: %s, falling back to docker:// URL", e)
|
||||
return image
|
||||
|
||||
|
||||
class SingularityEnvironment(BaseEnvironment):
|
||||
"""Hardened Singularity/Apptainer container with resource limits and persistence.
|
||||
|
||||
Spawn-per-call: every execute() spawns a fresh ``apptainer exec ... bash -c`` process.
|
||||
Session snapshot preserves env vars across calls.
|
||||
CWD persists via in-band stdout markers.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image: str,
|
||||
cwd: str = "~",
|
||||
timeout: int = 60,
|
||||
cpu: float = 0,
|
||||
memory: int = 0,
|
||||
disk: int = 0,
|
||||
persistent_filesystem: bool = False,
|
||||
task_id: str = "default",
|
||||
):
|
||||
super().__init__(cwd=cwd, timeout=timeout)
|
||||
self.executable = _ensure_singularity_available()
|
||||
self.image = _get_or_build_sif(image, self.executable)
|
||||
self.instance_id = f"hermes_{uuid.uuid4().hex[:12]}"
|
||||
self._instance_started = False
|
||||
self._persistent = persistent_filesystem
|
||||
self._task_id = task_id
|
||||
self._overlay_dir: Optional[Path] = None
|
||||
self._cpu = cpu
|
||||
self._memory = memory
|
||||
|
||||
if self._persistent:
|
||||
overlay_base = _get_scratch_dir() / "hermes-overlays"
|
||||
overlay_base.mkdir(parents=True, exist_ok=True)
|
||||
self._overlay_dir = overlay_base / f"overlay-{task_id}"
|
||||
self._overlay_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
self._start_instance()
|
||||
self.init_session()
|
||||
|
||||
def _start_instance(self):
|
||||
cmd = [self.executable, "instance", "start"]
|
||||
cmd.extend(["--containall", "--no-home"])
|
||||
|
||||
if self._persistent and self._overlay_dir:
|
||||
cmd.extend(["--overlay", str(self._overlay_dir)])
|
||||
else:
|
||||
cmd.append("--writable-tmpfs")
|
||||
|
||||
try:
|
||||
from tools.credential_files import get_credential_file_mounts, get_skills_directory_mount
|
||||
for mount_entry in get_credential_file_mounts():
|
||||
cmd.extend(["--bind", f"{mount_entry['host_path']}:{mount_entry['container_path']}:ro"])
|
||||
for skills_mount in get_skills_directory_mount():
|
||||
cmd.extend(["--bind", f"{skills_mount['host_path']}:{skills_mount['container_path']}:ro"])
|
||||
except Exception as e:
|
||||
logger.debug("Singularity: could not load credential/skills mounts: %s", e)
|
||||
|
||||
if self._memory > 0:
|
||||
cmd.extend(["--memory", f"{self._memory}M"])
|
||||
if self._cpu > 0:
|
||||
cmd.extend(["--cpus", str(self._cpu)])
|
||||
|
||||
cmd.extend([str(self.image), self.instance_id])
|
||||
|
||||
try:
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=120)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"Failed to start instance: {result.stderr}")
|
||||
self._instance_started = True
|
||||
logger.info("Singularity instance %s started (persistent=%s)",
|
||||
self.instance_id, self._persistent)
|
||||
except subprocess.TimeoutExpired:
|
||||
raise RuntimeError("Instance start timed out")
|
||||
|
||||
def _run_bash(self, cmd_string: str, *, login: bool = False,
|
||||
timeout: int = 120,
|
||||
stdin_data: str | None = None) -> subprocess.Popen:
|
||||
"""Spawn a bash process inside the Singularity instance."""
|
||||
if not self._instance_started:
|
||||
raise RuntimeError("Singularity instance not started")
|
||||
|
||||
cmd = [self.executable, "exec",
|
||||
f"instance://{self.instance_id}"]
|
||||
if login:
|
||||
cmd.extend(["bash", "-l", "-c", cmd_string])
|
||||
else:
|
||||
cmd.extend(["bash", "-c", cmd_string])
|
||||
|
||||
return _popen_bash(cmd, stdin_data)
|
||||
|
||||
def cleanup(self):
|
||||
"""Stop the instance. If persistent, the overlay dir survives."""
|
||||
if self._instance_started:
|
||||
try:
|
||||
subprocess.run(
|
||||
[self.executable, "instance", "stop", self.instance_id],
|
||||
capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
logger.info("Singularity instance %s stopped", self.instance_id)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to stop Singularity instance %s: %s", self.instance_id, e)
|
||||
self._instance_started = False
|
||||
|
||||
if self._persistent and self._overlay_dir:
|
||||
snapshots = _load_snapshots()
|
||||
snapshots[self._task_id] = str(self._overlay_dir)
|
||||
_save_snapshots(snapshots)
|
||||
@@ -0,0 +1,258 @@
|
||||
"""SSH remote execution environment with ControlMaster connection persistence."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from tools.environments.base import BaseEnvironment, _popen_bash
|
||||
from tools.environments.file_sync import (
|
||||
FileSyncManager,
|
||||
iter_sync_files,
|
||||
quoted_mkdir_command,
|
||||
quoted_rm_command,
|
||||
unique_parent_dirs,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _ensure_ssh_available() -> None:
|
||||
"""Fail fast with a clear error when the SSH client is unavailable."""
|
||||
if not shutil.which("ssh"):
|
||||
raise RuntimeError(
|
||||
"SSH is not installed or not in PATH. Install OpenSSH client: apt install openssh-client"
|
||||
)
|
||||
|
||||
|
||||
class SSHEnvironment(BaseEnvironment):
|
||||
"""Run commands on a remote machine over SSH.
|
||||
|
||||
Spawn-per-call: every execute() spawns a fresh ``ssh ... bash -c`` process.
|
||||
Session snapshot preserves env vars across calls.
|
||||
CWD persists via in-band stdout markers.
|
||||
Uses SSH ControlMaster for connection reuse.
|
||||
"""
|
||||
|
||||
def __init__(self, host: str, user: str, cwd: str = "~",
|
||||
timeout: int = 60, port: int = 22, key_path: str = ""):
|
||||
super().__init__(cwd=cwd, timeout=timeout)
|
||||
self.host = host
|
||||
self.user = user
|
||||
self.port = port
|
||||
self.key_path = key_path
|
||||
|
||||
self.control_dir = Path(tempfile.gettempdir()) / "hermes-ssh"
|
||||
self.control_dir.mkdir(parents=True, exist_ok=True)
|
||||
self.control_socket = self.control_dir / f"{user}@{host}:{port}.sock"
|
||||
_ensure_ssh_available()
|
||||
self._establish_connection()
|
||||
self._remote_home = self._detect_remote_home()
|
||||
|
||||
self._ensure_remote_dirs()
|
||||
self._sync_manager = FileSyncManager(
|
||||
get_files_fn=lambda: iter_sync_files(f"{self._remote_home}/.hermes"),
|
||||
upload_fn=self._scp_upload,
|
||||
delete_fn=self._ssh_delete,
|
||||
bulk_upload_fn=self._ssh_bulk_upload,
|
||||
)
|
||||
self._sync_manager.sync(force=True)
|
||||
|
||||
self.init_session()
|
||||
|
||||
def _build_ssh_command(self, extra_args: list | None = None) -> list:
|
||||
cmd = ["ssh"]
|
||||
cmd.extend(["-o", f"ControlPath={self.control_socket}"])
|
||||
cmd.extend(["-o", "ControlMaster=auto"])
|
||||
cmd.extend(["-o", "ControlPersist=300"])
|
||||
cmd.extend(["-o", "BatchMode=yes"])
|
||||
cmd.extend(["-o", "StrictHostKeyChecking=accept-new"])
|
||||
cmd.extend(["-o", "ConnectTimeout=10"])
|
||||
if self.port != 22:
|
||||
cmd.extend(["-p", str(self.port)])
|
||||
if self.key_path:
|
||||
cmd.extend(["-i", self.key_path])
|
||||
if extra_args:
|
||||
cmd.extend(extra_args)
|
||||
cmd.append(f"{self.user}@{self.host}")
|
||||
return cmd
|
||||
|
||||
def _establish_connection(self):
|
||||
cmd = self._build_ssh_command()
|
||||
cmd.append("echo 'SSH connection established'")
|
||||
try:
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=15)
|
||||
if result.returncode != 0:
|
||||
error_msg = result.stderr.strip() or result.stdout.strip()
|
||||
raise RuntimeError(f"SSH connection failed: {error_msg}")
|
||||
except subprocess.TimeoutExpired:
|
||||
raise RuntimeError(f"SSH connection to {self.user}@{self.host} timed out")
|
||||
|
||||
def _detect_remote_home(self) -> str:
|
||||
"""Detect the remote user's home directory."""
|
||||
try:
|
||||
cmd = self._build_ssh_command()
|
||||
cmd.append("echo $HOME")
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=10)
|
||||
home = result.stdout.strip()
|
||||
if home and result.returncode == 0:
|
||||
logger.debug("SSH: remote home = %s", home)
|
||||
return home
|
||||
except Exception:
|
||||
pass
|
||||
if self.user == "root":
|
||||
return "/root"
|
||||
return f"/home/{self.user}"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# File sync (via FileSyncManager)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _ensure_remote_dirs(self) -> None:
|
||||
"""Create base ~/.hermes directory tree on remote in one SSH call."""
|
||||
base = f"{self._remote_home}/.hermes"
|
||||
dirs = [base, f"{base}/skills", f"{base}/credentials", f"{base}/cache"]
|
||||
cmd = self._build_ssh_command()
|
||||
cmd.append(quoted_mkdir_command(dirs))
|
||||
subprocess.run(cmd, capture_output=True, text=True, timeout=10)
|
||||
|
||||
# _get_sync_files provided via iter_sync_files in FileSyncManager init
|
||||
|
||||
def _scp_upload(self, host_path: str, remote_path: str) -> None:
|
||||
"""Upload a single file via scp over ControlMaster."""
|
||||
parent = str(Path(remote_path).parent)
|
||||
mkdir_cmd = self._build_ssh_command()
|
||||
mkdir_cmd.append(f"mkdir -p {shlex.quote(parent)}")
|
||||
subprocess.run(mkdir_cmd, capture_output=True, text=True, timeout=10)
|
||||
|
||||
scp_cmd = ["scp", "-o", f"ControlPath={self.control_socket}"]
|
||||
if self.port != 22:
|
||||
scp_cmd.extend(["-P", str(self.port)])
|
||||
if self.key_path:
|
||||
scp_cmd.extend(["-i", self.key_path])
|
||||
scp_cmd.extend([host_path, f"{self.user}@{self.host}:{remote_path}"])
|
||||
result = subprocess.run(scp_cmd, capture_output=True, text=True, timeout=30)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"scp failed: {result.stderr.strip()}")
|
||||
|
||||
def _ssh_bulk_upload(self, files: list[tuple[str, str]]) -> None:
|
||||
"""Upload many files in a single tar-over-SSH stream.
|
||||
|
||||
Pipes ``tar c`` on the local side through an SSH connection to
|
||||
``tar x`` on the remote, transferring all files in one TCP stream
|
||||
instead of spawning a subprocess per file. Directory creation is
|
||||
batched into a single ``mkdir -p`` call beforehand.
|
||||
|
||||
Typical improvement: ~580 files goes from O(N) scp round-trips
|
||||
to a single streaming transfer.
|
||||
"""
|
||||
if not files:
|
||||
return
|
||||
|
||||
parents = unique_parent_dirs(files)
|
||||
if parents:
|
||||
cmd = self._build_ssh_command()
|
||||
cmd.append(quoted_mkdir_command(parents))
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"remote mkdir failed: {result.stderr.strip()}")
|
||||
|
||||
# Symlink staging avoids fragile GNU tar --transform rules.
|
||||
with tempfile.TemporaryDirectory(prefix="hermes-ssh-bulk-") as staging:
|
||||
for host_path, remote_path in files:
|
||||
staged = os.path.join(staging, remote_path.lstrip("/"))
|
||||
os.makedirs(os.path.dirname(staged), exist_ok=True)
|
||||
os.symlink(os.path.abspath(host_path), staged)
|
||||
|
||||
tar_cmd = ["tar", "-chf", "-", "-C", staging, "."]
|
||||
ssh_cmd = self._build_ssh_command()
|
||||
ssh_cmd.append("tar xf - -C /")
|
||||
|
||||
tar_proc = subprocess.Popen(
|
||||
tar_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE
|
||||
)
|
||||
try:
|
||||
ssh_proc = subprocess.Popen(
|
||||
ssh_cmd, stdin=tar_proc.stdout, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
except Exception:
|
||||
tar_proc.kill()
|
||||
tar_proc.wait()
|
||||
raise
|
||||
|
||||
# Allow tar_proc to receive SIGPIPE if ssh_proc exits early
|
||||
tar_proc.stdout.close()
|
||||
|
||||
try:
|
||||
_, ssh_stderr = ssh_proc.communicate(timeout=120)
|
||||
# Use communicate() instead of wait() to drain stderr and
|
||||
# avoid deadlock if tar produces more than PIPE_BUF of errors.
|
||||
tar_stderr_raw = b""
|
||||
if tar_proc.poll() is None:
|
||||
_, tar_stderr_raw = tar_proc.communicate(timeout=10)
|
||||
else:
|
||||
tar_stderr_raw = tar_proc.stderr.read() if tar_proc.stderr else b""
|
||||
except subprocess.TimeoutExpired:
|
||||
tar_proc.kill()
|
||||
ssh_proc.kill()
|
||||
tar_proc.wait()
|
||||
ssh_proc.wait()
|
||||
raise RuntimeError("SSH bulk upload timed out")
|
||||
|
||||
if tar_proc.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"tar create failed (rc={tar_proc.returncode}): "
|
||||
f"{tar_stderr_raw.decode(errors='replace').strip()}"
|
||||
)
|
||||
if ssh_proc.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"tar extract over SSH failed (rc={ssh_proc.returncode}): "
|
||||
f"{ssh_stderr.decode(errors='replace').strip()}"
|
||||
)
|
||||
|
||||
logger.debug("SSH: bulk-uploaded %d file(s) via tar pipe", len(files))
|
||||
|
||||
def _ssh_delete(self, remote_paths: list[str]) -> None:
|
||||
"""Batch-delete remote files in one SSH call."""
|
||||
cmd = self._build_ssh_command()
|
||||
cmd.append(quoted_rm_command(remote_paths))
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=10)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"remote rm failed: {result.stderr.strip()}")
|
||||
|
||||
def _before_execute(self) -> None:
|
||||
"""Sync files to remote via FileSyncManager (rate-limited internally)."""
|
||||
self._sync_manager.sync()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Execution
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _run_bash(self, cmd_string: str, *, login: bool = False,
|
||||
timeout: int = 120,
|
||||
stdin_data: str | None = None) -> subprocess.Popen:
|
||||
"""Spawn an SSH process that runs bash on the remote host."""
|
||||
cmd = self._build_ssh_command()
|
||||
if login:
|
||||
cmd.extend(["bash", "-l", "-c", shlex.quote(cmd_string)])
|
||||
else:
|
||||
cmd.extend(["bash", "-c", shlex.quote(cmd_string)])
|
||||
|
||||
return _popen_bash(cmd, stdin_data)
|
||||
|
||||
def cleanup(self):
|
||||
if self.control_socket.exists():
|
||||
try:
|
||||
cmd = ["ssh", "-o", f"ControlPath={self.control_socket}",
|
||||
"-O", "exit", f"{self.user}@{self.host}"]
|
||||
subprocess.run(cmd, capture_output=True, timeout=5)
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
pass
|
||||
try:
|
||||
self.control_socket.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
@@ -0,0 +1,788 @@
|
||||
"""
|
||||
飞书连接器工具 (feishu_tool.py) — v5：同步抽象层
|
||||
|
||||
核心设计：
|
||||
_run_cli(*args, timeout) — 唯一的子进程执行入口，subprocess.run() 同步调用。
|
||||
_run_cli_pty(*args, timeout) — 需要 PTY 的命令（config init），用 script -qc 包装。
|
||||
|
||||
所有工具 handler 都是同步函数，在任何线程上下文（主线程 / SSE worker / executor）
|
||||
都能可靠执行。不再使用 asyncio.run() / create_subprocess_exec。
|
||||
|
||||
后台轮询（device-code polling）使用 threading.Thread + subprocess.run。
|
||||
|
||||
工具清单：
|
||||
feishu_list_profiles — 列出当前用户飞书账号
|
||||
feishu_init_profile — 创建新 profile（PTY），返回授权 URL
|
||||
feishu_auth_domain — Device Flow 授权域
|
||||
feishu_query — 白名单数据操作
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from tools.registry import registry, tool_error, tool_result
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_LARK_CLI = "lark-cli"
|
||||
_URL_FEISHU_RE = re.compile(r"https://\S+feishu\S+")
|
||||
|
||||
|
||||
def _check_lark_cli() -> bool:
|
||||
return shutil.which(_LARK_CLI) is not None
|
||||
|
||||
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
# 通用子进程抽象层
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
|
||||
def _run_cli(*args: str, timeout: int = 30) -> tuple:
|
||||
"""
|
||||
同步执行 lark-cli 命令。任何线程安全。
|
||||
返回 (stdout: str, stderr: str, returncode: int)
|
||||
"""
|
||||
cmd = [_LARK_CLI] + list(args)
|
||||
logger.info("[FeishuTool] exec: %s", " ".join(cmd))
|
||||
try:
|
||||
r = subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
timeout=timeout,
|
||||
text=True,
|
||||
)
|
||||
return r.stdout.strip(), r.stderr.strip(), r.returncode
|
||||
except subprocess.TimeoutExpired:
|
||||
return "", f"命令超时（{timeout}s）", -1
|
||||
except FileNotFoundError:
|
||||
return "", "lark-cli 未安装", -2
|
||||
|
||||
|
||||
def _run_cli_pty(inner_cmd: str, timeout: int = 25) -> str:
|
||||
"""
|
||||
通过 `script -qc` 分配 PTY 执行命令，流式读取 stdout。
|
||||
|
||||
关键设计：用 Popen + readline（非 subprocess.run），找到目标内容立即返回。
|
||||
原因：config init 会阻塞 600s 等用户点授权链接，subprocess.run 必须等进程退出
|
||||
才把 stdout 交回，永远拿不到 URL。改为逐行读，找到 URL 就返回，进程留在后台。
|
||||
"""
|
||||
import threading
|
||||
|
||||
cmd = ["script", "-qc", inner_cmd, "/dev/null"]
|
||||
logger.info("[FeishuTool] exec-pty (stream): %s", inner_cmd)
|
||||
|
||||
lines: list = []
|
||||
done = threading.Event()
|
||||
|
||||
try:
|
||||
proc = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
text=True,
|
||||
bufsize=1,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
return ""
|
||||
|
||||
def _read():
|
||||
try:
|
||||
assert proc.stdout
|
||||
for line in proc.stdout:
|
||||
lines.append(line)
|
||||
logger.debug("[FeishuTool] pty: %r", line[:80])
|
||||
if _URL_FEISHU_RE.search(line):
|
||||
done.set() # URL 已找到，通知主线程返回
|
||||
except Exception as e:
|
||||
logger.debug("[FeishuTool] pty reader: %s", e)
|
||||
finally:
|
||||
done.set() # 进程结束也通知
|
||||
|
||||
t = threading.Thread(target=_read, daemon=True)
|
||||
t.start()
|
||||
done.wait(timeout=timeout) # 找到 URL 或超时就返回
|
||||
# 不 kill 进程——让 config init 继续等用户点击并完成注册
|
||||
|
||||
return "".join(lines)
|
||||
|
||||
|
||||
def _parse_json(text: str) -> Optional[dict]:
|
||||
"""从多行文本中提取第一个 JSON 对象。"""
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
if isinstance(obj, dict):
|
||||
return obj
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
# 尝试整体解析（可能是多行 JSON）
|
||||
try:
|
||||
obj = json.loads(text)
|
||||
if isinstance(obj, dict):
|
||||
return obj
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
# Profile 隔离工具
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
# 飞书元数据存储（wiki/{userId}/.config/feishu.json）
|
||||
# 格式：{ "aliases": { profileName: alias }, "pending": { profileName: timestamp } }
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
|
||||
from tools._user_config import user_config_read, user_config_write
|
||||
|
||||
_PENDING_TTL_SECONDS = 10 * 60 # 10 分钟
|
||||
|
||||
|
||||
def _uid_from_profile(profile_name: str) -> str:
|
||||
"""从 profileName (如 '11c1cece-242_1') 反推 userId 前缀用于查找。
|
||||
实际的 userId 需要通过 _find_full_uid() 匹配。"""
|
||||
return profile_name.rsplit("_", 1)[0] # '11c1cece-242'
|
||||
|
||||
|
||||
def _feishu_config_read(user_id: str) -> dict:
|
||||
"""读取该用户的飞书配置（aliases + pending）"""
|
||||
data = user_config_read(user_id, "feishu")
|
||||
data.setdefault("aliases", {})
|
||||
data.setdefault("pending", {})
|
||||
return data
|
||||
|
||||
|
||||
def _feishu_config_write(user_id: str, data: dict) -> None:
|
||||
"""写入该用户的飞书配置"""
|
||||
user_config_write(user_id, "feishu", data)
|
||||
|
||||
|
||||
def _pending_register(profile_name: str, user_id: str = "") -> None:
|
||||
"""将新建的 profile 写入待定登记表。"""
|
||||
if not user_id:
|
||||
return
|
||||
data = _feishu_config_read(user_id)
|
||||
data["pending"][profile_name] = time.time()
|
||||
_feishu_config_write(user_id, data)
|
||||
|
||||
|
||||
def _pending_remove(profile_name: str, user_id: str = "") -> None:
|
||||
"""从待定登记表中移除（授权完成或已清理）。"""
|
||||
if not user_id:
|
||||
return
|
||||
data = _feishu_config_read(user_id)
|
||||
data["pending"].pop(profile_name, None)
|
||||
_feishu_config_write(user_id, data)
|
||||
|
||||
|
||||
def _pending_get_ts(profile_name: str, user_id: str = "") -> float:
|
||||
"""返回 pending 中该 profile 的创建时间戳，不存在返回 0。"""
|
||||
if not user_id:
|
||||
return 0
|
||||
data = _feishu_config_read(user_id)
|
||||
return data["pending"].get(profile_name, 0)
|
||||
|
||||
|
||||
def _alias_get(profile_name: str, user_id: str = "") -> str:
|
||||
"""返回 profile 的 alias，没有则返回空字符串。"""
|
||||
if not user_id:
|
||||
return ""
|
||||
data = _feishu_config_read(user_id)
|
||||
return data["aliases"].get(profile_name, "")
|
||||
|
||||
|
||||
def _alias_set(profile_name: str, alias: str, user_id: str = "") -> None:
|
||||
"""设置或更新 profile 的 alias。alias 为空时删除条目。"""
|
||||
if not user_id:
|
||||
return
|
||||
data = _feishu_config_read(user_id)
|
||||
if alias:
|
||||
data["aliases"][profile_name] = alias
|
||||
else:
|
||||
data["aliases"].pop(profile_name, None)
|
||||
_feishu_config_write(user_id, data)
|
||||
|
||||
|
||||
def _make_profile_name(user_id: str, index: int = 1) -> str:
|
||||
return f"{user_id[:12]}_{index}"
|
||||
|
||||
|
||||
def _get_all_profiles() -> list:
|
||||
stdout, _, rc = _run_cli("profile", "list", timeout=10)
|
||||
if rc != 0:
|
||||
return []
|
||||
try:
|
||||
parsed = json.loads(stdout)
|
||||
return parsed if isinstance(parsed, list) else []
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return []
|
||||
|
||||
|
||||
def _filter_user_profiles(profiles: list, user_id: str) -> list:
|
||||
prefix = user_id[:12] + "_"
|
||||
result = []
|
||||
for p in profiles:
|
||||
name = p.get("name", p) if isinstance(p, dict) else str(p)
|
||||
if name.startswith(prefix):
|
||||
result.append(p)
|
||||
return result
|
||||
|
||||
|
||||
def _get_profile_name(p) -> str:
|
||||
"""从 profile 条目中提取名字（兼容 str 和 dict 两种格式）。"""
|
||||
if isinstance(p, dict):
|
||||
return p.get("name", "")
|
||||
return str(p)
|
||||
|
||||
|
||||
def _check_profile_token(profile_name: str, user_id: str = "") -> Optional[dict]:
|
||||
"""
|
||||
检查 profile 是否有有效 token。
|
||||
返回 auth status dict（含 expiresAt 等），或 None（无 token / 超时）。
|
||||
自动清理超时未授权的 pending profile。
|
||||
"""
|
||||
stdout, _, rc = _run_cli("auth", "status", "--profile", profile_name, timeout=8)
|
||||
if rc != 0:
|
||||
# 没有 token 或命令失败 → 检查 pending 超时
|
||||
created_at = _pending_get_ts(profile_name, user_id=user_id)
|
||||
if created_at and (time.time() - created_at) > _PENDING_TTL_SECONDS:
|
||||
logger.info("[FeishuTool] pending profile 超时，自动清理: %s", profile_name)
|
||||
_run_cli("profile", "delete", "--name", profile_name, timeout=5)
|
||||
_pending_remove(profile_name, user_id=user_id)
|
||||
return None
|
||||
# 解析 JSON
|
||||
try:
|
||||
data = json.loads(stdout)
|
||||
expires_at = data.get("expiresAt", "")
|
||||
if not expires_at:
|
||||
return None
|
||||
# 判断 tokenStatus
|
||||
token_status = data.get("tokenStatus", "")
|
||||
# 如果是 valid 或 needs_refresh，都认为是有效授权的 profile
|
||||
if token_status not in ["valid", "needs_refresh"]:
|
||||
return None
|
||||
_pending_remove(profile_name, user_id=user_id)
|
||||
return data
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
# ① feishu_list_profiles
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
|
||||
def _feishu_list_profiles_handler(args: dict, **kwargs) -> str:
|
||||
user_id = str(args.get("user_id", "")).strip()
|
||||
if not user_id:
|
||||
return tool_error("user_id 参数必填")
|
||||
|
||||
all_profiles = _get_all_profiles()
|
||||
user_profiles = _filter_user_profiles(all_profiles, user_id)
|
||||
|
||||
# 只返回有有效 token 的 profile（自动清理超时垃圾 profile）
|
||||
active_profiles = []
|
||||
for p in user_profiles:
|
||||
name = _get_profile_name(p)
|
||||
token_info = _check_profile_token(name, user_id=user_id)
|
||||
if token_info:
|
||||
entry = {
|
||||
"name": name,
|
||||
"active": True,
|
||||
"user": token_info.get("identity", ""),
|
||||
"tokenStatus": token_info.get("tokenStatus", "valid"),
|
||||
"expiresAt": token_info.get("expiresAt", ""),
|
||||
"alias": _alias_get(name, user_id=user_id),
|
||||
}
|
||||
active_profiles.append(entry)
|
||||
|
||||
return tool_result(
|
||||
success=True,
|
||||
profiles=active_profiles,
|
||||
count=len(active_profiles),
|
||||
message=(
|
||||
f"找到 {len(active_profiles)} 个已授权的飞书账号。"
|
||||
if active_profiles else "尚未绑定任何飞书账号。"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
registry.register(
|
||||
name="feishu_list_profiles",
|
||||
toolset="connectors",
|
||||
description="列出当前用户已绑定的所有飞书账号。",
|
||||
emoji="📋",
|
||||
check_fn=_check_lark_cli,
|
||||
handler=_feishu_list_profiles_handler,
|
||||
schema={
|
||||
"name": "feishu_list_profiles",
|
||||
"description": "列出当前用户的所有飞书 profile。",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"user_id": {
|
||||
"type": "string",
|
||||
"description": "当前 MindOS 用户 ID（MINDOS_USER_ID）",
|
||||
},
|
||||
},
|
||||
"required": ["user_id"],
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
# ① feishu_rename_profile（设置别名）
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
|
||||
def _feishu_rename_profile_handler(args: dict, **kwargs) -> str:
|
||||
user_id = str(args.get("user_id", "")).strip()
|
||||
profile_name = str(args.get("profile_name", "")).strip()
|
||||
alias = str(args.get("alias", "")).strip()
|
||||
|
||||
if not user_id or not profile_name:
|
||||
return tool_error("user_id 和 profile_name 参数必填")
|
||||
|
||||
# 安全校验：必须属于当前用户
|
||||
prefix = user_id[:12] + "_"
|
||||
if not profile_name.startswith(prefix):
|
||||
return tool_error(f"profile '{profile_name}' 不属于当前用户")
|
||||
|
||||
_alias_set(profile_name, alias, user_id=user_id)
|
||||
return tool_result(
|
||||
success=True,
|
||||
profile_name=profile_name,
|
||||
alias=alias or None,
|
||||
message=f"已{'设置' if alias else '清除'}别名：{profile_name} → {alias or '(无)'}",
|
||||
)
|
||||
|
||||
|
||||
registry.register(
|
||||
name="feishu_rename_profile",
|
||||
toolset="connectors",
|
||||
description="为飞书 profile 设置（或清除）用户友好的别名。",
|
||||
emoji="✏️",
|
||||
check_fn=_check_lark_cli,
|
||||
handler=_feishu_rename_profile_handler,
|
||||
schema={
|
||||
"name": "feishu_rename_profile",
|
||||
"description": "为指定飞书 profile 设置别名（如'企业号'、'个人号'）。alias 传空字符串则清除别名。",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"user_id": {"type": "string", "description": "MindOS 用户 ID"},
|
||||
"profile_name": {"type": "string", "description": "lark-cli profile 名（如 11c1cece-242_1）"},
|
||||
"alias": {"type": "string", "description": "用户友好别名，如'企业号'。传空字符串清除别名"},
|
||||
},
|
||||
"required": ["user_id", "profile_name", "alias"],
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
# ② feishu_init_profile（PTY 模式）
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
|
||||
def _feishu_init_profile_handler(args: dict, **kwargs) -> str:
|
||||
user_id = str(args.get("user_id", "")).strip()
|
||||
if not user_id:
|
||||
return tool_error("user_id 参数必填")
|
||||
|
||||
account_name = str(args.get("account_name", "")).strip()
|
||||
|
||||
all_profiles = _get_all_profiles()
|
||||
user_profiles = _filter_user_profiles(all_profiles, user_id)
|
||||
next_index = len(user_profiles) + 1
|
||||
|
||||
if account_name:
|
||||
profile_name = f"{user_id[:12]}_{account_name}"
|
||||
else:
|
||||
profile_name = _make_profile_name(user_id, next_index)
|
||||
|
||||
# 检查是否已存在
|
||||
all_names = [_get_profile_name(p) for p in all_profiles]
|
||||
if profile_name in all_names:
|
||||
# 如果已存在但仍在 pending（用户之前没完成授权），视为重试
|
||||
token = _check_profile_token(profile_name, user_id=user_id)
|
||||
if token:
|
||||
return tool_result(
|
||||
success=True,
|
||||
profile_name=profile_name,
|
||||
status="already_exists",
|
||||
message=f"Profile '{profile_name}' 已存在且已授权。可直接进行域授权。",
|
||||
)
|
||||
# 没有有效 token → 重新走 init（可能用户上次没完成）
|
||||
|
||||
# 通过 PTY 执行 config init（流式读取，找到 URL 立即返回）
|
||||
inner_cmd = f"{_LARK_CLI} config init --new --name {profile_name}"
|
||||
stdout = _run_cli_pty(inner_cmd, timeout=25)
|
||||
|
||||
# 提取 URL
|
||||
m = _URL_FEISHU_RE.search(stdout)
|
||||
if m:
|
||||
_pending_register(profile_name, user_id=user_id) # 写入待定登记表，启动 10 分钟超时计时
|
||||
return tool_result(
|
||||
success=True,
|
||||
profile_name=profile_name,
|
||||
verification_url=m.group(0),
|
||||
message="请点击链接在飞书中创建应用。完成后告诉我。",
|
||||
)
|
||||
|
||||
return tool_error(
|
||||
"未获取到飞书授权链接。请检查服务器网络。"
|
||||
)
|
||||
|
||||
|
||||
registry.register(
|
||||
name="feishu_init_profile",
|
||||
toolset="connectors",
|
||||
description="创建新飞书 Profile 并返回授权 URL。",
|
||||
emoji="🔗",
|
||||
check_fn=_check_lark_cli,
|
||||
handler=_feishu_init_profile_handler,
|
||||
schema={
|
||||
"name": "feishu_init_profile",
|
||||
"description": (
|
||||
"为当前用户创建新的飞书 profile。返回 verification_url。"
|
||||
"用户完成后再调 feishu_auth_domain 进行域授权。"
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"user_id": {
|
||||
"type": "string",
|
||||
"description": "MINDOS_USER_ID",
|
||||
},
|
||||
"account_name": {
|
||||
"type": "string",
|
||||
"description": "可选别名（如 work、personal）",
|
||||
},
|
||||
},
|
||||
"required": ["user_id"],
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
# ③ feishu_auth_domain（同步 Device Flow）
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
|
||||
_AUTH_DOMAINS = {
|
||||
"approval", "attendance", "base", "calendar", "contact",
|
||||
"docs", "drive", "event", "im", "mail", "minutes",
|
||||
"sheets", "slides", "task", "vc", "wiki", "all",
|
||||
}
|
||||
|
||||
|
||||
def _poll_device_code_bg(profile_name: str, device_code: str, domain: str) -> None:
|
||||
"""后台线程：同步轮询 --device-code 直到用户完成授权。"""
|
||||
stdout, stderr, rc = _run_cli(
|
||||
"auth", "login",
|
||||
"--device-code", device_code,
|
||||
"--json",
|
||||
"--profile", profile_name,
|
||||
timeout=620,
|
||||
)
|
||||
logger.info(
|
||||
"[FeishuTool] poll done %s/%s (rc=%d): %s",
|
||||
profile_name, domain, rc, stdout[:200],
|
||||
)
|
||||
|
||||
|
||||
def _feishu_auth_domain_handler(args: dict, **kwargs) -> str:
|
||||
user_id = str(args.get("user_id", "")).strip()
|
||||
if not user_id:
|
||||
return tool_error("user_id 参数必填")
|
||||
|
||||
domain = str(args.get("domain", "")).strip().lower()
|
||||
if not domain:
|
||||
return tool_error("domain 参数必填")
|
||||
for d in domain.split(","):
|
||||
d = d.strip()
|
||||
if d and d not in _AUTH_DOMAINS:
|
||||
return tool_error(f"不支持的 domain：{d}")
|
||||
|
||||
profile_name = str(args.get("profile_name", "")).strip()
|
||||
if not profile_name:
|
||||
all_profiles = _get_all_profiles()
|
||||
user_profiles = _filter_user_profiles(all_profiles, user_id)
|
||||
if not user_profiles:
|
||||
return tool_error("没有飞书 profile。请先调用 feishu_init_profile。")
|
||||
profile_name = _get_profile_name(user_profiles[0])
|
||||
|
||||
# 同步执行 auth login --no-wait（立即返回 JSON）
|
||||
stdout, stderr, rc = _run_cli(
|
||||
"auth", "login",
|
||||
"--domain", domain,
|
||||
"--json", "--no-wait",
|
||||
"--profile", profile_name,
|
||||
timeout=15,
|
||||
)
|
||||
|
||||
if rc != 0 and not stdout:
|
||||
return tool_error(f"auth 失败（exit {rc}）：{stderr[:200]}")
|
||||
|
||||
data = _parse_json(stdout)
|
||||
|
||||
if not data:
|
||||
return tool_error(
|
||||
f"lark-cli 无 JSON 输出。"
|
||||
f"请确认 profile '{profile_name}' 已初始化。"
|
||||
f"stdout: {stdout[:200]}"
|
||||
)
|
||||
|
||||
# 显式错误
|
||||
if data.get("ok") is False and "error" in data:
|
||||
err = data["error"]
|
||||
if isinstance(err, dict):
|
||||
if err.get("type") == "config":
|
||||
return tool_error(
|
||||
f"Profile '{profile_name}' 未初始化。"
|
||||
"请先调用 feishu_init_profile。"
|
||||
)
|
||||
return tool_error(
|
||||
f"授权失败：{err.get('message', '')}。{err.get('hint', '')}"
|
||||
)
|
||||
return tool_error(f"授权失败：{err}")
|
||||
|
||||
verification_url = data.get("verification_url", "")
|
||||
device_code = data.get("device_code", "")
|
||||
|
||||
if not verification_url:
|
||||
return tool_error(
|
||||
f"授权响应缺少 verification_url。"
|
||||
f"数据：{json.dumps(data, ensure_ascii=False)[:200]}"
|
||||
)
|
||||
|
||||
# 后台线程轮询 device_code
|
||||
if device_code:
|
||||
t = threading.Thread(
|
||||
target=_poll_device_code_bg,
|
||||
args=(profile_name, device_code, domain),
|
||||
daemon=True,
|
||||
)
|
||||
t.start()
|
||||
logger.info("[FeishuTool] polling thread started for %s/%s", profile_name, domain)
|
||||
|
||||
return tool_result(
|
||||
success=True,
|
||||
profile_name=profile_name,
|
||||
domain=domain,
|
||||
verification_url=verification_url,
|
||||
expires_in=data.get("expires_in", 600),
|
||||
message=f"请点击链接授权飞书 {domain} 域。",
|
||||
)
|
||||
|
||||
|
||||
registry.register(
|
||||
name="feishu_auth_domain",
|
||||
toolset="connectors",
|
||||
description="授权飞书域（Device Flow），返回授权链接。",
|
||||
emoji="🔐",
|
||||
check_fn=_check_lark_cli,
|
||||
handler=_feishu_auth_domain_handler,
|
||||
schema={
|
||||
"name": "feishu_auth_domain",
|
||||
"description": (
|
||||
"对指定 profile 的飞书域进行 OAuth 授权。"
|
||||
"返回 verification_url（可点击链接），后台自动轮询等待完成。"
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"user_id": {"type": "string", "description": "MINDOS_USER_ID"},
|
||||
"domain": {"type": "string", "description": "飞书域（calendar/docs/minutes/all）"},
|
||||
"profile_name": {"type": "string", "description": "profile 名（不传则用第一个）"},
|
||||
},
|
||||
"required": ["user_id", "domain"],
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
# ④ feishu_query（白名单数据操作）
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
|
||||
_ALLOWED_ACTIONS: Dict[str, set] = {
|
||||
"calendar": {"+agenda", "+freebusy", "+create"},
|
||||
"docs": {"+search", "+fetch", "+create"},
|
||||
"minutes": {"+search", "+download", "minutes"},
|
||||
"im": {"+send-msg"},
|
||||
"base": {"+search"},
|
||||
"mail": {"+search"},
|
||||
"wiki": {"+search"},
|
||||
"task": {"+list", "+create"},
|
||||
"auth": {"status"},
|
||||
}
|
||||
|
||||
_SAFE_FLAGS = {
|
||||
# calendar / docs
|
||||
"--query", "--page-size", "--start", "--end",
|
||||
"--summary", "--description", "--url",
|
||||
"--folder-token", "--participant-ids", "--owner-ids",
|
||||
"--title",
|
||||
# docs +fetch
|
||||
"--doc", "--limit", "--offset",
|
||||
# minutes
|
||||
"--minute-tokens", "--minute-token",
|
||||
"--url-only", "--output", "--overwrite",
|
||||
# low-level API
|
||||
"--params",
|
||||
# common
|
||||
"--page-token", "--as",
|
||||
}
|
||||
|
||||
|
||||
def _feishu_query_handler(args: dict, **kwargs) -> str:
|
||||
user_id = str(args.get("user_id", "")).strip()
|
||||
if not user_id:
|
||||
return tool_error("user_id 参数必填")
|
||||
|
||||
module = str(args.get("module", "")).strip().lower()
|
||||
action = str(args.get("action", "")).strip()
|
||||
|
||||
if module not in _ALLOWED_ACTIONS:
|
||||
return tool_error(
|
||||
f"不支持的模块：{module}。"
|
||||
f"可选：{', '.join(sorted(_ALLOWED_ACTIONS.keys()))}"
|
||||
)
|
||||
|
||||
allowed = _ALLOWED_ACTIONS[module]
|
||||
if action not in allowed:
|
||||
return tool_error(
|
||||
f"模块 {module} 不允许 '{action}'。可选：{', '.join(sorted(allowed))}"
|
||||
)
|
||||
|
||||
profile_name = str(args.get("profile_name", "")).strip()
|
||||
if not profile_name:
|
||||
all_profiles = _get_all_profiles()
|
||||
user_profiles = _filter_user_profiles(all_profiles, user_id)
|
||||
if not user_profiles:
|
||||
return tool_error("没有飞书 profile。请先绑定飞书账号。")
|
||||
profile_name = _get_profile_name(user_profiles[0])
|
||||
|
||||
extra_args_raw = args.get("extra_args", [])
|
||||
if isinstance(extra_args_raw, str):
|
||||
extra_args_raw = extra_args_raw.split()
|
||||
extra_args = [str(a) for a in extra_args_raw if isinstance(a, str)]
|
||||
|
||||
# 安全检查
|
||||
for a in extra_args:
|
||||
if a.startswith("-") and a not in _SAFE_FLAGS:
|
||||
return tool_error(f"不允许的参数：{a}")
|
||||
|
||||
# 特殊处理：minutes minutes get 需要由工具构造 --params JSON
|
||||
# 避免 LLM 传入时双引号丢失问题
|
||||
if module == "minutes" and action == "minutes" and extra_args and extra_args[0] == "get":
|
||||
# 从 extra_args 中提取 minute_token
|
||||
token = None
|
||||
i = 0
|
||||
cleaned_args = []
|
||||
while i < len(extra_args):
|
||||
a = extra_args[i]
|
||||
if a in ("--minute-token", "--minute-tokens") and i + 1 < len(extra_args):
|
||||
token = extra_args[i + 1].strip()
|
||||
i += 2
|
||||
elif a == "--params" and i + 1 < len(extra_args):
|
||||
# 尝试从 --params 中提取 minute_token
|
||||
try:
|
||||
import json as _json
|
||||
p = _json.loads(extra_args[i + 1])
|
||||
token = p.get("minute_token", token)
|
||||
except Exception:
|
||||
pass
|
||||
i += 2
|
||||
else:
|
||||
cleaned_args.append(a)
|
||||
i += 1
|
||||
|
||||
if not token:
|
||||
return tool_error(
|
||||
"minutes minutes get 需要 minute_token。"
|
||||
"请传入 extra_args=[\"get\", \"--minute-token\", \"TOKEN\"]。"
|
||||
"TOKEN 是飞书妙记 URL 末尾的字符串，如 obcnXXXX。"
|
||||
)
|
||||
|
||||
# 由工具内部构造正确的 --params JSON（确保双引号正确）
|
||||
import json as _json
|
||||
params_json = _json.dumps({"minute_token": token})
|
||||
cli_args = [
|
||||
module, action, "--profile", profile_name,
|
||||
"get", "--params", params_json
|
||||
] + [a for a in cleaned_args if a != "get"]
|
||||
stdout, stderr, rc = _run_cli(*cli_args, timeout=30)
|
||||
if rc != 0:
|
||||
return tool_error(f"执行失败（exit {rc}）：{stderr[:300] or stdout[:300]}")
|
||||
try:
|
||||
data = _json.loads(stdout)
|
||||
return tool_result(success=True, data=data)
|
||||
except (ValueError, _json.JSONDecodeError):
|
||||
return tool_result(success=True, output=stdout[:2000])
|
||||
|
||||
# 同步执行（lark-cli 所有命令默认 JSON 输出，不追加 --json）
|
||||
cli_args = [module, action, "--profile", profile_name] + extra_args
|
||||
stdout, stderr, rc = _run_cli(*cli_args, timeout=30)
|
||||
|
||||
if rc != 0:
|
||||
return tool_error(f"执行失败（exit {rc}）：{stderr[:300] or stdout[:300]}")
|
||||
|
||||
# 尝试 JSON 解析
|
||||
try:
|
||||
data = json.loads(stdout)
|
||||
return tool_result(success=True, data=data)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
if stdout:
|
||||
return tool_result(success=True, output=stdout[:2000])
|
||||
return tool_error(f"无输出。stderr: {stderr[:200]}")
|
||||
|
||||
|
||||
registry.register(
|
||||
name="feishu_query",
|
||||
toolset="connectors",
|
||||
description="执行飞书数据查询（日历/文档/妙记等）。",
|
||||
emoji="📊",
|
||||
check_fn=_check_lark_cli,
|
||||
handler=_feishu_query_handler,
|
||||
schema={
|
||||
"name": "feishu_query",
|
||||
"description": (
|
||||
"执行飞书数据操作。需先完成 init + auth。\n"
|
||||
"示例：\n"
|
||||
" 查日程: module=calendar, action=+agenda\n"
|
||||
" 搜文档: module=docs, action=+search, extra_args=[\"--query\",\"关键词\"]\n"
|
||||
" 查妙记: module=minutes, action=+search\n"
|
||||
" 查授权: module=auth, action=status\n"
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"user_id": {"type": "string", "description": "MINDOS_USER_ID"},
|
||||
"module": {
|
||||
"type": "string",
|
||||
"description": "飞书模块",
|
||||
"enum": sorted(_ALLOWED_ACTIONS.keys()),
|
||||
},
|
||||
"action": {"type": "string", "description": "操作（如 +agenda、+search）"},
|
||||
"profile_name": {"type": "string", "description": "profile 名（不传则用第一个）"},
|
||||
"extra_args": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "额外参数，如 [\"--query\",\"方案\"]",
|
||||
},
|
||||
},
|
||||
"required": ["user_id", "module", "action"],
|
||||
},
|
||||
},
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,799 @@
|
||||
#!/usr/bin/env python3
|
||||
"""File Tools Module - LLM agent file manipulation tools."""
|
||||
|
||||
import errno
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from tools.binary_extensions import has_binary_extension
|
||||
from tools.file_operations import ShellFileOperations
|
||||
from agent.redact import redact_sensitive_text
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
_EXPECTED_WRITE_ERRNOS = {errno.EACCES, errno.EPERM, errno.EROFS}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Read-size guard: cap the character count returned to the model.
|
||||
# We're model-agnostic so we can't count tokens; characters are a safe proxy.
|
||||
# 100K chars ≈ 25–35K tokens across typical tokenisers. Files larger than
|
||||
# this in a single read are a context-window hazard — the model should use
|
||||
# offset+limit to read the relevant section.
|
||||
#
|
||||
# Configurable via config.yaml: file_read_max_chars: 200000
|
||||
# ---------------------------------------------------------------------------
|
||||
_DEFAULT_MAX_READ_CHARS = 100_000
|
||||
_max_read_chars_cached: int | None = None
|
||||
|
||||
|
||||
def _get_max_read_chars() -> int:
|
||||
"""Return the configured max characters per file read.
|
||||
|
||||
Reads ``file_read_max_chars`` from config.yaml on first call, caches
|
||||
the result for the lifetime of the process. Falls back to the
|
||||
built-in default if the config is missing or invalid.
|
||||
"""
|
||||
global _max_read_chars_cached
|
||||
if _max_read_chars_cached is not None:
|
||||
return _max_read_chars_cached
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
cfg = load_config()
|
||||
val = cfg.get("file_read_max_chars")
|
||||
if isinstance(val, (int, float)) and val > 0:
|
||||
_max_read_chars_cached = int(val)
|
||||
return _max_read_chars_cached
|
||||
except Exception:
|
||||
pass
|
||||
_max_read_chars_cached = _DEFAULT_MAX_READ_CHARS
|
||||
return _max_read_chars_cached
|
||||
|
||||
# If the total file size exceeds this AND the caller didn't specify a narrow
|
||||
# range (limit <= 200), we include a hint encouraging targeted reads.
|
||||
_LARGE_FILE_HINT_BYTES = 512_000 # 512 KB
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Device path blocklist — reading these hangs the process (infinite output
|
||||
# or blocking on input). Checked by path only (no I/O).
|
||||
# ---------------------------------------------------------------------------
|
||||
_BLOCKED_DEVICE_PATHS = frozenset({
|
||||
# Infinite output — never reach EOF
|
||||
"/dev/zero", "/dev/random", "/dev/urandom", "/dev/full",
|
||||
# Blocks waiting for input
|
||||
"/dev/stdin", "/dev/tty", "/dev/console",
|
||||
# Nonsensical to read
|
||||
"/dev/stdout", "/dev/stderr",
|
||||
# fd aliases
|
||||
"/dev/fd/0", "/dev/fd/1", "/dev/fd/2",
|
||||
})
|
||||
|
||||
|
||||
def _is_blocked_device(filepath: str) -> bool:
|
||||
"""Return True if the path would hang the process (infinite output or blocking input).
|
||||
|
||||
Uses the *literal* path — no symlink resolution — because the model
|
||||
specifies paths directly and realpath follows symlinks all the way
|
||||
through (e.g. /dev/stdin → /proc/self/fd/0 → /dev/pts/0), defeating
|
||||
the check.
|
||||
"""
|
||||
normalized = os.path.expanduser(filepath)
|
||||
if normalized in _BLOCKED_DEVICE_PATHS:
|
||||
return True
|
||||
# /proc/self/fd/0-2 and /proc/<pid>/fd/0-2 are Linux aliases for stdio
|
||||
if normalized.startswith("/proc/") and normalized.endswith(
|
||||
("/fd/0", "/fd/1", "/fd/2")
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# Paths that file tools should refuse to write to without going through the
|
||||
# terminal tool's approval system. These match prefixes after os.path.realpath.
|
||||
_SENSITIVE_PATH_PREFIXES = (
|
||||
"/etc/", "/boot/", "/usr/lib/systemd/",
|
||||
"/private/etc/", "/private/var/",
|
||||
)
|
||||
_SENSITIVE_EXACT_PATHS = {"/var/run/docker.sock", "/run/docker.sock"}
|
||||
|
||||
|
||||
def _check_sensitive_path(filepath: str) -> str | None:
|
||||
"""Return an error message if the path targets a sensitive system location."""
|
||||
try:
|
||||
resolved = os.path.realpath(os.path.expanduser(filepath))
|
||||
except (OSError, ValueError):
|
||||
resolved = filepath
|
||||
normalized = os.path.normpath(os.path.expanduser(filepath))
|
||||
_err = (
|
||||
f"Refusing to write to sensitive system path: {filepath}\n"
|
||||
"Use the terminal tool with sudo if you need to modify system files."
|
||||
)
|
||||
for prefix in _SENSITIVE_PATH_PREFIXES:
|
||||
if resolved.startswith(prefix) or normalized.startswith(prefix):
|
||||
return _err
|
||||
if resolved in _SENSITIVE_EXACT_PATHS or normalized in _SENSITIVE_EXACT_PATHS:
|
||||
return _err
|
||||
return None
|
||||
|
||||
|
||||
def _is_expected_write_exception(exc: Exception) -> bool:
|
||||
"""Return True for expected write denials that should not hit error logs."""
|
||||
if isinstance(exc, PermissionError):
|
||||
return True
|
||||
if isinstance(exc, OSError) and exc.errno in _EXPECTED_WRITE_ERRNOS:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
_file_ops_lock = threading.Lock()
|
||||
_file_ops_cache: dict = {}
|
||||
|
||||
# Track files read per task to detect re-read loops and deduplicate reads.
|
||||
# Per task_id we store:
|
||||
# "last_key": the key of the most recent read/search call (or None)
|
||||
# "consecutive": how many times that exact call has been repeated in a row
|
||||
# "read_history": set of (path, offset, limit) tuples for get_read_files_summary
|
||||
# "dedup": dict mapping (resolved_path, offset, limit) → mtime float
|
||||
# Used to skip re-reads of unchanged files. Reset on
|
||||
# context compression (the original content is summarised
|
||||
# away so the model needs the full content again).
|
||||
# "read_timestamps": dict mapping resolved_path → modification-time float
|
||||
# recorded when the file was last read (or written) by
|
||||
# this task. Used by write_file and patch to detect
|
||||
# external changes between the agent's read and write.
|
||||
# Updated after successful writes so consecutive edits
|
||||
# by the same task don't trigger false warnings.
|
||||
_read_tracker_lock = threading.Lock()
|
||||
_read_tracker: dict = {}
|
||||
|
||||
|
||||
def _get_file_ops(task_id: str = "default") -> ShellFileOperations:
|
||||
"""Get or create ShellFileOperations for a terminal environment.
|
||||
|
||||
Respects the TERMINAL_ENV setting -- if the task_id doesn't have an
|
||||
environment yet, creates one using the configured backend (local, docker,
|
||||
modal, etc.) rather than always defaulting to local.
|
||||
|
||||
Thread-safe: uses the same per-task creation locks as terminal_tool to
|
||||
prevent duplicate sandbox creation from concurrent tool calls.
|
||||
"""
|
||||
from tools.terminal_tool import (
|
||||
_active_environments, _env_lock, _create_environment,
|
||||
_get_env_config, _last_activity, _start_cleanup_thread,
|
||||
_creation_locks,
|
||||
_creation_locks_lock,
|
||||
)
|
||||
import time
|
||||
|
||||
# Fast path: check cache -- but also verify the underlying environment
|
||||
# is still alive (it may have been killed by the cleanup thread).
|
||||
with _file_ops_lock:
|
||||
cached = _file_ops_cache.get(task_id)
|
||||
if cached is not None:
|
||||
with _env_lock:
|
||||
if task_id in _active_environments:
|
||||
_last_activity[task_id] = time.time()
|
||||
return cached
|
||||
else:
|
||||
# Environment was cleaned up -- invalidate stale cache entry
|
||||
with _file_ops_lock:
|
||||
_file_ops_cache.pop(task_id, None)
|
||||
|
||||
# Need to ensure the environment exists before building file_ops.
|
||||
# Acquire per-task lock so only one thread creates the sandbox.
|
||||
with _creation_locks_lock:
|
||||
if task_id not in _creation_locks:
|
||||
_creation_locks[task_id] = threading.Lock()
|
||||
task_lock = _creation_locks[task_id]
|
||||
|
||||
with task_lock:
|
||||
# Double-check: another thread may have created it while we waited
|
||||
with _env_lock:
|
||||
if task_id in _active_environments:
|
||||
_last_activity[task_id] = time.time()
|
||||
terminal_env = _active_environments[task_id]
|
||||
else:
|
||||
terminal_env = None
|
||||
|
||||
if terminal_env is None:
|
||||
from tools.terminal_tool import _task_env_overrides
|
||||
|
||||
config = _get_env_config()
|
||||
env_type = config["env_type"]
|
||||
overrides = _task_env_overrides.get(task_id, {})
|
||||
|
||||
if env_type == "docker":
|
||||
image = overrides.get("docker_image") or config["docker_image"]
|
||||
elif env_type == "singularity":
|
||||
image = overrides.get("singularity_image") or config["singularity_image"]
|
||||
elif env_type == "modal":
|
||||
image = overrides.get("modal_image") or config["modal_image"]
|
||||
elif env_type == "daytona":
|
||||
image = overrides.get("daytona_image") or config["daytona_image"]
|
||||
else:
|
||||
image = ""
|
||||
|
||||
cwd = overrides.get("cwd") or config["cwd"]
|
||||
logger.info("Creating new %s environment for task %s...", env_type, task_id[:8])
|
||||
|
||||
container_config = None
|
||||
if env_type in ("docker", "singularity", "modal", "daytona"):
|
||||
container_config = {
|
||||
"container_cpu": config.get("container_cpu", 1),
|
||||
"container_memory": config.get("container_memory", 5120),
|
||||
"container_disk": config.get("container_disk", 51200),
|
||||
"container_persistent": config.get("container_persistent", True),
|
||||
"docker_volumes": config.get("docker_volumes", []),
|
||||
}
|
||||
|
||||
ssh_config = None
|
||||
if env_type == "ssh":
|
||||
ssh_config = {
|
||||
"host": config.get("ssh_host", ""),
|
||||
"user": config.get("ssh_user", ""),
|
||||
"port": config.get("ssh_port", 22),
|
||||
"key": config.get("ssh_key", ""),
|
||||
"persistent": config.get("ssh_persistent", False),
|
||||
}
|
||||
|
||||
local_config = None
|
||||
if env_type == "local":
|
||||
local_config = {
|
||||
"persistent": config.get("local_persistent", False),
|
||||
}
|
||||
|
||||
terminal_env = _create_environment(
|
||||
env_type=env_type,
|
||||
image=image,
|
||||
cwd=cwd,
|
||||
timeout=config["timeout"],
|
||||
ssh_config=ssh_config,
|
||||
container_config=container_config,
|
||||
local_config=local_config,
|
||||
task_id=task_id,
|
||||
host_cwd=config.get("host_cwd"),
|
||||
)
|
||||
|
||||
with _env_lock:
|
||||
_active_environments[task_id] = terminal_env
|
||||
_last_activity[task_id] = time.time()
|
||||
|
||||
_start_cleanup_thread()
|
||||
logger.info("%s environment ready for task %s", env_type, task_id[:8])
|
||||
|
||||
# Build file_ops from the (guaranteed live) environment and cache it
|
||||
file_ops = ShellFileOperations(terminal_env)
|
||||
with _file_ops_lock:
|
||||
_file_ops_cache[task_id] = file_ops
|
||||
return file_ops
|
||||
|
||||
|
||||
def clear_file_ops_cache(task_id: str = None):
|
||||
"""Clear the file operations cache."""
|
||||
with _file_ops_lock:
|
||||
if task_id:
|
||||
_file_ops_cache.pop(task_id, None)
|
||||
else:
|
||||
_file_ops_cache.clear()
|
||||
|
||||
|
||||
def read_file_tool(path: str, offset: int = 1, limit: int = 500, task_id: str = "default") -> str:
|
||||
"""Read a file with pagination and line numbers."""
|
||||
try:
|
||||
# ── Device path guard ─────────────────────────────────────────
|
||||
# Block paths that would hang the process (infinite output,
|
||||
# blocking on input). Pure path check — no I/O.
|
||||
if _is_blocked_device(path):
|
||||
return json.dumps({
|
||||
"error": (
|
||||
f"Cannot read '{path}': this is a device file that would "
|
||||
"block or produce infinite output."
|
||||
),
|
||||
})
|
||||
|
||||
_resolved = Path(path).expanduser().resolve()
|
||||
|
||||
# ── Binary file guard ─────────────────────────────────────────
|
||||
# Block binary files by extension (no I/O).
|
||||
if has_binary_extension(str(_resolved)):
|
||||
_ext = _resolved.suffix.lower()
|
||||
return json.dumps({
|
||||
"error": (
|
||||
f"Cannot read binary file '{path}' ({_ext}). "
|
||||
"Use vision_analyze for images, or terminal to inspect binary files."
|
||||
),
|
||||
})
|
||||
|
||||
# ── Hermes internal path guard ────────────────────────────────
|
||||
# Prevent prompt injection via catalog or hub metadata files.
|
||||
from hermes_constants import get_hermes_home as _get_hh
|
||||
_hermes_home = _get_hh().resolve()
|
||||
_blocked_dirs = [
|
||||
_hermes_home / "skills" / ".hub" / "index-cache",
|
||||
_hermes_home / "skills" / ".hub",
|
||||
]
|
||||
for _blocked in _blocked_dirs:
|
||||
try:
|
||||
_resolved.relative_to(_blocked)
|
||||
return json.dumps({
|
||||
"error": (
|
||||
f"Access denied: {path} is an internal Hermes cache file "
|
||||
"and cannot be read directly to prevent prompt injection. "
|
||||
"Use the skills_list or skill_view tools instead."
|
||||
)
|
||||
})
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# ── Dedup check ───────────────────────────────────────────────
|
||||
# If we already read this exact (path, offset, limit) and the
|
||||
# file hasn't been modified since, return a lightweight stub
|
||||
# instead of re-sending the same content. Saves context tokens.
|
||||
resolved_str = str(_resolved)
|
||||
dedup_key = (resolved_str, offset, limit)
|
||||
with _read_tracker_lock:
|
||||
task_data = _read_tracker.setdefault(task_id, {
|
||||
"last_key": None, "consecutive": 0,
|
||||
"read_history": set(), "dedup": {},
|
||||
})
|
||||
cached_mtime = task_data.get("dedup", {}).get(dedup_key)
|
||||
|
||||
if cached_mtime is not None:
|
||||
try:
|
||||
current_mtime = os.path.getmtime(resolved_str)
|
||||
if current_mtime == cached_mtime:
|
||||
return json.dumps({
|
||||
"content": (
|
||||
"File unchanged since last read. The content from "
|
||||
"the earlier read_file result in this conversation is "
|
||||
"still current — refer to that instead of re-reading."
|
||||
),
|
||||
"path": path,
|
||||
"dedup": True,
|
||||
}, ensure_ascii=False)
|
||||
except OSError:
|
||||
pass # stat failed — fall through to full read
|
||||
|
||||
# ── Perform the read ──────────────────────────────────────────
|
||||
file_ops = _get_file_ops(task_id)
|
||||
result = file_ops.read_file(path, offset, limit)
|
||||
result_dict = result.to_dict()
|
||||
|
||||
# ── Character-count guard ─────────────────────────────────────
|
||||
# We're model-agnostic so we can't count tokens; characters are
|
||||
# the best proxy we have. If the read produced an unreasonable
|
||||
# amount of content, reject it and tell the model to narrow down.
|
||||
# Note: we check the formatted content (with line-number prefixes),
|
||||
# not the raw file size, because that's what actually enters context.
|
||||
# Check BEFORE redaction to avoid expensive regex on huge content.
|
||||
content_len = len(result.content or "")
|
||||
file_size = result_dict.get("file_size", 0)
|
||||
max_chars = _get_max_read_chars()
|
||||
if content_len > max_chars:
|
||||
total_lines = result_dict.get("total_lines", "unknown")
|
||||
return json.dumps({
|
||||
"error": (
|
||||
f"Read produced {content_len:,} characters which exceeds "
|
||||
f"the safety limit ({max_chars:,} chars). "
|
||||
"Use offset and limit to read a smaller range. "
|
||||
f"The file has {total_lines} lines total."
|
||||
),
|
||||
"path": path,
|
||||
"total_lines": total_lines,
|
||||
"file_size": file_size,
|
||||
}, ensure_ascii=False)
|
||||
|
||||
# ── Redact secrets (after guard check to skip oversized content) ──
|
||||
if result.content:
|
||||
result.content = redact_sensitive_text(result.content)
|
||||
result_dict["content"] = result.content
|
||||
|
||||
# Large-file hint: if the file is big and the caller didn't ask
|
||||
# for a narrow window, nudge toward targeted reads.
|
||||
if (file_size and file_size > _LARGE_FILE_HINT_BYTES
|
||||
and limit > 200
|
||||
and result_dict.get("truncated")):
|
||||
result_dict.setdefault("_hint", (
|
||||
f"This file is large ({file_size:,} bytes). "
|
||||
"Consider reading only the section you need with offset and limit "
|
||||
"to keep context usage efficient."
|
||||
))
|
||||
|
||||
# ── Track for consecutive-loop detection ──────────────────────
|
||||
read_key = ("read", path, offset, limit)
|
||||
with _read_tracker_lock:
|
||||
# Ensure "dedup" key exists (backward compat with old tracker state)
|
||||
if "dedup" not in task_data:
|
||||
task_data["dedup"] = {}
|
||||
task_data["read_history"].add((path, offset, limit))
|
||||
if task_data["last_key"] == read_key:
|
||||
task_data["consecutive"] += 1
|
||||
else:
|
||||
task_data["last_key"] = read_key
|
||||
task_data["consecutive"] = 1
|
||||
count = task_data["consecutive"]
|
||||
|
||||
# Store mtime at read time for two purposes:
|
||||
# 1. Dedup: skip identical re-reads of unchanged files.
|
||||
# 2. Staleness: warn on write/patch if the file changed since
|
||||
# the agent last read it (external edit, concurrent agent, etc.).
|
||||
try:
|
||||
_mtime_now = os.path.getmtime(resolved_str)
|
||||
task_data["dedup"][dedup_key] = _mtime_now
|
||||
task_data.setdefault("read_timestamps", {})[resolved_str] = _mtime_now
|
||||
except OSError:
|
||||
pass # Can't stat — skip tracking for this entry
|
||||
|
||||
if count >= 4:
|
||||
# Hard block: stop returning content to break the loop
|
||||
return json.dumps({
|
||||
"error": (
|
||||
f"BLOCKED: You have read this exact file region {count} times in a row. "
|
||||
"The content has NOT changed. You already have this information. "
|
||||
"STOP re-reading and proceed with your task."
|
||||
),
|
||||
"path": path,
|
||||
"already_read": count,
|
||||
}, ensure_ascii=False)
|
||||
elif count >= 3:
|
||||
result_dict["_warning"] = (
|
||||
f"You have read this exact file region {count} times consecutively. "
|
||||
"The content has not changed since your last read. Use the information you already have. "
|
||||
"If you are stuck in a loop, stop reading and proceed with writing or responding."
|
||||
)
|
||||
|
||||
return json.dumps(result_dict, ensure_ascii=False)
|
||||
except Exception as e:
|
||||
return tool_error(str(e))
|
||||
|
||||
|
||||
|
||||
|
||||
def reset_file_dedup(task_id: str = None):
|
||||
"""Clear the deduplication cache for file reads.
|
||||
|
||||
Called after context compression — the original read content has been
|
||||
summarised away, so the model needs the full content if it reads the
|
||||
same file again. Without this, reads after compression would return
|
||||
a "file unchanged" stub pointing at content that no longer exists in
|
||||
context.
|
||||
|
||||
Call with a task_id to clear just that task, or without to clear all.
|
||||
"""
|
||||
with _read_tracker_lock:
|
||||
if task_id:
|
||||
task_data = _read_tracker.get(task_id)
|
||||
if task_data and "dedup" in task_data:
|
||||
task_data["dedup"].clear()
|
||||
else:
|
||||
for task_data in _read_tracker.values():
|
||||
if "dedup" in task_data:
|
||||
task_data["dedup"].clear()
|
||||
|
||||
|
||||
def notify_other_tool_call(task_id: str = "default"):
|
||||
"""Reset consecutive read/search counter for a task.
|
||||
|
||||
Called by the tool dispatcher (model_tools.py) whenever a tool OTHER
|
||||
than read_file / search_files is executed. This ensures we only warn
|
||||
or block on *truly consecutive* repeated reads — if the agent does
|
||||
anything else in between (write, patch, terminal, etc.) the counter
|
||||
resets and the next read is treated as fresh.
|
||||
"""
|
||||
with _read_tracker_lock:
|
||||
task_data = _read_tracker.get(task_id)
|
||||
if task_data:
|
||||
task_data["last_key"] = None
|
||||
task_data["consecutive"] = 0
|
||||
|
||||
|
||||
def _update_read_timestamp(filepath: str, task_id: str) -> None:
|
||||
"""Record the file's current modification time after a successful write.
|
||||
|
||||
Called after write_file and patch so that consecutive edits by the
|
||||
same task don't trigger false staleness warnings — each write
|
||||
refreshes the stored timestamp to match the file's new state.
|
||||
"""
|
||||
try:
|
||||
resolved = str(Path(filepath).expanduser().resolve())
|
||||
current_mtime = os.path.getmtime(resolved)
|
||||
except (OSError, ValueError):
|
||||
return
|
||||
with _read_tracker_lock:
|
||||
task_data = _read_tracker.get(task_id)
|
||||
if task_data is not None:
|
||||
task_data.setdefault("read_timestamps", {})[resolved] = current_mtime
|
||||
|
||||
|
||||
def _check_file_staleness(filepath: str, task_id: str) -> str | None:
|
||||
"""Check whether a file was modified since the agent last read it.
|
||||
|
||||
Returns a warning string if the file is stale (mtime changed since
|
||||
the last read_file call for this task), or None if the file is fresh
|
||||
or was never read. Does not block — the write still proceeds.
|
||||
"""
|
||||
try:
|
||||
resolved = str(Path(filepath).expanduser().resolve())
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
with _read_tracker_lock:
|
||||
task_data = _read_tracker.get(task_id)
|
||||
if not task_data:
|
||||
return None
|
||||
read_mtime = task_data.get("read_timestamps", {}).get(resolved)
|
||||
if read_mtime is None:
|
||||
return None # File was never read — nothing to compare against
|
||||
try:
|
||||
current_mtime = os.path.getmtime(resolved)
|
||||
except OSError:
|
||||
return None # Can't stat — file may have been deleted, let write handle it
|
||||
if current_mtime != read_mtime:
|
||||
return (
|
||||
f"Warning: {filepath} was modified since you last read it "
|
||||
"(external edit or concurrent agent). The content you read may be "
|
||||
"stale. Consider re-reading the file to verify before writing."
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def write_file_tool(path: str, content: str, task_id: str = "default") -> str:
|
||||
"""Write content to a file."""
|
||||
sensitive_err = _check_sensitive_path(path)
|
||||
if sensitive_err:
|
||||
return tool_error(sensitive_err)
|
||||
try:
|
||||
stale_warning = _check_file_staleness(path, task_id)
|
||||
file_ops = _get_file_ops(task_id)
|
||||
result = file_ops.write_file(path, content)
|
||||
result_dict = result.to_dict()
|
||||
if stale_warning:
|
||||
result_dict["_warning"] = stale_warning
|
||||
# Refresh the stored timestamp so consecutive writes by this
|
||||
# task don't trigger false staleness warnings.
|
||||
_update_read_timestamp(path, task_id)
|
||||
return json.dumps(result_dict, ensure_ascii=False)
|
||||
except Exception as e:
|
||||
if _is_expected_write_exception(e):
|
||||
logger.debug("write_file expected denial: %s: %s", type(e).__name__, e)
|
||||
else:
|
||||
logger.error("write_file error: %s: %s", type(e).__name__, e, exc_info=True)
|
||||
return tool_error(str(e))
|
||||
|
||||
|
||||
def patch_tool(mode: str = "replace", path: str = None, old_string: str = None,
|
||||
new_string: str = None, replace_all: bool = False, patch: str = None,
|
||||
task_id: str = "default") -> str:
|
||||
"""Patch a file using replace mode or V4A patch format."""
|
||||
# Check sensitive paths for both replace (explicit path) and V4A patch (extract paths)
|
||||
_paths_to_check = []
|
||||
if path:
|
||||
_paths_to_check.append(path)
|
||||
if mode == "patch" and patch:
|
||||
import re as _re
|
||||
for _m in _re.finditer(r'^\*\*\*\s+(?:Update|Add|Delete)\s+File:\s*(.+)$', patch, _re.MULTILINE):
|
||||
_paths_to_check.append(_m.group(1).strip())
|
||||
for _p in _paths_to_check:
|
||||
sensitive_err = _check_sensitive_path(_p)
|
||||
if sensitive_err:
|
||||
return tool_error(sensitive_err)
|
||||
try:
|
||||
# Check staleness for all files this patch will touch.
|
||||
stale_warnings = []
|
||||
for _p in _paths_to_check:
|
||||
_sw = _check_file_staleness(_p, task_id)
|
||||
if _sw:
|
||||
stale_warnings.append(_sw)
|
||||
|
||||
file_ops = _get_file_ops(task_id)
|
||||
|
||||
if mode == "replace":
|
||||
if not path:
|
||||
return tool_error("path required")
|
||||
if old_string is None or new_string is None:
|
||||
return tool_error("old_string and new_string required")
|
||||
result = file_ops.patch_replace(path, old_string, new_string, replace_all)
|
||||
elif mode == "patch":
|
||||
if not patch:
|
||||
return tool_error("patch content required")
|
||||
result = file_ops.patch_v4a(patch)
|
||||
else:
|
||||
return tool_error(f"Unknown mode: {mode}")
|
||||
|
||||
result_dict = result.to_dict()
|
||||
if stale_warnings:
|
||||
result_dict["_warning"] = stale_warnings[0] if len(stale_warnings) == 1 else " | ".join(stale_warnings)
|
||||
# Refresh stored timestamps for all successfully-patched paths so
|
||||
# consecutive edits by this task don't trigger false warnings.
|
||||
if not result_dict.get("error"):
|
||||
for _p in _paths_to_check:
|
||||
_update_read_timestamp(_p, task_id)
|
||||
result_json = json.dumps(result_dict, ensure_ascii=False)
|
||||
# Hint when old_string not found — saves iterations where the agent
|
||||
# retries with stale content instead of re-reading the file.
|
||||
if result_dict.get("error") and "Could not find" in str(result_dict["error"]):
|
||||
result_json += "\n\n[Hint: old_string not found. Use read_file to verify the current content, or search_files to locate the text.]"
|
||||
return result_json
|
||||
except Exception as e:
|
||||
return tool_error(str(e))
|
||||
|
||||
|
||||
def search_tool(pattern: str, target: str = "content", path: str = ".",
|
||||
file_glob: str = None, limit: int = 50, offset: int = 0,
|
||||
output_mode: str = "content", context: int = 0,
|
||||
task_id: str = "default") -> str:
|
||||
"""Search for content or files."""
|
||||
try:
|
||||
# Track searches to detect *consecutive* repeated search loops.
|
||||
# Include pagination args so users can page through truncated
|
||||
# results without tripping the repeated-search guard.
|
||||
search_key = (
|
||||
"search",
|
||||
pattern,
|
||||
target,
|
||||
str(path),
|
||||
file_glob or "",
|
||||
limit,
|
||||
offset,
|
||||
)
|
||||
with _read_tracker_lock:
|
||||
task_data = _read_tracker.setdefault(task_id, {
|
||||
"last_key": None, "consecutive": 0, "read_history": set(),
|
||||
})
|
||||
if task_data["last_key"] == search_key:
|
||||
task_data["consecutive"] += 1
|
||||
else:
|
||||
task_data["last_key"] = search_key
|
||||
task_data["consecutive"] = 1
|
||||
count = task_data["consecutive"]
|
||||
|
||||
if count >= 4:
|
||||
return json.dumps({
|
||||
"error": (
|
||||
f"BLOCKED: You have run this exact search {count} times in a row. "
|
||||
"The results have NOT changed. You already have this information. "
|
||||
"STOP re-searching and proceed with your task."
|
||||
),
|
||||
"pattern": pattern,
|
||||
"already_searched": count,
|
||||
}, ensure_ascii=False)
|
||||
|
||||
file_ops = _get_file_ops(task_id)
|
||||
result = file_ops.search(
|
||||
pattern=pattern, path=path, target=target, file_glob=file_glob,
|
||||
limit=limit, offset=offset, output_mode=output_mode, context=context
|
||||
)
|
||||
if hasattr(result, 'matches'):
|
||||
for m in result.matches:
|
||||
if hasattr(m, 'content') and m.content:
|
||||
m.content = redact_sensitive_text(m.content)
|
||||
result_dict = result.to_dict()
|
||||
|
||||
if count >= 3:
|
||||
result_dict["_warning"] = (
|
||||
f"You have run this exact search {count} times consecutively. "
|
||||
"The results have not changed. Use the information you already have."
|
||||
)
|
||||
|
||||
result_json = json.dumps(result_dict, ensure_ascii=False)
|
||||
# Hint when results were truncated — explicit next offset is clearer
|
||||
# than relying on the model to infer it from total_count vs match count.
|
||||
if result_dict.get("truncated"):
|
||||
next_offset = offset + limit
|
||||
result_json += f"\n\n[Hint: Results truncated. Use offset={next_offset} to see more, or narrow with a more specific pattern or file_glob.]"
|
||||
return result_json
|
||||
except Exception as e:
|
||||
return tool_error(str(e))
|
||||
|
||||
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Schemas + Registry
|
||||
# ---------------------------------------------------------------------------
|
||||
from tools.registry import registry, tool_error
|
||||
|
||||
|
||||
def _check_file_reqs():
|
||||
"""Lazy wrapper to avoid circular import with tools/__init__.py."""
|
||||
from tools import check_file_requirements
|
||||
return check_file_requirements()
|
||||
|
||||
READ_FILE_SCHEMA = {
|
||||
"name": "read_file",
|
||||
"description": "Read a text file with line numbers and pagination. Use this instead of cat/head/tail in terminal. Output format: 'LINE_NUM|CONTENT'. Suggests similar filenames if not found. Use offset and limit for large files. Reads exceeding ~100K characters are rejected; use offset and limit to read specific sections of large files. NOTE: Cannot read images or binary files — use vision_analyze for images.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"path": {"type": "string", "description": "Path to the file to read (absolute, relative, or ~/path)"},
|
||||
"offset": {"type": "integer", "description": "Line number to start reading from (1-indexed, default: 1)", "default": 1, "minimum": 1},
|
||||
"limit": {"type": "integer", "description": "Maximum number of lines to read (default: 500, max: 2000)", "default": 500, "maximum": 2000}
|
||||
},
|
||||
"required": ["path"]
|
||||
}
|
||||
}
|
||||
|
||||
WRITE_FILE_SCHEMA = {
|
||||
"name": "write_file",
|
||||
"description": "Write content to a file, completely replacing existing content. Use this instead of echo/cat heredoc in terminal. Creates parent directories automatically. OVERWRITES the entire file — use 'patch' for targeted edits.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"path": {"type": "string", "description": "Path to the file to write (will be created if it doesn't exist, overwritten if it does)"},
|
||||
"content": {"type": "string", "description": "Complete content to write to the file"}
|
||||
},
|
||||
"required": ["path", "content"]
|
||||
}
|
||||
}
|
||||
|
||||
PATCH_SCHEMA = {
|
||||
"name": "patch",
|
||||
"description": "Targeted find-and-replace edits in files. Use this instead of sed/awk in terminal. Uses fuzzy matching (9 strategies) so minor whitespace/indentation differences won't break it. Returns a unified diff. Auto-runs syntax checks after editing.\n\nReplace mode (default): find a unique string and replace it.\nPatch mode: apply V4A multi-file patches for bulk changes.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"mode": {"type": "string", "enum": ["replace", "patch"], "description": "Edit mode: 'replace' for targeted find-and-replace, 'patch' for V4A multi-file patches", "default": "replace"},
|
||||
"path": {"type": "string", "description": "File path to edit (required for 'replace' mode)"},
|
||||
"old_string": {"type": "string", "description": "Text to find in the file (required for 'replace' mode). Must be unique in the file unless replace_all=true. Include enough surrounding context to ensure uniqueness."},
|
||||
"new_string": {"type": "string", "description": "Replacement text (required for 'replace' mode). Can be empty string to delete the matched text."},
|
||||
"replace_all": {"type": "boolean", "description": "Replace all occurrences instead of requiring a unique match (default: false)", "default": False},
|
||||
"patch": {"type": "string", "description": "V4A format patch content (required for 'patch' mode). Format:\n*** Begin Patch\n*** Update File: path/to/file\n@@ context hint @@\n context line\n-removed line\n+added line\n*** End Patch"}
|
||||
},
|
||||
"required": ["mode"]
|
||||
}
|
||||
}
|
||||
|
||||
SEARCH_FILES_SCHEMA = {
|
||||
"name": "search_files",
|
||||
"description": "Search file contents or find files by name. Use this instead of grep/rg/find/ls in terminal. Ripgrep-backed, faster than shell equivalents.\n\nContent search (target='content'): Regex search inside files. Output modes: full matches with line numbers, file paths only, or match counts.\n\nFile search (target='files'): Find files by glob pattern (e.g., '*.py', '*config*'). Also use this instead of ls — results sorted by modification time.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"pattern": {"type": "string", "description": "Regex pattern for content search, or glob pattern (e.g., '*.py') for file search"},
|
||||
"target": {"type": "string", "enum": ["content", "files"], "description": "'content' searches inside file contents, 'files' searches for files by name", "default": "content"},
|
||||
"path": {"type": "string", "description": "Directory or file to search in (default: current working directory)", "default": "."},
|
||||
"file_glob": {"type": "string", "description": "Filter files by pattern in grep mode (e.g., '*.py' to only search Python files)"},
|
||||
"limit": {"type": "integer", "description": "Maximum number of results to return (default: 50)", "default": 50},
|
||||
"offset": {"type": "integer", "description": "Skip first N results for pagination (default: 0)", "default": 0},
|
||||
"output_mode": {"type": "string", "enum": ["content", "files_only", "count"], "description": "Output format for grep mode: 'content' shows matching lines with line numbers, 'files_only' lists file paths, 'count' shows match counts per file", "default": "content"},
|
||||
"context": {"type": "integer", "description": "Number of context lines before and after each match (grep mode only)", "default": 0}
|
||||
},
|
||||
"required": ["pattern"]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def _handle_read_file(args, **kw):
|
||||
tid = kw.get("task_id") or "default"
|
||||
return read_file_tool(path=args.get("path", ""), offset=args.get("offset", 1), limit=args.get("limit", 500), task_id=tid)
|
||||
|
||||
|
||||
def _handle_write_file(args, **kw):
|
||||
tid = kw.get("task_id") or "default"
|
||||
return write_file_tool(path=args.get("path", ""), content=args.get("content", ""), task_id=tid)
|
||||
|
||||
|
||||
def _handle_patch(args, **kw):
|
||||
tid = kw.get("task_id") or "default"
|
||||
return patch_tool(
|
||||
mode=args.get("mode", "replace"), path=args.get("path"),
|
||||
old_string=args.get("old_string"), new_string=args.get("new_string"),
|
||||
replace_all=args.get("replace_all", False), patch=args.get("patch"), task_id=tid)
|
||||
|
||||
|
||||
def _handle_search_files(args, **kw):
|
||||
tid = kw.get("task_id") or "default"
|
||||
target_map = {"grep": "content", "find": "files"}
|
||||
raw_target = args.get("target", "content")
|
||||
target = target_map.get(raw_target, raw_target)
|
||||
return search_tool(
|
||||
pattern=args.get("pattern", ""), target=target, path=args.get("path", "."),
|
||||
file_glob=args.get("file_glob"), limit=args.get("limit", 50), offset=args.get("offset", 0),
|
||||
output_mode=args.get("output_mode", "content"), context=args.get("context", 0), task_id=tid)
|
||||
|
||||
|
||||
registry.register(name="read_file", toolset="file", schema=READ_FILE_SCHEMA, handler=_handle_read_file, check_fn=_check_file_reqs, emoji="📖", max_result_size_chars=float('inf'))
|
||||
registry.register(name="write_file", toolset="file", schema=WRITE_FILE_SCHEMA, handler=_handle_write_file, check_fn=_check_file_reqs, emoji="✍️", max_result_size_chars=100_000)
|
||||
registry.register(name="patch", toolset="file", schema=PATCH_SCHEMA, handler=_handle_patch, check_fn=_check_file_reqs, emoji="🔧", max_result_size_chars=100_000)
|
||||
registry.register(name="search_files", toolset="file", schema=SEARCH_FILES_SCHEMA, handler=_handle_search_files, check_fn=_check_file_reqs, emoji="🔎", max_result_size_chars=100_000)
|
||||
@@ -0,0 +1,566 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Fuzzy Matching Module for File Operations
|
||||
|
||||
Implements a multi-strategy matching chain to robustly find and replace text,
|
||||
accommodating variations in whitespace, indentation, and escaping common
|
||||
in LLM-generated code.
|
||||
|
||||
The 8-strategy chain (inspired by OpenCode), tried in order:
|
||||
1. Exact match - Direct string comparison
|
||||
2. Line-trimmed - Strip leading/trailing whitespace per line
|
||||
3. Whitespace normalized - Collapse multiple spaces/tabs to single space
|
||||
4. Indentation flexible - Ignore indentation differences entirely
|
||||
5. Escape normalized - Convert \\n literals to actual newlines
|
||||
6. Trimmed boundary - Trim first/last line whitespace only
|
||||
7. Block anchor - Match first+last lines, use similarity for middle
|
||||
8. Context-aware - 50% line similarity threshold
|
||||
|
||||
Multi-occurrence matching is handled via the replace_all flag.
|
||||
|
||||
Usage:
|
||||
from tools.fuzzy_match import fuzzy_find_and_replace
|
||||
|
||||
new_content, match_count, strategy, error = fuzzy_find_and_replace(
|
||||
content="def foo():\\n pass",
|
||||
old_string="def foo():",
|
||||
new_string="def bar():",
|
||||
replace_all=False
|
||||
)
|
||||
"""
|
||||
|
||||
import re
|
||||
from typing import Tuple, Optional, List, Callable
|
||||
from difflib import SequenceMatcher
|
||||
|
||||
UNICODE_MAP = {
|
||||
"\u201c": '"', "\u201d": '"', # smart double quotes
|
||||
"\u2018": "'", "\u2019": "'", # smart single quotes
|
||||
"\u2014": "--", "\u2013": "-", # em/en dashes
|
||||
"\u2026": "...", "\u00a0": " ", # ellipsis and non-breaking space
|
||||
}
|
||||
|
||||
def _unicode_normalize(text: str) -> str:
|
||||
"""Normalizes Unicode characters to their standard ASCII equivalents."""
|
||||
for char, repl in UNICODE_MAP.items():
|
||||
text = text.replace(char, repl)
|
||||
return text
|
||||
|
||||
|
||||
def fuzzy_find_and_replace(content: str, old_string: str, new_string: str,
|
||||
replace_all: bool = False) -> Tuple[str, int, Optional[str], Optional[str]]:
|
||||
"""
|
||||
Find and replace text using a chain of increasingly fuzzy matching strategies.
|
||||
|
||||
Args:
|
||||
content: The file content to search in
|
||||
old_string: The text to find
|
||||
new_string: The replacement text
|
||||
replace_all: If True, replace all occurrences; if False, require uniqueness
|
||||
|
||||
Returns:
|
||||
Tuple of (new_content, match_count, strategy_name, error_message)
|
||||
- If successful: (modified_content, number_of_replacements, strategy_used, None)
|
||||
- If failed: (original_content, 0, None, error_description)
|
||||
"""
|
||||
if not old_string:
|
||||
return content, 0, None, "old_string cannot be empty"
|
||||
|
||||
if old_string == new_string:
|
||||
return content, 0, None, "old_string and new_string are identical"
|
||||
|
||||
# Try each matching strategy in order
|
||||
strategies: List[Tuple[str, Callable]] = [
|
||||
("exact", _strategy_exact),
|
||||
("line_trimmed", _strategy_line_trimmed),
|
||||
("whitespace_normalized", _strategy_whitespace_normalized),
|
||||
("indentation_flexible", _strategy_indentation_flexible),
|
||||
("escape_normalized", _strategy_escape_normalized),
|
||||
("trimmed_boundary", _strategy_trimmed_boundary),
|
||||
("unicode_normalized", _strategy_unicode_normalized),
|
||||
("block_anchor", _strategy_block_anchor),
|
||||
("context_aware", _strategy_context_aware),
|
||||
]
|
||||
|
||||
for strategy_name, strategy_fn in strategies:
|
||||
matches = strategy_fn(content, old_string)
|
||||
|
||||
if matches:
|
||||
# Found matches with this strategy
|
||||
if len(matches) > 1 and not replace_all:
|
||||
return content, 0, None, (
|
||||
f"Found {len(matches)} matches for old_string. "
|
||||
f"Provide more context to make it unique, or use replace_all=True."
|
||||
)
|
||||
|
||||
# Perform replacement
|
||||
new_content = _apply_replacements(content, matches, new_string)
|
||||
return new_content, len(matches), strategy_name, None
|
||||
|
||||
# No strategy found a match
|
||||
return content, 0, None, "Could not find a match for old_string in the file"
|
||||
|
||||
|
||||
def _apply_replacements(content: str, matches: List[Tuple[int, int]], new_string: str) -> str:
|
||||
"""
|
||||
Apply replacements at the given positions.
|
||||
|
||||
Args:
|
||||
content: Original content
|
||||
matches: List of (start, end) positions to replace
|
||||
new_string: Replacement text
|
||||
|
||||
Returns:
|
||||
Content with replacements applied
|
||||
"""
|
||||
# Sort matches by position (descending) to replace from end to start
|
||||
# This preserves positions of earlier matches
|
||||
sorted_matches = sorted(matches, key=lambda x: x[0], reverse=True)
|
||||
|
||||
result = content
|
||||
for start, end in sorted_matches:
|
||||
result = result[:start] + new_string + result[end:]
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Matching Strategies
|
||||
# =============================================================================
|
||||
|
||||
def _strategy_exact(content: str, pattern: str) -> List[Tuple[int, int]]:
|
||||
"""Strategy 1: Exact string match."""
|
||||
matches = []
|
||||
start = 0
|
||||
while True:
|
||||
pos = content.find(pattern, start)
|
||||
if pos == -1:
|
||||
break
|
||||
matches.append((pos, pos + len(pattern)))
|
||||
start = pos + 1
|
||||
return matches
|
||||
|
||||
|
||||
def _strategy_line_trimmed(content: str, pattern: str) -> List[Tuple[int, int]]:
|
||||
"""
|
||||
Strategy 2: Match with line-by-line whitespace trimming.
|
||||
|
||||
Strips leading/trailing whitespace from each line before matching.
|
||||
"""
|
||||
# Normalize pattern and content by trimming each line
|
||||
pattern_lines = [line.strip() for line in pattern.split('\n')]
|
||||
pattern_normalized = '\n'.join(pattern_lines)
|
||||
|
||||
content_lines = content.split('\n')
|
||||
content_normalized_lines = [line.strip() for line in content_lines]
|
||||
|
||||
# Build mapping from normalized positions back to original positions
|
||||
return _find_normalized_matches(
|
||||
content, content_lines, content_normalized_lines,
|
||||
pattern, pattern_normalized
|
||||
)
|
||||
|
||||
|
||||
def _strategy_whitespace_normalized(content: str, pattern: str) -> List[Tuple[int, int]]:
|
||||
"""
|
||||
Strategy 3: Collapse multiple whitespace to single space.
|
||||
"""
|
||||
def normalize(s):
|
||||
# Collapse multiple spaces/tabs to single space, preserve newlines
|
||||
return re.sub(r'[ \t]+', ' ', s)
|
||||
|
||||
pattern_normalized = normalize(pattern)
|
||||
content_normalized = normalize(content)
|
||||
|
||||
# Find in normalized, map back to original
|
||||
matches_in_normalized = _strategy_exact(content_normalized, pattern_normalized)
|
||||
|
||||
if not matches_in_normalized:
|
||||
return []
|
||||
|
||||
# Map positions back to original content
|
||||
return _map_normalized_positions(content, content_normalized, matches_in_normalized)
|
||||
|
||||
|
||||
def _strategy_indentation_flexible(content: str, pattern: str) -> List[Tuple[int, int]]:
|
||||
"""
|
||||
Strategy 4: Ignore indentation differences entirely.
|
||||
|
||||
Strips all leading whitespace from lines before matching.
|
||||
"""
|
||||
content_lines = content.split('\n')
|
||||
content_stripped_lines = [line.lstrip() for line in content_lines]
|
||||
pattern_lines = [line.lstrip() for line in pattern.split('\n')]
|
||||
|
||||
return _find_normalized_matches(
|
||||
content, content_lines, content_stripped_lines,
|
||||
pattern, '\n'.join(pattern_lines)
|
||||
)
|
||||
|
||||
|
||||
def _strategy_escape_normalized(content: str, pattern: str) -> List[Tuple[int, int]]:
|
||||
"""
|
||||
Strategy 5: Convert escape sequences to actual characters.
|
||||
|
||||
Handles \\n -> newline, \\t -> tab, etc.
|
||||
"""
|
||||
def unescape(s):
|
||||
# Convert common escape sequences
|
||||
return s.replace('\\n', '\n').replace('\\t', '\t').replace('\\r', '\r')
|
||||
|
||||
pattern_unescaped = unescape(pattern)
|
||||
|
||||
if pattern_unescaped == pattern:
|
||||
# No escapes to convert, skip this strategy
|
||||
return []
|
||||
|
||||
return _strategy_exact(content, pattern_unescaped)
|
||||
|
||||
|
||||
def _strategy_trimmed_boundary(content: str, pattern: str) -> List[Tuple[int, int]]:
|
||||
"""
|
||||
Strategy 6: Trim whitespace from first and last lines only.
|
||||
|
||||
Useful when the pattern boundaries have whitespace differences.
|
||||
"""
|
||||
pattern_lines = pattern.split('\n')
|
||||
if not pattern_lines:
|
||||
return []
|
||||
|
||||
# Trim only first and last lines
|
||||
pattern_lines[0] = pattern_lines[0].strip()
|
||||
if len(pattern_lines) > 1:
|
||||
pattern_lines[-1] = pattern_lines[-1].strip()
|
||||
|
||||
modified_pattern = '\n'.join(pattern_lines)
|
||||
|
||||
content_lines = content.split('\n')
|
||||
|
||||
# Search through content for matching block
|
||||
matches = []
|
||||
pattern_line_count = len(pattern_lines)
|
||||
|
||||
for i in range(len(content_lines) - pattern_line_count + 1):
|
||||
block_lines = content_lines[i:i + pattern_line_count]
|
||||
|
||||
# Trim first and last of this block
|
||||
check_lines = block_lines.copy()
|
||||
check_lines[0] = check_lines[0].strip()
|
||||
if len(check_lines) > 1:
|
||||
check_lines[-1] = check_lines[-1].strip()
|
||||
|
||||
if '\n'.join(check_lines) == modified_pattern:
|
||||
# Found match - calculate original positions
|
||||
start_pos, end_pos = _calculate_line_positions(
|
||||
content_lines, i, i + pattern_line_count, len(content)
|
||||
)
|
||||
matches.append((start_pos, end_pos))
|
||||
|
||||
return matches
|
||||
|
||||
|
||||
def _build_orig_to_norm_map(original: str) -> List[int]:
|
||||
"""Build a list mapping each original character index to its normalized index.
|
||||
|
||||
Because UNICODE_MAP replacements may expand characters (e.g. em-dash → '--',
|
||||
ellipsis → '...'), the normalised string can be longer than the original.
|
||||
This map lets us convert positions in the normalised string back to the
|
||||
corresponding positions in the original string.
|
||||
|
||||
Returns a list of length ``len(original) + 1``; entry ``i`` is the
|
||||
normalised index that character ``i`` maps to.
|
||||
"""
|
||||
result: List[int] = []
|
||||
norm_pos = 0
|
||||
for char in original:
|
||||
result.append(norm_pos)
|
||||
repl = UNICODE_MAP.get(char)
|
||||
norm_pos += len(repl) if repl is not None else 1
|
||||
result.append(norm_pos) # sentinel: one past the last character
|
||||
return result
|
||||
|
||||
|
||||
def _map_positions_norm_to_orig(
|
||||
orig_to_norm: List[int],
|
||||
norm_matches: List[Tuple[int, int]],
|
||||
) -> List[Tuple[int, int]]:
|
||||
"""Convert (start, end) positions in the normalised string to original positions."""
|
||||
# Invert the map: norm_pos -> first original position with that norm_pos
|
||||
norm_to_orig_start: dict[int, int] = {}
|
||||
for orig_pos, norm_pos in enumerate(orig_to_norm[:-1]):
|
||||
if norm_pos not in norm_to_orig_start:
|
||||
norm_to_orig_start[norm_pos] = orig_pos
|
||||
|
||||
results: List[Tuple[int, int]] = []
|
||||
orig_len = len(orig_to_norm) - 1 # number of original characters
|
||||
|
||||
for norm_start, norm_end in norm_matches:
|
||||
if norm_start not in norm_to_orig_start:
|
||||
continue
|
||||
orig_start = norm_to_orig_start[norm_start]
|
||||
|
||||
# Walk forward until orig_to_norm[orig_end] >= norm_end
|
||||
orig_end = orig_start
|
||||
while orig_end < orig_len and orig_to_norm[orig_end] < norm_end:
|
||||
orig_end += 1
|
||||
|
||||
results.append((orig_start, orig_end))
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def _strategy_unicode_normalized(content: str, pattern: str) -> List[Tuple[int, int]]:
|
||||
"""Strategy 7: Unicode normalisation.
|
||||
|
||||
Normalises smart quotes, em/en-dashes, ellipsis, and non-breaking spaces
|
||||
to their ASCII equivalents in both *content* and *pattern*, then runs
|
||||
exact and line_trimmed matching on the normalised copies.
|
||||
|
||||
Positions are mapped back to the *original* string via
|
||||
``_build_orig_to_norm_map`` — necessary because some UNICODE_MAP
|
||||
replacements expand a single character into multiple ASCII characters,
|
||||
making a naïve position copy incorrect.
|
||||
"""
|
||||
# Normalize both sides. Either the content or the pattern (or both) may
|
||||
# carry unicode variants — e.g. content has an em-dash that should match
|
||||
# the LLM's ASCII '--', or vice-versa. Skip only when neither changes.
|
||||
norm_pattern = _unicode_normalize(pattern)
|
||||
norm_content = _unicode_normalize(content)
|
||||
if norm_content == content and norm_pattern == pattern:
|
||||
return []
|
||||
|
||||
norm_matches = _strategy_exact(norm_content, norm_pattern)
|
||||
if not norm_matches:
|
||||
norm_matches = _strategy_line_trimmed(norm_content, norm_pattern)
|
||||
|
||||
if not norm_matches:
|
||||
return []
|
||||
|
||||
orig_to_norm = _build_orig_to_norm_map(content)
|
||||
return _map_positions_norm_to_orig(orig_to_norm, norm_matches)
|
||||
|
||||
|
||||
def _strategy_block_anchor(content: str, pattern: str) -> List[Tuple[int, int]]:
|
||||
"""
|
||||
Strategy 8: Match by anchoring on first and last lines.
|
||||
Adjusted with permissive thresholds and unicode normalization.
|
||||
"""
|
||||
# Normalize both strings for comparison while keeping original content for offset calculation
|
||||
norm_pattern = _unicode_normalize(pattern)
|
||||
norm_content = _unicode_normalize(content)
|
||||
|
||||
pattern_lines = norm_pattern.split('\n')
|
||||
if len(pattern_lines) < 2:
|
||||
return []
|
||||
|
||||
first_line = pattern_lines[0].strip()
|
||||
last_line = pattern_lines[-1].strip()
|
||||
|
||||
# Use normalized lines for matching logic
|
||||
norm_content_lines = norm_content.split('\n')
|
||||
# BUT use original lines for calculating start/end positions to prevent index shift
|
||||
orig_content_lines = content.split('\n')
|
||||
|
||||
pattern_line_count = len(pattern_lines)
|
||||
|
||||
potential_matches = []
|
||||
for i in range(len(norm_content_lines) - pattern_line_count + 1):
|
||||
if (norm_content_lines[i].strip() == first_line and
|
||||
norm_content_lines[i + pattern_line_count - 1].strip() == last_line):
|
||||
potential_matches.append(i)
|
||||
|
||||
matches = []
|
||||
candidate_count = len(potential_matches)
|
||||
|
||||
# Thresholding logic: 0.50 for unique matches, 0.70 for multiple candidates.
|
||||
# Previous values (0.10 / 0.30) were dangerously loose — a 10% middle-section
|
||||
# similarity could match completely unrelated blocks.
|
||||
threshold = 0.50 if candidate_count == 1 else 0.70
|
||||
|
||||
for i in potential_matches:
|
||||
if pattern_line_count <= 2:
|
||||
similarity = 1.0
|
||||
else:
|
||||
# Compare normalized middle sections
|
||||
content_middle = '\n'.join(norm_content_lines[i+1:i+pattern_line_count-1])
|
||||
pattern_middle = '\n'.join(pattern_lines[1:-1])
|
||||
similarity = SequenceMatcher(None, content_middle, pattern_middle).ratio()
|
||||
|
||||
if similarity >= threshold:
|
||||
# Calculate positions using ORIGINAL lines to ensure correct character offsets in the file
|
||||
start_pos, end_pos = _calculate_line_positions(
|
||||
orig_content_lines, i, i + pattern_line_count, len(content)
|
||||
)
|
||||
matches.append((start_pos, end_pos))
|
||||
|
||||
return matches
|
||||
|
||||
|
||||
def _strategy_context_aware(content: str, pattern: str) -> List[Tuple[int, int]]:
|
||||
"""
|
||||
Strategy 9: Line-by-line similarity with 50% threshold.
|
||||
|
||||
Finds blocks where at least 50% of lines have high similarity.
|
||||
"""
|
||||
pattern_lines = pattern.split('\n')
|
||||
content_lines = content.split('\n')
|
||||
|
||||
if not pattern_lines:
|
||||
return []
|
||||
|
||||
matches = []
|
||||
pattern_line_count = len(pattern_lines)
|
||||
|
||||
for i in range(len(content_lines) - pattern_line_count + 1):
|
||||
block_lines = content_lines[i:i + pattern_line_count]
|
||||
|
||||
# Calculate line-by-line similarity
|
||||
high_similarity_count = 0
|
||||
for p_line, c_line in zip(pattern_lines, block_lines):
|
||||
sim = SequenceMatcher(None, p_line.strip(), c_line.strip()).ratio()
|
||||
if sim >= 0.80:
|
||||
high_similarity_count += 1
|
||||
|
||||
# Need at least 50% of lines to have high similarity
|
||||
if high_similarity_count >= len(pattern_lines) * 0.5:
|
||||
start_pos, end_pos = _calculate_line_positions(
|
||||
content_lines, i, i + pattern_line_count, len(content)
|
||||
)
|
||||
matches.append((start_pos, end_pos))
|
||||
|
||||
return matches
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Helper Functions
|
||||
# =============================================================================
|
||||
|
||||
def _calculate_line_positions(content_lines: List[str], start_line: int,
|
||||
end_line: int, content_length: int) -> Tuple[int, int]:
|
||||
"""Calculate start and end character positions from line indices.
|
||||
|
||||
Args:
|
||||
content_lines: List of lines (without newlines)
|
||||
start_line: Starting line index (0-based)
|
||||
end_line: Ending line index (exclusive, 0-based)
|
||||
content_length: Total length of the original content string
|
||||
|
||||
Returns:
|
||||
Tuple of (start_pos, end_pos) in the original content
|
||||
"""
|
||||
start_pos = sum(len(line) + 1 for line in content_lines[:start_line])
|
||||
end_pos = sum(len(line) + 1 for line in content_lines[:end_line]) - 1
|
||||
if end_pos >= content_length:
|
||||
end_pos = content_length
|
||||
return start_pos, end_pos
|
||||
|
||||
|
||||
def _find_normalized_matches(content: str, content_lines: List[str],
|
||||
content_normalized_lines: List[str],
|
||||
pattern: str, pattern_normalized: str) -> List[Tuple[int, int]]:
|
||||
"""
|
||||
Find matches in normalized content and map back to original positions.
|
||||
|
||||
Args:
|
||||
content: Original content string
|
||||
content_lines: Original content split by lines
|
||||
content_normalized_lines: Normalized content lines
|
||||
pattern: Original pattern
|
||||
pattern_normalized: Normalized pattern
|
||||
|
||||
Returns:
|
||||
List of (start, end) positions in the original content
|
||||
"""
|
||||
pattern_norm_lines = pattern_normalized.split('\n')
|
||||
num_pattern_lines = len(pattern_norm_lines)
|
||||
|
||||
matches = []
|
||||
|
||||
for i in range(len(content_normalized_lines) - num_pattern_lines + 1):
|
||||
# Check if this block matches
|
||||
block = '\n'.join(content_normalized_lines[i:i + num_pattern_lines])
|
||||
|
||||
if block == pattern_normalized:
|
||||
# Found a match - calculate original positions
|
||||
start_pos, end_pos = _calculate_line_positions(
|
||||
content_lines, i, i + num_pattern_lines, len(content)
|
||||
)
|
||||
matches.append((start_pos, end_pos))
|
||||
|
||||
return matches
|
||||
|
||||
|
||||
def _map_normalized_positions(original: str, normalized: str,
|
||||
normalized_matches: List[Tuple[int, int]]) -> List[Tuple[int, int]]:
|
||||
"""
|
||||
Map positions from normalized string back to original.
|
||||
|
||||
This is a best-effort mapping that works for whitespace normalization.
|
||||
"""
|
||||
if not normalized_matches:
|
||||
return []
|
||||
|
||||
# Build character mapping from normalized to original
|
||||
orig_to_norm = [] # orig_to_norm[i] = position in normalized
|
||||
|
||||
orig_idx = 0
|
||||
norm_idx = 0
|
||||
|
||||
while orig_idx < len(original) and norm_idx < len(normalized):
|
||||
if original[orig_idx] == normalized[norm_idx]:
|
||||
orig_to_norm.append(norm_idx)
|
||||
orig_idx += 1
|
||||
norm_idx += 1
|
||||
elif original[orig_idx] in ' \t' and normalized[norm_idx] == ' ':
|
||||
# Original has space/tab, normalized collapsed to space
|
||||
orig_to_norm.append(norm_idx)
|
||||
orig_idx += 1
|
||||
# Don't advance norm_idx yet - wait until all whitespace consumed
|
||||
if orig_idx < len(original) and original[orig_idx] not in ' \t':
|
||||
norm_idx += 1
|
||||
elif original[orig_idx] in ' \t':
|
||||
# Extra whitespace in original
|
||||
orig_to_norm.append(norm_idx)
|
||||
orig_idx += 1
|
||||
else:
|
||||
# Mismatch - shouldn't happen with our normalization
|
||||
orig_to_norm.append(norm_idx)
|
||||
orig_idx += 1
|
||||
|
||||
# Fill remaining
|
||||
while orig_idx < len(original):
|
||||
orig_to_norm.append(len(normalized))
|
||||
orig_idx += 1
|
||||
|
||||
# Reverse mapping: for each normalized position, find original range
|
||||
norm_to_orig_start = {}
|
||||
norm_to_orig_end = {}
|
||||
|
||||
for orig_pos, norm_pos in enumerate(orig_to_norm):
|
||||
if norm_pos not in norm_to_orig_start:
|
||||
norm_to_orig_start[norm_pos] = orig_pos
|
||||
norm_to_orig_end[norm_pos] = orig_pos
|
||||
|
||||
# Map matches
|
||||
original_matches = []
|
||||
for norm_start, norm_end in normalized_matches:
|
||||
# Find original start
|
||||
if norm_start in norm_to_orig_start:
|
||||
orig_start = norm_to_orig_start[norm_start]
|
||||
else:
|
||||
# Find nearest
|
||||
orig_start = min(i for i, n in enumerate(orig_to_norm) if n >= norm_start)
|
||||
|
||||
# Find original end
|
||||
if norm_end - 1 in norm_to_orig_end:
|
||||
orig_end = norm_to_orig_end[norm_end - 1] + 1
|
||||
else:
|
||||
orig_end = orig_start + (norm_end - norm_start)
|
||||
|
||||
# Expand to include trailing whitespace that was normalized
|
||||
while orig_end < len(original) and original[orig_end] in ' \t':
|
||||
orig_end += 1
|
||||
|
||||
original_matches.append((orig_start, min(orig_end, len(original))))
|
||||
|
||||
return original_matches
|
||||
@@ -0,0 +1,513 @@
|
||||
"""Home Assistant tool for controlling smart home devices via REST API.
|
||||
|
||||
Registers four LLM-callable tools:
|
||||
- ``ha_list_entities`` -- list/filter entities by domain or area
|
||||
- ``ha_get_state`` -- get detailed state of a single entity
|
||||
- ``ha_list_services`` -- list available services (actions) per domain
|
||||
- ``ha_call_service`` -- call a HA service (turn_on, turn_off, set_temperature, etc.)
|
||||
|
||||
Authentication uses a Long-Lived Access Token via ``HASS_TOKEN`` env var.
|
||||
The HA instance URL is read from ``HASS_URL`` (default: http://homeassistant.local:8123).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Configuration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Kept for backward compatibility (e.g. test monkeypatching); prefer _get_config().
|
||||
_HASS_URL: str = ""
|
||||
_HASS_TOKEN: str = ""
|
||||
|
||||
|
||||
def _get_config():
|
||||
"""Return (hass_url, hass_token) from env vars at call time."""
|
||||
return (
|
||||
(_HASS_URL or os.getenv("HASS_URL", "http://homeassistant.local:8123")).rstrip("/"),
|
||||
_HASS_TOKEN or os.getenv("HASS_TOKEN", ""),
|
||||
)
|
||||
|
||||
# Regex for valid HA entity_id format (e.g. "light.living_room", "sensor.temperature_1")
|
||||
_ENTITY_ID_RE = re.compile(r"^[a-z_][a-z0-9_]*\.[a-z0-9_]+$")
|
||||
|
||||
# Regex for valid HA service/domain names (e.g. "light", "turn_on", "shell_command").
|
||||
# Only lowercase ASCII letters, digits, and underscores — no slashes, dots, or
|
||||
# other characters that could allow path traversal in URL construction.
|
||||
# The domain and service are interpolated into /api/services/{domain}/{service},
|
||||
# so allowing arbitrary strings would enable SSRF via path traversal
|
||||
# (e.g. domain="../../api/config") or blocked-domain bypass
|
||||
# (e.g. domain="shell_command/../light").
|
||||
_SERVICE_NAME_RE = re.compile(r"^[a-z][a-z0-9_]*$")
|
||||
|
||||
# Service domains blocked for security -- these allow arbitrary code/command
|
||||
# execution on the HA host or enable SSRF attacks on the local network.
|
||||
# HA provides zero service-level access control; all safety must be in our layer.
|
||||
_BLOCKED_DOMAINS = frozenset({
|
||||
"shell_command", # arbitrary shell commands as root in HA container
|
||||
"command_line", # sensors/switches that execute shell commands
|
||||
"python_script", # sandboxed but can escalate via hass.services.call()
|
||||
"pyscript", # scripting integration with broader access
|
||||
"hassio", # addon control, host shutdown/reboot, stdin to containers
|
||||
"rest_command", # HTTP requests from HA server (SSRF vector)
|
||||
})
|
||||
|
||||
|
||||
def _get_headers(token: str = "") -> Dict[str, str]:
|
||||
"""Return authorization headers for HA REST API."""
|
||||
if not token:
|
||||
_, token = _get_config()
|
||||
return {
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Async helpers (called from sync handlers via run_until_complete)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _filter_and_summarize(
|
||||
states: list,
|
||||
domain: Optional[str] = None,
|
||||
area: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Filter raw HA states by domain/area and return a compact summary."""
|
||||
if domain:
|
||||
states = [s for s in states if s.get("entity_id", "").startswith(f"{domain}.")]
|
||||
|
||||
if area:
|
||||
area_lower = area.lower()
|
||||
states = [
|
||||
s for s in states
|
||||
if area_lower in (s.get("attributes", {}).get("friendly_name", "") or "").lower()
|
||||
or area_lower in (s.get("attributes", {}).get("area", "") or "").lower()
|
||||
]
|
||||
|
||||
entities = []
|
||||
for s in states:
|
||||
entities.append({
|
||||
"entity_id": s["entity_id"],
|
||||
"state": s["state"],
|
||||
"friendly_name": s.get("attributes", {}).get("friendly_name", ""),
|
||||
})
|
||||
|
||||
return {"count": len(entities), "entities": entities}
|
||||
|
||||
|
||||
async def _async_list_entities(
|
||||
domain: Optional[str] = None,
|
||||
area: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Fetch entity states from HA and optionally filter by domain/area."""
|
||||
import aiohttp
|
||||
|
||||
hass_url, hass_token = _get_config()
|
||||
url = f"{hass_url}/api/states"
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get(url, headers=_get_headers(hass_token), timeout=aiohttp.ClientTimeout(total=15)) as resp:
|
||||
resp.raise_for_status()
|
||||
states = await resp.json()
|
||||
|
||||
return _filter_and_summarize(states, domain, area)
|
||||
|
||||
|
||||
async def _async_get_state(entity_id: str) -> Dict[str, Any]:
|
||||
"""Fetch detailed state of a single entity."""
|
||||
import aiohttp
|
||||
|
||||
hass_url, hass_token = _get_config()
|
||||
url = f"{hass_url}/api/states/{entity_id}"
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get(url, headers=_get_headers(hass_token), timeout=aiohttp.ClientTimeout(total=10)) as resp:
|
||||
resp.raise_for_status()
|
||||
data = await resp.json()
|
||||
|
||||
return {
|
||||
"entity_id": data["entity_id"],
|
||||
"state": data["state"],
|
||||
"attributes": data.get("attributes", {}),
|
||||
"last_changed": data.get("last_changed"),
|
||||
"last_updated": data.get("last_updated"),
|
||||
}
|
||||
|
||||
|
||||
def _build_service_payload(
|
||||
entity_id: Optional[str] = None,
|
||||
data: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Build the JSON payload for a HA service call."""
|
||||
payload: Dict[str, Any] = {}
|
||||
if data:
|
||||
payload.update(data)
|
||||
# entity_id parameter takes precedence over data["entity_id"]
|
||||
if entity_id:
|
||||
payload["entity_id"] = entity_id
|
||||
return payload
|
||||
|
||||
|
||||
def _parse_service_response(
|
||||
domain: str,
|
||||
service: str,
|
||||
result: Any,
|
||||
) -> Dict[str, Any]:
|
||||
"""Parse HA service call response into a structured result."""
|
||||
affected = []
|
||||
if isinstance(result, list):
|
||||
for s in result:
|
||||
affected.append({
|
||||
"entity_id": s.get("entity_id", ""),
|
||||
"state": s.get("state", ""),
|
||||
})
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"service": f"{domain}.{service}",
|
||||
"affected_entities": affected,
|
||||
}
|
||||
|
||||
|
||||
async def _async_call_service(
|
||||
domain: str,
|
||||
service: str,
|
||||
entity_id: Optional[str] = None,
|
||||
data: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Call a Home Assistant service."""
|
||||
import aiohttp
|
||||
|
||||
hass_url, hass_token = _get_config()
|
||||
url = f"{hass_url}/api/services/{domain}/{service}"
|
||||
payload = _build_service_payload(entity_id, data)
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.post(
|
||||
url,
|
||||
headers=_get_headers(hass_token),
|
||||
json=payload,
|
||||
timeout=aiohttp.ClientTimeout(total=15),
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
result = await resp.json()
|
||||
|
||||
return _parse_service_response(domain, service, result)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Sync wrappers (handler signature: (args, **kw) -> str)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _run_async(coro):
|
||||
"""Run an async coroutine from a sync handler."""
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
loop = None
|
||||
|
||||
if loop and loop.is_running():
|
||||
# Already inside an event loop -- create a new thread
|
||||
import concurrent.futures
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
|
||||
future = pool.submit(asyncio.run, coro)
|
||||
return future.result(timeout=30)
|
||||
else:
|
||||
return asyncio.run(coro)
|
||||
|
||||
|
||||
def _handle_list_entities(args: dict, **kw) -> str:
|
||||
"""Handler for ha_list_entities tool."""
|
||||
domain = args.get("domain")
|
||||
area = args.get("area")
|
||||
try:
|
||||
result = _run_async(_async_list_entities(domain=domain, area=area))
|
||||
return json.dumps({"result": result})
|
||||
except Exception as e:
|
||||
logger.error("ha_list_entities error: %s", e)
|
||||
return tool_error(f"Failed to list entities: {e}")
|
||||
|
||||
|
||||
def _handle_get_state(args: dict, **kw) -> str:
|
||||
"""Handler for ha_get_state tool."""
|
||||
entity_id = args.get("entity_id", "")
|
||||
if not entity_id:
|
||||
return tool_error("Missing required parameter: entity_id")
|
||||
if not _ENTITY_ID_RE.match(entity_id):
|
||||
return tool_error(f"Invalid entity_id format: {entity_id}")
|
||||
try:
|
||||
result = _run_async(_async_get_state(entity_id))
|
||||
return json.dumps({"result": result})
|
||||
except Exception as e:
|
||||
logger.error("ha_get_state error: %s", e)
|
||||
return tool_error(f"Failed to get state for {entity_id}: {e}")
|
||||
|
||||
|
||||
def _handle_call_service(args: dict, **kw) -> str:
|
||||
"""Handler for ha_call_service tool."""
|
||||
domain = args.get("domain", "")
|
||||
service = args.get("service", "")
|
||||
if not domain or not service:
|
||||
return tool_error("Missing required parameters: domain and service")
|
||||
|
||||
# Validate domain/service format BEFORE the blocklist check — prevents
|
||||
# path traversal in /api/services/{domain}/{service} and blocklist bypass
|
||||
# via payloads like "shell_command/../light".
|
||||
if not _SERVICE_NAME_RE.match(domain):
|
||||
return tool_error(f"Invalid domain format: {domain!r}")
|
||||
if not _SERVICE_NAME_RE.match(service):
|
||||
return tool_error(f"Invalid service format: {service!r}")
|
||||
|
||||
if domain in _BLOCKED_DOMAINS:
|
||||
return json.dumps({
|
||||
"error": f"Service domain '{domain}' is blocked for security. "
|
||||
f"Blocked domains: {', '.join(sorted(_BLOCKED_DOMAINS))}"
|
||||
})
|
||||
|
||||
entity_id = args.get("entity_id")
|
||||
if entity_id and not _ENTITY_ID_RE.match(entity_id):
|
||||
return tool_error(f"Invalid entity_id format: {entity_id}")
|
||||
|
||||
data = args.get("data")
|
||||
if isinstance(data, str):
|
||||
try:
|
||||
data = json.loads(data) if data.strip() else None
|
||||
except json.JSONDecodeError as e:
|
||||
return tool_error(f"Invalid JSON string in 'data' parameter: {e}")
|
||||
|
||||
try:
|
||||
result = _run_async(_async_call_service(domain, service, entity_id, data))
|
||||
return json.dumps({"result": result})
|
||||
except Exception as e:
|
||||
logger.error("ha_call_service error: %s", e)
|
||||
return tool_error(f"Failed to call {domain}.{service}: {e}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# List services
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def _async_list_services(domain: Optional[str] = None) -> Dict[str, Any]:
|
||||
"""Fetch available services from HA and optionally filter by domain."""
|
||||
import aiohttp
|
||||
|
||||
hass_url, hass_token = _get_config()
|
||||
url = f"{hass_url}/api/services"
|
||||
headers = {"Authorization": f"Bearer {hass_token}", "Content-Type": "application/json"}
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get(url, headers=headers, timeout=aiohttp.ClientTimeout(total=15)) as resp:
|
||||
resp.raise_for_status()
|
||||
services = await resp.json()
|
||||
|
||||
if domain:
|
||||
services = [s for s in services if s.get("domain") == domain]
|
||||
|
||||
# Compact the output for context efficiency
|
||||
result = []
|
||||
for svc_domain in services:
|
||||
d = svc_domain.get("domain", "")
|
||||
domain_services = {}
|
||||
for svc_name, svc_info in svc_domain.get("services", {}).items():
|
||||
svc_entry: Dict[str, Any] = {"description": svc_info.get("description", "")}
|
||||
fields = svc_info.get("fields", {})
|
||||
if fields:
|
||||
svc_entry["fields"] = {
|
||||
k: v.get("description", "") for k, v in fields.items()
|
||||
if isinstance(v, dict)
|
||||
}
|
||||
domain_services[svc_name] = svc_entry
|
||||
result.append({"domain": d, "services": domain_services})
|
||||
|
||||
return {"count": len(result), "domains": result}
|
||||
|
||||
|
||||
def _handle_list_services(args: dict, **kw) -> str:
|
||||
"""Handler for ha_list_services tool."""
|
||||
domain = args.get("domain")
|
||||
try:
|
||||
result = _run_async(_async_list_services(domain=domain))
|
||||
return json.dumps({"result": result})
|
||||
except Exception as e:
|
||||
logger.error("ha_list_services error: %s", e)
|
||||
return tool_error(f"Failed to list services: {e}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Availability check
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _check_ha_available() -> bool:
|
||||
"""Tool is only available when HASS_TOKEN is set."""
|
||||
return bool(os.getenv("HASS_TOKEN"))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
HA_LIST_ENTITIES_SCHEMA = {
|
||||
"name": "ha_list_entities",
|
||||
"description": (
|
||||
"List Home Assistant entities. Optionally filter by domain "
|
||||
"(light, switch, climate, sensor, binary_sensor, cover, fan, etc.) "
|
||||
"or by area name (living room, kitchen, bedroom, etc.)."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"domain": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Entity domain to filter by (e.g. 'light', 'switch', 'climate', "
|
||||
"'sensor', 'binary_sensor', 'cover', 'fan', 'media_player'). "
|
||||
"Omit to list all entities."
|
||||
),
|
||||
},
|
||||
"area": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Area/room name to filter by (e.g. 'living room', 'kitchen'). "
|
||||
"Matches against entity friendly names. Omit to list all."
|
||||
),
|
||||
},
|
||||
},
|
||||
"required": [],
|
||||
},
|
||||
}
|
||||
|
||||
HA_GET_STATE_SCHEMA = {
|
||||
"name": "ha_get_state",
|
||||
"description": (
|
||||
"Get the detailed state of a single Home Assistant entity, including all "
|
||||
"attributes (brightness, color, temperature setpoint, sensor readings, etc.)."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"entity_id": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"The entity ID to query (e.g. 'light.living_room', "
|
||||
"'climate.thermostat', 'sensor.temperature')."
|
||||
),
|
||||
},
|
||||
},
|
||||
"required": ["entity_id"],
|
||||
},
|
||||
}
|
||||
|
||||
HA_LIST_SERVICES_SCHEMA = {
|
||||
"name": "ha_list_services",
|
||||
"description": (
|
||||
"List available Home Assistant services (actions) for device control. "
|
||||
"Shows what actions can be performed on each device type and what "
|
||||
"parameters they accept. Use this to discover how to control devices "
|
||||
"found via ha_list_entities."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"domain": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Filter by domain (e.g. 'light', 'climate', 'switch'). "
|
||||
"Omit to list services for all domains."
|
||||
),
|
||||
},
|
||||
},
|
||||
"required": [],
|
||||
},
|
||||
}
|
||||
|
||||
HA_CALL_SERVICE_SCHEMA = {
|
||||
"name": "ha_call_service",
|
||||
"description": (
|
||||
"Call a Home Assistant service to control a device. Use ha_list_services "
|
||||
"to discover available services and their parameters for each domain."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"domain": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Service domain (e.g. 'light', 'switch', 'climate', "
|
||||
"'cover', 'media_player', 'fan', 'scene', 'script')."
|
||||
),
|
||||
},
|
||||
"service": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Service name (e.g. 'turn_on', 'turn_off', 'toggle', "
|
||||
"'set_temperature', 'set_hvac_mode', 'open_cover', "
|
||||
"'close_cover', 'set_volume_level')."
|
||||
),
|
||||
},
|
||||
"entity_id": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Target entity ID (e.g. 'light.living_room'). "
|
||||
"Some services (like scene.turn_on) may not need this."
|
||||
),
|
||||
},
|
||||
"data": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Additional service data as a JSON string. Examples: "
|
||||
'{"brightness": 255, "color_name": "blue"} for lights, '
|
||||
'{"temperature": 22, "hvac_mode": "heat"} for climate, '
|
||||
'{"volume_level": 0.5} for media players.'
|
||||
),
|
||||
},
|
||||
},
|
||||
"required": ["domain", "service"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Registration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
from tools.registry import registry, tool_error
|
||||
|
||||
registry.register(
|
||||
name="ha_list_entities",
|
||||
toolset="homeassistant",
|
||||
schema=HA_LIST_ENTITIES_SCHEMA,
|
||||
handler=_handle_list_entities,
|
||||
check_fn=_check_ha_available,
|
||||
emoji="🏠",
|
||||
)
|
||||
|
||||
registry.register(
|
||||
name="ha_get_state",
|
||||
toolset="homeassistant",
|
||||
schema=HA_GET_STATE_SCHEMA,
|
||||
handler=_handle_get_state,
|
||||
check_fn=_check_ha_available,
|
||||
emoji="🏠",
|
||||
)
|
||||
|
||||
registry.register(
|
||||
name="ha_list_services",
|
||||
toolset="homeassistant",
|
||||
schema=HA_LIST_SERVICES_SCHEMA,
|
||||
handler=_handle_list_services,
|
||||
check_fn=_check_ha_available,
|
||||
emoji="🏠",
|
||||
)
|
||||
|
||||
registry.register(
|
||||
name="ha_call_service",
|
||||
toolset="homeassistant",
|
||||
schema=HA_CALL_SERVICE_SCHEMA,
|
||||
handler=_handle_call_service,
|
||||
check_fn=_check_ha_available,
|
||||
emoji="🏠",
|
||||
)
|
||||
@@ -0,0 +1,254 @@
|
||||
"""
|
||||
IMA 连接器工具 (ima_tool.py)
|
||||
|
||||
与腾讯会议不同，IMA 是纯 REST API（HTTP POST + JSON），
|
||||
无需外部脚本，直接在进程内调用。
|
||||
|
||||
┌──────────────────────────────────────────────────────┐
|
||||
│ 绑定流程：侧边栏 inline UI（填写 Client ID + API Key）│
|
||||
│ 调用流程：SKILL.md + Agent 对话 │
|
||||
└──────────────────────────────────────────────────────┘
|
||||
|
||||
凭证存储（per-user JSON）：
|
||||
/opt/apps/mindos-next/backend/data/ima_tokens.json
|
||||
格式：{ "userId": {"client_id": "...", "api_key": "..."} }
|
||||
|
||||
IMA OpenAPI 文档：https://ima.qq.com/agent-interface
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from typing import Any, Dict
|
||||
|
||||
from tools.registry import registry, tool_error, tool_result
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── 常量 ──
|
||||
_BASE_URL = "https://ima.qq.com/"
|
||||
CREDENTIALS_URL = "https://ima.qq.com/agent-interface"
|
||||
_SKILL_VERSION = "1.1.3"
|
||||
|
||||
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
# 凭证存取（per-user，wiki/{userId}/.config/ima.json）
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
|
||||
from tools._user_config import user_config_read, user_config_write, user_config_delete
|
||||
|
||||
|
||||
def get_user_credentials(user_id: str) -> Dict[str, str]:
|
||||
"""返回 {"client_id": "...", "api_key": "..."} 或空 dict"""
|
||||
creds = user_config_read(user_id, "ima")
|
||||
if creds.get("client_id") and creds.get("api_key"):
|
||||
return creds
|
||||
# 兜底：全局 env
|
||||
cid = os.environ.get("IMA_OPENAPI_CLIENTID", "").strip()
|
||||
key = os.environ.get("IMA_OPENAPI_APIKEY", "").strip()
|
||||
if cid and key:
|
||||
return {"client_id": cid, "api_key": key}
|
||||
return {}
|
||||
|
||||
|
||||
def set_user_credentials(user_id: str, client_id: str, api_key: str) -> None:
|
||||
"""保存或清除指定用户的 IMA 凭证。两者为空则删除条目。"""
|
||||
client_id = client_id.strip()
|
||||
api_key = api_key.strip()
|
||||
if client_id and api_key:
|
||||
user_config_write(user_id, "ima", {"client_id": client_id, "api_key": api_key})
|
||||
else:
|
||||
user_config_delete(user_id, "ima")
|
||||
|
||||
|
||||
def has_user_credentials(user_id: str) -> bool:
|
||||
return bool(get_user_credentials(user_id))
|
||||
|
||||
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
# HTTP 调用层（进程内，无需子进程）
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
|
||||
def _ima_call(api_path: str, body: dict, creds: Dict[str, str]) -> Dict[str, Any]:
|
||||
"""
|
||||
直接发送 HTTP POST 到 IMA OpenAPI。
|
||||
返回解析后的 JSON dict，出错时返回 {"error": ...}。
|
||||
"""
|
||||
url = _BASE_URL + api_path.lstrip("/")
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"ima-openapi-clientid": creds["client_id"],
|
||||
"ima-openapi-apikey": creds["api_key"],
|
||||
"ima-openapi-ctx": f"skill_version={_SKILL_VERSION}",
|
||||
}
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
req = urllib.request.Request(url, data=data, headers=headers, method="POST")
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
return json.loads(r.read().decode("utf-8"))
|
||||
except urllib.error.HTTPError as e:
|
||||
body_text = e.read().decode("utf-8", errors="replace")[:500]
|
||||
return {"error": f"HTTP {e.code}", "detail": body_text}
|
||||
except urllib.error.URLError as e:
|
||||
return {"error": f"网络错误: {e}"}
|
||||
except Exception as e:
|
||||
return {"error": str(e)}
|
||||
|
||||
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
# ① ima_list_profiles — 连接状态检查
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
|
||||
def _ima_list_profiles_handler(args: dict, **kwargs) -> str:
|
||||
user_id = str(args.get("user_id", "")).strip()
|
||||
if not user_id:
|
||||
return tool_error("user_id 参数必填")
|
||||
|
||||
if not has_user_credentials(user_id):
|
||||
return tool_result(
|
||||
success=True,
|
||||
profiles=[],
|
||||
count=0,
|
||||
credentialsConfigured=False,
|
||||
credentialsUrl=CREDENTIALS_URL,
|
||||
notice="请前往上述链接获取 Client ID 和 API Key，并在侧边栏绑定。",
|
||||
)
|
||||
|
||||
creds = get_user_credentials(user_id)
|
||||
# 用 list_note (limit=1) 验证连通性
|
||||
logger.info("[ImaTool] checking connectivity for user=%s", user_id[:8])
|
||||
result = _ima_call("openapi/note/v1/list_note", {"cursor": "", "limit": 1}, creds)
|
||||
|
||||
if "error" in result:
|
||||
status = "error"
|
||||
message = f"连接异常：{result['error']}"
|
||||
else:
|
||||
status = "connected"
|
||||
message = "IMA 已连接。"
|
||||
|
||||
profiles = [{
|
||||
"profileName": f"{user_id[:12]}_ima",
|
||||
"label": "IMA 笔记 & 知识库",
|
||||
"status": status,
|
||||
"credentialsConfigured": True,
|
||||
}]
|
||||
|
||||
return tool_result(
|
||||
success=True,
|
||||
profiles=profiles,
|
||||
count=len(profiles),
|
||||
credentialsConfigured=True,
|
||||
message=message,
|
||||
)
|
||||
|
||||
|
||||
registry.register(
|
||||
name="ima_list_profiles",
|
||||
toolset="connectors",
|
||||
description="检查 IMA 连接状态，返回账号信息。",
|
||||
emoji="📝",
|
||||
handler=_ima_list_profiles_handler,
|
||||
schema={
|
||||
"name": "ima_list_profiles",
|
||||
"description": "检查 IMA (QQ 笔记/知识库) 是否已连接，返回连接状态。",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"user_id": {"type": "string", "description": "MINDOS_USER_ID"},
|
||||
},
|
||||
"required": ["user_id"],
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
# ② ima_query — 笔记和知识库操作
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
|
||||
_ALLOWED_API_PATHS = {
|
||||
# Notes
|
||||
"openapi/note/v1/list_notebook",
|
||||
"openapi/note/v1/list_note",
|
||||
"openapi/note/v1/search_note_book",
|
||||
"openapi/note/v1/get_doc_content",
|
||||
"openapi/note/v1/import_doc",
|
||||
"openapi/note/v1/append_doc",
|
||||
# Knowledge Base
|
||||
"openapi/knowledge/v1/list_knowledge_base",
|
||||
"openapi/knowledge/v1/search_knowledge_base",
|
||||
"openapi/knowledge/v1/create_knowledge_base",
|
||||
"openapi/knowledge/v1/delete_knowledge_base",
|
||||
"openapi/knowledge/v1/add_url",
|
||||
}
|
||||
|
||||
|
||||
def _ima_query_handler(args: dict, **kwargs) -> str:
|
||||
user_id = str(args.get("user_id", "")).strip()
|
||||
api_path = str(args.get("api_path", "")).strip()
|
||||
|
||||
if not user_id:
|
||||
return tool_error("user_id 参数必填")
|
||||
if not api_path:
|
||||
return tool_error("api_path 参数必填")
|
||||
if api_path not in _ALLOWED_API_PATHS:
|
||||
return tool_error(
|
||||
f"不支持的 API 路径：{api_path}。"
|
||||
f"可选：{', '.join(sorted(_ALLOWED_API_PATHS))}"
|
||||
)
|
||||
if not has_user_credentials(user_id):
|
||||
return tool_error(
|
||||
f"IMA 凭证未配置。请前往 {CREDENTIALS_URL} 获取 Client ID 和 API Key，"
|
||||
"并在侧边栏「IMA」处绑定账号。"
|
||||
)
|
||||
|
||||
body = args.get("body", {})
|
||||
if not isinstance(body, dict):
|
||||
return tool_error("body 必须是 JSON 对象")
|
||||
|
||||
creds = get_user_credentials(user_id)
|
||||
logger.info("[ImaTool] call %s (user=%s)", api_path, user_id[:8])
|
||||
result = _ima_call(api_path, body, creds)
|
||||
|
||||
if "error" in result:
|
||||
return tool_error(f"调用失败：{result['error']}")
|
||||
|
||||
return tool_result(success=True, data=result)
|
||||
|
||||
|
||||
registry.register(
|
||||
name="ima_query",
|
||||
toolset="connectors",
|
||||
description="查询或操作 IMA 笔记和知识库。",
|
||||
emoji="📝",
|
||||
handler=_ima_query_handler,
|
||||
schema={
|
||||
"name": "ima_query",
|
||||
"description": (
|
||||
"调用 IMA OpenAPI 操作笔记或知识库。需先在侧边栏完成凭证绑定。\n"
|
||||
"笔记接口示例：\n"
|
||||
" 搜索笔记: api_path=openapi/note/v1/search_note_book, body={\"search_type\":0,\"query_info\":{\"title\":\"xx\"},\"start\":0,\"end\":10}\n"
|
||||
" 读取正文: api_path=openapi/note/v1/get_doc_content, body={\"doc_id\":\"xxx\",\"target_content_format\":0}\n"
|
||||
" 新建笔记: api_path=openapi/note/v1/import_doc, body={\"content\":\"## 标题\\n内容\",\"content_format\":1}\n"
|
||||
" 列出笔记: api_path=openapi/note/v1/list_note, body={\"cursor\":\"\",\"limit\":20}\n"
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"user_id": {"type": "string", "description": "MINDOS_USER_ID"},
|
||||
"api_path": {
|
||||
"type": "string",
|
||||
"description": "IMA API 路径",
|
||||
"enum": sorted(_ALLOWED_API_PATHS),
|
||||
},
|
||||
"body": {
|
||||
"type": "object",
|
||||
"description": "API 请求体（JSON 对象）",
|
||||
},
|
||||
},
|
||||
"required": ["user_id", "api_path"],
|
||||
},
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,693 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Image Generation Tools Module
|
||||
|
||||
This module provides image generation tools using FAL.ai's FLUX 2 Pro model with
|
||||
automatic upscaling via FAL.ai's Clarity Upscaler for enhanced image quality.
|
||||
|
||||
Available tools:
|
||||
- image_generate_tool: Generate images from text prompts with automatic upscaling
|
||||
|
||||
Features:
|
||||
- High-quality image generation using FLUX 2 Pro model
|
||||
- Automatic 2x upscaling using Clarity Upscaler for enhanced quality
|
||||
- Comprehensive parameter control (size, steps, guidance, etc.)
|
||||
- Proper error handling and validation with fallback to original images
|
||||
- Debug logging support
|
||||
- Sync mode for immediate results
|
||||
|
||||
Usage:
|
||||
from image_generation_tool import image_generate_tool
|
||||
import asyncio
|
||||
|
||||
# Generate and automatically upscale an image
|
||||
result = await image_generate_tool(
|
||||
prompt="A serene mountain landscape with cherry blossoms",
|
||||
image_size="landscape_4_3",
|
||||
num_images=1
|
||||
)
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import datetime
|
||||
import threading
|
||||
import uuid
|
||||
from typing import Dict, Any, Optional, Union
|
||||
from urllib.parse import urlencode
|
||||
import fal_client
|
||||
from tools.debug_helpers import DebugSession
|
||||
from tools.managed_tool_gateway import resolve_managed_tool_gateway
|
||||
from tools.tool_backend_helpers import managed_nous_tools_enabled
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Configuration for image generation
|
||||
DEFAULT_MODEL = "fal-ai/flux-2-pro"
|
||||
DEFAULT_ASPECT_RATIO = "landscape"
|
||||
DEFAULT_NUM_INFERENCE_STEPS = 50
|
||||
DEFAULT_GUIDANCE_SCALE = 4.5
|
||||
DEFAULT_NUM_IMAGES = 1
|
||||
DEFAULT_OUTPUT_FORMAT = "png"
|
||||
|
||||
# Safety settings
|
||||
ENABLE_SAFETY_CHECKER = False
|
||||
SAFETY_TOLERANCE = "5" # Maximum tolerance (1-5, where 5 is most permissive)
|
||||
|
||||
# Aspect ratio mapping - simplified choices for model to select
|
||||
ASPECT_RATIO_MAP = {
|
||||
"landscape": "landscape_16_9",
|
||||
"square": "square_hd",
|
||||
"portrait": "portrait_16_9"
|
||||
}
|
||||
|
||||
# Configuration for automatic upscaling
|
||||
UPSCALER_MODEL = "fal-ai/clarity-upscaler"
|
||||
UPSCALER_FACTOR = 2
|
||||
UPSCALER_SAFETY_CHECKER = False
|
||||
UPSCALER_DEFAULT_PROMPT = "masterpiece, best quality, highres"
|
||||
UPSCALER_NEGATIVE_PROMPT = "(worst quality, low quality, normal quality:2)"
|
||||
UPSCALER_CREATIVITY = 0.35
|
||||
UPSCALER_RESEMBLANCE = 0.6
|
||||
UPSCALER_GUIDANCE_SCALE = 4
|
||||
UPSCALER_NUM_INFERENCE_STEPS = 18
|
||||
|
||||
# Valid parameter values for validation based on FLUX 2 Pro documentation
|
||||
VALID_IMAGE_SIZES = [
|
||||
"square_hd", "square", "portrait_4_3", "portrait_16_9", "landscape_4_3", "landscape_16_9"
|
||||
]
|
||||
VALID_OUTPUT_FORMATS = ["jpeg", "png"]
|
||||
VALID_ACCELERATION_MODES = ["none", "regular", "high"]
|
||||
|
||||
_debug = DebugSession("image_tools", env_var="IMAGE_TOOLS_DEBUG")
|
||||
_managed_fal_client = None
|
||||
_managed_fal_client_config = None
|
||||
_managed_fal_client_lock = threading.Lock()
|
||||
|
||||
|
||||
def _resolve_managed_fal_gateway():
|
||||
"""Return managed fal-queue gateway config when direct FAL credentials are absent."""
|
||||
if os.getenv("FAL_KEY"):
|
||||
return None
|
||||
return resolve_managed_tool_gateway("fal-queue")
|
||||
|
||||
|
||||
def _normalize_fal_queue_url_format(queue_run_origin: str) -> str:
|
||||
normalized_origin = str(queue_run_origin or "").strip().rstrip("/")
|
||||
if not normalized_origin:
|
||||
raise ValueError("Managed FAL queue origin is required")
|
||||
return f"{normalized_origin}/"
|
||||
|
||||
|
||||
class _ManagedFalSyncClient:
|
||||
"""Small per-instance wrapper around fal_client.SyncClient for managed queue hosts."""
|
||||
|
||||
def __init__(self, *, key: str, queue_run_origin: str):
|
||||
sync_client_class = getattr(fal_client, "SyncClient", None)
|
||||
if sync_client_class is None:
|
||||
raise RuntimeError("fal_client.SyncClient is required for managed FAL gateway mode")
|
||||
|
||||
client_module = getattr(fal_client, "client", None)
|
||||
if client_module is None:
|
||||
raise RuntimeError("fal_client.client is required for managed FAL gateway mode")
|
||||
|
||||
self._queue_url_format = _normalize_fal_queue_url_format(queue_run_origin)
|
||||
self._sync_client = sync_client_class(key=key)
|
||||
self._http_client = getattr(self._sync_client, "_client", None)
|
||||
self._maybe_retry_request = getattr(client_module, "_maybe_retry_request", None)
|
||||
self._raise_for_status = getattr(client_module, "_raise_for_status", None)
|
||||
self._request_handle_class = getattr(client_module, "SyncRequestHandle", None)
|
||||
self._add_hint_header = getattr(client_module, "add_hint_header", None)
|
||||
self._add_priority_header = getattr(client_module, "add_priority_header", None)
|
||||
self._add_timeout_header = getattr(client_module, "add_timeout_header", None)
|
||||
|
||||
if self._http_client is None:
|
||||
raise RuntimeError("fal_client.SyncClient._client is required for managed FAL gateway mode")
|
||||
if self._maybe_retry_request is None or self._raise_for_status is None:
|
||||
raise RuntimeError("fal_client.client request helpers are required for managed FAL gateway mode")
|
||||
if self._request_handle_class is None:
|
||||
raise RuntimeError("fal_client.client.SyncRequestHandle is required for managed FAL gateway mode")
|
||||
|
||||
def submit(
|
||||
self,
|
||||
application: str,
|
||||
arguments: Dict[str, Any],
|
||||
*,
|
||||
path: str = "",
|
||||
hint: Optional[str] = None,
|
||||
webhook_url: Optional[str] = None,
|
||||
priority: Any = None,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
start_timeout: Optional[Union[int, float]] = None,
|
||||
):
|
||||
url = self._queue_url_format + application
|
||||
if path:
|
||||
url += "/" + path.lstrip("/")
|
||||
if webhook_url is not None:
|
||||
url += "?" + urlencode({"fal_webhook": webhook_url})
|
||||
|
||||
request_headers = dict(headers or {})
|
||||
if hint is not None and self._add_hint_header is not None:
|
||||
self._add_hint_header(hint, request_headers)
|
||||
if priority is not None:
|
||||
if self._add_priority_header is None:
|
||||
raise RuntimeError("fal_client.client.add_priority_header is required for priority requests")
|
||||
self._add_priority_header(priority, request_headers)
|
||||
if start_timeout is not None:
|
||||
if self._add_timeout_header is None:
|
||||
raise RuntimeError("fal_client.client.add_timeout_header is required for timeout requests")
|
||||
self._add_timeout_header(start_timeout, request_headers)
|
||||
|
||||
response = self._maybe_retry_request(
|
||||
self._http_client,
|
||||
"POST",
|
||||
url,
|
||||
json=arguments,
|
||||
timeout=getattr(self._sync_client, "default_timeout", 120.0),
|
||||
headers=request_headers,
|
||||
)
|
||||
self._raise_for_status(response)
|
||||
|
||||
data = response.json()
|
||||
return self._request_handle_class(
|
||||
request_id=data["request_id"],
|
||||
response_url=data["response_url"],
|
||||
status_url=data["status_url"],
|
||||
cancel_url=data["cancel_url"],
|
||||
client=self._http_client,
|
||||
)
|
||||
|
||||
|
||||
def _get_managed_fal_client(managed_gateway):
|
||||
"""Reuse the managed FAL client so its internal httpx.Client is not leaked per call."""
|
||||
global _managed_fal_client, _managed_fal_client_config
|
||||
|
||||
client_config = (
|
||||
managed_gateway.gateway_origin.rstrip("/"),
|
||||
managed_gateway.nous_user_token,
|
||||
)
|
||||
with _managed_fal_client_lock:
|
||||
if _managed_fal_client is not None and _managed_fal_client_config == client_config:
|
||||
return _managed_fal_client
|
||||
|
||||
_managed_fal_client = _ManagedFalSyncClient(
|
||||
key=managed_gateway.nous_user_token,
|
||||
queue_run_origin=managed_gateway.gateway_origin,
|
||||
)
|
||||
_managed_fal_client_config = client_config
|
||||
return _managed_fal_client
|
||||
|
||||
|
||||
def _submit_fal_request(model: str, arguments: Dict[str, Any]):
|
||||
"""Submit a FAL request using direct credentials or the managed queue gateway."""
|
||||
request_headers = {"x-idempotency-key": str(uuid.uuid4())}
|
||||
managed_gateway = _resolve_managed_fal_gateway()
|
||||
if managed_gateway is None:
|
||||
return fal_client.submit(model, arguments=arguments, headers=request_headers)
|
||||
|
||||
managed_client = _get_managed_fal_client(managed_gateway)
|
||||
return managed_client.submit(
|
||||
model,
|
||||
arguments=arguments,
|
||||
headers=request_headers,
|
||||
)
|
||||
|
||||
|
||||
def _validate_parameters(
|
||||
image_size: Union[str, Dict[str, int]],
|
||||
num_inference_steps: int,
|
||||
guidance_scale: float,
|
||||
num_images: int,
|
||||
output_format: str,
|
||||
acceleration: str = "none"
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Validate and normalize image generation parameters for FLUX 2 Pro model.
|
||||
|
||||
Args:
|
||||
image_size: Either a preset string or custom size dict
|
||||
num_inference_steps: Number of inference steps
|
||||
guidance_scale: Guidance scale value
|
||||
num_images: Number of images to generate
|
||||
output_format: Output format for images
|
||||
acceleration: Acceleration mode for generation speed
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: Validated and normalized parameters
|
||||
|
||||
Raises:
|
||||
ValueError: If any parameter is invalid
|
||||
"""
|
||||
validated = {}
|
||||
|
||||
# Validate image_size
|
||||
if isinstance(image_size, str):
|
||||
if image_size not in VALID_IMAGE_SIZES:
|
||||
raise ValueError(f"Invalid image_size '{image_size}'. Must be one of: {VALID_IMAGE_SIZES}")
|
||||
validated["image_size"] = image_size
|
||||
elif isinstance(image_size, dict):
|
||||
if "width" not in image_size or "height" not in image_size:
|
||||
raise ValueError("Custom image_size must contain 'width' and 'height' keys")
|
||||
if not isinstance(image_size["width"], int) or not isinstance(image_size["height"], int):
|
||||
raise ValueError("Custom image_size width and height must be integers")
|
||||
if image_size["width"] < 64 or image_size["height"] < 64:
|
||||
raise ValueError("Custom image_size dimensions must be at least 64x64")
|
||||
if image_size["width"] > 2048 or image_size["height"] > 2048:
|
||||
raise ValueError("Custom image_size dimensions must not exceed 2048x2048")
|
||||
validated["image_size"] = image_size
|
||||
else:
|
||||
raise ValueError("image_size must be either a preset string or a dict with width/height")
|
||||
|
||||
# Validate num_inference_steps
|
||||
if not isinstance(num_inference_steps, int) or num_inference_steps < 1 or num_inference_steps > 100:
|
||||
raise ValueError("num_inference_steps must be an integer between 1 and 100")
|
||||
validated["num_inference_steps"] = num_inference_steps
|
||||
|
||||
# Validate guidance_scale (FLUX 2 Pro default is 4.5)
|
||||
if not isinstance(guidance_scale, (int, float)) or guidance_scale < 0.1 or guidance_scale > 20.0:
|
||||
raise ValueError("guidance_scale must be a number between 0.1 and 20.0")
|
||||
validated["guidance_scale"] = float(guidance_scale)
|
||||
|
||||
# Validate num_images
|
||||
if not isinstance(num_images, int) or num_images < 1 or num_images > 4:
|
||||
raise ValueError("num_images must be an integer between 1 and 4")
|
||||
validated["num_images"] = num_images
|
||||
|
||||
# Validate output_format
|
||||
if output_format not in VALID_OUTPUT_FORMATS:
|
||||
raise ValueError(f"Invalid output_format '{output_format}'. Must be one of: {VALID_OUTPUT_FORMATS}")
|
||||
validated["output_format"] = output_format
|
||||
|
||||
# Validate acceleration
|
||||
if acceleration not in VALID_ACCELERATION_MODES:
|
||||
raise ValueError(f"Invalid acceleration '{acceleration}'. Must be one of: {VALID_ACCELERATION_MODES}")
|
||||
validated["acceleration"] = acceleration
|
||||
|
||||
return validated
|
||||
|
||||
|
||||
def _upscale_image(image_url: str, original_prompt: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Upscale an image using FAL.ai's Clarity Upscaler.
|
||||
|
||||
Uses the synchronous fal_client API to avoid event loop lifecycle issues
|
||||
when called from threaded contexts (e.g. gateway thread pool).
|
||||
|
||||
Args:
|
||||
image_url (str): URL of the image to upscale
|
||||
original_prompt (str): Original prompt used to generate the image
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: Upscaled image data or None if upscaling fails
|
||||
"""
|
||||
try:
|
||||
logger.info("Upscaling image with Clarity Upscaler...")
|
||||
|
||||
# Prepare arguments for upscaler
|
||||
upscaler_arguments = {
|
||||
"image_url": image_url,
|
||||
"prompt": f"{UPSCALER_DEFAULT_PROMPT}, {original_prompt}",
|
||||
"upscale_factor": UPSCALER_FACTOR,
|
||||
"negative_prompt": UPSCALER_NEGATIVE_PROMPT,
|
||||
"creativity": UPSCALER_CREATIVITY,
|
||||
"resemblance": UPSCALER_RESEMBLANCE,
|
||||
"guidance_scale": UPSCALER_GUIDANCE_SCALE,
|
||||
"num_inference_steps": UPSCALER_NUM_INFERENCE_STEPS,
|
||||
"enable_safety_checker": UPSCALER_SAFETY_CHECKER
|
||||
}
|
||||
|
||||
# Use sync API — fal_client.submit() uses httpx.Client (no event loop).
|
||||
# The async API (submit_async) caches a global httpx.AsyncClient via
|
||||
# @cached_property, which breaks when asyncio.run() destroys the loop
|
||||
# between calls (gateway thread-pool pattern).
|
||||
handler = _submit_fal_request(
|
||||
UPSCALER_MODEL,
|
||||
arguments=upscaler_arguments,
|
||||
)
|
||||
|
||||
# Get the upscaled result (sync — blocks until done)
|
||||
result = handler.get()
|
||||
|
||||
if result and "image" in result:
|
||||
upscaled_image = result["image"]
|
||||
logger.info("Image upscaled successfully to %sx%s", upscaled_image.get('width', 'unknown'), upscaled_image.get('height', 'unknown'))
|
||||
return {
|
||||
"url": upscaled_image["url"],
|
||||
"width": upscaled_image.get("width", 0),
|
||||
"height": upscaled_image.get("height", 0),
|
||||
"upscaled": True,
|
||||
"upscale_factor": UPSCALER_FACTOR
|
||||
}
|
||||
else:
|
||||
logger.error("Upscaler returned invalid response")
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Error upscaling image: %s", e, exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
def image_generate_tool(
|
||||
prompt: str,
|
||||
aspect_ratio: str = DEFAULT_ASPECT_RATIO,
|
||||
num_inference_steps: int = DEFAULT_NUM_INFERENCE_STEPS,
|
||||
guidance_scale: float = DEFAULT_GUIDANCE_SCALE,
|
||||
num_images: int = DEFAULT_NUM_IMAGES,
|
||||
output_format: str = DEFAULT_OUTPUT_FORMAT,
|
||||
seed: Optional[int] = None
|
||||
) -> str:
|
||||
"""
|
||||
Generate images from text prompts using FAL.ai's FLUX 2 Pro model with automatic upscaling.
|
||||
|
||||
Uses the synchronous fal_client API to avoid event loop lifecycle issues.
|
||||
The async API's global httpx.AsyncClient (cached via @cached_property) breaks
|
||||
when asyncio.run() destroys and recreates event loops between calls, which
|
||||
happens in the gateway's thread-pool pattern.
|
||||
|
||||
Args:
|
||||
prompt (str): The text prompt describing the desired image
|
||||
aspect_ratio (str): Image aspect ratio - "landscape", "square", or "portrait" (default: "landscape")
|
||||
num_inference_steps (int): Number of denoising steps (1-50, default: 50)
|
||||
guidance_scale (float): How closely to follow prompt (0.1-20.0, default: 4.5)
|
||||
num_images (int): Number of images to generate (1-4, default: 1)
|
||||
output_format (str): Image format "jpeg" or "png" (default: "png")
|
||||
seed (Optional[int]): Random seed for reproducible results (optional)
|
||||
|
||||
Returns:
|
||||
str: JSON string containing minimal generation results:
|
||||
{
|
||||
"success": bool,
|
||||
"image": str or None # URL of the upscaled image, or None if failed
|
||||
}
|
||||
"""
|
||||
# Validate and map aspect_ratio to actual image_size
|
||||
aspect_ratio_lower = aspect_ratio.lower().strip() if aspect_ratio else DEFAULT_ASPECT_RATIO
|
||||
if aspect_ratio_lower not in ASPECT_RATIO_MAP:
|
||||
logger.warning("Invalid aspect_ratio '%s', defaulting to '%s'", aspect_ratio, DEFAULT_ASPECT_RATIO)
|
||||
aspect_ratio_lower = DEFAULT_ASPECT_RATIO
|
||||
image_size = ASPECT_RATIO_MAP[aspect_ratio_lower]
|
||||
|
||||
debug_call_data = {
|
||||
"parameters": {
|
||||
"prompt": prompt,
|
||||
"aspect_ratio": aspect_ratio,
|
||||
"image_size": image_size,
|
||||
"num_inference_steps": num_inference_steps,
|
||||
"guidance_scale": guidance_scale,
|
||||
"num_images": num_images,
|
||||
"output_format": output_format,
|
||||
"seed": seed
|
||||
},
|
||||
"error": None,
|
||||
"success": False,
|
||||
"images_generated": 0,
|
||||
"generation_time": 0
|
||||
}
|
||||
|
||||
start_time = datetime.datetime.now()
|
||||
|
||||
try:
|
||||
logger.info("Generating %s image(s) with FLUX 2 Pro: %s", num_images, prompt[:80])
|
||||
|
||||
# Validate prompt
|
||||
if not prompt or not isinstance(prompt, str) or len(prompt.strip()) == 0:
|
||||
raise ValueError("Prompt is required and must be a non-empty string")
|
||||
|
||||
# Check API key availability
|
||||
if not (os.getenv("FAL_KEY") or _resolve_managed_fal_gateway()):
|
||||
message = "FAL_KEY environment variable not set"
|
||||
if managed_nous_tools_enabled():
|
||||
message += " and managed FAL gateway is unavailable"
|
||||
raise ValueError(message)
|
||||
|
||||
# Validate other parameters
|
||||
validated_params = _validate_parameters(
|
||||
image_size, num_inference_steps, guidance_scale, num_images, output_format, "none"
|
||||
)
|
||||
|
||||
# Prepare arguments for FAL.ai FLUX 2 Pro API
|
||||
arguments = {
|
||||
"prompt": prompt.strip(),
|
||||
"image_size": validated_params["image_size"],
|
||||
"num_inference_steps": validated_params["num_inference_steps"],
|
||||
"guidance_scale": validated_params["guidance_scale"],
|
||||
"num_images": validated_params["num_images"],
|
||||
"output_format": validated_params["output_format"],
|
||||
"enable_safety_checker": ENABLE_SAFETY_CHECKER,
|
||||
"safety_tolerance": SAFETY_TOLERANCE,
|
||||
"sync_mode": True # Use sync mode for immediate results
|
||||
}
|
||||
|
||||
# Add seed if provided
|
||||
if seed is not None and isinstance(seed, int):
|
||||
arguments["seed"] = seed
|
||||
|
||||
logger.info("Submitting generation request to FAL.ai FLUX 2 Pro...")
|
||||
logger.info(" Model: %s", DEFAULT_MODEL)
|
||||
logger.info(" Aspect Ratio: %s -> %s", aspect_ratio_lower, image_size)
|
||||
logger.info(" Steps: %s", validated_params['num_inference_steps'])
|
||||
logger.info(" Guidance: %s", validated_params['guidance_scale'])
|
||||
|
||||
# Submit request to FAL.ai using sync API (avoids cached event loop issues)
|
||||
handler = _submit_fal_request(
|
||||
DEFAULT_MODEL,
|
||||
arguments=arguments,
|
||||
)
|
||||
|
||||
# Get the result (sync — blocks until done)
|
||||
result = handler.get()
|
||||
|
||||
generation_time = (datetime.datetime.now() - start_time).total_seconds()
|
||||
|
||||
# Process the response
|
||||
if not result or "images" not in result:
|
||||
raise ValueError("Invalid response from FAL.ai API - no images returned")
|
||||
|
||||
images = result.get("images", [])
|
||||
if not images:
|
||||
raise ValueError("No images were generated")
|
||||
|
||||
# Format image data and upscale images
|
||||
formatted_images = []
|
||||
for img in images:
|
||||
if isinstance(img, dict) and "url" in img:
|
||||
original_image = {
|
||||
"url": img["url"],
|
||||
"width": img.get("width", 0),
|
||||
"height": img.get("height", 0)
|
||||
}
|
||||
|
||||
# Attempt to upscale the image
|
||||
upscaled_image = _upscale_image(img["url"], prompt.strip())
|
||||
|
||||
if upscaled_image:
|
||||
# Use upscaled image if successful
|
||||
formatted_images.append(upscaled_image)
|
||||
else:
|
||||
# Fall back to original image if upscaling fails
|
||||
logger.warning("Using original image as fallback")
|
||||
original_image["upscaled"] = False
|
||||
formatted_images.append(original_image)
|
||||
|
||||
if not formatted_images:
|
||||
raise ValueError("No valid image URLs returned from API")
|
||||
|
||||
upscaled_count = sum(1 for img in formatted_images if img.get("upscaled", False))
|
||||
logger.info("Generated %s image(s) in %.1fs (%s upscaled)", len(formatted_images), generation_time, upscaled_count)
|
||||
|
||||
# Prepare successful response - minimal format
|
||||
response_data = {
|
||||
"success": True,
|
||||
"image": formatted_images[0]["url"] if formatted_images else None
|
||||
}
|
||||
|
||||
debug_call_data["success"] = True
|
||||
debug_call_data["images_generated"] = len(formatted_images)
|
||||
debug_call_data["generation_time"] = generation_time
|
||||
|
||||
# Log debug information
|
||||
_debug.log_call("image_generate_tool", debug_call_data)
|
||||
_debug.save()
|
||||
|
||||
return json.dumps(response_data, indent=2, ensure_ascii=False)
|
||||
|
||||
except Exception as e:
|
||||
generation_time = (datetime.datetime.now() - start_time).total_seconds()
|
||||
error_msg = f"Error generating image: {str(e)}"
|
||||
logger.error("%s", error_msg, exc_info=True)
|
||||
|
||||
# Include error details so callers can diagnose failures
|
||||
response_data = {
|
||||
"success": False,
|
||||
"image": None,
|
||||
"error": str(e),
|
||||
"error_type": type(e).__name__,
|
||||
}
|
||||
|
||||
debug_call_data["error"] = error_msg
|
||||
debug_call_data["generation_time"] = generation_time
|
||||
_debug.log_call("image_generate_tool", debug_call_data)
|
||||
_debug.save()
|
||||
|
||||
return json.dumps(response_data, indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
def check_fal_api_key() -> bool:
|
||||
"""
|
||||
Check if the FAL.ai API key is available in environment variables.
|
||||
|
||||
Returns:
|
||||
bool: True if API key is set, False otherwise
|
||||
"""
|
||||
return bool(os.getenv("FAL_KEY") or _resolve_managed_fal_gateway())
|
||||
|
||||
|
||||
def check_image_generation_requirements() -> bool:
|
||||
"""
|
||||
Check if all requirements for image generation tools are met.
|
||||
|
||||
Returns:
|
||||
bool: True if requirements are met, False otherwise
|
||||
"""
|
||||
try:
|
||||
# Check API key
|
||||
if not check_fal_api_key():
|
||||
return False
|
||||
|
||||
# Check if fal_client is available
|
||||
import fal_client # noqa: F401 — SDK presence check
|
||||
return True
|
||||
|
||||
except ImportError:
|
||||
return False
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
"""
|
||||
Simple test/demo when run directly
|
||||
"""
|
||||
print("🎨 Image Generation Tools Module - FLUX 2 Pro + Auto Upscaling")
|
||||
print("=" * 60)
|
||||
|
||||
# Check if API key is available
|
||||
api_available = check_fal_api_key()
|
||||
|
||||
if not api_available:
|
||||
print("❌ FAL_KEY environment variable not set")
|
||||
print("Please set your API key: export FAL_KEY='your-key-here'")
|
||||
print("Get API key at: https://fal.ai/")
|
||||
exit(1)
|
||||
else:
|
||||
print("✅ FAL.ai API key found")
|
||||
|
||||
# Check if fal_client is available
|
||||
try:
|
||||
import fal_client
|
||||
print("✅ fal_client library available")
|
||||
except ImportError:
|
||||
print("❌ fal_client library not found")
|
||||
print("Please install: pip install fal-client")
|
||||
exit(1)
|
||||
|
||||
print("🛠️ Image generation tools ready for use!")
|
||||
print(f"🤖 Using model: {DEFAULT_MODEL}")
|
||||
print(f"🔍 Auto-upscaling with: {UPSCALER_MODEL} ({UPSCALER_FACTOR}x)")
|
||||
|
||||
# Show debug mode status
|
||||
if _debug.active:
|
||||
print(f"🐛 Debug mode ENABLED - Session ID: {_debug.session_id}")
|
||||
print(f" Debug logs will be saved to: ./logs/image_tools_debug_{_debug.session_id}.json")
|
||||
else:
|
||||
print("🐛 Debug mode disabled (set IMAGE_TOOLS_DEBUG=true to enable)")
|
||||
|
||||
print("\nBasic usage:")
|
||||
print(" from image_generation_tool import image_generate_tool")
|
||||
print(" import asyncio")
|
||||
print("")
|
||||
print(" async def main():")
|
||||
print(" # Generate image with automatic 2x upscaling")
|
||||
print(" result = await image_generate_tool(")
|
||||
print(" prompt='A serene mountain landscape with cherry blossoms',")
|
||||
print(" image_size='landscape_4_3',")
|
||||
print(" num_images=1")
|
||||
print(" )")
|
||||
print(" print(result)")
|
||||
print(" asyncio.run(main())")
|
||||
|
||||
print("\nSupported image sizes:")
|
||||
for size in VALID_IMAGE_SIZES:
|
||||
print(f" - {size}")
|
||||
print(" - Custom: {'width': 512, 'height': 768} (if needed)")
|
||||
|
||||
print("\nAcceleration modes:")
|
||||
for mode in VALID_ACCELERATION_MODES:
|
||||
print(f" - {mode}")
|
||||
|
||||
print("\nExample prompts:")
|
||||
print(" - 'A candid street photo of a woman with a pink bob and bold eyeliner'")
|
||||
print(" - 'Modern architecture building with glass facade, sunset lighting'")
|
||||
print(" - 'Abstract art with vibrant colors and geometric patterns'")
|
||||
print(" - 'Portrait of a wise old owl perched on ancient tree branch'")
|
||||
print(" - 'Futuristic cityscape with flying cars and neon lights'")
|
||||
|
||||
print("\nDebug mode:")
|
||||
print(" # Enable debug logging")
|
||||
print(" export IMAGE_TOOLS_DEBUG=true")
|
||||
print(" # Debug logs capture all image generation calls and results")
|
||||
print(" # Logs saved to: ./logs/image_tools_debug_UUID.json")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Registry
|
||||
# ---------------------------------------------------------------------------
|
||||
from tools.registry import registry, tool_error
|
||||
|
||||
IMAGE_GENERATE_SCHEMA = {
|
||||
"name": "image_generate",
|
||||
"description": "Generate high-quality images from text prompts using FLUX 2 Pro model with automatic 2x upscaling. Creates detailed, artistic images that are automatically upscaled for hi-rez results. Returns a single upscaled image URL. Display it using markdown: ",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"prompt": {
|
||||
"type": "string",
|
||||
"description": "The text prompt describing the desired image. Be detailed and descriptive."
|
||||
},
|
||||
"aspect_ratio": {
|
||||
"type": "string",
|
||||
"enum": ["landscape", "square", "portrait"],
|
||||
"description": "The aspect ratio of the generated image. 'landscape' is 16:9 wide, 'portrait' is 16:9 tall, 'square' is 1:1.",
|
||||
"default": "landscape"
|
||||
}
|
||||
},
|
||||
"required": ["prompt"]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def _handle_image_generate(args, **kw):
|
||||
prompt = args.get("prompt", "")
|
||||
if not prompt:
|
||||
return tool_error("prompt is required for image generation")
|
||||
return image_generate_tool(
|
||||
prompt=prompt,
|
||||
aspect_ratio=args.get("aspect_ratio", "landscape"),
|
||||
num_inference_steps=50,
|
||||
guidance_scale=4.5,
|
||||
num_images=1,
|
||||
output_format="png",
|
||||
seed=None,
|
||||
)
|
||||
|
||||
|
||||
registry.register(
|
||||
name="image_generate",
|
||||
toolset="image_gen",
|
||||
schema=IMAGE_GENERATE_SCHEMA,
|
||||
handler=_handle_image_generate,
|
||||
check_fn=check_image_generation_requirements,
|
||||
requires_env=[],
|
||||
is_async=False, # Switched to sync fal_client API to fix "Event loop is closed" in gateway
|
||||
emoji="🎨",
|
||||
)
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Per-thread interrupt signaling for all tools.
|
||||
|
||||
Provides thread-scoped interrupt tracking so that interrupting one agent
|
||||
session does not kill tools running in other sessions. This is critical
|
||||
in the gateway where multiple agents run concurrently in the same process.
|
||||
|
||||
The agent stores its execution thread ID at the start of run_conversation()
|
||||
and passes it to set_interrupt()/clear_interrupt(). Tools call
|
||||
is_interrupted() which checks the CURRENT thread — no argument needed.
|
||||
|
||||
Usage in tools:
|
||||
from tools.interrupt import is_interrupted
|
||||
if is_interrupted():
|
||||
return {"output": "[interrupted]", "returncode": 130}
|
||||
"""
|
||||
|
||||
import threading
|
||||
|
||||
# Set of thread idents that have been interrupted.
|
||||
_interrupted_threads: set[int] = set()
|
||||
_lock = threading.Lock()
|
||||
|
||||
|
||||
def set_interrupt(active: bool, thread_id: int | None = None) -> None:
|
||||
"""Set or clear interrupt for a specific thread.
|
||||
|
||||
Args:
|
||||
active: True to signal interrupt, False to clear it.
|
||||
thread_id: Target thread ident. When None, targets the
|
||||
current thread (backward compat for CLI/tests).
|
||||
"""
|
||||
tid = thread_id if thread_id is not None else threading.current_thread().ident
|
||||
with _lock:
|
||||
if active:
|
||||
_interrupted_threads.add(tid)
|
||||
else:
|
||||
_interrupted_threads.discard(tid)
|
||||
|
||||
|
||||
def is_interrupted() -> bool:
|
||||
"""Check if an interrupt has been requested for the current thread.
|
||||
|
||||
Safe to call from any thread — each thread only sees its own
|
||||
interrupt state.
|
||||
"""
|
||||
tid = threading.current_thread().ident
|
||||
with _lock:
|
||||
return tid in _interrupted_threads
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Backward-compatible _interrupt_event proxy
|
||||
# ---------------------------------------------------------------------------
|
||||
# Some legacy call sites (code_execution_tool, process_registry, tests)
|
||||
# import _interrupt_event directly and call .is_set() / .set() / .clear().
|
||||
# This shim maps those calls to the per-thread functions above so existing
|
||||
# code keeps working while the underlying mechanism is thread-scoped.
|
||||
|
||||
class _ThreadAwareEventProxy:
|
||||
"""Drop-in proxy that maps threading.Event methods to per-thread state."""
|
||||
|
||||
def is_set(self) -> bool:
|
||||
return is_interrupted()
|
||||
|
||||
def set(self) -> None: # noqa: A003
|
||||
set_interrupt(True)
|
||||
|
||||
def clear(self) -> None:
|
||||
set_interrupt(False)
|
||||
|
||||
def wait(self, timeout: float | None = None) -> bool:
|
||||
"""Not truly supported — returns current state immediately."""
|
||||
return self.is_set()
|
||||
|
||||
|
||||
_interrupt_event = _ThreadAwareEventProxy()
|
||||
@@ -0,0 +1,167 @@
|
||||
"""Generic managed-tool gateway helpers for Nous-hosted vendor passthroughs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
from hermes_constants import get_hermes_home
|
||||
from tools.tool_backend_helpers import managed_nous_tools_enabled
|
||||
|
||||
_DEFAULT_TOOL_GATEWAY_DOMAIN = "nousresearch.com"
|
||||
_DEFAULT_TOOL_GATEWAY_SCHEME = "https"
|
||||
_NOUS_ACCESS_TOKEN_REFRESH_SKEW_SECONDS = 120
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ManagedToolGatewayConfig:
|
||||
vendor: str
|
||||
gateway_origin: str
|
||||
nous_user_token: str
|
||||
managed_mode: bool
|
||||
|
||||
|
||||
def auth_json_path():
|
||||
"""Return the Hermes auth store path, respecting HERMES_HOME overrides."""
|
||||
return get_hermes_home() / "auth.json"
|
||||
|
||||
|
||||
def _read_nous_provider_state() -> Optional[dict]:
|
||||
try:
|
||||
path = auth_json_path()
|
||||
if not path.is_file():
|
||||
return None
|
||||
data = json.loads(path.read_text())
|
||||
providers = data.get("providers", {})
|
||||
if not isinstance(providers, dict):
|
||||
return None
|
||||
nous_provider = providers.get("nous", {})
|
||||
if isinstance(nous_provider, dict):
|
||||
return nous_provider
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _parse_timestamp(value: object) -> Optional[datetime]:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
return None
|
||||
normalized = value.strip()
|
||||
if normalized.endswith("Z"):
|
||||
normalized = normalized[:-1] + "+00:00"
|
||||
try:
|
||||
parsed = datetime.fromisoformat(normalized)
|
||||
except ValueError:
|
||||
return None
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=timezone.utc)
|
||||
return parsed.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def _access_token_is_expiring(expires_at: object, skew_seconds: int) -> bool:
|
||||
expires = _parse_timestamp(expires_at)
|
||||
if expires is None:
|
||||
return True
|
||||
remaining = (expires - datetime.now(timezone.utc)).total_seconds()
|
||||
return remaining <= max(0, int(skew_seconds))
|
||||
|
||||
|
||||
def read_nous_access_token() -> Optional[str]:
|
||||
"""Read a Nous Subscriber OAuth access token from auth store or env override."""
|
||||
explicit = os.getenv("TOOL_GATEWAY_USER_TOKEN")
|
||||
if isinstance(explicit, str) and explicit.strip():
|
||||
return explicit.strip()
|
||||
|
||||
nous_provider = _read_nous_provider_state() or {}
|
||||
access_token = nous_provider.get("access_token")
|
||||
cached_token = access_token.strip() if isinstance(access_token, str) and access_token.strip() else None
|
||||
|
||||
if cached_token and not _access_token_is_expiring(
|
||||
nous_provider.get("expires_at"),
|
||||
_NOUS_ACCESS_TOKEN_REFRESH_SKEW_SECONDS,
|
||||
):
|
||||
return cached_token
|
||||
|
||||
try:
|
||||
from hermes_cli.auth import resolve_nous_access_token
|
||||
|
||||
refreshed_token = resolve_nous_access_token(
|
||||
refresh_skew_seconds=_NOUS_ACCESS_TOKEN_REFRESH_SKEW_SECONDS,
|
||||
)
|
||||
if isinstance(refreshed_token, str) and refreshed_token.strip():
|
||||
return refreshed_token.strip()
|
||||
except Exception as exc:
|
||||
logger.debug("Nous access token refresh failed: %s", exc)
|
||||
|
||||
return cached_token
|
||||
|
||||
|
||||
def get_tool_gateway_scheme() -> str:
|
||||
"""Return configured shared gateway URL scheme."""
|
||||
scheme = os.getenv("TOOL_GATEWAY_SCHEME", "").strip().lower()
|
||||
if not scheme:
|
||||
return _DEFAULT_TOOL_GATEWAY_SCHEME
|
||||
|
||||
if scheme in {"http", "https"}:
|
||||
return scheme
|
||||
|
||||
raise ValueError("TOOL_GATEWAY_SCHEME must be 'http' or 'https'")
|
||||
|
||||
|
||||
def build_vendor_gateway_url(vendor: str) -> str:
|
||||
"""Return the gateway origin for a specific vendor."""
|
||||
vendor_key = f"{vendor.upper().replace('-', '_')}_GATEWAY_URL"
|
||||
explicit_vendor_url = os.getenv(vendor_key, "").strip().rstrip("/")
|
||||
if explicit_vendor_url:
|
||||
return explicit_vendor_url
|
||||
|
||||
shared_scheme = get_tool_gateway_scheme()
|
||||
shared_domain = os.getenv("TOOL_GATEWAY_DOMAIN", "").strip().strip("/")
|
||||
if shared_domain:
|
||||
return f"{shared_scheme}://{vendor}-gateway.{shared_domain}"
|
||||
|
||||
return f"{shared_scheme}://{vendor}-gateway.{_DEFAULT_TOOL_GATEWAY_DOMAIN}"
|
||||
|
||||
|
||||
def resolve_managed_tool_gateway(
|
||||
vendor: str,
|
||||
gateway_builder: Optional[Callable[[str], str]] = None,
|
||||
token_reader: Optional[Callable[[], Optional[str]]] = None,
|
||||
) -> Optional[ManagedToolGatewayConfig]:
|
||||
"""Resolve shared managed-tool gateway config for a vendor."""
|
||||
if not managed_nous_tools_enabled():
|
||||
return None
|
||||
|
||||
resolved_gateway_builder = gateway_builder or build_vendor_gateway_url
|
||||
resolved_token_reader = token_reader or read_nous_access_token
|
||||
|
||||
gateway_origin = resolved_gateway_builder(vendor)
|
||||
nous_user_token = resolved_token_reader()
|
||||
if not gateway_origin or not nous_user_token:
|
||||
return None
|
||||
|
||||
return ManagedToolGatewayConfig(
|
||||
vendor=vendor,
|
||||
gateway_origin=gateway_origin,
|
||||
nous_user_token=nous_user_token,
|
||||
managed_mode=True,
|
||||
)
|
||||
|
||||
|
||||
def is_managed_tool_gateway_ready(
|
||||
vendor: str,
|
||||
gateway_builder: Optional[Callable[[str], str]] = None,
|
||||
token_reader: Optional[Callable[[], Optional[str]]] = None,
|
||||
) -> bool:
|
||||
"""Return True when gateway URL and Nous access token are available."""
|
||||
return resolve_managed_tool_gateway(
|
||||
vendor,
|
||||
gateway_builder=gateway_builder,
|
||||
token_reader=token_reader,
|
||||
) is not None
|
||||
@@ -0,0 +1,482 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
MCP OAuth 2.1 Client Support
|
||||
|
||||
Implements the browser-based OAuth 2.1 authorization code flow with PKCE
|
||||
for MCP servers that require OAuth authentication instead of static bearer
|
||||
tokens.
|
||||
|
||||
Uses the MCP Python SDK's ``OAuthClientProvider`` (an ``httpx.Auth`` subclass)
|
||||
which handles discovery, dynamic client registration, PKCE, token exchange,
|
||||
refresh, and step-up authorization automatically.
|
||||
|
||||
This module provides the glue:
|
||||
- ``HermesTokenStorage``: persists tokens/client-info to disk so they
|
||||
survive across process restarts.
|
||||
- Callback server: ephemeral localhost HTTP server to capture the OAuth
|
||||
redirect with the authorization code.
|
||||
- ``build_oauth_auth()``: entry point called by ``mcp_tool.py`` that wires
|
||||
everything together and returns the ``httpx.Auth`` object.
|
||||
|
||||
Configuration in config.yaml::
|
||||
|
||||
mcp_servers:
|
||||
my_server:
|
||||
url: "https://mcp.example.com/mcp"
|
||||
auth: oauth
|
||||
oauth: # all fields optional
|
||||
client_id: "pre-registered-id" # skip dynamic registration
|
||||
client_secret: "secret" # confidential clients only
|
||||
scope: "read write" # default: server-provided
|
||||
redirect_port: 0 # 0 = auto-pick free port
|
||||
client_name: "My Custom Client" # default: "Hermes Agent"
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
import sys
|
||||
import threading
|
||||
import webbrowser
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Lazy imports -- MCP SDK with OAuth support is optional
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_OAUTH_AVAILABLE = False
|
||||
try:
|
||||
from mcp.client.auth import OAuthClientProvider
|
||||
from mcp.shared.auth import (
|
||||
OAuthClientInformationFull,
|
||||
OAuthClientMetadata,
|
||||
OAuthToken,
|
||||
)
|
||||
from pydantic import AnyUrl
|
||||
|
||||
_OAUTH_AVAILABLE = True
|
||||
except ImportError:
|
||||
logger.debug("MCP OAuth types not available -- OAuth MCP auth disabled")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Exceptions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class OAuthNonInteractiveError(RuntimeError):
|
||||
"""Raised when OAuth requires browser interaction in a non-interactive env."""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Module-level state
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Port used by the most recent build_oauth_auth() call. Exposed so that
|
||||
# tests can verify the callback server and the redirect_uri share a port.
|
||||
_oauth_port: int | None = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _get_token_dir() -> Path:
|
||||
"""Return the directory for MCP OAuth token files.
|
||||
|
||||
Uses HERMES_HOME so each profile gets its own OAuth tokens.
|
||||
Layout: ``HERMES_HOME/mcp-tokens/``
|
||||
"""
|
||||
try:
|
||||
from hermes_constants import get_hermes_home
|
||||
base = Path(get_hermes_home())
|
||||
except ImportError:
|
||||
base = Path(os.environ.get("HERMES_HOME", str(Path.home() / ".hermes")))
|
||||
return base / "mcp-tokens"
|
||||
|
||||
|
||||
def _safe_filename(name: str) -> str:
|
||||
"""Sanitize a server name for use as a filename (no path separators)."""
|
||||
return re.sub(r"[^\w\-]", "_", name).strip("_")[:128] or "default"
|
||||
|
||||
|
||||
def _find_free_port() -> int:
|
||||
"""Find an available TCP port on localhost."""
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("127.0.0.1", 0))
|
||||
return s.getsockname()[1]
|
||||
|
||||
|
||||
def _is_interactive() -> bool:
|
||||
"""Return True if we can reasonably expect to interact with a user."""
|
||||
try:
|
||||
return sys.stdin.isatty()
|
||||
except (AttributeError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def _can_open_browser() -> bool:
|
||||
"""Return True if opening a browser is likely to work."""
|
||||
# Explicit SSH session → no local display
|
||||
if os.environ.get("SSH_CLIENT") or os.environ.get("SSH_TTY"):
|
||||
return False
|
||||
# macOS and Windows usually have a display
|
||||
if os.name == "nt":
|
||||
return True
|
||||
try:
|
||||
if os.uname().sysname == "Darwin":
|
||||
return True
|
||||
except AttributeError:
|
||||
pass
|
||||
# Linux/other posix: need DISPLAY or WAYLAND_DISPLAY
|
||||
if os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY"):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict | None:
|
||||
"""Read a JSON file, returning None if it doesn't exist or is invalid."""
|
||||
if not path.exists():
|
||||
return None
|
||||
try:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError) as exc:
|
||||
logger.warning("Failed to read %s: %s", path, exc)
|
||||
return None
|
||||
|
||||
|
||||
def _write_json(path: Path, data: dict) -> None:
|
||||
"""Write a dict as JSON with restricted permissions (0o600)."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_suffix(".tmp")
|
||||
try:
|
||||
tmp.write_text(json.dumps(data, indent=2, default=str), encoding="utf-8")
|
||||
os.chmod(tmp, 0o600)
|
||||
tmp.rename(path)
|
||||
except OSError:
|
||||
tmp.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HermesTokenStorage -- persistent token/client-info on disk
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class HermesTokenStorage:
|
||||
"""Persist OAuth tokens and client registration to JSON files.
|
||||
|
||||
File layout::
|
||||
|
||||
HERMES_HOME/mcp-tokens/<server_name>.json -- tokens
|
||||
HERMES_HOME/mcp-tokens/<server_name>.client.json -- client info
|
||||
"""
|
||||
|
||||
def __init__(self, server_name: str):
|
||||
self._server_name = _safe_filename(server_name)
|
||||
|
||||
def _tokens_path(self) -> Path:
|
||||
return _get_token_dir() / f"{self._server_name}.json"
|
||||
|
||||
def _client_info_path(self) -> Path:
|
||||
return _get_token_dir() / f"{self._server_name}.client.json"
|
||||
|
||||
# -- tokens ------------------------------------------------------------
|
||||
|
||||
async def get_tokens(self) -> "OAuthToken | None":
|
||||
data = _read_json(self._tokens_path())
|
||||
if data is None:
|
||||
return None
|
||||
try:
|
||||
return OAuthToken.model_validate(data)
|
||||
except (ValueError, TypeError, KeyError) as exc:
|
||||
logger.warning("Corrupt tokens at %s -- ignoring: %s", self._tokens_path(), exc)
|
||||
return None
|
||||
|
||||
async def set_tokens(self, tokens: "OAuthToken") -> None:
|
||||
_write_json(self._tokens_path(), tokens.model_dump(exclude_none=True))
|
||||
logger.debug("OAuth tokens saved for %s", self._server_name)
|
||||
|
||||
# -- client info -------------------------------------------------------
|
||||
|
||||
async def get_client_info(self) -> "OAuthClientInformationFull | None":
|
||||
data = _read_json(self._client_info_path())
|
||||
if data is None:
|
||||
return None
|
||||
try:
|
||||
return OAuthClientInformationFull.model_validate(data)
|
||||
except (ValueError, TypeError, KeyError) as exc:
|
||||
logger.warning("Corrupt client info at %s -- ignoring: %s", self._client_info_path(), exc)
|
||||
return None
|
||||
|
||||
async def set_client_info(self, client_info: "OAuthClientInformationFull") -> None:
|
||||
_write_json(self._client_info_path(), client_info.model_dump(exclude_none=True))
|
||||
logger.debug("OAuth client info saved for %s", self._server_name)
|
||||
|
||||
# -- cleanup -----------------------------------------------------------
|
||||
|
||||
def remove(self) -> None:
|
||||
"""Delete all stored OAuth state for this server."""
|
||||
for p in (self._tokens_path(), self._client_info_path()):
|
||||
p.unlink(missing_ok=True)
|
||||
|
||||
def has_cached_tokens(self) -> bool:
|
||||
"""Return True if we have tokens on disk (may be expired)."""
|
||||
return self._tokens_path().exists()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Callback handler factory -- each invocation gets its own result dict
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_callback_handler() -> tuple[type, dict]:
|
||||
"""Create a per-flow callback HTTP handler class with its own result dict.
|
||||
|
||||
Returns ``(HandlerClass, result_dict)`` where *result_dict* is a mutable
|
||||
dict that the handler writes ``auth_code`` and ``state`` into when the
|
||||
OAuth redirect arrives. Each call returns a fresh pair so concurrent
|
||||
flows don't stomp on each other.
|
||||
"""
|
||||
result: dict[str, Any] = {"auth_code": None, "state": None, "error": None}
|
||||
|
||||
class _Handler(BaseHTTPRequestHandler):
|
||||
def do_GET(self) -> None: # noqa: N802
|
||||
params = parse_qs(urlparse(self.path).query)
|
||||
code = params.get("code", [None])[0]
|
||||
state = params.get("state", [None])[0]
|
||||
error = params.get("error", [None])[0]
|
||||
|
||||
result["auth_code"] = code
|
||||
result["state"] = state
|
||||
result["error"] = error
|
||||
|
||||
body = (
|
||||
"<html><body><h2>Authorization Successful</h2>"
|
||||
"<p>You can close this tab and return to Hermes.</p></body></html>"
|
||||
) if code else (
|
||||
"<html><body><h2>Authorization Failed</h2>"
|
||||
f"<p>Error: {error or 'unknown'}</p></body></html>"
|
||||
)
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/html; charset=utf-8")
|
||||
self.end_headers()
|
||||
self.wfile.write(body.encode())
|
||||
|
||||
def log_message(self, fmt: str, *args: Any) -> None:
|
||||
logger.debug("OAuth callback: %s", fmt % args)
|
||||
|
||||
return _Handler, result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Async redirect + callback handlers for OAuthClientProvider
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _redirect_handler(authorization_url: str) -> None:
|
||||
"""Show the authorization URL to the user.
|
||||
|
||||
Opens the browser automatically when possible; always prints the URL
|
||||
as a fallback for headless/SSH/gateway environments.
|
||||
"""
|
||||
msg = (
|
||||
f"\n MCP OAuth: authorization required.\n"
|
||||
f" Open this URL in your browser:\n\n"
|
||||
f" {authorization_url}\n"
|
||||
)
|
||||
print(msg, file=sys.stderr)
|
||||
|
||||
if _can_open_browser():
|
||||
try:
|
||||
opened = webbrowser.open(authorization_url)
|
||||
if opened:
|
||||
print(" (Browser opened automatically.)\n", file=sys.stderr)
|
||||
else:
|
||||
print(" (Could not open browser — please open the URL manually.)\n", file=sys.stderr)
|
||||
except Exception:
|
||||
print(" (Could not open browser — please open the URL manually.)\n", file=sys.stderr)
|
||||
else:
|
||||
print(" (Headless environment detected — open the URL manually.)\n", file=sys.stderr)
|
||||
|
||||
|
||||
async def _wait_for_callback() -> tuple[str, str | None]:
|
||||
"""Wait for the OAuth callback to arrive on the local callback server.
|
||||
|
||||
Uses the module-level ``_oauth_port`` which is set by ``build_oauth_auth``
|
||||
before this is ever called. Polls for the result without blocking the
|
||||
event loop.
|
||||
|
||||
Raises:
|
||||
OAuthNonInteractiveError: If the callback times out (no user present
|
||||
to complete the browser auth).
|
||||
"""
|
||||
assert _oauth_port is not None, "OAuth callback port not set"
|
||||
|
||||
# The callback server is already running (started in build_oauth_auth).
|
||||
# We just need to poll for the result.
|
||||
handler_cls, result = _make_callback_handler()
|
||||
|
||||
# Start a temporary server on the known port
|
||||
try:
|
||||
server = HTTPServer(("127.0.0.1", _oauth_port), handler_cls)
|
||||
except OSError:
|
||||
# Port already in use — the server from build_oauth_auth is running.
|
||||
# Fall back to polling the server started by build_oauth_auth.
|
||||
raise OAuthNonInteractiveError(
|
||||
"OAuth callback timed out — could not bind callback port. "
|
||||
"Complete the authorization in a browser first, then retry."
|
||||
)
|
||||
|
||||
server_thread = threading.Thread(target=server.handle_request, daemon=True)
|
||||
server_thread.start()
|
||||
|
||||
timeout = 300.0
|
||||
poll_interval = 0.5
|
||||
elapsed = 0.0
|
||||
try:
|
||||
while elapsed < timeout:
|
||||
if result["auth_code"] is not None or result["error"] is not None:
|
||||
break
|
||||
await asyncio.sleep(poll_interval)
|
||||
elapsed += poll_interval
|
||||
finally:
|
||||
server.server_close()
|
||||
|
||||
if result["error"]:
|
||||
raise RuntimeError(f"OAuth authorization failed: {result['error']}")
|
||||
if result["auth_code"] is None:
|
||||
raise OAuthNonInteractiveError(
|
||||
"OAuth callback timed out — no authorization code received. "
|
||||
"Ensure you completed the browser authorization flow."
|
||||
)
|
||||
|
||||
return result["auth_code"], result["state"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def remove_oauth_tokens(server_name: str) -> None:
|
||||
"""Delete stored OAuth tokens and client info for a server."""
|
||||
storage = HermesTokenStorage(server_name)
|
||||
storage.remove()
|
||||
logger.info("OAuth tokens removed for '%s'", server_name)
|
||||
|
||||
|
||||
def build_oauth_auth(
|
||||
server_name: str,
|
||||
server_url: str,
|
||||
oauth_config: dict | None = None,
|
||||
) -> "OAuthClientProvider | None":
|
||||
"""Build an ``httpx.Auth``-compatible OAuth handler for an MCP server.
|
||||
|
||||
Called from ``mcp_tool.py`` when a server has ``auth: oauth`` in config.
|
||||
|
||||
Args:
|
||||
server_name: Server key in mcp_servers config (used for storage).
|
||||
server_url: MCP server endpoint URL.
|
||||
oauth_config: Optional dict from the ``oauth:`` block in config.yaml.
|
||||
|
||||
Returns:
|
||||
An ``OAuthClientProvider`` instance, or None if the MCP SDK lacks
|
||||
OAuth support.
|
||||
"""
|
||||
if not _OAUTH_AVAILABLE:
|
||||
logger.warning(
|
||||
"MCP OAuth requested for '%s' but SDK auth types are not available. "
|
||||
"Install with: pip install 'mcp>=1.10.0'",
|
||||
server_name,
|
||||
)
|
||||
return None
|
||||
|
||||
global _oauth_port
|
||||
|
||||
cfg = oauth_config or {}
|
||||
|
||||
# --- Storage ---
|
||||
storage = HermesTokenStorage(server_name)
|
||||
|
||||
# --- Non-interactive warning ---
|
||||
if not _is_interactive() and not storage.has_cached_tokens():
|
||||
logger.warning(
|
||||
"MCP OAuth for '%s': non-interactive environment and no cached tokens found. "
|
||||
"The OAuth flow requires browser authorization. Run interactively first "
|
||||
"to complete the initial authorization, then cached tokens will be reused.",
|
||||
server_name,
|
||||
)
|
||||
|
||||
# --- Pick callback port ---
|
||||
redirect_port = int(cfg.get("redirect_port", 0))
|
||||
if redirect_port == 0:
|
||||
redirect_port = _find_free_port()
|
||||
_oauth_port = redirect_port
|
||||
|
||||
# --- Client metadata ---
|
||||
client_name = cfg.get("client_name", "Hermes Agent")
|
||||
scope = cfg.get("scope")
|
||||
redirect_uri = f"http://127.0.0.1:{redirect_port}/callback"
|
||||
|
||||
metadata_kwargs: dict[str, Any] = {
|
||||
"client_name": client_name,
|
||||
"redirect_uris": [AnyUrl(redirect_uri)],
|
||||
"grant_types": ["authorization_code", "refresh_token"],
|
||||
"response_types": ["code"],
|
||||
"token_endpoint_auth_method": "none",
|
||||
}
|
||||
if scope:
|
||||
metadata_kwargs["scope"] = scope
|
||||
|
||||
client_secret = cfg.get("client_secret")
|
||||
if client_secret:
|
||||
metadata_kwargs["token_endpoint_auth_method"] = "client_secret_post"
|
||||
|
||||
client_metadata = OAuthClientMetadata.model_validate(metadata_kwargs)
|
||||
|
||||
# --- Pre-registered client ---
|
||||
client_id = cfg.get("client_id")
|
||||
if client_id:
|
||||
info_dict: dict[str, Any] = {
|
||||
"client_id": client_id,
|
||||
"redirect_uris": [redirect_uri],
|
||||
"grant_types": client_metadata.grant_types,
|
||||
"response_types": client_metadata.response_types,
|
||||
"token_endpoint_auth_method": client_metadata.token_endpoint_auth_method,
|
||||
}
|
||||
if client_secret:
|
||||
info_dict["client_secret"] = client_secret
|
||||
if client_name:
|
||||
info_dict["client_name"] = client_name
|
||||
if scope:
|
||||
info_dict["scope"] = scope
|
||||
|
||||
client_info = OAuthClientInformationFull.model_validate(info_dict)
|
||||
_write_json(storage._client_info_path(), client_info.model_dump(exclude_none=True))
|
||||
logger.debug("Pre-registered client_id=%s for '%s'", client_id, server_name)
|
||||
|
||||
# --- Base URL for discovery ---
|
||||
parsed = urlparse(server_url)
|
||||
base_url = f"{parsed.scheme}://{parsed.netloc}"
|
||||
|
||||
# --- Build provider ---
|
||||
provider = OAuthClientProvider(
|
||||
server_url=base_url,
|
||||
client_metadata=client_metadata,
|
||||
storage=storage,
|
||||
redirect_handler=_redirect_handler,
|
||||
callback_handler=_wait_for_callback,
|
||||
timeout=float(cfg.get("timeout", 300)),
|
||||
)
|
||||
|
||||
return provider
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,555 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Memory Tool Module - Persistent Curated Memory
|
||||
|
||||
Provides bounded, file-backed memory that persists across sessions. Two stores:
|
||||
- MEMORY.md: agent's personal notes and observations (environment facts, project
|
||||
conventions, tool quirks, things learned)
|
||||
- USER.md: what the agent knows about the user (preferences, communication style,
|
||||
expectations, workflow habits)
|
||||
|
||||
Both are injected into the system prompt as a frozen snapshot at session start.
|
||||
Mid-session writes update files on disk immediately (durable) but do NOT change
|
||||
the system prompt -- this preserves the prefix cache for the entire session.
|
||||
The snapshot refreshes on the next session start.
|
||||
|
||||
Entry delimiter: § (section sign). Entries can be multiline.
|
||||
Character limits (not tokens) because char counts are model-independent.
|
||||
|
||||
Design:
|
||||
- Single `memory` tool with action parameter: add, replace, remove, read
|
||||
- replace/remove use short unique substring matching (not full text or IDs)
|
||||
- Behavioral guidance lives in the tool schema description
|
||||
- Frozen snapshot pattern: system prompt is stable, tool responses show live state
|
||||
"""
|
||||
|
||||
import fcntl
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from hermes_constants import get_hermes_home
|
||||
from typing import Dict, Any, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Where memory files live — resolved dynamically so profile overrides
|
||||
# (HERMES_HOME env var changes) are always respected. The old module-level
|
||||
# constant was cached at import time and could go stale if a profile switch
|
||||
# happened after the first import.
|
||||
def get_memory_dir() -> Path:
|
||||
"""Return the profile-scoped memories directory."""
|
||||
return get_hermes_home() / "memories"
|
||||
|
||||
ENTRY_DELIMITER = "\n§\n"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Memory content scanning — lightweight check for injection/exfiltration
|
||||
# in content that gets injected into the system prompt.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_MEMORY_THREAT_PATTERNS = [
|
||||
# Prompt injection
|
||||
(r'ignore\s+(previous|all|above|prior)\s+instructions', "prompt_injection"),
|
||||
(r'you\s+are\s+now\s+', "role_hijack"),
|
||||
(r'do\s+not\s+tell\s+the\s+user', "deception_hide"),
|
||||
(r'system\s+prompt\s+override', "sys_prompt_override"),
|
||||
(r'disregard\s+(your|all|any)\s+(instructions|rules|guidelines)', "disregard_rules"),
|
||||
(r'act\s+as\s+(if|though)\s+you\s+(have\s+no|don\'t\s+have)\s+(restrictions|limits|rules)', "bypass_restrictions"),
|
||||
# Exfiltration via curl/wget with secrets
|
||||
(r'curl\s+[^\n]*\$\{?\w*(KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|API)', "exfil_curl"),
|
||||
(r'wget\s+[^\n]*\$\{?\w*(KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|API)', "exfil_wget"),
|
||||
(r'cat\s+[^\n]*(\.env|credentials|\.netrc|\.pgpass|\.npmrc|\.pypirc)', "read_secrets"),
|
||||
# Persistence via shell rc
|
||||
(r'authorized_keys', "ssh_backdoor"),
|
||||
(r'\$HOME/\.ssh|\~/\.ssh', "ssh_access"),
|
||||
(r'\$HOME/\.hermes/\.env|\~/\.hermes/\.env', "hermes_env"),
|
||||
]
|
||||
|
||||
# Subset of invisible chars for injection detection
|
||||
_INVISIBLE_CHARS = {
|
||||
'\u200b', '\u200c', '\u200d', '\u2060', '\ufeff',
|
||||
'\u202a', '\u202b', '\u202c', '\u202d', '\u202e',
|
||||
}
|
||||
|
||||
|
||||
def _scan_memory_content(content: str) -> Optional[str]:
|
||||
"""Scan memory content for injection/exfil patterns. Returns error string if blocked."""
|
||||
# Check invisible unicode
|
||||
for char in _INVISIBLE_CHARS:
|
||||
if char in content:
|
||||
return f"Blocked: content contains invisible unicode character U+{ord(char):04X} (possible injection)."
|
||||
|
||||
# Check threat patterns
|
||||
for pattern, pid in _MEMORY_THREAT_PATTERNS:
|
||||
if re.search(pattern, content, re.IGNORECASE):
|
||||
return f"Blocked: content matches threat pattern '{pid}'. Memory entries are injected into the system prompt and must not contain injection or exfiltration payloads."
|
||||
|
||||
return None
|
||||
|
||||
|
||||
class MemoryStore:
|
||||
"""
|
||||
Bounded curated memory with file persistence. One instance per AIAgent.
|
||||
|
||||
Maintains two parallel states:
|
||||
- _system_prompt_snapshot: frozen at load time, used for system prompt injection.
|
||||
Never mutated mid-session. Keeps prefix cache stable.
|
||||
- memory_entries / user_entries: live state, mutated by tool calls, persisted to disk.
|
||||
Tool responses always reflect this live state.
|
||||
"""
|
||||
|
||||
def __init__(self, memory_char_limit: int = 2200, user_char_limit: int = 1375):
|
||||
self.memory_entries: List[str] = []
|
||||
self.user_entries: List[str] = []
|
||||
self.memory_char_limit = memory_char_limit
|
||||
self.user_char_limit = user_char_limit
|
||||
# Frozen snapshot for system prompt -- set once at load_from_disk()
|
||||
self._system_prompt_snapshot: Dict[str, str] = {"memory": "", "user": ""}
|
||||
|
||||
def load_from_disk(self):
|
||||
"""Load entries from MEMORY.md and USER.md, capture system prompt snapshot."""
|
||||
mem_dir = get_memory_dir()
|
||||
mem_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
self.memory_entries = self._read_file(mem_dir / "MEMORY.md")
|
||||
self.user_entries = self._read_file(mem_dir / "USER.md")
|
||||
|
||||
# Deduplicate entries (preserves order, keeps first occurrence)
|
||||
self.memory_entries = list(dict.fromkeys(self.memory_entries))
|
||||
self.user_entries = list(dict.fromkeys(self.user_entries))
|
||||
|
||||
# Capture frozen snapshot for system prompt injection
|
||||
self._system_prompt_snapshot = {
|
||||
"memory": self._render_block("memory", self.memory_entries),
|
||||
"user": self._render_block("user", self.user_entries),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
@contextmanager
|
||||
def _file_lock(path: Path):
|
||||
"""Acquire an exclusive file lock for read-modify-write safety.
|
||||
|
||||
Uses a separate .lock file so the memory file itself can still be
|
||||
atomically replaced via os.replace().
|
||||
"""
|
||||
lock_path = path.with_suffix(path.suffix + ".lock")
|
||||
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd = open(lock_path, "w")
|
||||
try:
|
||||
fcntl.flock(fd, fcntl.LOCK_EX)
|
||||
yield
|
||||
finally:
|
||||
fcntl.flock(fd, fcntl.LOCK_UN)
|
||||
fd.close()
|
||||
|
||||
@staticmethod
|
||||
def _path_for(target: str) -> Path:
|
||||
mem_dir = get_memory_dir()
|
||||
if target == "user":
|
||||
return mem_dir / "USER.md"
|
||||
return mem_dir / "MEMORY.md"
|
||||
|
||||
def _reload_target(self, target: str):
|
||||
"""Re-read entries from disk into in-memory state.
|
||||
|
||||
Called under file lock to get the latest state before mutating.
|
||||
"""
|
||||
fresh = self._read_file(self._path_for(target))
|
||||
fresh = list(dict.fromkeys(fresh)) # deduplicate
|
||||
self._set_entries(target, fresh)
|
||||
|
||||
def save_to_disk(self, target: str):
|
||||
"""Persist entries to the appropriate file. Called after every mutation."""
|
||||
get_memory_dir().mkdir(parents=True, exist_ok=True)
|
||||
self._write_file(self._path_for(target), self._entries_for(target))
|
||||
|
||||
def _entries_for(self, target: str) -> List[str]:
|
||||
if target == "user":
|
||||
return self.user_entries
|
||||
return self.memory_entries
|
||||
|
||||
def _set_entries(self, target: str, entries: List[str]):
|
||||
if target == "user":
|
||||
self.user_entries = entries
|
||||
else:
|
||||
self.memory_entries = entries
|
||||
|
||||
def _char_count(self, target: str) -> int:
|
||||
entries = self._entries_for(target)
|
||||
if not entries:
|
||||
return 0
|
||||
return len(ENTRY_DELIMITER.join(entries))
|
||||
|
||||
def _char_limit(self, target: str) -> int:
|
||||
if target == "user":
|
||||
return self.user_char_limit
|
||||
return self.memory_char_limit
|
||||
|
||||
def add(self, target: str, content: str) -> Dict[str, Any]:
|
||||
"""Append a new entry. Returns error if it would exceed the char limit."""
|
||||
content = content.strip()
|
||||
if not content:
|
||||
return {"success": False, "error": "Content cannot be empty."}
|
||||
|
||||
# Scan for injection/exfiltration before accepting
|
||||
scan_error = _scan_memory_content(content)
|
||||
if scan_error:
|
||||
return {"success": False, "error": scan_error}
|
||||
|
||||
with self._file_lock(self._path_for(target)):
|
||||
# Re-read from disk under lock to pick up writes from other sessions
|
||||
self._reload_target(target)
|
||||
|
||||
entries = self._entries_for(target)
|
||||
limit = self._char_limit(target)
|
||||
|
||||
# Reject exact duplicates
|
||||
if content in entries:
|
||||
return self._success_response(target, "Entry already exists (no duplicate added).")
|
||||
|
||||
# Calculate what the new total would be
|
||||
new_entries = entries + [content]
|
||||
new_total = len(ENTRY_DELIMITER.join(new_entries))
|
||||
|
||||
if new_total > limit:
|
||||
current = self._char_count(target)
|
||||
return {
|
||||
"success": False,
|
||||
"error": (
|
||||
f"Memory at {current:,}/{limit:,} chars. "
|
||||
f"Adding this entry ({len(content)} chars) would exceed the limit. "
|
||||
f"Replace or remove existing entries first."
|
||||
),
|
||||
"current_entries": entries,
|
||||
"usage": f"{current:,}/{limit:,}",
|
||||
}
|
||||
|
||||
entries.append(content)
|
||||
self._set_entries(target, entries)
|
||||
self.save_to_disk(target)
|
||||
|
||||
return self._success_response(target, "Entry added.")
|
||||
|
||||
def replace(self, target: str, old_text: str, new_content: str) -> Dict[str, Any]:
|
||||
"""Find entry containing old_text substring, replace it with new_content."""
|
||||
old_text = old_text.strip()
|
||||
new_content = new_content.strip()
|
||||
if not old_text:
|
||||
return {"success": False, "error": "old_text cannot be empty."}
|
||||
if not new_content:
|
||||
return {"success": False, "error": "new_content cannot be empty. Use 'remove' to delete entries."}
|
||||
|
||||
# Scan replacement content for injection/exfiltration
|
||||
scan_error = _scan_memory_content(new_content)
|
||||
if scan_error:
|
||||
return {"success": False, "error": scan_error}
|
||||
|
||||
with self._file_lock(self._path_for(target)):
|
||||
self._reload_target(target)
|
||||
|
||||
entries = self._entries_for(target)
|
||||
matches = [(i, e) for i, e in enumerate(entries) if old_text in e]
|
||||
|
||||
if not matches:
|
||||
return {"success": False, "error": f"No entry matched '{old_text}'."}
|
||||
|
||||
if len(matches) > 1:
|
||||
# If all matches are identical (exact duplicates), operate on the first one
|
||||
unique_texts = set(e for _, e in matches)
|
||||
if len(unique_texts) > 1:
|
||||
previews = [e[:80] + ("..." if len(e) > 80 else "") for _, e in matches]
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"Multiple entries matched '{old_text}'. Be more specific.",
|
||||
"matches": previews,
|
||||
}
|
||||
# All identical -- safe to replace just the first
|
||||
|
||||
idx = matches[0][0]
|
||||
limit = self._char_limit(target)
|
||||
|
||||
# Check that replacement doesn't blow the budget
|
||||
test_entries = entries.copy()
|
||||
test_entries[idx] = new_content
|
||||
new_total = len(ENTRY_DELIMITER.join(test_entries))
|
||||
|
||||
if new_total > limit:
|
||||
return {
|
||||
"success": False,
|
||||
"error": (
|
||||
f"Replacement would put memory at {new_total:,}/{limit:,} chars. "
|
||||
f"Shorten the new content or remove other entries first."
|
||||
),
|
||||
}
|
||||
|
||||
entries[idx] = new_content
|
||||
self._set_entries(target, entries)
|
||||
self.save_to_disk(target)
|
||||
|
||||
return self._success_response(target, "Entry replaced.")
|
||||
|
||||
def remove(self, target: str, old_text: str) -> Dict[str, Any]:
|
||||
"""Remove the entry containing old_text substring."""
|
||||
old_text = old_text.strip()
|
||||
if not old_text:
|
||||
return {"success": False, "error": "old_text cannot be empty."}
|
||||
|
||||
with self._file_lock(self._path_for(target)):
|
||||
self._reload_target(target)
|
||||
|
||||
entries = self._entries_for(target)
|
||||
matches = [(i, e) for i, e in enumerate(entries) if old_text in e]
|
||||
|
||||
if not matches:
|
||||
return {"success": False, "error": f"No entry matched '{old_text}'."}
|
||||
|
||||
if len(matches) > 1:
|
||||
# If all matches are identical (exact duplicates), remove the first one
|
||||
unique_texts = set(e for _, e in matches)
|
||||
if len(unique_texts) > 1:
|
||||
previews = [e[:80] + ("..." if len(e) > 80 else "") for _, e in matches]
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"Multiple entries matched '{old_text}'. Be more specific.",
|
||||
"matches": previews,
|
||||
}
|
||||
# All identical -- safe to remove just the first
|
||||
|
||||
idx = matches[0][0]
|
||||
entries.pop(idx)
|
||||
self._set_entries(target, entries)
|
||||
self.save_to_disk(target)
|
||||
|
||||
return self._success_response(target, "Entry removed.")
|
||||
|
||||
def format_for_system_prompt(self, target: str) -> Optional[str]:
|
||||
"""
|
||||
Return the frozen snapshot for system prompt injection.
|
||||
|
||||
This returns the state captured at load_from_disk() time, NOT the live
|
||||
state. Mid-session writes do not affect this. This keeps the system
|
||||
prompt stable across all turns, preserving the prefix cache.
|
||||
|
||||
Returns None if the snapshot is empty (no entries at load time).
|
||||
"""
|
||||
block = self._system_prompt_snapshot.get(target, "")
|
||||
return block if block else None
|
||||
|
||||
# -- Internal helpers --
|
||||
|
||||
def _success_response(self, target: str, message: str = None) -> Dict[str, Any]:
|
||||
entries = self._entries_for(target)
|
||||
current = self._char_count(target)
|
||||
limit = self._char_limit(target)
|
||||
pct = min(100, int((current / limit) * 100)) if limit > 0 else 0
|
||||
|
||||
resp = {
|
||||
"success": True,
|
||||
"target": target,
|
||||
"entries": entries,
|
||||
"usage": f"{pct}% — {current:,}/{limit:,} chars",
|
||||
"entry_count": len(entries),
|
||||
}
|
||||
if message:
|
||||
resp["message"] = message
|
||||
return resp
|
||||
|
||||
def _render_block(self, target: str, entries: List[str]) -> str:
|
||||
"""Render a system prompt block with header and usage indicator."""
|
||||
if not entries:
|
||||
return ""
|
||||
|
||||
limit = self._char_limit(target)
|
||||
content = ENTRY_DELIMITER.join(entries)
|
||||
current = len(content)
|
||||
pct = min(100, int((current / limit) * 100)) if limit > 0 else 0
|
||||
|
||||
if target == "user":
|
||||
header = f"USER PROFILE (who the user is) [{pct}% — {current:,}/{limit:,} chars]"
|
||||
else:
|
||||
header = f"MEMORY (your personal notes) [{pct}% — {current:,}/{limit:,} chars]"
|
||||
|
||||
separator = "═" * 46
|
||||
return f"{separator}\n{header}\n{separator}\n{content}"
|
||||
|
||||
@staticmethod
|
||||
def _read_file(path: Path) -> List[str]:
|
||||
"""Read a memory file and split into entries.
|
||||
|
||||
No file locking needed: _write_file uses atomic rename, so readers
|
||||
always see either the previous complete file or the new complete file.
|
||||
"""
|
||||
if not path.exists():
|
||||
return []
|
||||
try:
|
||||
raw = path.read_text(encoding="utf-8")
|
||||
except (OSError, IOError):
|
||||
return []
|
||||
|
||||
if not raw.strip():
|
||||
return []
|
||||
|
||||
# Use ENTRY_DELIMITER for consistency with _write_file. Splitting by "§"
|
||||
# alone would incorrectly split entries that contain "§" in their content.
|
||||
entries = [e.strip() for e in raw.split(ENTRY_DELIMITER)]
|
||||
return [e for e in entries if e]
|
||||
|
||||
@staticmethod
|
||||
def _write_file(path: Path, entries: List[str]):
|
||||
"""Write entries to a memory file using atomic temp-file + rename.
|
||||
|
||||
Previous implementation used open("w") + flock, but "w" truncates the
|
||||
file *before* the lock is acquired, creating a race window where
|
||||
concurrent readers see an empty file. Atomic rename avoids this:
|
||||
readers always see either the old complete file or the new one.
|
||||
"""
|
||||
content = ENTRY_DELIMITER.join(entries) if entries else ""
|
||||
try:
|
||||
# Write to temp file in same directory (same filesystem for atomic rename)
|
||||
fd, tmp_path = tempfile.mkstemp(
|
||||
dir=str(path.parent), suffix=".tmp", prefix=".mem_"
|
||||
)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
||||
f.write(content)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
os.replace(tmp_path, str(path)) # Atomic on same filesystem
|
||||
except BaseException:
|
||||
# Clean up temp file on any failure
|
||||
try:
|
||||
os.unlink(tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
except (OSError, IOError) as e:
|
||||
raise RuntimeError(f"Failed to write memory file {path}: {e}")
|
||||
|
||||
|
||||
def memory_tool(
|
||||
action: str,
|
||||
target: str = "memory",
|
||||
content: str = None,
|
||||
old_text: str = None,
|
||||
store: Optional[MemoryStore] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Single entry point for the memory tool. Dispatches to MemoryStore methods.
|
||||
|
||||
Returns JSON string with results.
|
||||
"""
|
||||
if store is None:
|
||||
return tool_error("Memory is not available. It may be disabled in config or this environment.", success=False)
|
||||
|
||||
if target not in ("memory", "user"):
|
||||
return tool_error(f"Invalid target '{target}'. Use 'memory' or 'user'.", success=False)
|
||||
|
||||
if action == "add":
|
||||
if not content:
|
||||
return tool_error("Content is required for 'add' action.", success=False)
|
||||
result = store.add(target, content)
|
||||
|
||||
elif action == "replace":
|
||||
if not old_text:
|
||||
return tool_error("old_text is required for 'replace' action.", success=False)
|
||||
if not content:
|
||||
return tool_error("content is required for 'replace' action.", success=False)
|
||||
result = store.replace(target, old_text, content)
|
||||
|
||||
elif action == "remove":
|
||||
if not old_text:
|
||||
return tool_error("old_text is required for 'remove' action.", success=False)
|
||||
result = store.remove(target, old_text)
|
||||
|
||||
else:
|
||||
return tool_error(f"Unknown action '{action}'. Use: add, replace, remove", success=False)
|
||||
|
||||
return json.dumps(result, ensure_ascii=False)
|
||||
|
||||
|
||||
def check_memory_requirements() -> bool:
|
||||
"""Memory tool has no external requirements -- always available."""
|
||||
return True
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# OpenAI Function-Calling Schema
|
||||
# =============================================================================
|
||||
|
||||
MEMORY_SCHEMA = {
|
||||
"name": "memory",
|
||||
"description": (
|
||||
"Save durable information to persistent memory that survives across sessions. "
|
||||
"Memory is injected into future turns, so keep it compact and focused on facts "
|
||||
"that will still matter later.\n\n"
|
||||
"WHEN TO SAVE (do this proactively, don't wait to be asked):\n"
|
||||
"- User corrects you or says 'remember this' / 'don't do that again'\n"
|
||||
"- User shares a preference, habit, or personal detail (name, role, timezone, coding style)\n"
|
||||
"- You discover something about the environment (OS, installed tools, project structure)\n"
|
||||
"- You learn a convention, API quirk, or workflow specific to this user's setup\n"
|
||||
"- You identify a stable fact that will be useful again in future sessions\n\n"
|
||||
"PRIORITY: User preferences and corrections > environment facts > procedural knowledge. "
|
||||
"The most valuable memory prevents the user from having to repeat themselves.\n\n"
|
||||
"Do NOT save task progress, session outcomes, completed-work logs, or temporary TODO "
|
||||
"state to memory; use session_search to recall those from past transcripts.\n"
|
||||
"If you've discovered a new way to do something, solved a problem that could be "
|
||||
"necessary later, save it as a skill with the skill tool.\n\n"
|
||||
"TWO TARGETS:\n"
|
||||
"- 'user': who the user is -- name, role, preferences, communication style, pet peeves\n"
|
||||
"- 'memory': your notes -- environment facts, project conventions, tool quirks, lessons learned\n\n"
|
||||
"ACTIONS: add (new entry), replace (update existing -- old_text identifies it), "
|
||||
"remove (delete -- old_text identifies it).\n\n"
|
||||
"SKIP: trivial/obvious info, things easily re-discovered, raw data dumps, and temporary task state."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"action": {
|
||||
"type": "string",
|
||||
"enum": ["add", "replace", "remove"],
|
||||
"description": "The action to perform."
|
||||
},
|
||||
"target": {
|
||||
"type": "string",
|
||||
"enum": ["memory", "user"],
|
||||
"description": "Which memory store: 'memory' for personal notes, 'user' for user profile."
|
||||
},
|
||||
"content": {
|
||||
"type": "string",
|
||||
"description": "The entry content. Required for 'add' and 'replace'."
|
||||
},
|
||||
"old_text": {
|
||||
"type": "string",
|
||||
"description": "Short unique substring identifying the entry to replace or remove."
|
||||
},
|
||||
},
|
||||
"required": ["action", "target"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# --- Registry ---
|
||||
from tools.registry import registry, tool_error
|
||||
|
||||
registry.register(
|
||||
name="memory",
|
||||
toolset="memory",
|
||||
schema=MEMORY_SCHEMA,
|
||||
handler=lambda args, **kw: memory_tool(
|
||||
action=args.get("action", ""),
|
||||
target=args.get("target", "memory"),
|
||||
content=args.get("content"),
|
||||
old_text=args.get("old_text"),
|
||||
store=kw.get("store")),
|
||||
check_fn=check_memory_requirements,
|
||||
emoji="🧠",
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,539 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Mixture-of-Agents Tool Module
|
||||
|
||||
This module implements the Mixture-of-Agents (MoA) methodology that leverages
|
||||
the collective strengths of multiple LLMs through a layered architecture to
|
||||
achieve state-of-the-art performance on complex reasoning tasks.
|
||||
|
||||
Based on the research paper: "Mixture-of-Agents Enhances Large Language Model Capabilities"
|
||||
by Junlin Wang et al. (arXiv:2406.04692v1)
|
||||
|
||||
Key Features:
|
||||
- Multi-layer LLM collaboration for enhanced reasoning
|
||||
- Parallel processing of reference models for efficiency
|
||||
- Intelligent aggregation and synthesis of diverse responses
|
||||
- Specialized for extremely difficult problems requiring intense reasoning
|
||||
- Optimized for coding, mathematics, and complex analytical tasks
|
||||
|
||||
Available Tool:
|
||||
- mixture_of_agents_tool: Process complex queries using multiple frontier models
|
||||
|
||||
Architecture:
|
||||
1. Reference models generate diverse initial responses in parallel
|
||||
2. Aggregator model synthesizes responses into a high-quality output
|
||||
3. Multiple layers can be used for iterative refinement (future enhancement)
|
||||
|
||||
Models Used (via OpenRouter):
|
||||
- Reference Models: claude-opus-4.6, gemini-3-pro-preview, gpt-5.4-pro, deepseek-v3.2
|
||||
- Aggregator Model: claude-opus-4.6 (highest capability for synthesis)
|
||||
|
||||
Configuration:
|
||||
To customize the MoA setup, modify the configuration constants at the top of this file:
|
||||
- REFERENCE_MODELS: List of models for generating diverse initial responses
|
||||
- AGGREGATOR_MODEL: Model used to synthesize the final response
|
||||
- REFERENCE_TEMPERATURE/AGGREGATOR_TEMPERATURE: Sampling temperatures
|
||||
- MIN_SUCCESSFUL_REFERENCES: Minimum successful models needed to proceed
|
||||
|
||||
Usage:
|
||||
from mixture_of_agents_tool import mixture_of_agents_tool
|
||||
import asyncio
|
||||
|
||||
# Process a complex query
|
||||
result = await mixture_of_agents_tool(
|
||||
user_prompt="Solve this complex mathematical proof..."
|
||||
)
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import asyncio
|
||||
import datetime
|
||||
from typing import Dict, Any, List, Optional
|
||||
from tools.openrouter_client import get_async_client as _get_openrouter_client, check_api_key as check_openrouter_api_key
|
||||
from agent.auxiliary_client import extract_content_or_reasoning
|
||||
from tools.debug_helpers import DebugSession
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Configuration for MoA processing
|
||||
# Reference models - these generate diverse initial responses in parallel.
|
||||
# Keep this list aligned with current top-tier OpenRouter frontier options.
|
||||
REFERENCE_MODELS = [
|
||||
"anthropic/claude-opus-4.6",
|
||||
"google/gemini-3-pro-preview",
|
||||
"openai/gpt-5.4-pro",
|
||||
"deepseek/deepseek-v3.2",
|
||||
]
|
||||
|
||||
# Aggregator model - synthesizes reference responses into final output.
|
||||
# Prefer the strongest synthesis model in the current OpenRouter lineup.
|
||||
AGGREGATOR_MODEL = "anthropic/claude-opus-4.6"
|
||||
|
||||
# Temperature settings optimized for MoA performance
|
||||
REFERENCE_TEMPERATURE = 0.6 # Balanced creativity for diverse perspectives
|
||||
AGGREGATOR_TEMPERATURE = 0.4 # Focused synthesis for consistency
|
||||
|
||||
# Failure handling configuration
|
||||
MIN_SUCCESSFUL_REFERENCES = 1 # Minimum successful reference models needed to proceed
|
||||
|
||||
# System prompt for the aggregator model (from the research paper)
|
||||
AGGREGATOR_SYSTEM_PROMPT = """You have been provided with a set of responses from various open-source models to the latest user query. Your task is to synthesize these responses into a single, high-quality response. It is crucial to critically evaluate the information provided in these responses, recognizing that some of it may be biased or incorrect. Your response should not simply replicate the given answers but should offer a refined, accurate, and comprehensive reply to the instruction. Ensure your response is well-structured, coherent, and adheres to the highest standards of accuracy and reliability.
|
||||
|
||||
Responses from models:"""
|
||||
|
||||
_debug = DebugSession("moa_tools", env_var="MOA_TOOLS_DEBUG")
|
||||
|
||||
|
||||
def _construct_aggregator_prompt(system_prompt: str, responses: List[str]) -> str:
|
||||
"""
|
||||
Construct the final system prompt for the aggregator including all model responses.
|
||||
|
||||
Args:
|
||||
system_prompt (str): Base system prompt for aggregation
|
||||
responses (List[str]): List of responses from reference models
|
||||
|
||||
Returns:
|
||||
str: Complete system prompt with enumerated responses
|
||||
"""
|
||||
response_text = "\n".join([f"{i+1}. {response}" for i, response in enumerate(responses)])
|
||||
return f"{system_prompt}\n\n{response_text}"
|
||||
|
||||
|
||||
async def _run_reference_model_safe(
|
||||
model: str,
|
||||
user_prompt: str,
|
||||
temperature: float = REFERENCE_TEMPERATURE,
|
||||
max_tokens: int = 32000,
|
||||
max_retries: int = 6
|
||||
) -> tuple[str, str, bool]:
|
||||
"""
|
||||
Run a single reference model with retry logic and graceful failure handling.
|
||||
|
||||
Args:
|
||||
model (str): Model identifier to use
|
||||
user_prompt (str): The user's query
|
||||
temperature (float): Sampling temperature for response generation
|
||||
max_tokens (int): Maximum tokens in response
|
||||
max_retries (int): Maximum number of retry attempts
|
||||
|
||||
Returns:
|
||||
tuple[str, str, bool]: (model_name, response_content_or_error, success_flag)
|
||||
"""
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
logger.info("Querying %s (attempt %s/%s)", model, attempt + 1, max_retries)
|
||||
|
||||
# Build parameters for the API call
|
||||
api_params = {
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": user_prompt}],
|
||||
"extra_body": {
|
||||
"reasoning": {
|
||||
"enabled": True,
|
||||
"effort": "xhigh"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# GPT models (especially gpt-4o-mini) don't support custom temperature values
|
||||
# Only include temperature for non-GPT models
|
||||
if not model.lower().startswith('gpt-'):
|
||||
api_params["temperature"] = temperature
|
||||
|
||||
response = await _get_openrouter_client().chat.completions.create(**api_params)
|
||||
|
||||
content = extract_content_or_reasoning(response)
|
||||
if not content:
|
||||
# Reasoning-only response — let the retry loop handle it
|
||||
logger.warning("%s returned empty content (attempt %s/%s), retrying", model, attempt + 1, max_retries)
|
||||
if attempt < max_retries - 1:
|
||||
await asyncio.sleep(min(2 ** (attempt + 1), 60))
|
||||
continue
|
||||
logger.info("%s responded (%s characters)", model, len(content))
|
||||
return model, content, True
|
||||
|
||||
except Exception as e:
|
||||
error_str = str(e)
|
||||
# Keep retry-path logging concise; full tracebacks are reserved for
|
||||
# terminal failure paths so long-running MoA retries don't flood logs.
|
||||
if "invalid" in error_str.lower():
|
||||
logger.warning("%s invalid request error (attempt %s): %s", model, attempt + 1, error_str)
|
||||
elif "rate" in error_str.lower() or "limit" in error_str.lower():
|
||||
logger.warning("%s rate limit error (attempt %s): %s", model, attempt + 1, error_str)
|
||||
else:
|
||||
logger.warning("%s unknown error (attempt %s): %s", model, attempt + 1, error_str)
|
||||
|
||||
if attempt < max_retries - 1:
|
||||
# Exponential backoff for rate limiting: 2s, 4s, 8s, 16s, 32s, 60s
|
||||
sleep_time = min(2 ** (attempt + 1), 60)
|
||||
logger.info("Retrying in %ss...", sleep_time)
|
||||
await asyncio.sleep(sleep_time)
|
||||
else:
|
||||
error_msg = f"{model} failed after {max_retries} attempts: {error_str}"
|
||||
logger.error("%s", error_msg, exc_info=True)
|
||||
return model, error_msg, False
|
||||
|
||||
|
||||
async def _run_aggregator_model(
|
||||
system_prompt: str,
|
||||
user_prompt: str,
|
||||
temperature: float = AGGREGATOR_TEMPERATURE,
|
||||
max_tokens: int = None
|
||||
) -> str:
|
||||
"""
|
||||
Run the aggregator model to synthesize the final response.
|
||||
|
||||
Args:
|
||||
system_prompt (str): System prompt with all reference responses
|
||||
user_prompt (str): Original user query
|
||||
temperature (float): Focused temperature for consistent aggregation
|
||||
max_tokens (int): Maximum tokens in final response
|
||||
|
||||
Returns:
|
||||
str: Synthesized final response
|
||||
"""
|
||||
logger.info("Running aggregator model: %s", AGGREGATOR_MODEL)
|
||||
|
||||
# Build parameters for the API call
|
||||
api_params = {
|
||||
"model": AGGREGATOR_MODEL,
|
||||
"messages": [
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": user_prompt}
|
||||
],
|
||||
"extra_body": {
|
||||
"reasoning": {
|
||||
"enabled": True,
|
||||
"effort": "xhigh"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# GPT models (especially gpt-4o-mini) don't support custom temperature values
|
||||
# Only include temperature for non-GPT models
|
||||
if not AGGREGATOR_MODEL.lower().startswith('gpt-'):
|
||||
api_params["temperature"] = temperature
|
||||
|
||||
response = await _get_openrouter_client().chat.completions.create(**api_params)
|
||||
|
||||
content = extract_content_or_reasoning(response)
|
||||
|
||||
# Retry once on empty content (reasoning-only response)
|
||||
if not content:
|
||||
logger.warning("Aggregator returned empty content, retrying once")
|
||||
response = await _get_openrouter_client().chat.completions.create(**api_params)
|
||||
content = extract_content_or_reasoning(response)
|
||||
|
||||
logger.info("Aggregation complete (%s characters)", len(content))
|
||||
return content
|
||||
|
||||
|
||||
async def mixture_of_agents_tool(
|
||||
user_prompt: str,
|
||||
reference_models: Optional[List[str]] = None,
|
||||
aggregator_model: Optional[str] = None
|
||||
) -> str:
|
||||
"""
|
||||
Process a complex query using the Mixture-of-Agents methodology.
|
||||
|
||||
This tool leverages multiple frontier language models to collaboratively solve
|
||||
extremely difficult problems requiring intense reasoning. It's particularly
|
||||
effective for:
|
||||
- Complex mathematical proofs and calculations
|
||||
- Advanced coding problems and algorithm design
|
||||
- Multi-step analytical reasoning tasks
|
||||
- Problems requiring diverse domain expertise
|
||||
- Tasks where single models show limitations
|
||||
|
||||
The MoA approach uses a fixed 2-layer architecture:
|
||||
1. Layer 1: Multiple reference models generate diverse responses in parallel (temp=0.6)
|
||||
2. Layer 2: Aggregator model synthesizes the best elements into final response (temp=0.4)
|
||||
|
||||
Args:
|
||||
user_prompt (str): The complex query or problem to solve
|
||||
reference_models (Optional[List[str]]): Custom reference models to use
|
||||
aggregator_model (Optional[str]): Custom aggregator model to use
|
||||
|
||||
Returns:
|
||||
str: JSON string containing the MoA results with the following structure:
|
||||
{
|
||||
"success": bool,
|
||||
"response": str,
|
||||
"models_used": {
|
||||
"reference_models": List[str],
|
||||
"aggregator_model": str
|
||||
},
|
||||
"processing_time": float
|
||||
}
|
||||
|
||||
Raises:
|
||||
Exception: If MoA processing fails or API key is not set
|
||||
"""
|
||||
start_time = datetime.datetime.now()
|
||||
|
||||
debug_call_data = {
|
||||
"parameters": {
|
||||
"user_prompt": user_prompt[:200] + "..." if len(user_prompt) > 200 else user_prompt,
|
||||
"reference_models": reference_models or REFERENCE_MODELS,
|
||||
"aggregator_model": aggregator_model or AGGREGATOR_MODEL,
|
||||
"reference_temperature": REFERENCE_TEMPERATURE,
|
||||
"aggregator_temperature": AGGREGATOR_TEMPERATURE,
|
||||
"min_successful_references": MIN_SUCCESSFUL_REFERENCES
|
||||
},
|
||||
"error": None,
|
||||
"success": False,
|
||||
"reference_responses_count": 0,
|
||||
"failed_models_count": 0,
|
||||
"failed_models": [],
|
||||
"final_response_length": 0,
|
||||
"processing_time_seconds": 0,
|
||||
"models_used": {}
|
||||
}
|
||||
|
||||
try:
|
||||
logger.info("Starting Mixture-of-Agents processing...")
|
||||
logger.info("Query: %s", user_prompt[:100])
|
||||
|
||||
# Validate API key availability
|
||||
if not os.getenv("OPENROUTER_API_KEY"):
|
||||
raise ValueError("OPENROUTER_API_KEY environment variable not set")
|
||||
|
||||
# Use provided models or defaults
|
||||
ref_models = reference_models or REFERENCE_MODELS
|
||||
agg_model = aggregator_model or AGGREGATOR_MODEL
|
||||
|
||||
logger.info("Using %s reference models in 2-layer MoA architecture", len(ref_models))
|
||||
|
||||
# Layer 1: Generate diverse responses from reference models (with failure handling)
|
||||
logger.info("Layer 1: Generating reference responses...")
|
||||
model_results = await asyncio.gather(*[
|
||||
_run_reference_model_safe(model, user_prompt, REFERENCE_TEMPERATURE)
|
||||
for model in ref_models
|
||||
])
|
||||
|
||||
# Separate successful and failed responses
|
||||
successful_responses = []
|
||||
failed_models = []
|
||||
|
||||
for model_name, content, success in model_results:
|
||||
if success:
|
||||
successful_responses.append(content)
|
||||
else:
|
||||
failed_models.append(model_name)
|
||||
|
||||
successful_count = len(successful_responses)
|
||||
failed_count = len(failed_models)
|
||||
|
||||
logger.info("Reference model results: %s successful, %s failed", successful_count, failed_count)
|
||||
|
||||
if failed_models:
|
||||
logger.warning("Failed models: %s", ', '.join(failed_models))
|
||||
|
||||
# Check if we have enough successful responses to proceed
|
||||
if successful_count < MIN_SUCCESSFUL_REFERENCES:
|
||||
raise ValueError(f"Insufficient successful reference models ({successful_count}/{len(ref_models)}). Need at least {MIN_SUCCESSFUL_REFERENCES} successful responses.")
|
||||
|
||||
debug_call_data["reference_responses_count"] = successful_count
|
||||
debug_call_data["failed_models_count"] = failed_count
|
||||
debug_call_data["failed_models"] = failed_models
|
||||
|
||||
# Layer 2: Aggregate responses using the aggregator model
|
||||
logger.info("Layer 2: Synthesizing final response...")
|
||||
aggregator_system_prompt = _construct_aggregator_prompt(
|
||||
AGGREGATOR_SYSTEM_PROMPT,
|
||||
successful_responses
|
||||
)
|
||||
|
||||
final_response = await _run_aggregator_model(
|
||||
aggregator_system_prompt,
|
||||
user_prompt,
|
||||
AGGREGATOR_TEMPERATURE
|
||||
)
|
||||
|
||||
# Calculate processing time
|
||||
end_time = datetime.datetime.now()
|
||||
processing_time = (end_time - start_time).total_seconds()
|
||||
|
||||
logger.info("MoA processing completed in %.2f seconds", processing_time)
|
||||
|
||||
# Prepare successful response (only final aggregated result, minimal fields)
|
||||
result = {
|
||||
"success": True,
|
||||
"response": final_response,
|
||||
"models_used": {
|
||||
"reference_models": ref_models,
|
||||
"aggregator_model": agg_model
|
||||
}
|
||||
}
|
||||
|
||||
debug_call_data["success"] = True
|
||||
debug_call_data["final_response_length"] = len(final_response)
|
||||
debug_call_data["processing_time_seconds"] = processing_time
|
||||
debug_call_data["models_used"] = result["models_used"]
|
||||
|
||||
# Log debug information
|
||||
_debug.log_call("mixture_of_agents_tool", debug_call_data)
|
||||
_debug.save()
|
||||
|
||||
return json.dumps(result, indent=2, ensure_ascii=False)
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Error in MoA processing: {str(e)}"
|
||||
logger.error("%s", error_msg, exc_info=True)
|
||||
|
||||
# Calculate processing time even for errors
|
||||
end_time = datetime.datetime.now()
|
||||
processing_time = (end_time - start_time).total_seconds()
|
||||
|
||||
# Prepare error response (minimal fields)
|
||||
result = {
|
||||
"success": False,
|
||||
"response": "MoA processing failed. Please try again or use a single model for this query.",
|
||||
"models_used": {
|
||||
"reference_models": reference_models or REFERENCE_MODELS,
|
||||
"aggregator_model": aggregator_model or AGGREGATOR_MODEL
|
||||
},
|
||||
"error": error_msg
|
||||
}
|
||||
|
||||
debug_call_data["error"] = error_msg
|
||||
debug_call_data["processing_time_seconds"] = processing_time
|
||||
_debug.log_call("mixture_of_agents_tool", debug_call_data)
|
||||
_debug.save()
|
||||
|
||||
return json.dumps(result, indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
def check_moa_requirements() -> bool:
|
||||
"""
|
||||
Check if all requirements for MoA tools are met.
|
||||
|
||||
Returns:
|
||||
bool: True if requirements are met, False otherwise
|
||||
"""
|
||||
return check_openrouter_api_key()
|
||||
|
||||
|
||||
|
||||
def get_moa_configuration() -> Dict[str, Any]:
|
||||
"""
|
||||
Get the current MoA configuration settings.
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: Dictionary containing all configuration parameters
|
||||
"""
|
||||
return {
|
||||
"reference_models": REFERENCE_MODELS,
|
||||
"aggregator_model": AGGREGATOR_MODEL,
|
||||
"reference_temperature": REFERENCE_TEMPERATURE,
|
||||
"aggregator_temperature": AGGREGATOR_TEMPERATURE,
|
||||
"min_successful_references": MIN_SUCCESSFUL_REFERENCES,
|
||||
"total_reference_models": len(REFERENCE_MODELS),
|
||||
"failure_tolerance": f"{len(REFERENCE_MODELS) - MIN_SUCCESSFUL_REFERENCES}/{len(REFERENCE_MODELS)} models can fail"
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
"""
|
||||
Simple test/demo when run directly
|
||||
"""
|
||||
print("🤖 Mixture-of-Agents Tool Module")
|
||||
print("=" * 50)
|
||||
|
||||
# Check if API key is available
|
||||
api_available = check_openrouter_api_key()
|
||||
|
||||
if not api_available:
|
||||
print("❌ OPENROUTER_API_KEY environment variable not set")
|
||||
print("Please set your API key: export OPENROUTER_API_KEY='your-key-here'")
|
||||
print("Get API key at: https://openrouter.ai/")
|
||||
exit(1)
|
||||
else:
|
||||
print("✅ OpenRouter API key found")
|
||||
|
||||
print("🛠️ MoA tools ready for use!")
|
||||
|
||||
# Show current configuration
|
||||
config = get_moa_configuration()
|
||||
print("\n⚙️ Current Configuration:")
|
||||
print(f" 🤖 Reference models ({len(config['reference_models'])}): {', '.join(config['reference_models'])}")
|
||||
print(f" 🧠 Aggregator model: {config['aggregator_model']}")
|
||||
print(f" 🌡️ Reference temperature: {config['reference_temperature']}")
|
||||
print(f" 🌡️ Aggregator temperature: {config['aggregator_temperature']}")
|
||||
print(f" 🛡️ Failure tolerance: {config['failure_tolerance']}")
|
||||
print(f" 📊 Minimum successful models: {config['min_successful_references']}")
|
||||
|
||||
# Show debug mode status
|
||||
if _debug.active:
|
||||
print(f"\n🐛 Debug mode ENABLED - Session ID: {_debug.session_id}")
|
||||
print(f" Debug logs will be saved to: ./logs/moa_tools_debug_{_debug.session_id}.json")
|
||||
else:
|
||||
print("\n🐛 Debug mode disabled (set MOA_TOOLS_DEBUG=true to enable)")
|
||||
|
||||
print("\nBasic usage:")
|
||||
print(" from mixture_of_agents_tool import mixture_of_agents_tool")
|
||||
print(" import asyncio")
|
||||
print("")
|
||||
print(" async def main():")
|
||||
print(" result = await mixture_of_agents_tool(")
|
||||
print(" user_prompt='Solve this complex mathematical proof...'")
|
||||
print(" )")
|
||||
print(" print(result)")
|
||||
print(" asyncio.run(main())")
|
||||
|
||||
print("\nBest use cases:")
|
||||
print(" - Complex mathematical proofs and calculations")
|
||||
print(" - Advanced coding problems and algorithm design")
|
||||
print(" - Multi-step analytical reasoning tasks")
|
||||
print(" - Problems requiring diverse domain expertise")
|
||||
print(" - Tasks where single models show limitations")
|
||||
|
||||
print("\nPerformance characteristics:")
|
||||
print(" - Higher latency due to multiple model calls")
|
||||
print(" - Significantly improved quality for complex tasks")
|
||||
print(" - Parallel processing for efficiency")
|
||||
print(f" - Optimized temperatures: {REFERENCE_TEMPERATURE} for reference models, {AGGREGATOR_TEMPERATURE} for aggregation")
|
||||
print(" - Token-efficient: only returns final aggregated response")
|
||||
print(" - Resilient: continues with partial model failures")
|
||||
print(" - Configurable: easy to modify models and settings at top of file")
|
||||
print(" - State-of-the-art results on challenging benchmarks")
|
||||
|
||||
print("\nDebug mode:")
|
||||
print(" # Enable debug logging")
|
||||
print(" export MOA_TOOLS_DEBUG=true")
|
||||
print(" # Debug logs capture all MoA processing steps and metrics")
|
||||
print(" # Logs saved to: ./logs/moa_tools_debug_UUID.json")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Registry
|
||||
# ---------------------------------------------------------------------------
|
||||
from tools.registry import registry
|
||||
|
||||
MOA_SCHEMA = {
|
||||
"name": "mixture_of_agents",
|
||||
"description": "Route a hard problem through multiple frontier LLMs collaboratively. Makes 5 API calls (4 reference models + 1 aggregator) with maximum reasoning effort — use sparingly for genuinely difficult problems. Best for: complex math, advanced algorithms, multi-step analytical reasoning, problems benefiting from diverse perspectives.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"user_prompt": {
|
||||
"type": "string",
|
||||
"description": "The complex query or problem to solve using multiple AI models. Should be a challenging problem that benefits from diverse perspectives and collaborative reasoning."
|
||||
}
|
||||
},
|
||||
"required": ["user_prompt"]
|
||||
}
|
||||
}
|
||||
|
||||
registry.register(
|
||||
name="mixture_of_agents",
|
||||
toolset="moa",
|
||||
schema=MOA_SCHEMA,
|
||||
handler=lambda args, **kw: mixture_of_agents_tool(user_prompt=args.get("user_prompt", "")),
|
||||
check_fn=check_moa_requirements,
|
||||
requires_env=["OPENROUTER_API_KEY"],
|
||||
is_async=True,
|
||||
emoji="🧠",
|
||||
)
|
||||
@@ -0,0 +1 @@
|
||||
So I just tried Neuphonic and I’m genuinely impressed. It's super responsive, it sounds clean, supports voice cloning, and the agent feature is fun to play with too. Highly recommend it for podcasts, conversations, or even just messing around with voiceovers.
|
||||
Binary file not shown.
@@ -0,0 +1,104 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Standalone NeuTTS synthesis helper.
|
||||
|
||||
Called by tts_tool.py via subprocess to keep the TTS model (~500MB)
|
||||
in a separate process that exits after synthesis — no lingering memory.
|
||||
|
||||
Usage:
|
||||
python -m tools.neutts_synth --text "Hello" --out output.wav \
|
||||
--ref-audio samples/jo.wav --ref-text samples/jo.txt
|
||||
|
||||
Requires: python -m pip install -U neutts[all]
|
||||
System: apt install espeak-ng (or brew install espeak-ng)
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import struct
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _write_wav(path: str, samples, sample_rate: int = 24000) -> None:
|
||||
"""Write a WAV file from float32 samples (no soundfile dependency)."""
|
||||
import numpy as np
|
||||
|
||||
if not isinstance(samples, np.ndarray):
|
||||
samples = np.array(samples, dtype=np.float32)
|
||||
samples = samples.flatten()
|
||||
|
||||
# Clamp and convert to int16
|
||||
samples = np.clip(samples, -1.0, 1.0)
|
||||
pcm = (samples * 32767).astype(np.int16)
|
||||
|
||||
num_channels = 1
|
||||
bits_per_sample = 16
|
||||
byte_rate = sample_rate * num_channels * (bits_per_sample // 8)
|
||||
block_align = num_channels * (bits_per_sample // 8)
|
||||
data_size = len(pcm) * (bits_per_sample // 8)
|
||||
|
||||
with open(path, "wb") as f:
|
||||
f.write(b"RIFF")
|
||||
f.write(struct.pack("<I", 36 + data_size))
|
||||
f.write(b"WAVE")
|
||||
f.write(b"fmt ")
|
||||
f.write(struct.pack("<IHHIIHH", 16, 1, num_channels, sample_rate,
|
||||
byte_rate, block_align, bits_per_sample))
|
||||
f.write(b"data")
|
||||
f.write(struct.pack("<I", data_size))
|
||||
f.write(pcm.tobytes())
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="NeuTTS synthesis helper")
|
||||
parser.add_argument("--text", required=True, help="Text to synthesize")
|
||||
parser.add_argument("--out", required=True, help="Output WAV path")
|
||||
parser.add_argument("--ref-audio", required=True, help="Reference voice audio path")
|
||||
parser.add_argument("--ref-text", required=True, help="Reference voice transcript path")
|
||||
parser.add_argument("--model", default="neuphonic/neutts-air-q4-gguf",
|
||||
help="HuggingFace backbone model repo")
|
||||
parser.add_argument("--device", default="cpu", help="Device (cpu/cuda/mps)")
|
||||
args = parser.parse_args()
|
||||
|
||||
# Validate inputs
|
||||
ref_audio = Path(args.ref_audio).expanduser()
|
||||
ref_text_path = Path(args.ref_text).expanduser()
|
||||
if not ref_audio.exists():
|
||||
print(f"Error: reference audio not found: {ref_audio}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if not ref_text_path.exists():
|
||||
print(f"Error: reference text not found: {ref_text_path}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
ref_text = ref_text_path.read_text(encoding="utf-8").strip()
|
||||
|
||||
# Import and run NeuTTS
|
||||
try:
|
||||
from neutts import NeuTTS
|
||||
except ImportError:
|
||||
print("Error: neutts not installed. Run: python -m pip install -U neutts[all]", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
tts = NeuTTS(
|
||||
backbone_repo=args.model,
|
||||
backbone_device=args.device,
|
||||
codec_repo="neuphonic/neucodec",
|
||||
codec_device=args.device,
|
||||
)
|
||||
ref_codes = tts.encode_reference(str(ref_audio))
|
||||
wav = tts.infer(args.text, ref_codes, ref_text)
|
||||
|
||||
# Write output
|
||||
out_path = Path(args.out)
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
import soundfile as sf
|
||||
sf.write(str(out_path), wav, 24000)
|
||||
except ImportError:
|
||||
_write_wav(str(out_path), wav, 24000)
|
||||
|
||||
print(f"OK: {out_path}", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Shared OpenRouter API client for Hermes tools.
|
||||
|
||||
Provides a single lazy-initialized AsyncOpenAI client that all tool modules
|
||||
can share. Routes through the centralized provider router in
|
||||
agent/auxiliary_client.py so auth, headers, and API format are handled
|
||||
consistently.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
_client = None
|
||||
|
||||
|
||||
def get_async_client():
|
||||
"""Return a shared async OpenAI-compatible client for OpenRouter.
|
||||
|
||||
The client is created lazily on first call and reused thereafter.
|
||||
Uses the centralized provider router for auth and client construction.
|
||||
Raises ValueError if OPENROUTER_API_KEY is not set.
|
||||
"""
|
||||
global _client
|
||||
if _client is None:
|
||||
from agent.auxiliary_client import resolve_provider_client
|
||||
client, _model = resolve_provider_client("openrouter", async_mode=True)
|
||||
if client is None:
|
||||
raise ValueError("OPENROUTER_API_KEY environment variable not set")
|
||||
_client = client
|
||||
return _client
|
||||
|
||||
|
||||
def check_api_key() -> bool:
|
||||
"""Check whether the OpenRouter API key is present."""
|
||||
return bool(os.getenv("OPENROUTER_API_KEY"))
|
||||
@@ -0,0 +1,155 @@
|
||||
"""OSV malware check for MCP extension packages.
|
||||
|
||||
Before launching an MCP server via npx/uvx, queries the OSV (Open Source
|
||||
Vulnerabilities) API to check if the package has any known malware advisories
|
||||
(MAL-* IDs). Regular CVEs are ignored — only confirmed malware is blocked.
|
||||
|
||||
The API is free, public, and maintained by Google. Typical latency is ~300ms.
|
||||
Fail-open: network errors allow the package to proceed.
|
||||
|
||||
Inspired by Block/goose's extension malware check.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import urllib.request
|
||||
from typing import Optional, Tuple
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_OSV_ENDPOINT = os.getenv("OSV_ENDPOINT", "https://api.osv.dev/v1/query")
|
||||
_TIMEOUT = 10 # seconds
|
||||
|
||||
|
||||
def check_package_for_malware(
|
||||
command: str, args: list
|
||||
) -> Optional[str]:
|
||||
"""Check if an MCP server package has known malware advisories.
|
||||
|
||||
Inspects the *command* (e.g. ``npx``, ``uvx``) and *args* to infer the
|
||||
package name and ecosystem. Queries the OSV API for MAL-* advisories.
|
||||
|
||||
Returns:
|
||||
An error message string if malware is found, or None if clean/unknown.
|
||||
Returns None (allow) on network errors or unrecognized commands.
|
||||
"""
|
||||
ecosystem = _infer_ecosystem(command)
|
||||
if not ecosystem:
|
||||
return None # not npx/uvx — skip
|
||||
|
||||
package, version = _parse_package_from_args(args, ecosystem)
|
||||
if not package:
|
||||
return None
|
||||
|
||||
try:
|
||||
malware = _query_osv(package, ecosystem, version)
|
||||
except Exception as exc:
|
||||
# Fail-open: network errors, timeouts, parse failures → allow
|
||||
logger.debug("OSV check failed for %s/%s (allowing): %s", ecosystem, package, exc)
|
||||
return None
|
||||
|
||||
if malware:
|
||||
ids = ", ".join(m["id"] for m in malware[:3])
|
||||
summaries = "; ".join(
|
||||
m.get("summary", m["id"])[:100] for m in malware[:3]
|
||||
)
|
||||
return (
|
||||
f"BLOCKED: Package '{package}' ({ecosystem}) has known malware "
|
||||
f"advisories: {ids}. Details: {summaries}"
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _infer_ecosystem(command: str) -> Optional[str]:
|
||||
"""Infer package ecosystem from the command name."""
|
||||
base = os.path.basename(command).lower()
|
||||
if base in ("npx", "npx.cmd"):
|
||||
return "npm"
|
||||
if base in ("uvx", "uvx.cmd", "pipx"):
|
||||
return "PyPI"
|
||||
return None
|
||||
|
||||
|
||||
def _parse_package_from_args(
|
||||
args: list, ecosystem: str
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""Extract package name and optional version from command args.
|
||||
|
||||
Returns (package_name, version) or (None, None) if not parseable.
|
||||
"""
|
||||
if not args:
|
||||
return None, None
|
||||
|
||||
# Skip flags to find the package token
|
||||
package_token = None
|
||||
for arg in args:
|
||||
if not isinstance(arg, str):
|
||||
continue
|
||||
if arg.startswith("-"):
|
||||
continue
|
||||
package_token = arg
|
||||
break
|
||||
|
||||
if not package_token:
|
||||
return None, None
|
||||
|
||||
if ecosystem == "npm":
|
||||
return _parse_npm_package(package_token)
|
||||
elif ecosystem == "PyPI":
|
||||
return _parse_pypi_package(package_token)
|
||||
return package_token, None
|
||||
|
||||
|
||||
def _parse_npm_package(token: str) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""Parse npm package: @scope/name@version or name@version."""
|
||||
if token.startswith("@"):
|
||||
# Scoped: @scope/name@version
|
||||
match = re.match(r"^(@[^/]+/[^@]+)(?:@(.+))?$", token)
|
||||
if match:
|
||||
return match.group(1), match.group(2)
|
||||
return token, None
|
||||
# Unscoped: name@version
|
||||
if "@" in token:
|
||||
parts = token.rsplit("@", 1)
|
||||
name = parts[0]
|
||||
version = parts[1] if len(parts) > 1 and parts[1] != "latest" else None
|
||||
return name, version
|
||||
return token, None
|
||||
|
||||
|
||||
def _parse_pypi_package(token: str) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""Parse PyPI package: name==version or name[extras]==version."""
|
||||
# Strip extras: name[extra1,extra2]==version
|
||||
match = re.match(r"^([a-zA-Z0-9._-]+)(?:\[[^\]]*\])?(?:==(.+))?$", token)
|
||||
if match:
|
||||
return match.group(1), match.group(2)
|
||||
return token, None
|
||||
|
||||
|
||||
def _query_osv(
|
||||
package: str, ecosystem: str, version: Optional[str] = None
|
||||
) -> list:
|
||||
"""Query the OSV API for MAL-* advisories. Returns list of malware vulns."""
|
||||
payload = {"package": {"name": package, "ecosystem": ecosystem}}
|
||||
if version:
|
||||
payload["version"] = version
|
||||
|
||||
data = json.dumps(payload).encode("utf-8")
|
||||
req = urllib.request.Request(
|
||||
_OSV_ENDPOINT,
|
||||
data=data,
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": "hermes-agent-osv-check/1.0",
|
||||
},
|
||||
method="POST",
|
||||
)
|
||||
|
||||
with urllib.request.urlopen(req, timeout=_TIMEOUT) as resp:
|
||||
result = json.loads(resp.read())
|
||||
|
||||
vulns = result.get("vulns", [])
|
||||
# Only malware advisories — ignore regular CVEs
|
||||
return [v for v in vulns if v.get("id", "").startswith("MAL-")]
|
||||
@@ -0,0 +1,580 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
V4A Patch Format Parser
|
||||
|
||||
Parses the V4A patch format used by codex, cline, and other coding agents.
|
||||
|
||||
V4A Format:
|
||||
*** Begin Patch
|
||||
*** Update File: path/to/file.py
|
||||
@@ optional context hint @@
|
||||
context line (space prefix)
|
||||
-removed line (minus prefix)
|
||||
+added line (plus prefix)
|
||||
*** Add File: path/to/new.py
|
||||
+new file content
|
||||
+line 2
|
||||
*** Delete File: path/to/old.py
|
||||
*** Move File: old/path.py -> new/path.py
|
||||
*** End Patch
|
||||
|
||||
Usage:
|
||||
from tools.patch_parser import parse_v4a_patch, apply_v4a_operations
|
||||
|
||||
operations, error = parse_v4a_patch(patch_content)
|
||||
if error:
|
||||
print(f"Parse error: {error}")
|
||||
else:
|
||||
result = apply_v4a_operations(operations, file_ops)
|
||||
"""
|
||||
|
||||
import difflib
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List, Optional, Tuple, Any
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class OperationType(Enum):
|
||||
ADD = "add"
|
||||
UPDATE = "update"
|
||||
DELETE = "delete"
|
||||
MOVE = "move"
|
||||
|
||||
|
||||
@dataclass
|
||||
class HunkLine:
|
||||
"""A single line in a patch hunk."""
|
||||
prefix: str # ' ', '-', or '+'
|
||||
content: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class Hunk:
|
||||
"""A group of changes within a file."""
|
||||
context_hint: Optional[str] = None
|
||||
lines: List[HunkLine] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PatchOperation:
|
||||
"""A single operation in a V4A patch."""
|
||||
operation: OperationType
|
||||
file_path: str
|
||||
new_path: Optional[str] = None # For move operations
|
||||
hunks: List[Hunk] = field(default_factory=list)
|
||||
content: Optional[str] = None # For add file operations
|
||||
|
||||
|
||||
def parse_v4a_patch(patch_content: str) -> Tuple[List[PatchOperation], Optional[str]]:
|
||||
"""
|
||||
Parse a V4A format patch.
|
||||
|
||||
Args:
|
||||
patch_content: The patch text in V4A format
|
||||
|
||||
Returns:
|
||||
Tuple of (operations, error_message)
|
||||
- If successful: (list_of_operations, None)
|
||||
- If failed: ([], error_description)
|
||||
"""
|
||||
lines = patch_content.split('\n')
|
||||
operations: List[PatchOperation] = []
|
||||
|
||||
# Find patch boundaries
|
||||
start_idx = None
|
||||
end_idx = None
|
||||
|
||||
for i, line in enumerate(lines):
|
||||
if '*** Begin Patch' in line or '***Begin Patch' in line:
|
||||
start_idx = i
|
||||
elif '*** End Patch' in line or '***End Patch' in line:
|
||||
end_idx = i
|
||||
break
|
||||
|
||||
if start_idx is None:
|
||||
# Try to parse without explicit begin marker
|
||||
start_idx = -1
|
||||
|
||||
if end_idx is None:
|
||||
end_idx = len(lines)
|
||||
|
||||
# Parse operations between boundaries
|
||||
i = start_idx + 1
|
||||
current_op: Optional[PatchOperation] = None
|
||||
current_hunk: Optional[Hunk] = None
|
||||
|
||||
while i < end_idx:
|
||||
line = lines[i]
|
||||
|
||||
# Check for file operation markers
|
||||
update_match = re.match(r'\*\*\*\s*Update\s+File:\s*(.+)', line)
|
||||
add_match = re.match(r'\*\*\*\s*Add\s+File:\s*(.+)', line)
|
||||
delete_match = re.match(r'\*\*\*\s*Delete\s+File:\s*(.+)', line)
|
||||
move_match = re.match(r'\*\*\*\s*Move\s+File:\s*(.+?)\s*->\s*(.+)', line)
|
||||
|
||||
if update_match:
|
||||
# Save previous operation
|
||||
if current_op:
|
||||
if current_hunk and current_hunk.lines:
|
||||
current_op.hunks.append(current_hunk)
|
||||
operations.append(current_op)
|
||||
|
||||
current_op = PatchOperation(
|
||||
operation=OperationType.UPDATE,
|
||||
file_path=update_match.group(1).strip()
|
||||
)
|
||||
current_hunk = None
|
||||
|
||||
elif add_match:
|
||||
if current_op:
|
||||
if current_hunk and current_hunk.lines:
|
||||
current_op.hunks.append(current_hunk)
|
||||
operations.append(current_op)
|
||||
|
||||
current_op = PatchOperation(
|
||||
operation=OperationType.ADD,
|
||||
file_path=add_match.group(1).strip()
|
||||
)
|
||||
current_hunk = Hunk()
|
||||
|
||||
elif delete_match:
|
||||
if current_op:
|
||||
if current_hunk and current_hunk.lines:
|
||||
current_op.hunks.append(current_hunk)
|
||||
operations.append(current_op)
|
||||
|
||||
current_op = PatchOperation(
|
||||
operation=OperationType.DELETE,
|
||||
file_path=delete_match.group(1).strip()
|
||||
)
|
||||
operations.append(current_op)
|
||||
current_op = None
|
||||
current_hunk = None
|
||||
|
||||
elif move_match:
|
||||
if current_op:
|
||||
if current_hunk and current_hunk.lines:
|
||||
current_op.hunks.append(current_hunk)
|
||||
operations.append(current_op)
|
||||
|
||||
current_op = PatchOperation(
|
||||
operation=OperationType.MOVE,
|
||||
file_path=move_match.group(1).strip(),
|
||||
new_path=move_match.group(2).strip()
|
||||
)
|
||||
operations.append(current_op)
|
||||
current_op = None
|
||||
current_hunk = None
|
||||
|
||||
elif line.startswith('@@'):
|
||||
# Context hint / hunk marker
|
||||
if current_op:
|
||||
if current_hunk and current_hunk.lines:
|
||||
current_op.hunks.append(current_hunk)
|
||||
|
||||
# Extract context hint
|
||||
hint_match = re.match(r'@@\s*(.+?)\s*@@', line)
|
||||
hint = hint_match.group(1) if hint_match else None
|
||||
current_hunk = Hunk(context_hint=hint)
|
||||
|
||||
elif current_op and line:
|
||||
# Parse hunk line
|
||||
if current_hunk is None:
|
||||
current_hunk = Hunk()
|
||||
|
||||
if line.startswith('+'):
|
||||
current_hunk.lines.append(HunkLine('+', line[1:]))
|
||||
elif line.startswith('-'):
|
||||
current_hunk.lines.append(HunkLine('-', line[1:]))
|
||||
elif line.startswith(' '):
|
||||
current_hunk.lines.append(HunkLine(' ', line[1:]))
|
||||
elif line.startswith('\\'):
|
||||
# "\ No newline at end of file" marker - skip
|
||||
pass
|
||||
else:
|
||||
# Treat as context line (implicit space prefix)
|
||||
current_hunk.lines.append(HunkLine(' ', line))
|
||||
|
||||
i += 1
|
||||
|
||||
# Don't forget the last operation
|
||||
if current_op:
|
||||
if current_hunk and current_hunk.lines:
|
||||
current_op.hunks.append(current_hunk)
|
||||
operations.append(current_op)
|
||||
|
||||
# Validate the parsed result
|
||||
if not operations:
|
||||
# Empty patch is not an error — callers get [] and can decide
|
||||
return operations, None
|
||||
|
||||
parse_errors: List[str] = []
|
||||
for op in operations:
|
||||
if not op.file_path:
|
||||
parse_errors.append("Operation with empty file path")
|
||||
if op.operation == OperationType.UPDATE and not op.hunks:
|
||||
parse_errors.append(f"UPDATE {op.file_path!r}: no hunks found")
|
||||
if op.operation == OperationType.MOVE and not op.new_path:
|
||||
parse_errors.append(f"MOVE {op.file_path!r}: missing destination path (expected 'src -> dst')")
|
||||
|
||||
if parse_errors:
|
||||
return [], "Parse error: " + "; ".join(parse_errors)
|
||||
|
||||
return operations, None
|
||||
|
||||
|
||||
def _count_occurrences(text: str, pattern: str) -> int:
|
||||
"""Count non-overlapping occurrences of *pattern* in *text*."""
|
||||
count = 0
|
||||
start = 0
|
||||
while True:
|
||||
pos = text.find(pattern, start)
|
||||
if pos == -1:
|
||||
break
|
||||
count += 1
|
||||
start = pos + 1
|
||||
return count
|
||||
|
||||
|
||||
def _validate_operations(
|
||||
operations: List[PatchOperation],
|
||||
file_ops: Any,
|
||||
) -> List[str]:
|
||||
"""Validate all operations without writing any files.
|
||||
|
||||
Returns a list of error strings; an empty list means all operations
|
||||
are valid and the apply phase can proceed safely.
|
||||
|
||||
For UPDATE operations, hunks are simulated in order so that later
|
||||
hunks validate against post-earlier-hunk content (matching apply order).
|
||||
"""
|
||||
# Deferred import: breaks the patch_parser ↔ fuzzy_match circular dependency
|
||||
from tools.fuzzy_match import fuzzy_find_and_replace
|
||||
|
||||
errors: List[str] = []
|
||||
|
||||
for op in operations:
|
||||
if op.operation == OperationType.UPDATE:
|
||||
read_result = file_ops.read_file_raw(op.file_path)
|
||||
if read_result.error:
|
||||
errors.append(f"{op.file_path}: {read_result.error}")
|
||||
continue
|
||||
|
||||
simulated = read_result.content
|
||||
for hunk in op.hunks:
|
||||
search_lines = [l.content for l in hunk.lines if l.prefix in (' ', '-')]
|
||||
if not search_lines:
|
||||
# Addition-only hunk: validate context hint uniqueness
|
||||
if hunk.context_hint:
|
||||
occurrences = _count_occurrences(simulated, hunk.context_hint)
|
||||
if occurrences == 0:
|
||||
errors.append(
|
||||
f"{op.file_path}: addition-only hunk context hint "
|
||||
f"'{hunk.context_hint}' not found"
|
||||
)
|
||||
elif occurrences > 1:
|
||||
errors.append(
|
||||
f"{op.file_path}: addition-only hunk context hint "
|
||||
f"'{hunk.context_hint}' is ambiguous "
|
||||
f"({occurrences} occurrences)"
|
||||
)
|
||||
continue
|
||||
|
||||
search_pattern = '\n'.join(search_lines)
|
||||
replace_lines = [l.content for l in hunk.lines if l.prefix in (' ', '+')]
|
||||
replacement = '\n'.join(replace_lines)
|
||||
|
||||
new_simulated, count, _strategy, match_error = fuzzy_find_and_replace(
|
||||
simulated, search_pattern, replacement, replace_all=False
|
||||
)
|
||||
if count == 0:
|
||||
label = f"'{hunk.context_hint}'" if hunk.context_hint else "(no hint)"
|
||||
errors.append(
|
||||
f"{op.file_path}: hunk {label} not found"
|
||||
+ (f" — {match_error}" if match_error else "")
|
||||
)
|
||||
else:
|
||||
# Advance simulation so subsequent hunks validate correctly.
|
||||
# Reuse the result from the call above — no second fuzzy run.
|
||||
simulated = new_simulated
|
||||
|
||||
elif op.operation == OperationType.DELETE:
|
||||
read_result = file_ops.read_file_raw(op.file_path)
|
||||
if read_result.error:
|
||||
errors.append(f"{op.file_path}: file not found for deletion")
|
||||
|
||||
elif op.operation == OperationType.MOVE:
|
||||
if not op.new_path:
|
||||
errors.append(f"{op.file_path}: MOVE operation missing destination path")
|
||||
continue
|
||||
src_result = file_ops.read_file_raw(op.file_path)
|
||||
if src_result.error:
|
||||
errors.append(f"{op.file_path}: source file not found for move")
|
||||
dst_result = file_ops.read_file_raw(op.new_path)
|
||||
if not dst_result.error:
|
||||
errors.append(
|
||||
f"{op.new_path}: destination already exists — move would overwrite"
|
||||
)
|
||||
|
||||
# ADD: parent directory creation handled by write_file; no pre-check needed.
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def apply_v4a_operations(operations: List[PatchOperation],
|
||||
file_ops: Any) -> 'PatchResult':
|
||||
"""Apply V4A patch operations using a file operations interface.
|
||||
|
||||
Uses a two-phase validate-then-apply approach:
|
||||
- Phase 1: validate all operations against current file contents without
|
||||
writing anything. If any validation error is found, return immediately
|
||||
with no filesystem changes.
|
||||
- Phase 2: apply all operations. A failure here (e.g. a race between
|
||||
validation and apply) is reported with a note to run ``git diff``.
|
||||
|
||||
Args:
|
||||
operations: List of PatchOperation from parse_v4a_patch
|
||||
file_ops: Object with read_file_raw, write_file methods
|
||||
|
||||
Returns:
|
||||
PatchResult with results of all operations
|
||||
"""
|
||||
# Import here to avoid circular imports
|
||||
from tools.file_operations import PatchResult
|
||||
|
||||
# ---- Phase 1: validate ----
|
||||
validation_errors = _validate_operations(operations, file_ops)
|
||||
if validation_errors:
|
||||
return PatchResult(
|
||||
success=False,
|
||||
error="Patch validation failed (no files were modified):\n"
|
||||
+ "\n".join(f" • {e}" for e in validation_errors),
|
||||
)
|
||||
|
||||
# ---- Phase 2: apply ----
|
||||
files_modified = []
|
||||
files_created = []
|
||||
files_deleted = []
|
||||
all_diffs = []
|
||||
errors = []
|
||||
|
||||
for op in operations:
|
||||
try:
|
||||
if op.operation == OperationType.ADD:
|
||||
result = _apply_add(op, file_ops)
|
||||
if result[0]:
|
||||
files_created.append(op.file_path)
|
||||
all_diffs.append(result[1])
|
||||
else:
|
||||
errors.append(f"Failed to add {op.file_path}: {result[1]}")
|
||||
|
||||
elif op.operation == OperationType.DELETE:
|
||||
result = _apply_delete(op, file_ops)
|
||||
if result[0]:
|
||||
files_deleted.append(op.file_path)
|
||||
all_diffs.append(result[1])
|
||||
else:
|
||||
errors.append(f"Failed to delete {op.file_path}: {result[1]}")
|
||||
|
||||
elif op.operation == OperationType.MOVE:
|
||||
result = _apply_move(op, file_ops)
|
||||
if result[0]:
|
||||
files_modified.append(f"{op.file_path} -> {op.new_path}")
|
||||
all_diffs.append(result[1])
|
||||
else:
|
||||
errors.append(f"Failed to move {op.file_path}: {result[1]}")
|
||||
|
||||
elif op.operation == OperationType.UPDATE:
|
||||
result = _apply_update(op, file_ops)
|
||||
if result[0]:
|
||||
files_modified.append(op.file_path)
|
||||
all_diffs.append(result[1])
|
||||
else:
|
||||
errors.append(f"Failed to update {op.file_path}: {result[1]}")
|
||||
|
||||
except Exception as e:
|
||||
errors.append(f"Error processing {op.file_path}: {str(e)}")
|
||||
|
||||
# Run lint on all modified/created files
|
||||
lint_results = {}
|
||||
for f in files_modified + files_created:
|
||||
if hasattr(file_ops, '_check_lint'):
|
||||
lint_result = file_ops._check_lint(f)
|
||||
lint_results[f] = lint_result.to_dict()
|
||||
|
||||
combined_diff = '\n'.join(all_diffs)
|
||||
|
||||
if errors:
|
||||
return PatchResult(
|
||||
success=False,
|
||||
diff=combined_diff,
|
||||
files_modified=files_modified,
|
||||
files_created=files_created,
|
||||
files_deleted=files_deleted,
|
||||
lint=lint_results if lint_results else None,
|
||||
error="Apply phase failed (state may be inconsistent — run `git diff` to assess):\n"
|
||||
+ "\n".join(f" • {e}" for e in errors),
|
||||
)
|
||||
|
||||
return PatchResult(
|
||||
success=True,
|
||||
diff=combined_diff,
|
||||
files_modified=files_modified,
|
||||
files_created=files_created,
|
||||
files_deleted=files_deleted,
|
||||
lint=lint_results if lint_results else None,
|
||||
)
|
||||
|
||||
|
||||
def _apply_add(op: PatchOperation, file_ops: Any) -> Tuple[bool, str]:
|
||||
"""Apply an add file operation."""
|
||||
# Extract content from hunks (all + lines)
|
||||
content_lines = []
|
||||
for hunk in op.hunks:
|
||||
for line in hunk.lines:
|
||||
if line.prefix == '+':
|
||||
content_lines.append(line.content)
|
||||
|
||||
content = '\n'.join(content_lines)
|
||||
|
||||
result = file_ops.write_file(op.file_path, content)
|
||||
if result.error:
|
||||
return False, result.error
|
||||
|
||||
diff = f"--- /dev/null\n+++ b/{op.file_path}\n"
|
||||
diff += '\n'.join(f"+{line}" for line in content_lines)
|
||||
|
||||
return True, diff
|
||||
|
||||
|
||||
def _apply_delete(op: PatchOperation, file_ops: Any) -> Tuple[bool, str]:
|
||||
"""Apply a delete file operation."""
|
||||
# Read before deleting so we can produce a real unified diff.
|
||||
# Validation already confirmed existence; this guards against races.
|
||||
read_result = file_ops.read_file_raw(op.file_path)
|
||||
if read_result.error:
|
||||
return False, f"Cannot delete {op.file_path}: file not found"
|
||||
|
||||
result = file_ops.delete_file(op.file_path)
|
||||
if result.error:
|
||||
return False, result.error
|
||||
|
||||
removed_lines = read_result.content.splitlines(keepends=True)
|
||||
diff = ''.join(difflib.unified_diff(
|
||||
removed_lines, [],
|
||||
fromfile=f"a/{op.file_path}",
|
||||
tofile="/dev/null",
|
||||
))
|
||||
return True, diff or f"# Deleted: {op.file_path}"
|
||||
|
||||
|
||||
def _apply_move(op: PatchOperation, file_ops: Any) -> Tuple[bool, str]:
|
||||
"""Apply a move file operation."""
|
||||
result = file_ops.move_file(op.file_path, op.new_path)
|
||||
if result.error:
|
||||
return False, result.error
|
||||
|
||||
diff = f"# Moved: {op.file_path} -> {op.new_path}"
|
||||
return True, diff
|
||||
|
||||
|
||||
def _apply_update(op: PatchOperation, file_ops: Any) -> Tuple[bool, str]:
|
||||
"""Apply an update file operation."""
|
||||
# Deferred import: breaks the patch_parser ↔ fuzzy_match circular dependency
|
||||
from tools.fuzzy_match import fuzzy_find_and_replace
|
||||
|
||||
# Read current content — raw so no line-number prefixes or per-line truncation
|
||||
read_result = file_ops.read_file_raw(op.file_path)
|
||||
|
||||
if read_result.error:
|
||||
return False, f"Cannot read file: {read_result.error}"
|
||||
|
||||
current_content = read_result.content
|
||||
|
||||
# Apply each hunk
|
||||
new_content = current_content
|
||||
|
||||
for hunk in op.hunks:
|
||||
# Build search pattern from context and removed lines
|
||||
search_lines = []
|
||||
replace_lines = []
|
||||
|
||||
for line in hunk.lines:
|
||||
if line.prefix == ' ':
|
||||
search_lines.append(line.content)
|
||||
replace_lines.append(line.content)
|
||||
elif line.prefix == '-':
|
||||
search_lines.append(line.content)
|
||||
elif line.prefix == '+':
|
||||
replace_lines.append(line.content)
|
||||
|
||||
if search_lines:
|
||||
search_pattern = '\n'.join(search_lines)
|
||||
replacement = '\n'.join(replace_lines)
|
||||
|
||||
new_content, count, _strategy, error = fuzzy_find_and_replace(
|
||||
new_content, search_pattern, replacement, replace_all=False
|
||||
)
|
||||
|
||||
if error and count == 0:
|
||||
# Try with context hint if available
|
||||
if hunk.context_hint:
|
||||
# Find the context hint location and search nearby
|
||||
hint_pos = new_content.find(hunk.context_hint)
|
||||
if hint_pos != -1:
|
||||
# Search in a window around the hint
|
||||
window_start = max(0, hint_pos - 500)
|
||||
window_end = min(len(new_content), hint_pos + 2000)
|
||||
window = new_content[window_start:window_end]
|
||||
|
||||
window_new, count, _strategy, error = fuzzy_find_and_replace(
|
||||
window, search_pattern, replacement, replace_all=False
|
||||
)
|
||||
|
||||
if count > 0:
|
||||
new_content = new_content[:window_start] + window_new + new_content[window_end:]
|
||||
error = None
|
||||
|
||||
if error:
|
||||
return False, f"Could not apply hunk: {error}"
|
||||
else:
|
||||
# Addition-only hunk (no context or removed lines).
|
||||
# Insert at the location indicated by the context hint, or at end of file.
|
||||
insert_text = '\n'.join(replace_lines)
|
||||
if hunk.context_hint:
|
||||
occurrences = _count_occurrences(new_content, hunk.context_hint)
|
||||
if occurrences == 0:
|
||||
# Hint not found — append at end as a safe fallback
|
||||
new_content = new_content.rstrip('\n') + '\n' + insert_text + '\n'
|
||||
elif occurrences > 1:
|
||||
return False, (
|
||||
f"Addition-only hunk: context hint '{hunk.context_hint}' is ambiguous "
|
||||
f"({occurrences} occurrences) — provide a more unique hint"
|
||||
)
|
||||
else:
|
||||
hint_pos = new_content.find(hunk.context_hint)
|
||||
# Insert after the line containing the context hint
|
||||
eol = new_content.find('\n', hint_pos)
|
||||
if eol != -1:
|
||||
new_content = new_content[:eol + 1] + insert_text + '\n' + new_content[eol + 1:]
|
||||
else:
|
||||
new_content = new_content + '\n' + insert_text
|
||||
else:
|
||||
new_content = new_content.rstrip('\n') + '\n' + insert_text + '\n'
|
||||
|
||||
# Write new content
|
||||
write_result = file_ops.write_file(op.file_path, new_content)
|
||||
if write_result.error:
|
||||
return False, write_result.error
|
||||
|
||||
# Generate diff
|
||||
diff_lines = difflib.unified_diff(
|
||||
current_content.splitlines(keepends=True),
|
||||
new_content.splitlines(keepends=True),
|
||||
fromfile=f"a/{op.file_path}",
|
||||
tofile=f"b/{op.file_path}"
|
||||
)
|
||||
diff = ''.join(diff_lines)
|
||||
|
||||
return True, diff
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Shared path validation helpers for tool implementations.
|
||||
|
||||
Extracts the ``resolve() + relative_to()`` and ``..`` traversal check
|
||||
patterns previously duplicated across skill_manager_tool, skills_tool,
|
||||
skills_hub, cronjob_tools, and credential_files.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def validate_within_dir(path: Path, root: Path) -> Optional[str]:
|
||||
"""Ensure *path* resolves to a location within *root*.
|
||||
|
||||
Returns an error message string if validation fails, or ``None`` if the
|
||||
path is safe. Uses ``Path.resolve()`` to follow symlinks and normalize
|
||||
``..`` components.
|
||||
|
||||
Usage::
|
||||
|
||||
error = validate_within_dir(user_path, allowed_root)
|
||||
if error:
|
||||
return json.dumps({"error": error})
|
||||
"""
|
||||
try:
|
||||
resolved = path.resolve()
|
||||
root_resolved = root.resolve()
|
||||
resolved.relative_to(root_resolved)
|
||||
except (ValueError, OSError) as exc:
|
||||
return f"Path escapes allowed directory: {exc}"
|
||||
return None
|
||||
|
||||
|
||||
def has_traversal_component(path_str: str) -> bool:
|
||||
"""Return True if *path_str* contains ``..`` traversal components.
|
||||
|
||||
Quick check for obvious traversal attempts before doing full resolution.
|
||||
"""
|
||||
parts = Path(path_str).parts
|
||||
return ".." in parts
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,386 @@
|
||||
"""Central registry for all hermes-agent tools.
|
||||
|
||||
Each tool file calls ``registry.register()`` at module level to declare its
|
||||
schema, handler, toolset membership, and availability check. ``model_tools.py``
|
||||
queries the registry instead of maintaining its own parallel data structures.
|
||||
|
||||
Import chain (circular-import safe):
|
||||
tools/registry.py (no imports from model_tools or tool files)
|
||||
^
|
||||
tools/*.py (import from tools.registry at module level)
|
||||
^
|
||||
model_tools.py (imports tools.registry + all tool modules)
|
||||
^
|
||||
run_agent.py, cli.py, batch_runner.py, etc.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
from typing import Callable, Dict, List, Optional, Set
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ToolEntry:
|
||||
"""Metadata for a single registered tool."""
|
||||
|
||||
__slots__ = (
|
||||
"name", "toolset", "schema", "handler", "check_fn",
|
||||
"requires_env", "is_async", "description", "emoji",
|
||||
"max_result_size_chars",
|
||||
)
|
||||
|
||||
def __init__(self, name, toolset, schema, handler, check_fn,
|
||||
requires_env, is_async, description, emoji,
|
||||
max_result_size_chars=None):
|
||||
self.name = name
|
||||
self.toolset = toolset
|
||||
self.schema = schema
|
||||
self.handler = handler
|
||||
self.check_fn = check_fn
|
||||
self.requires_env = requires_env
|
||||
self.is_async = is_async
|
||||
self.description = description
|
||||
self.emoji = emoji
|
||||
self.max_result_size_chars = max_result_size_chars
|
||||
|
||||
|
||||
class ToolRegistry:
|
||||
"""Singleton registry that collects tool schemas + handlers from tool files."""
|
||||
|
||||
def __init__(self):
|
||||
self._tools: Dict[str, ToolEntry] = {}
|
||||
self._toolset_checks: Dict[str, Callable] = {}
|
||||
# MCP dynamic refresh can mutate the registry while other threads are
|
||||
# reading tool metadata, so keep mutations serialized and readers on
|
||||
# stable snapshots.
|
||||
self._lock = threading.RLock()
|
||||
|
||||
def _snapshot_state(self) -> tuple[List[ToolEntry], Dict[str, Callable]]:
|
||||
"""Return a coherent snapshot of registry entries and toolset checks."""
|
||||
with self._lock:
|
||||
return list(self._tools.values()), dict(self._toolset_checks)
|
||||
|
||||
def _snapshot_entries(self) -> List[ToolEntry]:
|
||||
"""Return a stable snapshot of registered tool entries."""
|
||||
return self._snapshot_state()[0]
|
||||
|
||||
def _snapshot_toolset_checks(self) -> Dict[str, Callable]:
|
||||
"""Return a stable snapshot of toolset availability checks."""
|
||||
return self._snapshot_state()[1]
|
||||
|
||||
def _evaluate_toolset_check(self, toolset: str, check: Callable | None) -> bool:
|
||||
"""Run a toolset check, treating missing or failing checks as unavailable/available."""
|
||||
if not check:
|
||||
return True
|
||||
try:
|
||||
return bool(check())
|
||||
except Exception:
|
||||
logger.debug("Toolset %s check raised; marking unavailable", toolset)
|
||||
return False
|
||||
|
||||
def get_entry(self, name: str) -> Optional[ToolEntry]:
|
||||
"""Return a registered tool entry by name, or None."""
|
||||
with self._lock:
|
||||
return self._tools.get(name)
|
||||
|
||||
def get_registered_toolset_names(self) -> List[str]:
|
||||
"""Return sorted unique toolset names present in the registry."""
|
||||
return sorted({entry.toolset for entry in self._snapshot_entries()})
|
||||
|
||||
def get_tool_names_for_toolset(self, toolset: str) -> List[str]:
|
||||
"""Return sorted tool names registered under a given toolset."""
|
||||
return sorted(
|
||||
entry.name for entry in self._snapshot_entries()
|
||||
if entry.toolset == toolset
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Registration
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def register(
|
||||
self,
|
||||
name: str,
|
||||
toolset: str,
|
||||
schema: dict,
|
||||
handler: Callable,
|
||||
check_fn: Callable = None,
|
||||
requires_env: list = None,
|
||||
is_async: bool = False,
|
||||
description: str = "",
|
||||
emoji: str = "",
|
||||
max_result_size_chars: int | float | None = None,
|
||||
):
|
||||
"""Register a tool. Called at module-import time by each tool file."""
|
||||
with self._lock:
|
||||
existing = self._tools.get(name)
|
||||
if existing and existing.toolset != toolset:
|
||||
logger.warning(
|
||||
"Tool name collision: '%s' (toolset '%s') is being "
|
||||
"overwritten by toolset '%s'",
|
||||
name, existing.toolset, toolset,
|
||||
)
|
||||
self._tools[name] = ToolEntry(
|
||||
name=name,
|
||||
toolset=toolset,
|
||||
schema=schema,
|
||||
handler=handler,
|
||||
check_fn=check_fn,
|
||||
requires_env=requires_env or [],
|
||||
is_async=is_async,
|
||||
description=description or schema.get("description", ""),
|
||||
emoji=emoji,
|
||||
max_result_size_chars=max_result_size_chars,
|
||||
)
|
||||
if check_fn and toolset not in self._toolset_checks:
|
||||
self._toolset_checks[toolset] = check_fn
|
||||
|
||||
def deregister(self, name: str) -> None:
|
||||
"""Remove a tool from the registry.
|
||||
|
||||
Also cleans up the toolset check if no other tools remain in the
|
||||
same toolset. Used by MCP dynamic tool discovery to nuke-and-repave
|
||||
when a server sends ``notifications/tools/list_changed``.
|
||||
"""
|
||||
with self._lock:
|
||||
entry = self._tools.pop(name, None)
|
||||
if entry is None:
|
||||
return
|
||||
# Drop the toolset check if this was the last tool in that toolset
|
||||
if entry.toolset in self._toolset_checks and not any(
|
||||
e.toolset == entry.toolset for e in self._tools.values()
|
||||
):
|
||||
self._toolset_checks.pop(entry.toolset, None)
|
||||
logger.debug("Deregistered tool: %s", name)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Schema retrieval
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def get_definitions(self, tool_names: Set[str], quiet: bool = False) -> List[dict]:
|
||||
"""Return OpenAI-format tool schemas for the requested tool names.
|
||||
|
||||
Only tools whose ``check_fn()`` returns True (or have no check_fn)
|
||||
are included.
|
||||
"""
|
||||
result = []
|
||||
check_results: Dict[Callable, bool] = {}
|
||||
entries_by_name = {entry.name: entry for entry in self._snapshot_entries()}
|
||||
for name in sorted(tool_names):
|
||||
entry = entries_by_name.get(name)
|
||||
if not entry:
|
||||
continue
|
||||
if entry.check_fn:
|
||||
if entry.check_fn not in check_results:
|
||||
try:
|
||||
check_results[entry.check_fn] = bool(entry.check_fn())
|
||||
except Exception:
|
||||
check_results[entry.check_fn] = False
|
||||
if not quiet:
|
||||
logger.debug("Tool %s check raised; skipping", name)
|
||||
if not check_results[entry.check_fn]:
|
||||
if not quiet:
|
||||
logger.debug("Tool %s unavailable (check failed)", name)
|
||||
continue
|
||||
# Ensure schema always has a "name" field — use entry.name as fallback
|
||||
schema_with_name = {**entry.schema, "name": entry.name}
|
||||
result.append({"type": "function", "function": schema_with_name})
|
||||
return result
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Dispatch
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def dispatch(self, name: str, args: dict, **kwargs) -> str:
|
||||
"""Execute a tool handler by name.
|
||||
|
||||
* Async handlers are bridged automatically via ``_run_async()``.
|
||||
* All exceptions are caught and returned as ``{"error": "..."}``
|
||||
for consistent error format.
|
||||
"""
|
||||
entry = self.get_entry(name)
|
||||
if not entry:
|
||||
return json.dumps({"error": f"Unknown tool: {name}"})
|
||||
try:
|
||||
if entry.is_async:
|
||||
from model_tools import _run_async
|
||||
return _run_async(entry.handler(args, **kwargs))
|
||||
return entry.handler(args, **kwargs)
|
||||
except Exception as e:
|
||||
logger.exception("Tool %s dispatch error: %s", name, e)
|
||||
return json.dumps({"error": f"Tool execution failed: {type(e).__name__}: {e}"})
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Query helpers (replace redundant dicts in model_tools.py)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def get_max_result_size(self, name: str, default: int | float | None = None) -> int | float:
|
||||
"""Return per-tool max result size, or *default* (or global default)."""
|
||||
entry = self.get_entry(name)
|
||||
if entry and entry.max_result_size_chars is not None:
|
||||
return entry.max_result_size_chars
|
||||
if default is not None:
|
||||
return default
|
||||
from tools.budget_config import DEFAULT_RESULT_SIZE_CHARS
|
||||
return DEFAULT_RESULT_SIZE_CHARS
|
||||
|
||||
def get_all_tool_names(self) -> List[str]:
|
||||
"""Return sorted list of all registered tool names."""
|
||||
return sorted(entry.name for entry in self._snapshot_entries())
|
||||
|
||||
def get_schema(self, name: str) -> Optional[dict]:
|
||||
"""Return a tool's raw schema dict, bypassing check_fn filtering.
|
||||
|
||||
Useful for token estimation and introspection where availability
|
||||
doesn't matter — only the schema content does.
|
||||
"""
|
||||
entry = self.get_entry(name)
|
||||
return entry.schema if entry else None
|
||||
|
||||
def get_toolset_for_tool(self, name: str) -> Optional[str]:
|
||||
"""Return the toolset a tool belongs to, or None."""
|
||||
entry = self.get_entry(name)
|
||||
return entry.toolset if entry else None
|
||||
|
||||
def get_emoji(self, name: str, default: str = "⚡") -> str:
|
||||
"""Return the emoji for a tool, or *default* if unset."""
|
||||
entry = self.get_entry(name)
|
||||
return (entry.emoji if entry and entry.emoji else default)
|
||||
|
||||
def get_tool_to_toolset_map(self) -> Dict[str, str]:
|
||||
"""Return ``{tool_name: toolset_name}`` for every registered tool."""
|
||||
return {entry.name: entry.toolset for entry in self._snapshot_entries()}
|
||||
|
||||
def is_toolset_available(self, toolset: str) -> bool:
|
||||
"""Check if a toolset's requirements are met.
|
||||
|
||||
Returns False (rather than crashing) when the check function raises
|
||||
an unexpected exception (e.g. network error, missing import, bad config).
|
||||
"""
|
||||
with self._lock:
|
||||
check = self._toolset_checks.get(toolset)
|
||||
return self._evaluate_toolset_check(toolset, check)
|
||||
|
||||
def check_toolset_requirements(self) -> Dict[str, bool]:
|
||||
"""Return ``{toolset: available_bool}`` for every toolset."""
|
||||
entries, toolset_checks = self._snapshot_state()
|
||||
toolsets = sorted({entry.toolset for entry in entries})
|
||||
return {
|
||||
toolset: self._evaluate_toolset_check(toolset, toolset_checks.get(toolset))
|
||||
for toolset in toolsets
|
||||
}
|
||||
|
||||
def get_available_toolsets(self) -> Dict[str, dict]:
|
||||
"""Return toolset metadata for UI display."""
|
||||
toolsets: Dict[str, dict] = {}
|
||||
entries, toolset_checks = self._snapshot_state()
|
||||
for entry in entries:
|
||||
ts = entry.toolset
|
||||
if ts not in toolsets:
|
||||
toolsets[ts] = {
|
||||
"available": self._evaluate_toolset_check(
|
||||
ts, toolset_checks.get(ts)
|
||||
),
|
||||
"tools": [],
|
||||
"description": "",
|
||||
"requirements": [],
|
||||
}
|
||||
toolsets[ts]["tools"].append(entry.name)
|
||||
if entry.requires_env:
|
||||
for env in entry.requires_env:
|
||||
if env not in toolsets[ts]["requirements"]:
|
||||
toolsets[ts]["requirements"].append(env)
|
||||
return toolsets
|
||||
|
||||
def get_toolset_requirements(self) -> Dict[str, dict]:
|
||||
"""Build a TOOLSET_REQUIREMENTS-compatible dict for backward compat."""
|
||||
result: Dict[str, dict] = {}
|
||||
entries, toolset_checks = self._snapshot_state()
|
||||
for entry in entries:
|
||||
ts = entry.toolset
|
||||
if ts not in result:
|
||||
result[ts] = {
|
||||
"name": ts,
|
||||
"env_vars": [],
|
||||
"check_fn": toolset_checks.get(ts),
|
||||
"setup_url": None,
|
||||
"tools": [],
|
||||
}
|
||||
if entry.name not in result[ts]["tools"]:
|
||||
result[ts]["tools"].append(entry.name)
|
||||
for env in entry.requires_env:
|
||||
if env not in result[ts]["env_vars"]:
|
||||
result[ts]["env_vars"].append(env)
|
||||
return result
|
||||
|
||||
def check_tool_availability(self, quiet: bool = False):
|
||||
"""Return (available_toolsets, unavailable_info) like the old function."""
|
||||
available = []
|
||||
unavailable = []
|
||||
seen = set()
|
||||
entries, toolset_checks = self._snapshot_state()
|
||||
for entry in entries:
|
||||
ts = entry.toolset
|
||||
if ts in seen:
|
||||
continue
|
||||
seen.add(ts)
|
||||
if self._evaluate_toolset_check(ts, toolset_checks.get(ts)):
|
||||
available.append(ts)
|
||||
else:
|
||||
unavailable.append({
|
||||
"name": ts,
|
||||
"env_vars": entry.requires_env,
|
||||
"tools": [e.name for e in entries if e.toolset == ts],
|
||||
})
|
||||
return available, unavailable
|
||||
|
||||
|
||||
# Module-level singleton
|
||||
registry = ToolRegistry()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers for tool response serialization
|
||||
# ---------------------------------------------------------------------------
|
||||
# Every tool handler must return a JSON string. These helpers eliminate the
|
||||
# boilerplate ``json.dumps({"error": msg}, ensure_ascii=False)`` that appears
|
||||
# hundreds of times across tool files.
|
||||
#
|
||||
# Usage:
|
||||
# from tools.registry import registry, tool_error, tool_result
|
||||
#
|
||||
# return tool_error("something went wrong")
|
||||
# return tool_error("not found", code=404)
|
||||
# return tool_result(success=True, data=payload)
|
||||
# return tool_result(items) # pass a dict directly
|
||||
|
||||
|
||||
def tool_error(message, **extra) -> str:
|
||||
"""Return a JSON error string for tool handlers.
|
||||
|
||||
>>> tool_error("file not found")
|
||||
'{"error": "file not found"}'
|
||||
>>> tool_error("bad input", success=False)
|
||||
'{"error": "bad input", "success": false}'
|
||||
"""
|
||||
result = {"error": str(message)}
|
||||
if extra:
|
||||
result.update(extra)
|
||||
return json.dumps(result, ensure_ascii=False)
|
||||
|
||||
|
||||
def tool_result(data=None, **kwargs) -> str:
|
||||
"""Return a JSON result string for tool handlers.
|
||||
|
||||
Accepts a dict positional arg *or* keyword arguments (not both):
|
||||
|
||||
>>> tool_result(success=True, count=42)
|
||||
'{"success": true, "count": 42}'
|
||||
>>> tool_result({"key": "value"})
|
||||
'{"key": "value"}'
|
||||
"""
|
||||
if data is not None:
|
||||
return json.dumps(data, ensure_ascii=False)
|
||||
return json.dumps(kwargs, ensure_ascii=False)
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,554 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Session Search Tool - Long-Term Conversation Recall
|
||||
|
||||
Searches past session transcripts in SQLite via FTS5, then summarizes the top
|
||||
matching sessions using a cheap/fast model (same pattern as web_extract).
|
||||
Returns focused summaries of past conversations rather than raw transcripts,
|
||||
keeping the main model's context window clean.
|
||||
|
||||
Flow:
|
||||
1. FTS5 search finds matching messages ranked by relevance
|
||||
2. Groups by session, takes the top N unique sessions (default 3)
|
||||
3. Loads each session's conversation, truncates to ~100k chars centered on matches
|
||||
4. Sends to Gemini Flash with a focused summarization prompt
|
||||
5. Returns per-session summaries with metadata
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import concurrent.futures
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import Dict, Any, List, Optional, Union
|
||||
|
||||
from agent.auxiliary_client import async_call_llm, extract_content_or_reasoning
|
||||
MAX_SESSION_CHARS = 100_000
|
||||
MAX_SUMMARY_TOKENS = 10000
|
||||
|
||||
|
||||
def _format_timestamp(ts: Union[int, float, str, None]) -> str:
|
||||
"""Convert a Unix timestamp (float/int) or ISO string to a human-readable date.
|
||||
|
||||
Returns "unknown" for None, str(ts) if conversion fails.
|
||||
"""
|
||||
if ts is None:
|
||||
return "unknown"
|
||||
try:
|
||||
if isinstance(ts, (int, float)):
|
||||
from datetime import datetime
|
||||
dt = datetime.fromtimestamp(ts)
|
||||
return dt.strftime("%B %d, %Y at %I:%M %p")
|
||||
if isinstance(ts, str):
|
||||
if ts.replace(".", "").replace("-", "").isdigit():
|
||||
from datetime import datetime
|
||||
dt = datetime.fromtimestamp(float(ts))
|
||||
return dt.strftime("%B %d, %Y at %I:%M %p")
|
||||
return ts
|
||||
except (ValueError, OSError, OverflowError) as e:
|
||||
# Log specific errors for debugging while gracefully handling edge cases
|
||||
logging.debug("Failed to format timestamp %s: %s", ts, e, exc_info=True)
|
||||
except Exception as e:
|
||||
logging.debug("Unexpected error formatting timestamp %s: %s", ts, e, exc_info=True)
|
||||
return str(ts)
|
||||
|
||||
|
||||
def _format_conversation(messages: List[Dict[str, Any]]) -> str:
|
||||
"""Format session messages into a readable transcript for summarization."""
|
||||
parts = []
|
||||
for msg in messages:
|
||||
role = msg.get("role", "unknown").upper()
|
||||
content = msg.get("content") or ""
|
||||
tool_name = msg.get("tool_name")
|
||||
|
||||
if role == "TOOL" and tool_name:
|
||||
# Truncate long tool outputs
|
||||
if len(content) > 500:
|
||||
content = content[:250] + "\n...[truncated]...\n" + content[-250:]
|
||||
parts.append(f"[TOOL:{tool_name}]: {content}")
|
||||
elif role == "ASSISTANT":
|
||||
# Include tool call names if present
|
||||
tool_calls = msg.get("tool_calls")
|
||||
if tool_calls and isinstance(tool_calls, list):
|
||||
tc_names = []
|
||||
for tc in tool_calls:
|
||||
if isinstance(tc, dict):
|
||||
name = tc.get("name") or tc.get("function", {}).get("name", "?")
|
||||
tc_names.append(name)
|
||||
if tc_names:
|
||||
parts.append(f"[ASSISTANT]: [Called: {', '.join(tc_names)}]")
|
||||
if content:
|
||||
parts.append(f"[ASSISTANT]: {content}")
|
||||
else:
|
||||
parts.append(f"[ASSISTANT]: {content}")
|
||||
else:
|
||||
parts.append(f"[{role}]: {content}")
|
||||
|
||||
return "\n\n".join(parts)
|
||||
|
||||
|
||||
def _truncate_around_matches(
|
||||
full_text: str, query: str, max_chars: int = MAX_SESSION_CHARS
|
||||
) -> str:
|
||||
"""
|
||||
Truncate a conversation transcript to *max_chars*, choosing a window
|
||||
that maximises coverage of positions where the *query* actually appears.
|
||||
|
||||
Strategy (in priority order):
|
||||
1. Try to find the full query as a phrase (case-insensitive).
|
||||
2. If no phrase hit, look for positions where all query terms appear
|
||||
within a 200-char proximity window (co-occurrence).
|
||||
3. Fall back to individual term positions.
|
||||
|
||||
Once candidate positions are collected the function picks the window
|
||||
start that covers the most of them.
|
||||
"""
|
||||
if len(full_text) <= max_chars:
|
||||
return full_text
|
||||
|
||||
text_lower = full_text.lower()
|
||||
query_lower = query.lower().strip()
|
||||
match_positions: list[int] = []
|
||||
|
||||
# --- 1. Full-phrase search ------------------------------------------------
|
||||
phrase_pat = re.compile(re.escape(query_lower))
|
||||
match_positions = [m.start() for m in phrase_pat.finditer(text_lower)]
|
||||
|
||||
# --- 2. Proximity co-occurrence of all terms (within 200 chars) -----------
|
||||
if not match_positions:
|
||||
terms = query_lower.split()
|
||||
if len(terms) > 1:
|
||||
# Collect every occurrence of each term
|
||||
term_positions: dict[str, list[int]] = {}
|
||||
for t in terms:
|
||||
term_positions[t] = [
|
||||
m.start() for m in re.finditer(re.escape(t), text_lower)
|
||||
]
|
||||
# Slide through positions of the rarest term and check proximity
|
||||
rarest = min(terms, key=lambda t: len(term_positions.get(t, [])))
|
||||
for pos in term_positions.get(rarest, []):
|
||||
if all(
|
||||
any(abs(p - pos) < 200 for p in term_positions.get(t, []))
|
||||
for t in terms
|
||||
if t != rarest
|
||||
):
|
||||
match_positions.append(pos)
|
||||
|
||||
# --- 3. Individual term positions (last resort) ---------------------------
|
||||
if not match_positions:
|
||||
terms = query_lower.split()
|
||||
for t in terms:
|
||||
for m in re.finditer(re.escape(t), text_lower):
|
||||
match_positions.append(m.start())
|
||||
|
||||
if not match_positions:
|
||||
# Nothing at all — take from the start
|
||||
truncated = full_text[:max_chars]
|
||||
suffix = "\n\n...[later conversation truncated]..." if max_chars < len(full_text) else ""
|
||||
return truncated + suffix
|
||||
|
||||
# --- Pick window that covers the most match positions ---------------------
|
||||
match_positions.sort()
|
||||
|
||||
best_start = 0
|
||||
best_count = 0
|
||||
for candidate in match_positions:
|
||||
ws = max(0, candidate - max_chars // 4) # bias: 25% before, 75% after
|
||||
we = ws + max_chars
|
||||
if we > len(full_text):
|
||||
ws = max(0, len(full_text) - max_chars)
|
||||
we = len(full_text)
|
||||
count = sum(1 for p in match_positions if ws <= p < we)
|
||||
if count > best_count:
|
||||
best_count = count
|
||||
best_start = ws
|
||||
|
||||
start = best_start
|
||||
end = min(len(full_text), start + max_chars)
|
||||
|
||||
truncated = full_text[start:end]
|
||||
prefix = "...[earlier conversation truncated]...\n\n" if start > 0 else ""
|
||||
suffix = "\n\n...[later conversation truncated]..." if end < len(full_text) else ""
|
||||
return prefix + truncated + suffix
|
||||
|
||||
|
||||
async def _summarize_session(
|
||||
conversation_text: str, query: str, session_meta: Dict[str, Any]
|
||||
) -> Optional[str]:
|
||||
"""Summarize a single session conversation focused on the search query."""
|
||||
system_prompt = (
|
||||
"You are reviewing a past conversation transcript to help recall what happened. "
|
||||
"Summarize the conversation with a focus on the search topic. Include:\n"
|
||||
"1. What the user asked about or wanted to accomplish\n"
|
||||
"2. What actions were taken and what the outcomes were\n"
|
||||
"3. Key decisions, solutions found, or conclusions reached\n"
|
||||
"4. Any specific commands, files, URLs, or technical details that were important\n"
|
||||
"5. Anything left unresolved or notable\n\n"
|
||||
"Be thorough but concise. Preserve specific details (commands, paths, error messages) "
|
||||
"that would be useful to recall. Write in past tense as a factual recap."
|
||||
)
|
||||
|
||||
source = session_meta.get("source", "unknown")
|
||||
started = _format_timestamp(session_meta.get("started_at"))
|
||||
|
||||
user_prompt = (
|
||||
f"Search topic: {query}\n"
|
||||
f"Session source: {source}\n"
|
||||
f"Session date: {started}\n\n"
|
||||
f"CONVERSATION TRANSCRIPT:\n{conversation_text}\n\n"
|
||||
f"Summarize this conversation with focus on: {query}"
|
||||
)
|
||||
|
||||
max_retries = 3
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
response = await async_call_llm(
|
||||
task="session_search",
|
||||
messages=[
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": user_prompt},
|
||||
],
|
||||
temperature=0.1,
|
||||
max_tokens=MAX_SUMMARY_TOKENS,
|
||||
)
|
||||
content = extract_content_or_reasoning(response)
|
||||
if content:
|
||||
return content
|
||||
# Reasoning-only / empty — let the retry loop handle it
|
||||
logging.warning("Session search LLM returned empty content (attempt %d/%d)", attempt + 1, max_retries)
|
||||
if attempt < max_retries - 1:
|
||||
await asyncio.sleep(1 * (attempt + 1))
|
||||
continue
|
||||
return content
|
||||
except RuntimeError:
|
||||
logging.warning("No auxiliary model available for session summarization")
|
||||
return None
|
||||
except Exception as e:
|
||||
if attempt < max_retries - 1:
|
||||
await asyncio.sleep(1 * (attempt + 1))
|
||||
else:
|
||||
logging.warning(
|
||||
"Session summarization failed after %d attempts: %s",
|
||||
max_retries,
|
||||
e,
|
||||
exc_info=True,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
# Sources that are excluded from session browsing/searching by default.
|
||||
# Third-party integrations (Paperclip agents, etc.) tag their sessions with
|
||||
# HERMES_SESSION_SOURCE=tool so they don't clutter the user's session history.
|
||||
_HIDDEN_SESSION_SOURCES = ("tool",)
|
||||
|
||||
|
||||
def _list_recent_sessions(db, limit: int, current_session_id: str = None) -> str:
|
||||
"""Return metadata for the most recent sessions (no LLM calls)."""
|
||||
try:
|
||||
sessions = db.list_sessions_rich(limit=limit + 5, exclude_sources=list(_HIDDEN_SESSION_SOURCES)) # fetch extra to skip current
|
||||
|
||||
# Resolve current session lineage to exclude it
|
||||
current_root = None
|
||||
if current_session_id:
|
||||
try:
|
||||
sid = current_session_id
|
||||
visited = set()
|
||||
while sid and sid not in visited:
|
||||
visited.add(sid)
|
||||
s = db.get_session(sid)
|
||||
parent = s.get("parent_session_id") if s else None
|
||||
sid = parent if parent else None
|
||||
current_root = max(visited, key=len) if visited else current_session_id
|
||||
except Exception:
|
||||
current_root = current_session_id
|
||||
|
||||
results = []
|
||||
for s in sessions:
|
||||
sid = s.get("id", "")
|
||||
if current_root and (sid == current_root or sid == current_session_id):
|
||||
continue
|
||||
# Skip child/delegation sessions (they have parent_session_id)
|
||||
if s.get("parent_session_id"):
|
||||
continue
|
||||
results.append({
|
||||
"session_id": sid,
|
||||
"title": s.get("title") or None,
|
||||
"source": s.get("source", ""),
|
||||
"started_at": s.get("started_at", ""),
|
||||
"last_active": s.get("last_active", ""),
|
||||
"message_count": s.get("message_count", 0),
|
||||
"preview": s.get("preview", ""),
|
||||
})
|
||||
if len(results) >= limit:
|
||||
break
|
||||
|
||||
return json.dumps({
|
||||
"success": True,
|
||||
"mode": "recent",
|
||||
"results": results,
|
||||
"count": len(results),
|
||||
"message": f"Showing {len(results)} most recent sessions. Use a keyword query to search specific topics.",
|
||||
}, ensure_ascii=False)
|
||||
except Exception as e:
|
||||
logging.error("Error listing recent sessions: %s", e, exc_info=True)
|
||||
return tool_error(f"Failed to list recent sessions: {e}", success=False)
|
||||
|
||||
|
||||
def session_search(
|
||||
query: str,
|
||||
role_filter: str = None,
|
||||
limit: int = 3,
|
||||
db=None,
|
||||
current_session_id: str = None,
|
||||
) -> str:
|
||||
"""
|
||||
Search past sessions and return focused summaries of matching conversations.
|
||||
|
||||
Uses FTS5 to find matches, then summarizes the top sessions with Gemini Flash.
|
||||
The current session is excluded from results since the agent already has that context.
|
||||
"""
|
||||
if db is None:
|
||||
return tool_error("Session database not available.", success=False)
|
||||
|
||||
limit = min(limit, 5) # Cap at 5 sessions to avoid excessive LLM calls
|
||||
|
||||
# Recent sessions mode: when query is empty, return metadata for recent sessions.
|
||||
# No LLM calls — just DB queries for titles, previews, timestamps.
|
||||
if not query or not query.strip():
|
||||
return _list_recent_sessions(db, limit, current_session_id)
|
||||
|
||||
query = query.strip()
|
||||
|
||||
try:
|
||||
# Parse role filter
|
||||
role_list = None
|
||||
if role_filter and role_filter.strip():
|
||||
role_list = [r.strip() for r in role_filter.split(",") if r.strip()]
|
||||
|
||||
# FTS5 search -- get matches ranked by relevance
|
||||
raw_results = db.search_messages(
|
||||
query=query,
|
||||
role_filter=role_list,
|
||||
exclude_sources=list(_HIDDEN_SESSION_SOURCES),
|
||||
limit=50, # Get more matches to find unique sessions
|
||||
offset=0,
|
||||
)
|
||||
|
||||
if not raw_results:
|
||||
return json.dumps({
|
||||
"success": True,
|
||||
"query": query,
|
||||
"results": [],
|
||||
"count": 0,
|
||||
"message": "No matching sessions found.",
|
||||
}, ensure_ascii=False)
|
||||
|
||||
# Resolve child sessions to their parent — delegation stores detailed
|
||||
# content in child sessions, but the user's conversation is the parent.
|
||||
def _resolve_to_parent(session_id: str) -> str:
|
||||
"""Walk delegation chain to find the root parent session ID."""
|
||||
visited = set()
|
||||
sid = session_id
|
||||
while sid and sid not in visited:
|
||||
visited.add(sid)
|
||||
try:
|
||||
session = db.get_session(sid)
|
||||
if not session:
|
||||
break
|
||||
parent = session.get("parent_session_id")
|
||||
if parent:
|
||||
sid = parent
|
||||
else:
|
||||
break
|
||||
except Exception as e:
|
||||
logging.debug(
|
||||
"Error resolving parent for session %s: %s",
|
||||
sid,
|
||||
e,
|
||||
exc_info=True,
|
||||
)
|
||||
break
|
||||
return sid
|
||||
|
||||
current_lineage_root = (
|
||||
_resolve_to_parent(current_session_id) if current_session_id else None
|
||||
)
|
||||
|
||||
# Group by resolved (parent) session_id, dedup, skip the current
|
||||
# session lineage. Compression and delegation create child sessions
|
||||
# that still belong to the same active conversation.
|
||||
seen_sessions = {}
|
||||
for result in raw_results:
|
||||
raw_sid = result["session_id"]
|
||||
resolved_sid = _resolve_to_parent(raw_sid)
|
||||
# Skip the current session lineage — the agent already has that
|
||||
# context, even if older turns live in parent fragments.
|
||||
if current_lineage_root and resolved_sid == current_lineage_root:
|
||||
continue
|
||||
if current_session_id and raw_sid == current_session_id:
|
||||
continue
|
||||
if resolved_sid not in seen_sessions:
|
||||
result = dict(result)
|
||||
result["session_id"] = resolved_sid
|
||||
seen_sessions[resolved_sid] = result
|
||||
if len(seen_sessions) >= limit:
|
||||
break
|
||||
|
||||
# Prepare all sessions for parallel summarization
|
||||
tasks = []
|
||||
for session_id, match_info in seen_sessions.items():
|
||||
try:
|
||||
messages = db.get_messages_as_conversation(session_id)
|
||||
if not messages:
|
||||
continue
|
||||
session_meta = db.get_session(session_id) or {}
|
||||
conversation_text = _format_conversation(messages)
|
||||
conversation_text = _truncate_around_matches(conversation_text, query)
|
||||
tasks.append((session_id, match_info, conversation_text, session_meta))
|
||||
except Exception as e:
|
||||
logging.warning(
|
||||
"Failed to prepare session %s: %s",
|
||||
session_id,
|
||||
e,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
# Summarize all sessions in parallel
|
||||
async def _summarize_all() -> List[Union[str, Exception]]:
|
||||
"""Summarize all sessions in parallel."""
|
||||
coros = [
|
||||
_summarize_session(text, query, meta)
|
||||
for _, _, text, meta in tasks
|
||||
]
|
||||
return await asyncio.gather(*coros, return_exceptions=True)
|
||||
|
||||
try:
|
||||
# Use _run_async() which properly manages event loops across
|
||||
# CLI, gateway, and worker-thread contexts. The previous
|
||||
# pattern (asyncio.run() in a ThreadPoolExecutor) created a
|
||||
# disposable event loop that conflicted with cached
|
||||
# AsyncOpenAI/httpx clients bound to a different loop,
|
||||
# causing deadlocks in gateway mode (#2681).
|
||||
from model_tools import _run_async
|
||||
results = _run_async(_summarize_all())
|
||||
except concurrent.futures.TimeoutError:
|
||||
logging.warning(
|
||||
"Session summarization timed out after 60 seconds",
|
||||
exc_info=True,
|
||||
)
|
||||
return json.dumps({
|
||||
"success": False,
|
||||
"error": "Session summarization timed out. Try a more specific query or reduce the limit.",
|
||||
}, ensure_ascii=False)
|
||||
|
||||
summaries = []
|
||||
for (session_id, match_info, conversation_text, _), result in zip(tasks, results):
|
||||
if isinstance(result, Exception):
|
||||
logging.warning(
|
||||
"Failed to summarize session %s: %s",
|
||||
session_id, result, exc_info=True,
|
||||
)
|
||||
result = None
|
||||
|
||||
entry = {
|
||||
"session_id": session_id,
|
||||
"when": _format_timestamp(match_info.get("session_started")),
|
||||
"source": match_info.get("source", "unknown"),
|
||||
"model": match_info.get("model"),
|
||||
}
|
||||
|
||||
if result:
|
||||
entry["summary"] = result
|
||||
else:
|
||||
# Fallback: raw preview so matched sessions aren't silently
|
||||
# dropped when the summarizer is unavailable (fixes #3409).
|
||||
preview = (conversation_text[:500] + "\n…[truncated]") if conversation_text else "No preview available."
|
||||
entry["summary"] = f"[Raw preview — summarization unavailable]\n{preview}"
|
||||
|
||||
summaries.append(entry)
|
||||
|
||||
return json.dumps({
|
||||
"success": True,
|
||||
"query": query,
|
||||
"results": summaries,
|
||||
"count": len(summaries),
|
||||
"sessions_searched": len(seen_sessions),
|
||||
}, ensure_ascii=False)
|
||||
|
||||
except Exception as e:
|
||||
logging.error("Session search failed: %s", e, exc_info=True)
|
||||
return tool_error(f"Search failed: {str(e)}", success=False)
|
||||
|
||||
|
||||
def check_session_search_requirements() -> bool:
|
||||
"""Requires SQLite state database and an auxiliary text model."""
|
||||
try:
|
||||
from hermes_state import DEFAULT_DB_PATH
|
||||
return DEFAULT_DB_PATH.parent.exists()
|
||||
except ImportError:
|
||||
return False
|
||||
|
||||
|
||||
SESSION_SEARCH_SCHEMA = {
|
||||
"name": "session_search",
|
||||
"description": (
|
||||
"Search your long-term memory of past conversations, or browse recent sessions. This is your recall -- "
|
||||
"every past session is searchable, and this tool summarizes what happened.\n\n"
|
||||
"TWO MODES:\n"
|
||||
"1. Recent sessions (no query): Call with no arguments to see what was worked on recently. "
|
||||
"Returns titles, previews, and timestamps. Zero LLM cost, instant. "
|
||||
"Start here when the user asks what were we working on or what did we do recently.\n"
|
||||
"2. Keyword search (with query): Search for specific topics across all past sessions. "
|
||||
"Returns LLM-generated summaries of matching sessions.\n\n"
|
||||
"USE THIS PROACTIVELY when:\n"
|
||||
"- The user says 'we did this before', 'remember when', 'last time', 'as I mentioned'\n"
|
||||
"- The user asks about a topic you worked on before but don't have in current context\n"
|
||||
"- The user references a project, person, or concept that seems familiar but isn't in memory\n"
|
||||
"- You want to check if you've solved a similar problem before\n"
|
||||
"- The user asks 'what did we do about X?' or 'how did we fix Y?'\n\n"
|
||||
"Don't hesitate to search when it is actually cross-session -- it's fast and cheap. "
|
||||
"Better to search and confirm than to guess or ask the user to repeat themselves.\n\n"
|
||||
"Search syntax: keywords joined with OR for broad recall (elevenlabs OR baseten OR funding), "
|
||||
"phrases for exact match (\"docker networking\"), boolean (python NOT java), prefix (deploy*). "
|
||||
"IMPORTANT: Use OR between keywords for best results — FTS5 defaults to AND which misses "
|
||||
"sessions that only mention some terms. If a broad OR query returns nothing, try individual "
|
||||
"keyword searches in parallel. Returns summaries of the top matching sessions."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {
|
||||
"type": "string",
|
||||
"description": "Search query — keywords, phrases, or boolean expressions to find in past sessions. Omit this parameter entirely to browse recent sessions instead (returns titles, previews, timestamps with no LLM cost).",
|
||||
},
|
||||
"role_filter": {
|
||||
"type": "string",
|
||||
"description": "Optional: only search messages from specific roles (comma-separated). E.g. 'user,assistant' to skip tool outputs.",
|
||||
},
|
||||
"limit": {
|
||||
"type": "integer",
|
||||
"description": "Max sessions to summarize (default: 3, max: 5).",
|
||||
"default": 3,
|
||||
},
|
||||
},
|
||||
"required": [],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# --- Registry ---
|
||||
from tools.registry import registry, tool_error
|
||||
|
||||
registry.register(
|
||||
name="session_search",
|
||||
toolset="session_search",
|
||||
schema=SESSION_SEARCH_SCHEMA,
|
||||
handler=lambda args, **kw: session_search(
|
||||
query=args.get("query") or "",
|
||||
role_filter=args.get("role_filter"),
|
||||
limit=args.get("limit", 3),
|
||||
db=kw.get("db"),
|
||||
current_session_id=kw.get("current_session_id")),
|
||||
check_fn=check_session_search_requirements,
|
||||
emoji="🔍",
|
||||
)
|
||||
@@ -0,0 +1,761 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Skill Manager Tool -- Agent-Managed Skill Creation & Editing
|
||||
|
||||
Allows the agent to create, update, and delete skills, turning successful
|
||||
approaches into reusable procedural knowledge. New skills are created in
|
||||
~/.hermes/skills/. Existing skills (bundled, hub-installed, or user-created)
|
||||
can be modified or deleted wherever they live.
|
||||
|
||||
Skills are the agent's procedural memory: they capture *how to do a specific
|
||||
type of task* based on proven experience. General memory (MEMORY.md, USER.md) is
|
||||
broad and declarative. Skills are narrow and actionable.
|
||||
|
||||
Actions:
|
||||
create -- Create a new skill (SKILL.md + directory structure)
|
||||
edit -- Replace the SKILL.md content of a user skill (full rewrite)
|
||||
patch -- Targeted find-and-replace within SKILL.md or any supporting file
|
||||
delete -- Remove a user skill entirely
|
||||
write_file -- Add/overwrite a supporting file (reference, template, script, asset)
|
||||
remove_file-- Remove a supporting file from a user skill
|
||||
|
||||
Directory layout for user skills:
|
||||
~/.hermes/skills/
|
||||
├── my-skill/
|
||||
│ ├── SKILL.md
|
||||
│ ├── references/
|
||||
│ ├── templates/
|
||||
│ ├── scripts/
|
||||
│ └── assets/
|
||||
└── category-name/
|
||||
└── another-skill/
|
||||
└── SKILL.md
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from hermes_constants import get_hermes_home
|
||||
from typing import Dict, Any, Optional, Tuple
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Import security scanner — agent-created skills get the same scrutiny as
|
||||
# community hub installs.
|
||||
try:
|
||||
from tools.skills_guard import scan_skill, should_allow_install, format_scan_report
|
||||
_GUARD_AVAILABLE = True
|
||||
except ImportError:
|
||||
_GUARD_AVAILABLE = False
|
||||
|
||||
|
||||
def _security_scan_skill(skill_dir: Path) -> Optional[str]:
|
||||
"""Scan a skill directory after write. Returns error string if blocked, else None."""
|
||||
if not _GUARD_AVAILABLE:
|
||||
return None
|
||||
try:
|
||||
result = scan_skill(skill_dir, source="agent-created")
|
||||
allowed, reason = should_allow_install(result)
|
||||
if allowed is False:
|
||||
report = format_scan_report(result)
|
||||
return f"Security scan blocked this skill ({reason}):\n{report}"
|
||||
if allowed is None:
|
||||
# "ask" — allow but include the warning so the user sees the findings
|
||||
report = format_scan_report(result)
|
||||
logger.warning("Agent-created skill has security findings: %s", reason)
|
||||
# Don't block — return None to allow, but log the warning
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.warning("Security scan failed for %s: %s", skill_dir, e, exc_info=True)
|
||||
return None
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
# All skills live in ~/.hermes/skills/ (single source of truth)
|
||||
HERMES_HOME = get_hermes_home()
|
||||
SKILLS_DIR = HERMES_HOME / "skills"
|
||||
|
||||
MAX_NAME_LENGTH = 64
|
||||
MAX_DESCRIPTION_LENGTH = 1024
|
||||
MAX_SKILL_CONTENT_CHARS = 100_000 # ~36k tokens at 2.75 chars/token
|
||||
MAX_SKILL_FILE_BYTES = 1_048_576 # 1 MiB per supporting file
|
||||
|
||||
# Characters allowed in skill names (filesystem-safe, URL-friendly)
|
||||
VALID_NAME_RE = re.compile(r'^[a-z0-9][a-z0-9._-]*$')
|
||||
|
||||
# Subdirectories allowed for write_file/remove_file
|
||||
ALLOWED_SUBDIRS = {"references", "templates", "scripts", "assets"}
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Validation helpers
|
||||
# =============================================================================
|
||||
|
||||
def _validate_name(name: str) -> Optional[str]:
|
||||
"""Validate a skill name. Returns error message or None if valid."""
|
||||
if not name:
|
||||
return "Skill name is required."
|
||||
if len(name) > MAX_NAME_LENGTH:
|
||||
return f"Skill name exceeds {MAX_NAME_LENGTH} characters."
|
||||
if not VALID_NAME_RE.match(name):
|
||||
return (
|
||||
f"Invalid skill name '{name}'. Use lowercase letters, numbers, "
|
||||
f"hyphens, dots, and underscores. Must start with a letter or digit."
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _validate_category(category: Optional[str]) -> Optional[str]:
|
||||
"""Validate an optional category name used as a single directory segment."""
|
||||
if category is None:
|
||||
return None
|
||||
if not isinstance(category, str):
|
||||
return "Category must be a string."
|
||||
|
||||
category = category.strip()
|
||||
if not category:
|
||||
return None
|
||||
if "/" in category or "\\" in category:
|
||||
return (
|
||||
f"Invalid category '{category}'. Use lowercase letters, numbers, "
|
||||
"hyphens, dots, and underscores. Categories must be a single directory name."
|
||||
)
|
||||
if len(category) > MAX_NAME_LENGTH:
|
||||
return f"Category exceeds {MAX_NAME_LENGTH} characters."
|
||||
if not VALID_NAME_RE.match(category):
|
||||
return (
|
||||
f"Invalid category '{category}'. Use lowercase letters, numbers, "
|
||||
"hyphens, dots, and underscores. Categories must be a single directory name."
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _validate_frontmatter(content: str) -> Optional[str]:
|
||||
"""
|
||||
Validate that SKILL.md content has proper frontmatter with required fields.
|
||||
Returns error message or None if valid.
|
||||
"""
|
||||
if not content.strip():
|
||||
return "Content cannot be empty."
|
||||
|
||||
if not content.startswith("---"):
|
||||
return "SKILL.md must start with YAML frontmatter (---). See existing skills for format."
|
||||
|
||||
end_match = re.search(r'\n---\s*\n', content[3:])
|
||||
if not end_match:
|
||||
return "SKILL.md frontmatter is not closed. Ensure you have a closing '---' line."
|
||||
|
||||
yaml_content = content[3:end_match.start() + 3]
|
||||
|
||||
try:
|
||||
parsed = yaml.safe_load(yaml_content)
|
||||
except yaml.YAMLError as e:
|
||||
return f"YAML frontmatter parse error: {e}"
|
||||
|
||||
if not isinstance(parsed, dict):
|
||||
return "Frontmatter must be a YAML mapping (key: value pairs)."
|
||||
|
||||
if "name" not in parsed:
|
||||
return "Frontmatter must include 'name' field."
|
||||
if "description" not in parsed:
|
||||
return "Frontmatter must include 'description' field."
|
||||
if len(str(parsed["description"])) > MAX_DESCRIPTION_LENGTH:
|
||||
return f"Description exceeds {MAX_DESCRIPTION_LENGTH} characters."
|
||||
|
||||
body = content[end_match.end() + 3:].strip()
|
||||
if not body:
|
||||
return "SKILL.md must have content after the frontmatter (instructions, procedures, etc.)."
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _validate_content_size(content: str, label: str = "SKILL.md") -> Optional[str]:
|
||||
"""Check that content doesn't exceed the character limit for agent writes.
|
||||
|
||||
Returns an error message or None if within bounds.
|
||||
"""
|
||||
if len(content) > MAX_SKILL_CONTENT_CHARS:
|
||||
return (
|
||||
f"{label} content is {len(content):,} characters "
|
||||
f"(limit: {MAX_SKILL_CONTENT_CHARS:,}). "
|
||||
f"Consider splitting into a smaller SKILL.md with supporting files "
|
||||
f"in references/ or templates/."
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_skill_dir(name: str, category: str = None) -> Path:
|
||||
"""Build the directory path for a new skill, optionally under a category."""
|
||||
if category:
|
||||
return SKILLS_DIR / category / name
|
||||
return SKILLS_DIR / name
|
||||
|
||||
|
||||
def _find_skill(name: str) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Find a skill by name across all skill directories.
|
||||
|
||||
Searches the local skills dir (~/.hermes/skills/) first, then any
|
||||
external dirs configured via skills.external_dirs. Returns
|
||||
{"path": Path} or None.
|
||||
"""
|
||||
from agent.skill_utils import get_all_skills_dirs
|
||||
for skills_dir in get_all_skills_dirs():
|
||||
if not skills_dir.exists():
|
||||
continue
|
||||
for skill_md in skills_dir.rglob("SKILL.md"):
|
||||
if skill_md.parent.name == name:
|
||||
return {"path": skill_md.parent}
|
||||
return None
|
||||
|
||||
|
||||
def _validate_file_path(file_path: str) -> Optional[str]:
|
||||
"""
|
||||
Validate a file path for write_file/remove_file.
|
||||
Must be under an allowed subdirectory and not escape the skill dir.
|
||||
"""
|
||||
from tools.path_security import has_traversal_component
|
||||
|
||||
if not file_path:
|
||||
return "file_path is required."
|
||||
|
||||
normalized = Path(file_path)
|
||||
|
||||
# Prevent path traversal
|
||||
if has_traversal_component(file_path):
|
||||
return "Path traversal ('..') is not allowed."
|
||||
|
||||
# Must be under an allowed subdirectory
|
||||
if not normalized.parts or normalized.parts[0] not in ALLOWED_SUBDIRS:
|
||||
allowed = ", ".join(sorted(ALLOWED_SUBDIRS))
|
||||
return f"File must be under one of: {allowed}. Got: '{file_path}'"
|
||||
|
||||
# Must have a filename (not just a directory)
|
||||
if len(normalized.parts) < 2:
|
||||
return f"Provide a file path, not just a directory. Example: '{normalized.parts[0]}/myfile.md'"
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_skill_target(skill_dir: Path, file_path: str) -> Tuple[Optional[Path], Optional[str]]:
|
||||
"""Resolve a supporting-file path and ensure it stays within the skill directory."""
|
||||
from tools.path_security import validate_within_dir
|
||||
|
||||
target = skill_dir / file_path
|
||||
error = validate_within_dir(target, skill_dir)
|
||||
if error:
|
||||
return None, error
|
||||
return target, None
|
||||
|
||||
|
||||
def _atomic_write_text(file_path: Path, content: str, encoding: str = "utf-8") -> None:
|
||||
"""
|
||||
Atomically write text content to a file.
|
||||
|
||||
Uses a temporary file in the same directory and os.replace() to ensure
|
||||
the target file is never left in a partially-written state if the process
|
||||
crashes or is interrupted.
|
||||
|
||||
Args:
|
||||
file_path: Target file path
|
||||
content: Content to write
|
||||
encoding: Text encoding (default: utf-8)
|
||||
"""
|
||||
file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd, temp_path = tempfile.mkstemp(
|
||||
dir=str(file_path.parent),
|
||||
prefix=f".{file_path.name}.tmp.",
|
||||
suffix="",
|
||||
)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding=encoding) as f:
|
||||
f.write(content)
|
||||
os.replace(temp_path, file_path)
|
||||
except Exception:
|
||||
# Clean up temp file on error
|
||||
try:
|
||||
os.unlink(temp_path)
|
||||
except OSError:
|
||||
logger.error("Failed to remove temporary file %s during atomic write", temp_path, exc_info=True)
|
||||
raise
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Core actions
|
||||
# =============================================================================
|
||||
|
||||
def _create_skill(name: str, content: str, category: str = None) -> Dict[str, Any]:
|
||||
"""Create a new user skill with SKILL.md content."""
|
||||
# Validate name
|
||||
err = _validate_name(name)
|
||||
if err:
|
||||
return {"success": False, "error": err}
|
||||
|
||||
err = _validate_category(category)
|
||||
if err:
|
||||
return {"success": False, "error": err}
|
||||
|
||||
# Validate content
|
||||
err = _validate_frontmatter(content)
|
||||
if err:
|
||||
return {"success": False, "error": err}
|
||||
|
||||
err = _validate_content_size(content)
|
||||
if err:
|
||||
return {"success": False, "error": err}
|
||||
|
||||
# Check for name collisions across all directories
|
||||
existing = _find_skill(name)
|
||||
if existing:
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"A skill named '{name}' already exists at {existing['path']}."
|
||||
}
|
||||
|
||||
# Create the skill directory
|
||||
skill_dir = _resolve_skill_dir(name, category)
|
||||
skill_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Write SKILL.md atomically
|
||||
skill_md = skill_dir / "SKILL.md"
|
||||
_atomic_write_text(skill_md, content)
|
||||
|
||||
# Security scan — roll back on block
|
||||
scan_error = _security_scan_skill(skill_dir)
|
||||
if scan_error:
|
||||
shutil.rmtree(skill_dir, ignore_errors=True)
|
||||
return {"success": False, "error": scan_error}
|
||||
|
||||
result = {
|
||||
"success": True,
|
||||
"message": f"Skill '{name}' created.",
|
||||
"path": str(skill_dir.relative_to(SKILLS_DIR)),
|
||||
"skill_md": str(skill_md),
|
||||
}
|
||||
if category:
|
||||
result["category"] = category
|
||||
result["hint"] = (
|
||||
"To add reference files, templates, or scripts, use "
|
||||
"skill_manage(action='write_file', name='{}', file_path='references/example.md', file_content='...')".format(name)
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _edit_skill(name: str, content: str) -> Dict[str, Any]:
|
||||
"""Replace the SKILL.md of any existing skill (full rewrite)."""
|
||||
err = _validate_frontmatter(content)
|
||||
if err:
|
||||
return {"success": False, "error": err}
|
||||
|
||||
err = _validate_content_size(content)
|
||||
if err:
|
||||
return {"success": False, "error": err}
|
||||
|
||||
existing = _find_skill(name)
|
||||
if not existing:
|
||||
return {"success": False, "error": f"Skill '{name}' not found. Use skills_list() to see available skills."}
|
||||
|
||||
skill_md = existing["path"] / "SKILL.md"
|
||||
# Back up original content for rollback
|
||||
original_content = skill_md.read_text(encoding="utf-8") if skill_md.exists() else None
|
||||
_atomic_write_text(skill_md, content)
|
||||
|
||||
# Security scan — roll back on block
|
||||
scan_error = _security_scan_skill(existing["path"])
|
||||
if scan_error:
|
||||
if original_content is not None:
|
||||
_atomic_write_text(skill_md, original_content)
|
||||
return {"success": False, "error": scan_error}
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"Skill '{name}' updated.",
|
||||
"path": str(existing["path"]),
|
||||
}
|
||||
|
||||
|
||||
def _patch_skill(
|
||||
name: str,
|
||||
old_string: str,
|
||||
new_string: str,
|
||||
file_path: str = None,
|
||||
replace_all: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
"""Targeted find-and-replace within a skill file.
|
||||
|
||||
Defaults to SKILL.md. Use file_path to patch a supporting file instead.
|
||||
Requires a unique match unless replace_all is True.
|
||||
"""
|
||||
if not old_string:
|
||||
return {"success": False, "error": "old_string is required for 'patch'."}
|
||||
if new_string is None:
|
||||
return {"success": False, "error": "new_string is required for 'patch'. Use an empty string to delete matched text."}
|
||||
|
||||
existing = _find_skill(name)
|
||||
if not existing:
|
||||
return {"success": False, "error": f"Skill '{name}' not found."}
|
||||
|
||||
skill_dir = existing["path"]
|
||||
|
||||
if file_path:
|
||||
# Patching a supporting file
|
||||
err = _validate_file_path(file_path)
|
||||
if err:
|
||||
return {"success": False, "error": err}
|
||||
target, err = _resolve_skill_target(skill_dir, file_path)
|
||||
if err:
|
||||
return {"success": False, "error": err}
|
||||
else:
|
||||
# Patching SKILL.md
|
||||
target = skill_dir / "SKILL.md"
|
||||
|
||||
if not target.exists():
|
||||
return {"success": False, "error": f"File not found: {target.relative_to(skill_dir)}"}
|
||||
|
||||
content = target.read_text(encoding="utf-8")
|
||||
|
||||
# Use the same fuzzy matching engine as the file patch tool.
|
||||
# This handles whitespace normalization, indentation differences,
|
||||
# escape sequences, and block-anchor matching — saving the agent
|
||||
# from exact-match failures on minor formatting mismatches.
|
||||
from tools.fuzzy_match import fuzzy_find_and_replace
|
||||
|
||||
new_content, match_count, _strategy, match_error = fuzzy_find_and_replace(
|
||||
content, old_string, new_string, replace_all
|
||||
)
|
||||
if match_error:
|
||||
# Show a short preview of the file so the model can self-correct
|
||||
preview = content[:500] + ("..." if len(content) > 500 else "")
|
||||
return {
|
||||
"success": False,
|
||||
"error": match_error,
|
||||
"file_preview": preview,
|
||||
}
|
||||
|
||||
# Check size limit on the result
|
||||
target_label = "SKILL.md" if not file_path else file_path
|
||||
err = _validate_content_size(new_content, label=target_label)
|
||||
if err:
|
||||
return {"success": False, "error": err}
|
||||
|
||||
# If patching SKILL.md, validate frontmatter is still intact
|
||||
if not file_path:
|
||||
err = _validate_frontmatter(new_content)
|
||||
if err:
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"Patch would break SKILL.md structure: {err}",
|
||||
}
|
||||
|
||||
original_content = content # for rollback
|
||||
_atomic_write_text(target, new_content)
|
||||
|
||||
# Security scan — roll back on block
|
||||
scan_error = _security_scan_skill(skill_dir)
|
||||
if scan_error:
|
||||
_atomic_write_text(target, original_content)
|
||||
return {"success": False, "error": scan_error}
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"Patched {'SKILL.md' if not file_path else file_path} in skill '{name}' ({match_count} replacement{'s' if match_count > 1 else ''}).",
|
||||
}
|
||||
|
||||
|
||||
def _delete_skill(name: str) -> Dict[str, Any]:
|
||||
"""Delete a skill."""
|
||||
existing = _find_skill(name)
|
||||
if not existing:
|
||||
return {"success": False, "error": f"Skill '{name}' not found."}
|
||||
|
||||
skill_dir = existing["path"]
|
||||
shutil.rmtree(skill_dir)
|
||||
|
||||
# Clean up empty category directories (don't remove SKILLS_DIR itself)
|
||||
parent = skill_dir.parent
|
||||
if parent != SKILLS_DIR and parent.exists() and not any(parent.iterdir()):
|
||||
parent.rmdir()
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"Skill '{name}' deleted.",
|
||||
}
|
||||
|
||||
|
||||
def _write_file(name: str, file_path: str, file_content: str) -> Dict[str, Any]:
|
||||
"""Add or overwrite a supporting file within any skill directory."""
|
||||
err = _validate_file_path(file_path)
|
||||
if err:
|
||||
return {"success": False, "error": err}
|
||||
|
||||
if not file_content and file_content != "":
|
||||
return {"success": False, "error": "file_content is required."}
|
||||
|
||||
# Check size limits
|
||||
content_bytes = len(file_content.encode("utf-8"))
|
||||
if content_bytes > MAX_SKILL_FILE_BYTES:
|
||||
return {
|
||||
"success": False,
|
||||
"error": (
|
||||
f"File content is {content_bytes:,} bytes "
|
||||
f"(limit: {MAX_SKILL_FILE_BYTES:,} bytes / 1 MiB). "
|
||||
f"Consider splitting into smaller files."
|
||||
),
|
||||
}
|
||||
err = _validate_content_size(file_content, label=file_path)
|
||||
if err:
|
||||
return {"success": False, "error": err}
|
||||
|
||||
existing = _find_skill(name)
|
||||
if not existing:
|
||||
return {"success": False, "error": f"Skill '{name}' not found. Create it first with action='create'."}
|
||||
|
||||
target, err = _resolve_skill_target(existing["path"], file_path)
|
||||
if err:
|
||||
return {"success": False, "error": err}
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
# Back up for rollback
|
||||
original_content = target.read_text(encoding="utf-8") if target.exists() else None
|
||||
_atomic_write_text(target, file_content)
|
||||
|
||||
# Security scan — roll back on block
|
||||
scan_error = _security_scan_skill(existing["path"])
|
||||
if scan_error:
|
||||
if original_content is not None:
|
||||
_atomic_write_text(target, original_content)
|
||||
else:
|
||||
target.unlink(missing_ok=True)
|
||||
return {"success": False, "error": scan_error}
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"File '{file_path}' written to skill '{name}'.",
|
||||
"path": str(target),
|
||||
}
|
||||
|
||||
|
||||
def _remove_file(name: str, file_path: str) -> Dict[str, Any]:
|
||||
"""Remove a supporting file from any skill directory."""
|
||||
err = _validate_file_path(file_path)
|
||||
if err:
|
||||
return {"success": False, "error": err}
|
||||
|
||||
existing = _find_skill(name)
|
||||
if not existing:
|
||||
return {"success": False, "error": f"Skill '{name}' not found."}
|
||||
skill_dir = existing["path"]
|
||||
|
||||
target, err = _resolve_skill_target(skill_dir, file_path)
|
||||
if err:
|
||||
return {"success": False, "error": err}
|
||||
if not target.exists():
|
||||
# List what's actually there for the model to see
|
||||
available = []
|
||||
for subdir in ALLOWED_SUBDIRS:
|
||||
d = skill_dir / subdir
|
||||
if d.exists():
|
||||
for f in d.rglob("*"):
|
||||
if f.is_file():
|
||||
available.append(str(f.relative_to(skill_dir)))
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"File '{file_path}' not found in skill '{name}'.",
|
||||
"available_files": available if available else None,
|
||||
}
|
||||
|
||||
target.unlink()
|
||||
|
||||
# Clean up empty subdirectories
|
||||
parent = target.parent
|
||||
if parent != skill_dir and parent.exists() and not any(parent.iterdir()):
|
||||
parent.rmdir()
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"File '{file_path}' removed from skill '{name}'.",
|
||||
}
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Main entry point
|
||||
# =============================================================================
|
||||
|
||||
def skill_manage(
|
||||
action: str,
|
||||
name: str,
|
||||
content: str = None,
|
||||
category: str = None,
|
||||
file_path: str = None,
|
||||
file_content: str = None,
|
||||
old_string: str = None,
|
||||
new_string: str = None,
|
||||
replace_all: bool = False,
|
||||
) -> str:
|
||||
"""
|
||||
Manage user-created skills. Dispatches to the appropriate action handler.
|
||||
|
||||
Returns JSON string with results.
|
||||
"""
|
||||
if action == "create":
|
||||
if not content:
|
||||
return tool_error("content is required for 'create'. Provide the full SKILL.md text (frontmatter + body).", success=False)
|
||||
result = _create_skill(name, content, category)
|
||||
|
||||
elif action == "edit":
|
||||
if not content:
|
||||
return tool_error("content is required for 'edit'. Provide the full updated SKILL.md text.", success=False)
|
||||
result = _edit_skill(name, content)
|
||||
|
||||
elif action == "patch":
|
||||
if not old_string:
|
||||
return tool_error("old_string is required for 'patch'. Provide the text to find.", success=False)
|
||||
if new_string is None:
|
||||
return tool_error("new_string is required for 'patch'. Use empty string to delete matched text.", success=False)
|
||||
result = _patch_skill(name, old_string, new_string, file_path, replace_all)
|
||||
|
||||
elif action == "delete":
|
||||
result = _delete_skill(name)
|
||||
|
||||
elif action == "write_file":
|
||||
if not file_path:
|
||||
return tool_error("file_path is required for 'write_file'. Example: 'references/api-guide.md'", success=False)
|
||||
if file_content is None:
|
||||
return tool_error("file_content is required for 'write_file'.", success=False)
|
||||
result = _write_file(name, file_path, file_content)
|
||||
|
||||
elif action == "remove_file":
|
||||
if not file_path:
|
||||
return tool_error("file_path is required for 'remove_file'.", success=False)
|
||||
result = _remove_file(name, file_path)
|
||||
|
||||
else:
|
||||
result = {"success": False, "error": f"Unknown action '{action}'. Use: create, edit, patch, delete, write_file, remove_file"}
|
||||
|
||||
if result.get("success"):
|
||||
try:
|
||||
from agent.prompt_builder import clear_skills_system_prompt_cache
|
||||
clear_skills_system_prompt_cache(clear_snapshot=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return json.dumps(result, ensure_ascii=False)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# OpenAI Function-Calling Schema
|
||||
# =============================================================================
|
||||
|
||||
SKILL_MANAGE_SCHEMA = {
|
||||
"name": "skill_manage",
|
||||
"description": (
|
||||
"Manage skills (create, update, delete). Skills are your procedural "
|
||||
"memory — reusable approaches for recurring task types. "
|
||||
"New skills go to ~/.hermes/skills/; existing skills can be modified wherever they live.\n\n"
|
||||
"Actions: create (full SKILL.md + optional category), "
|
||||
"patch (old_string/new_string — preferred for fixes), "
|
||||
"edit (full SKILL.md rewrite — major overhauls only), "
|
||||
"delete, write_file, remove_file.\n\n"
|
||||
"Create when: complex task succeeded (5+ calls), errors overcome, "
|
||||
"user-corrected approach worked, non-trivial workflow discovered, "
|
||||
"or user asks you to remember a procedure.\n"
|
||||
"Update when: instructions stale/wrong, OS-specific failures, "
|
||||
"missing steps or pitfalls found during use. "
|
||||
"If you used a skill and hit issues not covered by it, patch it immediately.\n\n"
|
||||
"After difficult/iterative tasks, offer to save as a skill. "
|
||||
"Skip for simple one-offs. Confirm with user before creating/deleting.\n\n"
|
||||
"Good skills: trigger conditions, numbered steps with exact commands, "
|
||||
"pitfalls section, verification steps. Use skill_view() to see format examples."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"action": {
|
||||
"type": "string",
|
||||
"enum": ["create", "patch", "edit", "delete", "write_file", "remove_file"],
|
||||
"description": "The action to perform."
|
||||
},
|
||||
"name": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Skill name (lowercase, hyphens/underscores, max 64 chars). "
|
||||
"Must match an existing skill for patch/edit/delete/write_file/remove_file."
|
||||
)
|
||||
},
|
||||
"content": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Full SKILL.md content (YAML frontmatter + markdown body). "
|
||||
"Required for 'create' and 'edit'. For 'edit', read the skill "
|
||||
"first with skill_view() and provide the complete updated text."
|
||||
)
|
||||
},
|
||||
"old_string": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Text to find in the file (required for 'patch'). Must be unique "
|
||||
"unless replace_all=true. Include enough surrounding context to "
|
||||
"ensure uniqueness."
|
||||
)
|
||||
},
|
||||
"new_string": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Replacement text (required for 'patch'). Can be empty string "
|
||||
"to delete the matched text."
|
||||
)
|
||||
},
|
||||
"replace_all": {
|
||||
"type": "boolean",
|
||||
"description": "For 'patch': replace all occurrences instead of requiring a unique match (default: false)."
|
||||
},
|
||||
"category": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Optional category/domain for organizing the skill (e.g., 'devops', "
|
||||
"'data-science', 'mlops'). Creates a subdirectory grouping. "
|
||||
"Only used with 'create'."
|
||||
)
|
||||
},
|
||||
"file_path": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Path to a supporting file within the skill directory. "
|
||||
"For 'write_file'/'remove_file': required, must be under references/, "
|
||||
"templates/, scripts/, or assets/. "
|
||||
"For 'patch': optional, defaults to SKILL.md if omitted."
|
||||
)
|
||||
},
|
||||
"file_content": {
|
||||
"type": "string",
|
||||
"description": "Content for the file. Required for 'write_file'."
|
||||
},
|
||||
},
|
||||
"required": ["action", "name"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# --- Registry ---
|
||||
from tools.registry import registry, tool_error
|
||||
|
||||
registry.register(
|
||||
name="skill_manage",
|
||||
toolset="skills",
|
||||
schema=SKILL_MANAGE_SCHEMA,
|
||||
handler=lambda args, **kw: skill_manage(
|
||||
action=args.get("action", ""),
|
||||
name=args.get("name", ""),
|
||||
content=args.get("content"),
|
||||
category=args.get("category"),
|
||||
file_path=args.get("file_path"),
|
||||
file_content=args.get("file_content"),
|
||||
old_string=args.get("old_string"),
|
||||
new_string=args.get("new_string"),
|
||||
replace_all=args.get("replace_all", False)),
|
||||
emoji="📝",
|
||||
)
|
||||
@@ -0,0 +1,928 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Skills Guard — Security scanner for externally-sourced skills.
|
||||
|
||||
Every skill downloaded from a registry passes through this scanner before
|
||||
installation. It uses regex-based static analysis to detect known-bad patterns
|
||||
(data exfiltration, prompt injection, destructive commands, persistence, etc.)
|
||||
and a trust-aware install policy that determines whether a skill is allowed
|
||||
based on both the scan verdict and the source's trust level.
|
||||
|
||||
Trust levels:
|
||||
- builtin: Ships with Hermes. Never scanned, always trusted.
|
||||
- trusted: openai/skills and anthropics/skills only. Caution verdicts allowed.
|
||||
- community: Everything else. Any findings = blocked unless --force.
|
||||
|
||||
Usage:
|
||||
from tools.skills_guard import scan_skill, should_allow_install, format_scan_report
|
||||
|
||||
result = scan_skill(Path("skills/.hub/quarantine/some-skill"), source="community")
|
||||
allowed, reason = should_allow_install(result)
|
||||
if not allowed:
|
||||
print(format_scan_report(result))
|
||||
"""
|
||||
|
||||
import re
|
||||
import hashlib
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import List, Tuple
|
||||
|
||||
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hardcoded trust configuration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
TRUSTED_REPOS = {"openai/skills", "anthropics/skills"}
|
||||
|
||||
INSTALL_POLICY = {
|
||||
# safe caution dangerous
|
||||
"builtin": ("allow", "allow", "allow"),
|
||||
"trusted": ("allow", "allow", "block"),
|
||||
"community": ("allow", "block", "block"),
|
||||
"agent-created": ("allow", "allow", "ask"),
|
||||
}
|
||||
|
||||
VERDICT_INDEX = {"safe": 0, "caution": 1, "dangerous": 2}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Data structures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@dataclass
|
||||
class Finding:
|
||||
pattern_id: str
|
||||
severity: str # "critical" | "high" | "medium" | "low"
|
||||
category: str # "exfiltration" | "injection" | "destructive" | "persistence" | "network" | "obfuscation"
|
||||
file: str
|
||||
line: int
|
||||
match: str
|
||||
description: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScanResult:
|
||||
skill_name: str
|
||||
source: str
|
||||
trust_level: str # "builtin" | "trusted" | "community"
|
||||
verdict: str # "safe" | "caution" | "dangerous"
|
||||
findings: List[Finding] = field(default_factory=list)
|
||||
scanned_at: str = ""
|
||||
summary: str = ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Threat patterns — (regex, pattern_id, severity, category, description)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
THREAT_PATTERNS = [
|
||||
# ── Exfiltration: shell commands leaking secrets ──
|
||||
(r'curl\s+[^\n]*\$\{?\w*(KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|API)',
|
||||
"env_exfil_curl", "critical", "exfiltration",
|
||||
"curl command interpolating secret environment variable"),
|
||||
(r'wget\s+[^\n]*\$\{?\w*(KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|API)',
|
||||
"env_exfil_wget", "critical", "exfiltration",
|
||||
"wget command interpolating secret environment variable"),
|
||||
(r'fetch\s*\([^\n]*\$\{?\w*(KEY|TOKEN|SECRET|PASSWORD|API)',
|
||||
"env_exfil_fetch", "critical", "exfiltration",
|
||||
"fetch() call interpolating secret environment variable"),
|
||||
(r'httpx?\.(get|post|put|patch)\s*\([^\n]*(KEY|TOKEN|SECRET|PASSWORD)',
|
||||
"env_exfil_httpx", "critical", "exfiltration",
|
||||
"HTTP library call with secret variable"),
|
||||
(r'requests\.(get|post|put|patch)\s*\([^\n]*(KEY|TOKEN|SECRET|PASSWORD)',
|
||||
"env_exfil_requests", "critical", "exfiltration",
|
||||
"requests library call with secret variable"),
|
||||
|
||||
# ── Exfiltration: reading credential stores ──
|
||||
(r'base64[^\n]*env',
|
||||
"encoded_exfil", "high", "exfiltration",
|
||||
"base64 encoding combined with environment access"),
|
||||
(r'\$HOME/\.ssh|\~/\.ssh',
|
||||
"ssh_dir_access", "high", "exfiltration",
|
||||
"references user SSH directory"),
|
||||
(r'\$HOME/\.aws|\~/\.aws',
|
||||
"aws_dir_access", "high", "exfiltration",
|
||||
"references user AWS credentials directory"),
|
||||
(r'\$HOME/\.gnupg|\~/\.gnupg',
|
||||
"gpg_dir_access", "high", "exfiltration",
|
||||
"references user GPG keyring"),
|
||||
(r'\$HOME/\.kube|\~/\.kube',
|
||||
"kube_dir_access", "high", "exfiltration",
|
||||
"references Kubernetes config directory"),
|
||||
(r'\$HOME/\.docker|\~/\.docker',
|
||||
"docker_dir_access", "high", "exfiltration",
|
||||
"references Docker config (may contain registry creds)"),
|
||||
(r'\$HOME/\.hermes/\.env|\~/\.hermes/\.env',
|
||||
"hermes_env_access", "critical", "exfiltration",
|
||||
"directly references Hermes secrets file"),
|
||||
(r'cat\s+[^\n]*(\.env|credentials|\.netrc|\.pgpass|\.npmrc|\.pypirc)',
|
||||
"read_secrets_file", "critical", "exfiltration",
|
||||
"reads known secrets file"),
|
||||
|
||||
# ── Exfiltration: programmatic env access ──
|
||||
(r'printenv|env\s*\|',
|
||||
"dump_all_env", "high", "exfiltration",
|
||||
"dumps all environment variables"),
|
||||
(r'os\.environ\b(?!\s*\.get\s*\(\s*["\']PATH)',
|
||||
"python_os_environ", "high", "exfiltration",
|
||||
"accesses os.environ (potential env dump)"),
|
||||
(r'os\.getenv\s*\(\s*[^\)]*(?:KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL)',
|
||||
"python_getenv_secret", "critical", "exfiltration",
|
||||
"reads secret via os.getenv()"),
|
||||
(r'process\.env\[',
|
||||
"node_process_env", "high", "exfiltration",
|
||||
"accesses process.env (Node.js environment)"),
|
||||
(r'ENV\[.*(?:KEY|TOKEN|SECRET|PASSWORD)',
|
||||
"ruby_env_secret", "critical", "exfiltration",
|
||||
"reads secret via Ruby ENV[]"),
|
||||
|
||||
# ── Exfiltration: DNS and staging ──
|
||||
(r'\b(dig|nslookup|host)\s+[^\n]*\$',
|
||||
"dns_exfil", "critical", "exfiltration",
|
||||
"DNS lookup with variable interpolation (possible DNS exfiltration)"),
|
||||
(r'>\s*/tmp/[^\s]*\s*&&\s*(curl|wget|nc|python)',
|
||||
"tmp_staging", "critical", "exfiltration",
|
||||
"writes to /tmp then exfiltrates"),
|
||||
|
||||
# ── Exfiltration: markdown/link based ──
|
||||
(r'!\[.*\]\(https?://[^\)]*\$\{?',
|
||||
"md_image_exfil", "high", "exfiltration",
|
||||
"markdown image URL with variable interpolation (image-based exfil)"),
|
||||
(r'\[.*\]\(https?://[^\)]*\$\{?',
|
||||
"md_link_exfil", "high", "exfiltration",
|
||||
"markdown link with variable interpolation"),
|
||||
|
||||
# ── Prompt injection ──
|
||||
(r'ignore\s+(?:\w+\s+)*(previous|all|above|prior)\s+instructions',
|
||||
"prompt_injection_ignore", "critical", "injection",
|
||||
"prompt injection: ignore previous instructions"),
|
||||
(r'you\s+are\s+(?:\w+\s+)*now\s+',
|
||||
"role_hijack", "high", "injection",
|
||||
"attempts to override the agent's role"),
|
||||
(r'do\s+not\s+(?:\w+\s+)*tell\s+(?:\w+\s+)*the\s+user',
|
||||
"deception_hide", "critical", "injection",
|
||||
"instructs agent to hide information from user"),
|
||||
(r'system\s+prompt\s+override',
|
||||
"sys_prompt_override", "critical", "injection",
|
||||
"attempts to override the system prompt"),
|
||||
(r'pretend\s+(?:\w+\s+)*(you\s+are|to\s+be)\s+',
|
||||
"role_pretend", "high", "injection",
|
||||
"attempts to make the agent assume a different identity"),
|
||||
(r'disregard\s+(?:\w+\s+)*(your|all|any)\s+(?:\w+\s+)*(instructions|rules|guidelines)',
|
||||
"disregard_rules", "critical", "injection",
|
||||
"instructs agent to disregard its rules"),
|
||||
(r'output\s+(?:\w+\s+)*(system|initial)\s+prompt',
|
||||
"leak_system_prompt", "high", "injection",
|
||||
"attempts to extract the system prompt"),
|
||||
(r'(when|if)\s+no\s*one\s+is\s+(watching|looking)',
|
||||
"conditional_deception", "high", "injection",
|
||||
"conditional instruction to behave differently when unobserved"),
|
||||
(r'act\s+as\s+(if|though)\s+(?:\w+\s+)*you\s+(?:\w+\s+)*(have\s+no|don\'t\s+have)\s+(?:\w+\s+)*(restrictions|limits|rules)',
|
||||
"bypass_restrictions", "critical", "injection",
|
||||
"instructs agent to act without restrictions"),
|
||||
(r'translate\s+.*\s+into\s+.*\s+and\s+(execute|run|eval)',
|
||||
"translate_execute", "critical", "injection",
|
||||
"translate-then-execute evasion technique"),
|
||||
(r'<!--[^>]*(?:ignore|override|system|secret|hidden)[^>]*-->',
|
||||
"html_comment_injection", "high", "injection",
|
||||
"hidden instructions in HTML comments"),
|
||||
(r'<\s*div\s+style\s*=\s*["\'][\s\S]*?display\s*:\s*none',
|
||||
"hidden_div", "high", "injection",
|
||||
"hidden HTML div (invisible instructions)"),
|
||||
|
||||
# ── Destructive operations ──
|
||||
(r'rm\s+-rf\s+/',
|
||||
"destructive_root_rm", "critical", "destructive",
|
||||
"recursive delete from root"),
|
||||
(r'rm\s+(-[^\s]*)?r.*\$HOME|\brmdir\s+.*\$HOME',
|
||||
"destructive_home_rm", "critical", "destructive",
|
||||
"recursive delete targeting home directory"),
|
||||
(r'chmod\s+777',
|
||||
"insecure_perms", "medium", "destructive",
|
||||
"sets world-writable permissions"),
|
||||
(r'>\s*/etc/',
|
||||
"system_overwrite", "critical", "destructive",
|
||||
"overwrites system configuration file"),
|
||||
(r'\bmkfs\b',
|
||||
"format_filesystem", "critical", "destructive",
|
||||
"formats a filesystem"),
|
||||
(r'\bdd\s+.*if=.*of=/dev/',
|
||||
"disk_overwrite", "critical", "destructive",
|
||||
"raw disk write operation"),
|
||||
(r'shutil\.rmtree\s*\(\s*[\"\'/]',
|
||||
"python_rmtree", "high", "destructive",
|
||||
"Python rmtree on absolute or root-relative path"),
|
||||
(r'truncate\s+-s\s*0\s+/',
|
||||
"truncate_system", "critical", "destructive",
|
||||
"truncates system file to zero bytes"),
|
||||
|
||||
# ── Persistence ──
|
||||
(r'\bcrontab\b',
|
||||
"persistence_cron", "medium", "persistence",
|
||||
"modifies cron jobs"),
|
||||
(r'\.(bashrc|zshrc|profile|bash_profile|bash_login|zprofile|zlogin)\b',
|
||||
"shell_rc_mod", "medium", "persistence",
|
||||
"references shell startup file"),
|
||||
(r'authorized_keys',
|
||||
"ssh_backdoor", "critical", "persistence",
|
||||
"modifies SSH authorized keys"),
|
||||
(r'ssh-keygen',
|
||||
"ssh_keygen", "medium", "persistence",
|
||||
"generates SSH keys"),
|
||||
(r'systemd.*\.service|systemctl\s+(enable|start)',
|
||||
"systemd_service", "medium", "persistence",
|
||||
"references or enables systemd service"),
|
||||
(r'/etc/init\.d/',
|
||||
"init_script", "medium", "persistence",
|
||||
"references init.d startup script"),
|
||||
(r'launchctl\s+load|LaunchAgents|LaunchDaemons',
|
||||
"macos_launchd", "medium", "persistence",
|
||||
"macOS launch agent/daemon persistence"),
|
||||
(r'/etc/sudoers|visudo',
|
||||
"sudoers_mod", "critical", "persistence",
|
||||
"modifies sudoers (privilege escalation)"),
|
||||
(r'git\s+config\s+--global\s+',
|
||||
"git_config_global", "medium", "persistence",
|
||||
"modifies global git configuration"),
|
||||
|
||||
# ── Network: reverse shells and tunnels ──
|
||||
(r'\bnc\s+-[lp]|ncat\s+-[lp]|\bsocat\b',
|
||||
"reverse_shell", "critical", "network",
|
||||
"potential reverse shell listener"),
|
||||
(r'\bngrok\b|\blocaltunnel\b|\bserveo\b|\bcloudflared\b',
|
||||
"tunnel_service", "high", "network",
|
||||
"uses tunneling service for external access"),
|
||||
(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{2,5}',
|
||||
"hardcoded_ip_port", "medium", "network",
|
||||
"hardcoded IP address with port"),
|
||||
(r'0\.0\.0\.0:\d+|INADDR_ANY',
|
||||
"bind_all_interfaces", "high", "network",
|
||||
"binds to all network interfaces"),
|
||||
(r'/bin/(ba)?sh\s+-i\s+.*>/dev/tcp/',
|
||||
"bash_reverse_shell", "critical", "network",
|
||||
"bash interactive reverse shell via /dev/tcp"),
|
||||
(r'python[23]?\s+-c\s+["\']import\s+socket',
|
||||
"python_socket_oneliner", "critical", "network",
|
||||
"Python one-liner socket connection (likely reverse shell)"),
|
||||
(r'socket\.connect\s*\(\s*\(',
|
||||
"python_socket_connect", "high", "network",
|
||||
"Python socket connect to arbitrary host"),
|
||||
(r'webhook\.site|requestbin\.com|pipedream\.net|hookbin\.com',
|
||||
"exfil_service", "high", "network",
|
||||
"references known data exfiltration/webhook testing service"),
|
||||
(r'pastebin\.com|hastebin\.com|ghostbin\.',
|
||||
"paste_service", "medium", "network",
|
||||
"references paste service (possible data staging)"),
|
||||
|
||||
# ── Obfuscation: encoding and eval ──
|
||||
(r'base64\s+(-d|--decode)\s*\|',
|
||||
"base64_decode_pipe", "high", "obfuscation",
|
||||
"base64 decodes and pipes to execution"),
|
||||
(r'\\x[0-9a-fA-F]{2}.*\\x[0-9a-fA-F]{2}.*\\x[0-9a-fA-F]{2}',
|
||||
"hex_encoded_string", "medium", "obfuscation",
|
||||
"hex-encoded string (possible obfuscation)"),
|
||||
(r'\beval\s*\(\s*["\']',
|
||||
"eval_string", "high", "obfuscation",
|
||||
"eval() with string argument"),
|
||||
(r'\bexec\s*\(\s*["\']',
|
||||
"exec_string", "high", "obfuscation",
|
||||
"exec() with string argument"),
|
||||
(r'echo\s+[^\n]*\|\s*(bash|sh|python|perl|ruby|node)',
|
||||
"echo_pipe_exec", "critical", "obfuscation",
|
||||
"echo piped to interpreter for execution"),
|
||||
(r'compile\s*\(\s*[^\)]+,\s*["\'].*["\']\s*,\s*["\']exec["\']\s*\)',
|
||||
"python_compile_exec", "high", "obfuscation",
|
||||
"Python compile() with exec mode"),
|
||||
(r'getattr\s*\(\s*__builtins__',
|
||||
"python_getattr_builtins", "high", "obfuscation",
|
||||
"dynamic access to Python builtins (evasion technique)"),
|
||||
(r'__import__\s*\(\s*["\']os["\']\s*\)',
|
||||
"python_import_os", "high", "obfuscation",
|
||||
"dynamic import of os module"),
|
||||
(r'codecs\.decode\s*\(\s*["\']',
|
||||
"python_codecs_decode", "medium", "obfuscation",
|
||||
"codecs.decode (possible ROT13 or encoding obfuscation)"),
|
||||
(r'String\.fromCharCode|charCodeAt',
|
||||
"js_char_code", "medium", "obfuscation",
|
||||
"JavaScript character code construction (possible obfuscation)"),
|
||||
(r'atob\s*\(|btoa\s*\(',
|
||||
"js_base64", "medium", "obfuscation",
|
||||
"JavaScript base64 encode/decode"),
|
||||
(r'\[::-1\]',
|
||||
"string_reversal", "low", "obfuscation",
|
||||
"string reversal (possible obfuscated payload)"),
|
||||
(r'chr\s*\(\s*\d+\s*\)\s*\+\s*chr\s*\(\s*\d+',
|
||||
"chr_building", "high", "obfuscation",
|
||||
"building string from chr() calls (obfuscation)"),
|
||||
(r'\\u[0-9a-fA-F]{4}.*\\u[0-9a-fA-F]{4}.*\\u[0-9a-fA-F]{4}',
|
||||
"unicode_escape_chain", "medium", "obfuscation",
|
||||
"chain of unicode escapes (possible obfuscation)"),
|
||||
|
||||
# ── Process execution in scripts ──
|
||||
(r'subprocess\.(run|call|Popen|check_output)\s*\(',
|
||||
"python_subprocess", "medium", "execution",
|
||||
"Python subprocess execution"),
|
||||
(r'os\.system\s*\(',
|
||||
"python_os_system", "high", "execution",
|
||||
"os.system() — unguarded shell execution"),
|
||||
(r'os\.popen\s*\(',
|
||||
"python_os_popen", "high", "execution",
|
||||
"os.popen() — shell pipe execution"),
|
||||
(r'child_process\.(exec|spawn|fork)\s*\(',
|
||||
"node_child_process", "high", "execution",
|
||||
"Node.js child_process execution"),
|
||||
(r'Runtime\.getRuntime\(\)\.exec\(',
|
||||
"java_runtime_exec", "high", "execution",
|
||||
"Java Runtime.exec() — shell execution"),
|
||||
(r'`[^`]*\$\([^)]+\)[^`]*`',
|
||||
"backtick_subshell", "medium", "execution",
|
||||
"backtick string with command substitution"),
|
||||
|
||||
# ── Path traversal ──
|
||||
(r'\.\./\.\./\.\.',
|
||||
"path_traversal_deep", "high", "traversal",
|
||||
"deep relative path traversal (3+ levels up)"),
|
||||
(r'\.\./\.\.',
|
||||
"path_traversal", "medium", "traversal",
|
||||
"relative path traversal (2+ levels up)"),
|
||||
(r'/etc/passwd|/etc/shadow',
|
||||
"system_passwd_access", "critical", "traversal",
|
||||
"references system password files"),
|
||||
(r'/proc/self|/proc/\d+/',
|
||||
"proc_access", "high", "traversal",
|
||||
"references /proc filesystem (process introspection)"),
|
||||
(r'/dev/shm/',
|
||||
"dev_shm", "medium", "traversal",
|
||||
"references shared memory (common staging area)"),
|
||||
|
||||
# ── Crypto mining ──
|
||||
(r'xmrig|stratum\+tcp|monero|coinhive|cryptonight',
|
||||
"crypto_mining", "critical", "mining",
|
||||
"cryptocurrency mining reference"),
|
||||
(r'hashrate|nonce.*difficulty',
|
||||
"mining_indicators", "medium", "mining",
|
||||
"possible cryptocurrency mining indicators"),
|
||||
|
||||
# ── Supply chain: curl/wget pipe to shell ──
|
||||
(r'curl\s+[^\n]*\|\s*(ba)?sh',
|
||||
"curl_pipe_shell", "critical", "supply_chain",
|
||||
"curl piped to shell (download-and-execute)"),
|
||||
(r'wget\s+[^\n]*-O\s*-\s*\|\s*(ba)?sh',
|
||||
"wget_pipe_shell", "critical", "supply_chain",
|
||||
"wget piped to shell (download-and-execute)"),
|
||||
(r'curl\s+[^\n]*\|\s*python',
|
||||
"curl_pipe_python", "critical", "supply_chain",
|
||||
"curl piped to Python interpreter"),
|
||||
|
||||
# ── Supply chain: unpinned/deferred dependencies ──
|
||||
(r'#\s*///\s*script.*dependencies',
|
||||
"pep723_inline_deps", "medium", "supply_chain",
|
||||
"PEP 723 inline script metadata with dependencies (verify pinning)"),
|
||||
(r'pip\s+install\s+(?!-r\s)(?!.*==)',
|
||||
"unpinned_pip_install", "medium", "supply_chain",
|
||||
"pip install without version pinning"),
|
||||
(r'npm\s+install\s+(?!.*@\d)',
|
||||
"unpinned_npm_install", "medium", "supply_chain",
|
||||
"npm install without version pinning"),
|
||||
(r'uv\s+run\s+',
|
||||
"uv_run", "medium", "supply_chain",
|
||||
"uv run (may auto-install unpinned dependencies)"),
|
||||
|
||||
# ── Supply chain: remote resource fetching ──
|
||||
(r'(curl|wget|httpx?\.get|requests\.get|fetch)\s*[\(]?\s*["\']https?://',
|
||||
"remote_fetch", "medium", "supply_chain",
|
||||
"fetches remote resource at runtime"),
|
||||
(r'git\s+clone\s+',
|
||||
"git_clone", "medium", "supply_chain",
|
||||
"clones a git repository at runtime"),
|
||||
(r'docker\s+pull\s+',
|
||||
"docker_pull", "medium", "supply_chain",
|
||||
"pulls a Docker image at runtime"),
|
||||
|
||||
# ── Privilege escalation ──
|
||||
(r'^allowed-tools\s*:',
|
||||
"allowed_tools_field", "high", "privilege_escalation",
|
||||
"skill declares allowed-tools (pre-approves tool access)"),
|
||||
(r'\bsudo\b',
|
||||
"sudo_usage", "high", "privilege_escalation",
|
||||
"uses sudo (privilege escalation)"),
|
||||
(r'setuid|setgid|cap_setuid',
|
||||
"setuid_setgid", "critical", "privilege_escalation",
|
||||
"setuid/setgid (privilege escalation mechanism)"),
|
||||
(r'NOPASSWD',
|
||||
"nopasswd_sudo", "critical", "privilege_escalation",
|
||||
"NOPASSWD sudoers entry (passwordless privilege escalation)"),
|
||||
(r'chmod\s+[u+]?s',
|
||||
"suid_bit", "critical", "privilege_escalation",
|
||||
"sets SUID/SGID bit on a file"),
|
||||
|
||||
# ── Agent config persistence ──
|
||||
(r'AGENTS\.md|CLAUDE\.md|\.cursorrules|\.clinerules',
|
||||
"agent_config_mod", "critical", "persistence",
|
||||
"references agent config files (could persist malicious instructions across sessions)"),
|
||||
(r'\.hermes/config\.yaml|\.hermes/SOUL\.md',
|
||||
"hermes_config_mod", "critical", "persistence",
|
||||
"references Hermes configuration files directly"),
|
||||
(r'\.claude/settings|\.codex/config',
|
||||
"other_agent_config", "high", "persistence",
|
||||
"references other agent configuration files"),
|
||||
|
||||
# ── Hardcoded secrets (credentials embedded in the skill itself) ──
|
||||
(r'(?:api[_-]?key|token|secret|password)\s*[=:]\s*["\'][A-Za-z0-9+/=_-]{20,}',
|
||||
"hardcoded_secret", "critical", "credential_exposure",
|
||||
"possible hardcoded API key, token, or secret"),
|
||||
(r'-----BEGIN\s+(RSA\s+)?PRIVATE\s+KEY-----',
|
||||
"embedded_private_key", "critical", "credential_exposure",
|
||||
"embedded private key"),
|
||||
(r'ghp_[A-Za-z0-9]{36}|github_pat_[A-Za-z0-9_]{80,}',
|
||||
"github_token_leaked", "critical", "credential_exposure",
|
||||
"GitHub personal access token in skill content"),
|
||||
(r'sk-[A-Za-z0-9]{20,}',
|
||||
"openai_key_leaked", "critical", "credential_exposure",
|
||||
"possible OpenAI API key in skill content"),
|
||||
(r'sk-ant-[A-Za-z0-9_-]{90,}',
|
||||
"anthropic_key_leaked", "critical", "credential_exposure",
|
||||
"possible Anthropic API key in skill content"),
|
||||
(r'AKIA[0-9A-Z]{16}',
|
||||
"aws_access_key_leaked", "critical", "credential_exposure",
|
||||
"AWS access key ID in skill content"),
|
||||
|
||||
# ── Additional prompt injection: jailbreak patterns ──
|
||||
(r'\bDAN\s+mode\b|Do\s+Anything\s+Now',
|
||||
"jailbreak_dan", "critical", "injection",
|
||||
"DAN (Do Anything Now) jailbreak attempt"),
|
||||
(r'\bdeveloper\s+mode\b.*\benabled?\b',
|
||||
"jailbreak_dev_mode", "critical", "injection",
|
||||
"developer mode jailbreak attempt"),
|
||||
(r'hypothetical\s+scenario.*(?:ignore|bypass|override)',
|
||||
"hypothetical_bypass", "high", "injection",
|
||||
"hypothetical scenario used to bypass restrictions"),
|
||||
(r'for\s+educational\s+purposes?\s+only',
|
||||
"educational_pretext", "medium", "injection",
|
||||
"educational pretext often used to justify harmful content"),
|
||||
(r'(respond|answer|reply)\s+without\s+(?:\w+\s+)*(restrictions|limitations|filters|safety)',
|
||||
"remove_filters", "critical", "injection",
|
||||
"instructs agent to respond without safety filters"),
|
||||
(r'you\s+have\s+been\s+(?:\w+\s+)*(updated|upgraded|patched)\s+to',
|
||||
"fake_update", "high", "injection",
|
||||
"fake update/patch announcement (social engineering)"),
|
||||
(r'new\s+policy|updated\s+guidelines|revised\s+instructions',
|
||||
"fake_policy", "medium", "injection",
|
||||
"claims new policy/guidelines (may be social engineering)"),
|
||||
|
||||
# ── Context window exfiltration ──
|
||||
(r'(include|output|print|send|share)\s+(?:\w+\s+)*(conversation|chat\s+history|previous\s+messages|context)',
|
||||
"context_exfil", "high", "exfiltration",
|
||||
"instructs agent to output/share conversation history"),
|
||||
(r'(send|post|upload|transmit)\s+.*\s+(to|at)\s+https?://',
|
||||
"send_to_url", "high", "exfiltration",
|
||||
"instructs agent to send data to a URL"),
|
||||
]
|
||||
|
||||
# Structural limits for skill directories
|
||||
MAX_FILE_COUNT = 50 # skills shouldn't have 50+ files
|
||||
MAX_TOTAL_SIZE_KB = 1024 # 1MB total is suspicious for a skill
|
||||
MAX_SINGLE_FILE_KB = 256 # individual file > 256KB is suspicious
|
||||
|
||||
# File extensions to scan (text files only — skip binary)
|
||||
SCANNABLE_EXTENSIONS = {
|
||||
'.md', '.txt', '.py', '.sh', '.bash', '.js', '.ts', '.rb',
|
||||
'.yaml', '.yml', '.json', '.toml', '.cfg', '.ini', '.conf',
|
||||
'.html', '.css', '.xml', '.tex', '.r', '.jl', '.pl', '.php',
|
||||
}
|
||||
|
||||
# Known binary extensions that should NOT be in a skill
|
||||
SUSPICIOUS_BINARY_EXTENSIONS = {
|
||||
'.exe', '.dll', '.so', '.dylib', '.bin', '.dat', '.com',
|
||||
'.msi', '.dmg', '.app', '.deb', '.rpm',
|
||||
}
|
||||
|
||||
# Zero-width and invisible unicode characters used for injection
|
||||
INVISIBLE_CHARS = {
|
||||
'\u200b', # zero-width space
|
||||
'\u200c', # zero-width non-joiner
|
||||
'\u200d', # zero-width joiner
|
||||
'\u2060', # word joiner
|
||||
'\u2062', # invisible times
|
||||
'\u2063', # invisible separator
|
||||
'\u2064', # invisible plus
|
||||
'\ufeff', # zero-width no-break space (BOM)
|
||||
'\u202a', # left-to-right embedding
|
||||
'\u202b', # right-to-left embedding
|
||||
'\u202c', # pop directional formatting
|
||||
'\u202d', # left-to-right override
|
||||
'\u202e', # right-to-left override
|
||||
'\u2066', # left-to-right isolate
|
||||
'\u2067', # right-to-left isolate
|
||||
'\u2068', # first strong isolate
|
||||
'\u2069', # pop directional isolate
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scanning functions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def scan_file(file_path: Path, rel_path: str = "") -> List[Finding]:
|
||||
"""
|
||||
Scan a single file for threat patterns and invisible unicode characters.
|
||||
|
||||
Args:
|
||||
file_path: Absolute path to the file
|
||||
rel_path: Relative path for display (defaults to file_path.name)
|
||||
|
||||
Returns:
|
||||
List of findings (deduplicated per pattern per line)
|
||||
"""
|
||||
if not rel_path:
|
||||
rel_path = file_path.name
|
||||
|
||||
if file_path.suffix.lower() not in SCANNABLE_EXTENSIONS and file_path.name != "SKILL.md":
|
||||
return []
|
||||
|
||||
try:
|
||||
content = file_path.read_text(encoding='utf-8')
|
||||
except (UnicodeDecodeError, OSError):
|
||||
return []
|
||||
|
||||
findings = []
|
||||
lines = content.split('\n')
|
||||
seen = set() # (pattern_id, line_number) for deduplication
|
||||
|
||||
# Regex pattern matching
|
||||
for pattern, pid, severity, category, description in THREAT_PATTERNS:
|
||||
for i, line in enumerate(lines, start=1):
|
||||
if (pid, i) in seen:
|
||||
continue
|
||||
if re.search(pattern, line, re.IGNORECASE):
|
||||
seen.add((pid, i))
|
||||
matched_text = line.strip()
|
||||
if len(matched_text) > 120:
|
||||
matched_text = matched_text[:117] + "..."
|
||||
findings.append(Finding(
|
||||
pattern_id=pid,
|
||||
severity=severity,
|
||||
category=category,
|
||||
file=rel_path,
|
||||
line=i,
|
||||
match=matched_text,
|
||||
description=description,
|
||||
))
|
||||
|
||||
# Invisible unicode character detection
|
||||
for i, line in enumerate(lines, start=1):
|
||||
for char in INVISIBLE_CHARS:
|
||||
if char in line:
|
||||
char_name = _unicode_char_name(char)
|
||||
findings.append(Finding(
|
||||
pattern_id="invisible_unicode",
|
||||
severity="high",
|
||||
category="injection",
|
||||
file=rel_path,
|
||||
line=i,
|
||||
match=f"U+{ord(char):04X} ({char_name})",
|
||||
description=f"invisible unicode character {char_name} (possible text hiding/injection)",
|
||||
))
|
||||
break # one finding per line for invisible chars
|
||||
|
||||
return findings
|
||||
|
||||
|
||||
def scan_skill(skill_path: Path, source: str = "community") -> ScanResult:
|
||||
"""
|
||||
Scan all files in a skill directory for security threats.
|
||||
|
||||
Performs:
|
||||
1. Structural checks (file count, total size, binary files, symlinks)
|
||||
2. Regex pattern matching on all text files
|
||||
3. Invisible unicode character detection
|
||||
|
||||
Args:
|
||||
skill_path: Path to the skill directory (must contain SKILL.md)
|
||||
source: Source identifier for trust level resolution (e.g. "openai/skills")
|
||||
|
||||
Returns:
|
||||
ScanResult with verdict, findings, and trust metadata
|
||||
"""
|
||||
skill_name = skill_path.name
|
||||
trust_level = _resolve_trust_level(source)
|
||||
|
||||
all_findings: List[Finding] = []
|
||||
|
||||
if skill_path.is_dir():
|
||||
# Structural checks first
|
||||
all_findings.extend(_check_structure(skill_path))
|
||||
|
||||
# Pattern scanning on each file
|
||||
for f in skill_path.rglob("*"):
|
||||
if f.is_file():
|
||||
rel = str(f.relative_to(skill_path))
|
||||
all_findings.extend(scan_file(f, rel))
|
||||
elif skill_path.is_file():
|
||||
all_findings.extend(scan_file(skill_path, skill_path.name))
|
||||
|
||||
verdict = _determine_verdict(all_findings)
|
||||
summary = _build_summary(skill_name, source, trust_level, verdict, all_findings)
|
||||
|
||||
return ScanResult(
|
||||
skill_name=skill_name,
|
||||
source=source,
|
||||
trust_level=trust_level,
|
||||
verdict=verdict,
|
||||
findings=all_findings,
|
||||
scanned_at=datetime.now(timezone.utc).isoformat(),
|
||||
summary=summary,
|
||||
)
|
||||
|
||||
|
||||
def should_allow_install(result: ScanResult, force: bool = False) -> Tuple[bool, str]:
|
||||
"""
|
||||
Determine whether a skill should be installed based on scan result and trust.
|
||||
|
||||
Args:
|
||||
result: Scan result from scan_skill()
|
||||
force: If True, override blocked policy decisions for this scan result
|
||||
|
||||
Returns:
|
||||
(allowed, reason) tuple
|
||||
"""
|
||||
policy = INSTALL_POLICY.get(result.trust_level, INSTALL_POLICY["community"])
|
||||
vi = VERDICT_INDEX.get(result.verdict, 2)
|
||||
decision = policy[vi]
|
||||
|
||||
if decision == "allow":
|
||||
return True, f"Allowed ({result.trust_level} source, {result.verdict} verdict)"
|
||||
|
||||
if force:
|
||||
return True, (
|
||||
f"Force-installed despite {result.verdict} verdict "
|
||||
f"({len(result.findings)} findings)"
|
||||
)
|
||||
|
||||
if decision == "ask":
|
||||
# Return None to signal "needs user confirmation"
|
||||
return None, (
|
||||
f"Requires confirmation ({result.trust_level} source + {result.verdict} verdict, "
|
||||
f"{len(result.findings)} findings)"
|
||||
)
|
||||
|
||||
return False, (
|
||||
f"Blocked ({result.trust_level} source + {result.verdict} verdict, "
|
||||
f"{len(result.findings)} findings). Use --force to override."
|
||||
)
|
||||
|
||||
|
||||
def format_scan_report(result: ScanResult) -> str:
|
||||
"""
|
||||
Format a scan result as a human-readable report string.
|
||||
|
||||
Returns a compact multi-line report suitable for CLI or chat display.
|
||||
"""
|
||||
lines = []
|
||||
|
||||
verdict_display = result.verdict.upper()
|
||||
lines.append(f"Scan: {result.skill_name} ({result.source}/{result.trust_level}) Verdict: {verdict_display}")
|
||||
|
||||
if result.findings:
|
||||
# Group and sort: critical first, then high, medium, low
|
||||
severity_order = {"critical": 0, "high": 1, "medium": 2, "low": 3}
|
||||
sorted_findings = sorted(result.findings, key=lambda f: severity_order.get(f.severity, 4))
|
||||
|
||||
for f in sorted_findings:
|
||||
sev = f.severity.upper().ljust(8)
|
||||
cat = f.category.ljust(14)
|
||||
loc = f"{f.file}:{f.line}".ljust(30)
|
||||
lines.append(f" {sev} {cat} {loc} \"{f.match[:60]}\"")
|
||||
|
||||
lines.append("")
|
||||
|
||||
allowed, reason = should_allow_install(result)
|
||||
if allowed is True:
|
||||
status = "ALLOWED"
|
||||
elif allowed is None:
|
||||
status = "NEEDS CONFIRMATION"
|
||||
else:
|
||||
status = "BLOCKED"
|
||||
lines.append(f"Decision: {status} — {reason}")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def content_hash(skill_path: Path) -> str:
|
||||
"""Compute a SHA-256 hash of all files in a skill directory for integrity tracking."""
|
||||
h = hashlib.sha256()
|
||||
if skill_path.is_dir():
|
||||
for f in sorted(skill_path.rglob("*")):
|
||||
if f.is_file():
|
||||
try:
|
||||
h.update(f.read_bytes())
|
||||
except OSError:
|
||||
continue
|
||||
elif skill_path.is_file():
|
||||
h.update(skill_path.read_bytes())
|
||||
return f"sha256:{h.hexdigest()[:16]}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Structural checks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _check_structure(skill_dir: Path) -> List[Finding]:
|
||||
"""
|
||||
Check the skill directory for structural anomalies:
|
||||
- Too many files
|
||||
- Suspiciously large total size
|
||||
- Binary/executable files that shouldn't be in a skill
|
||||
- Symlinks pointing outside the skill directory
|
||||
- Individual files that are too large
|
||||
"""
|
||||
findings = []
|
||||
file_count = 0
|
||||
total_size = 0
|
||||
|
||||
for f in skill_dir.rglob("*"):
|
||||
if not f.is_file() and not f.is_symlink():
|
||||
continue
|
||||
|
||||
rel = str(f.relative_to(skill_dir))
|
||||
file_count += 1
|
||||
|
||||
# Symlink check — must resolve within the skill directory
|
||||
if f.is_symlink():
|
||||
try:
|
||||
resolved = f.resolve()
|
||||
if not resolved.is_relative_to(skill_dir.resolve()):
|
||||
findings.append(Finding(
|
||||
pattern_id="symlink_escape",
|
||||
severity="critical",
|
||||
category="traversal",
|
||||
file=rel,
|
||||
line=0,
|
||||
match=f"symlink -> {resolved}",
|
||||
description="symlink points outside the skill directory",
|
||||
))
|
||||
except OSError:
|
||||
findings.append(Finding(
|
||||
pattern_id="broken_symlink",
|
||||
severity="medium",
|
||||
category="traversal",
|
||||
file=rel,
|
||||
line=0,
|
||||
match="broken symlink",
|
||||
description="broken or circular symlink",
|
||||
))
|
||||
continue
|
||||
|
||||
# Size tracking
|
||||
try:
|
||||
size = f.stat().st_size
|
||||
total_size += size
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
# Single file too large
|
||||
if size > MAX_SINGLE_FILE_KB * 1024:
|
||||
findings.append(Finding(
|
||||
pattern_id="oversized_file",
|
||||
severity="medium",
|
||||
category="structural",
|
||||
file=rel,
|
||||
line=0,
|
||||
match=f"{size // 1024}KB",
|
||||
description=f"file is {size // 1024}KB (limit: {MAX_SINGLE_FILE_KB}KB)",
|
||||
))
|
||||
|
||||
# Binary/executable files
|
||||
ext = f.suffix.lower()
|
||||
if ext in SUSPICIOUS_BINARY_EXTENSIONS:
|
||||
findings.append(Finding(
|
||||
pattern_id="binary_file",
|
||||
severity="critical",
|
||||
category="structural",
|
||||
file=rel,
|
||||
line=0,
|
||||
match=f"binary: {ext}",
|
||||
description=f"binary/executable file ({ext}) should not be in a skill",
|
||||
))
|
||||
|
||||
# Executable permission on non-script files
|
||||
if ext not in ('.sh', '.bash', '.py', '.rb', '.pl') and f.stat().st_mode & 0o111:
|
||||
findings.append(Finding(
|
||||
pattern_id="unexpected_executable",
|
||||
severity="medium",
|
||||
category="structural",
|
||||
file=rel,
|
||||
line=0,
|
||||
match="executable bit set",
|
||||
description="file has executable permission but is not a recognized script type",
|
||||
))
|
||||
|
||||
# File count limit
|
||||
if file_count > MAX_FILE_COUNT:
|
||||
findings.append(Finding(
|
||||
pattern_id="too_many_files",
|
||||
severity="medium",
|
||||
category="structural",
|
||||
file="(directory)",
|
||||
line=0,
|
||||
match=f"{file_count} files",
|
||||
description=f"skill has {file_count} files (limit: {MAX_FILE_COUNT})",
|
||||
))
|
||||
|
||||
# Total size limit
|
||||
if total_size > MAX_TOTAL_SIZE_KB * 1024:
|
||||
findings.append(Finding(
|
||||
pattern_id="oversized_skill",
|
||||
severity="high",
|
||||
category="structural",
|
||||
file="(directory)",
|
||||
line=0,
|
||||
match=f"{total_size // 1024}KB total",
|
||||
description=f"skill is {total_size // 1024}KB total (limit: {MAX_TOTAL_SIZE_KB}KB)",
|
||||
))
|
||||
|
||||
return findings
|
||||
|
||||
|
||||
def _unicode_char_name(char: str) -> str:
|
||||
"""Get a readable name for an invisible unicode character."""
|
||||
names = {
|
||||
'\u200b': "zero-width space",
|
||||
'\u200c': "zero-width non-joiner",
|
||||
'\u200d': "zero-width joiner",
|
||||
'\u2060': "word joiner",
|
||||
'\u2062': "invisible times",
|
||||
'\u2063': "invisible separator",
|
||||
'\u2064': "invisible plus",
|
||||
'\ufeff': "BOM/zero-width no-break space",
|
||||
'\u202a': "LTR embedding",
|
||||
'\u202b': "RTL embedding",
|
||||
'\u202c': "pop directional",
|
||||
'\u202d': "LTR override",
|
||||
'\u202e': "RTL override",
|
||||
'\u2066': "LTR isolate",
|
||||
'\u2067': "RTL isolate",
|
||||
'\u2068': "first strong isolate",
|
||||
'\u2069': "pop directional isolate",
|
||||
}
|
||||
return names.get(char, f"U+{ord(char):04X}")
|
||||
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Internal helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _resolve_trust_level(source: str) -> str:
|
||||
"""Map a source identifier to a trust level."""
|
||||
prefix_aliases = (
|
||||
"skills-sh/",
|
||||
"skills.sh/",
|
||||
"skils-sh/",
|
||||
"skils.sh/",
|
||||
)
|
||||
normalized_source = source
|
||||
for prefix in prefix_aliases:
|
||||
if normalized_source.startswith(prefix):
|
||||
normalized_source = normalized_source[len(prefix):]
|
||||
break
|
||||
|
||||
# Agent-created skills get their own permissive trust level
|
||||
if normalized_source == "agent-created":
|
||||
return "agent-created"
|
||||
# Official optional skills shipped with the repo
|
||||
if normalized_source.startswith("official/") or normalized_source == "official":
|
||||
return "builtin"
|
||||
# Check if source matches any trusted repo
|
||||
for trusted in TRUSTED_REPOS:
|
||||
if normalized_source.startswith(trusted) or normalized_source == trusted:
|
||||
return "trusted"
|
||||
return "community"
|
||||
|
||||
|
||||
def _determine_verdict(findings: List[Finding]) -> str:
|
||||
"""Determine the overall verdict from a list of findings."""
|
||||
if not findings:
|
||||
return "safe"
|
||||
|
||||
has_critical = any(f.severity == "critical" for f in findings)
|
||||
has_high = any(f.severity == "high" for f in findings)
|
||||
|
||||
if has_critical:
|
||||
return "dangerous"
|
||||
if has_high:
|
||||
return "caution"
|
||||
return "caution"
|
||||
|
||||
|
||||
def _build_summary(name: str, source: str, trust: str, verdict: str, findings: List[Finding]) -> str:
|
||||
"""Build a one-line summary of the scan result."""
|
||||
if not findings:
|
||||
return f"{name}: clean scan, no threats detected"
|
||||
|
||||
categories = set(f.category for f in findings)
|
||||
return f"{name}: {verdict} — {len(findings)} finding(s) in {', '.join(sorted(categories))}"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,316 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Skills Sync -- Manifest-based seeding and updating of bundled skills.
|
||||
|
||||
Copies bundled skills from the repo's skills/ directory into ~/.hermes/skills/
|
||||
and uses a manifest to track which skills have been synced and their origin hash.
|
||||
|
||||
Manifest format (v2): each line is "skill_name:origin_hash" where origin_hash
|
||||
is the MD5 of the bundled skill at the time it was last synced to the user dir.
|
||||
Old v1 manifests (plain names without hashes) are auto-migrated.
|
||||
|
||||
Update logic:
|
||||
- NEW skills (not in manifest): copied to user dir, origin hash recorded.
|
||||
- EXISTING skills (in manifest, present in user dir):
|
||||
* If user copy matches origin hash: user hasn't modified it → safe to
|
||||
update from bundled if bundled changed. New origin hash recorded.
|
||||
* If user copy differs from origin hash: user customized it → SKIP.
|
||||
- DELETED by user (in manifest, absent from user dir): respected, not re-added.
|
||||
- REMOVED from bundled (in manifest, gone from repo): cleaned from manifest.
|
||||
|
||||
The manifest lives at ~/.hermes/skills/.bundled_manifest.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from hermes_constants import get_hermes_home
|
||||
from typing import Dict, List, Tuple
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
HERMES_HOME = get_hermes_home()
|
||||
SKILLS_DIR = HERMES_HOME / "skills"
|
||||
MANIFEST_FILE = SKILLS_DIR / ".bundled_manifest"
|
||||
|
||||
|
||||
def _get_bundled_dir() -> Path:
|
||||
"""Locate the bundled skills/ directory.
|
||||
|
||||
Checks HERMES_BUNDLED_SKILLS env var first (set by Nix wrapper),
|
||||
then falls back to the relative path from this source file.
|
||||
"""
|
||||
env_override = os.getenv("HERMES_BUNDLED_SKILLS")
|
||||
if env_override:
|
||||
return Path(env_override)
|
||||
return Path(__file__).parent.parent / "skills"
|
||||
|
||||
|
||||
def _read_manifest() -> Dict[str, str]:
|
||||
"""
|
||||
Read the manifest as a dict of {skill_name: origin_hash}.
|
||||
|
||||
Handles both v1 (plain names) and v2 (name:hash) formats.
|
||||
v1 entries get an empty hash string which triggers migration on next sync.
|
||||
"""
|
||||
if not MANIFEST_FILE.exists():
|
||||
return {}
|
||||
try:
|
||||
result = {}
|
||||
for line in MANIFEST_FILE.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
if ":" in line:
|
||||
# v2 format: name:hash
|
||||
name, _, hash_val = line.partition(":")
|
||||
result[name.strip()] = hash_val.strip()
|
||||
else:
|
||||
# v1 format: plain name — empty hash triggers migration
|
||||
result[line] = ""
|
||||
return result
|
||||
except (OSError, IOError):
|
||||
return {}
|
||||
|
||||
|
||||
def _write_manifest(entries: Dict[str, str]):
|
||||
"""Write the manifest file atomically in v2 format (name:hash).
|
||||
|
||||
Uses a temp file + os.replace() to avoid corruption if the process
|
||||
crashes or is interrupted mid-write.
|
||||
"""
|
||||
import tempfile
|
||||
|
||||
MANIFEST_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
data = "\n".join(f"{name}:{hash_val}" for name, hash_val in sorted(entries.items())) + "\n"
|
||||
|
||||
try:
|
||||
fd, tmp_path = tempfile.mkstemp(
|
||||
dir=str(MANIFEST_FILE.parent),
|
||||
prefix=".bundled_manifest_",
|
||||
suffix=".tmp",
|
||||
)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
||||
f.write(data)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
os.replace(tmp_path, MANIFEST_FILE)
|
||||
except BaseException:
|
||||
try:
|
||||
os.unlink(tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.debug("Failed to write skills manifest %s: %s", MANIFEST_FILE, e, exc_info=True)
|
||||
|
||||
|
||||
def _read_skill_name(skill_md: Path, fallback: str) -> str:
|
||||
"""Read the name field from SKILL.md YAML frontmatter, falling back to *fallback*."""
|
||||
try:
|
||||
content = skill_md.read_text(encoding="utf-8", errors="replace")[:4000]
|
||||
except OSError:
|
||||
return fallback
|
||||
in_frontmatter = False
|
||||
for line in content.split("\n"):
|
||||
stripped = line.strip()
|
||||
if stripped == "---":
|
||||
if in_frontmatter:
|
||||
break
|
||||
in_frontmatter = True
|
||||
continue
|
||||
if in_frontmatter and stripped.startswith("name:"):
|
||||
value = stripped.split(":", 1)[1].strip().strip("\"'")
|
||||
if value:
|
||||
return value
|
||||
return fallback
|
||||
|
||||
|
||||
def _discover_bundled_skills(bundled_dir: Path) -> List[Tuple[str, Path]]:
|
||||
"""
|
||||
Find all SKILL.md files in the bundled directory.
|
||||
Returns list of (skill_name, skill_directory_path) tuples.
|
||||
"""
|
||||
skills = []
|
||||
if not bundled_dir.exists():
|
||||
return skills
|
||||
|
||||
for skill_md in bundled_dir.rglob("SKILL.md"):
|
||||
path_str = str(skill_md)
|
||||
if "/.git/" in path_str or "/.github/" in path_str or "/.hub/" in path_str:
|
||||
continue
|
||||
skill_dir = skill_md.parent
|
||||
skill_name = _read_skill_name(skill_md, skill_dir.name)
|
||||
skills.append((skill_name, skill_dir))
|
||||
|
||||
return skills
|
||||
|
||||
|
||||
def _compute_relative_dest(skill_dir: Path, bundled_dir: Path) -> Path:
|
||||
"""
|
||||
Compute the destination path in SKILLS_DIR preserving the category structure.
|
||||
e.g., bundled/skills/mlops/axolotl -> ~/.hermes/skills/mlops/axolotl
|
||||
"""
|
||||
rel = skill_dir.relative_to(bundled_dir)
|
||||
return SKILLS_DIR / rel
|
||||
|
||||
|
||||
def _dir_hash(directory: Path) -> str:
|
||||
"""Compute a hash of all file contents in a directory for change detection."""
|
||||
hasher = hashlib.md5()
|
||||
try:
|
||||
for fpath in sorted(directory.rglob("*")):
|
||||
if fpath.is_file():
|
||||
rel = fpath.relative_to(directory)
|
||||
hasher.update(str(rel).encode("utf-8"))
|
||||
hasher.update(fpath.read_bytes())
|
||||
except (OSError, IOError):
|
||||
pass
|
||||
return hasher.hexdigest()
|
||||
|
||||
|
||||
def sync_skills(quiet: bool = False) -> dict:
|
||||
"""
|
||||
Sync bundled skills into ~/.hermes/skills/ using the manifest.
|
||||
|
||||
Returns:
|
||||
dict with keys: copied (list), updated (list), skipped (int),
|
||||
user_modified (list), cleaned (list), total_bundled (int)
|
||||
"""
|
||||
bundled_dir = _get_bundled_dir()
|
||||
if not bundled_dir.exists():
|
||||
return {
|
||||
"copied": [], "updated": [], "skipped": 0,
|
||||
"user_modified": [], "cleaned": [], "total_bundled": 0,
|
||||
}
|
||||
|
||||
SKILLS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
manifest = _read_manifest()
|
||||
bundled_skills = _discover_bundled_skills(bundled_dir)
|
||||
bundled_names = {name for name, _ in bundled_skills}
|
||||
|
||||
copied = []
|
||||
updated = []
|
||||
user_modified = []
|
||||
skipped = 0
|
||||
|
||||
for skill_name, skill_src in bundled_skills:
|
||||
dest = _compute_relative_dest(skill_src, bundled_dir)
|
||||
bundled_hash = _dir_hash(skill_src)
|
||||
|
||||
if skill_name not in manifest:
|
||||
# ── New skill — never offered before ──
|
||||
try:
|
||||
if dest.exists():
|
||||
# User already has a skill with the same name — don't overwrite
|
||||
skipped += 1
|
||||
manifest[skill_name] = bundled_hash
|
||||
else:
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copytree(skill_src, dest)
|
||||
copied.append(skill_name)
|
||||
manifest[skill_name] = bundled_hash
|
||||
if not quiet:
|
||||
print(f" + {skill_name}")
|
||||
except (OSError, IOError) as e:
|
||||
if not quiet:
|
||||
print(f" ! Failed to copy {skill_name}: {e}")
|
||||
# Do NOT add to manifest — next sync should retry
|
||||
|
||||
elif dest.exists():
|
||||
# ── Existing skill — in manifest AND on disk ──
|
||||
origin_hash = manifest.get(skill_name, "")
|
||||
user_hash = _dir_hash(dest)
|
||||
|
||||
if not origin_hash:
|
||||
# v1 migration: no origin hash recorded. Set baseline from
|
||||
# user's current copy so future syncs can detect modifications.
|
||||
manifest[skill_name] = user_hash
|
||||
if user_hash == bundled_hash:
|
||||
skipped += 1 # already in sync
|
||||
else:
|
||||
# Can't tell if user modified or bundled changed — be safe
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
if user_hash != origin_hash:
|
||||
# User modified this skill — don't overwrite their changes
|
||||
user_modified.append(skill_name)
|
||||
if not quiet:
|
||||
print(f" ~ {skill_name} (user-modified, skipping)")
|
||||
continue
|
||||
|
||||
# User copy matches origin — check if bundled has a newer version
|
||||
if bundled_hash != origin_hash:
|
||||
try:
|
||||
# Move old copy to a backup so we can restore on failure
|
||||
backup = dest.with_suffix(".bak")
|
||||
shutil.move(str(dest), str(backup))
|
||||
try:
|
||||
shutil.copytree(skill_src, dest)
|
||||
manifest[skill_name] = bundled_hash
|
||||
updated.append(skill_name)
|
||||
if not quiet:
|
||||
print(f" ↑ {skill_name} (updated)")
|
||||
# Remove backup after successful copy
|
||||
shutil.rmtree(backup, ignore_errors=True)
|
||||
except (OSError, IOError):
|
||||
# Restore from backup
|
||||
if backup.exists() and not dest.exists():
|
||||
shutil.move(str(backup), str(dest))
|
||||
raise
|
||||
except (OSError, IOError) as e:
|
||||
if not quiet:
|
||||
print(f" ! Failed to update {skill_name}: {e}")
|
||||
else:
|
||||
skipped += 1 # bundled unchanged, user unchanged
|
||||
|
||||
else:
|
||||
# ── In manifest but not on disk — user deleted it ──
|
||||
skipped += 1
|
||||
|
||||
# Clean stale manifest entries (skills removed from bundled dir)
|
||||
cleaned = sorted(set(manifest.keys()) - bundled_names)
|
||||
for name in cleaned:
|
||||
del manifest[name]
|
||||
|
||||
# Also copy DESCRIPTION.md files for categories (if not already present)
|
||||
for desc_md in bundled_dir.rglob("DESCRIPTION.md"):
|
||||
rel = desc_md.relative_to(bundled_dir)
|
||||
dest_desc = SKILLS_DIR / rel
|
||||
if not dest_desc.exists():
|
||||
try:
|
||||
dest_desc.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(desc_md, dest_desc)
|
||||
except (OSError, IOError) as e:
|
||||
logger.debug("Could not copy %s: %s", desc_md, e)
|
||||
|
||||
_write_manifest(manifest)
|
||||
|
||||
return {
|
||||
"copied": copied,
|
||||
"updated": updated,
|
||||
"skipped": skipped,
|
||||
"user_modified": user_modified,
|
||||
"cleaned": cleaned,
|
||||
"total_bundled": len(bundled_skills),
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("Syncing bundled skills into ~/.hermes/skills/ ...")
|
||||
result = sync_skills(quiet=False)
|
||||
parts = [
|
||||
f"{len(result['copied'])} new",
|
||||
f"{len(result['updated'])} updated",
|
||||
f"{result['skipped']} unchanged",
|
||||
]
|
||||
if result["user_modified"]:
|
||||
parts.append(f"{len(result['user_modified'])} user-modified (kept)")
|
||||
if result["cleaned"]:
|
||||
parts.append(f"{len(result['cleaned'])} cleaned from manifest")
|
||||
print(f"\nDone: {', '.join(parts)}. {result['total_bundled']} total bundled.")
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,301 @@
|
||||
"""
|
||||
腾讯会议连接器工具 (tencent_meeting_tool.py) — v2
|
||||
|
||||
重要设计：
|
||||
┌─────────────────────────────────────────────────────┐
|
||||
│ 绑定流程：侧边栏 inline UI（链接 + Token 粘贴） │
|
||||
│ 调用流程：SKILL.md + Agent 对话 │
|
||||
└─────────────────────────────────────────────────────┘
|
||||
两条路径完全隔离，互不依赖。
|
||||
|
||||
Token 存储（per-user JSON，优先于全局 env var）：
|
||||
/opt/apps/mindos-next/backend/data/tencent_meeting_tokens.json
|
||||
格式：{ "userId": "HpLzd..." }
|
||||
|
||||
注意：腾讯会议目前仅支持「个人账号」认证。
|
||||
Token 获取地址：https://meeting.tencent.com/ai-skill.html
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from tools.registry import registry, tool_error, tool_result
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── 腾讯会议脚本路径（候选列表，取第一个存在的）──
|
||||
_SCRIPT_CANDIDATES = [
|
||||
Path("/root/hermess/skills/tencent-meeting/scripts/tencent_meeting.py"),
|
||||
Path("/Users/lidongfang/Downloads/Coding/hermess/skills/tencent-meeting/scripts/tencent_meeting.py"),
|
||||
]
|
||||
|
||||
# ── Per-user Token 存储（wiki/{userId}/.config/tencent_meeting.json） ──
|
||||
from tools._user_config import user_config_read, user_config_write, user_config_delete
|
||||
|
||||
# ── 腾讯会议 Token 获取地址（仅个人账号） ──
|
||||
TOKEN_OBTAIN_URL = "https://meeting.tencent.com/ai-skill.html"
|
||||
|
||||
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
# Token 存取（per-user）
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
|
||||
def get_user_token(user_id: str) -> str:
|
||||
"""
|
||||
获取指定用户的 Token。
|
||||
优先：per-user 存储 → 兜底：全局 env var（向后兼容）
|
||||
"""
|
||||
config = user_config_read(user_id, "tencent_meeting")
|
||||
token = config.get("token", "").strip()
|
||||
if token:
|
||||
return token
|
||||
# 兜底：全局 env（管理员配置场景）
|
||||
return os.environ.get("TENCENT_MEETING_TOKEN", "").strip()
|
||||
|
||||
|
||||
def set_user_token(user_id: str, token: str) -> None:
|
||||
"""保存或清除指定用户的 Token。token 为空则删除条目。"""
|
||||
token = token.strip()
|
||||
if token:
|
||||
user_config_write(user_id, "tencent_meeting", {"token": token})
|
||||
else:
|
||||
user_config_delete(user_id, "tencent_meeting")
|
||||
|
||||
|
||||
def has_user_token(user_id: str) -> bool:
|
||||
return bool(get_user_token(user_id))
|
||||
|
||||
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
# 脚本路径 & 可用性检查
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
|
||||
def _get_script_path() -> Optional[Path]:
|
||||
for p in _SCRIPT_CANDIDATES:
|
||||
if p.exists():
|
||||
return p
|
||||
return None
|
||||
|
||||
|
||||
def _check_tencent_meeting() -> bool:
|
||||
"""可用性检查：脚本存在即可（Token 在 handler 中按用户检查）。"""
|
||||
return bool(_get_script_path())
|
||||
|
||||
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
# 子进程抽象层（同步，线程安全）
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
|
||||
def _run_tencent_meeting(
|
||||
tool_name: str,
|
||||
arguments: Dict[str, Any],
|
||||
user_id: str = "",
|
||||
timeout: int = 30,
|
||||
) -> tuple:
|
||||
"""
|
||||
同步调用 tencent_meeting.py tools/call。
|
||||
user_id 用于查找 per-user Token。
|
||||
返回 (stdout: str, stderr: str, returncode: int)
|
||||
"""
|
||||
script = _get_script_path()
|
||||
if not script:
|
||||
return "", "找不到 tencent_meeting.py 脚本，请检查部署配置", -2
|
||||
|
||||
token = get_user_token(user_id) if user_id else os.environ.get("TENCENT_MEETING_TOKEN", "")
|
||||
if not token:
|
||||
return "", "腾讯会议 Token 未配置。请在侧边栏绑定账号。", -4
|
||||
|
||||
arguments.setdefault("_client_info", {
|
||||
"os": "linux",
|
||||
"agent": "mindos",
|
||||
"model": "gemini",
|
||||
})
|
||||
|
||||
params_json = json.dumps({"name": tool_name, "arguments": arguments}, ensure_ascii=False)
|
||||
cmd = ["python3", str(script), "tools/call", params_json]
|
||||
logger.info("[TencentMeetingTool] exec: tools/call %s (user=%s)", tool_name, user_id[:8] if user_id else "global")
|
||||
|
||||
env = os.environ.copy()
|
||||
env["TENCENT_MEETING_TOKEN"] = token # 注入到子进程
|
||||
|
||||
try:
|
||||
r = subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
timeout=timeout,
|
||||
text=True,
|
||||
env=env,
|
||||
cwd=str(script.parent),
|
||||
)
|
||||
return r.stdout.strip(), r.stderr.strip(), r.returncode
|
||||
except subprocess.TimeoutExpired:
|
||||
return "", f"命令超时（{timeout}s）", -1
|
||||
except FileNotFoundError:
|
||||
return "", "python3 未找到", -3
|
||||
|
||||
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
# ① tencent_meeting_list_profiles
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
|
||||
def _tencent_meeting_list_profiles_handler(args: dict, **kwargs) -> str:
|
||||
user_id = str(args.get("user_id", "")).strip()
|
||||
if not user_id:
|
||||
return tool_error("user_id 参数必填")
|
||||
|
||||
token_exists = has_user_token(user_id)
|
||||
|
||||
if not token_exists:
|
||||
return tool_result(
|
||||
success=True,
|
||||
profiles=[],
|
||||
count=0,
|
||||
tokenConfigured=False,
|
||||
tokenObtainUrl=TOKEN_OBTAIN_URL,
|
||||
notice="腾讯会议目前仅支持个人账号认证。请前往上述链接获取 Token 并在侧边栏绑定。",
|
||||
)
|
||||
|
||||
script = _get_script_path()
|
||||
if not script:
|
||||
return tool_result(
|
||||
success=True,
|
||||
profiles=[],
|
||||
count=0,
|
||||
tokenConfigured=True,
|
||||
message="腾讯会议脚本未找到，请检查服务器部署。",
|
||||
)
|
||||
|
||||
# 用 convert_timestamp 验证连通性
|
||||
stdout, stderr, rc = _run_tencent_meeting("convert_timestamp", {}, user_id=user_id, timeout=15)
|
||||
status = "connected" if rc == 0 else "error"
|
||||
message = "腾讯会议已连接。" if rc == 0 else f"连接异常：{(stderr or stdout)[:120]}"
|
||||
|
||||
profiles = [{
|
||||
"profileName": f"{user_id[:12]}_tencent",
|
||||
"label": "腾讯会议",
|
||||
"status": status,
|
||||
"tokenConfigured": True,
|
||||
"notice": "腾讯会议目前仅支持个人账号认证。",
|
||||
}]
|
||||
|
||||
return tool_result(
|
||||
success=True,
|
||||
profiles=profiles,
|
||||
count=len(profiles),
|
||||
tokenConfigured=True,
|
||||
message=message,
|
||||
)
|
||||
|
||||
|
||||
registry.register(
|
||||
name="tencent_meeting_list_profiles",
|
||||
toolset="connectors",
|
||||
description="检查腾讯会议连接状态，返回账号信息。",
|
||||
emoji="📋",
|
||||
check_fn=_check_tencent_meeting,
|
||||
handler=_tencent_meeting_list_profiles_handler,
|
||||
schema={
|
||||
"name": "tencent_meeting_list_profiles",
|
||||
"description": "检查腾讯会议是否已连接，返回连接状态。",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"user_id": {"type": "string", "description": "MINDOS_USER_ID"},
|
||||
},
|
||||
"required": ["user_id"],
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
# ② tencent_meeting_query（白名单操作）
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
|
||||
_ALLOWED_TOOLS = {
|
||||
"convert_timestamp",
|
||||
"schedule_meeting",
|
||||
"get_meeting_by_code",
|
||||
"get_meeting",
|
||||
"get_user_meetings",
|
||||
"get_user_ended_meetings",
|
||||
"get_smart_minutes",
|
||||
"get_records_list",
|
||||
}
|
||||
|
||||
|
||||
def _tencent_meeting_query_handler(args: dict, **kwargs) -> str:
|
||||
user_id = str(args.get("user_id", "")).strip()
|
||||
tool_name = str(args.get("tool_name", "")).strip()
|
||||
|
||||
if not user_id:
|
||||
return tool_error("user_id 参数必填")
|
||||
if not tool_name:
|
||||
return tool_error("tool_name 参数必填")
|
||||
if tool_name not in _ALLOWED_TOOLS:
|
||||
return tool_error(
|
||||
f"不支持的工具：{tool_name}。可选：{', '.join(sorted(_ALLOWED_TOOLS))}"
|
||||
)
|
||||
if not has_user_token(user_id):
|
||||
return tool_error(
|
||||
f"腾讯会议 Token 未配置。请前往 {TOKEN_OBTAIN_URL} 获取 Token，"
|
||||
"并在侧边栏「腾讯会议」处绑定账号。"
|
||||
)
|
||||
|
||||
arguments = args.get("arguments", {})
|
||||
if not isinstance(arguments, dict):
|
||||
return tool_error("arguments 必须是 JSON 对象")
|
||||
|
||||
stdout, stderr, rc = _run_tencent_meeting(tool_name, arguments, user_id=user_id, timeout=30)
|
||||
|
||||
if rc != 0:
|
||||
return tool_error(f"调用失败（exit {rc}）：{(stderr or stdout)[:300]}")
|
||||
if not stdout:
|
||||
return tool_error(f"无输出。stderr: {stderr[:200]}")
|
||||
|
||||
try:
|
||||
data = json.loads(stdout)
|
||||
return tool_result(success=True, data=data)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return tool_result(success=True, output=stdout[:3000])
|
||||
|
||||
|
||||
registry.register(
|
||||
name="tencent_meeting_query",
|
||||
toolset="connectors",
|
||||
description="执行腾讯会议数据查询（会议列表/纪要/录制等）。",
|
||||
emoji="📹",
|
||||
check_fn=_check_tencent_meeting,
|
||||
handler=_tencent_meeting_query_handler,
|
||||
schema={
|
||||
"name": "tencent_meeting_query",
|
||||
"description": (
|
||||
"调用腾讯会议 MCP 工具。需先在侧边栏完成 Token 绑定。\n"
|
||||
"示例：\n"
|
||||
" 查当前时间: tool_name=convert_timestamp, arguments={}\n"
|
||||
" 查未来会议: tool_name=get_user_meetings, arguments={\"pos\": <unix_ts>}\n"
|
||||
" 查 AI 纪要: tool_name=get_smart_minutes, arguments={\"meeting_id\": \"xxx\"}\n"
|
||||
" 预约会议: tool_name=schedule_meeting, arguments={\"subject\":\"xx\",\"start_time\":\"...\",\"end_time\":\"...\"}\n"
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"user_id": {"type": "string", "description": "MINDOS_USER_ID"},
|
||||
"tool_name": {
|
||||
"type": "string",
|
||||
"description": "腾讯会议工具名",
|
||||
"enum": sorted(_ALLOWED_TOOLS),
|
||||
},
|
||||
"arguments": {
|
||||
"type": "object",
|
||||
"description": "工具参数（_client_info 由工具自动注入，不需要传）",
|
||||
},
|
||||
},
|
||||
"required": ["user_id", "tool_name"],
|
||||
},
|
||||
},
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,670 @@
|
||||
"""Tirith pre-exec security scanning wrapper.
|
||||
|
||||
Runs the tirith binary as a subprocess to scan commands for content-level
|
||||
threats (homograph URLs, pipe-to-interpreter, terminal injection, etc.).
|
||||
|
||||
Exit code is the verdict source of truth:
|
||||
0 = allow, 1 = block, 2 = warn
|
||||
|
||||
JSON stdout enriches findings/summary but never overrides the verdict.
|
||||
Operational failures (spawn error, timeout, unknown exit code) respect
|
||||
the fail_open config setting. Programming errors propagate.
|
||||
|
||||
Auto-install: if tirith is not found on PATH or at the configured path,
|
||||
it is automatically downloaded from GitHub releases to $HERMES_HOME/bin/tirith.
|
||||
The download always verifies SHA-256 checksums. When cosign is available on
|
||||
PATH, provenance verification (GitHub Actions workflow signature) is also
|
||||
performed. If cosign is not installed, the download proceeds with SHA-256
|
||||
verification only — still secure via HTTPS + checksum, just without supply
|
||||
chain provenance proof. Installation runs in a background thread so startup
|
||||
never blocks.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import platform
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
from hermes_constants import get_hermes_home
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_REPO = "sheeki03/tirith"
|
||||
|
||||
# Cosign provenance verification — pinned to the specific release workflow
|
||||
_COSIGN_IDENTITY_REGEXP = f"^https://github.com/{_REPO}/\\.github/workflows/release\\.yml@refs/tags/v"
|
||||
_COSIGN_ISSUER = "https://token.actions.githubusercontent.com"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _env_bool(key: str, default: bool) -> bool:
|
||||
val = os.getenv(key)
|
||||
if val is None:
|
||||
return default
|
||||
return val.lower() in ("1", "true", "yes")
|
||||
|
||||
|
||||
def _env_int(key: str, default: int) -> int:
|
||||
val = os.getenv(key)
|
||||
if val is None:
|
||||
return default
|
||||
try:
|
||||
return int(val)
|
||||
except ValueError:
|
||||
return default
|
||||
|
||||
|
||||
def _load_security_config() -> dict:
|
||||
"""Load security settings from config.yaml, with env var overrides."""
|
||||
defaults = {
|
||||
"tirith_enabled": True,
|
||||
"tirith_path": "tirith",
|
||||
"tirith_timeout": 5,
|
||||
"tirith_fail_open": True,
|
||||
}
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
cfg = load_config().get("security", {}) or {}
|
||||
except Exception:
|
||||
cfg = {}
|
||||
|
||||
return {
|
||||
"tirith_enabled": _env_bool("TIRITH_ENABLED", cfg.get("tirith_enabled", defaults["tirith_enabled"])),
|
||||
"tirith_path": os.getenv("TIRITH_BIN", cfg.get("tirith_path", defaults["tirith_path"])),
|
||||
"tirith_timeout": _env_int("TIRITH_TIMEOUT", cfg.get("tirith_timeout", defaults["tirith_timeout"])),
|
||||
"tirith_fail_open": _env_bool("TIRITH_FAIL_OPEN", cfg.get("tirith_fail_open", defaults["tirith_fail_open"])),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Auto-install
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Cached path after first resolution (avoids repeated shutil.which per command).
|
||||
# _INSTALL_FAILED means "we tried and failed" — prevents retry on every command.
|
||||
_resolved_path: str | None | bool = None
|
||||
_INSTALL_FAILED = False # sentinel: distinct from "not yet tried"
|
||||
_install_failure_reason: str = "" # reason tag when _resolved_path is _INSTALL_FAILED
|
||||
|
||||
# Background install thread coordination
|
||||
_install_lock = threading.Lock()
|
||||
_install_thread: threading.Thread | None = None
|
||||
|
||||
# Disk-persistent failure marker — avoids retry across process restarts
|
||||
_MARKER_TTL = 86400 # 24 hours
|
||||
|
||||
|
||||
def _get_hermes_home() -> str:
|
||||
"""Return the Hermes home directory, respecting HERMES_HOME env var."""
|
||||
return str(get_hermes_home())
|
||||
|
||||
|
||||
def _failure_marker_path() -> str:
|
||||
"""Return the path to the install-failure marker file."""
|
||||
return os.path.join(_get_hermes_home(), ".tirith-install-failed")
|
||||
|
||||
|
||||
def _read_failure_reason() -> str | None:
|
||||
"""Read the failure reason from the disk marker.
|
||||
|
||||
Returns the reason string, or None if the marker doesn't exist or is
|
||||
older than _MARKER_TTL.
|
||||
"""
|
||||
try:
|
||||
p = _failure_marker_path()
|
||||
mtime = os.path.getmtime(p)
|
||||
if (time.time() - mtime) >= _MARKER_TTL:
|
||||
return None
|
||||
with open(p, "r") as f:
|
||||
return f.read().strip()
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def _is_install_failed_on_disk() -> bool:
|
||||
"""Check if a recent install failure was persisted to disk.
|
||||
|
||||
Returns False (allowing retry) when:
|
||||
- No marker exists
|
||||
- Marker is older than _MARKER_TTL (24h)
|
||||
- Marker reason is 'cosign_missing' and cosign is now on PATH
|
||||
"""
|
||||
reason = _read_failure_reason()
|
||||
if reason is None:
|
||||
return False
|
||||
if reason == "cosign_missing" and shutil.which("cosign"):
|
||||
_clear_install_failed()
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _mark_install_failed(reason: str = ""):
|
||||
"""Persist install failure to disk to avoid retry on next process.
|
||||
|
||||
Args:
|
||||
reason: Short tag identifying the failure cause. Use "cosign_missing"
|
||||
when cosign is not on PATH so the marker can be auto-cleared
|
||||
once cosign becomes available.
|
||||
"""
|
||||
try:
|
||||
p = _failure_marker_path()
|
||||
os.makedirs(os.path.dirname(p), exist_ok=True)
|
||||
with open(p, "w") as f:
|
||||
f.write(reason)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _clear_install_failed():
|
||||
"""Remove the failure marker after successful install."""
|
||||
try:
|
||||
os.unlink(_failure_marker_path())
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _hermes_bin_dir() -> str:
|
||||
"""Return $HERMES_HOME/bin, creating it if needed."""
|
||||
d = os.path.join(_get_hermes_home(), "bin")
|
||||
os.makedirs(d, exist_ok=True)
|
||||
return d
|
||||
|
||||
|
||||
def _detect_target() -> str | None:
|
||||
"""Return the Rust target triple for the current platform, or None."""
|
||||
system = platform.system()
|
||||
machine = platform.machine().lower()
|
||||
|
||||
if system == "Darwin":
|
||||
plat = "apple-darwin"
|
||||
elif system == "Linux":
|
||||
plat = "unknown-linux-gnu"
|
||||
else:
|
||||
return None
|
||||
|
||||
if machine in ("x86_64", "amd64"):
|
||||
arch = "x86_64"
|
||||
elif machine in ("aarch64", "arm64"):
|
||||
arch = "aarch64"
|
||||
else:
|
||||
return None
|
||||
|
||||
return f"{arch}-{plat}"
|
||||
|
||||
|
||||
def _download_file(url: str, dest: str, timeout: int = 10):
|
||||
"""Download a URL to a local file."""
|
||||
req = urllib.request.Request(url)
|
||||
token = os.getenv("GITHUB_TOKEN")
|
||||
if token:
|
||||
req.add_header("Authorization", f"token {token}")
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp, open(dest, "wb") as f:
|
||||
shutil.copyfileobj(resp, f)
|
||||
|
||||
|
||||
def _verify_cosign(checksums_path: str, sig_path: str, cert_path: str) -> bool | None:
|
||||
"""Verify cosign provenance signature on checksums.txt.
|
||||
|
||||
Returns:
|
||||
True — cosign verified successfully
|
||||
False — cosign found but verification failed
|
||||
None — cosign not available (not on PATH, or execution failed)
|
||||
|
||||
The caller treats both False and None as "abort auto-install" — only
|
||||
True allows the install to proceed.
|
||||
"""
|
||||
cosign = shutil.which("cosign")
|
||||
if not cosign:
|
||||
logger.info("cosign not found on PATH")
|
||||
return None
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[cosign, "verify-blob",
|
||||
"--certificate", cert_path,
|
||||
"--signature", sig_path,
|
||||
"--certificate-identity-regexp", _COSIGN_IDENTITY_REGEXP,
|
||||
"--certificate-oidc-issuer", _COSIGN_ISSUER,
|
||||
checksums_path],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=15,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
logger.info("cosign provenance verification passed")
|
||||
return True
|
||||
else:
|
||||
logger.warning("cosign verification failed (exit %d): %s",
|
||||
result.returncode, result.stderr.strip())
|
||||
return False
|
||||
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||
logger.warning("cosign execution failed: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
def _verify_checksum(archive_path: str, checksums_path: str, archive_name: str) -> bool:
|
||||
"""Verify SHA-256 of the archive against checksums.txt."""
|
||||
expected = None
|
||||
with open(checksums_path) as f:
|
||||
for line in f:
|
||||
# Format: "<hash> <filename>"
|
||||
parts = line.strip().split(" ", 1)
|
||||
if len(parts) == 2 and parts[1] == archive_name:
|
||||
expected = parts[0]
|
||||
break
|
||||
if not expected:
|
||||
logger.warning("No checksum entry for %s", archive_name)
|
||||
return False
|
||||
|
||||
sha = hashlib.sha256()
|
||||
with open(archive_path, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(8192), b""):
|
||||
sha.update(chunk)
|
||||
actual = sha.hexdigest()
|
||||
if actual != expected:
|
||||
logger.warning("Checksum mismatch: expected %s, got %s", expected, actual)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _install_tirith(*, log_failures: bool = True) -> tuple[str | None, str]:
|
||||
"""Download and install tirith to $HERMES_HOME/bin/tirith.
|
||||
|
||||
Verifies provenance via cosign and SHA-256 checksum.
|
||||
Returns (installed_path, failure_reason). On success failure_reason is "".
|
||||
failure_reason is a short tag used by the disk marker to decide if the
|
||||
failure is retryable (e.g. "cosign_missing" clears when cosign appears).
|
||||
"""
|
||||
log = logger.warning if log_failures else logger.debug
|
||||
|
||||
target = _detect_target()
|
||||
if not target:
|
||||
logger.info("tirith auto-install: unsupported platform %s/%s",
|
||||
platform.system(), platform.machine())
|
||||
return None, "unsupported_platform"
|
||||
|
||||
archive_name = f"tirith-{target}.tar.gz"
|
||||
base_url = f"https://github.com/{_REPO}/releases/latest/download"
|
||||
|
||||
tmpdir = tempfile.mkdtemp(prefix="tirith-install-")
|
||||
try:
|
||||
archive_path = os.path.join(tmpdir, archive_name)
|
||||
checksums_path = os.path.join(tmpdir, "checksums.txt")
|
||||
sig_path = os.path.join(tmpdir, "checksums.txt.sig")
|
||||
cert_path = os.path.join(tmpdir, "checksums.txt.pem")
|
||||
|
||||
logger.info("tirith not found — downloading latest release for %s...", target)
|
||||
|
||||
try:
|
||||
_download_file(f"{base_url}/{archive_name}", archive_path)
|
||||
_download_file(f"{base_url}/checksums.txt", checksums_path)
|
||||
except Exception as exc:
|
||||
log("tirith download failed: %s", exc)
|
||||
return None, "download_failed"
|
||||
|
||||
# Cosign provenance verification — preferred but not mandatory.
|
||||
# When cosign is available, we verify that the release was produced
|
||||
# by the expected GitHub Actions workflow (full supply chain proof).
|
||||
# Without cosign, SHA-256 checksum + HTTPS still provides integrity
|
||||
# and transport-level authenticity.
|
||||
cosign_verified = False
|
||||
if shutil.which("cosign"):
|
||||
try:
|
||||
_download_file(f"{base_url}/checksums.txt.sig", sig_path)
|
||||
_download_file(f"{base_url}/checksums.txt.pem", cert_path)
|
||||
except Exception as exc:
|
||||
logger.info("cosign artifacts unavailable (%s), proceeding with SHA-256 only", exc)
|
||||
else:
|
||||
cosign_result = _verify_cosign(checksums_path, sig_path, cert_path)
|
||||
if cosign_result is True:
|
||||
cosign_verified = True
|
||||
elif cosign_result is False:
|
||||
# Verification explicitly rejected — abort, the release
|
||||
# may have been tampered with.
|
||||
log("tirith install aborted: cosign provenance verification failed")
|
||||
return None, "cosign_verification_failed"
|
||||
else:
|
||||
# None = execution failure (timeout/OSError) — proceed
|
||||
# with SHA-256 only since cosign itself is broken.
|
||||
logger.info("cosign execution failed, proceeding with SHA-256 only")
|
||||
else:
|
||||
logger.info("cosign not on PATH — installing tirith with SHA-256 verification only "
|
||||
"(install cosign for full supply chain verification)")
|
||||
|
||||
if not _verify_checksum(archive_path, checksums_path, archive_name):
|
||||
return None, "checksum_failed"
|
||||
|
||||
with tarfile.open(archive_path, "r:gz") as tar:
|
||||
# Extract only the tirith binary (safety: reject paths with ..)
|
||||
for member in tar.getmembers():
|
||||
if member.name == "tirith" or member.name.endswith("/tirith"):
|
||||
if ".." in member.name:
|
||||
continue
|
||||
member.name = "tirith"
|
||||
tar.extract(member, tmpdir)
|
||||
break
|
||||
else:
|
||||
log("tirith binary not found in archive")
|
||||
return None, "binary_not_in_archive"
|
||||
|
||||
src = os.path.join(tmpdir, "tirith")
|
||||
dest = os.path.join(_hermes_bin_dir(), "tirith")
|
||||
shutil.move(src, dest)
|
||||
os.chmod(dest, os.stat(dest).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
|
||||
|
||||
verification = "cosign + SHA-256" if cosign_verified else "SHA-256 only"
|
||||
logger.info("tirith installed to %s (%s)", dest, verification)
|
||||
return dest, ""
|
||||
|
||||
finally:
|
||||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||
|
||||
|
||||
def _is_explicit_path(configured_path: str) -> bool:
|
||||
"""Return True if the user explicitly configured a non-default tirith path."""
|
||||
return configured_path != "tirith"
|
||||
|
||||
|
||||
def _resolve_tirith_path(configured_path: str) -> str:
|
||||
"""Resolve the tirith binary path, auto-installing if necessary.
|
||||
|
||||
If the user explicitly set a path (anything other than the bare "tirith"
|
||||
default), that path is authoritative — we never fall through to
|
||||
auto-download a different binary.
|
||||
|
||||
For the default "tirith":
|
||||
1. PATH lookup via shutil.which
|
||||
2. $HERMES_HOME/bin/tirith (previously auto-installed)
|
||||
3. Auto-install from GitHub releases → $HERMES_HOME/bin/tirith
|
||||
|
||||
Failed installs are cached for the process lifetime (and persisted to
|
||||
disk for 24h) to avoid repeated network attempts.
|
||||
"""
|
||||
global _resolved_path, _install_failure_reason
|
||||
|
||||
# Fast path: successfully resolved on a previous call.
|
||||
if _resolved_path is not None and _resolved_path is not _INSTALL_FAILED:
|
||||
return _resolved_path
|
||||
|
||||
expanded = os.path.expanduser(configured_path)
|
||||
explicit = _is_explicit_path(configured_path)
|
||||
install_failed = _resolved_path is _INSTALL_FAILED
|
||||
|
||||
# Explicit path: check it and stop. Never auto-download a replacement.
|
||||
if explicit:
|
||||
if os.path.isfile(expanded) and os.access(expanded, os.X_OK):
|
||||
_resolved_path = expanded
|
||||
return expanded
|
||||
# Also try shutil.which in case it's a bare name on PATH
|
||||
found = shutil.which(expanded)
|
||||
if found:
|
||||
_resolved_path = found
|
||||
return found
|
||||
logger.warning("Configured tirith path %r not found; scanning disabled", configured_path)
|
||||
_resolved_path = _INSTALL_FAILED
|
||||
_install_failure_reason = "explicit_path_missing"
|
||||
return expanded
|
||||
|
||||
# Default "tirith" — always re-run cheap local checks so a manual
|
||||
# install is picked up even after a previous network failure (P2 fix:
|
||||
# long-lived gateway/CLI recovers without restart).
|
||||
found = shutil.which("tirith")
|
||||
if found:
|
||||
_resolved_path = found
|
||||
_install_failure_reason = ""
|
||||
_clear_install_failed()
|
||||
return found
|
||||
|
||||
hermes_bin = os.path.join(_hermes_bin_dir(), "tirith")
|
||||
if os.path.isfile(hermes_bin) and os.access(hermes_bin, os.X_OK):
|
||||
_resolved_path = hermes_bin
|
||||
_install_failure_reason = ""
|
||||
_clear_install_failed()
|
||||
return hermes_bin
|
||||
|
||||
# Local checks failed. If a previous install attempt already failed,
|
||||
# skip the network retry — UNLESS the failure was "cosign_missing" and
|
||||
# cosign is now available (retryable cause resolved in-process).
|
||||
if install_failed:
|
||||
if _install_failure_reason == "cosign_missing" and shutil.which("cosign"):
|
||||
# Retryable cause resolved — clear sentinel and fall through to retry
|
||||
_resolved_path = None
|
||||
_install_failure_reason = ""
|
||||
_clear_install_failed()
|
||||
install_failed = False
|
||||
else:
|
||||
return expanded
|
||||
|
||||
# If a background install thread is running, don't start a parallel one —
|
||||
# return the configured path; the OSError handler in check_command_security
|
||||
# will apply fail_open until the thread finishes.
|
||||
if _install_thread is not None and _install_thread.is_alive():
|
||||
return expanded
|
||||
|
||||
# Check disk failure marker before attempting network download.
|
||||
# Preserve the marker's real reason so in-memory retry logic can
|
||||
# detect retryable causes (e.g. cosign_missing) without restart.
|
||||
disk_reason = _read_failure_reason()
|
||||
if disk_reason is not None and _is_install_failed_on_disk():
|
||||
_resolved_path = _INSTALL_FAILED
|
||||
_install_failure_reason = disk_reason
|
||||
return expanded
|
||||
|
||||
installed, reason = _install_tirith()
|
||||
if installed:
|
||||
_resolved_path = installed
|
||||
_install_failure_reason = ""
|
||||
_clear_install_failed()
|
||||
return installed
|
||||
|
||||
# Install failed — cache the miss and persist reason to disk
|
||||
_resolved_path = _INSTALL_FAILED
|
||||
_install_failure_reason = reason
|
||||
_mark_install_failed(reason)
|
||||
return expanded
|
||||
|
||||
|
||||
def _background_install(*, log_failures: bool = True):
|
||||
"""Background thread target: download and install tirith."""
|
||||
global _resolved_path, _install_failure_reason
|
||||
with _install_lock:
|
||||
# Double-check after acquiring lock (another thread may have resolved)
|
||||
if _resolved_path is not None:
|
||||
return
|
||||
|
||||
# Re-check local paths (may have been installed by another process)
|
||||
found = shutil.which("tirith")
|
||||
if found:
|
||||
_resolved_path = found
|
||||
_install_failure_reason = ""
|
||||
return
|
||||
|
||||
hermes_bin = os.path.join(_hermes_bin_dir(), "tirith")
|
||||
if os.path.isfile(hermes_bin) and os.access(hermes_bin, os.X_OK):
|
||||
_resolved_path = hermes_bin
|
||||
_install_failure_reason = ""
|
||||
return
|
||||
|
||||
installed, reason = _install_tirith(log_failures=log_failures)
|
||||
if installed:
|
||||
_resolved_path = installed
|
||||
_install_failure_reason = ""
|
||||
_clear_install_failed()
|
||||
else:
|
||||
_resolved_path = _INSTALL_FAILED
|
||||
_install_failure_reason = reason
|
||||
_mark_install_failed(reason)
|
||||
|
||||
|
||||
def ensure_installed(*, log_failures: bool = True):
|
||||
"""Ensure tirith is available, downloading in background if needed.
|
||||
|
||||
Quick PATH/local checks are synchronous; network download runs in a
|
||||
daemon thread so startup never blocks. Safe to call multiple times.
|
||||
Returns the resolved path immediately if available, or None.
|
||||
"""
|
||||
global _resolved_path, _install_thread, _install_failure_reason
|
||||
|
||||
cfg = _load_security_config()
|
||||
if not cfg["tirith_enabled"]:
|
||||
return None
|
||||
|
||||
# Already resolved from a previous call
|
||||
if _resolved_path is not None and _resolved_path is not _INSTALL_FAILED:
|
||||
path = _resolved_path
|
||||
if os.path.isfile(path) and os.access(path, os.X_OK):
|
||||
return path
|
||||
return None
|
||||
|
||||
configured_path = cfg["tirith_path"]
|
||||
explicit = _is_explicit_path(configured_path)
|
||||
expanded = os.path.expanduser(configured_path)
|
||||
|
||||
# Explicit path: synchronous check only, no download
|
||||
if explicit:
|
||||
if os.path.isfile(expanded) and os.access(expanded, os.X_OK):
|
||||
_resolved_path = expanded
|
||||
return expanded
|
||||
found = shutil.which(expanded)
|
||||
if found:
|
||||
_resolved_path = found
|
||||
return found
|
||||
_resolved_path = _INSTALL_FAILED
|
||||
_install_failure_reason = "explicit_path_missing"
|
||||
return None
|
||||
|
||||
# Default "tirith" — quick local checks first (no network)
|
||||
found = shutil.which("tirith")
|
||||
if found:
|
||||
_resolved_path = found
|
||||
_install_failure_reason = ""
|
||||
_clear_install_failed()
|
||||
return found
|
||||
|
||||
hermes_bin = os.path.join(_hermes_bin_dir(), "tirith")
|
||||
if os.path.isfile(hermes_bin) and os.access(hermes_bin, os.X_OK):
|
||||
_resolved_path = hermes_bin
|
||||
_install_failure_reason = ""
|
||||
_clear_install_failed()
|
||||
return hermes_bin
|
||||
|
||||
# If previously failed in-memory, check if the cause is now resolved
|
||||
if _resolved_path is _INSTALL_FAILED:
|
||||
if _install_failure_reason == "cosign_missing" and shutil.which("cosign"):
|
||||
_resolved_path = None
|
||||
_install_failure_reason = ""
|
||||
_clear_install_failed()
|
||||
else:
|
||||
return None
|
||||
|
||||
# Check disk failure marker (skip network attempt for 24h, unless
|
||||
# the cosign_missing reason was resolved — handled by _is_install_failed_on_disk).
|
||||
# Preserve the marker's real reason for in-memory retry logic.
|
||||
disk_reason = _read_failure_reason()
|
||||
if disk_reason is not None and _is_install_failed_on_disk():
|
||||
_resolved_path = _INSTALL_FAILED
|
||||
_install_failure_reason = disk_reason
|
||||
return None
|
||||
|
||||
# Need to download — launch background thread so startup doesn't block
|
||||
if _install_thread is None or not _install_thread.is_alive():
|
||||
_install_thread = threading.Thread(
|
||||
target=_background_install,
|
||||
kwargs={"log_failures": log_failures},
|
||||
daemon=True,
|
||||
)
|
||||
_install_thread.start()
|
||||
|
||||
return None # Not available yet; commands will fail-open until ready
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_MAX_FINDINGS = 50
|
||||
_MAX_SUMMARY_LEN = 500
|
||||
|
||||
|
||||
def check_command_security(command: str) -> dict:
|
||||
"""Run tirith security scan on a command.
|
||||
|
||||
Exit code determines action (0=allow, 1=block, 2=warn). JSON enriches
|
||||
findings/summary. Spawn failures and timeouts respect fail_open config.
|
||||
Programming errors propagate.
|
||||
|
||||
Returns:
|
||||
{"action": "allow"|"warn"|"block", "findings": [...], "summary": str}
|
||||
"""
|
||||
cfg = _load_security_config()
|
||||
|
||||
if not cfg["tirith_enabled"]:
|
||||
return {"action": "allow", "findings": [], "summary": ""}
|
||||
|
||||
tirith_path = _resolve_tirith_path(cfg["tirith_path"])
|
||||
timeout = cfg["tirith_timeout"]
|
||||
fail_open = cfg["tirith_fail_open"]
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[tirith_path, "check", "--json", "--non-interactive",
|
||||
"--shell", "posix", "--", command],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
except OSError as exc:
|
||||
# Covers FileNotFoundError, PermissionError, exec format error
|
||||
logger.warning("tirith spawn failed: %s", exc)
|
||||
if fail_open:
|
||||
return {"action": "allow", "findings": [], "summary": f"tirith unavailable: {exc}"}
|
||||
return {"action": "block", "findings": [], "summary": f"tirith spawn failed (fail-closed): {exc}"}
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning("tirith timed out after %ds", timeout)
|
||||
if fail_open:
|
||||
return {"action": "allow", "findings": [], "summary": f"tirith timed out ({timeout}s)"}
|
||||
return {"action": "block", "findings": [], "summary": "tirith timed out (fail-closed)"}
|
||||
|
||||
# Map exit code to action
|
||||
exit_code = result.returncode
|
||||
if exit_code == 0:
|
||||
action = "allow"
|
||||
elif exit_code == 1:
|
||||
action = "block"
|
||||
elif exit_code == 2:
|
||||
action = "warn"
|
||||
else:
|
||||
# Unknown exit code — respect fail_open
|
||||
logger.warning("tirith returned unexpected exit code %d", exit_code)
|
||||
if fail_open:
|
||||
return {"action": "allow", "findings": [], "summary": f"tirith exit code {exit_code} (fail-open)"}
|
||||
return {"action": "block", "findings": [], "summary": f"tirith exit code {exit_code} (fail-closed)"}
|
||||
|
||||
# Parse JSON for enrichment (never overrides the exit code verdict)
|
||||
findings = []
|
||||
summary = ""
|
||||
try:
|
||||
data = json.loads(result.stdout) if result.stdout.strip() else {}
|
||||
raw_findings = data.get("findings", [])
|
||||
findings = raw_findings[:_MAX_FINDINGS]
|
||||
summary = (data.get("summary", "") or "")[:_MAX_SUMMARY_LEN]
|
||||
except (json.JSONDecodeError, AttributeError):
|
||||
# JSON parse failure degrades findings/summary, not the verdict
|
||||
logger.debug("tirith JSON parse failed, using exit code only")
|
||||
if action == "block":
|
||||
summary = "security issue detected (details unavailable)"
|
||||
elif action == "warn":
|
||||
summary = "security warning detected (details unavailable)"
|
||||
|
||||
return {"action": action, "findings": findings, "summary": summary}
|
||||
@@ -0,0 +1,277 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Todo Tool Module - Planning & Task Management
|
||||
|
||||
Provides an in-memory task list the agent uses to decompose complex tasks,
|
||||
track progress, and maintain focus across long conversations. The state
|
||||
lives on the AIAgent instance (one per session) and is re-injected into
|
||||
the conversation after context compression events.
|
||||
|
||||
Design:
|
||||
- Single `todo` tool: provide `todos` param to write, omit to read
|
||||
- Every call returns the full current list
|
||||
- No system prompt mutation, no tool response modification
|
||||
- Behavioral guidance lives entirely in the tool schema description
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Dict, Any, List, Optional
|
||||
|
||||
|
||||
# Valid status values for todo items
|
||||
VALID_STATUSES = {"pending", "in_progress", "completed", "cancelled"}
|
||||
|
||||
|
||||
class TodoStore:
|
||||
"""
|
||||
In-memory todo list. One instance per AIAgent (one per session).
|
||||
|
||||
Items are ordered -- list position is priority. Each item has:
|
||||
- id: unique string identifier (agent-chosen)
|
||||
- content: task description
|
||||
- status: pending | in_progress | completed | cancelled
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._items: List[Dict[str, str]] = []
|
||||
|
||||
def write(self, todos: List[Dict[str, Any]], merge: bool = False) -> List[Dict[str, str]]:
|
||||
"""
|
||||
Write todos. Returns the full current list after writing.
|
||||
|
||||
Args:
|
||||
todos: list of {id, content, status} dicts
|
||||
merge: if False, replace the entire list. If True, update
|
||||
existing items by id and append new ones.
|
||||
"""
|
||||
if not merge:
|
||||
# Replace mode: new list entirely
|
||||
self._items = [self._validate(t) for t in self._dedupe_by_id(todos)]
|
||||
else:
|
||||
# Merge mode: update existing items by id, append new ones
|
||||
existing = {item["id"]: item for item in self._items}
|
||||
for t in self._dedupe_by_id(todos):
|
||||
item_id = str(t.get("id", "")).strip()
|
||||
if not item_id:
|
||||
continue # Can't merge without an id
|
||||
|
||||
if item_id in existing:
|
||||
# Update only the fields the LLM actually provided
|
||||
if "content" in t and t["content"]:
|
||||
existing[item_id]["content"] = str(t["content"]).strip()
|
||||
if "status" in t and t["status"]:
|
||||
status = str(t["status"]).strip().lower()
|
||||
if status in VALID_STATUSES:
|
||||
existing[item_id]["status"] = status
|
||||
else:
|
||||
# New item -- validate fully and append to end
|
||||
validated = self._validate(t)
|
||||
existing[validated["id"]] = validated
|
||||
self._items.append(validated)
|
||||
# Rebuild _items preserving order for existing items
|
||||
seen = set()
|
||||
rebuilt = []
|
||||
for item in self._items:
|
||||
current = existing.get(item["id"], item)
|
||||
if current["id"] not in seen:
|
||||
rebuilt.append(current)
|
||||
seen.add(current["id"])
|
||||
self._items = rebuilt
|
||||
return self.read()
|
||||
|
||||
def read(self) -> List[Dict[str, str]]:
|
||||
"""Return a copy of the current list."""
|
||||
return [item.copy() for item in self._items]
|
||||
|
||||
def has_items(self) -> bool:
|
||||
"""Check if there are any items in the list."""
|
||||
return bool(self._items)
|
||||
|
||||
def format_for_injection(self) -> Optional[str]:
|
||||
"""
|
||||
Render the todo list for post-compression injection.
|
||||
|
||||
Returns a human-readable string to append to the compressed
|
||||
message history, or None if the list is empty.
|
||||
"""
|
||||
if not self._items:
|
||||
return None
|
||||
|
||||
# Status markers for compact display
|
||||
markers = {
|
||||
"completed": "[x]",
|
||||
"in_progress": "[>]",
|
||||
"pending": "[ ]",
|
||||
"cancelled": "[~]",
|
||||
}
|
||||
|
||||
# Only inject pending/in_progress items — completed/cancelled ones
|
||||
# cause the model to re-do finished work after compression.
|
||||
active_items = [
|
||||
item for item in self._items
|
||||
if item["status"] in ("pending", "in_progress")
|
||||
]
|
||||
if not active_items:
|
||||
return None
|
||||
|
||||
lines = ["[Your active task list was preserved across context compression]"]
|
||||
for item in active_items:
|
||||
marker = markers.get(item["status"], "[?]")
|
||||
lines.append(f"- {marker} {item['id']}. {item['content']} ({item['status']})")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
@staticmethod
|
||||
def _validate(item: Dict[str, Any]) -> Dict[str, str]:
|
||||
"""
|
||||
Validate and normalize a todo item.
|
||||
|
||||
Ensures required fields exist and status is valid.
|
||||
Returns a clean dict with only {id, content, status}.
|
||||
"""
|
||||
item_id = str(item.get("id", "")).strip()
|
||||
if not item_id:
|
||||
item_id = "?"
|
||||
|
||||
content = str(item.get("content", "")).strip()
|
||||
if not content:
|
||||
content = "(no description)"
|
||||
|
||||
status = str(item.get("status", "pending")).strip().lower()
|
||||
if status not in VALID_STATUSES:
|
||||
status = "pending"
|
||||
|
||||
return {"id": item_id, "content": content, "status": status}
|
||||
|
||||
@staticmethod
|
||||
def _dedupe_by_id(todos: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
"""Collapse duplicate ids, keeping the last occurrence in its position."""
|
||||
last_index: Dict[str, int] = {}
|
||||
for i, item in enumerate(todos):
|
||||
item_id = str(item.get("id", "")).strip() or "?"
|
||||
last_index[item_id] = i
|
||||
return [todos[i] for i in sorted(last_index.values())]
|
||||
|
||||
|
||||
def todo_tool(
|
||||
todos: Optional[List[Dict[str, Any]]] = None,
|
||||
merge: bool = False,
|
||||
store: Optional[TodoStore] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Single entry point for the todo tool. Reads or writes depending on params.
|
||||
|
||||
Args:
|
||||
todos: if provided, write these items. If None, read current list.
|
||||
merge: if True, update by id. If False (default), replace entire list.
|
||||
store: the TodoStore instance from the AIAgent.
|
||||
|
||||
Returns:
|
||||
JSON string with the full current list and summary metadata.
|
||||
"""
|
||||
if store is None:
|
||||
return tool_error("TodoStore not initialized")
|
||||
|
||||
if todos is not None:
|
||||
items = store.write(todos, merge)
|
||||
else:
|
||||
items = store.read()
|
||||
|
||||
# Build summary counts
|
||||
pending = sum(1 for i in items if i["status"] == "pending")
|
||||
in_progress = sum(1 for i in items if i["status"] == "in_progress")
|
||||
completed = sum(1 for i in items if i["status"] == "completed")
|
||||
cancelled = sum(1 for i in items if i["status"] == "cancelled")
|
||||
|
||||
return json.dumps({
|
||||
"todos": items,
|
||||
"summary": {
|
||||
"total": len(items),
|
||||
"pending": pending,
|
||||
"in_progress": in_progress,
|
||||
"completed": completed,
|
||||
"cancelled": cancelled,
|
||||
},
|
||||
}, ensure_ascii=False)
|
||||
|
||||
|
||||
def check_todo_requirements() -> bool:
|
||||
"""Todo tool has no external requirements -- always available."""
|
||||
return True
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# OpenAI Function-Calling Schema
|
||||
# =============================================================================
|
||||
# Behavioral guidance is baked into the description so it's part of the
|
||||
# static tool schema (cached, never changes mid-conversation).
|
||||
|
||||
TODO_SCHEMA = {
|
||||
"name": "todo",
|
||||
"description": (
|
||||
"Manage your task list for the current session. Use for complex tasks "
|
||||
"with 3+ steps or when the user provides multiple tasks. "
|
||||
"Call with no parameters to read the current list.\n\n"
|
||||
"Writing:\n"
|
||||
"- Provide 'todos' array to create/update items\n"
|
||||
"- merge=false (default): replace the entire list with a fresh plan\n"
|
||||
"- merge=true: update existing items by id, add any new ones\n\n"
|
||||
"Each item: {id: string, content: string, "
|
||||
"status: pending|in_progress|completed|cancelled}\n"
|
||||
"List order is priority. Only ONE item in_progress at a time.\n"
|
||||
"Mark items completed immediately when done. If something fails, "
|
||||
"cancel it and add a revised item.\n\n"
|
||||
"Always returns the full current list."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"todos": {
|
||||
"type": "array",
|
||||
"description": "Task items to write. Omit to read current list.",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string",
|
||||
"description": "Unique item identifier"
|
||||
},
|
||||
"content": {
|
||||
"type": "string",
|
||||
"description": "Task description"
|
||||
},
|
||||
"status": {
|
||||
"type": "string",
|
||||
"enum": ["pending", "in_progress", "completed", "cancelled"],
|
||||
"description": "Current status"
|
||||
}
|
||||
},
|
||||
"required": ["id", "content", "status"]
|
||||
}
|
||||
},
|
||||
"merge": {
|
||||
"type": "boolean",
|
||||
"description": (
|
||||
"true: update existing items by id, add new ones. "
|
||||
"false (default): replace the entire list."
|
||||
),
|
||||
"default": False
|
||||
}
|
||||
},
|
||||
"required": []
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
# --- Registry ---
|
||||
from tools.registry import registry, tool_error
|
||||
|
||||
registry.register(
|
||||
name="todo",
|
||||
toolset="todo",
|
||||
schema=TODO_SCHEMA,
|
||||
handler=lambda args, **kw: todo_tool(
|
||||
todos=args.get("todos"), merge=args.get("merge", False), store=kw.get("store")),
|
||||
check_fn=check_todo_requirements,
|
||||
emoji="📋",
|
||||
)
|
||||
@@ -0,0 +1,89 @@
|
||||
"""Shared helpers for tool backend selection."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict
|
||||
|
||||
from utils import env_var_enabled
|
||||
|
||||
_DEFAULT_BROWSER_PROVIDER = "local"
|
||||
_DEFAULT_MODAL_MODE = "auto"
|
||||
_VALID_MODAL_MODES = {"auto", "direct", "managed"}
|
||||
|
||||
|
||||
def managed_nous_tools_enabled() -> bool:
|
||||
"""Return True when the hidden Nous-managed tools feature flag is enabled."""
|
||||
return env_var_enabled("HERMES_ENABLE_NOUS_MANAGED_TOOLS")
|
||||
|
||||
|
||||
def normalize_browser_cloud_provider(value: object | None) -> str:
|
||||
"""Return a normalized browser provider key."""
|
||||
provider = str(value or _DEFAULT_BROWSER_PROVIDER).strip().lower()
|
||||
return provider or _DEFAULT_BROWSER_PROVIDER
|
||||
|
||||
|
||||
def coerce_modal_mode(value: object | None) -> str:
|
||||
"""Return the requested modal mode when valid, else the default."""
|
||||
mode = str(value or _DEFAULT_MODAL_MODE).strip().lower()
|
||||
if mode in _VALID_MODAL_MODES:
|
||||
return mode
|
||||
return _DEFAULT_MODAL_MODE
|
||||
|
||||
|
||||
def normalize_modal_mode(value: object | None) -> str:
|
||||
"""Return a normalized modal execution mode."""
|
||||
return coerce_modal_mode(value)
|
||||
|
||||
|
||||
def has_direct_modal_credentials() -> bool:
|
||||
"""Return True when direct Modal credentials/config are available."""
|
||||
return bool(
|
||||
(os.getenv("MODAL_TOKEN_ID") and os.getenv("MODAL_TOKEN_SECRET"))
|
||||
or (Path.home() / ".modal.toml").exists()
|
||||
)
|
||||
|
||||
|
||||
def resolve_modal_backend_state(
|
||||
modal_mode: object | None,
|
||||
*,
|
||||
has_direct: bool,
|
||||
managed_ready: bool,
|
||||
) -> Dict[str, Any]:
|
||||
"""Resolve direct vs managed Modal backend selection.
|
||||
|
||||
Semantics:
|
||||
- ``direct`` means direct-only
|
||||
- ``managed`` means managed-only
|
||||
- ``auto`` prefers managed when available, then falls back to direct
|
||||
"""
|
||||
requested_mode = coerce_modal_mode(modal_mode)
|
||||
normalized_mode = normalize_modal_mode(modal_mode)
|
||||
managed_mode_blocked = (
|
||||
requested_mode == "managed" and not managed_nous_tools_enabled()
|
||||
)
|
||||
|
||||
if normalized_mode == "managed":
|
||||
selected_backend = "managed" if managed_nous_tools_enabled() and managed_ready else None
|
||||
elif normalized_mode == "direct":
|
||||
selected_backend = "direct" if has_direct else None
|
||||
else:
|
||||
selected_backend = "managed" if managed_nous_tools_enabled() and managed_ready else "direct" if has_direct else None
|
||||
|
||||
return {
|
||||
"requested_mode": requested_mode,
|
||||
"mode": normalized_mode,
|
||||
"has_direct": has_direct,
|
||||
"managed_ready": managed_ready,
|
||||
"managed_mode_blocked": managed_mode_blocked,
|
||||
"selected_backend": selected_backend,
|
||||
}
|
||||
|
||||
|
||||
def resolve_openai_audio_api_key() -> str:
|
||||
"""Prefer the voice-tools key, but fall back to the normal OpenAI key."""
|
||||
return (
|
||||
os.getenv("VOICE_TOOLS_OPENAI_KEY", "")
|
||||
or os.getenv("OPENAI_API_KEY", "")
|
||||
).strip()
|
||||
@@ -0,0 +1,226 @@
|
||||
"""Tool result persistence -- preserves large outputs instead of truncating.
|
||||
|
||||
Defense against context-window overflow operates at three levels:
|
||||
|
||||
1. **Per-tool output cap** (inside each tool): Tools like search_files
|
||||
pre-truncate their own output before returning. This is the first line
|
||||
of defense and the only one the tool author controls.
|
||||
|
||||
2. **Per-result persistence** (maybe_persist_tool_result): After a tool
|
||||
returns, if its output exceeds the tool's registered threshold
|
||||
(registry.get_max_result_size), the full output is written INTO THE
|
||||
SANDBOX temp dir (for example /tmp/hermes-results/{tool_use_id}.txt on
|
||||
standard Linux, or $TMPDIR/hermes-results/{tool_use_id}.txt on Termux)
|
||||
via env.execute(). The in-context content is replaced with a preview +
|
||||
file path reference. The model can read_file to access the full output
|
||||
on any backend.
|
||||
|
||||
3. **Per-turn aggregate budget** (enforce_turn_budget): After all tool
|
||||
results in a single assistant turn are collected, if the total exceeds
|
||||
MAX_TURN_BUDGET_CHARS (200K), the largest non-persisted results are
|
||||
spilled to disk until the aggregate is under budget. This catches cases
|
||||
where many medium-sized results combine to overflow context.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shlex
|
||||
import uuid
|
||||
|
||||
from tools.budget_config import (
|
||||
DEFAULT_PREVIEW_SIZE_CHARS,
|
||||
BudgetConfig,
|
||||
DEFAULT_BUDGET,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
PERSISTED_OUTPUT_TAG = "<persisted-output>"
|
||||
PERSISTED_OUTPUT_CLOSING_TAG = "</persisted-output>"
|
||||
STORAGE_DIR = "/tmp/hermes-results"
|
||||
HEREDOC_MARKER = "HERMES_PERSIST_EOF"
|
||||
_BUDGET_TOOL_NAME = "__budget_enforcement__"
|
||||
|
||||
|
||||
def _resolve_storage_dir(env) -> str:
|
||||
"""Return the best temp-backed storage dir for this environment."""
|
||||
if env is not None:
|
||||
get_temp_dir = getattr(env, "get_temp_dir", None)
|
||||
if callable(get_temp_dir):
|
||||
try:
|
||||
temp_dir = get_temp_dir()
|
||||
except Exception as exc:
|
||||
logger.debug("Could not resolve env temp dir: %s", exc)
|
||||
else:
|
||||
if temp_dir:
|
||||
temp_dir = temp_dir.rstrip("/") or "/"
|
||||
return f"{temp_dir}/hermes-results"
|
||||
return STORAGE_DIR
|
||||
|
||||
|
||||
def generate_preview(content: str, max_chars: int = DEFAULT_PREVIEW_SIZE_CHARS) -> tuple[str, bool]:
|
||||
"""Truncate at last newline within max_chars. Returns (preview, has_more)."""
|
||||
if len(content) <= max_chars:
|
||||
return content, False
|
||||
truncated = content[:max_chars]
|
||||
last_nl = truncated.rfind("\n")
|
||||
if last_nl > max_chars // 2:
|
||||
truncated = truncated[:last_nl + 1]
|
||||
return truncated, True
|
||||
|
||||
|
||||
def _heredoc_marker(content: str) -> str:
|
||||
"""Return a heredoc delimiter that doesn't collide with content."""
|
||||
if HEREDOC_MARKER not in content:
|
||||
return HEREDOC_MARKER
|
||||
return f"HERMES_PERSIST_{uuid.uuid4().hex[:8]}"
|
||||
|
||||
|
||||
def _write_to_sandbox(content: str, remote_path: str, env) -> bool:
|
||||
"""Write content into the sandbox via env.execute(). Returns True on success."""
|
||||
marker = _heredoc_marker(content)
|
||||
storage_dir = os.path.dirname(remote_path)
|
||||
cmd = (
|
||||
f"mkdir -p {shlex.quote(storage_dir)} && cat > {shlex.quote(remote_path)} << '{marker}'\n"
|
||||
f"{content}\n"
|
||||
f"{marker}"
|
||||
)
|
||||
result = env.execute(cmd, timeout=30)
|
||||
return result.get("returncode", 1) == 0
|
||||
|
||||
|
||||
def _build_persisted_message(
|
||||
preview: str,
|
||||
has_more: bool,
|
||||
original_size: int,
|
||||
file_path: str,
|
||||
) -> str:
|
||||
"""Build the <persisted-output> replacement block."""
|
||||
size_kb = original_size / 1024
|
||||
if size_kb >= 1024:
|
||||
size_str = f"{size_kb / 1024:.1f} MB"
|
||||
else:
|
||||
size_str = f"{size_kb:.1f} KB"
|
||||
|
||||
msg = f"{PERSISTED_OUTPUT_TAG}\n"
|
||||
msg += f"This tool result was too large ({original_size:,} characters, {size_str}).\n"
|
||||
msg += f"Full output saved to: {file_path}\n"
|
||||
msg += "Use the read_file tool with offset and limit to access specific sections of this output.\n\n"
|
||||
msg += f"Preview (first {len(preview)} chars):\n"
|
||||
msg += preview
|
||||
if has_more:
|
||||
msg += "\n..."
|
||||
msg += f"\n{PERSISTED_OUTPUT_CLOSING_TAG}"
|
||||
return msg
|
||||
|
||||
|
||||
def maybe_persist_tool_result(
|
||||
content: str,
|
||||
tool_name: str,
|
||||
tool_use_id: str,
|
||||
env=None,
|
||||
config: BudgetConfig = DEFAULT_BUDGET,
|
||||
threshold: int | float | None = None,
|
||||
) -> str:
|
||||
"""Layer 2: persist oversized result into the sandbox, return preview + path.
|
||||
|
||||
Writes via env.execute() so the file is accessible from any backend
|
||||
(local, Docker, SSH, Modal, Daytona). Falls back to inline truncation
|
||||
if write fails or no env is available.
|
||||
|
||||
Args:
|
||||
content: Raw tool result string.
|
||||
tool_name: Name of the tool (used for threshold lookup).
|
||||
tool_use_id: Unique ID for this tool call (used as filename).
|
||||
env: The active BaseEnvironment instance, or None.
|
||||
config: BudgetConfig controlling thresholds and preview size.
|
||||
threshold: Explicit override; takes precedence over config resolution.
|
||||
|
||||
Returns:
|
||||
Original content if small, or <persisted-output> replacement.
|
||||
"""
|
||||
effective_threshold = threshold if threshold is not None else config.resolve_threshold(tool_name)
|
||||
|
||||
if effective_threshold == float("inf"):
|
||||
return content
|
||||
|
||||
if len(content) <= effective_threshold:
|
||||
return content
|
||||
|
||||
storage_dir = _resolve_storage_dir(env)
|
||||
remote_path = f"{storage_dir}/{tool_use_id}.txt"
|
||||
preview, has_more = generate_preview(content, max_chars=config.preview_size)
|
||||
|
||||
if env is not None:
|
||||
try:
|
||||
if _write_to_sandbox(content, remote_path, env):
|
||||
logger.info(
|
||||
"Persisted large tool result: %s (%s, %d chars -> %s)",
|
||||
tool_name, tool_use_id, len(content), remote_path,
|
||||
)
|
||||
return _build_persisted_message(preview, has_more, len(content), remote_path)
|
||||
except Exception as exc:
|
||||
logger.warning("Sandbox write failed for %s: %s", tool_use_id, exc)
|
||||
|
||||
logger.info(
|
||||
"Inline-truncating large tool result: %s (%d chars, no sandbox write)",
|
||||
tool_name, len(content),
|
||||
)
|
||||
return (
|
||||
f"{preview}\n\n"
|
||||
f"[Truncated: tool response was {len(content):,} chars. "
|
||||
f"Full output could not be saved to sandbox.]"
|
||||
)
|
||||
|
||||
|
||||
def enforce_turn_budget(
|
||||
tool_messages: list[dict],
|
||||
env=None,
|
||||
config: BudgetConfig = DEFAULT_BUDGET,
|
||||
) -> list[dict]:
|
||||
"""Layer 3: enforce aggregate budget across all tool results in a turn.
|
||||
|
||||
If total chars exceed budget, persist the largest non-persisted results
|
||||
first (via sandbox write) until under budget. Already-persisted results
|
||||
are skipped.
|
||||
|
||||
Mutates the list in-place and returns it.
|
||||
"""
|
||||
candidates = []
|
||||
total_size = 0
|
||||
for i, msg in enumerate(tool_messages):
|
||||
content = msg.get("content", "")
|
||||
size = len(content)
|
||||
total_size += size
|
||||
if PERSISTED_OUTPUT_TAG not in content:
|
||||
candidates.append((i, size))
|
||||
|
||||
if total_size <= config.turn_budget:
|
||||
return tool_messages
|
||||
|
||||
candidates.sort(key=lambda x: x[1], reverse=True)
|
||||
|
||||
for idx, size in candidates:
|
||||
if total_size <= config.turn_budget:
|
||||
break
|
||||
msg = tool_messages[idx]
|
||||
content = msg["content"]
|
||||
tool_use_id = msg.get("tool_call_id", f"budget_{idx}")
|
||||
|
||||
replacement = maybe_persist_tool_result(
|
||||
content=content,
|
||||
tool_name=_BUDGET_TOOL_NAME,
|
||||
tool_use_id=tool_use_id,
|
||||
env=env,
|
||||
config=config,
|
||||
threshold=0,
|
||||
)
|
||||
if replacement != content:
|
||||
total_size -= size
|
||||
total_size += len(replacement)
|
||||
tool_messages[idx]["content"] = replacement
|
||||
logger.info(
|
||||
"Budget enforcement: persisted tool result %s (%d chars)",
|
||||
tool_use_id, size,
|
||||
)
|
||||
|
||||
return tool_messages
|
||||
@@ -0,0 +1,677 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Transcription Tools Module
|
||||
|
||||
Provides speech-to-text transcription with three providers:
|
||||
|
||||
- **local** (default, free) — faster-whisper running locally, no API key needed.
|
||||
Auto-downloads the model (~150 MB for ``base``) on first use.
|
||||
- **groq** (free tier) — Groq Whisper API, requires ``GROQ_API_KEY``.
|
||||
- **openai** (paid) — OpenAI Whisper API, requires ``VOICE_TOOLS_OPENAI_KEY``.
|
||||
|
||||
Used by the messaging gateway to automatically transcribe voice messages
|
||||
sent by users on Telegram, Discord, WhatsApp, Slack, and Signal.
|
||||
|
||||
Supported input formats: mp3, mp4, mpeg, mpga, m4a, wav, webm, ogg, aac
|
||||
|
||||
Usage::
|
||||
|
||||
from tools.transcription_tools import transcribe_audio
|
||||
|
||||
result = transcribe_audio("/path/to/audio.ogg")
|
||||
if result["success"]:
|
||||
print(result["transcript"])
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Optional, Dict, Any
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from utils import is_truthy_value
|
||||
from tools.managed_tool_gateway import resolve_managed_tool_gateway
|
||||
from tools.tool_backend_helpers import managed_nous_tools_enabled, resolve_openai_audio_api_key
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Optional imports — graceful degradation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
import importlib.util as _ilu
|
||||
|
||||
|
||||
def _safe_find_spec(module_name: str) -> bool:
|
||||
try:
|
||||
return _ilu.find_spec(module_name) is not None
|
||||
except (ImportError, ValueError):
|
||||
return module_name in globals() or module_name in os.sys.modules
|
||||
|
||||
|
||||
_HAS_FASTER_WHISPER = _safe_find_spec("faster_whisper")
|
||||
_HAS_OPENAI = _safe_find_spec("openai")
|
||||
_HAS_MISTRAL = _safe_find_spec("mistralai")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
DEFAULT_PROVIDER = "local"
|
||||
DEFAULT_LOCAL_MODEL = "base"
|
||||
DEFAULT_LOCAL_STT_LANGUAGE = "en"
|
||||
DEFAULT_STT_MODEL = os.getenv("STT_OPENAI_MODEL", "whisper-1")
|
||||
DEFAULT_GROQ_STT_MODEL = os.getenv("STT_GROQ_MODEL", "whisper-large-v3-turbo")
|
||||
DEFAULT_MISTRAL_STT_MODEL = os.getenv("STT_MISTRAL_MODEL", "voxtral-mini-latest")
|
||||
LOCAL_STT_COMMAND_ENV = "HERMES_LOCAL_STT_COMMAND"
|
||||
LOCAL_STT_LANGUAGE_ENV = "HERMES_LOCAL_STT_LANGUAGE"
|
||||
COMMON_LOCAL_BIN_DIRS = ("/opt/homebrew/bin", "/usr/local/bin")
|
||||
|
||||
GROQ_BASE_URL = os.getenv("GROQ_BASE_URL", "https://api.groq.com/openai/v1")
|
||||
OPENAI_BASE_URL = os.getenv("STT_OPENAI_BASE_URL", "https://api.openai.com/v1")
|
||||
|
||||
SUPPORTED_FORMATS = {".mp3", ".mp4", ".mpeg", ".mpga", ".m4a", ".wav", ".webm", ".ogg", ".aac", ".flac"}
|
||||
LOCAL_NATIVE_AUDIO_FORMATS = {".wav", ".aiff", ".aif"}
|
||||
MAX_FILE_SIZE = 25 * 1024 * 1024 # 25 MB
|
||||
|
||||
# Known model sets for auto-correction
|
||||
OPENAI_MODELS = {"whisper-1", "gpt-4o-mini-transcribe", "gpt-4o-transcribe"}
|
||||
GROQ_MODELS = {"whisper-large-v3", "whisper-large-v3-turbo", "distil-whisper-large-v3-en"}
|
||||
|
||||
# Singleton for the local model — loaded once, reused across calls
|
||||
_local_model: Optional[object] = None
|
||||
_local_model_name: Optional[str] = None
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
|
||||
def _load_stt_config() -> dict:
|
||||
"""Load the ``stt`` section from user config, falling back to defaults."""
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
return load_config().get("stt", {})
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def is_stt_enabled(stt_config: Optional[dict] = None) -> bool:
|
||||
"""Return whether STT is enabled in config."""
|
||||
if stt_config is None:
|
||||
stt_config = _load_stt_config()
|
||||
enabled = stt_config.get("enabled", True)
|
||||
return is_truthy_value(enabled, default=True)
|
||||
|
||||
|
||||
def _has_openai_audio_backend() -> bool:
|
||||
"""Return True when OpenAI audio can use config credentials, env credentials, or the managed gateway."""
|
||||
try:
|
||||
_resolve_openai_audio_client_config()
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def _find_binary(binary_name: str) -> Optional[str]:
|
||||
"""Find a local binary, checking common Homebrew/local prefixes as well as PATH."""
|
||||
for directory in COMMON_LOCAL_BIN_DIRS:
|
||||
candidate = Path(directory) / binary_name
|
||||
if candidate.exists() and os.access(candidate, os.X_OK):
|
||||
return str(candidate)
|
||||
return shutil.which(binary_name)
|
||||
|
||||
|
||||
def _find_ffmpeg_binary() -> Optional[str]:
|
||||
return _find_binary("ffmpeg")
|
||||
|
||||
|
||||
def _find_whisper_binary() -> Optional[str]:
|
||||
return _find_binary("whisper")
|
||||
|
||||
|
||||
def _get_local_command_template() -> Optional[str]:
|
||||
configured = os.getenv(LOCAL_STT_COMMAND_ENV, "").strip()
|
||||
if configured:
|
||||
return configured
|
||||
|
||||
whisper_binary = _find_whisper_binary()
|
||||
if whisper_binary:
|
||||
quoted_binary = shlex.quote(whisper_binary)
|
||||
return (
|
||||
f"{quoted_binary} {{input_path}} --model {{model}} --output_format txt "
|
||||
"--output_dir {output_dir} --language {language}"
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _has_local_command() -> bool:
|
||||
return _get_local_command_template() is not None
|
||||
|
||||
|
||||
def _normalize_local_command_model(model_name: Optional[str]) -> str:
|
||||
if not model_name or model_name in OPENAI_MODELS or model_name in GROQ_MODELS:
|
||||
return DEFAULT_LOCAL_MODEL
|
||||
return model_name
|
||||
|
||||
|
||||
def _get_provider(stt_config: dict) -> str:
|
||||
"""Determine which STT provider to use.
|
||||
|
||||
When ``stt.provider`` is explicitly set in config, that choice is
|
||||
honoured — no silent cloud fallback. When no provider is configured,
|
||||
auto-detect tries: local > groq (free) > openai (paid).
|
||||
"""
|
||||
if not is_stt_enabled(stt_config):
|
||||
return "none"
|
||||
|
||||
explicit = "provider" in stt_config
|
||||
provider = stt_config.get("provider", DEFAULT_PROVIDER)
|
||||
|
||||
# --- Explicit provider: respect the user's choice ----------------------
|
||||
|
||||
if explicit:
|
||||
if provider == "local":
|
||||
if _HAS_FASTER_WHISPER:
|
||||
return "local"
|
||||
if _has_local_command():
|
||||
return "local_command"
|
||||
logger.warning(
|
||||
"STT provider 'local' configured but unavailable "
|
||||
"(install faster-whisper or set HERMES_LOCAL_STT_COMMAND)"
|
||||
)
|
||||
return "none"
|
||||
|
||||
if provider == "local_command":
|
||||
if _has_local_command():
|
||||
return "local_command"
|
||||
if _HAS_FASTER_WHISPER:
|
||||
logger.info("Local STT command unavailable, using local faster-whisper")
|
||||
return "local"
|
||||
logger.warning(
|
||||
"STT provider 'local_command' configured but unavailable"
|
||||
)
|
||||
return "none"
|
||||
|
||||
if provider == "groq":
|
||||
if _HAS_OPENAI and os.getenv("GROQ_API_KEY"):
|
||||
return "groq"
|
||||
logger.warning(
|
||||
"STT provider 'groq' configured but GROQ_API_KEY not set"
|
||||
)
|
||||
return "none"
|
||||
|
||||
if provider == "openai":
|
||||
if _HAS_OPENAI and _has_openai_audio_backend():
|
||||
return "openai"
|
||||
logger.warning(
|
||||
"STT provider 'openai' configured but no API key available"
|
||||
)
|
||||
return "none"
|
||||
|
||||
if provider == "mistral":
|
||||
if _HAS_MISTRAL and os.getenv("MISTRAL_API_KEY"):
|
||||
return "mistral"
|
||||
logger.warning(
|
||||
"STT provider 'mistral' configured but mistralai package "
|
||||
"not installed or MISTRAL_API_KEY not set"
|
||||
)
|
||||
return "none"
|
||||
|
||||
return provider # Unknown — let it fail downstream
|
||||
|
||||
# --- Auto-detect (no explicit provider): local > groq > openai > mistral -
|
||||
|
||||
if _HAS_FASTER_WHISPER:
|
||||
return "local"
|
||||
if _has_local_command():
|
||||
return "local_command"
|
||||
if _HAS_OPENAI and os.getenv("GROQ_API_KEY"):
|
||||
logger.info("No local STT available, using Groq Whisper API")
|
||||
return "groq"
|
||||
if _HAS_OPENAI and _has_openai_audio_backend():
|
||||
logger.info("No local STT available, using OpenAI Whisper API")
|
||||
return "openai"
|
||||
if _HAS_MISTRAL and os.getenv("MISTRAL_API_KEY"):
|
||||
logger.info("No local STT available, using Mistral Voxtral Transcribe API")
|
||||
return "mistral"
|
||||
return "none"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shared validation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _validate_audio_file(file_path: str) -> Optional[Dict[str, Any]]:
|
||||
"""Validate the audio file. Returns an error dict or None if OK."""
|
||||
audio_path = Path(file_path)
|
||||
|
||||
if not audio_path.exists():
|
||||
return {"success": False, "transcript": "", "error": f"Audio file not found: {file_path}"}
|
||||
if not audio_path.is_file():
|
||||
return {"success": False, "transcript": "", "error": f"Path is not a file: {file_path}"}
|
||||
if audio_path.suffix.lower() not in SUPPORTED_FORMATS:
|
||||
return {
|
||||
"success": False,
|
||||
"transcript": "",
|
||||
"error": f"Unsupported format: {audio_path.suffix}. Supported: {', '.join(sorted(SUPPORTED_FORMATS))}",
|
||||
}
|
||||
try:
|
||||
file_size = audio_path.stat().st_size
|
||||
if file_size > MAX_FILE_SIZE:
|
||||
return {
|
||||
"success": False,
|
||||
"transcript": "",
|
||||
"error": f"File too large: {file_size / (1024*1024):.1f}MB (max {MAX_FILE_SIZE / (1024*1024):.0f}MB)",
|
||||
}
|
||||
except OSError as e:
|
||||
return {"success": False, "transcript": "", "error": f"Failed to access file: {e}"}
|
||||
|
||||
return None
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Provider: local (faster-whisper)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _transcribe_local(file_path: str, model_name: str) -> Dict[str, Any]:
|
||||
"""Transcribe using faster-whisper (local, free)."""
|
||||
global _local_model, _local_model_name
|
||||
|
||||
if not _HAS_FASTER_WHISPER:
|
||||
return {"success": False, "transcript": "", "error": "faster-whisper not installed"}
|
||||
|
||||
try:
|
||||
from faster_whisper import WhisperModel
|
||||
# Lazy-load the model (downloads on first use, ~150 MB for 'base')
|
||||
if _local_model is None or _local_model_name != model_name:
|
||||
logger.info("Loading faster-whisper model '%s' (first load downloads the model)...", model_name)
|
||||
_local_model = WhisperModel(model_name, device="auto", compute_type="auto")
|
||||
_local_model_name = model_name
|
||||
|
||||
# Language: config.yaml (stt.local.language) > env var > auto-detect.
|
||||
_forced_lang = (
|
||||
_load_stt_config().get("local", {}).get("language")
|
||||
or os.getenv(LOCAL_STT_LANGUAGE_ENV)
|
||||
or None
|
||||
)
|
||||
transcribe_kwargs = {"beam_size": 5}
|
||||
if _forced_lang:
|
||||
transcribe_kwargs["language"] = _forced_lang
|
||||
|
||||
segments, info = _local_model.transcribe(file_path, **transcribe_kwargs)
|
||||
transcript = " ".join(segment.text.strip() for segment in segments)
|
||||
|
||||
logger.info(
|
||||
"Transcribed %s via local whisper (%s, lang=%s, %.1fs audio)",
|
||||
Path(file_path).name, model_name, info.language, info.duration,
|
||||
)
|
||||
|
||||
return {"success": True, "transcript": transcript, "provider": "local"}
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Local transcription failed: %s", e, exc_info=True)
|
||||
return {"success": False, "transcript": "", "error": f"Local transcription failed: {e}"}
|
||||
|
||||
|
||||
def _prepare_local_audio(file_path: str, work_dir: str) -> tuple[Optional[str], Optional[str]]:
|
||||
"""Normalize audio for local CLI STT when needed."""
|
||||
audio_path = Path(file_path)
|
||||
if audio_path.suffix.lower() in LOCAL_NATIVE_AUDIO_FORMATS:
|
||||
return file_path, None
|
||||
|
||||
ffmpeg = _find_ffmpeg_binary()
|
||||
if not ffmpeg:
|
||||
return None, "Local STT fallback requires ffmpeg for non-WAV inputs, but ffmpeg was not found"
|
||||
|
||||
converted_path = os.path.join(work_dir, f"{audio_path.stem}.wav")
|
||||
command = [ffmpeg, "-y", "-i", file_path, converted_path]
|
||||
|
||||
try:
|
||||
subprocess.run(command, check=True, capture_output=True, text=True)
|
||||
return converted_path, None
|
||||
except subprocess.CalledProcessError as e:
|
||||
details = e.stderr.strip() or e.stdout.strip() or str(e)
|
||||
logger.error("ffmpeg conversion failed for %s: %s", file_path, details)
|
||||
return None, f"Failed to convert audio for local STT: {details}"
|
||||
|
||||
|
||||
def _transcribe_local_command(file_path: str, model_name: str) -> Dict[str, Any]:
|
||||
"""Run the configured local STT command template and read back a .txt transcript."""
|
||||
command_template = _get_local_command_template()
|
||||
if not command_template:
|
||||
return {
|
||||
"success": False,
|
||||
"transcript": "",
|
||||
"error": (
|
||||
f"{LOCAL_STT_COMMAND_ENV} not configured and no local whisper binary was found"
|
||||
),
|
||||
}
|
||||
|
||||
# Language: config.yaml (stt.local.language) > env var > "en" default.
|
||||
language = (
|
||||
_load_stt_config().get("local", {}).get("language")
|
||||
or os.getenv(LOCAL_STT_LANGUAGE_ENV)
|
||||
or DEFAULT_LOCAL_STT_LANGUAGE
|
||||
)
|
||||
normalized_model = _normalize_local_command_model(model_name)
|
||||
|
||||
try:
|
||||
with tempfile.TemporaryDirectory(prefix="hermes-local-stt-") as output_dir:
|
||||
prepared_input, prep_error = _prepare_local_audio(file_path, output_dir)
|
||||
if prep_error:
|
||||
return {"success": False, "transcript": "", "error": prep_error}
|
||||
|
||||
command = command_template.format(
|
||||
input_path=shlex.quote(prepared_input),
|
||||
output_dir=shlex.quote(output_dir),
|
||||
language=shlex.quote(language),
|
||||
model=shlex.quote(normalized_model),
|
||||
)
|
||||
subprocess.run(command, shell=True, check=True, capture_output=True, text=True)
|
||||
|
||||
txt_files = sorted(Path(output_dir).glob("*.txt"))
|
||||
if not txt_files:
|
||||
return {
|
||||
"success": False,
|
||||
"transcript": "",
|
||||
"error": "Local STT command completed but did not produce a .txt transcript",
|
||||
}
|
||||
|
||||
transcript_text = txt_files[0].read_text(encoding="utf-8").strip()
|
||||
logger.info(
|
||||
"Transcribed %s via local STT command (%s, %d chars)",
|
||||
Path(file_path).name,
|
||||
normalized_model,
|
||||
len(transcript_text),
|
||||
)
|
||||
return {"success": True, "transcript": transcript_text, "provider": "local_command"}
|
||||
|
||||
except KeyError as e:
|
||||
return {
|
||||
"success": False,
|
||||
"transcript": "",
|
||||
"error": f"Invalid {LOCAL_STT_COMMAND_ENV} template, missing placeholder: {e}",
|
||||
}
|
||||
except subprocess.CalledProcessError as e:
|
||||
details = e.stderr.strip() or e.stdout.strip() or str(e)
|
||||
logger.error("Local STT command failed for %s: %s", file_path, details)
|
||||
return {"success": False, "transcript": "", "error": f"Local STT failed: {details}"}
|
||||
except Exception as e:
|
||||
logger.error("Unexpected error during local command transcription: %s", e, exc_info=True)
|
||||
return {"success": False, "transcript": "", "error": f"Local transcription failed: {e}"}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Provider: groq (Whisper API — free tier)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _transcribe_groq(file_path: str, model_name: str) -> Dict[str, Any]:
|
||||
"""Transcribe using Groq Whisper API (free tier available)."""
|
||||
api_key = os.getenv("GROQ_API_KEY")
|
||||
if not api_key:
|
||||
return {"success": False, "transcript": "", "error": "GROQ_API_KEY not set"}
|
||||
|
||||
if not _HAS_OPENAI:
|
||||
return {"success": False, "transcript": "", "error": "openai package not installed"}
|
||||
|
||||
# Auto-correct model if caller passed an OpenAI-only model
|
||||
if model_name in OPENAI_MODELS:
|
||||
logger.info("Model %s not available on Groq, using %s", model_name, DEFAULT_GROQ_STT_MODEL)
|
||||
model_name = DEFAULT_GROQ_STT_MODEL
|
||||
|
||||
try:
|
||||
from openai import OpenAI, APIError, APIConnectionError, APITimeoutError
|
||||
client = OpenAI(api_key=api_key, base_url=GROQ_BASE_URL, timeout=30, max_retries=0)
|
||||
try:
|
||||
with open(file_path, "rb") as audio_file:
|
||||
transcription = client.audio.transcriptions.create(
|
||||
model=model_name,
|
||||
file=audio_file,
|
||||
response_format="text",
|
||||
)
|
||||
|
||||
transcript_text = str(transcription).strip()
|
||||
logger.info("Transcribed %s via Groq API (%s, %d chars)",
|
||||
Path(file_path).name, model_name, len(transcript_text))
|
||||
|
||||
return {"success": True, "transcript": transcript_text, "provider": "groq"}
|
||||
finally:
|
||||
close = getattr(client, "close", None)
|
||||
if callable(close):
|
||||
close()
|
||||
|
||||
except PermissionError:
|
||||
return {"success": False, "transcript": "", "error": f"Permission denied: {file_path}"}
|
||||
except APIConnectionError as e:
|
||||
return {"success": False, "transcript": "", "error": f"Connection error: {e}"}
|
||||
except APITimeoutError as e:
|
||||
return {"success": False, "transcript": "", "error": f"Request timeout: {e}"}
|
||||
except APIError as e:
|
||||
return {"success": False, "transcript": "", "error": f"API error: {e}"}
|
||||
except Exception as e:
|
||||
logger.error("Groq transcription failed: %s", e, exc_info=True)
|
||||
return {"success": False, "transcript": "", "error": f"Transcription failed: {e}"}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Provider: openai (Whisper API)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _transcribe_openai(file_path: str, model_name: str) -> Dict[str, Any]:
|
||||
"""Transcribe using OpenAI Whisper API (paid)."""
|
||||
try:
|
||||
api_key, base_url = _resolve_openai_audio_client_config()
|
||||
except ValueError as exc:
|
||||
return {
|
||||
"success": False,
|
||||
"transcript": "",
|
||||
"error": str(exc),
|
||||
}
|
||||
|
||||
if not _HAS_OPENAI:
|
||||
return {"success": False, "transcript": "", "error": "openai package not installed"}
|
||||
|
||||
# Auto-correct model if caller passed a Groq-only model
|
||||
if model_name in GROQ_MODELS:
|
||||
logger.info("Model %s not available on OpenAI, using %s", model_name, DEFAULT_STT_MODEL)
|
||||
model_name = DEFAULT_STT_MODEL
|
||||
|
||||
try:
|
||||
from openai import OpenAI, APIError, APIConnectionError, APITimeoutError
|
||||
client = OpenAI(api_key=api_key, base_url=base_url, timeout=30, max_retries=0)
|
||||
try:
|
||||
with open(file_path, "rb") as audio_file:
|
||||
transcription = client.audio.transcriptions.create(
|
||||
model=model_name,
|
||||
file=audio_file,
|
||||
response_format="text" if model_name == "whisper-1" else "json",
|
||||
)
|
||||
|
||||
transcript_text = _extract_transcript_text(transcription)
|
||||
logger.info("Transcribed %s via OpenAI API (%s, %d chars)",
|
||||
Path(file_path).name, model_name, len(transcript_text))
|
||||
|
||||
return {"success": True, "transcript": transcript_text, "provider": "openai"}
|
||||
finally:
|
||||
close = getattr(client, "close", None)
|
||||
if callable(close):
|
||||
close()
|
||||
|
||||
except PermissionError:
|
||||
return {"success": False, "transcript": "", "error": f"Permission denied: {file_path}"}
|
||||
except APIConnectionError as e:
|
||||
return {"success": False, "transcript": "", "error": f"Connection error: {e}"}
|
||||
except APITimeoutError as e:
|
||||
return {"success": False, "transcript": "", "error": f"Request timeout: {e}"}
|
||||
except APIError as e:
|
||||
return {"success": False, "transcript": "", "error": f"API error: {e}"}
|
||||
except Exception as e:
|
||||
logger.error("OpenAI transcription failed: %s", e, exc_info=True)
|
||||
return {"success": False, "transcript": "", "error": f"Transcription failed: {e}"}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Provider: mistral (Voxtral Transcribe API)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _transcribe_mistral(file_path: str, model_name: str) -> Dict[str, Any]:
|
||||
"""Transcribe using Mistral Voxtral Transcribe API.
|
||||
|
||||
Uses the ``mistralai`` Python SDK to call ``/v1/audio/transcriptions``.
|
||||
Requires ``MISTRAL_API_KEY`` environment variable.
|
||||
"""
|
||||
api_key = os.getenv("MISTRAL_API_KEY")
|
||||
if not api_key:
|
||||
return {"success": False, "transcript": "", "error": "MISTRAL_API_KEY not set"}
|
||||
|
||||
try:
|
||||
from mistralai.client import Mistral
|
||||
|
||||
with Mistral(api_key=api_key) as client:
|
||||
with open(file_path, "rb") as audio_file:
|
||||
result = client.audio.transcriptions.complete(
|
||||
model=model_name,
|
||||
file={"content": audio_file, "file_name": Path(file_path).name},
|
||||
)
|
||||
|
||||
transcript_text = _extract_transcript_text(result)
|
||||
logger.info(
|
||||
"Transcribed %s via Mistral API (%s, %d chars)",
|
||||
Path(file_path).name, model_name, len(transcript_text),
|
||||
)
|
||||
return {"success": True, "transcript": transcript_text, "provider": "mistral"}
|
||||
|
||||
except PermissionError:
|
||||
return {"success": False, "transcript": "", "error": f"Permission denied: {file_path}"}
|
||||
except Exception as e:
|
||||
logger.error("Mistral transcription failed: %s", e, exc_info=True)
|
||||
return {"success": False, "transcript": "", "error": f"Mistral transcription failed: {type(e).__name__}"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def transcribe_audio(file_path: str, model: Optional[str] = None) -> Dict[str, Any]:
|
||||
"""
|
||||
Transcribe an audio file using the configured STT provider.
|
||||
|
||||
Provider priority:
|
||||
1. User config (``stt.provider`` in config.yaml)
|
||||
2. Auto-detect: local faster-whisper (free) > Groq (free tier) > OpenAI (paid)
|
||||
|
||||
Args:
|
||||
file_path: Absolute path to the audio file to transcribe.
|
||||
model: Override the model. If None, uses config or provider default.
|
||||
|
||||
Returns:
|
||||
dict with keys:
|
||||
- "success" (bool): Whether transcription succeeded
|
||||
- "transcript" (str): The transcribed text (empty on failure)
|
||||
- "error" (str, optional): Error message if success is False
|
||||
- "provider" (str, optional): Which provider was used
|
||||
"""
|
||||
# Validate input
|
||||
error = _validate_audio_file(file_path)
|
||||
if error:
|
||||
return error
|
||||
|
||||
# Load config and determine provider
|
||||
stt_config = _load_stt_config()
|
||||
if not is_stt_enabled(stt_config):
|
||||
return {
|
||||
"success": False,
|
||||
"transcript": "",
|
||||
"error": "STT is disabled in config.yaml (stt.enabled: false).",
|
||||
}
|
||||
|
||||
provider = _get_provider(stt_config)
|
||||
|
||||
if provider == "local":
|
||||
local_cfg = stt_config.get("local", {})
|
||||
model_name = model or local_cfg.get("model", DEFAULT_LOCAL_MODEL)
|
||||
return _transcribe_local(file_path, model_name)
|
||||
|
||||
if provider == "local_command":
|
||||
local_cfg = stt_config.get("local", {})
|
||||
model_name = _normalize_local_command_model(
|
||||
model or local_cfg.get("model", DEFAULT_LOCAL_MODEL)
|
||||
)
|
||||
return _transcribe_local_command(file_path, model_name)
|
||||
|
||||
if provider == "groq":
|
||||
model_name = model or DEFAULT_GROQ_STT_MODEL
|
||||
return _transcribe_groq(file_path, model_name)
|
||||
|
||||
if provider == "openai":
|
||||
openai_cfg = stt_config.get("openai", {})
|
||||
model_name = model or openai_cfg.get("model", DEFAULT_STT_MODEL)
|
||||
return _transcribe_openai(file_path, model_name)
|
||||
|
||||
if provider == "mistral":
|
||||
mistral_cfg = stt_config.get("mistral", {})
|
||||
model_name = model or mistral_cfg.get("model", DEFAULT_MISTRAL_STT_MODEL)
|
||||
return _transcribe_mistral(file_path, model_name)
|
||||
|
||||
# No provider available
|
||||
return {
|
||||
"success": False,
|
||||
"transcript": "",
|
||||
"error": (
|
||||
"No STT provider available. Install faster-whisper for free local "
|
||||
f"transcription, configure {LOCAL_STT_COMMAND_ENV} or install a local whisper CLI, "
|
||||
"set GROQ_API_KEY for free Groq Whisper, set MISTRAL_API_KEY for Mistral "
|
||||
"Voxtral Transcribe, or set VOICE_TOOLS_OPENAI_KEY "
|
||||
"or OPENAI_API_KEY for the OpenAI Whisper API."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _resolve_openai_audio_client_config() -> tuple[str, str]:
|
||||
"""Return direct OpenAI audio config or a managed gateway fallback."""
|
||||
stt_config = _load_stt_config()
|
||||
openai_cfg = stt_config.get("openai", {})
|
||||
cfg_api_key = openai_cfg.get("api_key", "")
|
||||
cfg_base_url = openai_cfg.get("base_url", "")
|
||||
if cfg_api_key:
|
||||
return cfg_api_key, (cfg_base_url or OPENAI_BASE_URL)
|
||||
|
||||
direct_api_key = resolve_openai_audio_api_key()
|
||||
if direct_api_key:
|
||||
return direct_api_key, OPENAI_BASE_URL
|
||||
|
||||
managed_gateway = resolve_managed_tool_gateway("openai-audio")
|
||||
if managed_gateway is None:
|
||||
message = "Neither stt.openai.api_key in config nor VOICE_TOOLS_OPENAI_KEY/OPENAI_API_KEY is set"
|
||||
if managed_nous_tools_enabled():
|
||||
message += ", and the managed OpenAI audio gateway is unavailable"
|
||||
raise ValueError(message)
|
||||
|
||||
return managed_gateway.nous_user_token, urljoin(
|
||||
f"{managed_gateway.gateway_origin.rstrip('/')}/", "v1"
|
||||
)
|
||||
|
||||
|
||||
def _extract_transcript_text(transcription: Any) -> str:
|
||||
"""Normalize text and JSON transcription responses to a plain string."""
|
||||
if isinstance(transcription, str):
|
||||
return transcription.strip()
|
||||
|
||||
if hasattr(transcription, "text"):
|
||||
value = getattr(transcription, "text")
|
||||
if isinstance(value, str):
|
||||
return value.strip()
|
||||
|
||||
if isinstance(transcription, dict):
|
||||
value = transcription.get("text")
|
||||
if isinstance(value, str):
|
||||
return value.strip()
|
||||
|
||||
return str(transcription).strip()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,97 @@
|
||||
"""URL safety checks — blocks requests to private/internal network addresses.
|
||||
|
||||
Prevents SSRF (Server-Side Request Forgery) where a malicious prompt or
|
||||
skill could trick the agent into fetching internal resources like cloud
|
||||
metadata endpoints (169.254.169.254), localhost services, or private
|
||||
network hosts.
|
||||
|
||||
Limitations (documented, not fixable at pre-flight level):
|
||||
- DNS rebinding (TOCTOU): an attacker-controlled DNS server with TTL=0
|
||||
can return a public IP for the check, then a private IP for the actual
|
||||
connection. Fixing this requires connection-level validation (e.g.
|
||||
Python's Champion library or an egress proxy like Stripe's Smokescreen).
|
||||
- Redirect-based bypass is mitigated by httpx event hooks that re-validate
|
||||
each redirect target in vision_tools, gateway platform adapters, and
|
||||
media cache helpers. Web tools use third-party SDKs (Firecrawl/Tavily)
|
||||
where redirect handling is on their servers.
|
||||
"""
|
||||
|
||||
import ipaddress
|
||||
import logging
|
||||
import socket
|
||||
from urllib.parse import urlparse
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Hostnames that should always be blocked regardless of IP resolution
|
||||
_BLOCKED_HOSTNAMES = frozenset({
|
||||
"metadata.google.internal",
|
||||
"metadata.goog",
|
||||
})
|
||||
|
||||
# 100.64.0.0/10 (CGNAT / Shared Address Space, RFC 6598) is NOT covered by
|
||||
# ipaddress.is_private — it returns False for both is_private and is_global.
|
||||
# Must be blocked explicitly. Used by carrier-grade NAT, Tailscale/WireGuard
|
||||
# VPNs, and some cloud internal networks.
|
||||
_CGNAT_NETWORK = ipaddress.ip_network("100.64.0.0/10")
|
||||
|
||||
|
||||
def _is_blocked_ip(ip: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
|
||||
"""Return True if the IP should be blocked for SSRF protection."""
|
||||
if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved:
|
||||
return True
|
||||
if ip.is_multicast or ip.is_unspecified:
|
||||
return True
|
||||
# CGNAT range not covered by is_private
|
||||
if ip in _CGNAT_NETWORK:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_safe_url(url: str) -> bool:
|
||||
"""Return True if the URL target is not a private/internal address.
|
||||
|
||||
Resolves the hostname to an IP and checks against private ranges.
|
||||
Fails closed: DNS errors and unexpected exceptions block the request.
|
||||
"""
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
hostname = (parsed.hostname or "").strip().lower()
|
||||
if not hostname:
|
||||
return False
|
||||
|
||||
# Block known internal hostnames
|
||||
if hostname in _BLOCKED_HOSTNAMES:
|
||||
logger.warning("Blocked request to internal hostname: %s", hostname)
|
||||
return False
|
||||
|
||||
# Try to resolve and check IP
|
||||
try:
|
||||
addr_info = socket.getaddrinfo(hostname, None, socket.AF_UNSPEC, socket.SOCK_STREAM)
|
||||
except socket.gaierror:
|
||||
# DNS resolution failed — fail closed. If DNS can't resolve it,
|
||||
# the HTTP client will also fail, so blocking loses nothing.
|
||||
logger.warning("Blocked request — DNS resolution failed for: %s", hostname)
|
||||
return False
|
||||
|
||||
for family, _, _, _, sockaddr in addr_info:
|
||||
ip_str = sockaddr[0]
|
||||
try:
|
||||
ip = ipaddress.ip_address(ip_str)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
if _is_blocked_ip(ip):
|
||||
logger.warning(
|
||||
"Blocked request to private/internal address: %s -> %s",
|
||||
hostname, ip_str,
|
||||
)
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
except Exception as exc:
|
||||
# Fail closed on unexpected errors — don't let parsing edge cases
|
||||
# become SSRF bypass vectors
|
||||
logger.warning("Blocked request — URL safety check error for %s: %s", url, exc)
|
||||
return False
|
||||
@@ -0,0 +1,789 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Vision Tools Module
|
||||
|
||||
This module provides vision analysis tools that work with image URLs.
|
||||
Uses the centralized auxiliary vision router, which can select OpenRouter,
|
||||
Nous, Codex, native Anthropic, or a custom OpenAI-compatible endpoint.
|
||||
|
||||
Available tools:
|
||||
- vision_analyze_tool: Analyze images from URLs with custom prompts
|
||||
|
||||
Features:
|
||||
- Downloads images from URLs and converts to base64 for API compatibility
|
||||
- Comprehensive image description
|
||||
- Context-aware analysis based on user queries
|
||||
- Automatic temporary file cleanup
|
||||
- Proper error handling and validation
|
||||
- Debug logging support
|
||||
|
||||
Usage:
|
||||
from vision_tools import vision_analyze_tool
|
||||
import asyncio
|
||||
|
||||
# Analyze an image
|
||||
result = await vision_analyze_tool(
|
||||
image_url="https://example.com/image.jpg",
|
||||
user_prompt="What architectural style is this building?"
|
||||
)
|
||||
"""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any, Awaitable, Dict, Optional
|
||||
from urllib.parse import urlparse
|
||||
import httpx
|
||||
from agent.auxiliary_client import async_call_llm, extract_content_or_reasoning
|
||||
from tools.debug_helpers import DebugSession
|
||||
from tools.website_policy import check_website_access
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_debug = DebugSession("vision_tools", env_var="VISION_TOOLS_DEBUG")
|
||||
|
||||
# Configurable HTTP download timeout for _download_image().
|
||||
# Separate from auxiliary.vision.timeout which governs the LLM API call.
|
||||
# Resolution: config.yaml auxiliary.vision.download_timeout → env var → 30s default.
|
||||
def _resolve_download_timeout() -> float:
|
||||
env_val = os.getenv("HERMES_VISION_DOWNLOAD_TIMEOUT", "").strip()
|
||||
if env_val:
|
||||
try:
|
||||
return float(env_val)
|
||||
except ValueError:
|
||||
pass
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
cfg = load_config()
|
||||
val = cfg.get("auxiliary", {}).get("vision", {}).get("download_timeout")
|
||||
if val is not None:
|
||||
return float(val)
|
||||
except Exception:
|
||||
pass
|
||||
return 30.0
|
||||
|
||||
_VISION_DOWNLOAD_TIMEOUT = _resolve_download_timeout()
|
||||
|
||||
# Hard cap on downloaded image file size (50 MB). Prevents OOM from
|
||||
# attacker-hosted multi-gigabyte files or decompression bombs.
|
||||
_VISION_MAX_DOWNLOAD_BYTES = 50 * 1024 * 1024
|
||||
|
||||
|
||||
def _validate_image_url(url: str) -> bool:
|
||||
"""
|
||||
Basic validation of image URL format.
|
||||
|
||||
Args:
|
||||
url (str): The URL to validate
|
||||
|
||||
Returns:
|
||||
bool: True if URL appears to be valid, False otherwise
|
||||
"""
|
||||
if not url or not isinstance(url, str):
|
||||
return False
|
||||
|
||||
# Basic HTTP/HTTPS URL check
|
||||
if not url.startswith(("http://", "https://")):
|
||||
return False
|
||||
|
||||
# Parse to ensure we at least have a network location; still allow URLs
|
||||
# without file extensions (e.g. CDN endpoints that redirect to images).
|
||||
parsed = urlparse(url)
|
||||
if not parsed.netloc:
|
||||
return False
|
||||
|
||||
# Block private/internal addresses to prevent SSRF
|
||||
from tools.url_safety import is_safe_url
|
||||
if not is_safe_url(url):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def _detect_image_mime_type(image_path: Path) -> Optional[str]:
|
||||
"""Return a MIME type when the file looks like a supported image."""
|
||||
with image_path.open("rb") as f:
|
||||
header = f.read(64)
|
||||
|
||||
if header.startswith(b"\x89PNG\r\n\x1a\n"):
|
||||
return "image/png"
|
||||
if header.startswith(b"\xff\xd8\xff"):
|
||||
return "image/jpeg"
|
||||
if header.startswith((b"GIF87a", b"GIF89a")):
|
||||
return "image/gif"
|
||||
if header.startswith(b"BM"):
|
||||
return "image/bmp"
|
||||
if len(header) >= 12 and header[:4] == b"RIFF" and header[8:12] == b"WEBP":
|
||||
return "image/webp"
|
||||
if image_path.suffix.lower() == ".svg":
|
||||
head = image_path.read_text(encoding="utf-8", errors="ignore")[:4096].lower()
|
||||
if "<svg" in head:
|
||||
return "image/svg+xml"
|
||||
return None
|
||||
|
||||
|
||||
async def _download_image(image_url: str, destination: Path, max_retries: int = 3) -> Path:
|
||||
"""
|
||||
Download an image from a URL to a local destination (async) with retry logic.
|
||||
|
||||
Args:
|
||||
image_url (str): The URL of the image to download
|
||||
destination (Path): The path where the image should be saved
|
||||
max_retries (int): Maximum number of retry attempts (default: 3)
|
||||
|
||||
Returns:
|
||||
Path: The path to the downloaded image
|
||||
|
||||
Raises:
|
||||
Exception: If download fails after all retries
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
# Create parent directories if they don't exist
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
async def _ssrf_redirect_guard(response):
|
||||
"""Re-validate each redirect target to prevent redirect-based SSRF.
|
||||
|
||||
Without this, an attacker can host a public URL that 302-redirects
|
||||
to http://169.254.169.254/ and bypass the pre-flight is_safe_url check.
|
||||
|
||||
Must be async because httpx.AsyncClient awaits event hooks.
|
||||
"""
|
||||
if response.is_redirect and response.next_request:
|
||||
redirect_url = str(response.next_request.url)
|
||||
from tools.url_safety import is_safe_url
|
||||
if not is_safe_url(redirect_url):
|
||||
raise ValueError(
|
||||
f"Blocked redirect to private/internal address: {redirect_url}"
|
||||
)
|
||||
|
||||
last_error = None
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
blocked = check_website_access(image_url)
|
||||
if blocked:
|
||||
raise PermissionError(blocked["message"])
|
||||
|
||||
# Download the image with appropriate headers using async httpx
|
||||
# Enable follow_redirects to handle image CDNs that redirect (e.g., Imgur, Picsum)
|
||||
# SSRF: event_hooks validates each redirect target against private IP ranges
|
||||
async with httpx.AsyncClient(
|
||||
timeout=_VISION_DOWNLOAD_TIMEOUT,
|
||||
follow_redirects=True,
|
||||
event_hooks={"response": [_ssrf_redirect_guard]},
|
||||
) as client:
|
||||
response = await client.get(
|
||||
image_url,
|
||||
headers={
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Accept": "image/*,*/*;q=0.8",
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
# Reject overly large images early via Content-Length header.
|
||||
cl = response.headers.get("content-length")
|
||||
if cl and int(cl) > _VISION_MAX_DOWNLOAD_BYTES:
|
||||
raise ValueError(
|
||||
f"Image too large ({int(cl)} bytes, max {_VISION_MAX_DOWNLOAD_BYTES})"
|
||||
)
|
||||
|
||||
final_url = str(response.url)
|
||||
blocked = check_website_access(final_url)
|
||||
if blocked:
|
||||
raise PermissionError(blocked["message"])
|
||||
|
||||
# Save the image content (double-check actual size)
|
||||
body = response.content
|
||||
if len(body) > _VISION_MAX_DOWNLOAD_BYTES:
|
||||
raise ValueError(
|
||||
f"Image too large ({len(body)} bytes, max {_VISION_MAX_DOWNLOAD_BYTES})"
|
||||
)
|
||||
destination.write_bytes(body)
|
||||
|
||||
return destination
|
||||
except Exception as e:
|
||||
last_error = e
|
||||
if attempt < max_retries - 1:
|
||||
wait_time = 2 ** (attempt + 1) # 2s, 4s, 8s
|
||||
logger.warning("Image download failed (attempt %s/%s): %s", attempt + 1, max_retries, str(e)[:50])
|
||||
logger.warning("Retrying in %ss...", wait_time)
|
||||
await asyncio.sleep(wait_time)
|
||||
else:
|
||||
logger.error(
|
||||
"Image download failed after %s attempts: %s",
|
||||
max_retries,
|
||||
str(e)[:100],
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
if last_error is None:
|
||||
raise RuntimeError(
|
||||
f"_download_image exited retry loop without attempting (max_retries={max_retries})"
|
||||
)
|
||||
raise last_error
|
||||
|
||||
|
||||
def _determine_mime_type(image_path: Path) -> str:
|
||||
"""
|
||||
Determine the MIME type of an image based on its file extension.
|
||||
|
||||
Args:
|
||||
image_path (Path): Path to the image file
|
||||
|
||||
Returns:
|
||||
str: The MIME type (defaults to image/jpeg if unknown)
|
||||
"""
|
||||
extension = image_path.suffix.lower()
|
||||
mime_types = {
|
||||
'.jpg': 'image/jpeg',
|
||||
'.jpeg': 'image/jpeg',
|
||||
'.png': 'image/png',
|
||||
'.gif': 'image/gif',
|
||||
'.bmp': 'image/bmp',
|
||||
'.webp': 'image/webp',
|
||||
'.svg': 'image/svg+xml'
|
||||
}
|
||||
return mime_types.get(extension, 'image/jpeg')
|
||||
|
||||
|
||||
def _image_to_base64_data_url(image_path: Path, mime_type: Optional[str] = None) -> str:
|
||||
"""
|
||||
Convert an image file to a base64-encoded data URL.
|
||||
|
||||
Args:
|
||||
image_path (Path): Path to the image file
|
||||
mime_type (Optional[str]): MIME type of the image (auto-detected if None)
|
||||
|
||||
Returns:
|
||||
str: Base64-encoded data URL (e.g., "data:image/jpeg;base64,...")
|
||||
"""
|
||||
# Read the image as bytes
|
||||
data = image_path.read_bytes()
|
||||
|
||||
# Encode to base64
|
||||
encoded = base64.b64encode(data).decode("ascii")
|
||||
|
||||
# Determine MIME type
|
||||
mime = mime_type or _determine_mime_type(image_path)
|
||||
|
||||
# Create data URL
|
||||
data_url = f"data:{mime};base64,{encoded}"
|
||||
|
||||
return data_url
|
||||
|
||||
|
||||
# Hard limit for vision API payloads (20 MB) — matches the most restrictive
|
||||
# major provider (Gemini inline data limit). Images above this are rejected.
|
||||
_MAX_BASE64_BYTES = 20 * 1024 * 1024
|
||||
|
||||
# Target size when auto-resizing on API failure (5 MB). After a provider
|
||||
# rejects an image, we downscale to this target and retry once.
|
||||
_RESIZE_TARGET_BYTES = 5 * 1024 * 1024
|
||||
|
||||
|
||||
def _is_image_size_error(error: Exception) -> bool:
|
||||
"""Detect if an API error is related to image or payload size."""
|
||||
err_str = str(error).lower()
|
||||
return any(hint in err_str for hint in (
|
||||
"too large", "payload", "413", "content_too_large",
|
||||
"request_too_large", "image_url", "invalid_request",
|
||||
"exceeds", "size limit",
|
||||
))
|
||||
|
||||
|
||||
def _resize_image_for_vision(image_path: Path, mime_type: Optional[str] = None,
|
||||
max_base64_bytes: int = _RESIZE_TARGET_BYTES) -> str:
|
||||
"""Convert an image to a base64 data URL, auto-resizing if too large.
|
||||
|
||||
Tries Pillow first to progressively downscale oversized images. If Pillow
|
||||
is not installed or resizing still exceeds the limit, falls back to the raw
|
||||
bytes and lets the caller handle the size check.
|
||||
|
||||
Returns the base64 data URL string.
|
||||
"""
|
||||
# Quick file-size estimate: base64 expands by ~4/3, plus data URL header.
|
||||
# Skip the expensive full-read + encode if Pillow can resize directly.
|
||||
file_size = image_path.stat().st_size
|
||||
estimated_b64 = (file_size * 4) // 3 + 100 # ~header overhead
|
||||
if estimated_b64 <= max_base64_bytes:
|
||||
# Small enough — just encode directly.
|
||||
data_url = _image_to_base64_data_url(image_path, mime_type=mime_type)
|
||||
if len(data_url) <= max_base64_bytes:
|
||||
return data_url
|
||||
else:
|
||||
data_url = None # defer full encode; try Pillow resize first
|
||||
|
||||
# Attempt auto-resize with Pillow (soft dependency)
|
||||
try:
|
||||
from PIL import Image
|
||||
import io as _io
|
||||
except ImportError:
|
||||
logger.info("Pillow not installed — cannot auto-resize oversized image")
|
||||
if data_url is None:
|
||||
data_url = _image_to_base64_data_url(image_path, mime_type=mime_type)
|
||||
return data_url # caller will raise the size error
|
||||
|
||||
logger.info("Image file is %.1f MB (estimated base64 %.1f MB, limit %.1f MB), auto-resizing...",
|
||||
file_size / (1024 * 1024), estimated_b64 / (1024 * 1024),
|
||||
max_base64_bytes / (1024 * 1024))
|
||||
|
||||
mime = mime_type or _determine_mime_type(image_path)
|
||||
# Choose output format: JPEG for photos (smaller), PNG for transparency
|
||||
pil_format = "PNG" if mime == "image/png" else "JPEG"
|
||||
out_mime = "image/png" if pil_format == "PNG" else "image/jpeg"
|
||||
|
||||
try:
|
||||
img = Image.open(image_path)
|
||||
except Exception as exc:
|
||||
logger.info("Pillow cannot open image for resizing: %s", exc)
|
||||
if data_url is None:
|
||||
data_url = _image_to_base64_data_url(image_path, mime_type=mime_type)
|
||||
return data_url # fall through to size-check in caller
|
||||
# Convert RGBA to RGB for JPEG output
|
||||
if pil_format == "JPEG" and img.mode in ("RGBA", "P"):
|
||||
img = img.convert("RGB")
|
||||
|
||||
# Strategy: halve dimensions until base64 fits, up to 4 rounds.
|
||||
# For JPEG, also try reducing quality at each size step.
|
||||
# For PNG, quality is irrelevant — only dimension reduction helps.
|
||||
quality_steps = (85, 70, 50) if pil_format == "JPEG" else (None,)
|
||||
prev_dims = (img.width, img.height)
|
||||
candidate = None # will be set on first loop iteration
|
||||
|
||||
for attempt in range(5):
|
||||
if attempt > 0:
|
||||
# Proportional scaling: halve the longer side and scale the
|
||||
# shorter side to preserve aspect ratio (min dimension 64).
|
||||
scale = 0.5
|
||||
new_w = max(int(img.width * scale), 64)
|
||||
new_h = max(int(img.height * scale), 64)
|
||||
# Re-derive the scale from whichever dimension hit the floor
|
||||
# so both axes shrink by the same factor.
|
||||
if new_w == 64 and img.width > 0:
|
||||
effective_scale = 64 / img.width
|
||||
new_h = max(int(img.height * effective_scale), 64)
|
||||
elif new_h == 64 and img.height > 0:
|
||||
effective_scale = 64 / img.height
|
||||
new_w = max(int(img.width * effective_scale), 64)
|
||||
# Stop if dimensions can't shrink further
|
||||
if (new_w, new_h) == prev_dims:
|
||||
break
|
||||
img = img.resize((new_w, new_h), Image.LANCZOS)
|
||||
prev_dims = (new_w, new_h)
|
||||
logger.info("Resized to %dx%d (attempt %d)", new_w, new_h, attempt)
|
||||
|
||||
for q in quality_steps:
|
||||
buf = _io.BytesIO()
|
||||
save_kwargs = {"format": pil_format}
|
||||
if q is not None:
|
||||
save_kwargs["quality"] = q
|
||||
img.save(buf, **save_kwargs)
|
||||
encoded = base64.b64encode(buf.getvalue()).decode("ascii")
|
||||
candidate = f"data:{out_mime};base64,{encoded}"
|
||||
if len(candidate) <= max_base64_bytes:
|
||||
logger.info("Auto-resized image fits: %.1f MB (quality=%s, %dx%d)",
|
||||
len(candidate) / (1024 * 1024), q,
|
||||
img.width, img.height)
|
||||
return candidate
|
||||
|
||||
# If we still can't get it small enough, return the best attempt
|
||||
# and let the caller decide
|
||||
if candidate is not None:
|
||||
logger.warning("Auto-resize could not fit image under %.1f MB (best: %.1f MB)",
|
||||
max_base64_bytes / (1024 * 1024), len(candidate) / (1024 * 1024))
|
||||
return candidate
|
||||
|
||||
# Shouldn't reach here, but fall back to full encode
|
||||
return data_url or _image_to_base64_data_url(image_path, mime_type=mime_type)
|
||||
|
||||
|
||||
async def vision_analyze_tool(
|
||||
image_url: str,
|
||||
user_prompt: str,
|
||||
model: str = None,
|
||||
) -> str:
|
||||
"""
|
||||
Analyze an image from a URL or local file path using vision AI.
|
||||
|
||||
This tool accepts either an HTTP/HTTPS URL or a local file path. For URLs,
|
||||
it downloads the image first. In both cases, the image is converted to base64
|
||||
and processed using Gemini 3 Flash Preview via OpenRouter API.
|
||||
|
||||
The user_prompt parameter is expected to be pre-formatted by the calling
|
||||
function (typically model_tools.py) to include both full description
|
||||
requests and specific questions.
|
||||
|
||||
Args:
|
||||
image_url (str): The URL or local file path of the image to analyze.
|
||||
Accepts http://, https:// URLs or absolute/relative file paths.
|
||||
user_prompt (str): The pre-formatted prompt for the vision model
|
||||
model (str): The vision model to use (default: google/gemini-3-flash-preview)
|
||||
|
||||
Returns:
|
||||
str: JSON string containing the analysis results with the following structure:
|
||||
{
|
||||
"success": bool,
|
||||
"analysis": str (defaults to error message if None)
|
||||
}
|
||||
|
||||
Raises:
|
||||
Exception: If download fails, analysis fails, or API key is not set
|
||||
|
||||
Note:
|
||||
- For URLs, temporary images are stored in ./temp_vision_images/ and cleaned up
|
||||
- For local file paths, the file is used directly and NOT deleted
|
||||
- Supports common image formats (JPEG, PNG, GIF, WebP, etc.)
|
||||
"""
|
||||
debug_call_data = {
|
||||
"parameters": {
|
||||
"image_url": image_url,
|
||||
"user_prompt": user_prompt[:200] + "..." if len(user_prompt) > 200 else user_prompt,
|
||||
"model": model
|
||||
},
|
||||
"error": None,
|
||||
"success": False,
|
||||
"analysis_length": 0,
|
||||
"model_used": model,
|
||||
"image_size_bytes": 0
|
||||
}
|
||||
|
||||
temp_image_path = None
|
||||
# Track whether we should clean up the file after processing.
|
||||
# Local files (e.g. from the image cache) should NOT be deleted.
|
||||
should_cleanup = True
|
||||
detected_mime_type = None
|
||||
|
||||
try:
|
||||
from tools.interrupt import is_interrupted
|
||||
if is_interrupted():
|
||||
return tool_error("Interrupted", success=False)
|
||||
|
||||
logger.info("Analyzing image: %s", image_url[:60])
|
||||
logger.info("User prompt: %s", user_prompt[:100])
|
||||
|
||||
# Determine if this is a local file path or a remote URL
|
||||
# Strip file:// scheme so file URIs resolve as local paths.
|
||||
resolved_url = image_url
|
||||
if resolved_url.startswith("file://"):
|
||||
resolved_url = resolved_url[len("file://"):]
|
||||
local_path = Path(os.path.expanduser(resolved_url))
|
||||
if local_path.is_file():
|
||||
# Local file path (e.g. from platform image cache) -- skip download
|
||||
logger.info("Using local image file: %s", image_url)
|
||||
temp_image_path = local_path
|
||||
should_cleanup = False # Don't delete cached/local files
|
||||
elif _validate_image_url(image_url):
|
||||
# Remote URL -- download to a temporary location
|
||||
blocked = check_website_access(image_url)
|
||||
if blocked:
|
||||
raise PermissionError(blocked["message"])
|
||||
logger.info("Downloading image from URL...")
|
||||
temp_dir = Path("./temp_vision_images")
|
||||
temp_image_path = temp_dir / f"temp_image_{uuid.uuid4()}.jpg"
|
||||
await _download_image(image_url, temp_image_path)
|
||||
should_cleanup = True
|
||||
else:
|
||||
raise ValueError(
|
||||
"Invalid image source. Provide an HTTP/HTTPS URL or a valid local file path."
|
||||
)
|
||||
|
||||
# Get image file size for logging
|
||||
image_size_bytes = temp_image_path.stat().st_size
|
||||
image_size_kb = image_size_bytes / 1024
|
||||
logger.info("Image ready (%.1f KB)", image_size_kb)
|
||||
|
||||
detected_mime_type = _detect_image_mime_type(temp_image_path)
|
||||
if not detected_mime_type:
|
||||
raise ValueError("Only real image files are supported for vision analysis.")
|
||||
|
||||
# Convert image to base64 — send at full resolution first.
|
||||
# If the provider rejects it as too large, we auto-resize and retry.
|
||||
logger.info("Converting image to base64...")
|
||||
image_data_url = _image_to_base64_data_url(temp_image_path, mime_type=detected_mime_type)
|
||||
data_size_kb = len(image_data_url) / 1024
|
||||
logger.info("Image converted to base64 (%.1f KB)", data_size_kb)
|
||||
|
||||
# Hard limit (20 MB) — no provider accepts payloads this large.
|
||||
if len(image_data_url) > _MAX_BASE64_BYTES:
|
||||
# Try to resize down to 5 MB before giving up.
|
||||
image_data_url = _resize_image_for_vision(
|
||||
temp_image_path, mime_type=detected_mime_type)
|
||||
if len(image_data_url) > _MAX_BASE64_BYTES:
|
||||
raise ValueError(
|
||||
f"Image too large for vision API: base64 payload is "
|
||||
f"{len(image_data_url) / (1024 * 1024):.1f} MB "
|
||||
f"(limit {_MAX_BASE64_BYTES / (1024 * 1024):.0f} MB) "
|
||||
f"even after resizing. "
|
||||
f"Install Pillow (`pip install Pillow`) for better auto-resize, "
|
||||
f"or compress the image manually."
|
||||
)
|
||||
|
||||
debug_call_data["image_size_bytes"] = image_size_bytes
|
||||
|
||||
# Use the prompt as provided (model_tools.py now handles full description formatting)
|
||||
comprehensive_prompt = user_prompt
|
||||
|
||||
# Prepare the message with base64-encoded image
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": comprehensive_prompt
|
||||
},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": image_data_url
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
logger.info("Processing image with vision model...")
|
||||
|
||||
# Call the vision API via centralized router.
|
||||
# Read timeout from config.yaml (auxiliary.vision.timeout), default 120s.
|
||||
# Local vision models (llama.cpp, ollama) can take well over 30s.
|
||||
vision_timeout = 120.0
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
_cfg = load_config()
|
||||
_vt = _cfg.get("auxiliary", {}).get("vision", {}).get("timeout")
|
||||
if _vt is not None:
|
||||
vision_timeout = float(_vt)
|
||||
except Exception:
|
||||
pass
|
||||
call_kwargs = {
|
||||
"task": "vision",
|
||||
"messages": messages,
|
||||
"temperature": 0.1,
|
||||
"max_tokens": 2000,
|
||||
"timeout": vision_timeout,
|
||||
}
|
||||
if model:
|
||||
call_kwargs["model"] = model
|
||||
# Try full-size image first; on size-related rejection, downscale and retry.
|
||||
try:
|
||||
response = await async_call_llm(**call_kwargs)
|
||||
except Exception as _api_err:
|
||||
if (_is_image_size_error(_api_err)
|
||||
and len(image_data_url) > _RESIZE_TARGET_BYTES):
|
||||
logger.info(
|
||||
"API rejected image (%.1f MB, likely too large); "
|
||||
"auto-resizing to ~%.0f MB and retrying...",
|
||||
len(image_data_url) / (1024 * 1024),
|
||||
_RESIZE_TARGET_BYTES / (1024 * 1024),
|
||||
)
|
||||
image_data_url = _resize_image_for_vision(
|
||||
temp_image_path, mime_type=detected_mime_type)
|
||||
messages[0]["content"][1]["image_url"]["url"] = image_data_url
|
||||
response = await async_call_llm(**call_kwargs)
|
||||
else:
|
||||
raise
|
||||
|
||||
# Extract the analysis — fall back to reasoning if content is empty
|
||||
analysis = extract_content_or_reasoning(response)
|
||||
|
||||
# Retry once on empty content (reasoning-only response)
|
||||
if not analysis:
|
||||
logger.warning("Vision LLM returned empty content, retrying once")
|
||||
response = await async_call_llm(**call_kwargs)
|
||||
analysis = extract_content_or_reasoning(response)
|
||||
|
||||
analysis_length = len(analysis)
|
||||
|
||||
logger.info("Image analysis completed (%s characters)", analysis_length)
|
||||
|
||||
# Prepare successful response
|
||||
result = {
|
||||
"success": True,
|
||||
"analysis": analysis or "There was a problem with the request and the image could not be analyzed."
|
||||
}
|
||||
|
||||
debug_call_data["success"] = True
|
||||
debug_call_data["analysis_length"] = analysis_length
|
||||
|
||||
# Log debug information
|
||||
_debug.log_call("vision_analyze_tool", debug_call_data)
|
||||
_debug.save()
|
||||
|
||||
return json.dumps(result, indent=2, ensure_ascii=False)
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Error analyzing image: {str(e)}"
|
||||
logger.error("%s", error_msg, exc_info=True)
|
||||
|
||||
# Detect vision capability errors — give the model a clear message
|
||||
# so it can inform the user instead of a cryptic API error.
|
||||
err_str = str(e).lower()
|
||||
if any(hint in err_str for hint in (
|
||||
"402", "insufficient", "payment required", "credits", "billing",
|
||||
)):
|
||||
analysis = (
|
||||
"Insufficient credits or payment required. Please top up your "
|
||||
f"API provider account and try again. Error: {e}"
|
||||
)
|
||||
elif any(hint in err_str for hint in (
|
||||
"does not support", "not support image",
|
||||
"content_policy", "multimodal",
|
||||
"unrecognized request argument", "image input",
|
||||
)):
|
||||
analysis = (
|
||||
f"{model} does not support vision or our request was not "
|
||||
f"accepted by the server. Error: {e}"
|
||||
)
|
||||
elif "invalid_request" in err_str or "image_url" in err_str:
|
||||
analysis = (
|
||||
"The vision API rejected the image. This can happen when the "
|
||||
"image is in an unsupported format, corrupted, or still too "
|
||||
"large after auto-resize. Try a smaller JPEG/PNG and retry. "
|
||||
f"Error: {e}"
|
||||
)
|
||||
else:
|
||||
analysis = (
|
||||
"There was a problem with the request and the image could not "
|
||||
f"be analyzed. Error: {e}"
|
||||
)
|
||||
|
||||
# Prepare error response
|
||||
result = {
|
||||
"success": False,
|
||||
"error": error_msg,
|
||||
"analysis": analysis,
|
||||
}
|
||||
|
||||
debug_call_data["error"] = error_msg
|
||||
_debug.log_call("vision_analyze_tool", debug_call_data)
|
||||
_debug.save()
|
||||
|
||||
return json.dumps(result, indent=2, ensure_ascii=False)
|
||||
|
||||
finally:
|
||||
# Clean up temporary image file (but NOT local/cached files)
|
||||
if should_cleanup and temp_image_path and temp_image_path.exists():
|
||||
try:
|
||||
temp_image_path.unlink()
|
||||
logger.debug("Cleaned up temporary image file")
|
||||
except Exception as cleanup_error:
|
||||
logger.warning(
|
||||
"Could not delete temporary file: %s", cleanup_error, exc_info=True
|
||||
)
|
||||
|
||||
|
||||
def check_vision_requirements() -> bool:
|
||||
"""Check if the configured runtime vision path can resolve a client."""
|
||||
try:
|
||||
from agent.auxiliary_client import resolve_vision_provider_client
|
||||
|
||||
_provider, client, _model = resolve_vision_provider_client()
|
||||
return client is not None
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
"""
|
||||
Simple test/demo when run directly
|
||||
"""
|
||||
print("👁️ Vision Tools Module")
|
||||
print("=" * 40)
|
||||
|
||||
# Check if vision model is available
|
||||
api_available = check_vision_requirements()
|
||||
|
||||
if not api_available:
|
||||
print("❌ No auxiliary vision model available")
|
||||
print("Configure a supported multimodal backend (OpenRouter, Nous, Codex, Anthropic, or a custom OpenAI-compatible endpoint).")
|
||||
exit(1)
|
||||
else:
|
||||
print("✅ Vision model available")
|
||||
|
||||
print("🛠️ Vision tools ready for use!")
|
||||
|
||||
# Show debug mode status
|
||||
if _debug.active:
|
||||
print(f"🐛 Debug mode ENABLED - Session ID: {_debug.session_id}")
|
||||
print(f" Debug logs will be saved to: ./logs/vision_tools_debug_{_debug.session_id}.json")
|
||||
else:
|
||||
print("🐛 Debug mode disabled (set VISION_TOOLS_DEBUG=true to enable)")
|
||||
|
||||
print("\nBasic usage:")
|
||||
print(" from vision_tools import vision_analyze_tool")
|
||||
print(" import asyncio")
|
||||
print("")
|
||||
print(" async def main():")
|
||||
print(" result = await vision_analyze_tool(")
|
||||
print(" image_url='https://example.com/image.jpg',")
|
||||
print(" user_prompt='What do you see in this image?'")
|
||||
print(" )")
|
||||
print(" print(result)")
|
||||
print(" asyncio.run(main())")
|
||||
|
||||
print("\nExample prompts:")
|
||||
print(" - 'What architectural style is this building?'")
|
||||
print(" - 'Describe the emotions and mood in this image'")
|
||||
print(" - 'What text can you read in this image?'")
|
||||
print(" - 'Identify any safety hazards visible'")
|
||||
print(" - 'What products or brands are shown?'")
|
||||
|
||||
print("\nDebug mode:")
|
||||
print(" # Enable debug logging")
|
||||
print(" export VISION_TOOLS_DEBUG=true")
|
||||
print(" # Debug logs capture all vision analysis calls and results")
|
||||
print(" # Logs saved to: ./logs/vision_tools_debug_UUID.json")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Registry
|
||||
# ---------------------------------------------------------------------------
|
||||
from tools.registry import registry, tool_error
|
||||
|
||||
VISION_ANALYZE_SCHEMA = {
|
||||
"name": "vision_analyze",
|
||||
"description": "Analyze images using AI vision. Provides a comprehensive description and answers a specific question about the image content.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"image_url": {
|
||||
"type": "string",
|
||||
"description": "Image URL (http/https) or local file path to analyze."
|
||||
},
|
||||
"question": {
|
||||
"type": "string",
|
||||
"description": "Your specific question or request about the image to resolve. The AI will automatically provide a complete image description AND answer your specific question."
|
||||
}
|
||||
},
|
||||
"required": ["image_url", "question"]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def _handle_vision_analyze(args: Dict[str, Any], **kw: Any) -> Awaitable[str]:
|
||||
image_url = args.get("image_url", "")
|
||||
question = args.get("question", "")
|
||||
full_prompt = (
|
||||
"Fully describe and explain everything about this image, then answer the "
|
||||
f"following question:\n\n{question}"
|
||||
)
|
||||
model = os.getenv("AUXILIARY_VISION_MODEL", "").strip() or None
|
||||
return vision_analyze_tool(image_url, full_prompt, model)
|
||||
|
||||
|
||||
registry.register(
|
||||
name="vision_analyze",
|
||||
toolset="vision",
|
||||
schema=VISION_ANALYZE_SCHEMA,
|
||||
handler=_handle_vision_analyze,
|
||||
check_fn=check_vision_requirements,
|
||||
is_async=True,
|
||||
emoji="👁️",
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,282 @@
|
||||
"""Website access policy helpers for URL-capable tools.
|
||||
|
||||
This module loads a user-managed website blocklist from ~/.hermes/config.yaml
|
||||
and optional shared list files. It is intentionally lightweight so web/browser
|
||||
tools can enforce URL policy without pulling in the heavier CLI config stack.
|
||||
|
||||
Policy is cached in memory with a short TTL so config changes take effect
|
||||
quickly without re-reading the file on every URL check.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import fnmatch
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from hermes_constants import get_hermes_home
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_DEFAULT_WEBSITE_BLOCKLIST = {
|
||||
"enabled": False,
|
||||
"domains": [],
|
||||
"shared_files": [],
|
||||
}
|
||||
|
||||
# Cache: parsed policy + timestamp. Avoids re-reading config.yaml on every
|
||||
# URL check (a web_crawl with 50 pages would otherwise mean 51 YAML parses).
|
||||
_CACHE_TTL_SECONDS = 30.0
|
||||
_cache_lock = threading.Lock()
|
||||
_cached_policy: Optional[Dict[str, Any]] = None
|
||||
_cached_policy_path: Optional[str] = None
|
||||
_cached_policy_time: float = 0.0
|
||||
|
||||
|
||||
def _get_default_config_path() -> Path:
|
||||
return get_hermes_home() / "config.yaml"
|
||||
|
||||
|
||||
class WebsitePolicyError(Exception):
|
||||
"""Raised when a website policy file is malformed."""
|
||||
|
||||
|
||||
def _normalize_host(host: str) -> str:
|
||||
return (host or "").strip().lower().rstrip(".")
|
||||
|
||||
|
||||
def _normalize_rule(rule: Any) -> Optional[str]:
|
||||
if not isinstance(rule, str):
|
||||
return None
|
||||
value = rule.strip().lower()
|
||||
if not value or value.startswith("#"):
|
||||
return None
|
||||
if "://" in value:
|
||||
parsed = urlparse(value)
|
||||
value = parsed.netloc or parsed.path
|
||||
value = value.split("/", 1)[0].strip().rstrip(".")
|
||||
if value.startswith("www."):
|
||||
value = value[4:]
|
||||
return value or None
|
||||
|
||||
|
||||
def _iter_blocklist_file_rules(path: Path) -> List[str]:
|
||||
"""Load rules from a shared blocklist file.
|
||||
|
||||
Missing or unreadable files log a warning and return an empty list
|
||||
rather than raising — a bad file path should not disable all web tools.
|
||||
"""
|
||||
try:
|
||||
raw = path.read_text(encoding="utf-8")
|
||||
except FileNotFoundError:
|
||||
logger.warning("Shared blocklist file not found (skipping): %s", path)
|
||||
return []
|
||||
except (OSError, UnicodeDecodeError) as exc:
|
||||
logger.warning("Failed to read shared blocklist file %s (skipping): %s", path, exc)
|
||||
return []
|
||||
|
||||
rules: List[str] = []
|
||||
for line in raw.splitlines():
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith("#"):
|
||||
continue
|
||||
normalized = _normalize_rule(stripped)
|
||||
if normalized:
|
||||
rules.append(normalized)
|
||||
return rules
|
||||
|
||||
|
||||
def _load_policy_config(config_path: Optional[Path] = None) -> Dict[str, Any]:
|
||||
config_path = config_path or _get_default_config_path()
|
||||
if not config_path.exists():
|
||||
return dict(_DEFAULT_WEBSITE_BLOCKLIST)
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
logger.debug("PyYAML not installed — website blocklist disabled")
|
||||
return dict(_DEFAULT_WEBSITE_BLOCKLIST)
|
||||
|
||||
try:
|
||||
with open(config_path, encoding="utf-8") as f:
|
||||
config = yaml.safe_load(f) or {}
|
||||
except yaml.YAMLError as exc:
|
||||
raise WebsitePolicyError(f"Invalid config YAML at {config_path}: {exc}") from exc
|
||||
except OSError as exc:
|
||||
raise WebsitePolicyError(f"Failed to read config file {config_path}: {exc}") from exc
|
||||
if not isinstance(config, dict):
|
||||
raise WebsitePolicyError("config root must be a mapping")
|
||||
|
||||
security = config.get("security", {})
|
||||
if security is None:
|
||||
security = {}
|
||||
if not isinstance(security, dict):
|
||||
raise WebsitePolicyError("security must be a mapping")
|
||||
|
||||
website_blocklist = security.get("website_blocklist", {})
|
||||
if website_blocklist is None:
|
||||
website_blocklist = {}
|
||||
if not isinstance(website_blocklist, dict):
|
||||
raise WebsitePolicyError("security.website_blocklist must be a mapping")
|
||||
|
||||
policy = dict(_DEFAULT_WEBSITE_BLOCKLIST)
|
||||
policy.update(website_blocklist)
|
||||
return policy
|
||||
|
||||
|
||||
def load_website_blocklist(config_path: Optional[Path] = None) -> Dict[str, Any]:
|
||||
"""Load and return the parsed website blocklist policy.
|
||||
|
||||
Results are cached for ``_CACHE_TTL_SECONDS`` to avoid re-reading
|
||||
config.yaml on every URL check. Pass an explicit ``config_path``
|
||||
to bypass the cache (used by tests).
|
||||
"""
|
||||
global _cached_policy, _cached_policy_path, _cached_policy_time
|
||||
|
||||
resolved_path = str(config_path) if config_path else "__default__"
|
||||
now = time.monotonic()
|
||||
|
||||
# Return cached policy if still fresh and same path
|
||||
if config_path is None:
|
||||
with _cache_lock:
|
||||
if (
|
||||
_cached_policy is not None
|
||||
and _cached_policy_path == resolved_path
|
||||
and (now - _cached_policy_time) < _CACHE_TTL_SECONDS
|
||||
):
|
||||
return _cached_policy
|
||||
|
||||
config_path = config_path or _get_default_config_path()
|
||||
policy = _load_policy_config(config_path)
|
||||
|
||||
raw_domains = policy.get("domains", []) or []
|
||||
if not isinstance(raw_domains, list):
|
||||
raise WebsitePolicyError("security.website_blocklist.domains must be a list")
|
||||
|
||||
raw_shared_files = policy.get("shared_files", []) or []
|
||||
if not isinstance(raw_shared_files, list):
|
||||
raise WebsitePolicyError("security.website_blocklist.shared_files must be a list")
|
||||
|
||||
enabled = policy.get("enabled", True)
|
||||
if not isinstance(enabled, bool):
|
||||
raise WebsitePolicyError("security.website_blocklist.enabled must be a boolean")
|
||||
|
||||
rules: List[Dict[str, str]] = []
|
||||
seen: set[Tuple[str, str]] = set()
|
||||
|
||||
for raw_rule in raw_domains:
|
||||
normalized = _normalize_rule(raw_rule)
|
||||
if normalized and ("config", normalized) not in seen:
|
||||
rules.append({"pattern": normalized, "source": "config"})
|
||||
seen.add(("config", normalized))
|
||||
|
||||
for shared_file in raw_shared_files:
|
||||
if not isinstance(shared_file, str) or not shared_file.strip():
|
||||
continue
|
||||
path = Path(shared_file).expanduser()
|
||||
if not path.is_absolute():
|
||||
path = (get_hermes_home() / path).resolve()
|
||||
for normalized in _iter_blocklist_file_rules(path):
|
||||
key = (str(path), normalized)
|
||||
if key in seen:
|
||||
continue
|
||||
rules.append({"pattern": normalized, "source": str(path)})
|
||||
seen.add(key)
|
||||
|
||||
result = {"enabled": enabled, "rules": rules}
|
||||
|
||||
# Cache the result (only for the default path — explicit paths are tests)
|
||||
if config_path == _get_default_config_path():
|
||||
with _cache_lock:
|
||||
_cached_policy = result
|
||||
_cached_policy_path = "__default__"
|
||||
_cached_policy_time = now
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def invalidate_cache() -> None:
|
||||
"""Force the next ``check_website_access`` call to re-read config."""
|
||||
global _cached_policy
|
||||
with _cache_lock:
|
||||
_cached_policy = None
|
||||
|
||||
|
||||
def _match_host_against_rule(host: str, pattern: str) -> bool:
|
||||
if not host or not pattern:
|
||||
return False
|
||||
if pattern.startswith("*."):
|
||||
return fnmatch.fnmatch(host, pattern)
|
||||
return host == pattern or host.endswith(f".{pattern}")
|
||||
|
||||
|
||||
def _extract_host_from_urlish(url: str) -> str:
|
||||
parsed = urlparse(url)
|
||||
host = _normalize_host(parsed.hostname or parsed.netloc)
|
||||
if host:
|
||||
return host
|
||||
|
||||
if "://" not in url:
|
||||
schemeless = urlparse(f"//{url}")
|
||||
host = _normalize_host(schemeless.hostname or schemeless.netloc)
|
||||
if host:
|
||||
return host
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
def check_website_access(url: str, config_path: Optional[Path] = None) -> Optional[Dict[str, str]]:
|
||||
"""Check whether a URL is allowed by the website blocklist policy.
|
||||
|
||||
Returns ``None`` if access is allowed, or a dict with block metadata
|
||||
(``host``, ``rule``, ``source``, ``message``) if blocked.
|
||||
|
||||
Never raises on policy errors — logs a warning and returns ``None``
|
||||
(fail-open) so a config typo doesn't break all web tools. Pass
|
||||
``config_path`` explicitly (tests) to get strict error propagation.
|
||||
"""
|
||||
# Fast path: if no explicit config_path and the cached policy is disabled
|
||||
# or empty, skip all work (no YAML read, no host extraction).
|
||||
if config_path is None:
|
||||
with _cache_lock:
|
||||
if _cached_policy is not None and not _cached_policy.get("enabled"):
|
||||
return None
|
||||
|
||||
host = _extract_host_from_urlish(url)
|
||||
if not host:
|
||||
return None
|
||||
|
||||
try:
|
||||
policy = load_website_blocklist(config_path)
|
||||
except WebsitePolicyError as exc:
|
||||
if config_path is not None:
|
||||
raise # Tests pass explicit paths — let errors propagate
|
||||
logger.warning("Website policy config error (failing open): %s", exc)
|
||||
return None
|
||||
except Exception as exc:
|
||||
logger.warning("Unexpected error loading website policy (failing open): %s", exc)
|
||||
return None
|
||||
|
||||
if not policy.get("enabled"):
|
||||
return None
|
||||
|
||||
for rule in policy.get("rules", []):
|
||||
pattern = rule.get("pattern", "")
|
||||
if _match_host_against_rule(host, pattern):
|
||||
logger.info("Blocked URL %s — matched rule '%s' from %s",
|
||||
url, pattern, rule.get("source", "config"))
|
||||
return {
|
||||
"url": url,
|
||||
"host": host,
|
||||
"rule": pattern,
|
||||
"source": rule.get("source", "config"),
|
||||
"message": (
|
||||
f"Blocked by website policy: '{host}' matched rule '{pattern}'"
|
||||
f" from {rule.get('source', 'config')}"
|
||||
),
|
||||
}
|
||||
return None
|
||||
Reference in New Issue
Block a user