fix: 补全 vendor 快照（model_tools/utils/toolsets/gateway/cron/hermes_time）+ python-dotenv 依赖
- vendor_hermes.sh: 补全遗漏的单文件模块和目录 - pyproject.toml: 添加 python-dotenv 隐性依赖 - 全量 import 测试通过（216 .py, 180K 行） - firecrawl/fal_client 为可选工具，缺失时优雅跳过
This commit is contained in:
+150
-38
@@ -31,7 +31,7 @@ T = TypeVar("T")
|
||||
|
||||
DEFAULT_DB_PATH = get_hermes_home() / "state.db"
|
||||
|
||||
SCHEMA_VERSION = 8
|
||||
SCHEMA_VERSION = 9
|
||||
|
||||
SCHEMA_SQL = """
|
||||
CREATE TABLE IF NOT EXISTS schema_version (
|
||||
@@ -81,7 +81,8 @@ CREATE TABLE IF NOT EXISTS messages (
|
||||
finish_reason TEXT,
|
||||
reasoning TEXT,
|
||||
reasoning_details TEXT,
|
||||
codex_reasoning_items TEXT
|
||||
codex_reasoning_items TEXT,
|
||||
archived INTEGER DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_sessions_source ON sessions(source);
|
||||
@@ -366,6 +367,13 @@ class SessionDB:
|
||||
except Exception:
|
||||
pass
|
||||
cursor.execute("UPDATE schema_version SET version = 8")
|
||||
if current_version < 9:
|
||||
# v9: messages 表增加 archived 列（Context Compaction 消息归档）
|
||||
try:
|
||||
cursor.execute("ALTER TABLE messages ADD COLUMN archived INTEGER DEFAULT 0")
|
||||
except sqlite3.OperationalError:
|
||||
pass # Column already exists
|
||||
cursor.execute("UPDATE schema_version SET version = 9")
|
||||
|
||||
# Unique title index — always ensure it exists (safe to run after migrations
|
||||
# since the title column is guaranteed to exist at this point)
|
||||
@@ -920,19 +928,125 @@ class SessionDB:
|
||||
result.append(msg)
|
||||
return result
|
||||
|
||||
def get_messages_as_conversation(self, session_id: str) -> List[Dict[str, Any]]:
|
||||
def archive_messages(self, session_id: str) -> int:
|
||||
"""将指定 session 的所有活跃消息标记为 archived。
|
||||
|
||||
用于 Context Compaction：原始消息保留在 DB 中供前端历史查看，
|
||||
但不再被 get_messages_as_conversation() 返回给 LLM。
|
||||
|
||||
Returns:
|
||||
归档的消息数量。
|
||||
"""
|
||||
Load messages in the OpenAI conversation format (role + content dicts).
|
||||
Used by the gateway to restore conversation history.
|
||||
"""
|
||||
with self._lock:
|
||||
cursor = self._conn.execute(
|
||||
"SELECT role, content, tool_call_id, tool_calls, tool_name, "
|
||||
"reasoning, reasoning_details, codex_reasoning_items "
|
||||
"FROM messages WHERE session_id = ? ORDER BY timestamp, id",
|
||||
def _do(conn):
|
||||
cursor = conn.execute(
|
||||
"UPDATE messages SET archived = 1 "
|
||||
"WHERE session_id = ? AND archived = 0",
|
||||
(session_id,),
|
||||
)
|
||||
rows = cursor.fetchall()
|
||||
return cursor.rowcount
|
||||
return self._execute_write(_do)
|
||||
|
||||
def get_messages_as_conversation(self, session_id: str) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Load active (non-archived) messages in the OpenAI conversation format.
|
||||
Used by the gateway to restore conversation history for LLM context.
|
||||
Archived messages (from Context Compaction) are excluded.
|
||||
"""
|
||||
with self._lock:
|
||||
rows = self._conn.execute(
|
||||
"SELECT role, content, tool_call_id, tool_calls, tool_name, "
|
||||
"reasoning, reasoning_details, codex_reasoning_items "
|
||||
"FROM messages WHERE session_id = ? AND archived = 0 ORDER BY timestamp, id",
|
||||
(session_id,),
|
||||
).fetchall()
|
||||
return self._rows_to_conversation(rows)
|
||||
|
||||
# ── Compaction 标记前缀（与 context_compressor.py 保持一致） ──
|
||||
_COMPACTION_MARKER = "[CONTEXT COMPACTION"
|
||||
|
||||
def get_all_messages_as_conversation(self, session_id: str) -> List[Dict[str, Any]]:
|
||||
"""Load ALL messages (including archived) for frontend history display.
|
||||
|
||||
智能合并策略：
|
||||
1. 如果没有归档消息 → 直接返回活跃消息（未发生过 compaction）。
|
||||
2. 如果有归档消息 → 以归档消息为"基础历史"，然后从活跃消息中
|
||||
**去除 compaction summary + tail 重复副本**，只追加真正的新消息。
|
||||
|
||||
这确保前端看到完整、无重复的对话历史，同时 LLM 侧的
|
||||
get_messages_as_conversation() 仍然只返回活跃（精简）上下文。
|
||||
"""
|
||||
with self._lock:
|
||||
archived_rows = self._conn.execute(
|
||||
"SELECT role, content, tool_call_id, tool_calls, tool_name, "
|
||||
"reasoning, reasoning_details, codex_reasoning_items "
|
||||
"FROM messages WHERE session_id = ? AND archived = 1 "
|
||||
"ORDER BY timestamp, id",
|
||||
(session_id,),
|
||||
).fetchall()
|
||||
|
||||
active_rows = self._conn.execute(
|
||||
"SELECT role, content, tool_call_id, tool_calls, tool_name, "
|
||||
"reasoning, reasoning_details, codex_reasoning_items "
|
||||
"FROM messages WHERE session_id = ? AND archived = 0 "
|
||||
"ORDER BY timestamp, id",
|
||||
(session_id,),
|
||||
).fetchall()
|
||||
|
||||
# 无归档 → 从未压缩过，直接返回全部活跃消息
|
||||
if not archived_rows:
|
||||
return self._rows_to_conversation(active_rows)
|
||||
|
||||
archived_msgs = self._rows_to_conversation(archived_rows)
|
||||
|
||||
# 过滤归档中的 compaction summary（多次压缩可能产生多条）
|
||||
archived_msgs = [
|
||||
m for m in archived_msgs
|
||||
if not (m.get("content") or "").lstrip().startswith(self._COMPACTION_MARKER)
|
||||
]
|
||||
|
||||
# 构建归档消息的签名集（role + content 前 300 字），用于去重
|
||||
# tail 消息的 timestamp 在 flush 时被重写，可能出现在 archived
|
||||
# 的任意位置，因此必须对全量 archived 构建签名
|
||||
all_sigs: set = set()
|
||||
for msg in archived_msgs:
|
||||
sig = self._msg_dedup_sig(msg)
|
||||
all_sigs.add(sig)
|
||||
|
||||
# 从活跃消息中筛选真正的新消息
|
||||
active_msgs = self._rows_to_conversation(active_rows)
|
||||
new_msgs = []
|
||||
for msg in active_msgs:
|
||||
content = msg.get("content") or ""
|
||||
# 跳过 compaction summary（LLM 内部参考，不应展示给用户）
|
||||
if content.lstrip().startswith(self._COMPACTION_MARKER):
|
||||
continue
|
||||
# 跳过 tail 重复副本
|
||||
sig = self._msg_dedup_sig(msg)
|
||||
if sig in all_sigs:
|
||||
continue
|
||||
new_msgs.append(msg)
|
||||
|
||||
return archived_msgs + new_msgs
|
||||
|
||||
@staticmethod
|
||||
def _msg_dedup_sig(msg: dict) -> tuple:
|
||||
"""生成消息去重签名。
|
||||
|
||||
签名组成：(role, content_prefix, tool_call_id, tc_fingerprint)
|
||||
- tool_call_id: tool 角色消息的关联 ID
|
||||
- tc_fingerprint: assistant 消息携带的 tool_calls 首个 call ID
|
||||
（防止空 content 但不同 tool_calls 的 assistant 消息误判重复）
|
||||
"""
|
||||
content = (msg.get("content") or "")[:300]
|
||||
tc_fp = ""
|
||||
tool_calls = msg.get("tool_calls")
|
||||
if tool_calls and isinstance(tool_calls, list) and len(tool_calls) > 0:
|
||||
first_tc = tool_calls[0] if isinstance(tool_calls[0], dict) else {}
|
||||
tc_fp = first_tc.get("id", "") or first_tc.get("call_id", "")
|
||||
return (msg.get("role", ""), content, msg.get("tool_call_id") or "", tc_fp)
|
||||
|
||||
def _rows_to_conversation(self, rows) -> List[Dict[str, Any]]:
|
||||
"""将 DB rows 转换为 OpenAI conversation 格式（共享解析逻辑）。"""
|
||||
messages = []
|
||||
for row in rows:
|
||||
msg = {"role": row["role"], "content": row["content"]}
|
||||
@@ -944,11 +1058,8 @@ class SessionDB:
|
||||
try:
|
||||
msg["tool_calls"] = json.loads(row["tool_calls"])
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
logger.warning("Failed to deserialize tool_calls in conversation replay, falling back to []")
|
||||
logger.warning("Failed to deserialize tool_calls, falling back to []")
|
||||
msg["tool_calls"] = []
|
||||
# Restore reasoning fields on assistant messages so providers
|
||||
# that replay reasoning (OpenRouter, OpenAI, Nous) receive
|
||||
# coherent multi-turn reasoning context.
|
||||
if row["role"] == "assistant":
|
||||
if row["reasoning"]:
|
||||
msg["reasoning"] = row["reasoning"]
|
||||
@@ -956,13 +1067,11 @@ class SessionDB:
|
||||
try:
|
||||
msg["reasoning_details"] = json.loads(row["reasoning_details"])
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
logger.warning("Failed to deserialize reasoning_details, falling back to None")
|
||||
msg["reasoning_details"] = None
|
||||
if row["codex_reasoning_items"]:
|
||||
try:
|
||||
msg["codex_reasoning_items"] = json.loads(row["codex_reasoning_items"])
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
logger.warning("Failed to deserialize codex_reasoning_items, falling back to None")
|
||||
msg["codex_reasoning_items"] = None
|
||||
messages.append(msg)
|
||||
return messages
|
||||
@@ -1162,13 +1271,24 @@ class SessionDB:
|
||||
cursor = self._conn.execute("SELECT COUNT(*) FROM sessions")
|
||||
return cursor.fetchone()[0]
|
||||
|
||||
def message_count(self, session_id: str = None) -> int:
|
||||
"""Count messages, optionally for a specific session."""
|
||||
def message_count(self, session_id: str = None, active_only: bool = False) -> int:
|
||||
"""Count messages, optionally for a specific session.
|
||||
|
||||
Args:
|
||||
session_id: If provided, count only messages for this session.
|
||||
active_only: If True, exclude archived messages (from Context Compaction).
|
||||
"""
|
||||
with self._lock:
|
||||
if session_id:
|
||||
cursor = self._conn.execute(
|
||||
"SELECT COUNT(*) FROM messages WHERE session_id = ?", (session_id,)
|
||||
)
|
||||
if active_only:
|
||||
cursor = self._conn.execute(
|
||||
"SELECT COUNT(*) FROM messages WHERE session_id = ? AND archived = 0",
|
||||
(session_id,),
|
||||
)
|
||||
else:
|
||||
cursor = self._conn.execute(
|
||||
"SELECT COUNT(*) FROM messages WHERE session_id = ?", (session_id,)
|
||||
)
|
||||
else:
|
||||
cursor = self._conn.execute("SELECT COUNT(*) FROM messages")
|
||||
return cursor.fetchone()[0]
|
||||
@@ -1305,21 +1425,13 @@ class SessionDB:
|
||||
s.started_at
|
||||
) AS last_active,
|
||||
(
|
||||
COALESCE(
|
||||
(SELECT SUM(
|
||||
(LENGTH(m3.tool_calls) - LENGTH(REPLACE(m3.tool_calls, '"write_file"', ''))) / LENGTH('"write_file"')
|
||||
) FROM messages m3
|
||||
WHERE m3.session_id = s.id
|
||||
AND m3.tool_calls LIKE '%"write_file"%'),
|
||||
0)
|
||||
+
|
||||
(SELECT COUNT(*) FROM messages m4
|
||||
WHERE m4.session_id = s.id
|
||||
AND m4.tool_calls LIKE '%"patch"%')
|
||||
+
|
||||
(SELECT COUNT(*) FROM messages m5
|
||||
WHERE m5.session_id = s.id
|
||||
AND m5.role = 'assistant' AND m5.content LIKE '%"type"%audio"%')
|
||||
SELECT COUNT(*) FROM messages m3
|
||||
WHERE m3.session_id = s.id
|
||||
AND (
|
||||
m3.tool_calls LIKE '%"write_file"%'
|
||||
OR m3.tool_calls LIKE '%"patch"%'
|
||||
OR (m3.role = 'assistant' AND m3.content LIKE '%"type"%audio"%')
|
||||
)
|
||||
) AS work_product_count
|
||||
FROM sessions s
|
||||
WHERE s.user_id = ?
|
||||
|
||||
Reference in New Issue
Block a user