mp_1787300862_s8r25x/ai-assistant-travel
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat: v2.1 BYOK + 自动流水线

Browse Source 
- 统一为 DASHSCOPE_API_KEY（百炼 Key 通吃 LLM + Embedding）
- import-voc 后自动触发 UDE 转写 + 向量化（后台 asyncio task）
- 新增 GET /pipeline-status 查询流水线进度
- run_clustering 变纯 CPU（向量已预计算）
- 新增独立 run_vectorization 函数
- 修复 Python 3.9 类型注解兼容性
This commit is contained in:
lidf
2026-04-07 23:09:11 +08:00
parent 3cd7d4776d
commit f61c255b9d
4 changed files with 201 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/server.py
+103 -16
View File
@@ -1,16 +1,19 @@
"""
黑手党提案 — 独立后端（阿里云内闭环）
黑手党提案 — 独立后端（BYOK v2.1）
FastAPI 服务，端口 8093。
VOC 数据通过公网 API 导入，不直读 VOC DB。
合伙人通过 X-DashScope-Key Header 传入百炼 Key，服务器统一用于 LLM + Embedding。
import-voc 完成后自动触发 UDE 转写 + 向量化流水线。
"""
import os
import logging
import asyncio
import httpx
from fastapi import FastAPI, Header, HTTPException, Query
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel
from typing import Optional
from dotenv import load_dotenv
load_dotenv()
@@ -20,7 +23,7 @@ from db import get_case_conn, init_case_db, list_cases as _list_cases, DATA_DIR,
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(name)s] %(levelname)s %(message)s")
logger = logging.getLogger("mafia")
app = FastAPI(title="黑手党提案后端", version="2.0.0", description="独立后端：阿里云内闭环，VOC 通过 API 导入")
app = FastAPI(title="黑手党提案后端", version="2.1.0", description="独立后端：阿里云内闭环 + BYOK，合伙人自带 API Key")
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
@@ -29,6 +32,20 @@ app.add_middleware(
)
# ═══════════ BYOK Key 解析 ═══════════
def resolve_key(header_key: Optional[str]) -> str:
"""优先使用合伙人 Header 传入的 DashScope Key（百炼 Key 一个通吃 LLM + Embedding）"""
if header_key:
logger.info(f"[BYOK] 使用合伙人 Key: ...{header_key[-6:]}")
return header_key
env_val = os.getenv("DASHSCOPE_API_KEY", "")
if env_val:
logger.info(f"[BYOK] 使用 fallback Key: ...{env_val[-6:]}")
return env_val
raise HTTPException(401, "缺少 DashScope API Key。请在 CoPaw 中配置 DASHSCOPE_API_KEY 并通过 X-DashScope-Key Header 传入")
# ═══════════ Models ═══════════
class CreateCaseRequest(BaseModel):
@@ -91,7 +108,35 @@ async def delete_case(case_id: str):
raise HTTPException(404, "案例不存在")
# ═══════════ VOC 导入（跨云 API） ═══════════
# ═══════════ VOC 导入 + 自动流水线 ═══════════
# 后台流水线状态（内存缓存，进程级）
_pipeline_status: dict[str, dict] = {}
async def _auto_pipeline(case_id: str, dashscope_key: str):
"""后台自动流水线：UDE 转写 → 向量化"""
from tools.ude_extract import run_ude_extraction, run_vectorization
_pipeline_status[case_id] = {"stage": "extracting", "progress": {}}
try:
# Stage 1: UDE 转写
result = await run_ude_extraction(case_id, limit=0, dashscope_key=dashscope_key)
_pipeline_status[case_id]["progress"]["extraction"] = result
logger.info(f"[Pipeline] {case_id} UDE 转写完成: {result.get('totalUdes', 0)} 条")
# Stage 2: 向量化
_pipeline_status[case_id]["stage"] = "vectorizing"
vec_result = run_vectorization(case_id, dashscope_key=dashscope_key)
_pipeline_status[case_id]["progress"]["vectorization"] = vec_result
logger.info(f"[Pipeline] {case_id} 向量化完成: {vec_result.get('vectorized', 0)} 条")
_pipeline_status[case_id]["stage"] = "done"
except Exception as e:
logger.error(f"[Pipeline] {case_id} 失败: {e}")
_pipeline_status[case_id]["stage"] = "error"
_pipeline_status[case_id]["error"] = str(e)
@app.post("/api/cases/{case_id}/link-voc")
async def link_voc(case_id: str, req: LinkVocRequest):
@@ -106,8 +151,15 @@ async def link_voc(case_id: str, req: LinkVocRequest):
@app.post("/api/cases/{case_id}/import-voc")
async def import_voc(case_id: str, page: int = Query(1), pageSize: int = Query(100)):
"""从 VOC 公网 API 拉取评论数据，存入本地案例 DB"""
async def import_voc(
case_id: str,
page: int = Query(1),
pageSize: int = Query(100),
x_dashscope_key: str = Header(None),
):
"""从 VOC 公网 API 拉取评论，完成后自动触发 UDE 转写 + 向量化"""
dashscope_key = resolve_key(x_dashscope_key)
try:
with get_case_conn(case_id) as conn:
card = conn.execute("SELECT voc_research_id, voc_api_base FROM case_card LIMIT 1").fetchone()
@@ -120,7 +172,6 @@ async def import_voc(case_id: str, page: int = Query(1), pageSize: int = Query(1
voc_rid = card["voc_research_id"]
api_base = card["voc_api_base"] or VOC_API_BASE
# 从 VOC API 拉取（只读，不需要 TikHub Key）
total_imported = 0
current_page = page
@@ -167,18 +218,45 @@ async def import_voc(case_id: str, page: int = Query(1), pageSize: int = Query(1
break
current_page += 1
# 更新统计
with get_case_conn(case_id) as conn:
local_count = conn.execute("SELECT count(*) FROM comments").fetchone()[0]
# 自动触发后台流水线（UDE 转写 + 向量化）
pipeline_triggered = False
if local_count > 0:
asyncio.create_task(_auto_pipeline(case_id, dashscope_key))
pipeline_triggered = True
logger.info(f"[Import] {case_id} 自动流水线已触发（{local_count} 条评论）")
return {
"imported": total_imported,
"totalLocal": local_count,
"vocResearchId": voc_rid,
"pagesProcessed": current_page - page + 1,
"pipelineTriggered": pipeline_triggered,
}
@app.get("/api/cases/{case_id}/pipeline-status")
async def pipeline_status(case_id: str):
"""查询后台流水线进度"""
status = _pipeline_status.get(case_id)
if not status:
# 没有正在运行的流水线，从 DB 推断状态
try:
with get_case_conn(case_id) as conn:
comments = conn.execute("SELECT count(*) FROM comments").fetchone()[0]
udes = conn.execute("SELECT count(*) FROM ude_sentences").fetchone()[0]
vectorized = conn.execute("SELECT count(*) FROM ude_sentences WHERE vector IS NOT NULL").fetchone()[0]
except FileNotFoundError:
raise HTTPException(404, "案例不存在")
return {
"stage": "idle",
"progress": {"comments": comments, "udesExtracted": udes, "udesVectorized": vectorized},
}
return status
@app.get("/api/cases/{case_id}/comments")
async def get_comments(case_id: str, page: int = 1, pageSize: int = 50):
"""查看本地导入的评论"""
@@ -195,13 +273,19 @@ async def get_comments(case_id: str, page: int = 1, pageSize: int = 50):
return {"total": total, "page": page, "items": [dict(r) for r in rows]}
# ═══════════ UDE 分析 ═══════════
# ═══════════ UDE 分析（BYOK） ═══════════
@app.post("/api/cases/{case_id}/ude/extract")
async def extract_ude(case_id: str, limit: int = Query(0)):
async def extract_ude(
case_id: str,
limit: int = Query(0),
x_dashscope_key: str = Header(None),
):
"""手动触发 UDE 转写（通常由 import-voc 自动触发，此接口用于补跑）"""
from tools.ude_extract import run_ude_extraction
key = resolve_key(x_dashscope_key)
try:
result = await run_ude_extraction(case_id, limit)
result = await run_ude_extraction(case_id, limit, dashscope_key=key)
except FileNotFoundError as e:
raise HTTPException(404, str(e))
return result
@@ -212,12 +296,11 @@ async def cluster_ude(
case_id: str,
eps: float = Query(0.25),
minSamples: int = Query(3),
x_dashscope_key: str = Header(None),
):
"""向量聚类（纯 CPU，不调外部 API。向量已在流水线中预计算）"""
from tools.ude_extract import run_clustering
key = x_dashscope_key or os.getenv("DASHSCOPE_API_KEY", "")
try:
result = run_clustering(case_id, eps, minSamples, dashscope_key=key)
result = run_clustering(case_id, eps, minSamples)
except FileNotFoundError as e:
raise HTTPException(404, str(e))
return result
@@ -251,10 +334,14 @@ async def get_coverage(case_id: str):
async def health():
return {
"status": "ok",
"version": "2.0.0",
"architecture": "independent (Aliyun self-contained)",
"version": "2.1.0",
"architecture": "BYOK (Bring Your Own Key)",
"vocApiBase": VOC_API_BASE,
"caseDataDir": str(DATA_DIR),
"byok": {
"bailianFallback": bool(os.getenv("BAILIAN_API_KEY")),
"dashscopeFallback": bool(os.getenv("DASHSCOPE_API_KEY")),
},
}