lidf/chanpinhsd
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat: v2.1 BYOK + 自动流水线

Browse Source 
- 统一为 DASHSCOPE_API_KEY（百炼 Key 通吃 LLM + Embedding）
- import-voc 后自动触发 UDE 转写 + 向量化（后台 asyncio task）
- 新增 GET /pipeline-status 查询流水线进度
- run_clustering 变纯 CPU（向量已预计算）
- 新增独立 run_vectorization 函数
- 修复 Python 3.9 类型注解兼容性
This commit is contained in:
lidf
2026-04-07 23:09:11 +08:00
parent 3cd7d4776d
commit f61c255b9d
4 changed files with 201 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/tools/ude_extract.py
+78 -46
View File
@@ -1,9 +1,10 @@
"""
黑手党提案 — UDE 提取工具（阿里云内闭环）
黑手党提案 — UDE 提取工具（BYOK v2.1）
流程：本地 comments → LLM 转写 UDE → DashScope 向量化 → DBSCAN 聚类
所有数据读写都在案例 DB 内，不跨云。
所有外部 API 调用统一使用 DASHSCOPE_API_KEY（百炼 Key 一个通吃 LLM + Embedding）。
向量化在 import-voc 流水线中预完成，聚类为纯 CPU 操作。
"""
from __future__ import annotations
@@ -29,28 +30,28 @@ EMBED_BATCH_SIZE = 25
PROMPT_PATH = Path(__file__).parent.parent / "prompts" / "voc_to_ude.txt"
def _get_llm_client() -> AsyncOpenAI:
return AsyncOpenAI(
api_key=os.getenv("LITELLM_MASTER_KEY"),
base_url=os.getenv("LITELLM_PROXY_URL"),
)
# DashScope OpenAI 兼容端点（LLM + Embedding 共用）
DASHSCOPE_BASE = "https://dashscope.aliyuncs.com/compatible-mode/v1"
def _get_embed_client(key: str) -> OpenAI:
def _get_llm_client(dashscope_key: str = None) -> AsyncOpenAI:
"""百炼 Key 一个通吃 LLM + Embedding"""
key = dashscope_key or os.getenv("DASHSCOPE_API_KEY", "")
return AsyncOpenAI(api_key=key, base_url=DASHSCOPE_BASE)
def _get_embed_client(dashscope_key: str = None) -> OpenAI:
key = dashscope_key or os.getenv("DASHSCOPE_API_KEY", "")
if not key:
raise ValueError("DashScope API Key 未配置。请通过 Header 或 .env 传入。")
return OpenAI(
api_key=key,
base_url="https://dashscope.aliyuncs.com/compatible-mode/v1",
)
raise ValueError("DashScope API Key 未配置。")
return OpenAI(api_key=key, base_url=DASHSCOPE_BASE)
# ═══════════ Step 1: 本地评论 → UDE 转写 ═══════════
async def _call_ude_llm(prompt: str, comments: list[dict]) -> list[dict]:
async def _call_ude_llm(prompt: str, comments: list[dict], dashscope_key: str = None) -> list[dict]:
"""单批 LLM 转写"""
client = _get_llm_client()
client = _get_llm_client(dashscope_key)
user_msg = "请将以下消费者评论转写为 UDE 格式句，返回 JSON：\n\n"
for c in comments:
user_msg += f"[{c['id']}] 平台:{c['platform']} 原文: \"{c['text'][:300]}\"\n\n"
@@ -80,12 +81,12 @@ async def _call_ude_llm(prompt: str, comments: list[dict]) -> list[dict]:
return []
async def _process_ude_batch(comments, prompt, semaphore):
async def _process_ude_batch(comments, prompt, semaphore, dashscope_key=None):
async with semaphore:
return await _call_ude_llm(prompt, comments)
return await _call_ude_llm(prompt, comments, dashscope_key)
async def run_ude_extraction(case_id: str, limit: int = 0) -> dict:
async def run_ude_extraction(case_id: str, limit: int = 0, dashscope_key: str = None) -> dict:
"""从本地 comments 表读取评论，转写为 UDE，存入 ude_sentences"""
from db import get_case_conn
@@ -94,12 +95,10 @@ async def run_ude_extraction(case_id: str, limit: int = 0) -> dict:
return {"error": "UDE 转写 prompt 未找到 (prompts/voc_to_ude.txt)"}
with get_case_conn(case_id) as conn:
# 获取已转写的 comment_ids
done_ids = {r[0] for r in conn.execute(
"SELECT comment_id FROM ude_sentences"
).fetchall()}
# 从本地 comments 表读取
rows = conn.execute("""
SELECT id, platform, text
FROM comments WHERE length(text) > 10
@@ -117,17 +116,15 @@ async def run_ude_extraction(case_id: str, limit: int = 0) -> dict:
if limit > 0:
pending = pending[:limit]
# 切批
batches = []
for i in range(0, len(pending), BATCH_SIZE):
chunk = pending[i:i + BATCH_SIZE]
batches.append([{"id": r["id"], "platform": r["platform"], "text": r["text"]} for r in chunk])
semaphore = asyncio.Semaphore(CONCURRENCY)
tasks = [asyncio.create_task(_process_ude_batch(b, prompt, semaphore)) for b in batches]
tasks = [asyncio.create_task(_process_ude_batch(b, prompt, semaphore, dashscope_key)) for b in batches]
all_results = await asyncio.gather(*tasks)
# 写入案例 DB
ok = 0
with get_case_conn(case_id) as conn:
for results in all_results:
@@ -160,7 +157,7 @@ async def run_ude_extraction(case_id: str, limit: int = 0) -> dict:
}
# ═══════════ Step 2 & 3: 向量化 + 聚类 ═══════════
# ═══════════ Step 2: 向量化（独立函数，流水线自动调用） ═══════════
def _embed_texts(client: OpenAI, texts: list[str]) -> list[list[float]]:
all_vectors = []
@@ -171,38 +168,75 @@ def _embed_texts(client: OpenAI, texts: list[str]) -> list[list[float]]:
return all_vectors
def run_clustering(case_id: str, eps: float = 0.25, min_samples: int = 3,
dashscope_key: str = None) -> dict:
"""向量化 + DBSCAN 聚类（全部在本地案例 DB 内）"""
def run_vectorization(case_id: str, dashscope_key: str = None) -> dict:
"""为所有未向量化的 UDE 生成 embedding（独立于聚类，可被流水线自动调用）"""
from db import get_case_conn
embed_client = _get_embed_client(dashscope_key)
with get_case_conn(case_id) as conn:
# 只处理未向量化的 UDE
rows = conn.execute(
"SELECT id, ude_text FROM ude_sentences WHERE vector IS NULL ORDER BY id"
).fetchall()
if not rows:
total = conn.execute("SELECT count(*) FROM ude_sentences").fetchone()[0]
return {"vectorized": 0, "totalUdes": total, "message": "全部已向量化"}
texts = [r["ude_text"] for r in rows]
ids = [r["id"] for r in rows]
vectors = _embed_texts(embed_client, texts)
for i, uid in enumerate(ids):
conn.execute("UPDATE ude_sentences SET vector = ? WHERE id = ?",
(json.dumps(vectors[i]), uid))
conn.commit()
total = conn.execute("SELECT count(*) FROM ude_sentences").fetchone()[0]
vectorized_total = conn.execute("SELECT count(*) FROM ude_sentences WHERE vector IS NOT NULL").fetchone()[0]
return {
"vectorized": len(rows),
"totalUdes": total,
"totalVectorized": vectorized_total,
}
# ═══════════ Step 3: 聚类（纯 CPU，不调外部 API） ═══════════
def run_clustering(case_id: str, eps: float = 0.25, min_samples: int = 3) -> dict:
"""纯 CPU 聚类：从 DB 读取已有向量，DBSCAN 计算。不调任何外部 API。"""
from sklearn.cluster import DBSCAN
from sklearn.metrics.pairwise import cosine_distances
from db import get_case_conn
key = dashscope_key or os.getenv("DASHSCOPE_API_KEY", "")
if not key:
return {"error": "DashScope API Key 未配置。"}
embed_client = _get_embed_client(key)
with get_case_conn(case_id) as conn:
rows = conn.execute("SELECT id, comment_id, ude_text FROM ude_sentences ORDER BY id").fetchall()
rows = conn.execute("SELECT id, comment_id, ude_text, vector FROM ude_sentences ORDER BY id").fetchall()
if len(rows) < min_samples:
return {"error": f"UDE 不足 ({len(rows)} 条)，至少需要 {min_samples} 条。"}
# 检查向量完备性
has_vector = [r for r in rows if r["vector"]]
missing = len(rows) - len(has_vector)
if missing > 0:
return {
"error": f"{missing} 条 UDE 尚未向量化，请等待流水线完成或手动调用 /ude/extract",
"totalUdes": len(rows),
"vectorized": len(has_vector),
"missing": missing,
}
ude_texts = [r["ude_text"] for r in rows]
ude_ids = [r["id"] for r in rows]
comment_ids = [r["comment_id"] for r in rows]
# 向量化
vectors = _embed_texts(embed_client, ude_texts)
vec_array = np.array(vectors)
# 从 DB 读取已有向量（纯 CPU）
vec_array = np.array([json.loads(r["vector"]) for r in rows])
# 保存向量
for i, uid in enumerate(ude_ids):
conn.execute("UPDATE ude_sentences SET vector = ? WHERE id = ?",
(json.dumps(vectors[i]), uid))
# DBSCAN
# DBSCAN（纯 CPU）
dist_matrix = cosine_distances(vec_array)
clustering = DBSCAN(eps=eps, min_samples=min_samples, metric="precomputed").fit(dist_matrix)
labels = clustering.labels_
@@ -224,12 +258,10 @@ def run_clustering(case_id: str, eps: float = 0.25, min_samples: int = 3,
member_vectors = vec_array[member_indices]
member_cids = [comment_ids[i] for i in member_indices]
# 簇中心
centroid = member_vectors.mean(axis=0)
dists = cosine_distances([centroid], member_vectors)[0]
representative = member_texts[dists.argmin()]
# 原声采样（从本地 comments 表）
sample_voices = []
for cid in member_cids[:5]:
voice = conn.execute(