lidf/chanpinhsd
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

refactor: v2.0 完全解耦 — 阿里云内闭环

Browse Source 
- 删除 VOC_DATA_DIR / get_voc_conn（不再跨云直读 SQLite）
- 案例 DB 自带 comments 表，自包含所有数据
- 新增 POST /import-voc：通过 VOC 公网 API 导入评论
- VOC_API_BASE 环境变量控制 API 地址
- 新增 httpx 依赖
This commit is contained in:
lidf
2026-04-07 19:47:34 +08:00
parent ec8eaa0b36
commit c5e2a58258
5 changed files with 194 additions and 191 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/tools/ude_extract.py
+55 -85
View File
@@ -1,10 +1,9 @@
"""
黑手党提案 — UDE 提取工具
黑手党提案 — UDE 提取工具（阿里云内闭环）
流程：VOC 原始评论 → LLM 转写 UDE → DashScope 向量化 → DBSCAN 聚类 → 覆盖扫描
流程：本地 comments → LLM 转写 UDE → DashScope 向量化 → DBSCAN 聚类
数据来源：只读访问共享 VOC 数据层
分析结果：写入本项目的案例 DB
所有数据读写都在案例 DB 内，不跨云。
"""
from __future__ import annotations
@@ -47,7 +46,7 @@ def _get_embed_client(key: str) -> OpenAI:
)
# ═══════════ Step 1: VOC → UDE 转写 ═══════════
# ═══════════ Step 1: 本地评论 → UDE 转写 ═══════════
async def _call_ude_llm(prompt: str, comments: list[dict]) -> list[dict]:
"""单批 LLM 转写"""
@@ -87,40 +86,33 @@ async def _process_ude_batch(comments, prompt, semaphore):
async def run_ude_extraction(case_id: str, limit: int = 0) -> dict:
"""从共享 VOC 数据读取原始评论，转写为 UDE，存入案例 DB"""
from db import get_case_conn, get_voc_conn
"""从本地 comments 表读取评论，转写为 UDE，存入 ude_sentences"""
from db import get_case_conn
prompt = PROMPT_PATH.read_text("utf-8") if PROMPT_PATH.exists() else ""
if not prompt:
return {"error": "UDE 转写 prompt 未找到 (prompts/voc_to_ude.txt)"}
with get_case_conn(case_id) as case_conn:
card = case_conn.execute("SELECT voc_research_id FROM case_card LIMIT 1").fetchone()
if not card or not card["voc_research_id"]:
return {"error": "未关联 VOC 研究。请先调用 link-voc。"}
voc_research_id = card["voc_research_id"]
# 获取已转写的 voc_comment_ids
done_ids = {r[0] for r in case_conn.execute(
"SELECT voc_comment_id FROM ude_sentences"
with get_case_conn(case_id) as conn:
# 获取已转写的 comment_ids
done_ids = {r[0] for r in conn.execute(
"SELECT comment_id FROM ude_sentences"
).fetchall()}
# 从 VOC DB 只读获取原始评论
with get_voc_conn(voc_research_id) as voc_conn:
rows = voc_conn.execute("""
# 从本地 comments 表读取
rows = conn.execute("""
SELECT id, platform, text
FROM comments
WHERE length(text) > 10
FROM comments WHERE length(text) > 10
ORDER BY id
""").fetchall()
# 过滤已完成的
total_comments = len(rows)
pending = [r for r in rows if r["id"] not in done_ids]
if not pending:
with get_case_conn(case_id) as conn:
total = conn.execute("SELECT count(*) FROM ude_sentences").fetchone()[0]
return {"message": "全部已转写完成", "total_udes": total, "new": 0}
return {"message": "全部已转写完成", "totalUdes": total, "new": 0}
if limit > 0:
pending = pending[:limit]
@@ -137,7 +129,7 @@ async def run_ude_extraction(case_id: str, limit: int = 0) -> dict:
# 写入案例 DB
ok = 0
with get_case_conn(case_id) as case_conn:
with get_case_conn(case_id) as conn:
for results in all_results:
for r in (results or []):
if not isinstance(r, dict):
@@ -149,21 +141,21 @@ async def run_ude_extraction(case_id: str, limit: int = 0) -> dict:
if not cid:
continue
try:
case_conn.execute(
"INSERT OR IGNORE INTO ude_sentences (voc_comment_id, ude_text, confidence) VALUES (?, ?, ?)",
conn.execute(
"INSERT OR IGNORE INTO ude_sentences (comment_id, ude_text, confidence) VALUES (?, ?, ?)",
(int(cid), ude_text, r.get("confidence", 0.5))
)
ok += 1
except Exception as e:
logger.warning(f"[UDE] 写入失败 id={cid}: {e}")
case_conn.commit()
total = case_conn.execute("SELECT count(*) FROM ude_sentences").fetchone()[0]
conn.commit()
total = conn.execute("SELECT count(*) FROM ude_sentences").fetchone()[0]
return {
"new_udes": ok,
"total_udes": total,
"total_voc_comments": len(rows),
"remaining": len(rows) - total,
"newUdes": ok,
"totalUdes": total,
"totalComments": total_comments,
"remaining": total_comments - total,
"batches": len(batches),
}
@@ -181,10 +173,10 @@ def _embed_texts(client: OpenAI, texts: list[str]) -> list[list[float]]:
def run_clustering(case_id: str, eps: float = 0.25, min_samples: int = 3,
dashscope_key: str = None) -> dict:
"""向量化 + DBSCAN 聚类"""
"""向量化 + DBSCAN 聚类（全部在本地案例 DB 内）"""
from sklearn.cluster import DBSCAN
from sklearn.metrics.pairwise import cosine_distances
from db import get_case_conn, get_voc_conn
from db import get_case_conn
key = dashscope_key or os.getenv("DASHSCOPE_API_KEY", "")
if not key:
@@ -193,13 +185,13 @@ def run_clustering(case_id: str, eps: float = 0.25, min_samples: int = 3,
embed_client = _get_embed_client(key)
with get_case_conn(case_id) as conn:
rows = conn.execute("SELECT id, voc_comment_id, ude_text FROM ude_sentences ORDER BY id").fetchall()
rows = conn.execute("SELECT id, comment_id, ude_text FROM ude_sentences ORDER BY id").fetchall()
if len(rows) < min_samples:
return {"error": f"UDE 不足 ({len(rows)} 条)，至少需要 {min_samples} 条。"}
ude_texts = [r["ude_text"] for r in rows]
ude_ids = [r["id"] for r in rows]
comment_ids = [r["voc_comment_id"] for r in rows]
comment_ids = [r["comment_id"] for r in rows]
# 向量化
vectors = _embed_texts(embed_client, ude_texts)
@@ -223,10 +215,6 @@ def run_clustering(case_id: str, eps: float = 0.25, min_samples: int = 3,
# 清空旧聚类，写入新聚类
conn.execute("DELETE FROM ude_clusters")
# 获取关联的 VOC research_id 用于读取原声
card = conn.execute("SELECT voc_research_id FROM case_card LIMIT 1").fetchone()
voc_rid = card["voc_research_id"] if card else None
clusters = []
unique_labels = sorted(set(labels) - {-1})
@@ -241,30 +229,24 @@ def run_clustering(case_id: str, eps: float = 0.25, min_samples: int = 3,
dists = cosine_distances([centroid], member_vectors)[0]
representative = member_texts[dists.argmin()]
# 取原声
# 原声采样（从本地 comments 表）
sample_voices = []
if voc_rid:
try:
voc_conn = get_voc_conn(voc_rid)
for cid in member_cids[:5]:
voice = voc_conn.execute(
"SELECT text, platform FROM comments WHERE id = ?", (cid,)
).fetchone()
if voice:
sample_voices.append({"text": voice["text"][:200], "platform": voice["platform"]})
voc_conn.close()
except Exception:
pass
for cid in member_cids[:5]:
voice = conn.execute(
"SELECT text, platform FROM comments WHERE id = ?", (cid,)
).fetchone()
if voice:
sample_voices.append({"text": voice["text"][:200], "platform": voice["platform"]})
conn.execute(
"INSERT INTO ude_clusters (representative_ude, coverage, sample_voices) VALUES (?, ?, ?)",
(representative, len(member_indices), json.dumps(sample_voices, ensure_ascii=False))
)
clusters.append({
"cluster_id": int(cluster_id),
"representative_ude": representative,
"clusterId": int(cluster_id),
"representativeUde": representative,
"coverage": len(member_indices),
"sample_voices": sample_voices,
"sampleVoices": sample_voices,
})
conn.commit()
@@ -272,24 +254,22 @@ def run_clustering(case_id: str, eps: float = 0.25, min_samples: int = 3,
noise_count = int((labels == -1).sum())
return {
"total_udes": len(labels),
"num_clusters": len(clusters),
"noise_count": noise_count,
"noise_pct": round(noise_count / len(labels) * 100, 1) if len(labels) else 0,
"totalUdes": len(labels),
"numClusters": len(clusters),
"noiseCount": noise_count,
"noisePct": round(noise_count / len(labels) * 100, 1) if len(labels) else 0,
"clusters": clusters,
"params": {"eps": eps, "min_samples": min_samples},
"params": {"eps": eps, "minSamples": min_samples},
}
# ═══════════ Step 5: 覆盖扫描 ═══════════
# ═══════════ 覆盖扫描 ═══════════
def run_coverage_scan(case_id: str) -> dict:
from db import get_case_conn, get_voc_conn
from db import get_case_conn
with get_case_conn(case_id) as conn:
card = conn.execute("SELECT voc_research_id FROM case_card LIMIT 1").fetchone()
voc_rid = card["voc_research_id"] if card else None
total_comments = conn.execute("SELECT count(*) FROM comments").fetchone()[0]
total_udes = conn.execute("SELECT count(*) FROM ude_sentences").fetchone()[0]
clustered = conn.execute("SELECT count(*) FROM ude_sentences WHERE cluster_id >= 0").fetchone()[0]
noise = conn.execute("SELECT count(*) FROM ude_sentences WHERE cluster_id = -1").fetchone()[0]
@@ -299,27 +279,17 @@ def run_coverage_scan(case_id: str) -> dict:
).fetchall()]
noise_samples = [dict(r) for r in conn.execute(
"SELECT ude_text, voc_comment_id, confidence FROM ude_sentences WHERE cluster_id = -1 ORDER BY confidence DESC LIMIT 10"
"SELECT ude_text, comment_id, confidence FROM ude_sentences WHERE cluster_id = -1 ORDER BY confidence DESC LIMIT 10"
).fetchall()]
total_voc = 0
if voc_rid:
try:
with get_voc_conn(voc_rid) as voc:
total_voc = voc.execute(
"SELECT count(*) FROM comments WHERE length(text) > 10 "
).fetchone()[0]
except Exception:
pass
return {
"total_voc_comments": total_voc,
"total_udes": total_udes,
"udes_clustered": clustered,
"udes_noise": noise,
"coverage_rate": round(clustered / total_voc * 100, 1) if total_voc else 0,
"cluster_distribution": cluster_stats,
"noise_samples": noise_samples,
"totalComments": total_comments,
"totalUdes": total_udes,
"udesClustered": clustered,
"udesNoise": noise,
"coverageRate": round(clustered / total_comments * 100, 1) if total_comments else 0,
"clusterDistribution": cluster_stats,
"noiseSamples": noise_samples,
"verdict": "充分" if (total_udes > 0 and noise / total_udes < 0.1) else
("需关注" if (total_udes > 0 and noise / total_udes < 0.2) else "需调参"),
}