267 lines
9.9 KiB
Python
267 lines
9.9 KiB
Python
"""Extract knowledge notes from threads / searches / tool results.
|
||
|
||
First-release strategy is rule-based (wiki-capture aligned, §11 of the dev doc):
|
||
keep high-value content (user questions, final assistant answers, cited
|
||
sources), drop noise (empty/greeting/intermediate reasoning), and write a
|
||
*declarative* note rather than a chat transcript. An LLM extractor can later
|
||
replace :func:`extract_thread` while keeping the same :class:`NoteDraft` output.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import re
|
||
from typing import Any
|
||
|
||
from deerflow.knowledge.schemas import NoteDraft, SourceDraft
|
||
|
||
# Greeting / low-value lines filtered from extracted turns.
|
||
_GREETING_RE = re.compile(r"^(你好|您好|hi|hello|hey|thanks?|谢谢|好的|ok|okay)[\s!！。.,，~]*$", re.IGNORECASE)
|
||
_WS_RE = re.compile(r"\s+")
|
||
|
||
|
||
def _one_line(text: str, limit: int = 300) -> str:
|
||
"""Collapse whitespace and clip ``text`` to a single line."""
|
||
collapsed = _WS_RE.sub(" ", text or "").strip()
|
||
return collapsed[:limit]
|
||
|
||
|
||
def message_text(msg: dict[str, Any]) -> str:
|
||
"""Return the plain-text content of a serialized LangChain message."""
|
||
content = msg.get("content")
|
||
if isinstance(content, str):
|
||
return content.strip()
|
||
if isinstance(content, list):
|
||
parts: list[str] = []
|
||
for block in content:
|
||
if isinstance(block, str):
|
||
parts.append(block)
|
||
elif isinstance(block, dict):
|
||
if block.get("type") in (None, "text") and isinstance(block.get("text"), str):
|
||
parts.append(block["text"])
|
||
return "\n".join(p for p in parts if p).strip()
|
||
return ""
|
||
|
||
|
||
def _msg_type(msg: dict[str, Any]) -> str:
|
||
return str(msg.get("type") or msg.get("role") or "").lower()
|
||
|
||
|
||
def split_turns(messages: list[dict[str, Any]], *, max_messages: int = 80) -> list[dict[str, str]]:
|
||
"""Split messages into ``{user, ai}`` turns keyed by human messages.
|
||
|
||
For each turn we keep the last substantial AI message (with text body).
|
||
"""
|
||
msgs = messages[-max_messages:] if max_messages and len(messages) > max_messages else list(messages)
|
||
turns: list[dict[str, str]] = []
|
||
current: dict[str, str] | None = None
|
||
for msg in msgs:
|
||
mtype = _msg_type(msg)
|
||
text = message_text(msg)
|
||
if mtype == "human":
|
||
if current and (current.get("user") or current.get("ai")):
|
||
turns.append(current)
|
||
current = {"user": text, "ai": ""}
|
||
elif mtype == "ai" and text:
|
||
if current is None:
|
||
current = {"user": "", "ai": text}
|
||
else:
|
||
current["ai"] = text # last substantial AI message wins
|
||
if current and (current.get("user") or current.get("ai")):
|
||
turns.append(current)
|
||
# Drop turns that are pure greetings with no answer.
|
||
cleaned = []
|
||
for t in turns:
|
||
user = (t.get("user") or "").strip()
|
||
ai = (t.get("ai") or "").strip()
|
||
if not ai and _GREETING_RE.match(user):
|
||
continue
|
||
if not user and not ai:
|
||
continue
|
||
cleaned.append({"user": user, "ai": ai})
|
||
return cleaned
|
||
|
||
|
||
def _auto_title(thread_title: str | None, turns: list[dict[str, str]]) -> str:
|
||
if thread_title and thread_title.strip() and thread_title.strip().lower() != "untitled":
|
||
return thread_title.strip()[:200]
|
||
for t in turns:
|
||
if t.get("user"):
|
||
first = t["user"].splitlines()[0].strip()
|
||
return (first[:40] + ("…" if len(first) > 40 else "")) or "知识笔记"
|
||
return "知识笔记"
|
||
|
||
|
||
def _summary(turns: list[dict[str, str]], *, limit: int = 280) -> str:
|
||
"""Simple rule-based summary: the first answer, clipped."""
|
||
for t in turns:
|
||
if t.get("ai"):
|
||
text = re.sub(r"\s+", " ", t["ai"]).strip()
|
||
return text[:limit] + ("…" if len(text) > limit else "")
|
||
for t in turns:
|
||
if t.get("user"):
|
||
text = re.sub(r"\s+", " ", t["user"]).strip()
|
||
return text[:limit] + ("…" if len(text) > limit else "")
|
||
return ""
|
||
|
||
|
||
def related_links(project_name: str | None) -> list[str]: # noqa: ARG001
|
||
"""Structural wikilinks every note used to carry.
|
||
|
||
Product decision: notes no longer auto-link the vault meta/hub pages
|
||
(``index`` / ``_meta/taxonomy`` / the project hub) — they were noise in every
|
||
note's ``## Related``. Only real, content-specific related notes (injected by
|
||
the service layer, which has DB access) are linked now. Kept as a function
|
||
(returns empty) so existing callers and the wiki-capture contract stay intact.
|
||
"""
|
||
return []
|
||
|
||
|
||
# Backwards-compatible internal alias.
|
||
_related_links = related_links
|
||
|
||
|
||
def _related_section_md(related_lines: list[str], source_lines: list[str] | None = None) -> str: # noqa: ARG001
|
||
"""Render the ``## Related`` section, or ``""`` when there is nothing to show.
|
||
|
||
Sources are intentionally **not** rendered into the body anymore. The note
|
||
detail page already shows every source once via the structured ``来源`` block
|
||
(``note.sources``); emitting a ``### Sources`` sub-section here duplicated that
|
||
list (and broke the frontend's clean ``## Related`` extraction). The
|
||
``source_lines`` argument is kept for backward compatibility and ignored.
|
||
"""
|
||
if not related_lines:
|
||
return ""
|
||
inner = "\n".join(related_lines)
|
||
return f"## Related\n\n{inner}\n"
|
||
|
||
|
||
def _build_body(
|
||
*,
|
||
title: str,
|
||
turns: list[dict[str, str]],
|
||
mode: str,
|
||
project_name: str | None,
|
||
sources: list[SourceDraft],
|
||
) -> str:
|
||
"""Render the wiki-capture 5-section declarative body."""
|
||
full = mode == "full"
|
||
answer_clip = 4000 if full else 800
|
||
|
||
context_lines = []
|
||
for t in turns:
|
||
if t.get("user"):
|
||
q = re.sub(r"\s+", " ", t["user"]).strip()
|
||
context_lines.append(f"- {q[:300]}")
|
||
context = "\n".join(context_lines[:8]) or "- 来自一次对话沉淀。^[inferred]"
|
||
|
||
finding_lines = []
|
||
for i, t in enumerate(turns, 1):
|
||
if t.get("ai"):
|
||
ans = t["ai"].strip()
|
||
ans = ans if full else (ans[:answer_clip] + ("…" if len(ans) > answer_clip else ""))
|
||
finding_lines.append(f"### 结论 {i}\n\n{ans}")
|
||
finding = "\n\n".join(finding_lines) or "_暂无可提炼的结论。_"
|
||
|
||
reasoning = "本页由对话自动沉淀而成，结论来自助手最终回答；过程性推理已省略。判断与归纳部分标记为 ^[inferred]。"
|
||
|
||
implications = "- 后续可在相关问答中参考本页知识。\n- 如内容过期或冲突，请编辑或归档。"
|
||
|
||
related_lines = [f"- {link}" for link in _related_links(project_name)]
|
||
source_lines = [f"- {s.title or s.url or '来源'}" + (f" — {s.url}" if s.url else "") for s in sources[:20]]
|
||
related_section = _related_section_md(related_lines, source_lines)
|
||
|
||
return (
|
||
f"# {title}\n\n"
|
||
f"## Context\n\n{context}\n\n"
|
||
f"## Finding / Decision\n\n{finding}\n\n"
|
||
f"## Reasoning\n\n{reasoning}\n\n"
|
||
f"## Implications\n\n{implications}\n\n"
|
||
f"{related_section}"
|
||
)
|
||
|
||
|
||
def extract_thread(
|
||
messages: list[dict[str, Any]],
|
||
*,
|
||
thread_id: str,
|
||
thread_title: str | None = None,
|
||
mode: str = "summary",
|
||
title: str | None = None,
|
||
tags: list[str] | None = None,
|
||
project_name: str | None = "zncm",
|
||
sources: list[SourceDraft] | None = None,
|
||
max_messages: int = 80,
|
||
) -> NoteDraft | None:
|
||
"""Build a :class:`NoteDraft` from thread messages, or ``None`` if empty."""
|
||
turns = split_turns(messages, max_messages=max_messages)
|
||
if not turns:
|
||
return None
|
||
final_title = (title or "").strip() or _auto_title(thread_title, turns)
|
||
summary = _summary(turns)
|
||
sources = sources or []
|
||
body = _build_body(title=final_title, turns=turns, mode=mode, project_name=project_name, sources=sources)
|
||
base_tags = list(tags or [])
|
||
for default in (project_name or "zncm", "knowledge-base"):
|
||
if default and default not in base_tags:
|
||
base_tags.append(default)
|
||
return NoteDraft(
|
||
title=final_title,
|
||
summary=summary,
|
||
content_md=body,
|
||
category="synthesis",
|
||
source_type="thread",
|
||
source_id=thread_id,
|
||
tags=base_tags,
|
||
confidence=0.6,
|
||
related=_related_links(project_name),
|
||
sources=sources,
|
||
source_key=f"thread:{thread_id}",
|
||
)
|
||
|
||
|
||
def extract_search(
|
||
query: str,
|
||
sources: list[SourceDraft],
|
||
*,
|
||
thread_id: str | None = None,
|
||
project_name: str | None = "zncm",
|
||
title: str | None = None,
|
||
tags: list[str] | None = None,
|
||
) -> NoteDraft:
|
||
"""Build a reference-category :class:`NoteDraft` from search/tool results."""
|
||
final_title = (title or "").strip() or f"检索：{query[:40]}"
|
||
body_lines = [f"# {final_title}\n", "## Context\n", f"- 检索关键词：{query}\n", "## Finding / Decision\n"]
|
||
for s in sources:
|
||
label = s.title or s.url or "来源"
|
||
line = f"- **{label}**"
|
||
if s.url:
|
||
line += f" — {s.url}"
|
||
if s.snippet:
|
||
line += "\n - " + _one_line(s.snippet)
|
||
body_lines.append(line)
|
||
body_lines += [
|
||
"\n## Reasoning\n",
|
||
"本页汇总外部检索来源，供后续问答引用。来源可信度需结合原文判断。^[ambiguous]\n",
|
||
"## Implications\n",
|
||
"- 引用前请核对原始来源。\n",
|
||
"## Related\n",
|
||
*[f"- {link}" for link in _related_links(project_name)],
|
||
]
|
||
base_tags = list(tags or [])
|
||
for default in (project_name or "zncm", "knowledge-base", "reference"):
|
||
if default and default not in base_tags:
|
||
base_tags.append(default)
|
||
return NoteDraft(
|
||
title=final_title,
|
||
summary=f"检索“{query}”的来源汇总，共 {len(sources)} 条。",
|
||
content_md="\n".join(body_lines),
|
||
category="references",
|
||
source_type="search",
|
||
source_id=f"search:{query[:120]}",
|
||
tags=base_tags,
|
||
confidence=0.4,
|
||
related=_related_links(project_name),
|
||
sources=sources,
|
||
source_key=None,
|
||
)
|