"""Extract knowledge notes from threads / searches / tool results. First-release strategy is rule-based (wiki-capture aligned, §11 of the dev doc): keep high-value content (user questions, final assistant answers, cited sources), drop noise (empty/greeting/intermediate reasoning), and write a *declarative* note rather than a chat transcript. An LLM extractor can later replace :func:`extract_thread` while keeping the same :class:`NoteDraft` output. """ from __future__ import annotations import re from typing import Any from deerflow.knowledge.schemas import NoteDraft, SourceDraft # Greeting / low-value lines filtered from extracted turns. _GREETING_RE = re.compile(r"^(你好|您好|hi|hello|hey|thanks?|谢谢|好的|ok|okay)[\s!!。.,,~]*$", re.IGNORECASE) _WS_RE = re.compile(r"\s+") def _one_line(text: str, limit: int = 300) -> str: """Collapse whitespace and clip ``text`` to a single line.""" collapsed = _WS_RE.sub(" ", text or "").strip() return collapsed[:limit] def message_text(msg: dict[str, Any]) -> str: """Return the plain-text content of a serialized LangChain message.""" content = msg.get("content") if isinstance(content, str): return content.strip() if isinstance(content, list): parts: list[str] = [] for block in content: if isinstance(block, str): parts.append(block) elif isinstance(block, dict): if block.get("type") in (None, "text") and isinstance(block.get("text"), str): parts.append(block["text"]) return "\n".join(p for p in parts if p).strip() return "" def _msg_type(msg: dict[str, Any]) -> str: return str(msg.get("type") or msg.get("role") or "").lower() def split_turns(messages: list[dict[str, Any]], *, max_messages: int = 80) -> list[dict[str, str]]: """Split messages into ``{user, ai}`` turns keyed by human messages. For each turn we keep the last substantial AI message (with text body). """ msgs = messages[-max_messages:] if max_messages and len(messages) > max_messages else list(messages) turns: list[dict[str, str]] = [] current: dict[str, str] | None = None for msg in msgs: mtype = _msg_type(msg) text = message_text(msg) if mtype == "human": if current and (current.get("user") or current.get("ai")): turns.append(current) current = {"user": text, "ai": ""} elif mtype == "ai" and text: if current is None: current = {"user": "", "ai": text} else: current["ai"] = text # last substantial AI message wins if current and (current.get("user") or current.get("ai")): turns.append(current) # Drop turns that are pure greetings with no answer. cleaned = [] for t in turns: user = (t.get("user") or "").strip() ai = (t.get("ai") or "").strip() if not ai and _GREETING_RE.match(user): continue if not user and not ai: continue cleaned.append({"user": user, "ai": ai}) return cleaned def _auto_title(thread_title: str | None, turns: list[dict[str, str]]) -> str: if thread_title and thread_title.strip() and thread_title.strip().lower() != "untitled": return thread_title.strip()[:200] for t in turns: if t.get("user"): first = t["user"].splitlines()[0].strip() return (first[:40] + ("…" if len(first) > 40 else "")) or "知识笔记" return "知识笔记" def _summary(turns: list[dict[str, str]], *, limit: int = 280) -> str: """Simple rule-based summary: the first answer, clipped.""" for t in turns: if t.get("ai"): text = re.sub(r"\s+", " ", t["ai"]).strip() return text[:limit] + ("…" if len(text) > limit else "") for t in turns: if t.get("user"): text = re.sub(r"\s+", " ", t["user"]).strip() return text[:limit] + ("…" if len(text) > limit else "") return "" def related_links(project_name: str | None) -> list[str]: # noqa: ARG001 """Structural wikilinks every note used to carry. Product decision: notes no longer auto-link the vault meta/hub pages (``index`` / ``_meta/taxonomy`` / the project hub) — they were noise in every note's ``## Related``. Only real, content-specific related notes (injected by the service layer, which has DB access) are linked now. Kept as a function (returns empty) so existing callers and the wiki-capture contract stay intact. """ return [] # Backwards-compatible internal alias. _related_links = related_links def _related_section_md(related_lines: list[str], source_lines: list[str] | None = None) -> str: # noqa: ARG001 """Render the ``## Related`` section, or ``""`` when there is nothing to show. Sources are intentionally **not** rendered into the body anymore. The note detail page already shows every source once via the structured ``来源`` block (``note.sources``); emitting a ``### Sources`` sub-section here duplicated that list (and broke the frontend's clean ``## Related`` extraction). The ``source_lines`` argument is kept for backward compatibility and ignored. """ if not related_lines: return "" inner = "\n".join(related_lines) return f"## Related\n\n{inner}\n" def _build_body( *, title: str, turns: list[dict[str, str]], mode: str, project_name: str | None, sources: list[SourceDraft], ) -> str: """Render the wiki-capture 5-section declarative body.""" full = mode == "full" answer_clip = 4000 if full else 800 context_lines = [] for t in turns: if t.get("user"): q = re.sub(r"\s+", " ", t["user"]).strip() context_lines.append(f"- {q[:300]}") context = "\n".join(context_lines[:8]) or "- 来自一次对话沉淀。^[inferred]" finding_lines = [] for i, t in enumerate(turns, 1): if t.get("ai"): ans = t["ai"].strip() ans = ans if full else (ans[:answer_clip] + ("…" if len(ans) > answer_clip else "")) finding_lines.append(f"### 结论 {i}\n\n{ans}") finding = "\n\n".join(finding_lines) or "_暂无可提炼的结论。_" reasoning = "本页由对话自动沉淀而成,结论来自助手最终回答;过程性推理已省略。判断与归纳部分标记为 ^[inferred]。" implications = "- 后续可在相关问答中参考本页知识。\n- 如内容过期或冲突,请编辑或归档。" related_lines = [f"- {link}" for link in _related_links(project_name)] source_lines = [f"- {s.title or s.url or '来源'}" + (f" — {s.url}" if s.url else "") for s in sources[:20]] related_section = _related_section_md(related_lines, source_lines) return ( f"# {title}\n\n" f"## Context\n\n{context}\n\n" f"## Finding / Decision\n\n{finding}\n\n" f"## Reasoning\n\n{reasoning}\n\n" f"## Implications\n\n{implications}\n\n" f"{related_section}" ) def extract_thread( messages: list[dict[str, Any]], *, thread_id: str, thread_title: str | None = None, mode: str = "summary", title: str | None = None, tags: list[str] | None = None, project_name: str | None = "zncm", sources: list[SourceDraft] | None = None, max_messages: int = 80, ) -> NoteDraft | None: """Build a :class:`NoteDraft` from thread messages, or ``None`` if empty.""" turns = split_turns(messages, max_messages=max_messages) if not turns: return None final_title = (title or "").strip() or _auto_title(thread_title, turns) summary = _summary(turns) sources = sources or [] body = _build_body(title=final_title, turns=turns, mode=mode, project_name=project_name, sources=sources) base_tags = list(tags or []) for default in (project_name or "zncm", "knowledge-base"): if default and default not in base_tags: base_tags.append(default) return NoteDraft( title=final_title, summary=summary, content_md=body, category="synthesis", source_type="thread", source_id=thread_id, tags=base_tags, confidence=0.6, related=_related_links(project_name), sources=sources, source_key=f"thread:{thread_id}", ) def extract_search( query: str, sources: list[SourceDraft], *, thread_id: str | None = None, project_name: str | None = "zncm", title: str | None = None, tags: list[str] | None = None, ) -> NoteDraft: """Build a reference-category :class:`NoteDraft` from search/tool results.""" final_title = (title or "").strip() or f"检索:{query[:40]}" body_lines = [f"# {final_title}\n", "## Context\n", f"- 检索关键词:{query}\n", "## Finding / Decision\n"] for s in sources: label = s.title or s.url or "来源" line = f"- **{label}**" if s.url: line += f" — {s.url}" if s.snippet: line += "\n - " + _one_line(s.snippet) body_lines.append(line) body_lines += [ "\n## Reasoning\n", "本页汇总外部检索来源,供后续问答引用。来源可信度需结合原文判断。^[ambiguous]\n", "## Implications\n", "- 引用前请核对原始来源。\n", "## Related\n", *[f"- {link}" for link in _related_links(project_name)], ] base_tags = list(tags or []) for default in (project_name or "zncm", "knowledge-base", "reference"): if default and default not in base_tags: base_tags.append(default) return NoteDraft( title=final_title, summary=f"检索“{query}”的来源汇总,共 {len(sources)} 条。", content_md="\n".join(body_lines), category="references", source_type="search", source_id=f"search:{query[:120]}", tags=base_tags, confidence=0.4, related=_related_links(project_name), sources=sources, source_key=None, )