83 lines
2.9 KiB
Python
83 lines
2.9 KiB
Python
"""Normalize knowledge sources from threads / searches / tool results.
|
|
|
|
Source items come from a few shapes: backend reference-batches (built by
|
|
``deerflow.runtime.references``), raw search-result payloads, and tool-call
|
|
messages. They are all normalized into :class:`SourceDraft` so the repository
|
|
and frontend see one consistent structure.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
from deerflow.knowledge.schemas import SourceDraft
|
|
|
|
|
|
def _clip(value: Any, limit: int = 2000) -> str | None:
|
|
if value is None:
|
|
return None
|
|
text = str(value).strip()
|
|
if not text:
|
|
return None
|
|
return text[:limit]
|
|
|
|
|
|
def from_reference_batches(batches: list[dict[str, Any]] | None, *, thread_id: str | None = None, max_items: int = 20) -> list[SourceDraft]:
|
|
"""Convert backend reference-batches into :class:`SourceDraft`."""
|
|
out: list[SourceDraft] = []
|
|
seen: set[str] = set()
|
|
for batch in batches or []:
|
|
msg_id = batch.get("assistant_message_id") or batch.get("id")
|
|
for src in batch.get("sources", []) or []:
|
|
if not isinstance(src, dict):
|
|
continue
|
|
url = _clip(src.get("url"), 1024)
|
|
title = _clip(src.get("title"), 512) or _clip(src.get("source"), 512) or "来源"
|
|
dedup_key = url or f"{title}:{_clip(src.get('snippet'), 80)}"
|
|
if dedup_key in seen:
|
|
continue
|
|
seen.add(dedup_key)
|
|
out.append(
|
|
SourceDraft(
|
|
source_type="search" if url else "tool",
|
|
thread_id=thread_id,
|
|
message_id=_clip(msg_id, 128),
|
|
tool_name=_clip(src.get("skillName"), 128),
|
|
title=title,
|
|
url=url,
|
|
snippet=_clip(src.get("snippet")),
|
|
raw=None,
|
|
)
|
|
)
|
|
if len(out) >= max_items:
|
|
return out
|
|
return out
|
|
|
|
|
|
def from_search_results(
|
|
query: str,
|
|
results: list[dict[str, Any]] | None,
|
|
*,
|
|
thread_id: str | None = None,
|
|
message_id: str | None = None,
|
|
max_items: int = 20,
|
|
) -> list[SourceDraft]:
|
|
"""Convert raw search results (title/url/snippet) into :class:`SourceDraft`."""
|
|
out: list[SourceDraft] = []
|
|
for item in (results or [])[:max_items]:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
out.append(
|
|
SourceDraft(
|
|
source_type="search",
|
|
thread_id=thread_id,
|
|
message_id=message_id,
|
|
tool_name=_clip(item.get("tool_name") or item.get("tool"), 128),
|
|
title=_clip(item.get("title"), 512) or query[:200],
|
|
url=_clip(item.get("url"), 1024),
|
|
snippet=_clip(item.get("snippet") or item.get("content")),
|
|
raw={k: item.get(k) for k in ("score",) if item.get(k) is not None} or None,
|
|
)
|
|
)
|
|
return out
|