76 lines
2.7 KiB
Python
76 lines
2.7 KiB
Python
"""Duplicate detection for knowledge notes (phase 3).
|
|
|
|
First-release heuristic: token Jaccard over ``title + summary + tags`` plus an
|
|
exact ``content_md`` hash match. Cheap, deterministic and embedding-free so it
|
|
works on offline deployments. Returns *suggestions*; merging is an explicit,
|
|
user-driven action.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from typing import Any
|
|
|
|
from deerflow.knowledge.manifest import content_hash
|
|
|
|
_TOKEN_RE = re.compile(r"[\w一-鿿]+", re.UNICODE)
|
|
|
|
|
|
def _tokens(*texts: str) -> set[str]:
|
|
out: set[str] = set()
|
|
for t in texts:
|
|
out.update(m.lower() for m in _TOKEN_RE.findall(t or ""))
|
|
return out
|
|
|
|
|
|
def _jaccard(a: set[str], b: set[str]) -> float:
|
|
if not a or not b:
|
|
return 0.0
|
|
inter = len(a & b)
|
|
union = len(a | b)
|
|
return inter / union if union else 0.0
|
|
|
|
|
|
def find_duplicate_groups(notes: list[dict[str, Any]], *, threshold: float = 0.82, max_notes: int = 800) -> list[dict[str, Any]]:
|
|
"""Return groups ``{primary, duplicates:[{id,title,score,reason}]}``.
|
|
|
|
``primary`` is the most recently updated note in each cluster.
|
|
"""
|
|
notes = notes[:max_notes]
|
|
feats: list[tuple[dict[str, Any], set[str], str]] = []
|
|
for n in notes:
|
|
toks = _tokens(n.get("title") or "", n.get("summary") or "", " ".join(n.get("tags") or []))
|
|
feats.append((n, toks, content_hash(n.get("content_md") or "")))
|
|
|
|
used: set[str] = set()
|
|
groups: list[dict[str, Any]] = []
|
|
for i in range(len(feats)):
|
|
ni, ti, hi = feats[i]
|
|
if ni["id"] in used:
|
|
continue
|
|
dups: list[dict[str, Any]] = []
|
|
for j in range(i + 1, len(feats)):
|
|
nj, tj, hj = feats[j]
|
|
if nj["id"] in used:
|
|
continue
|
|
if hi and hi == hj:
|
|
score, reason = 1.0, "identical-content"
|
|
else:
|
|
score = _jaccard(ti, tj)
|
|
reason = "similar-title-summary"
|
|
if score >= threshold:
|
|
dups.append({"id": nj["id"], "title": nj.get("title"), "score": round(score, 4), "reason": reason})
|
|
used.add(nj["id"])
|
|
if dups:
|
|
used.add(ni["id"])
|
|
# Primary = newest by updated_at.
|
|
cluster = [ni, *[next(f[0] for f in feats if f[0]["id"] == d["id"]) for d in dups]]
|
|
primary = max(cluster, key=lambda x: x.get("updated_at") or "")
|
|
others = [
|
|
{"id": c["id"], "title": c.get("title"), "score": next((d["score"] for d in dups if d["id"] == c["id"]), 1.0)}
|
|
for c in cluster
|
|
if c["id"] != primary["id"]
|
|
]
|
|
groups.append({"primary": {"id": primary["id"], "title": primary.get("title")}, "duplicates": others})
|
|
return groups
|