"""Duplicate detection for knowledge notes (phase 3).

First-release heuristic: token Jaccard over ``title + summary + tags`` plus an
exact ``content_md`` hash match. Cheap, deterministic and embedding-free so it
works on offline deployments. Returns *suggestions*; merging is an explicit,
user-driven action.
"""

from __future__ import annotations

import re
from typing import Any

from deerflow.knowledge.manifest import content_hash

_TOKEN_RE = re.compile(r"[\w一-鿿]+", re.UNICODE)


def _tokens(*texts: str) -> set[str]:
    out: set[str] = set()
    for t in texts:
        out.update(m.lower() for m in _TOKEN_RE.findall(t or ""))
    return out


def _jaccard(a: set[str], b: set[str]) -> float:
    if not a or not b:
        return 0.0
    inter = len(a & b)
    union = len(a | b)
    return inter / union if union else 0.0


def find_duplicate_groups(notes: list[dict[str, Any]], *, threshold: float = 0.82, max_notes: int = 800) -> list[dict[str, Any]]:
    """Return groups ``{primary, duplicates:[{id,title,score,reason}]}``.

    ``primary`` is the most recently updated note in each cluster.
    """
    notes = notes[:max_notes]
    feats: list[tuple[dict[str, Any], set[str], str]] = []
    for n in notes:
        toks = _tokens(n.get("title") or "", n.get("summary") or "", " ".join(n.get("tags") or []))
        feats.append((n, toks, content_hash(n.get("content_md") or "")))

    used: set[str] = set()
    groups: list[dict[str, Any]] = []
    for i in range(len(feats)):
        ni, ti, hi = feats[i]
        if ni["id"] in used:
            continue
        dups: list[dict[str, Any]] = []
        for j in range(i + 1, len(feats)):
            nj, tj, hj = feats[j]
            if nj["id"] in used:
                continue
            if hi and hi == hj:
                score, reason = 1.0, "identical-content"
            else:
                score = _jaccard(ti, tj)
                reason = "similar-title-summary"
            if score >= threshold:
                dups.append({"id": nj["id"], "title": nj.get("title"), "score": round(score, 4), "reason": reason})
                used.add(nj["id"])
        if dups:
            used.add(ni["id"])
            # Primary = newest by updated_at.
            cluster = [ni, *[next(f[0] for f in feats if f[0]["id"] == d["id"]) for d in dups]]
            primary = max(cluster, key=lambda x: x.get("updated_at") or "")
            others = [
                {"id": c["id"], "title": c.get("title"), "score": next((d["score"] for d in dups if d["id"] == c["id"]), 1.0)}
                for c in cluster
                if c["id"] != primary["id"]
            ]
            groups.append({"primary": {"id": primary["id"], "title": primary.get("title")}, "duplicates": others})
    return groups