deerflow-code/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/exporter.py
2026-09-07 18:24:55 +08:00

464 lines
19 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""obsidian-wiki ``wiki-export`` compatible graph generation.
Builds a knowledge graph from the notes (nodes) plus their wikilinks and shared
tags (edges) and writes the four native artifacts into ``<vault>/wiki-export/``:
``graph.json``, ``graph.graphml``, ``cypher.txt`` and a self-contained
``graph.html`` (no external CDN, safe to iframe offline).
"""
from __future__ import annotations
import html
import json
from pathlib import Path
from typing import Any
from deerflow.knowledge.markdown_vault import extract_wikilinks
def _vault_key(vault_path: str | None) -> str | None:
if not vault_path:
return None
key = vault_path.replace("\\", "/")
if key.endswith(".md"):
key = key[:-3]
return key
def normalize_blacklist(labels: list[str] | set[str] | None) -> set[str]:
"""Normalize blacklist labels for matching (trimmed, lower-cased, non-empty)."""
out: set[str] = set()
for raw in labels or []:
label = (raw or "").strip().lower()
if label:
out.add(label)
return out
def build_graph(
notes: list[dict[str, Any]],
entities: list[dict[str, Any]] | None = None,
relations: list[dict[str, Any]] | None = None,
*,
blacklist: set[str] | None = None,
) -> dict[str, Any]:
"""Return ``{nodes, edges}`` from notes (+ optional entities/relations).
Note nodes come from ``notes`` (id, title, tags, vault_path, content_md);
when ``entities``/``relations`` are supplied (phase 4), entity nodes and
note→entity / entity→entity edges are added on top of the wikilink and
shared-tag edges.
``blacklist`` is a set of *normalized* labels (see :func:`normalize_blacklist`);
notes whose title and entities whose name match are dropped, together with
every edge touching them (edges only join surviving nodes).
"""
blocked = blacklist or set()
def _is_blocked(label: str | None) -> bool:
return bool(blocked) and (label or "").strip().lower() in blocked
# Drop blacklisted notes up front so the wikilink / shared-tag edge loops
# below never reference them either.
notes = [n for n in notes if not _is_blocked(n.get("title"))]
nodes = []
by_key: dict[str, str] = {}
note_ids: set[str] = set()
for n in notes:
nodes.append({"id": n["id"], "type": "note", "label": n.get("title") or n["id"], "category": n.get("source_type")})
note_ids.add(n["id"])
key = _vault_key(n.get("vault_path"))
if key:
by_key[key] = n["id"]
# Only surface entities that are still mentioned by a LIVE note. Entities whose
# only referencing notes were deleted/archived would otherwise linger forever
# as orphan nodes ("graph doesn't follow my deletions").
mentioned: set[str] = set()
for rel in relations or []:
if rel.get("from_note_id") in note_ids and rel.get("to_entity_id"):
mentioned.add(rel["to_entity_id"])
entity_ids: set[str] = set()
for ent in entities or []:
if ent["id"] not in mentioned:
continue
if _is_blocked(ent.get("name")):
continue
nodes.append({"id": ent["id"], "type": "entity", "label": ent.get("name") or ent["id"], "category": ent.get("entity_type")})
entity_ids.add(ent["id"])
edges: list[dict[str, Any]] = []
edge_seen: set[tuple[str, str, str]] = set()
def add_edge(a: str, b: str, label: str, weight: float) -> None:
if a == b:
return
key = (min(a, b), max(a, b), label)
if key in edge_seen:
return
edge_seen.add(key)
edges.append({"id": f"rel_{len(edges) + 1}", "source": a, "target": b, "label": label, "weight": weight})
# Wikilink edges.
for n in notes:
for target in extract_wikilinks(n.get("content_md") or ""):
tid = by_key.get(target.replace("\\", "/").removesuffix(".md"))
if tid:
add_edge(n["id"], tid, "links", 2)
# Shared-tag edges.
tag_to_notes: dict[str, list[str]] = {}
for n in notes:
for tag in n.get("tags") or []:
tag_to_notes.setdefault(tag.lower(), []).append(n["id"])
for tag, ids in tag_to_notes.items():
if len(ids) < 2 or len(ids) > 50: # skip ubiquitous tags (e.g. knowledge-base)
continue
for i in range(len(ids)):
for j in range(i + 1, len(ids)):
add_edge(ids[i], ids[j], "related", 1)
# Entity / relation edges (phase 4) — only when both endpoints exist as nodes.
valid = note_ids | entity_ids
for rel in relations or []:
src = rel.get("from_note_id") or rel.get("from_entity_id")
tgt = rel.get("to_note_id") or rel.get("to_entity_id")
if src in valid and tgt in valid:
add_edge(src, tgt, rel.get("relation_type") or "related", float(rel.get("weight") or 1))
return {"nodes": nodes, "edges": edges}
def to_graphml(graph: dict[str, Any]) -> str:
lines = [
'<?xml version="1.0" encoding="UTF-8"?>',
'<graphml xmlns="http://graphml.graphdrawing.org/xmlns">',
' <key id="label" for="node" attr.name="label" attr.type="string"/>',
' <key id="elabel" for="edge" attr.name="label" attr.type="string"/>',
' <graph edgedefault="undirected">',
]
for n in graph["nodes"]:
lines.append(f' <node id="{html.escape(str(n["id"]))}"><data key="label">{html.escape(str(n["label"]))}</data></node>')
for e in graph["edges"]:
lines.append(
f' <edge id="{html.escape(str(e["id"]))}" source="{html.escape(str(e["source"]))}" '
f'target="{html.escape(str(e["target"]))}"><data key="elabel">{html.escape(str(e["label"]))}</data></edge>'
)
lines += [" </graph>", "</graphml>"]
return "\n".join(lines)
def to_cypher(graph: dict[str, Any]) -> str:
lines = []
for n in graph["nodes"]:
label = str(n["label"]).replace("'", "\\'")
lines.append(f"CREATE (:Note {{id:'{n['id']}', title:'{label}'}});")
for e in graph["edges"]:
rel = str(e["label"]).upper()
lines.append(f"MATCH (a:Note {{id:'{e['source']}'}}),(b:Note {{id:'{e['target']}'}}) CREATE (a)-[:{rel} {{weight:{e['weight']}}}]->(b);")
return "\n".join(lines)
def to_html(graph: dict[str, Any]) -> str:
"""Self-contained, theme-aware force-directed graph (canvas + vanilla JS, no CDN).
The page reads ``?theme=light|dark`` from its own URL so the embedding iframe
can stay in sync with the app's day/night mode (an iframe cannot inherit the
parent's CSS). The layout settles and then **freezes** so a dense graph stops
drifting and stays clickable; node size scales with degree; and hovering or
clicking a node focuses it + its direct neighbours while dimming everything
else, which cuts through heavy node/edge clutter.
"""
data_json = json.dumps(graph, ensure_ascii=False)
return _HTML_TEMPLATE.replace("__GRAPH_DATA__", data_json)
def export_all(
vault_root: Path | str,
notes: list[dict[str, Any]],
*,
entities: list[dict[str, Any]] | None = None,
relations: list[dict[str, Any]] | None = None,
write_html: bool = True,
write_json: bool = True,
blacklist: set[str] | None = None,
) -> dict[str, Any]:
"""Generate graph artifacts under ``<vault>/wiki-export/`` and return stats."""
out_dir = Path(vault_root).resolve() / "wiki-export"
out_dir.mkdir(parents=True, exist_ok=True)
graph = build_graph(notes, entities, relations, blacklist=blacklist)
written: list[str] = []
if write_json:
(out_dir / "graph.json").write_text(json.dumps(graph, ensure_ascii=False, indent=2), encoding="utf-8")
written.append("graph.json")
(out_dir / "graph.graphml").write_text(to_graphml(graph), encoding="utf-8")
written.append("graph.graphml")
(out_dir / "cypher.txt").write_text(to_cypher(graph), encoding="utf-8")
written.append("cypher.txt")
if write_html:
(out_dir / "graph.html").write_text(to_html(graph), encoding="utf-8")
written.append("graph.html")
return {
"files": written,
"stats": {"nodes": len(graph["nodes"]), "edges": len(graph["edges"])},
"graph": graph,
}
_HTML_TEMPLATE = """<!DOCTYPE html>
<html lang="zh">
<head>
<meta charset="utf-8"/>
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<title>Knowledge Graph</title>
<style>
:root{
--bg:#0b1020; --fg:#e7ecff; --sub:rgba(231,236,255,.78);
--panel:rgba(17,26,54,.92); --panel-border:#2a3a66;
}
html.light{
--bg:#f6f8fc; --fg:#1f2433; --sub:rgba(31,36,51,.72);
--panel:rgba(255,255,255,.94); --panel-border:#d7deec;
}
html,body{margin:0;height:100%;background:var(--bg);color:var(--fg);font-family:system-ui,Segoe UI,Arial,sans-serif;overflow:hidden}
canvas{display:block;cursor:grab}
canvas.grabbing{cursor:grabbing}
.card{position:fixed;background:var(--panel);border:1px solid var(--panel-border);border-radius:8px;font-size:12px;z-index:3;backdrop-filter:blur(6px)}
#hud{top:10px;left:12px;padding:6px 10px;color:var(--sub);z-index:2}
#legend{top:10px;right:12px;padding:8px 10px;display:flex;flex-direction:column;gap:6px}
#legend .row{display:flex;align-items:center;gap:6px;color:var(--sub)}
#legend .dot{width:10px;height:10px;border-radius:50%}
#tip{padding:4px 8px;pointer-events:none;display:none}
#panel{left:12px;bottom:12px;max-width:300px;padding:12px 14px;display:none}
#panel h3{margin:0 0 6px;font-size:14px;line-height:1.3;word-break:break-word}
#panel .meta{color:var(--sub);margin-bottom:8px}
#panel .nb{display:flex;flex-wrap:wrap;gap:5px;max-height:120px;overflow:auto}
#panel .nb span{padding:2px 7px;border:1px solid var(--panel-border);border-radius:999px;line-height:1.4;max-width:100%;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
#panel .close{position:absolute;top:8px;right:10px;cursor:pointer;color:var(--sub);font-size:14px}
#reset{bottom:12px;right:12px;padding:7px 12px;cursor:pointer;color:var(--fg);user-select:none}
#reset:hover{filter:brightness(1.1)}
#empty{position:fixed;inset:0;display:flex;align-items:center;justify-content:center;opacity:.55}
</style>
</head>
<body>
<div id="hud" class="card"></div>
<div id="legend" class="card">
<div class="row"><span class="dot" id="lg-note"></span>知识笔记</div>
<div class="row"><span class="dot" id="lg-entity"></span>实体</div>
</div>
<div id="tip" class="card"></div>
<div id="panel" class="card"><span class="close" id="panel-close">×</span><h3></h3><div class="meta"></div><div class="nb"></div></div>
<div id="reset" class="card">重置视图</div>
<canvas id="c"></canvas>
<script>
const GRAPH = __GRAPH_DATA__;
// Theme is supplied by the embedding iframe (it can't read the parent's CSS).
const THEME = (new URLSearchParams(location.search).get('theme') === 'light') ? 'light' : 'dark';
const PAL = {
dark: {note:'#5b8cff', entity:'#ffb454', label:'rgba(231,236,255,.92)', edge:[150,178,255], halo:'rgba(11,16,32,.55)'},
light: {note:'#2f6df6', entity:'#d97706', label:'rgba(31,36,51,.92)', edge:[63,96,168], halo:'rgba(255,255,255,.7)'},
}[THEME];
document.documentElement.classList.toggle('light', THEME === 'light');
document.getElementById('lg-note').style.background = PAL.note;
document.getElementById('lg-entity').style.background = PAL.entity;
const canvas = document.getElementById('c'), ctx = canvas.getContext('2d');
const hud = document.getElementById('hud'), tip = document.getElementById('tip');
const panel = document.getElementById('panel');
let W=0,H=0,DPR=Math.min(window.devicePixelRatio||1,2);
function resize(){W=innerWidth;H=innerHeight;canvas.width=W*DPR;canvas.height=H*DPR;canvas.style.width=W+'px';canvas.style.height=H+'px';}
addEventListener('resize',()=>{resize();bump();});resize();
hud.textContent = GRAPH.nodes.length + ' 个知识节点 · ' + GRAPH.edges.length + ' 条关系';
if(!GRAPH.nodes.length){document.body.insertAdjacentHTML('beforeend','<div id="empty">暂无知识图谱数据</div>');}
// --- index, adjacency, degree, node sizing -------------------------------
const idx={}, adj={};
GRAPH.nodes.forEach(n=>{idx[n.id]=n;adj[n.id]=new Set();});
const edges=[];
for(const e of GRAPH.edges){
const s=idx[e.source], t=idx[e.target];
if(!s||!t||s===t) continue;
edges.push({s,t,w:e.weight||e.w||1,label:e.label||''});
adj[e.source].add(e.target); adj[e.target].add(e.source);
}
const N=GRAPH.nodes.length;
GRAPH.nodes.forEach((n,i)=>{
n.deg=adj[n.id].size;
n.r=4.5+Math.min(11,Math.sqrt(n.deg)*2.4); // hubs are visibly bigger → easier to hit
const a=i*2.399963, rad=18*Math.sqrt(i); // phyllotaxis spread = no initial overlap
n.x=Math.cos(a)*rad; n.y=Math.sin(a)*rad; n.vx=0; n.vy=0;
});
// "Hub" nodes keep their label even when zoomed out; the rest declutter.
const degs=GRAPH.nodes.map(n=>n.deg).sort((a,b)=>b-a);
const HUB=Math.max(3,(degs[Math.floor(N*0.08)]||3));
// --- force layout that cools down and FREEZES (frozen graph = clickable) --
const REP=9000, LINK=92, CENTER=0.0016;
let alpha=1, drag=null;
function physics(){
for(let i=0;i<N;i++){
const n=GRAPH.nodes[i];
for(let j=i+1;j<N;j++){
const m=GRAPH.nodes[j];
let dx=n.x-m.x,dy=n.y-m.y,d2=dx*dx+dy*dy+0.01,f=REP/d2;
let fx=dx*f,fy=dy*f; n.vx+=fx;n.vy+=fy;m.vx-=fx;m.vy-=fy;
}
}
for(const e of edges){
let dx=e.t.x-e.s.x,dy=e.t.y-e.s.y,d=Math.hypot(dx,dy)||1,f=(d-LINK)*0.02;
e.s.vx+=dx/d*f;e.s.vy+=dy/d*f;e.t.vx-=dx/d*f;e.t.vy-=dy/d*f;
}
for(const n of GRAPH.nodes){
if(n===drag){n.vx=n.vy=0;continue;}
n.vx-=n.x*CENTER; n.vy-=n.y*CENTER;
n.x+=(n.vx*=0.82)*alpha; n.y+=(n.vy*=0.82)*alpha;
}
alpha*=0.985;
}
// --- camera / fit-to-view -------------------------------------------------
let cam={x:0,y:0,k:1};
function toWorld(x,y){return {x:(x-cam.x)/cam.k,y:(y-cam.y)/cam.k};}
function fitView(){
if(!N){return;}
let a=1e9,b=1e9,c=-1e9,d=-1e9;
for(const n of GRAPH.nodes){a=Math.min(a,n.x);b=Math.min(b,n.y);c=Math.max(c,n.x);d=Math.max(d,n.y);}
const gw=Math.max(c-a,1),gh=Math.max(d-b,1),pad=90;
cam.k=Math.max(0.12,Math.min(2.2,Math.min((W-pad)/gw,(H-pad)/gh)));
cam.x=W/2-((a+c)/2)*cam.k; cam.y=H/2-((b+d)/2)*cam.k;
}
let fitted=false;
function bump(){alpha=Math.max(alpha,0.35);} // re-warm the sim on interaction
// --- focus / hover state --------------------------------------------------
let hoverNode=null, focusNode=null;
function active(){return focusNode||hoverNode;}
function neighbors(n){return n?adj[n.id]:null;}
function draw(){
ctx.setTransform(DPR,0,0,DPR,0,0);
ctx.clearRect(0,0,W,H);
ctx.translate(cam.x,cam.y); ctx.scale(cam.k,cam.k);
const act=active(), nb=neighbors(act), zoomed=cam.k>1.35;
const [er,eg,eb]=PAL.edge;
// edges: clearly visible by default; when a node is focused, only its edges
// stay bright and the rest fade back (but never to invisible).
for(const e of edges){
const lit = act && (e.s===act||e.t===act);
const op = act ? (lit?0.98:0.12) : Math.min(0.85,0.55+e.w*0.06);
// Keep a stable on-screen thickness regardless of zoom (>=1px), so lines
// never thin out to a hairline when zoomed in or on dense graphs.
const screenW = lit ? 2.6 : Math.min(2.4,1.4+e.w*0.45);
ctx.strokeStyle='rgba('+er+','+eg+','+eb+','+op+')';
ctx.lineWidth=Math.max(1,screenW)/cam.k;
ctx.beginPath();ctx.moveTo(e.s.x,e.s.y);ctx.lineTo(e.t.x,e.t.y);ctx.stroke();
}
// nodes
for(const n of GRAPH.nodes){
const dim = act && n!==act && !(nb&&nb.has(n.id));
ctx.globalAlpha = dim?0.22:1;
ctx.fillStyle = n.type==='entity'?PAL.entity:PAL.note;
ctx.beginPath();ctx.arc(n.x,n.y,n.r,0,6.2832);ctx.fill();
if(n===act){ctx.lineWidth=2/cam.k;ctx.strokeStyle=PAL.label;ctx.stroke();}
ctx.globalAlpha=1;
}
// labels: only for focus/neighbors, hubs, or when zoomed in — kills clutter.
ctx.textBaseline='middle';
for(const n of GRAPH.nodes){
const show = act ? (n===act||(nb&&nb.has(n.id))) : (zoomed||n.deg>=HUB);
if(!show) continue;
const txt=(n.label||'').slice(0,20);
if(!txt) continue;
ctx.font=(n===act?'bold ':'')+(11/cam.k)+'px system-ui';
const tx=n.x+n.r+4/cam.k, ty=n.y;
ctx.lineWidth=3/cam.k; ctx.strokeStyle=PAL.halo;
ctx.strokeText(txt,tx,ty);
ctx.fillStyle=PAL.label; ctx.fillText(txt,tx,ty);
}
}
function loop(){
if(alpha>0.02 || drag){physics();} else if(!fitted){fitView();fitted=true;}
draw();
requestAnimationFrame(loop);
}
// Pre-settle a few iterations so the first paint is already roughly laid out.
for(let i=0;i<60;i++)physics();
fitView();
loop();
// --- picking & interaction ------------------------------------------------
function pick(p){
let best=null,bd=1e9;
for(const n of GRAPH.nodes){
const dx=n.x-p.x,dy=n.y-p.y,d=dx*dx+dy*dy, hit=(n.r+8/cam.k);
if(d<hit*hit && d<bd){bd=d;best=n;}
}
return best;
}
let pan=null, downXY=null, moved=false;
canvas.addEventListener('mousedown',ev=>{
const p=toWorld(ev.clientX,ev.clientY), hit=pick(p);
downXY={x:ev.clientX,y:ev.clientY,hit}; moved=false;
if(hit){drag=hit;} else {pan={x:ev.clientX,y:ev.clientY,cx:cam.x,cy:cam.y};}
canvas.classList.add('grabbing');
});
addEventListener('mousemove',ev=>{
const p=toWorld(ev.clientX,ev.clientY);
if(downXY && (Math.abs(ev.clientX-downXY.x)>3||Math.abs(ev.clientY-downXY.y)>3)) moved=true;
if(drag){drag.x=p.x;drag.y=p.y;drag.vx=drag.vy=0;bump();}
else if(pan){cam.x=pan.cx+ev.clientX-pan.x;cam.y=pan.cy+ev.clientY-pan.y;}
hoverNode=drag||pan?hoverNode:pick(p);
if(hoverNode && !drag && !pan){
tip.style.display='block';
tip.style.left=Math.min(ev.clientX+14,W-220)+'px';
tip.style.top=(ev.clientY+14)+'px';
tip.textContent=hoverNode.label||'';
} else { tip.style.display='none'; }
});
addEventListener('mouseup',ev=>{
if(downXY && !moved){ // a click, not a drag → toggle focus / detail panel
focusNode = (downXY.hit && downXY.hit!==focusNode) ? downXY.hit : null;
renderPanel();
}
drag=null;pan=null;downXY=null;canvas.classList.remove('grabbing');
});
canvas.addEventListener('wheel',ev=>{
ev.preventDefault();
const s=ev.deltaY<0?1.12:0.89;
const nk=Math.max(0.08,Math.min(6,cam.k*s)), r=nk/cam.k;
cam.x=ev.clientX-(ev.clientX-cam.x)*r; cam.y=ev.clientY-(ev.clientY-cam.y)*r; cam.k=nk;
},{passive:false});
// --- detail panel ---------------------------------------------------------
const TYPE_LABEL={note:'知识笔记',entity:'实体'};
function renderPanel(){
if(!focusNode){panel.style.display='none';return;}
const n=focusNode;
panel.querySelector('h3').textContent=n.label||n.id;
const cat=n.category?' · '+n.category:'';
panel.querySelector('.meta').textContent=(TYPE_LABEL[n.type]||n.type||'节点')+cat+' · '+n.deg+' 条关系';
const nb=panel.querySelector('.nb'); nb.innerHTML='';
let count=0;
for(const id of adj[n.id]){
if(count++>=24)break;
const m=idx[id]; if(!m)continue;
const s=document.createElement('span'); s.textContent=m.label||id;
s.title=m.label||id;
s.style.cursor='pointer';
s.onclick=()=>{focusNode=m;renderPanel();};
nb.appendChild(s);
}
if(!adj[n.id].size){const s=document.createElement('span');s.textContent='(无关联)';s.style.opacity=.6;nb.appendChild(s);}
panel.style.display='block';
}
document.getElementById('panel-close').onclick=()=>{focusNode=null;renderPanel();};
document.getElementById('reset').onclick=()=>{focusNode=null;renderPanel();fitView();};
</script>
</body>
</html>"""