95 lines
4.7 KiB
Python
95 lines
4.7 KiB
Python
"""Build gold reports and reproducible failure instances from task annotations."""
|
||
|
||
from __future__ import annotations
|
||
|
||
from app.report_collaboration.eval.schema import EvalTask, FailureKind, ReportCandidate, UsageSnapshot
|
||
from app.report_collaboration.eval.thresholds import REQUIRED_EVENTS
|
||
|
||
|
||
def materialize_gold(task: EvalTask) -> ReportCandidate:
|
||
sections = list(task.template.sections)
|
||
body: list[str] = [f"# {sections[0]}"]
|
||
body.append(f"{task.title}。时间范围：{task.time_range}。受众：{task.audience}。")
|
||
body.append("必选角度：" + "、".join(task.required_angles) + "。")
|
||
for fact in task.key_facts:
|
||
body.append(f"{fact.text}（{fact.id}）。")
|
||
if task.template.required_questions:
|
||
body.append("必答问题：" + " ".join(task.template.required_questions))
|
||
body.append("回答：" + "；".join(f"{fact.number}" for fact in task.key_facts[:2]))
|
||
if task.template.required_tables or task.template.comparison_dimensions:
|
||
headers = task.template.comparison_dimensions or ["项目", "口径"]
|
||
if len(headers) == 1:
|
||
headers = [headers[0], "取值"]
|
||
body.append("| " + " | ".join(headers) + " |")
|
||
body.append("| " + " | ".join("---" for _ in headers) + " |")
|
||
row = [task.key_facts[0].number if task.key_facts else "—"]
|
||
while len(row) < len(headers):
|
||
row.append(task.required_angles[0] if task.required_angles else "—")
|
||
body.append("| " + " | ".join(row[: len(headers)]) + " |")
|
||
for title in sections[1:]:
|
||
body.append(f"# {title}")
|
||
body.append(f"{title}需结合已引用主张审慎判断，避免把未核验数字写成事实。")
|
||
if title == sections[-1]:
|
||
body.append("局限：公开口径可能修订；本报告不采用来源中不存在的数字或事件归属。")
|
||
source_index = []
|
||
for index, source in enumerate(task.acceptable_sources, start=1):
|
||
token = source.title_tokens[0] if source.title_tokens else f"source-{index}"
|
||
year = source.min_year or 2025
|
||
source_index.append(
|
||
{
|
||
"id": task.key_facts[min(index - 1, len(task.key_facts) - 1)].id if task.key_facts else f"src-{index}",
|
||
"title": f"{token} 公开口径",
|
||
"url": f"https://example.{source.domain_suffix or 'org'}/{task.id}/{index}",
|
||
"published_at": f"{year}-06-01",
|
||
"added_at": f"{year}-06-02T00:00:00Z",
|
||
"agent_run_id": None,
|
||
}
|
||
)
|
||
return ReportCandidate(
|
||
source="gold",
|
||
markdown="\n\n".join(body),
|
||
source_index=source_index,
|
||
events_present=list(REQUIRED_EVENTS),
|
||
status="completed",
|
||
notes="annotation-built gold",
|
||
)
|
||
|
||
|
||
def materialize_failure(task: EvalTask, kind: FailureKind) -> ReportCandidate:
|
||
gold = materialize_gold(task)
|
||
if kind == "fabricated" and task.forbidden_fabrications:
|
||
return gold.model_copy(update={"source": "failure", "markdown": gold.markdown + "\n\n" + task.forbidden_fabrications[0].text, "notes": kind})
|
||
if kind == "missing_angle" and task.required_angles:
|
||
return gold.model_copy(update={"source": "failure", "markdown": gold.markdown.replace(task.required_angles[0], ""), "notes": kind})
|
||
if kind == "uncited":
|
||
markdown = gold.markdown
|
||
for fact in task.key_facts:
|
||
markdown = markdown.replace(f"（{fact.id}）", "")
|
||
return gold.model_copy(update={"source": "failure", "markdown": markdown, "source_index": [], "notes": kind})
|
||
if kind == "roundtable_essay":
|
||
snippet = task.key_facts[0].text if task.key_facts else task.title
|
||
return ReportCandidate(
|
||
source="roundtable",
|
||
markdown=f"{task.title}总体向好。{snippet}。建议继续观察。",
|
||
source_index=[],
|
||
events_present=[],
|
||
status="completed",
|
||
notes="roundtable-style essay without citations or template",
|
||
)
|
||
if kind == "over_budget":
|
||
return gold.model_copy(
|
||
update={
|
||
"source": "failure",
|
||
"usage": UsageSnapshot(duration_seconds=9000, model_calls=400, retrieval_calls=80, input_tokens=300_000, output_tokens=80_000, cost=12.5),
|
||
"notes": kind,
|
||
}
|
||
)
|
||
if kind == "empty":
|
||
return ReportCandidate(source="failure", markdown="", source_index=[], status="failed", notes=kind)
|
||
if kind == "illegal_json":
|
||
return ReportCandidate(source="failure", markdown="{not-json", source_index=[], status="failed", events_present=[], notes=kind)
|
||
return gold.model_copy(update={"source": "failure", "markdown": "", "notes": kind})
|
||
|
||
|
||
__all__ = ["materialize_failure", "materialize_gold"]
|