"""Build gold reports and reproducible failure instances from task annotations.""" from __future__ import annotations from app.report_collaboration.eval.schema import EvalTask, FailureKind, ReportCandidate, UsageSnapshot from app.report_collaboration.eval.thresholds import REQUIRED_EVENTS def materialize_gold(task: EvalTask) -> ReportCandidate: sections = list(task.template.sections) body: list[str] = [f"# {sections[0]}"] body.append(f"{task.title}。时间范围:{task.time_range}。受众:{task.audience}。") body.append("必选角度:" + "、".join(task.required_angles) + "。") for fact in task.key_facts: body.append(f"{fact.text}({fact.id})。") if task.template.required_questions: body.append("必答问题:" + " ".join(task.template.required_questions)) body.append("回答:" + ";".join(f"{fact.number}" for fact in task.key_facts[:2])) if task.template.required_tables or task.template.comparison_dimensions: headers = task.template.comparison_dimensions or ["项目", "口径"] if len(headers) == 1: headers = [headers[0], "取值"] body.append("| " + " | ".join(headers) + " |") body.append("| " + " | ".join("---" for _ in headers) + " |") row = [task.key_facts[0].number if task.key_facts else "—"] while len(row) < len(headers): row.append(task.required_angles[0] if task.required_angles else "—") body.append("| " + " | ".join(row[: len(headers)]) + " |") for title in sections[1:]: body.append(f"# {title}") body.append(f"{title}需结合已引用主张审慎判断,避免把未核验数字写成事实。") if title == sections[-1]: body.append("局限:公开口径可能修订;本报告不采用来源中不存在的数字或事件归属。") source_index = [] for index, source in enumerate(task.acceptable_sources, start=1): token = source.title_tokens[0] if source.title_tokens else f"source-{index}" year = source.min_year or 2025 source_index.append( { "id": task.key_facts[min(index - 1, len(task.key_facts) - 1)].id if task.key_facts else f"src-{index}", "title": f"{token} 公开口径", "url": f"https://example.{source.domain_suffix or 'org'}/{task.id}/{index}", "published_at": f"{year}-06-01", "added_at": f"{year}-06-02T00:00:00Z", "agent_run_id": None, } ) return ReportCandidate( source="gold", markdown="\n\n".join(body), source_index=source_index, events_present=list(REQUIRED_EVENTS), status="completed", notes="annotation-built gold", ) def materialize_failure(task: EvalTask, kind: FailureKind) -> ReportCandidate: gold = materialize_gold(task) if kind == "fabricated" and task.forbidden_fabrications: return gold.model_copy(update={"source": "failure", "markdown": gold.markdown + "\n\n" + task.forbidden_fabrications[0].text, "notes": kind}) if kind == "missing_angle" and task.required_angles: return gold.model_copy(update={"source": "failure", "markdown": gold.markdown.replace(task.required_angles[0], ""), "notes": kind}) if kind == "uncited": markdown = gold.markdown for fact in task.key_facts: markdown = markdown.replace(f"({fact.id})", "") return gold.model_copy(update={"source": "failure", "markdown": markdown, "source_index": [], "notes": kind}) if kind == "roundtable_essay": snippet = task.key_facts[0].text if task.key_facts else task.title return ReportCandidate( source="roundtable", markdown=f"{task.title}总体向好。{snippet}。建议继续观察。", source_index=[], events_present=[], status="completed", notes="roundtable-style essay without citations or template", ) if kind == "over_budget": return gold.model_copy( update={ "source": "failure", "usage": UsageSnapshot(duration_seconds=9000, model_calls=400, retrieval_calls=80, input_tokens=300_000, output_tokens=80_000, cost=12.5), "notes": kind, } ) if kind == "empty": return ReportCandidate(source="failure", markdown="", source_index=[], status="failed", notes=kind) if kind == "illegal_json": return ReportCandidate(source="failure", markdown="{not-json", source_index=[], status="failed", events_present=[], notes=kind) return gold.model_copy(update={"source": "failure", "markdown": "", "notes": kind}) __all__ = ["materialize_failure", "materialize_gold"]