30 lines
1.2 KiB
Python
30 lines
1.2 KiB
Python
"""Count early-stop / empty / illegal-JSON / missing-event outcomes across repeated runs."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from app.report_collaboration.eval.schema import ReliabilityCounts, ReportCandidate
|
|
from app.report_collaboration.eval.thresholds import REQUIRED_EVENTS
|
|
|
|
|
|
def summarize_reliability(runs: list[ReportCandidate]) -> ReliabilityCounts:
|
|
counts = ReliabilityCounts(runs=len(runs))
|
|
for item in runs:
|
|
empty = not (item.markdown or "").strip()
|
|
illegal = item.notes == "illegal_json" or ((item.markdown or "").lstrip().startswith("{") and "schema_version" not in (item.markdown or ""))
|
|
missing = bool(item.events_present) and any(name not in set(item.events_present) for name in REQUIRED_EVENTS)
|
|
early = item.status not in {"completed", "cancelled"} or item.notes == "early_stop"
|
|
if empty:
|
|
counts.empty_output += 1
|
|
elif illegal:
|
|
counts.illegal_json += 1
|
|
elif missing:
|
|
counts.missing_events += 1
|
|
elif early:
|
|
counts.early_stop += 1
|
|
elif item.status == "completed":
|
|
counts.completed += 1
|
|
return counts
|
|
|
|
|
|
__all__ = ["summarize_reliability"]
|