183 lines
4.4 KiB
Python
183 lines
4.4 KiB
Python
"""Frozen RC-BE-018 quality-eval contracts. No model calls."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Literal
|
|
|
|
from pydantic import BaseModel, ConfigDict, Field
|
|
|
|
EvalCategory = Literal["policy", "market", "enterprise", "event", "comparison"]
|
|
FailureKind = Literal["fabricated", "missing_angle", "uncited", "roundtable_essay", "over_budget", "empty", "illegal_json"]
|
|
|
|
|
|
class KeyFact(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
id: str
|
|
text: str
|
|
number: str
|
|
must_cite: bool = True
|
|
acceptable_source_tokens: list[str] = Field(default_factory=list)
|
|
|
|
|
|
class ForbiddenItem(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
text: str
|
|
reason: str
|
|
|
|
|
|
class AcceptableSource(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
title_tokens: list[str]
|
|
domain_suffix: str | None = None
|
|
min_year: int | None = 2024
|
|
|
|
|
|
class TemplateSpec(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
title: str
|
|
sections: list[str]
|
|
required_questions: list[str] = Field(default_factory=list)
|
|
required_tables: list[str] = Field(default_factory=list)
|
|
comparison_dimensions: list[str] = Field(default_factory=list)
|
|
length_target: str | None = None
|
|
|
|
|
|
class DirectedModification(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
id: str
|
|
command: str
|
|
intent: str
|
|
must_reopen_roles: list[str]
|
|
must_not_reopen_roles: list[str] = Field(default_factory=list)
|
|
|
|
|
|
class UsageSnapshot(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
duration_seconds: float = 0
|
|
model_calls: int = 0
|
|
retrieval_calls: int = 0
|
|
input_tokens: int = 0
|
|
output_tokens: int = 0
|
|
cost: float = 0
|
|
|
|
|
|
class ReportCandidate(BaseModel):
|
|
"""A scoreable report, from the collaboration runtime or a roundtable export."""
|
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
source: Literal["collaboration", "roundtable", "gold", "failure"] = "collaboration"
|
|
markdown: str
|
|
source_index: list[dict] = Field(default_factory=list)
|
|
usage: UsageSnapshot | None = None
|
|
events_present: list[str] = Field(default_factory=list)
|
|
status: str = "completed"
|
|
notes: str | None = None
|
|
|
|
|
|
class EvalTask(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
id: str
|
|
category: EvalCategory
|
|
title: str
|
|
prompt: str
|
|
time_range: str = "2024 年至今"
|
|
audience: str = "企业战略决策人员"
|
|
required_angles: list[str]
|
|
key_facts: list[KeyFact]
|
|
forbidden_fabrications: list[ForbiddenItem]
|
|
template: TemplateSpec
|
|
review_rules: list[str]
|
|
acceptable_sources: list[AcceptableSource]
|
|
directed_modifications: list[DirectedModification] = Field(default_factory=list)
|
|
|
|
|
|
class MetricScore(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
name: str
|
|
score: float = Field(ge=0, le=1)
|
|
passed: bool
|
|
issues: list[str] = Field(default_factory=list)
|
|
|
|
|
|
class TaskScore(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
task_id: str
|
|
source: str
|
|
metrics: list[MetricScore]
|
|
passed: bool
|
|
composite: float
|
|
|
|
def metric(self, name: str) -> MetricScore:
|
|
for item in self.metrics:
|
|
if item.name == name:
|
|
return item
|
|
raise KeyError(name)
|
|
|
|
|
|
class ComparisonResult(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
task_id: str
|
|
winner: Literal["collaboration", "roundtable", "tie"]
|
|
collaboration: TaskScore
|
|
roundtable: TaskScore
|
|
deltas: dict[str, float]
|
|
notes: str
|
|
|
|
|
|
class ReliabilityCounts(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
runs: int
|
|
early_stop: int = 0
|
|
empty_output: int = 0
|
|
illegal_json: int = 0
|
|
missing_events: int = 0
|
|
completed: int = 0
|
|
|
|
@property
|
|
def failure_rate(self) -> float:
|
|
if self.runs <= 0:
|
|
return 1.0
|
|
failed = self.early_stop + self.empty_output + self.illegal_json + self.missing_events
|
|
return failed / self.runs
|
|
|
|
|
|
class CatalogScore(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
tasks: int
|
|
passed: int
|
|
scores: list[TaskScore]
|
|
composite: float
|
|
by_category: dict[str, float]
|
|
|
|
|
|
__all__ = [
|
|
"AcceptableSource",
|
|
"CatalogScore",
|
|
"ComparisonResult",
|
|
"DirectedModification",
|
|
"EvalCategory",
|
|
"EvalTask",
|
|
"FailureKind",
|
|
"ForbiddenItem",
|
|
"KeyFact",
|
|
"MetricScore",
|
|
"ReliabilityCounts",
|
|
"ReportCandidate",
|
|
"TaskScore",
|
|
"TemplateSpec",
|
|
"UsageSnapshot",
|
|
]
|