"""Runtime command orchestration with safe-point application.""" from __future__ import annotations from datetime import UTC, datetime, timedelta from typing import Any from app.report_collaboration.contracts.commands import CollaborationCommand, IntentDecision from app.report_collaboration.execution.ledger import TaskLedger from app.report_collaboration.execution.quality_gate import requirement_from_session_record from app.report_collaboration.interventions.impact_analyzer import CommandImpact, ImpactAnalyzer from app.report_collaboration.interventions.intent_router import RuntimeIntentContext, RuntimeIntentRouterAgent from deerflow.config.report_collaboration_config import ReportCollaborationConfig from deerflow.persistence.report_collaboration.base import ( ReportCollaborationConflictError, ReportCollaborationNotFoundError, ReportCollaborationStore, ReportCollaborationValidationError, ) from deerflow.persistence.report_collaboration.codec import parse_lease_until _PROCESSABLE = {"accepted", "accepted_waiting_boundary"} _CANCELLABLE = {"pending", "classified", "confirmation_required", "accepted", "accepted_waiting_boundary"} class RuntimeCommandService: """Classify, confirm and apply user commands without delegating scheduling.""" def __init__( self, store: ReportCollaborationStore, *, router: RuntimeIntentRouterAgent | None = None, analyzer: ImpactAnalyzer | None = None, config: ReportCollaborationConfig | None = None, ) -> None: self._store = store self._router = router or RuntimeIntentRouterAgent() self._analyzer = analyzer or ImpactAnalyzer( confirmation_threshold=(config or ReportCollaborationConfig()).intent_confirmation_threshold ) self._config = config or ReportCollaborationConfig() self._ledger = TaskLedger(store) async def submit( self, run_id: str, *, text: str, idempotency_key: str, target_node_id: str | None = None, agent_run_id: str | None = None, ) -> dict[str, Any]: run, session, plan, node_runs, agent_runs = await self._context(run_id) existing = next( (item for item in await self._store.list_commands(run["session_id"], run_id=run_id) if item["idempotency_key"] == idempotency_key), None, ) if existing is not None: if existing.get("status") != "pending": return existing # Resume an interrupted classification from the durable command, # never from a retry request carrying a different payload. text = str(existing.get("text") or text) stored_targets = list(existing.get("target_ids") or []) target_node_id = stored_targets[0] if stored_targets else None agent_run_id = None target_nodes, target_node_run_id = self._resolve_targets( plan=plan, node_runs=node_runs, agent_runs=agent_runs, target_node_id=target_node_id, agent_run_id=agent_run_id, ) command = existing or await self._store.create_command( run["session_id"], idempotency_key=idempotency_key, text=text, run_id=run_id, operation="command", target_type="node" if target_nodes else None, target_ids=target_nodes, expected_revision=int(session.get("requirement_revision") or 0), status="pending", ) # A concurrent duplicate may have won the unique idempotency key. if command.get("status") != "pending": return command message = await self._store.create_message( run["session_id"], role="human", content=text, # Command and message use separate uniqueness domains. Reusing the # client key lets the frontend replace its optimistic bubble. idempotency_key=idempotency_key, metadata={ "idempotency_key": idempotency_key, "target_node_id": target_node_id, "agent_run_id": agent_run_id, "runtime_command": True, }, command_id=command["command_id"], agent_run_id=agent_run_id, node_run_id=target_node_run_id, completed=True, ) await self._store.append_event( session_id=run["session_id"], run_id=run_id, event_type="message.created", data={"message": message}, ) requirement = requirement_from_session_record(session) latest = _latest_nodes(node_runs) report_versions = await self._store.list_report_versions(run["session_id"]) report_summary = None if report_versions: report_summary = str(report_versions[-1].get("markdown") or "")[:1200] decision = await self._router.route( RuntimeIntentContext( text=text, requirement=requirement.model_dump(mode="json") if requirement else {}, plan_nodes=list(plan.get("nodes") or []), target_node_ids=target_nodes, target_agent_run_id=agent_run_id, awaiting_node_ids=[node_id for node_id, row in latest.items() if row.get("status") == "awaiting_input"], report_summary=report_summary, ) ) valid_node_ids = {str(item.get("id")) for item in plan.get("nodes") or [] if item.get("id")} decision_targets = [node_id for node_id in decision.requested_target_ids if node_id in valid_node_ids] impact = self._analyzer.analyze( decision, plan=plan, node_runs=node_runs, agent_runs=agent_runs, explicit_target_node_ids=target_nodes or decision_targets, ) if decision.intent == "add_angle" and not impact.target_node_ids: rejected = await self._reject_classified( run, command, decision, impact, error="没有可用的研究支路承接新角度", ) return rejected final_status = "confirmation_required" if impact.requires_confirmation else ( "accepted_waiting_boundary" if impact.waiting_boundary_nodes else "accepted" ) patch = _classification_patch(decision, impact, status=final_status) classified = {**command, **patch, "status": "classified"} await self._store.append_event( session_id=run["session_id"], run_id=run_id, event_type="command.classified", data={"command": CollaborationCommand.model_validate(classified).model_dump(mode="json")}, ) try: command = await self._store.patch_command( command["command_id"], patch=patch, expected_statuses={"pending"}, ) except ReportCollaborationConflictError: # Another request carrying the same key classified it first. The # unique command remains the only source of effects. return await self._store.get_command(command["command_id"]) or command if final_status == "confirmation_required": await self._store.append_event( session_id=run["session_id"], run_id=run_id, event_type="command.confirmation.required", data={"command": command}, ) return command await self._emit_accepted(run, command, bool(impact.waiting_boundary_nodes)) if not impact.waiting_boundary_nodes: await self.process_safe_point(run_id) command = await self._store.get_command(command["command_id"]) or command return command async def confirm(self, run_id: str, command_id: str, *, idempotency_key: str) -> dict[str, Any]: run, _session, plan, node_runs, agent_runs = await self._context(run_id) existing_audit = next( (item for item in await self._store.list_commands(run["session_id"], run_id=run_id) if item["idempotency_key"] == idempotency_key), None, ) command = await self._command_for_run(run, command_id) if existing_audit is not None: return command if command.get("status") in {"accepted", "accepted_waiting_boundary", "executing", "completed"}: return command if command.get("status") != "confirmation_required": raise ReportCollaborationConflictError("COMMAND_STATE_CONFLICT", f"命令当前状态为 {command.get('status')}，不能确认") await self._store.create_command( run["session_id"], idempotency_key=idempotency_key, text="", run_id=run_id, operation="confirm_command", target_type="command", target_ids=[command_id], status="completed", ) decision = _decision_from_command(command) impact = self._analyzer.analyze( decision, plan=plan, node_runs=node_runs, agent_runs=agent_runs, explicit_target_node_ids=list(command.get("target_ids") or []), ) status = "accepted_waiting_boundary" if impact.waiting_boundary_nodes else "accepted" try: command = await self._store.patch_command( command_id, patch={"status": status, "impact_nodes": impact.impact_nodes, "error": None}, expected_statuses={"confirmation_required"}, ) except ReportCollaborationConflictError: current = await self._command_for_run(run, command_id) if current.get("status") in {"accepted", "accepted_waiting_boundary", "executing", "completed"}: return current raise await self._emit_accepted(run, command, bool(impact.waiting_boundary_nodes)) if not impact.waiting_boundary_nodes: await self.process_safe_point(run_id) command = await self._store.get_command(command_id) or command return command async def cancel(self, run_id: str, command_id: str, *, idempotency_key: str) -> dict[str, Any]: run, _session, _plan, _nodes, _agents = await self._context(run_id) command = await self._command_for_run(run, command_id) audits = await self._store.list_commands(run["session_id"], run_id=run_id) if any(item["idempotency_key"] == idempotency_key for item in audits): return command await self._store.create_command( run["session_id"], idempotency_key=idempotency_key, text="", run_id=run_id, operation="cancel_command", target_type="command", target_ids=[command_id], status="completed", ) if command.get("status") == "cancelled": return command try: cancelled = await self._store.patch_command( command_id, patch={"status": "cancelled"}, expected_statuses=_CANCELLABLE, ) except ReportCollaborationConflictError: current = await self._command_for_run(run, command_id) if current.get("status") == "cancelled": return current raise await self._store.append_event( session_id=run["session_id"], run_id=run["id"], event_type="command.cancelled", data={"command_id": command_id, "command": cancelled}, ) return cancelled async def process_safe_point(self, run_id: str, *, completed_node_id: str | None = None) -> list[str]: """Apply accepted commands in creation order at a model/tool/commit boundary.""" run, _session, _plan, node_runs, agent_runs = await self._context(run_id) applied: list[str] = [] await self.reclaim_executing(run_id, force=False) commands = await self._store.list_commands(run["session_id"], run_id=run_id) # One command may mutate several attempts and artifacts. Never overlap # two such mutations for the same run, even across Gateway/worker # processes. CAS on the oldest command elects the processor. if any(item.get("status") == "executing" for item in commands): return applied for command in commands: if command.get("operation", "command") != "command" and not command.get("intent"): continue if command.get("status") not in _PROCESSABLE: continue running = _running_impacted_nodes(node_runs, agent_runs, list(command.get("impact_nodes") or [])) if completed_node_id: running.discard(completed_node_id) if running: if command.get("status") == "accepted": await self._store.patch_command( command["command_id"], patch={"status": "accepted_waiting_boundary"}, expected_statuses={"accepted"}, ) continue try: claimed = await self._store.patch_command( command["command_id"], patch={"status": "executing", "error": None}, expected_statuses=_PROCESSABLE, ) except ReportCollaborationConflictError: return applied try: result = await self._apply(run, claimed) await self._store.patch_command( command["command_id"], patch={"status": "completed", "reason": result}, expected_statuses={"executing"}, ) await self._store.append_event( session_id=run["session_id"], run_id=run_id, event_type="command.completed", data={"command_id": command["command_id"], "result_summary": result}, ) applied.append(command["command_id"]) except Exception as exc: message = str(exc)[:500] await self._store.patch_command( command["command_id"], patch={"status": "failed", "error": message}, expected_statuses={"executing"}, ) await self._store.append_event( session_id=run["session_id"], run_id=run_id, event_type="command.failed", data={"command_id": command["command_id"], "error": message}, ) return applied async def _apply(self, run: dict[str, Any], command: dict[str, Any]) -> str: intent = str(command.get("intent") or "") instruction = str(command.get("normalized_instruction") or command.get("text") or "") current_run = await self._store.get_run(run["id"]) if ( current_run and (current_run.get("cancel_requested") or current_run.get("status") == "cancelled") and intent not in {"cancel", "ask_report"} ): raise ReportCollaborationConflictError("RUN_CANCELLED", "运行已取消，不能再应用修改") if intent == "cancel": await self._store.request_cancel_run(run["id"], idempotency_key=f"{command['idempotency_key']}:natural-cancel") return "已提交取消执行请求" if intent == "ask_report": versions = await self._store.list_report_versions(run["session_id"]) artifacts = await self._store.list_artifacts(run["id"]) if versions: answer = f"当前报告为 v{versions[-1]['version']}，正文已生成；已有 {len(artifacts)} 项协作成果。" else: answer = f"报告仍在生成中，目前已有 {len(artifacts)} 项协作成果。" message = await self._store.create_message( run["session_id"], role="ai", content=answer, idempotency_key=f"{command['idempotency_key']}:answer", name="协作协调者", command_id=command["command_id"], completed=True, ) await self._store.append_event( session_id=run["session_id"], run_id=run["id"], event_type="message.created", data={"message": message}, ) return "已回答当前报告状态" if intent == "answer_question": resumed = await self._ledger.apply_user_answer( run["id"], command_id=command["command_id"], instruction=instruction, node_ids=list(command.get("impact_nodes") or command.get("target_ids") or []), ) if not resumed: raise ReportCollaborationValidationError("NO_AWAITING_NODE", "当前没有可由该回答恢复的等待节点") return f"已将回答交给 {len(resumed)} 个等待节点" impact_nodes = list(command.get("impact_nodes") or []) if not impact_nodes: raise ReportCollaborationValidationError("NO_IMPACT_NODE", "没有找到可安全返工的节点") changed = await self._ledger.apply_command_rework( run["id"], command_id=command["command_id"], instruction=instruction, node_ids=impact_nodes, ) if not changed: raise ReportCollaborationValidationError("COMMAND_NO_EFFECT", "目标节点已过期或命令已经应用") await self._apply_requirement_patch(run, command) return f"已在安全边界应用修改，重新排队 {len(changed)} 个节点" async def reclaim_executing(self, run_id: str, *, force: bool = True) -> list[str]: """Return stuck ``executing`` commands to ``accepted`` so a takeover can finish them. ``force=True`` is for lease reclaim: the previous worker is gone. ``force=False`` only releases commands whose ``updated_at`` exceeded the lease TTL. """ run = await self._store.get_run(run_id) if run is None: return [] ttl = max(30, int(self._config.lease_seconds)) now = datetime.now(UTC) reclaimed: list[str] = [] for command in await self._store.list_commands(run["session_id"], run_id=run_id): if command.get("status") != "executing": continue if not force: updated = parse_lease_until(command.get("updated_at")) if updated is not None and now - updated < timedelta(seconds=ttl): continue try: await self._store.patch_command( command["command_id"], patch={"status": "accepted", "error": None}, expected_statuses={"executing"}, ) except ReportCollaborationConflictError: continue reclaimed.append(command["command_id"]) return reclaimed async def _reject_classified( self, run: dict[str, Any], command: dict[str, Any], decision: IntentDecision, impact: CommandImpact, *, error: str, ) -> dict[str, Any]: patch = _classification_patch(decision, impact, status="rejected") patch["error"] = error await self._store.append_event( session_id=run["session_id"], run_id=run["id"], event_type="command.classified", data={"command": {**command, **patch}}, ) try: rejected = await self._store.patch_command( command["command_id"], patch=patch, expected_statuses={"pending"}, ) except ReportCollaborationConflictError: return await self._store.get_command(command["command_id"]) or command await self._store.append_event( session_id=run["session_id"], run_id=run["id"], event_type="command.failed", data={"command_id": rejected["command_id"], "error": error}, ) return rejected async def _apply_requirement_patch(self, run: dict[str, Any], command: dict[str, Any]) -> None: intent = str(command.get("intent") or "") if intent not in {"add_angle", "update_requirements"}: return session = await self._store.get_session_record(run["session_id"]) if not session: raise ReportCollaborationNotFoundError("session", run["session_id"]) current = requirement_from_session_record(session) if current is None: return # Recreate the same deterministic patch used by the router. It is not # persisted on the command row, so confirmed commands cannot trust an # arbitrary client payload. context = RuntimeIntentContext( text=str(command.get("text") or ""), requirement=current.model_dump(mode="json"), plan_nodes=[], ) decision = await RuntimeIntentRouterAgent().route(context) if decision.requirement_patch is None: return state = dict(session.get("requirement_json") or {}) snapshot = decision.requirement_patch.model_dump(mode="json") if isinstance(state.get("snapshot"), dict): state["snapshot"] = snapshot else: state = snapshot await self._store.save_requirement_state( run["session_id"], requirement_json=state, requirement_revision=int(snapshot["revision"]), status="running", ) await self._store.append_event( session_id=run["session_id"], run_id=run["id"], event_type="requirement.updated", data={"requirement": snapshot, "revision": snapshot["revision"], "command_id": command["command_id"]}, ) async def _emit_accepted(self, run: dict[str, Any], command: dict[str, Any], waiting: bool) -> None: await self._store.append_event( session_id=run["session_id"], run_id=run["id"], event_type="command.accepted", data={ "command_id": command["command_id"], "waiting_boundary": waiting, "impact_nodes": list(command.get("impact_nodes") or []), }, ) async def _command_for_run(self, run: dict[str, Any], command_id: str) -> dict[str, Any]: command = await self._store.get_command(command_id) if command is None: raise ReportCollaborationNotFoundError("command", command_id) commands = await self._store.list_commands(run["session_id"], run_id=run["id"]) if not any(item["command_id"] == command_id for item in commands): raise ReportCollaborationNotFoundError("command", command_id) return command async def _context(self, run_id: str) -> tuple[dict, dict, dict, list[dict], list[dict]]: run = await self._store.get_run(run_id) if run is None: raise ReportCollaborationNotFoundError("run", run_id) session = await self._store.get_session_record(run["session_id"]) if session is None: raise ReportCollaborationNotFoundError("session", run["session_id"]) plan = await self._store.get_plan(run["plan_id"]) if plan is None: raise ReportCollaborationNotFoundError("plan", run["plan_id"]) return run, session, plan, await self._store.list_node_runs(run_id), await self._store.list_agent_runs(run_id) @staticmethod def _resolve_targets( *, plan: dict, node_runs: list[dict], agent_runs: list[dict], target_node_id: str | None, agent_run_id: str | None, ) -> tuple[list[str], str | None]: valid_nodes = {str(item.get("id")) for item in plan.get("nodes") or [] if item.get("id")} node_by_run = {str(item.get("node_run_id")): str(item.get("node_id")) for item in node_runs if item.get("node_run_id")} resolved: list[str] = [] node_run_id: str | None = None if target_node_id: if target_node_id not in valid_nodes: raise ReportCollaborationValidationError("TARGET_NOT_FOUND", f"目标节点不存在：{target_node_id}") resolved.append(target_node_id) latest = [item for item in node_runs if item.get("node_id") == target_node_id] if latest: node_run_id = max(latest, key=lambda item: int(item.get("attempt") or 0)).get("node_run_id") if agent_run_id: agent = next((item for item in agent_runs if item.get("agent_run_id") == agent_run_id), None) if agent is None: raise ReportCollaborationValidationError("TARGET_NOT_FOUND", f"智能体运行不存在：{agent_run_id}") agent_node_run = str(agent.get("node_run_id") or "") agent_node = node_by_run.get(agent_node_run) if not agent_node: raise ReportCollaborationValidationError("TARGET_NOT_FOUND", f"智能体运行没有对应节点：{agent_run_id}") if resolved and resolved[0] != agent_node: raise ReportCollaborationValidationError("TARGET_MISMATCH", "目标节点与智能体运行不属于同一节点") if agent_node not in resolved: resolved.append(agent_node) node_run_id = agent_node_run return resolved, node_run_id def _classification_patch(decision: IntentDecision, impact: CommandImpact, *, status: str) -> dict[str, Any]: return { "intent": decision.intent, "confidence": decision.confidence, "target_type": impact.target_type, "target_ids": impact.target_node_ids, "normalized_instruction": decision.normalized_instruction, "requires_confirmation": impact.requires_confirmation, "cost_level": impact.cost_level, "reason": impact.reason, "impact_nodes": impact.impact_nodes, "status": status, "error": None, } def _decision_from_command(command: dict[str, Any]) -> IntentDecision: intent = command.get("intent") if not intent: raise ReportCollaborationValidationError("COMMAND_NOT_CLASSIFIED", "命令尚未完成意图识别") return IntentDecision.model_validate( { "intent": intent, "confidence": command.get("confidence") or 0, "normalized_instruction": command.get("normalized_instruction") or command.get("text") or "", "scope": command.get("target_type") or "conversation", "requested_target_ids": command.get("target_ids") or [], "requirement_patch": None, "requires_confirmation": False, "cost_level": command.get("cost_level") or "low", "reason": command.get("reason") or "用户已确认", } ) def _latest_nodes(rows: list[dict]) -> dict[str, dict]: latest: dict[str, dict] = {} for row in rows: node_id = str(row.get("node_id") or "") if node_id and (node_id not in latest or int(row.get("attempt") or 0) >= int(latest[node_id].get("attempt") or 0)): latest[node_id] = row return latest def _running_impacted_nodes(node_runs: list[dict], agent_runs: list[dict], impact_nodes: list[str]) -> set[str]: wanted = set(impact_nodes) latest = _latest_nodes(node_runs) node_by_run = {str(row.get("node_run_id")): node_id for node_id, row in latest.items()} running = { node_id for node_id, row in latest.items() if node_id in wanted and row.get("status") in {"running", "validating"} } for agent in agent_runs: if agent.get("status") == "running": node_id = node_by_run.get(str(agent.get("node_run_id") or "")) if node_id in wanted: running.add(node_id) return running __all__ = ["RuntimeCommandService"]