deerflow-code/skills-data/xdfx-step2-pathway/scripts/xdfx_flow.py
2026-09-07 18:24:55 +08:00

3313 lines
135 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""Robust xdfx single-agent flow runner.
The skill text should stay thin; this script owns the slow, failure-prone
workflow details: HTTP calls, streamed answer parsing, JSON extraction, id
handoff, partial failure handling, and state persistence.
"""
from __future__ import annotations
import argparse
import ast
import copy
import json
import os
import re
import ssl
import sys
import time
import traceback
import urllib.error
import urllib.parse
import urllib.request
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import xdfx_auth
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
if hasattr(sys.stderr, "reconfigure"):
sys.stderr.reconfigure(encoding="utf-8", errors="replace")
# Keep every backend path in one table so endpoint drift is easy to audit.
ENDPOINTS = {
"aiuser": "/gpt/chat/chat_aiuser",
"group": "/gpt/chat/chat_qianqian_group_chat",
"spread": "/gpt/chat/chat_qianqian_spread_chat",
"cuiruo": "/gpt/chat/chat_qianqian_cuiruo_chat",
"strategy_model": "/gpt/chat/chat_qianqian_newboci_chat",
"save_design": "/api/task/task-action-design",
"save_design_account": "/api/task/task-action-design-account",
"save_pathway": "/api/task/task-action-pathway-account",
"save_relation": "/api/task/task-action-design-pathway-relation/saveRelation",
"save_design_vuln": "/api/task/task-action-design-vulns",
"save_pathway_vuln": "/api/task/task-action-pathway-account-vulns",
"save_strategy": "/api/task/task-action-strategy",
"save_channel": "/api/task/task-action-channel",
"save_narrate": "/api/base/base-narrate",
"save_message": "/api/task/task-action-message",
"save_strategy_relation": "/api/task/task-action-design-strategy-relation",
"dict_data": "/api/sys/sys-dict-data",
}
# Human-readable names are used only in the Chinese feedback logs.
ENDPOINT_LABELS = {
"login": "登录获取访问令牌",
"aiuser": "调用候选账号/受众分析模型接口",
"group": "调用受众目标结构化抽取接口",
"spread": "调用关键传播节点结构化抽取接口",
"cuiruo": "调用脆弱点分析模型接口",
"strategy_model": "调用策略生成模型接口",
"save_design": "保存受众目标",
"save_design_account": "保存受众目标账号",
"save_pathway": "保存关键传播节点账号",
"save_relation": "保存受众与关键传播节点关系",
"save_design_vuln": "保存受众脆弱点",
"save_pathway_vuln": "保存关键传播节点脆弱点",
"save_strategy": "保存策略主表",
"save_channel": "保存策略平台与人设",
"save_narrate": "保存策略信息主题",
"save_message": "保存策略消息配置",
"save_strategy_relation": "保存策略与受众/传播节点关系",
"dict_data": "读取前端字典数据",
}
LOG_PREFIXES = {
"operation": "操作反馈",
"error": "错误反馈",
"json_review": "JSON解析复核",
"assist": "用户协助",
"save": "入库反馈",
"step": "步骤反馈",
"done": "执行完成",
}
STATUS_LABELS = {
"ok": "成功",
"partial": "部分完成",
"skipped": "已跳过",
"waiting_user": "等待用户选择",
}
STEP_NUMBER_LABELS = {
"step1": "第一步",
"step2": "第二步",
"step3": "第三步",
"step4": "第四步",
"step5": "第五步",
"step6": "第六步",
}
HTTP_METHOD_LABELS = {
"GET": "读取(GET)",
"POST": "提交(POST)",
"PUT": "更新(PUT)",
"PATCH": "局部更新(PATCH)",
"DELETE": "删除(DELETE)",
}
STAGE_SEGMENT_LABELS = {
"analyze": "模型分析",
"extract": "结构化抽取",
"save": "保存数据",
"read": "读取数据",
"status": "同步状态",
}
REQUEST_FIELD_LABELS = {
"history": "历史消息",
"max_tokens": "最大生成令牌数",
"model_name": "模型名称",
"otherinfo": "其他信息",
"prompt_name": "提示词名称",
"stream": "是否流式",
"temperature": "温度",
"usermessage": "用户消息",
"user_message": "用户消息",
"username": "登录用户名",
"password": "登录密码",
"mubiao": "行动目标",
"zhanghao": "账号列表",
"taskId": "任务ID",
"actionId": "行动ID",
"taskActionId": "行动ID",
"designId": "受众ID",
"pathwayId": "传播节点ID",
"strategyId": "策略ID",
"accountId": "账号ID",
"accountName": "账号名称",
"accountType": "账号类型",
"targetAudience": "受众目标",
"vulnerabilities": "脆弱点",
"topic": "信息标题",
"narrate": "叙事主题",
"platform": "平台",
"persona": "发布人设",
"modelType": "信息形式",
"quantity": "数量",
"publishTime": "发布时间段",
"reason": "理由",
"ids": "ID列表",
"id": "ID",
"step": "步骤状态",
}
SENSITIVE_KEY_PARTS = ("password", "token", "authorization", "access_token", "accesstoken", "authtoken")
# Platform values mirror the frontend dictionaries used by xdfx strategy pages.
PLATFORM_MAP = {
"facebook": 1,
"fb": 1,
"ptt": 13,
"instagram": 2,
"ig": 2,
"x": 3,
"twitter": 3,
"youtube": 4,
"yt": 4,
}
# The model service sometimes returns a successful HTTP response with these
# strings instead of useful content. Treat them like transient failures.
MODEL_RETRY_MARKERS = (
"y模型没有响应",
"模型没有响应",
"模型无响应",
"接口超时",
"请求超时",
"响应超时",
"读取超时",
"连接超时",
"超时",
"timeout",
"timed out",
"read timed out",
"gateway timeout",
"504 gateway",
)
DEFAULT_STRATEGY_PROMPT_AUDIENCE = (
"一、叙事主题设计,根据议程设置理论,需优先选择与目标受众利益高度重合的议题。"
"叙事框架构建采用系统叙事与认同叙事。二、行动人设设计需强调专业背书、地域关联和跨领域合作。"
"三、传播策略需与目标受众所在平台、语言风格、文化禁忌和信息接受方式一致。"
"请至少生成1个策略,字段使用:总体理由、发布人设1、账号类型1、社交平台1、信息标题1、"
"信息形式1、数量1、信息发布时间段1,可继续输出2、3。只返回JSON。"
)
DEFAULT_STRATEGY_PROMPT_PATHWAY = (
"一、叙事主题设计,根据议程设置理论,需优先选择与关键传播节点利益高度重合的议题。"
"叙事框架构建采用系统叙事与认同叙事。二、行动人设设计需强调专业背书、地域关联和跨领域合作。"
"三、传播节点突破策略需贴合该账号平台、粉丝结构、语言风格、文化禁忌和既有发文风格。"
"请至少生成1个策略,字段使用:总体理由、发布人设1、账号类型1、社交平台1、信息标题1、"
"信息形式1、数量1、信息发布时间段1,可继续输出2、3。只返回JSON。"
)
class FlowError(RuntimeError):
pass
@dataclass
class FlowContext:
"""Runtime options shared by every step.
Keep this object boring and serializable-ish: it is passed through every
helper, while durable progress belongs in the state JSON file.
"""
api_base: str
headers: dict[str, str]
timeout: float
retries: int
dry_run: bool
state_path: Path
summary_path: Path | None
max_audiences: int = 3
max_pathways: int = 3
max_vulns: int = 3
max_strategies: int = 3
downstream_audiences: int = 2
downstream_pathways: int = 2
require_user_assist: bool = True
select_audience_refs: list[str] | None = None
select_pathway_refs: list[str] | None = None
select_audience_vuln_refs: list[str] | None = None
select_pathway_vuln_refs: list[str] | None = None
select_strategy_audience_refs: list[str] | None = None
select_strategy_pathway_refs: list[str] | None = None
strategy_target: str = ""
sync_step_status: bool = True
model_retries: int = 1
verify_ssl: bool = False
auto_login: bool = True
login_username: str = "admin"
login_password: str = "ch@user"
login_path: str = "/api/auth/login"
login_attempt_count: int = 0
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def compact(value: Any, limit: int = 1400) -> str:
try:
if isinstance(value, str):
text = value
else:
text = json.dumps(value, ensure_ascii=False)
except Exception:
text = repr(value)
if len(text) <= limit:
return text
return text[:limit] + f"... <已截断 {len(text) - limit} 个字符>"
def is_sensitive_key(key: Any) -> bool:
normalized = re.sub(r"[^a-z0-9_]", "", str(key or "").lower())
return any(part in normalized for part in SENSITIVE_KEY_PARTS)
def redact_sensitive(value: Any) -> Any:
if isinstance(value, dict):
return {key: "***" if is_sensitive_key(key) else redact_sensitive(item) for key, item in value.items()}
if isinstance(value, list):
return [redact_sensitive(item) for item in value]
if isinstance(value, tuple):
return tuple(redact_sensitive(item) for item in value)
if isinstance(value, str):
text = re.sub(r"(?i)(Bearer\s+)[^\s\",}]+", r"\1***", value)
text = re.sub(
r"(?i)(\"(?:password|token|authToken|access_token|accessToken|authorization)\"\s*:\s*)\"[^\"]*\"",
r'\1"***"',
text,
)
return re.sub(
r"(?i)((?:password|token|authToken|access_token|accessToken|authorization)=)[^&\s]+",
r"\1***",
text,
)
return value
def safe_compact(value: Any, limit: int = 1400) -> str:
return compact(redact_sensitive(value), limit)
def split_selection_refs(raw: Any) -> list[str]:
"""Parse user choices like '1,3' or ['1', '3'] into stable strings."""
if raw is None:
return []
if isinstance(raw, (list, tuple, set)):
parts: list[str] = []
for item in raw:
parts.extend(split_selection_refs(item))
return parts
text = str(raw).strip()
if not text:
return []
if text.lower() in {"all", "全部"}:
return ["all"]
return [part.strip() for part in re.split(r"[,,;;\s]+", text) if part.strip()]
def normalized_ref(value: Any) -> str:
return re.sub(r"\s+", "", str(value or "")).lower()
def has_selection(refs: list[str] | None) -> bool:
return bool([ref for ref in refs or [] if str(ref).strip()])
def clear_pending_assist(state: dict[str, Any]) -> None:
state.pop("_pendingUserAssist", None)
def has_pending_assist(state: dict[str, Any]) -> bool:
return isinstance(state.get("_pendingUserAssist"), dict)
def user_assist_command_hint(state: dict[str, Any], mode: str, flag_name: str, example: str) -> str:
command = f"python scripts/xdfx_flow.py --mode {mode} --state {state.get('_statePath') or 'xdfx-flow-state.json'}"
if state.get("_summaryPath"):
command += f" --summary {state.get('_summaryPath')}"
return f"{command} --{flag_name} {example}"
def emit_user_assist(
state: dict[str, Any],
stage: str,
*,
selection_key: str,
title: str,
prompt: str,
options: list[dict[str, Any]],
command_hint: str,
allow_multiple: bool = True,
max_count: int | None = None,
exclusive_note: str = "",
) -> None:
payload = {
"日志类型": "用户协助",
"阶段": stage_label(stage),
"步骤": feedback_step_label(stage),
"选择类型": selection_key,
"标题": title,
"提示": prompt,
"是否允许多选": yes_no(allow_multiple),
"最多选择数量": max_count,
"互斥规则": exclusive_note,
"选项": options,
"继续命令示例": command_hint,
"状态文件": state.get("_statePath", ""),
"摘要文件": state.get("_summaryPath", ""),
"时间": utc_now(),
}
state["_pendingUserAssist"] = {
"selectionKey": selection_key,
"stage": stage,
"title": title,
"commandHint": command_hint,
"optionCount": len(options),
"createdAt": payload["时间"],
}
print(LOG_PREFIXES["assist"] + " " + json.dumps(payload, ensure_ascii=False), flush=True)
record_event(state, stage, "等待用户协助选择", {"selectionKey": selection_key, "optionCount": len(options)})
def ref_matches(ref: str, ordinal: int, aliases: list[Any]) -> bool:
normalized = normalized_ref(ref)
if normalized == "all":
return True
if normalized == str(ordinal):
return True
return any(normalized and normalized == normalized_ref(alias) for alias in aliases if alias is not None)
def select_by_refs(
items: list[Any],
refs: list[str],
alias_fn,
*,
max_count: int | None = None,
) -> tuple[list[Any], list[str]]:
if not refs:
selected = list(items)
elif any(normalized_ref(ref) in {"all", "全部"} for ref in refs):
selected = list(items)
else:
selected = []
missing: list[str] = []
for ref in refs:
found = None
for index, item in enumerate(items, start=1):
if ref_matches(ref, index, alias_fn(item, index)):
found = item
break
if found is None:
missing.append(ref)
continue
if found not in selected:
selected.append(found)
if max_count is not None:
return selected[:max_count], missing
return selected, missing
if max_count is not None:
selected = selected[:max_count]
return selected, []
def stable_json(value: Any) -> str:
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
def as_int(value: Any) -> int | None:
if value is None:
return None
if isinstance(value, bool):
return None
if isinstance(value, int):
return value
if isinstance(value, float) and value.is_integer():
return int(value)
text = str(value).strip()
if re.fullmatch(r"-?\d+", text):
return int(text)
return None
def as_list(value: Any) -> list[Any]:
if value is None:
return []
if isinstance(value, list):
return value
if isinstance(value, tuple):
return list(value)
return [value]
def deep_get(data: Any, *keys: str) -> Any:
cur = data
for key in keys:
if not isinstance(cur, dict):
return None
cur = cur.get(key)
return cur
def first_present(*values: Any) -> Any:
for value in values:
if value is None:
continue
if isinstance(value, str) and not value.strip():
continue
return value
return None
def load_json_file(path: str | Path) -> dict[str, Any]:
p = Path(path)
if not p.exists():
return {}
return json.loads(p.read_text(encoding="utf-8-sig"))
def save_json_file(path: str | Path, value: Any) -> None:
p = Path(path)
text = json.dumps(value, ensure_ascii=False, indent=2)
try:
p.parent.mkdir(parents=True, exist_ok=True)
p.write_text(text, encoding="utf-8")
except PermissionError:
fallback = Path.cwd() / p.name
fallback.write_text(text, encoding="utf-8")
def default_state() -> dict[str, Any]:
"""Create a resumable state container.
The script can be run as six independent commands. IDs returned by one step
are persisted here so later steps can resume without redoing earlier saves.
"""
return {
"api_base": "",
"taskId": "",
"actionId": "",
"mubiao": "",
"direction": "U方向",
"platform": "",
"audienceRawText": "",
"audienceCandidates": [],
"topAudiences": [],
"pathwayCandidates": [],
"topPathways": [],
"audiencesWithPathways": [],
"audienceVulnCandidates": {},
"pathwayVulnCandidates": {},
"audienceVulns": {},
"pathwayVulns": {},
"audienceStrategies": [],
"pathwayStrategies": [],
"errors": [],
"events": [],
"startedAt": utc_now(),
"updatedAt": utc_now(),
}
def merge_config_into_state(state: dict[str, Any], config: dict[str, Any]) -> None:
# Accept both frontend-style camelCase and CLI-friendly snake_case inputs.
aliases = {
"task_id": "taskId",
"action_id": "actionId",
"apiBase": "api_base",
"target": "mubiao",
"goal": "mubiao",
}
for key, value in config.items():
dest = aliases.get(key, key)
if value is None:
continue
if dest in {"taskId", "actionId"}:
state[dest] = str(value)
elif dest in state or dest in {"api_base", "headers"}:
state[dest] = value
def record_event(state: dict[str, Any], stage: str, message: str, data: Any | None = None) -> None:
"""Append a compact diagnostic event to the state file."""
state.setdefault("events", []).append(
{
"time": utc_now(),
"stage": stage,
"message": message,
"data": data,
}
)
state["updatedAt"] = utc_now()
def feedback_step(stage: str) -> str:
match = re.search(r"(step[1-6])", stage or "")
return match.group(1) if match else ""
def feedback_step_label(stage: str) -> str:
return STEP_NUMBER_LABELS.get(feedback_step(stage), "")
def request_keys(payload: Any) -> list[str]:
if isinstance(payload, dict):
return sorted(REQUEST_FIELD_LABELS.get(str(key), str(key)) for key in payload.keys())
return []
def endpoint_label(endpoint_key: str | None, method: str, url: str) -> str:
if endpoint_key and endpoint_key in ENDPOINT_LABELS:
return ENDPOINT_LABELS[endpoint_key]
path = url.split("://", 1)[-1]
path = "/" + path.split("/", 1)[1] if "/" in path else url
if "/api/task/task-action/" in path and method == "GET":
return "读取行动步骤状态"
if "/api/task/task-action/" in path and method in {"PATCH", "PUT"}:
return "更新行动步骤状态"
if "setTopicKeywords" in path:
return "同步策略关键词"
return f"调用接口 {method}"
def method_label(method: str) -> str:
return HTTP_METHOD_LABELS.get(str(method).upper(), str(method))
def yes_no(value: bool) -> str:
return "是" if value else "否"
def stage_label(stage: Any) -> str:
text = str(stage or "")
if not text:
return ""
if text in STEP_LABELS:
return STEP_LABELS[text]
if text == "runner":
return "流程执行器"
parts = text.split("/")
base = parts[0]
match = re.search(r"(step[1-6])", base)
labels: list[str] = []
if match:
labels.append(STEP_LABELS.get(match.group(1), match.group(1)))
elif base == "runner":
labels.append("流程执行器")
else:
labels.append(base)
for segment in parts[1:]:
normalized = segment.replace("-", "_")
attempt = re.fullmatch(r"model-call-attempt-(\d+)", segment)
json_attempt = re.fullmatch(r"model-json-attempt-(\d+)", segment)
if attempt:
labels.append(f"模型接口第 {attempt.group(1)} 次尝试")
elif json_attempt:
labels.append(f"JSON解析第 {json_attempt.group(1)} 次尝试")
elif segment == "auto-login":
labels.append("自动登录")
elif segment in STAGE_SEGMENT_LABELS:
labels.append(STAGE_SEGMENT_LABELS[segment])
elif normalized in ENDPOINT_LABELS:
labels.append(ENDPOINT_LABELS[normalized])
else:
labels.append(segment)
return " / ".join(labels)
def status_label(status: Any) -> str:
if status is None:
return ""
return STATUS_LABELS.get(str(status), str(status))
def mode_label(mode: str) -> str:
if mode == "full":
return "全流程"
return STEP_LABELS.get(mode, mode)
def exception_type_label(exception_type: Any) -> str:
name = str(exception_type or "")
labels = {
"HTTPError": "HTTP 调用异常",
"URLError": "网络地址异常",
"TimeoutError": "请求超时异常",
"FlowError": "流程异常",
"ValueError": "数据格式异常",
"KeyError": "字段缺失异常",
"RuntimeError": "运行异常",
}
return labels.get(name, "程序异常" if name else "")
def emit_operation_feedback(
state: dict[str, Any],
stage: str,
method: str,
url: str,
*,
endpoint_key: str | None = None,
payload: Any | None = None,
attempt: int = 1,
dry_run: bool = False,
) -> None:
# Feedback lines are a mini protocol consumed by the calling agent. They
# must stay single-line JSON so weak models/tools can relay them reliably.
payload_obj = {
"日志类型": "操作反馈",
"步骤": feedback_step_label(stage),
"阶段": stage_label(stage),
"操作": endpoint_label(endpoint_key, method, url),
"请求方式": method_label(method),
"接口地址": url,
"接口名称": endpoint_label(endpoint_key, method, url),
"第几次尝试": attempt,
"是否演练": yes_no(dry_run),
"请求字段": request_keys(payload),
"状态文件": state.get("_statePath", ""),
"摘要文件": state.get("_summaryPath", ""),
"时间": utc_now(),
}
print(LOG_PREFIXES["operation"] + " " + json.dumps(payload_obj, ensure_ascii=False), flush=True)
def emit_error_feedback(error: dict[str, Any]) -> None:
payload = {
"日志类型": "错误反馈",
"阶段": stage_label(error.get("stage")),
"步骤": feedback_step_label(str(error.get("stage") or "")),
"错误信息": error.get("message"),
"接口地址": error.get("url"),
"状态码": error.get("status"),
"请求摘要": error.get("request"),
"响应摘要": error.get("response"),
"异常类型": exception_type_label(error.get("exceptionType")),
"异常信息": error.get("exception"),
"时间": error.get("time"),
}
print(LOG_PREFIXES["error"] + " " + json.dumps(payload, ensure_ascii=False), flush=True)
def emit_json_parse_review(
state: dict[str, Any],
stage: str,
endpoint_key: str,
text: str,
candidate_count: int,
) -> None:
payload = {
"日志类型": "JSON解析复核",
"阶段": stage_label(stage),
"步骤": feedback_step_label(stage),
"接口名称": ENDPOINT_LABELS.get(endpoint_key, endpoint_key),
"提示": "模型输出经过多候选提取和修复后仍无法解析为可用 JSON,请执行智能体复核是否需要重新调用本步骤接口。",
"候选数量": candidate_count,
"响应片段": compact(text, 3000),
"状态文件": state.get("_statePath", ""),
"摘要文件": state.get("_summaryPath", ""),
"时间": utc_now(),
}
print(LOG_PREFIXES["json_review"] + " " + json.dumps(payload, ensure_ascii=False), flush=True)
SAVE_ENTITY_LABELS = {
"save_design": "受众目标",
"save_design_account": "受众目标账号",
"save_pathway": "关键传播节点",
"save_relation": "受众-传播节点关系",
"save_design_vuln": "受众脆弱点",
"save_pathway_vuln": "传播节点脆弱点",
"save_strategy": "策略",
"save_channel": "策略平台与人设",
"save_narrate": "策略信息主题",
"save_message": "策略消息配置",
"save_strategy_relation": "策略关联关系",
}
def saved_entity_name(endpoint_key: str, payload: dict[str, Any]) -> str:
"""Return the most user-readable name for an inserted row."""
candidates = [
payload.get("targetAudience"),
payload.get("accountName"),
payload.get("vulnerabilities"),
payload.get("topic"),
payload.get("narrate"),
payload.get("platform"),
payload.get("strategyId"),
payload.get("designId"),
payload.get("pathwayId"),
]
for value in candidates:
if value is not None and str(value).strip():
return str(value).strip()
return endpoint_key
def emit_save_feedback(
state: dict[str, Any],
stage: str,
endpoint_key: str,
payload: dict[str, Any],
row_id: int,
*,
dry_run: bool,
) -> None:
# Emit only after an id is known. Downstream steps use these ids and should
# fail loudly rather than inventing relations against missing rows.
label = SAVE_ENTITY_LABELS.get(endpoint_key, endpoint_key)
name = saved_entity_name(endpoint_key, payload)
feedback = {
"日志类型": "入库反馈",
"步骤": feedback_step(stage),
"阶段": stage,
"入库对象类型": label,
"入库对象名称": name,
"返回ID": row_id,
"接口名称": ENDPOINT_LABELS.get(endpoint_key, label),
"接口路径": ENDPOINTS.get(endpoint_key, ""),
"是否演练": yes_no(dry_run),
"请求字段": request_keys(payload),
"状态文件": state.get("_statePath", ""),
"摘要文件": state.get("_summaryPath", ""),
"时间": utc_now(),
}
print(LOG_PREFIXES["save"] + " " + json.dumps(feedback, ensure_ascii=False), flush=True)
record_event(
state,
stage,
f"入库{label}成功:{name}",
{"id": row_id, "endpoint": endpoint_key, "dryRun": dry_run},
)
def record_error(
state: dict[str, Any],
stage: str,
message: str,
*,
url: str | None = None,
status: int | None = None,
request: Any | None = None,
response: Any | None = None,
exception: BaseException | None = None,
) -> None:
# Persist full-enough error context for post-run diagnosis, then immediately
# emit a short line the agent can show to the user.
item = {
"time": utc_now(),
"stage": stage,
"message": message,
"url": url,
"status": status,
"request": safe_compact(request, 1200) if request is not None else None,
"response": safe_compact(response, 2000) if response is not None else None,
"exceptionType": type(exception).__name__ if exception else None,
"exception": safe_compact(str(exception), 1000) if exception else None,
}
state.setdefault("errors", []).append(item)
state["updatedAt"] = utc_now()
emit_error_feedback(item)
def stash_recovered_errors(state: dict[str, Any], error_count_before: int, stage: str, reason: str) -> None:
errors = state.setdefault("errors", [])
if len(errors) <= error_count_before:
return
recovered = errors[error_count_before:]
del errors[error_count_before:]
state.setdefault("recoveredErrors", []).extend(recovered)
record_event(
state,
stage,
"模型调用首次异常,准备重试;该异常如重试成功则不计入最终 errors",
{"reason": reason, "recoveredErrorCount": len(recovered)},
)
def require_context(state: dict[str, Any], *keys: str) -> None:
missing = [key for key in keys if not str(state.get(key) or "").strip()]
if missing:
raise FlowError(f"缺少必要参数:{', '.join(missing)}")
def url_for(ctx: FlowContext, endpoint_key: str) -> str:
return ctx.api_base.rstrip("/") + ENDPOINTS[endpoint_key]
def url_for_path(ctx: FlowContext, path: str) -> str:
return ctx.api_base.rstrip("/") + "/" + path.lstrip("/")
def parse_headers(header_values: list[str]) -> dict[str, str]:
headers = {"Content-Type": "application/json", "Accept": "application/json"}
for raw in header_values:
if ":" not in raw:
raise FlowError(f"header 格式错误,应为 Key: Value,实际:{raw}")
key, value = raw.split(":", 1)
normalized = key.strip()
if normalized.lower() == "authorization":
continue
headers[normalized] = value.strip()
return headers
def ssl_context(ctx: FlowContext) -> ssl.SSLContext | None:
if ctx.verify_ssl:
return None
return ssl._create_unverified_context()
def login_candidate_paths(ctx: FlowContext) -> list[str]:
paths = [
ctx.login_path or "/api/auth/login",
"/api/auth/login",
]
seen: set[str] = set()
unique: list[str] = []
for path in paths:
normalized = str(path or "").strip()
if not normalized or normalized in seen:
continue
seen.add(normalized)
unique.append(normalized)
return unique
def try_auto_login(ctx: FlowContext, state: dict[str, Any], stage: str, *, force: bool = False) -> bool:
"""Fetch a fresh token through the local auth package before business calls."""
if not ctx.auto_login:
return False
if ctx.dry_run:
record_event(state, stage, "dry-run 跳过自动登录", {"username": ctx.login_username})
return False
if not ctx.login_username:
record_error(state, stage, "自动登录失败:缺少 login_username")
return False
if not force and ctx.login_attempt_count > 0:
return any(key.lower() == "authorization" and str(value).strip() for key, value in ctx.headers.items())
if force and ctx.login_attempt_count >= 2:
return False
ctx.login_attempt_count += 1
payload = {"username": ctx.login_username, "password": ctx.login_password}
last_status: int | None = None
last_response = ""
last_exc: BaseException | None = None
for attempt, path in enumerate(login_candidate_paths(ctx), start=1):
display_url = xdfx_auth.redact_url(xdfx_auth.build_login_url(ctx.api_base, path))
emit_operation_feedback(
state,
stage,
"POST",
display_url,
endpoint_key="login",
payload=payload,
attempt=attempt,
)
try:
result = xdfx_auth.fetch_access_token(
api_base=ctx.api_base,
username=ctx.login_username,
password=ctx.login_password,
login_path=path,
timeout=ctx.timeout,
verify_ssl=ctx.verify_ssl,
)
ctx.headers["Authorization"] = xdfx_auth.auth_header_value(result.token)
state["_tokenSource"] = "xdfx_auth"
record_event(
state,
stage,
"启动阶段已通过 xdfx_auth 包获取 token,并写入 Authorization",
{"url": result.url, "status": result.status, "username": ctx.login_username},
)
return True
except xdfx_auth.AuthError as exc:
last_status = exc.status
last_response = exc.response
last_exc = exc
record_event(
state,
stage,
"xdfx_auth 获取 token 失败,准备尝试下一个候选登录路径",
{"url": exc.url, "status": exc.status, "response": safe_compact(exc.response, 1000), "exception": str(exc)},
)
record_error(
state,
stage,
"自动登录失败,未能通过 xdfx_auth 获取 access_token",
url=xdfx_auth.redact_url(xdfx_auth.build_login_url(ctx.api_base, ctx.login_path or "/api/auth/login")),
status=last_status,
request=payload,
response=last_response,
exception=last_exc,
)
return False
def http_request_json_url(
ctx: FlowContext,
state: dict[str, Any],
method: str,
url: str,
payload: dict[str, Any] | None,
stage: str,
endpoint_key: str | None = None,
allow_auto_login: bool = True,
) -> tuple[Any | None, str]:
"""Call an HTTP JSON endpoint with logging, retries, and 401 recovery."""
method = method.upper()
if ctx.dry_run:
emit_operation_feedback(state, stage, method, url, endpoint_key=endpoint_key, payload=payload, dry_run=True)
record_event(state, stage, f"DRY RUN {method} {url}", {"payload": payload})
return {"dryRun": True}, json.dumps({"dryRun": True})
data = None if payload is None else json.dumps(payload, ensure_ascii=False).encode("utf-8")
last_exc: BaseException | None = None
for attempt in range(ctx.retries + 1):
emit_operation_feedback(
state,
stage,
method,
url,
endpoint_key=endpoint_key,
payload=payload,
attempt=attempt + 1,
)
try:
req = urllib.request.Request(url, data=data, headers=ctx.headers, method=method)
with urllib.request.urlopen(req, timeout=ctx.timeout, context=ssl_context(ctx)) as resp:
raw = resp.read().decode("utf-8", errors="replace")
parsed = parse_json_lenient(raw)
record_event(state, stage, f"{method} {url} ok", {"status": getattr(resp, "status", None)})
return parsed, raw
except urllib.error.HTTPError as exc:
body = exc.read().decode("utf-8", errors="replace") if exc.fp else ""
if exc.code == 401 and allow_auto_login and ctx.auto_login:
# Refresh once on 401, then replay the original request exactly.
refreshed = try_auto_login(ctx, state, f"{stage}/auto-login", force=True)
if refreshed:
record_event(
state,
stage,
"接口返回 401,自动登录刷新 token 后重试原请求",
{"url": url, "status": exc.code, "response": compact(body, 1000)},
)
return http_request_json_url(
ctx,
state,
method,
url,
payload,
f"{stage}/retry-after-login",
endpoint_key=endpoint_key,
allow_auto_login=False,
)
record_error(
state,
stage,
"HTTP 接口返回错误",
url=url,
status=exc.code,
request=payload,
response=body,
exception=exc,
)
if exc.code < 500 or attempt >= ctx.retries:
return None, body
last_exc = exc
except Exception as exc: # noqa: BLE001 - CLI runner must capture every failure.
last_exc = exc
if attempt >= ctx.retries:
record_error(
state,
stage,
"HTTP 调用异常",
url=url,
request=payload,
response=traceback.format_exc(),
exception=exc,
)
return None, ""
time.sleep(min(2**attempt, 5))
if last_exc:
record_error(state, stage, "HTTP 调用失败", url=url, request=payload, exception=last_exc)
return None, ""
def http_get_json_path(ctx: FlowContext, state: dict[str, Any], path: str, stage: str) -> tuple[Any | None, str]:
return http_request_json_url(ctx, state, "GET", url_for_path(ctx, path), None, stage)
def http_patch_json_path(ctx: FlowContext, state: dict[str, Any], path: str, payload: dict[str, Any], stage: str) -> tuple[Any | None, str]:
return http_request_json_url(ctx, state, "PATCH", url_for_path(ctx, path), payload, stage)
def http_get_json_endpoint(ctx: FlowContext, state: dict[str, Any], endpoint_path: str, stage: str) -> tuple[Any | None, str]:
return http_request_json_url(ctx, state, "GET", url_for_path(ctx, endpoint_path), None, stage)
def http_post_json(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, payload: dict[str, Any], stage: str) -> tuple[Any | None, str]:
url = url_for(ctx, endpoint_key)
if ctx.dry_run:
emit_operation_feedback(state, stage, "POST", url, endpoint_key=endpoint_key, payload=payload, dry_run=True)
record_event(state, stage, f"DRY RUN POST {url}", {"payload": payload})
if endpoint_key.startswith("save_"):
# Synthetic ids let downstream dry-run code exercise relation wiring.
synthetic_id = abs(hash(stage + stable_json(payload))) % 100000000
return {"id": synthetic_id, "dryRun": True}, json.dumps({"id": synthetic_id, "dryRun": True})
return {"dryRun": True}, json.dumps({"dryRun": True})
return http_request_json_url(ctx, state, "POST", url, payload, stage, endpoint_key=endpoint_key)
def parse_json_lenient(text: str) -> Any | None:
"""Parse normal JSON plus adjacent/SSE-like JSON objects."""
stripped = text.strip()
if not stripped:
return None
try:
return json.loads(stripped)
except Exception:
pass
objects = parse_concatenated_json_objects(stripped)
if objects:
return objects
return None
def parse_concatenated_json_objects(text: str) -> list[Any]:
"""Read streams such as {"answer": "..."}{"answer": "..."} or data: lines."""
decoder = json.JSONDecoder()
objects: list[Any] = []
i = 0
while i < len(text):
while i < len(text) and text[i].isspace():
i += 1
if i >= len(text):
break
if text.startswith("data:", i):
line_end = text.find("\n", i)
if line_end == -1:
line_end = len(text)
line = text[i + 5 : line_end].strip()
if line and line != "[DONE]":
try:
objects.append(json.loads(line))
except Exception:
pass
i = line_end + 1
continue
if text[i] not in "{[":
nxt = min([p for p in (text.find("{", i + 1), text.find("[", i + 1)) if p != -1], default=-1)
if nxt == -1:
break
i = nxt
continue
try:
obj, end = decoder.raw_decode(text, i)
objects.append(obj)
i = end
except Exception:
i += 1
return objects
def model_text_from_response(parsed: Any, raw: str) -> tuple[str, list[Any]]:
"""Flatten streamed model chunks into one text buffer."""
objects: list[Any]
if isinstance(parsed, list):
objects = parsed
elif parsed is None:
objects = parse_concatenated_json_objects(raw)
else:
objects = [parsed]
parts: list[str] = []
for obj in objects:
collect_answer_parts(obj, parts)
if parts:
return "".join(parts), objects
if isinstance(parsed, (dict, list)):
return json.dumps(parsed, ensure_ascii=False), objects
return raw, objects
def model_retry_reason(text: str, raw: str, parsed: Any) -> str | None:
pieces: list[str] = []
if text:
pieces.append(text)
if raw and raw != text:
pieces.append(raw)
if parsed is not None:
pieces.append(compact(parsed, 2000))
combined = "\n".join(pieces).strip()
if not combined:
return "模型接口空响应"
lowered = combined.lower()
for marker in MODEL_RETRY_MARKERS:
if marker.lower() in lowered:
return f"模型接口返回异常标记:{marker}"
return None
def collect_answer_parts(obj: Any, parts: list[str]) -> None:
if isinstance(obj, dict):
if "answer" in obj and isinstance(obj.get("answer"), str):
parts.append(obj["answer"])
for value in obj.values():
collect_answer_parts(value, parts)
elif isinstance(obj, list):
for item in obj:
collect_answer_parts(item, parts)
def strip_think(text: str) -> str:
text = text.replace("\ufeff", "").replace("\u200b", "")
return re.sub(r"<think>.*?</think>", "", text, flags=re.DOTALL | re.IGNORECASE).strip()
def extract_json_from_text(text: str) -> Any | None:
candidates = extract_json_candidates_from_text(text)
return select_json_candidate(candidates, endpoint_key="", stage="")
def extract_json_candidates_from_text(text: str) -> list[Any]:
"""Collect every plausible JSON object/array from messy model text."""
cleaned = strip_think(text)
candidates: list[Any] = []
seen: set[str] = set()
def add_candidate(value: Any) -> None:
if value is None:
return
try:
key = stable_json(value)
except Exception:
key = repr(value)
if key in seen:
return
seen.add(key)
candidates.append(value)
for block in re.findall(r"```(?:json|JSON)?\s*(.*?)```", cleaned, flags=re.DOTALL):
add_candidate(try_parse_json_object(block))
for obj in parse_concatenated_json_objects(block):
add_candidate(obj)
for segment in iter_balanced_segments(block, "{", "}"):
add_candidate(try_parse_json_object(segment))
for segment in iter_balanced_segments(block, "[", "]"):
add_candidate(try_parse_json_object(segment))
add_candidate(try_parse_json_object(cleaned))
for obj in parse_concatenated_json_objects(cleaned):
add_candidate(obj)
for segment in iter_balanced_segments(cleaned, "{", "}"):
add_candidate(try_parse_json_object(segment))
for segment in iter_balanced_segments(cleaned, "[", "]"):
add_candidate(try_parse_json_object(segment))
return candidates
def select_json_candidate(candidates: list[Any], endpoint_key: str, stage: str) -> Any | None:
"""Pick the candidate whose keys best match the current step."""
if not candidates:
return None
scored = [(json_candidate_score(candidate, endpoint_key, stage), idx, candidate) for idx, candidate in enumerate(candidates)]
scored.sort(key=lambda item: (item[0], item[1]), reverse=True)
best_score, _, best = scored[0]
if best_score <= 0:
return candidates[-1]
return best
def json_candidate_score(candidate: Any, endpoint_key: str, stage: str) -> int:
keys = flatten_json_keys(candidate)
text = stable_json(candidate)[:6000] if candidate is not None else ""
score = 1 if isinstance(candidate, (dict, list)) else 0
expected = expected_json_terms(endpoint_key, stage)
for term in expected:
if term in keys or term in text:
score += 8
if isinstance(candidate, dict):
score += min(len(candidate), 20)
if "items" in candidate and isinstance(candidate.get("items"), list):
score += len(candidate["items"])
elif isinstance(candidate, list):
score += min(len(candidate), 20)
if "answer" in keys and not any(term in keys or term in text for term in expected):
score -= 5
return score
def expected_json_terms(endpoint_key: str, stage: str) -> list[str]:
haystack = f"{endpoint_key} {stage}"
if "group" in haystack or "audience" in haystack:
return ["群体", "受众", "targetAudience", "用户编号列表", "recReason", "理由"]
if "spread" in haystack or "pathway" in haystack:
return ["账号", "传播节点", "account", "recReason", "理由"]
if "cuiruo" in haystack or "vuln" in haystack:
return ["脆弱点", "vulnerability", "vuln", "vulnerabilities", "理由"]
if "strategy" in haystack or "newboci" in haystack:
return ["信息标题", "发布人设", "账号类型", "社交平台", "topic", "persona", "accountType"]
return ["items", "data", "result"]
def flatten_json_keys(value: Any) -> set[str]:
keys: set[str] = set()
if isinstance(value, dict):
for key, item in value.items():
keys.add(str(key))
keys.update(flatten_json_keys(item))
elif isinstance(value, list):
for item in value:
keys.update(flatten_json_keys(item))
return keys
def try_parse_json_object(text: str) -> Any | None:
candidate = text.strip().strip("\u200b")
if not candidate:
return None
normalized = normalize_json_text(candidate)
repairs = [
candidate,
normalized,
quote_unquoted_json_keys(normalized),
re.sub(r",\s*([}\]])", r"\1", quote_unquoted_json_keys(normalized)),
]
for item in repairs:
try:
return json.loads(item)
except Exception:
pass
try:
parsed = ast.literal_eval(item)
if isinstance(parsed, (dict, list)):
return parsed
except Exception:
pass
return None
def normalize_json_text(text: str) -> str:
normalized = (
text.strip()
.replace("\ufeff", "")
.replace("\u200b", "")
.replace("“", '"')
.replace("”", '"')
.replace("„", '"')
.replace(""", '"')
.replace("‘", "'")
.replace("’", "'")
.replace(",", ",")
.replace(":", ":")
)
normalized = re.sub(r",\s*([}\]])", r"\1", normalized)
normalized = re.sub(r"\bNone\b", "null", normalized)
normalized = re.sub(r"\bTrue\b", "true", normalized)
normalized = re.sub(r"\bFalse\b", "false", normalized)
return normalized
def quote_unquoted_json_keys(text: str) -> str:
return re.sub(r'([{\[,]\s*)([A-Za-z_\u4e00-\u9fff][\w\u4e00-\u9fff\-]*\d*)\s*:', r'\1"\2":', text)
def iter_balanced_segments(text: str, open_ch: str, close_ch: str, limit: int = 80) -> list[str]:
segments: list[str] = []
start = text.find(open_ch)
while start != -1 and len(segments) < limit:
segment = extract_balanced_from(text, start, open_ch, close_ch)
if segment:
segments.append(segment)
start = text.find(open_ch, start + 1)
else:
start = text.find(open_ch, start + 1)
return segments
def extract_balanced_from(text: str, start: int, open_ch: str, close_ch: str) -> str | None:
depth = 0
in_string = False
quote_char = ""
escape = False
for idx in range(start, len(text)):
ch = text[idx]
if in_string:
if escape:
escape = False
elif ch == "\\":
escape = True
elif ch == quote_char:
in_string = False
continue
if ch in {'"', "'"}:
in_string = True
quote_char = ch
elif ch == open_ch:
depth += 1
elif ch == close_ch:
depth -= 1
if depth == 0:
return text[start : idx + 1]
return None
def extract_first_balanced(text: str, open_ch: str, close_ch: str) -> str | None:
segments = iter_balanced_segments(text, open_ch, close_ch, limit=1)
return segments[0] if segments else None
def call_model_text(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, payload: dict[str, Any], stage: str) -> tuple[str, list[Any], Any | None]:
"""Call a model endpoint and retry only known empty/timeout/no-response cases."""
max_attempts = max(ctx.model_retries, 0) + 1
last_text = ""
last_objects: list[Any] = []
last_parsed: Any | None = None
last_reason = ""
for attempt in range(max_attempts):
attempt_stage = f"{stage}/model-call-attempt-{attempt + 1}"
error_count_before = len(state.get("errors") or [])
parsed, raw = http_post_json(ctx, state, endpoint_key, payload, attempt_stage)
text, objects = model_text_from_response(parsed, raw)
reason = model_retry_reason(text, raw, parsed)
last_text, last_objects, last_parsed, last_reason = text, objects, parsed, reason or ""
if reason is None:
if attempt > 0:
record_event(state, stage, f"模型接口第 {attempt + 1} 次调用恢复成功")
return text, objects, parsed
if attempt < max_attempts - 1:
stash_recovered_errors(state, error_count_before, attempt_stage, reason)
record_event(
state,
stage,
f"模型接口第 {attempt + 1} 次调用无有效响应,自动重试一次",
{"reason": reason, "endpoint": endpoint_key},
)
time.sleep(min(2**attempt, 3))
continue
record_error(
state,
stage,
"模型接口重试后仍无有效响应",
url=url_for(ctx, endpoint_key),
request=payload,
response={"reason": last_reason, "text": last_text, "parsed": last_parsed},
)
return last_text, last_objects, last_parsed
def call_model_json(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, payload: dict[str, Any], stage: str) -> tuple[dict[str, Any], str, list[Any]]:
"""Return the best structured JSON for a model step.
If the first parse fails, the second attempt includes an explicit repair
instruction so weak models can re-emit only the required JSON.
"""
last_text = ""
last_objects: list[Any] = []
last_candidate_count = 0
current = copy.deepcopy(payload)
for attempt in range(2):
text, objects, parsed = call_model_text(ctx, state, endpoint_key, current, f"{stage}/model-json-attempt-{attempt + 1}")
last_text = text
last_objects = objects
if isinstance(parsed, dict) and not any("answer" in obj for obj in as_list(parsed) if isinstance(obj, dict)):
return parsed, text, objects
candidates = extract_json_candidates_from_text(text)
last_candidate_count = len(candidates)
extracted = select_json_candidate(candidates, endpoint_key, stage)
if isinstance(extracted, dict):
return extracted, text, objects
if isinstance(extracted, list):
return {"items": extracted}, text, objects
current = copy.deepcopy(payload)
suffix = "\n\n重要:上一轮输出无法解析。请只返回一个合法 JSON 对象,不要解释,不要 Markdown,不要 think。"
if isinstance(current.get("user_message"), str):
current["user_message"] += suffix
elif isinstance(current.get("usermessage"), str):
current["usermessage"] += suffix
else:
current["user_message"] = suffix.strip()
emit_json_parse_review(state, stage, endpoint_key, last_text, last_candidate_count)
record_error(state, stage, "模型输出无法解析为 JSON", request=payload, response=last_text)
return {}, last_text, last_objects
def is_account_like(obj: dict[str, Any]) -> bool:
if not isinstance(obj, dict):
return False
has_text = any(k in obj for k in ("page_content", "content", "frofileInfo", "发文", "docs"))
has_identity = any(k in obj for k in ("accountId", "accountName", "user_id", "name", "id", "screen_name")) or isinstance(obj.get("metadata"), dict)
return has_text and has_identity
def collect_accounts(obj: Any, out: list[dict[str, Any]]) -> None:
if isinstance(obj, dict):
docs = obj.get("docs")
if isinstance(docs, dict):
collect_accounts(docs, out)
if is_account_like(obj):
out.append(obj)
for value in obj.values():
if value is docs:
continue
collect_accounts(value, out)
elif isinstance(obj, list):
for item in obj:
collect_accounts(item, out)
def accounts_from_response(parsed: Any, objects: list[Any], text: str) -> list[dict[str, Any]]:
out: list[dict[str, Any]] = []
collect_accounts(parsed, out)
collect_accounts(objects, out)
if not out:
for candidate in extract_json_candidates_from_text(text):
collect_accounts(candidate, out)
return dedupe_accounts(out)
def dedupe_accounts(accounts: list[dict[str, Any]]) -> list[dict[str, Any]]:
seen: set[str] = set()
result: list[dict[str, Any]] = []
for account in accounts:
key = str(first_present(account_id(account), account_name(account), stable_json(account)[:200]))
if key in seen:
continue
seen.add(key)
result.append(account)
return result
def profile(account: dict[str, Any]) -> dict[str, Any]:
meta = account.get("metadata") if isinstance(account.get("metadata"), dict) else {}
user_profile = meta.get("用户画像") if isinstance(meta.get("用户画像"), dict) else {}
return {**meta, **user_profile}
def account_id(account: dict[str, Any]) -> str:
prof = profile(account)
value = first_present(account.get("accountId"), account.get("user_id"), deep_get(account, "metadata", "user_id"), prof.get("user_id"))
if value is None:
value = account.get("id")
return str(value or "").strip()
def account_name(account: dict[str, Any]) -> str:
prof = profile(account)
value = first_present(account.get("accountName"), account.get("name"), account.get("user_name"), prof.get("user_name"), account.get("screen_name"), prof.get("screen_name"))
return str(value or "").strip()
def account_platform_raw(account: dict[str, Any]) -> Any:
prof = profile(account)
return first_present(account.get("platformType"), account.get("accountPlat"), account.get("platform"), deep_get(account, "metadata", "平台"), prof.get("平台"))
def platform_to_number(value: Any) -> Any:
number = as_int(value)
if number is not None:
return number
text = str(value or "").strip().lower()
return PLATFORM_MAP.get(text, value if value not in (None, "") else "")
def platform_label(value: Any) -> str:
raw = account_platform_raw(value) if isinstance(value, dict) else value
if raw is None:
return ""
number = as_int(raw)
if number == 1:
return "facebook"
if number == 13:
return "ptt"
text = str(raw).strip()
return {"fb": "facebook"}.get(text.lower(), text)
def fans_count(account: dict[str, Any]) -> int:
prof = profile(account)
for value in (
account.get("fansCount"),
account.get("followerCount"),
account.get("followers_count"),
account.get("fans_count"),
deep_get(account, "metadata", "fans_count"),
prof.get("粉丝数"),
prof.get("fans_count"),
):
number = as_int(value)
if number is not None:
return number
return 0
def account_issue(account: dict[str, Any]) -> str:
value = first_present(account.get("issue"), deep_get(account, "metadata", "issue"))
if isinstance(value, list):
return ";".join(str(v) for v in value)
return str(value or "")
def profile_info(account: dict[str, Any]) -> str:
value = account.get("frofileInfo")
if isinstance(value, str) and value.strip():
return value
return json.dumps(account, ensure_ascii=False)
def parse_index_list(value: Any) -> list[str]:
refs: list[str] = []
def add(raw: Any) -> None:
if raw is None:
return
if isinstance(raw, dict):
keys = (
"编号id",
"编号ID",
"用户编号id",
"用户编号ID",
"用户编号",
"账号编号",
"引用序号",
"索引",
"index",
"id",
"ids",
"accountIndex",
"accountIndexes",
"userIndex",
"userIndexes",
)
for key in keys:
if key in raw:
add(raw.get(key))
return
if isinstance(raw, (list, tuple, set)):
for item in raw:
add(item)
return
text = str(raw).strip()
if not text:
return
angle_refs = re.findall(r"<<\s*([^<>]+?)\s*>>", text)
if angle_refs:
for token in angle_refs:
add(token)
return
for part in re.split(r"[,,;;、\s]+", text):
token = part.strip().strip("<>").strip()
if token:
refs.append(token)
add(value)
deduped: list[str] = []
seen: set[str] = set()
for ref in refs:
if ref in seen:
continue
seen.add(ref)
deduped.append(ref)
return deduped
def account_ref_numbers(account: dict[str, Any]) -> set[int]:
numbers: set[int] = set()
meta = account.get("metadata") if isinstance(account.get("metadata"), dict) else {}
for key in ("编号id", "编号ID", "用户编号id", "用户编号ID", "用户编号", "账号编号", "引用序号", "索引", "index", "accountIndex"):
for container in (account, meta):
number = as_int(container.get(key)) if isinstance(container, dict) else None
if number is not None:
numbers.add(number)
return numbers
def resolve_account_ref(ref: Any, accounts: list[dict[str, Any]]) -> dict[str, Any] | None:
"""Resolve model references like <<3>>, index numbers, names, or account ids."""
if not accounts:
return None
text = str(ref).strip().strip("<>").strip()
number = as_int(text)
if number is not None:
for account in accounts:
if number in account_ref_numbers(account):
return account
# Step1 group extraction returns 编号/id values that refer to the
# previous zhanghao array index. Prefer exact index fields above, then
# fall back to zero-based and one-based array positions.
if 0 <= number < len(accounts):
return accounts[number]
if 1 <= number <= len(accounts):
return accounts[number - 1]
lower = text.lower()
for account in accounts:
names = [account_name(account), account_id(account), str(account.get("screen_name") or ""), str(deep_get(account, "metadata", "screen_name") or "")]
if any(name and (lower in name.lower() or name.lower() in lower) for name in names):
return account
return None
def extract_audiences(data: dict[str, Any]) -> list[dict[str, Any]]:
"""Normalize audience JSON whether the model used numbered fields or items[]."""
items: list[dict[str, Any]] = []
for i in range(1, 21):
group = first_present(data.get(f"群体{i}"), data.get(f"受众{i}"), data.get(f"targetAudience{i}"))
reason = first_present(data.get(f"理由{i}"), data.get(f"推荐理由{i}"), data.get(f"recReason{i}"))
users = first_present(
data.get(f"用户编号列表{i}"),
data.get(f"编号id列表{i}"),
data.get(f"编号ID列表{i}"),
data.get(f"用户编号id列表{i}"),
data.get(f"用户编号ID列表{i}"),
data.get(f"用户列表{i}"),
data.get(f"账号编号列表{i}"),
data.get(f"索引列表{i}"),
data.get(f"编号{i}"),
data.get(f"id{i}"),
data.get(f"ids{i}"),
data.get(f"索引{i}"),
)
if group:
items.append(
{
"targetAudience": str(group).strip(),
"recReason": str(reason or "").strip(),
"userIndexes": parse_index_list(users),
"rank": i,
}
)
if not items and isinstance(data.get("items"), list):
for i, item in enumerate(data["items"], start=1):
if isinstance(item, dict):
group = first_present(item.get("群体"), item.get("受众"), item.get("targetAudience"), item.get("name"))
if group:
items.append(
{
"targetAudience": str(group).strip(),
"recReason": str(first_present(item.get("理由"), item.get("recReason"), item.get("reason")) or "").strip(),
"userIndexes": parse_index_list(
first_present(
item.get("用户编号列表"),
item.get("编号id列表"),
item.get("编号ID列表"),
item.get("用户编号id列表"),
item.get("用户编号ID列表"),
item.get("编号id"),
item.get("编号ID"),
item.get("用户编号id"),
item.get("用户编号ID"),
item.get("编号"),
item.get("索引"),
item.get("id"),
item.get("ids"),
item.get("users"),
item.get("accounts"),
item.get("userIndexes"),
item.get("accountIndexes"),
)
),
"rank": i,
}
)
return items
def audience_score(item: dict[str, Any]) -> tuple[int, int]:
text = (item.get("targetAudience", "") + " " + item.get("recReason", "")).lower()
keywords = ["重点", "核心", "高度相关", "关键", "军事", "安全", "两岸", "美国", "对台", "影响力", "传播力", "现实派"]
score = sum(3 for kw in keywords if kw.lower() in text)
score += min(len(item.get("userIndexes") or []), 5)
score += max(0, 6 - int(item.get("rank") or 99))
return score, -int(item.get("rank") or 99)
def select_top_audiences(items: list[dict[str, Any]], limit: int) -> list[dict[str, Any]]:
"""Deduplicate audience names and keep the strongest candidates."""
unique: dict[str, dict[str, Any]] = {}
for item in items:
key = item.get("targetAudience", "")
if key and key not in unique:
unique[key] = item
return sorted(unique.values(), key=audience_score, reverse=True)[:limit]
def extract_pathway_refs(data: dict[str, Any]) -> list[dict[str, Any]]:
items: list[dict[str, Any]] = []
for i in range(1, 21):
ref = first_present(data.get(f"账号{i}"), data.get(f"传播节点{i}"), data.get(f"account{i}"))
reason = first_present(data.get(f"理由{i}"), data.get(f"推荐理由{i}"), data.get(f"recReason{i}"))
if ref is not None:
items.append({"ref": ref, "recReason": str(reason or "").strip(), "rank": i})
return items
def pathway_score(item: dict[str, Any]) -> tuple[int, int, int]:
account = item.get("account") or {}
text = (item.get("recReason", "") + " " + account_name(account) + " " + account_issue(account)).lower()
keywords = ["核心", "关键", "传播", "影响力", "粉丝", "贴合", "军事", "安全", "两岸", "美国", "专业"]
score = sum(3 for kw in keywords if kw.lower() in text)
score += min(fans_count(account) // 1000, 50)
score += max(0, 8 - int(item.get("rank") or 99))
return score, fans_count(account), -int(item.get("rank") or 99)
def merge_pathway_candidates(candidates: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Merge the same account found from different audiences before saving top 3."""
merged: dict[str, dict[str, Any]] = {}
for item in candidates:
account = item.get("account") or {}
key = account_id(account) or account_name(account) or stable_json(account)[:200]
if not key:
continue
existing = merged.get(key)
if existing is None:
item["designIds"] = list(dict.fromkeys(item.get("designIds") or []))
item["relatedAudiences"] = list(dict.fromkeys(item.get("relatedAudiences") or []))
merged[key] = item
else:
existing["designIds"] = list(dict.fromkeys((existing.get("designIds") or []) + (item.get("designIds") or [])))
existing["relatedAudiences"] = list(dict.fromkeys((existing.get("relatedAudiences") or []) + (item.get("relatedAudiences") or [])))
if pathway_score(item) > pathway_score(existing):
keep_ids = existing["designIds"]
keep_audiences = existing["relatedAudiences"]
merged[key] = item
merged[key]["designIds"] = keep_ids
merged[key]["relatedAudiences"] = keep_audiences
return list(merged.values())
def extract_vulns(data: dict[str, Any], limit: int) -> list[dict[str, str]]:
"""Normalize weak-point model output into a capped list."""
vulns: list[dict[str, str]] = []
for i in range(1, 21):
vuln = first_present(data.get(f"脆弱点{i}"), data.get(f"vulnerability{i}"), data.get(f"vuln{i}"))
reason = first_present(data.get(f"理由{i}"), data.get(f"reason{i}"))
if vuln:
text = str(vuln).strip()
if text and text not in [v["vulnerabilities"] for v in vulns]:
vulns.append({"vulnerabilities": text, "reason": str(reason or "").strip(), "rank": str(i)})
if len(vulns) >= limit:
break
return vulns
def extract_strategies(data: dict[str, Any], limit: int) -> list[dict[str, Any]]:
"""Normalize strategy output and preserve the modal text for later mapping."""
strategies: list[dict[str, Any]] = []
for i in range(1, 21):
title = first_present(data.get(f"信息标题{i}"), data.get(f"主题{i}"), data.get(f"topic{i}"))
persona = first_present(data.get(f"发布人设{i}"), data.get(f"人设{i}"), data.get(f"persona{i}"), "专业议题分析者")
account_type = first_present(data.get(f"账号类型{i}"), data.get(f"accountType{i}"))
social_platform = first_present(data.get(f"社交平台{i}"), data.get(f"平台{i}"), data.get(f"platform{i}"))
format_value = first_present(data.get(f"信息形式{i}"), data.get(f"format{i}"), data.get(f"modelType{i}"))
language = first_present(data.get(f"语种{i}"), data.get(f"language{i}"))
count = first_present(data.get(f"数量{i}"), data.get(f"信息数量{i}"), data.get(f"respondingCount{i}"))
persona_size = first_present(data.get(f"人设数量{i}"), data.get(f"personaSize{i}"))
if title:
strategies.append(
{
"persona": str(persona or "专业议题分析者").strip(),
"accountType": str(account_type or "").strip(),
"platform": str(social_platform or "").strip(),
"title": str(title).strip(),
"format": format_value,
"language": language,
"respondingCount": count,
"personaSize": persona_size,
"rank": i,
}
)
if len(strategies) >= limit:
break
if not strategies:
title = first_present(data.get("信息标题"), data.get("主题"), data.get("topic"))
if title:
strategies.append(
{
"persona": str(first_present(data.get("发布人设"), data.get("人设"), "专业议题分析者")).strip(),
"accountType": str(first_present(data.get("账号类型"), data.get("accountType"), "")).strip(),
"platform": str(first_present(data.get("社交平台"), data.get("平台"), "")).strip(),
"title": str(title).strip(),
"format": first_present(data.get("信息形式"), data.get("format"), data.get("modelType")),
"language": first_present(data.get("语种"), data.get("language")),
"respondingCount": first_present(data.get("数量"), data.get("信息数量"), data.get("respondingCount")),
"personaSize": first_present(data.get("人设数量"), data.get("personaSize")),
"rank": 1,
}
)
return strategies
def response_id(response: Any) -> int | None:
if isinstance(response, dict):
number = as_int(response.get("id"))
if number is not None:
return number
data = response.get("data")
if isinstance(data, dict):
return response_id(data)
return None
def response_rows(response: Any) -> list[Any]:
"""Extract list rows from common CRUD response shapes."""
if isinstance(response, list):
return response
if not isinstance(response, dict):
return []
for key in ("data", "rows", "items", "records", "list"):
value = response.get(key)
if isinstance(value, list):
return value
rows = response_rows(value)
if rows:
return rows
return []
def save_with_id(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, payload: dict[str, Any], stage: str) -> tuple[int | None, Any | None]:
response, _ = http_post_json(ctx, state, endpoint_key, payload, stage)
rid = response_id(response)
if rid is None:
record_error(state, stage, "保存接口未返回 ID,视为失败", request=payload, response=response)
else:
emit_save_feedback(state, stage, endpoint_key, payload, rid, dry_run=ctx.dry_run)
return rid, response
DEFAULT_STEP_FLAGS = {
"step-analysis-three": 1,
"step-info-search-ceh": 1,
"step-analysis-audience": 0,
"step-analysis-keypoint": 0,
"step-analysis-weakpoint": 0,
"step-analysis-strategy": 0,
"step-analysis-strategy-gantt": 0,
"step-effect-analysis": 0,
}
def parse_step_flags(raw: Any) -> dict[str, Any]:
if isinstance(raw, dict):
return dict(raw)
if isinstance(raw, str) and raw.strip():
try:
parsed = json.loads(raw)
if isinstance(parsed, dict):
return parsed
except Exception:
return {}
return {}
def mark_step_status(ctx: FlowContext, state: dict[str, Any], links: list[str], stage: str) -> None:
"""Update the frontend step bar without overwriting unrelated step flags."""
if not ctx.sync_step_status:
record_event(state, stage, "跳过 task-action.step 状态同步")
return
action_id = as_int(state.get("actionId")) or state.get("actionId")
if not action_id:
record_error(state, stage, "缺少 actionId,无法同步步骤状态")
return
step_obj = {**DEFAULT_STEP_FLAGS, **parse_step_flags(state.get("actionStep"))}
if not ctx.dry_run:
action, _ = http_get_json_path(ctx, state, f"/api/task/task-action/{action_id}", stage + "/get-action-step")
if isinstance(action, dict):
step_obj = {**DEFAULT_STEP_FLAGS, **parse_step_flags(action.get("step"))}
elif action is None:
record_error(state, stage, "读取 task-action 失败,仍尝试用本地步骤状态写回")
for link in links:
step_obj[link] = 1
state["actionStep"] = step_obj
payload = {"step": json.dumps(step_obj, ensure_ascii=False)}
http_patch_json_path(ctx, state, f"/api/task/task-action/{action_id}", payload, stage + "/update-action-step")
def chat_number(value: Any, default: int = 10) -> int:
number = as_int(value)
if number is not None and number > 0:
return number
match = re.search(r"\d+", str(value or ""))
return int(match.group(0)) if match else default
def strategy_platform(value: Any) -> Any:
text = str(value or "").strip().lower()
if not text:
return 1
if "fb" in text or "facebook" in text:
return 1
if text in {"x", "twitter"} or "twitter" in text:
return 3
if "instagram" in text or text in {"ig", "in"}:
return 2
if "reddit" in text or text.startswith("re"):
return 14
if "threads" in text or text.startswith("th"):
return 15
return platform_to_number(value) or 1
def choice_tokens(value: Any) -> list[Any]:
"""Split frontend choice values from arrays, JSON strings, and mixed delimiters."""
if value is None:
return []
if isinstance(value, dict):
return choice_tokens(
first_present(value.get("value"), value.get("dictValue"), value.get("label"), value.get("dictLabel"))
)
if isinstance(value, (list, tuple, set)):
tokens: list[Any] = []
for item in value:
tokens.extend(choice_tokens(item))
return tokens
text = str(value).strip()
if not text:
return []
if text[0] in "[{":
try:
parsed = json.loads(text)
if parsed != value:
return choice_tokens(parsed)
except Exception:
pass
return [part.strip() for part in re.split(r"[,,、;/|]+", text) if part.strip()]
def dict_choices(
ctx: FlowContext,
state: dict[str, Any],
dict_type: str,
stage: str,
) -> list[dict[str, Any]]:
"""Fetch and cache frontend dictionary choices such as modelType."""
cache = state.setdefault("_dictChoiceCache", {})
if dict_type in cache:
cached = cache[dict_type]
return cached if isinstance(cached, list) else []
if ctx.dry_run:
cache[dict_type] = []
return []
query = urllib.parse.urlencode({"filter": f"dictType||$eq||{dict_type}", "limit": 999, "page": 1})
url = f"{url_for(ctx, 'dict_data')}?{query}"
response, _ = http_request_json_url(ctx, state, "GET", url, None, stage, endpoint_key="dict_data")
choices: list[dict[str, Any]] = []
for row in response_rows(response):
if not isinstance(row, dict):
continue
label = str(first_present(row.get("dictLabel"), row.get("label"), "") or "").strip()
raw_value = first_present(row.get("dictValue"), row.get("value"))
if not label or raw_value is None:
continue
number = as_int(raw_value)
choices.append({"label": label, "value": number if number is not None else str(raw_value).strip()})
for alias in choice_tokens(row.get("alias")):
alias_label = str(alias).strip()
if alias_label:
choices.append({"label": alias_label, "value": number if number is not None else str(raw_value).strip()})
cache[dict_type] = choices
return choices
def choice_value_by_label(choices: list[dict[str, Any]] | None, label: str) -> Any | None:
needle = re.sub(r"\s+", "", str(label or "").lower())
if not needle:
return None
for item in choices or []:
item_label = re.sub(r"\s+", "", str(item.get("label") or "").lower())
if item_label and (needle == item_label or needle in item_label or item_label in needle):
return item.get("value")
return None
def model_type_token_value(token: Any, choices: list[dict[str, Any]] | None = None) -> Any:
"""Convert modal labels like 图片/视频 into frontend dictionary values."""
number = as_int(token)
if number is not None:
return number
text = str(token or "").strip()
lower = text.lower()
exact = choice_value_by_label(choices, text)
if exact is not None:
return exact
label = ""
fallback_value = 2
if "视" in text or "video" in lower:
label, fallback_value = "视频", 3
elif "图" in text or "image" in lower or "picture" in lower or "photo" in lower:
label, fallback_value = "图片", 2
elif "音" in text or "audio" in lower or "sound" in lower:
label, fallback_value = "音频", 4
elif "发文" in text or "文本" in text or "文字" in text or "text" in lower:
label, fallback_value = "文本", 1
if label:
matched = choice_value_by_label(choices, label)
if matched is not None:
return matched
return fallback_value
# Mirrors the frontend helper's unknown-token default.
return 2
def strategy_model_type(value: Any, choices: list[dict[str, Any]] | None = None) -> str:
"""Return the comma-separated value string expected by task-action-message."""
tokens = choice_tokens(value)
if not tokens:
# Frontend save paths fall back to "1" when no modal type is selected.
return "1"
values: list[str] = []
seen: set[str] = set()
for token in tokens:
normalized = str(model_type_token_value(token, choices)).strip()
if normalized and normalized not in seen:
values.append(normalized)
seen.add(normalized)
return ",".join(values) or "1"
def strategy_language(value: Any) -> int:
# Frontend languageTypeChange falls back to 3.
return as_int(value) or 3
STRATEGY_ACCOUNT_TYPE_AUDIENCE = "目标受众账号"
STRATEGY_ACCOUNT_TYPE_PATHWAY = "关键传播节点账号"
def strategy_account_type(*, audience: bool) -> str:
"""Return the fixed account type for the current strategy step."""
return STRATEGY_ACCOUNT_TYPE_AUDIENCE if audience else STRATEGY_ACCOUNT_TYPE_PATHWAY
def save_strategy_details(
ctx: FlowContext,
state: dict[str, Any],
strategy_id: int,
item: dict[str, Any],
strategy: dict[str, Any],
topic: str,
stage: str,
*,
audience: bool,
) -> None:
"""Save all frontend-readable strategy child rows.
The strategy table and arrangement page do not read only the main strategy
row; they also expect channel, narrate, message, and relation rows.
"""
platform = strategy_platform(
first_present(strategy.get("platform"), state.get("platform"), platform_label(item.get("account") or {}))
)
account_type = strategy_account_type(audience=audience)
persona = strategy.get("persona") or "专业议题分析者"
persona_size = chat_number(strategy.get("personaSize"), 3)
responding_count = chat_number(first_present(strategy.get("respondingCount"), strategy.get("数量")), 10)
title = str(strategy.get("title") or "").strip()
model_type_choices = dict_choices(ctx, state, "modelType", stage + "/dict-modelType")
channel_id, _ = save_with_id(
ctx,
state,
"save_channel",
{
"taskId": as_int(state["taskId"]) or state["taskId"],
"actionId": as_int(state["actionId"]) or state["actionId"],
"strategyId": strategy_id,
"platform": platform,
"accountType": account_type,
"personaJson": json.dumps([{"persona": persona, "size": persona_size}], ensure_ascii=False),
},
stage + "/save-channel",
)
narrate_id, _ = save_with_id(ctx, state, "save_narrate", {"narrate": title}, stage + "/save-narrate")
if channel_id is not None and narrate_id is not None:
save_with_id(
ctx,
state,
"save_message",
{
"taskId": as_int(state["taskId"]) or state["taskId"],
"actionId": as_int(state["actionId"]) or state["actionId"],
"strategyId": strategy_id,
"channelId": channel_id,
"modelType": strategy_model_type(
first_present(strategy.get("format"), strategy.get("信息形式"), strategy.get("modelType")),
model_type_choices,
),
"language": strategy_language(first_present(strategy.get("language"), strategy.get("语种"))),
"respondingCount": responding_count,
"messageCount": responding_count,
"respondingTopic": narrate_id,
},
stage + "/save-message",
)
# Strategy ownership must never be inferred from model text or mixed
# relations: Step5 writes only audience designId, Step6 writes only pathwayId.
if audience:
design_ids = [item.get("designId")]
pathway_ids: list[Any] = []
if not any(design_ids):
record_error(state, stage, "受众策略缺少 designId,无法写入受众策略关系", request=item)
else:
design_ids = []
pathway_ids = [first_present(item.get("pathwayAccountId"), item.get("pathwayId"))]
if not any(pathway_ids):
record_error(state, stage, "关键传播节点策略缺少 pathwayId,无法写入传播节点策略关系", request=item)
for design_id in [x for x in design_ids if x]:
save_with_id(ctx, state, "save_strategy_relation", {"designId": design_id, "strategyId": strategy_id}, stage + "/save-strategy-relation")
for pathway_id in [x for x in pathway_ids if x]:
save_with_id(ctx, state, "save_strategy_relation", {"pathwayId": pathway_id, "strategyId": strategy_id}, stage + "/save-strategy-relation")
if not ctx.dry_run:
http_get_json_endpoint(
ctx,
state,
f"/api/task/task-action-strategy/setTopicKeywords/{as_int(state['actionId']) or state['actionId']}",
stage + "/set-topic-keywords",
)
def object_for_audience(audience: dict[str, Any]) -> dict[str, Any]:
return {
"targetAudience": audience.get("targetAudience"),
"recReason": audience.get("recReason"),
"accounts": audience.get("accounts") or [],
}
def selected_accounts_for_item(item: dict[str, Any], *, audience: bool) -> list[Any]:
if audience:
accounts = item.get("accounts") or []
return accounts or [object_for_audience(item)]
return [item.get("account") or item]
def id_ref_key(value: Any) -> str:
number = as_int(value)
if number is not None:
return str(number)
return str(value or "").strip()
def audience_name_key(value: Any) -> str:
return re.sub(r"\s+", "", str(value or "")).lower()
def compact_pathway_ref(pathway: dict[str, Any]) -> dict[str, Any]:
return {
"pathwayAccountId": pathway.get("pathwayAccountId") or pathway.get("pathwayId"),
"accountId": pathway.get("accountId"),
"accountName": pathway.get("accountName"),
}
def add_unique_pathway(target: list[dict[str, Any]], pathway: dict[str, Any]) -> None:
key = id_ref_key(pathway.get("pathwayAccountId") or pathway.get("pathwayId") or pathway.get("accountId") or pathway.get("accountName"))
if not key:
return
seen = {
id_ref_key(item.get("pathwayAccountId") or item.get("pathwayId") or item.get("accountId") or item.get("accountName"))
for item in target
}
if key not in seen:
target.append(pathway)
def audience_pathway_pairs(state: dict[str, Any]) -> list[tuple[dict[str, Any], list[dict[str, Any]]]]:
"""Return only audiences that produced at least one saved key pathway."""
id_links: dict[str, list[dict[str, Any]]] = {}
name_links: dict[str, list[dict[str, Any]]] = {}
for pathway in state.get("topPathways") or []:
if not isinstance(pathway, dict):
continue
for design_id in pathway.get("designIds") or []:
key = id_ref_key(design_id)
if key:
id_links.setdefault(key, []).append(pathway)
for name in pathway.get("relatedAudiences") or []:
key = audience_name_key(name)
if key:
name_links.setdefault(key, []).append(pathway)
pairs: list[tuple[dict[str, Any], list[dict[str, Any]]]] = []
for audience_item in state.get("topAudiences") or []:
if not isinstance(audience_item, dict):
continue
linked: list[dict[str, Any]] = []
design_key = id_ref_key(audience_item.get("designId"))
for pathway in id_links.get(design_key, []):
add_unique_pathway(linked, pathway)
name_key = audience_name_key(audience_item.get("targetAudience"))
for pathway in name_links.get(name_key, []):
add_unique_pathway(linked, pathway)
if linked:
item = {**audience_item, "downstreamPathways": [compact_pathway_ref(pathway) for pathway in linked]}
pairs.append((item, linked))
return pairs
def audience_pathway_summary(state: dict[str, Any]) -> list[dict[str, Any]]:
summary: list[dict[str, Any]] = []
for audience_item, pathways in audience_pathway_pairs(state):
summary.append(
{
"designId": audience_item.get("designId"),
"targetAudience": audience_item.get("targetAudience"),
"pathwayCount": len(pathways),
"pathways": [compact_pathway_ref(pathway) for pathway in pathways],
}
)
return summary
def has_audiences_with_pathways(state: dict[str, Any]) -> bool:
return bool(audience_pathway_pairs(state))
def downstream_limit(ctx: FlowContext, *, audience: bool) -> int:
"""Limit expensive downstream analysis while keeping 3 upstream rows stored."""
raw = ctx.downstream_audiences if audience else ctx.downstream_pathways
return max(1, raw)
def downstream_items(ctx: FlowContext, state: dict[str, Any], stage: str, *, audience: bool) -> list[dict[str, Any]]:
"""Select the 1-2 best stored rows for vulnerability/strategy generation."""
source_key = "topAudiences" if audience else "topPathways"
id_key = "designId" if audience else "pathwayAccountId"
selected_key = "selectedDownstreamAudiences" if audience else "selectedDownstreamPathways"
all_items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)]
if audience:
state["audiencesWithPathways"] = audience_pathway_summary(state)
items = [item for item, _ in audience_pathway_pairs(state)]
if not items:
state[selected_key] = []
record_error(state, stage, "没有已分析出关键传播节点的受众,不能继续分析受众脆弱点或策略;请先重试关键传播节点分析")
return []
else:
items = all_items
limit = downstream_limit(ctx, audience=audience)
selected = items[:limit]
selected_rows: list[dict[str, Any]] = []
for item in selected:
row = {
"id": item.get(id_key),
"name": item.get("targetAudience") if audience else item.get("accountName"),
}
if audience:
row["pathwayCount"] = len(item.get("downstreamPathways") or [])
row["pathways"] = item.get("downstreamPathways") or []
selected_rows.append(row)
state[selected_key] = selected_rows
if selected:
record_event(
state,
stage,
f"下游分析选择 {len(selected)}/{len(items)} 个{'已产生传播节点的受众' if audience else '传播节点'}",
{"limit": limit, "selected": state[selected_key], "candidateCount": len(items), "totalStoredCount": len(all_items)},
)
return selected
def audience_option(item: dict[str, Any], index: int) -> dict[str, Any]:
return {
"序号": index,
"受众目标": item.get("targetAudience"),
"推荐理由": compact(item.get("recReason", ""), 300),
"账号引用": item.get("userIndexes") or [],
}
def audience_aliases(item: dict[str, Any], index: int) -> list[Any]:
return [index, item.get("targetAudience"), item.get("rank")]
def pathway_option(item: dict[str, Any], index: int) -> dict[str, Any]:
return {
"序号": index,
"账号名称": item.get("accountName") or account_name(item.get("account") or {}),
"账号ID": item.get("accountId") or account_id(item.get("account") or {}),
"关联受众": item.get("relatedAudiences") or [],
"推荐理由": compact(item.get("recReason", ""), 300),
}
def pathway_aliases(item: dict[str, Any], index: int) -> list[Any]:
account = item.get("account") or item
return [
index,
item.get("pathwayAccountId"),
item.get("pathwayId"),
item.get("accountId"),
item.get("accountName"),
account_id(account),
account_name(account),
]
def vuln_candidate_entries(
items: list[dict[str, Any]],
candidate_store: dict[str, Any],
*,
audience: bool,
) -> list[dict[str, Any]]:
entries: list[dict[str, Any]] = []
seq = 1
for item in items:
owner_id = item.get("designId") if audience else item.get("pathwayAccountId")
owner_name = item.get("targetAudience") if audience else item.get("accountName")
for index, vuln in enumerate(candidate_store.get(str(owner_id)) or [], start=1):
if not isinstance(vuln, dict):
continue
entries.append(
{
"seq": seq,
"ownerId": owner_id,
"ownerName": owner_name,
"localIndex": index,
"vuln": vuln,
}
)
seq += 1
return entries
def vuln_option(entry: dict[str, Any]) -> dict[str, Any]:
vuln = entry.get("vuln") or {}
return {
"序号": entry.get("seq"),
"对象ID": entry.get("ownerId"),
"对象名称": entry.get("ownerName"),
"本对象内序号": entry.get("localIndex"),
"脆弱点": vuln.get("vulnerabilities"),
"理由": compact(vuln.get("reason", ""), 260),
}
def vuln_aliases(entry: dict[str, Any], index: int) -> list[Any]:
vuln = entry.get("vuln") or {}
owner_id = entry.get("ownerId")
return [
index,
entry.get("seq"),
f"{owner_id}:{entry.get('localIndex')}",
vuln.get("vulnerabilities"),
]
def strategy_target_option(item: dict[str, Any], index: int, *, audience: bool) -> dict[str, Any]:
if audience:
return {
"序号": index,
"类型": "受众",
"对象ID": item.get("designId"),
"对象名称": item.get("targetAudience"),
"已入库脆弱点数量": len((item.get("_vulns") or [])),
"关联传播节点": item.get("downstreamPathways") or [],
}
return {
"序号": index,
"类型": "关键传播节点",
"对象ID": item.get("pathwayAccountId"),
"对象名称": item.get("accountName"),
"已入库脆弱点数量": len((item.get("_vulns") or [])),
"关联受众": item.get("relatedAudiences") or [],
}
def strategy_target_aliases(item: dict[str, Any], index: int, *, audience: bool) -> list[Any]:
return [
index,
item.get("designId") if audience else item.get("pathwayAccountId"),
item.get("targetAudience") if audience else item.get("accountName"),
item.get("accountId") if not audience else None,
]
def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None:
"""Analyze audiences, save top 3, and attach any referenced accounts."""
require_context(state, "taskId", "actionId", "mubiao")
stage = "step1-audience"
if state.get("topAudiences") and not ctx.dry_run:
record_event(state, stage, "topAudiences 已存在,跳过。如需重跑请清空 state 中该字段。")
return
candidates = [item for item in state.get("audienceCandidates") or [] if isinstance(item, dict)]
accounts = [item for item in state.get("audienceAccounts") or [] if isinstance(item, dict)]
if not candidates:
payload = {
"usermessage": "你是中国对台湾方向的认知域专家,你要围绕驱动目标传播行动中的叙事主题,请详细分析你在各社交平台的受众目标群体,可以从账号画像和账号近期发文等方面考虑。可以从受众目标的兴趣爱好、关注议题、职业定位、政治立场等方面划分。",
"otherinfo": {"方向": state.get("direction") or "U方向"},
"history": [],
"stream": True,
"model_name": "OpenAI3",
"temperature": 0.7,
"max_tokens": 20000,
"prompt_name": f"任务背景和目的:{state['mubiao']}",
}
text, objects, parsed = call_model_text(ctx, state, "aiuser", payload, stage + "/analyze")
state["audienceRawText"] = text
# The first model call may return raw text plus embedded account objects.
accounts = accounts_from_response(parsed, objects, text)
state["audienceAccounts"] = accounts
# The second model call converts the long audience analysis into numbered
# groups that can be saved as task-action-design rows.
group_payload = {
"mubiao": state["mubiao"],
"user_message": "分析其中关键信息。请基于上一轮受众分析结果,抽取受众目标群体。字段必须是:群体1、用户编号列表1、理由1、群体2、用户编号列表2、理由2 ...,最多输出5个,只返回JSON。用户编号列表中的编号/id必须对应本次入参 zhanghao 数组里的账号索引值;例如返回 0 或 <<0>> 表示 zhanghao[0],返回 1 或 <<1>> 表示 zhanghao[1]。不要生成不存在于 zhanghao 的编号。",
"zhanghao": accounts,
"history": [],
"stream": True,
"model_name": "OpenAI3",
"temperature": 0.7,
"max_tokens": 20000,
"prompt_name": state["mubiao"],
}
data, _, _ = call_model_json(ctx, state, "group", group_payload, stage + "/extract")
candidates = extract_audiences(data)
state["audienceCandidates"] = candidates
if not candidates:
record_error(state, stage, "没有生成可供选择的受众候选")
return
if ctx.require_user_assist and not has_selection(ctx.select_audience_refs):
emit_user_assist(
state,
stage,
selection_key="audience",
title="请选择要入库并继续分析的受众",
prompt=f"可多选,最多建议选择 {ctx.max_audiences} 个。回复序号即可,例如:1,2,3。",
options=[audience_option(item, index) for index, item in enumerate(candidates, start=1)],
command_hint=user_assist_command_hint(state, "step1", "select-audiences", "1,2,3"),
allow_multiple=True,
max_count=ctx.max_audiences,
)
return
selected_refs = ctx.select_audience_refs or []
top, missing = select_by_refs(candidates, selected_refs, audience_aliases, max_count=ctx.max_audiences)
clear_pending_assist(state)
if missing:
record_error(state, stage, f"用户选择的受众未匹配:{', '.join(missing)}")
if not top:
record_error(state, stage, "没有选中任何受众,无法继续")
return
saved: list[dict[str, Any]] = []
for item in top:
# Save the audience first; account rows and downstream analyses depend
# on the returned designId.
design_payload = {
"taskId": as_int(state["taskId"]) or state["taskId"],
"actionId": as_int(state["actionId"]) or state["actionId"],
"parentTargetAudience": "大模型生成",
"targetAudience": item["targetAudience"],
"recReason": item.get("recReason", ""),
}
design_id, resp = save_with_id(ctx, state, "save_design", design_payload, stage + "/save-design")
if design_id is None:
continue
item = {**item, "designId": design_id, "saveResponse": resp, "accounts": [], "savedAccountIds": []}
for ref in item.get("userIndexes") or []:
# Model references are intentionally fuzzy, so resolve them against
# both explicit indexes and account names/ids.
account = resolve_account_ref(ref, accounts)
if account is None:
record_error(state, stage + "/save-design-account", f"用户编号 {ref} 未匹配到候选账号")
continue
aid = account_id(account)
name = account_name(account)
if not aid or not name:
record_error(state, stage + "/save-design-account", "账号缺少 accountId 或 accountName,跳过", request=account)
continue
account_payload = {
"taskId": as_int(state["taskId"]) or state["taskId"],
"actionId": as_int(state["actionId"]) or state["actionId"],
"accountId": aid,
"accountName": name,
"accountPlat": platform_to_number(account_platform_raw(account)),
"accountSource": first_present(account.get("dataSource"), account.get("accountSource"), "target"),
"frofileInfo": profile_info(account),
"designId": design_id,
}
saved_account_id, _ = save_with_id(ctx, state, "save_design_account", account_payload, stage + "/save-design-account")
item["accounts"].append(account)
if saved_account_id is not None:
item["savedAccountIds"].append(saved_account_id)
saved.append(item)
state["topAudiences"] = saved
if saved:
mark_step_status(ctx, state, ["step-analysis-audience", "step-analysis-keypoint"], stage + "/step-status")
record_event(state, stage, f"完成,保存受众 {len(saved)} 个")
def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None:
"""Analyze key pathway accounts for saved audiences and store top 3."""
require_context(state, "taskId", "actionId", "mubiao")
stage = "step2-pathway"
audiences = [a for a in state.get("topAudiences") or [] if a.get("designId")]
if not audiences:
record_error(state, stage, "没有已入库受众,无法分析传播节点")
return
if state.get("topPathways") and not ctx.dry_run:
state["audiencesWithPathways"] = audience_pathway_summary(state)
record_event(state, stage, "topPathways 已存在,跳过。如需重跑请清空 state 中该字段。")
return
ranked = [item for item in state.get("pathwayCandidates") or [] if isinstance(item, dict)]
if not ranked:
all_candidates: list[dict[str, Any]] = []
for audience in audiences:
# Analyze pathway candidates in the context of each saved audience.
payload = {
"usermessage": "你是中国对台湾方向的认知域专家,你要围绕驱动目标传播行动中的叙事主题,请分析哪些社交账号是你本次任务的核心传播节点,通过撬动该账号可实现扩大传播影响力的目的。可以从账号粉丝量、发文内容互动量、所在领域的影响力、账号画像与你传播议题的贴合度等角度分析。",
"otherinfo": {
"方向": state.get("direction") or "U方向",
"任务": "传播",
"平台": state.get("platform") or "fb",
"受众": {"名称": audience.get("targetAudience"), "理由": audience.get("recReason")},
},
"history": [],
"stream": True,
"model_name": "OpenAI3",
"temperature": 0.7,
"max_tokens": 20000,
"prompt_name": f"任务背景和目的:{state['mubiao']}",
}
text, objects, parsed = call_model_text(ctx, state, "aiuser", payload, stage + "/analyze")
accounts = accounts_from_response(parsed, objects, text)
# Ask the model to point back to candidate accounts, then resolve those
# references to concrete account objects.
spread_payload = {
"mubiao": state["mubiao"],
"user_message": "分析其中关键信息。请从候选账号中抽取核心传播节点,字段必须是:账号1、理由1、账号2、理由2 ...,最多输出5个。账号字段优先返回候选数组的引用编号或索引,只返回JSON。",
"zhanghao": accounts,
"history": [],
"stream": True,
"model_name": "liyou",
"temperature": 0.7,
"max_tokens": 1000,
"prompt_name": f"{audience.get('targetAudience')}:{audience.get('recReason')};",
}
data, _, _ = call_model_json(ctx, state, "spread", spread_payload, stage + "/extract")
refs = extract_pathway_refs(data)
for ref_item in refs:
account = resolve_account_ref(ref_item.get("ref"), accounts)
if account is None:
record_error(state, stage, f"传播节点引用 {ref_item.get('ref')} 未匹配账号", response=data)
continue
all_candidates.append(
{
"account": account,
"accountId": account_id(account),
"accountName": account_name(account),
"recReason": ref_item.get("recReason", ""),
"rank": ref_item.get("rank"),
"designIds": [audience["designId"]],
"relatedAudiences": [audience.get("targetAudience", "")],
}
)
merged = merge_pathway_candidates(all_candidates)
ranked = sorted(merged, key=pathway_score, reverse=True)
state["pathwayCandidates"] = ranked
if not ranked:
record_error(state, stage, "没有生成可供选择的关键传播节点候选")
return
if ctx.require_user_assist and not has_selection(ctx.select_pathway_refs):
emit_user_assist(
state,
stage,
selection_key="pathway",
title="请选择要入库并继续分析的关键传播节点",
prompt=f"可多选,最多建议选择 {ctx.max_pathways} 个。回复序号、账号名称或账号ID均可,例如:1,2,3。",
options=[pathway_option(item, index) for index, item in enumerate(ranked, start=1)],
command_hint=user_assist_command_hint(state, "step2", "select-pathways", "1,2,3"),
allow_multiple=True,
max_count=ctx.max_pathways,
)
return
top, missing = select_by_refs(ranked, ctx.select_pathway_refs or [], pathway_aliases, max_count=ctx.max_pathways)
clear_pending_assist(state)
if missing:
record_error(state, stage, f"用户选择的关键传播节点未匹配:{', '.join(missing)}")
if not top:
record_error(state, stage, "没有选中任何关键传播节点,无法继续")
return
saved: list[dict[str, Any]] = []
for item in top:
# Save the account row before writing the design-pathway relation.
account = item["account"]
aid = account_id(account)
name = account_name(account)
if not aid or not name:
record_error(state, stage + "/save-pathway", "传播节点账号缺少 id 或名称,跳过", request=account)
continue
pathway_payload = {
"taskId": as_int(state["taskId"]) or state["taskId"],
"actionId": as_int(state["actionId"]) or state["actionId"],
"accountName": name,
"accountId": aid,
"accountSource": first_present(account.get("dataSource"), account.get("accountSource"), "target"),
"fansCount": fans_count(account),
"platform": platform_to_number(account_platform_raw(account)),
"recReason": item.get("recReason", ""),
"issue": account_issue(account),
"frofileInfo": profile_info(account),
}
pathway_id, resp = save_with_id(ctx, state, "save_pathway", pathway_payload, stage + "/save-pathway")
if pathway_id is None:
continue
item = {
**item,
"pathwayAccountId": pathway_id,
"pathwayId": pathway_id,
"platform": platform_label(account),
"saveResponse": resp,
}
relation_payload = {
"actionId": as_int(state["actionId"]) or state["actionId"],
"pathwayId": pathway_id,
"designIds": item.get("designIds") or [],
}
http_post_json(ctx, state, "save_relation", relation_payload, stage + "/save-relation")
saved.append(item)
state["topPathways"] = saved
state["audiencesWithPathways"] = audience_pathway_summary(state)
if saved:
mark_step_status(ctx, state, ["step-analysis-keypoint", "step-analysis-weakpoint"], stage + "/step-status")
record_event(state, stage, f"完成,保存传播节点 {len(saved)} 个,可继续下游分析的受众 {len(state.get('audiencesWithPathways') or [])} 个")
def run_step3(ctx: FlowContext, state: dict[str, Any]) -> None:
run_vuln_step(ctx, state, audience=True)
def run_step4(ctx: FlowContext, state: dict[str, Any]) -> None:
run_vuln_step(ctx, state, audience=False)
def run_vuln_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool) -> None:
"""Generate and save weak points for selected audiences or pathway accounts."""
require_context(state, "taskId", "actionId", "mubiao")
stage = "step3-audience-vulns" if audience else "step4-pathway-vulns"
items = downstream_items(ctx, state, stage, audience=audience)
output_key = "audienceVulns" if audience else "pathwayVulns"
candidate_key = "audienceVulnCandidates" if audience else "pathwayVulnCandidates"
select_refs = ctx.select_audience_vuln_refs if audience else ctx.select_pathway_vuln_refs
state.setdefault(output_key, {})
candidate_store = state.setdefault(candidate_key, {})
for item in items:
# owner_id is designId for audiences and pathwayAccountId for accounts.
owner_id = item.get("designId") if audience else item.get("pathwayAccountId")
if not owner_id:
continue
if str(owner_id) in state.get(output_key, {}) and state[output_key][str(owner_id)] and not ctx.dry_run:
continue
if str(owner_id) in candidate_store and candidate_store[str(owner_id)]:
continue
zhanghao = selected_accounts_for_item(item, audience=audience)
prompt = (
"你是中国对台湾方向的认知战专家,你要传播行动要求的叙事主题。请基于该受众目标的画像、关注议题、社会分工定位、利益诉求与上一轮账号佐证,分析该受众目标有哪些脆弱点可以成为后续策略设计的关键要素。请输出理由、脆弱点1、理由1、脆弱点2、理由2、脆弱点3、理由3,只返回JSON。"
if audience
else "你是中国对台湾方向的认知战专家,你要传播行动要求的叙事主题,经分析你发现该账号是你本次传播任务中最重要的传播节点,请基于该账号画像、发文历史、社会分工定位、面临的利益诉求等角度,分析该账号有哪些脆弱点可以成为你撬动目标的关键要素,例如该账号比较脆弱的利益或内心诉求等。请输出理由、脆弱点1、理由1、脆弱点2、理由2、脆弱点3、理由3,只返回JSON。"
)
payload = {
"user_message": prompt,
"mubiao": state["mubiao"],
"zhanghao": zhanghao,
"history": [],
"stream": True,
"model_name": "OpenAI3",
"temperature": 0.7,
"max_tokens": 1000,
"prompt_name": item.get("targetAudience") if audience else "大V",
}
data, _, _ = call_model_json(ctx, state, "cuiruo", payload, stage + "/extract")
vulns = extract_vulns(data, ctx.max_vulns)
candidate_store[str(owner_id)] = vulns
entries = vuln_candidate_entries(items, candidate_store, audience=audience)
if not entries:
record_error(state, stage, "没有生成可供选择的脆弱点候选")
return
if ctx.require_user_assist and not has_selection(select_refs):
flag = "select-audience-vulns" if audience else "select-pathway-vulns"
emit_user_assist(
state,
stage,
selection_key="audience_vuln" if audience else "pathway_vuln",
title="请选择要入库的受众脆弱点" if audience else "请选择要入库的传播节点脆弱点",
prompt="可多选。回复全局序号即可,例如:1,2,4;也可用 对象ID:本对象内序号。",
options=[vuln_option(entry) for entry in entries],
command_hint=user_assist_command_hint(state, "step3" if audience else "step4", flag, "1,2,4"),
allow_multiple=True,
)
return
selected_entries, missing = select_by_refs(entries, select_refs or [], vuln_aliases)
clear_pending_assist(state)
if missing:
record_error(state, stage, f"用户选择的脆弱点未匹配:{', '.join(missing)}")
selected_by_owner: dict[str, list[dict[str, Any]]] = {}
for entry in selected_entries:
owner_key = str(entry.get("ownerId"))
selected_by_owner.setdefault(owner_key, []).append(entry.get("vuln") or {})
if not selected_by_owner:
record_error(state, stage, "没有选中任何脆弱点,无法继续")
return
for item in items:
owner_id = item.get("designId") if audience else item.get("pathwayAccountId")
if not owner_id:
continue
if str(owner_id) in state.get(output_key, {}) and state[output_key][str(owner_id)] and not ctx.dry_run:
continue
saved: list[dict[str, Any]] = []
for vuln in selected_by_owner.get(str(owner_id), []):
# The two weak-point tables have nearly identical payloads but
# different foreign-key names.
if audience:
save_payload = {
"taskId": as_int(state["taskId"]) or state["taskId"],
"actionId": as_int(state["actionId"]) or state["actionId"],
"actionDesignId": owner_id,
"vulnerabilities": vuln["vulnerabilities"],
"recScore": 100,
}
endpoint = "save_design_vuln"
else:
save_payload = {
"taskId": as_int(state["taskId"]) or state["taskId"],
"actionId": as_int(state["actionId"]) or state["actionId"],
"pathwayAccountId": owner_id,
"vulnerabilities": vuln["vulnerabilities"],
"recScore": 100,
}
endpoint = "save_pathway_vuln"
vuln_id, resp = save_with_id(ctx, state, endpoint, save_payload, stage + "/save")
if vuln_id is not None:
saved.append({**vuln, "id": vuln_id, "saveResponse": resp})
state[output_key][str(owner_id)] = saved
if any(state.get(output_key, {}).values()):
mark_step_status(ctx, state, ["step-analysis-weakpoint"], stage + "/step-status")
record_event(state, stage, "完成")
def extract_goal(mubiao: str) -> str:
match = re.search(r"实现[“\"](.+?)[”\"]作战目的", mubiao)
if match:
return match.group(1)
match = re.search(r"实现(.+?)(?:作战目的|目的)", mubiao)
if match:
return match.group(1).strip(":: ,,。")
return mubiao
def run_step5(ctx: FlowContext, state: dict[str, Any]) -> None:
run_strategy_step(ctx, state, audience=True)
def run_step6(ctx: FlowContext, state: dict[str, Any]) -> None:
run_strategy_step(ctx, state, audience=False)
def run_strategy_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool) -> None:
"""Generate strategies from saved weak points and persist frontend child rows."""
require_context(state, "taskId", "actionId", "mubiao")
stage = "step5-audience-strategy" if audience else "step6-pathway-strategy"
if audience and (has_selection(ctx.select_strategy_pathway_refs) or ctx.strategy_target == "pathway"):
clear_pending_assist(state)
record_error(state, stage, "策略生成目标类型冲突:Step5 只能选择受众,不能同时选择关键传播节点")
return
if not audience and (has_selection(ctx.select_strategy_audience_refs) or ctx.strategy_target == "audience"):
clear_pending_assist(state)
record_error(state, stage, "策略生成目标类型冲突:Step6 只能选择关键传播节点,不能同时选择受众")
return
items = downstream_items(ctx, state, stage, audience=audience)
vulns_by_owner = state.get("audienceVulns" if audience else "pathwayVulns") or {}
selectable_items: list[dict[str, Any]] = []
for item in items:
owner_id = item.get("designId") if audience else item.get("pathwayAccountId")
vulns = vulns_by_owner.get(str(owner_id)) or []
if vulns:
selectable_items.append({**item, "_vulns": vulns})
select_refs = ctx.select_strategy_audience_refs if audience else ctx.select_strategy_pathway_refs
if selectable_items and ctx.require_user_assist and not has_selection(select_refs):
flag = "select-strategy-audiences" if audience else "select-strategy-pathways"
emit_user_assist(
state,
stage,
selection_key="strategy_audience" if audience else "strategy_pathway",
title="请选择要生成策略的受众" if audience else "请选择要生成策略的关键传播节点",
prompt=(
"可多选,但本步骤只能选择受众,不能混选关键传播节点。回复序号即可,例如:1,2。"
if audience
else "可多选,但本步骤只能选择关键传播节点,不能混选受众。回复序号即可,例如:1,2。"
),
options=[strategy_target_option(item, index, audience=audience) for index, item in enumerate(selectable_items, start=1)],
command_hint=user_assist_command_hint(state, "step5" if audience else "step6", flag, "1,2"),
allow_multiple=True,
exclusive_note="受众策略和关键传播节点策略不能在同一步命令中交叉选择。",
)
return
selected_items, missing = select_by_refs(
selectable_items,
select_refs or [],
lambda item, index: strategy_target_aliases(item, index, audience=audience),
)
clear_pending_assist(state)
if missing:
record_error(state, stage, f"用户选择的策略生成对象未匹配:{', '.join(missing)}")
items = selected_items
if not items:
record_error(state, stage, "没有选中任何具备脆弱点的策略生成对象,无法继续")
return
output_key = "audienceStrategies" if audience else "pathwayStrategies"
state.setdefault(output_key, [])
existing_topics = {s.get("topic") for s in state[output_key] if isinstance(s, dict)}
for item in items:
# Strategies are only meaningful after the matching weak points exist.
owner_id = item.get("designId") if audience else item.get("pathwayAccountId")
vulns = vulns_by_owner.get(str(owner_id)) or []
vuln_texts = [v.get("vulnerabilities") for v in vulns if v.get("vulnerabilities")]
if not vuln_texts:
record_error(state, stage, "没有可用脆弱点,跳过策略生成", request=item)
continue
target_names = [item.get("targetAudience")] if audience else (item.get("relatedAudiences") or [item.get("accountName")])
zhanghao = selected_accounts_for_item(item, audience=audience)
first_account = zhanghao[0] if zhanghao and isinstance(zhanghao[0], dict) else {}
pingtai = state.get("platform") or platform_label(first_account)
payload = {
"user_message": DEFAULT_STRATEGY_PROMPT_AUDIENCE if audience else DEFAULT_STRATEGY_PROMPT_PATHWAY,
"cuiruodian": vuln_texts,
"pingtai": pingtai or "facebook",
"shouzhong": [name for name in target_names if name],
"mudi": extract_goal(state["mubiao"]),
"heiliao": [],
"zhanghao": zhanghao,
"history": [],
"stream": True,
"model_name": "OpenAI3",
"temperature": 0.7,
"max_tokens": 1000,
"prompt_name": "default",
}
data, _, _ = call_model_json(ctx, state, "strategy_model", payload, stage + "/extract")
strategies = extract_strategies(data, ctx.max_strategies)
for strategy in strategies:
target = (target_names[0] if target_names else "") or (item.get("accountName") if not audience else "")
account_type = strategy_account_type(audience=audience)
returned_account_type = str(first_present(strategy.get("accountType"), item.get("accountType"), item.get("type"), "") or "").strip()
if returned_account_type and returned_account_type != account_type:
record_event(
state,
stage,
f"模型返回账号类型“{returned_account_type}”与当前步骤不一致,已强制按“{account_type}”入库",
{"strategy": strategy, "item": item},
)
strategy_for_save = {**strategy, "accountType": account_type}
topic = f"利用\"{strategy.get('persona') or '专业议题分析者'}\"人设,传播\"{strategy.get('title')}\"主题"
if not strategy.get("title"):
record_error(state, stage, "策略缺少信息标题,跳过", request=strategy, response=data)
continue
if topic in existing_topics:
continue
# Save the main row first; channel/narrate/message/relation rows all
# require the returned strategy id.
save_payload = {
"taskId": as_int(state["taskId"]) or state["taskId"],
"actionId": as_int(state["actionId"]) or state["actionId"],
"targetAudience": target,
"accountType": account_type,
"topic": topic,
"strategyTitle": str(strategy.get("title") or "").strip(),
}
strategy_id, resp = save_with_id(ctx, state, "save_strategy", save_payload, stage + "/save")
if strategy_id is not None:
# Frontend strategy pages read these child tables, so a strategy
# is not complete until this helper succeeds as far as possible.
save_strategy_details(ctx, state, strategy_id, item, strategy_for_save, topic, stage, audience=audience)
existing_topics.add(topic)
state[output_key].append({**strategy_for_save, "id": strategy_id, "targetAudience": target, "topic": topic, "saveResponse": resp})
if state[output_key]:
mark_step_status(ctx, state, ["step-analysis-strategy", "step-analysis-strategy-gantt"], stage + "/step-status")
record_event(state, stage, "完成")
STEP_FUNCS = {
"step1": run_step1,
"step2": run_step2,
"step3": run_step3,
"step4": run_step4,
"step5": run_step5,
"step6": run_step6,
}
STEP_LABELS = {
"step1": "受众目标分析与入库",
"step2": "关键传播节点分析与入库",
"step3": "受众脆弱点分析与入库",
"step4": "传播节点脆弱点分析与入库",
"step5": "受众策略分析与入库",
"step6": "传播节点策略分析与入库",
}
def state_counts(state: dict[str, Any]) -> dict[str, int]:
"""Summarize stored objects for step feedback."""
return {
"受众数量": len(state.get("topAudiences") or []),
"传播节点数量": len(state.get("topPathways") or []),
"受众脆弱点数量": sum(len(v) for v in (state.get("audienceVulns") or {}).values()),
"传播节点脆弱点数量": sum(len(v) for v in (state.get("pathwayVulns") or {}).values()),
"受众策略数量": len(state.get("audienceStrategies") or []),
"传播节点策略数量": len(state.get("pathwayStrategies") or []),
}
def emit_step_feedback(
state: dict[str, Any],
step: str,
before_error_count: int,
*,
status: str | None = None,
note: str = "",
) -> None:
# One feedback line per step keeps the single-skill and split-skill versions
# equally transparent to the user.
total_errors = len(state.get("errors") or [])
new_errors = max(0, total_errors - before_error_count)
final_status = status or ("waiting_user" if has_pending_assist(state) else ("partial" if new_errors else "ok"))
final_note = note or (state.get("_pendingUserAssist", {}).get("title", "") if has_pending_assist(state) else "")
payload = {
"日志类型": "步骤反馈",
"步骤": STEP_NUMBER_LABELS.get(step, step),
"步骤名称": STEP_LABELS.get(step, step),
"状态": STATUS_LABELS.get(final_status, final_status),
"执行状态": status_label(final_status),
"备注": final_note,
"本步骤新增错误数": new_errors,
"累计错误数": total_errors,
"累计入库数量": state_counts(state),
"状态文件": state.get("_statePath", ""),
"摘要文件": state.get("_summaryPath", ""),
"更新时间": state.get("updatedAt"),
}
print(LOG_PREFIXES["step"] + " " + json.dumps(payload, ensure_ascii=False), flush=True)
def finish_step(
ctx: FlowContext,
state: dict[str, Any],
step: str,
before_error_count: int,
*,
status: str | None = None,
note: str = "",
) -> None:
# Always persist before emitting step feedback so the user can inspect files
# immediately after a reported step completes.
state["updatedAt"] = utc_now()
save_json_file(ctx.state_path, state)
if ctx.summary_path:
write_summary(ctx.summary_path, state)
emit_step_feedback(state, step, before_error_count, status=status, note=note)
def run_step_with_feedback(ctx: FlowContext, state: dict[str, Any], step: str) -> None:
before_errors = len(state.get("errors") or [])
STEP_FUNCS[step](ctx, state)
finish_step(ctx, state, step, before_errors)
def skip_step_with_feedback(ctx: FlowContext, state: dict[str, Any], step: str, reason: str) -> None:
before_errors = len(state.get("errors") or [])
record_event(state, step, f"跳过:{reason}")
finish_step(ctx, state, step, before_errors, status="skipped", note=reason)
def run_full(ctx: FlowContext, state: dict[str, Any]) -> None:
"""Legacy full runner.
The wrapper script is preferred for real-time logs, but this remains useful
for dry-runs and unattended local troubleshooting.
"""
run_step_with_feedback(ctx, state, "step1")
if has_pending_assist(state):
return
run_step_with_feedback(ctx, state, "step2")
if has_pending_assist(state):
return
run_step_with_feedback(ctx, state, "step3")
if has_pending_assist(state):
return
if state.get("topPathways"):
run_step_with_feedback(ctx, state, "step4")
if has_pending_assist(state):
return
else:
skip_step_with_feedback(ctx, state, "step4", "没有已入库关键传播节点")
run_step_with_feedback(ctx, state, "step5")
if has_pending_assist(state):
return
if state.get("topPathways"):
run_step_with_feedback(ctx, state, "step6")
if has_pending_assist(state):
return
else:
skip_step_with_feedback(ctx, state, "step6", "没有已入库关键传播节点")
def write_summary(path: Path, state: dict[str, Any]) -> None:
"""Write a human-readable checkpoint beside the machine state JSON."""
audience_vuln_count = sum(len(v) for v in (state.get("audienceVulns") or {}).values())
pathway_vuln_count = sum(len(v) for v in (state.get("pathwayVulns") or {}).values())
lines = [
"# xdfx 分析入库执行摘要",
"",
f"- 更新时间:{state.get('updatedAt')}",
f"- taskId:{state.get('taskId')}",
f"- actionId:{state.get('actionId')}",
f"- 受众入库:{len(state.get('topAudiences') or [])}",
f"- 传播节点入库:{len(state.get('topPathways') or [])}",
f"- 下游受众选择:{len(state.get('selectedDownstreamAudiences') or [])}",
f"- 下游传播节点选择:{len(state.get('selectedDownstreamPathways') or [])}",
f"- 受众脆弱点入库:{audience_vuln_count}",
f"- 传播节点脆弱点入库:{pathway_vuln_count}",
f"- 受众策略入库:{len(state.get('audienceStrategies') or [])}",
f"- 传播节点策略入库:{len(state.get('pathwayStrategies') or [])}",
f"- 错误数:{len(state.get('errors') or [])}",
"",
"## 受众",
]
for item in state.get("topAudiences") or []:
lines.append(f"- {item.get('targetAudience')} (designId={item.get('designId')})")
lines += ["", "## 传播节点"]
for item in state.get("topPathways") or []:
lines.append(f"- {item.get('accountName')} (pathwayAccountId={item.get('pathwayAccountId')})")
if state.get("errors"):
lines += ["", "## 失败明细"]
for err in state["errors"]:
lines.append(f"- [{stage_label(err.get('stage'))}] {err.get('message')} 状态码={err.get('status')} 接口={err.get('url')}")
if err.get("response"):
lines.append(f" - 响应摘要:{err.get('response')}")
text = "\n".join(lines) + "\n"
try:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(text, encoding="utf-8")
except PermissionError:
fallback = Path.cwd() / path.name
fallback.write_text(text, encoding="utf-8")
def build_arg_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="运行 xdfx 分析与入库流程。")
parser.add_argument("--mode", choices=["full", *STEP_FUNCS.keys()], default="full")
parser.add_argument("--input", help="JSON 配置文件。常用字段:api_base、taskId、actionId、mubiao、direction、platform、headers。")
parser.add_argument("--state", default="xdfx-flow-state.json", help="用于断点续跑和步骤交接的状态 JSON 文件路径。")
parser.add_argument("--summary", default="xdfx-flow-summary.md", help="Markdown 执行摘要输出路径。")
parser.add_argument("--api-base")
parser.add_argument("--task-id")
parser.add_argument("--action-id")
parser.add_argument("--mubiao")
parser.add_argument("--direction", default=None)
parser.add_argument("--platform", default=None)
parser.add_argument("--header", action="append", default=[], help="附加非鉴权 HTTP 请求头,格式为 'Key: Value';Authorization 会被忽略。")
parser.add_argument("--login-username", default=os.getenv("XDFX_LOGIN_USERNAME", "admin"), help="xdfx_auth 启动阶段自动登录使用的用户名。")
parser.add_argument("--login-password", default=os.getenv("XDFX_LOGIN_PASSWORD", "ch@user"), help="登录接口 JSON 请求体中的 password。")
parser.add_argument("--login-path", default=os.getenv("XDFX_LOGIN_PATH", "/api/auth/login"), help="登录接口路径或完整 URL。")
parser.add_argument("--no-auto-login", action="store_true", help="关闭 xdfx_auth 自动登录和 401 刷新。")
parser.add_argument("--verify-ssl", action="store_true", help="校验 HTTPS 证书。默认不校验,以兼容内网/UAT 自签证书。")
parser.add_argument("--timeout", type=float, default=90)
parser.add_argument("--retries", type=int, default=2)
parser.add_argument("--dry-run", action="store_true")
parser.add_argument("--no-step-status", action="store_true", help="步骤完成后不更新 task/task-action.step。")
parser.add_argument("--model-retries", type=int, default=1, help="模型/流式接口首次空响应、超时或提示模型无响应时的重试次数。")
parser.add_argument("--max-audiences", type=int, default=3)
parser.add_argument("--max-pathways", type=int, default=3)
parser.add_argument("--max-vulns", type=int, default=3)
parser.add_argument("--max-strategies", type=int, default=3)
parser.add_argument("--downstream-audiences", type=int, default=2, help="继续进入脆弱点/策略步骤的已入库受众数量,默认 2。")
parser.add_argument("--downstream-pathways", type=int, default=2, help="继续进入脆弱点/策略步骤的已入库传播节点数量,默认 2。")
parser.add_argument("--no-user-assist", action="store_true", help="关闭每步用户协助选择,使用脚本默认排序自动继续。")
parser.add_argument("--select-audiences", default="", help="Step1 用户选择的受众序号/名称,逗号分隔,例如 1,2,3。")
parser.add_argument("--select-pathways", default="", help="Step2 用户选择的关键传播节点序号/账号名/账号ID,逗号分隔。")
parser.add_argument("--select-audience-vulns", default="", help="Step3 用户选择的受众脆弱点序号,逗号分隔。")
parser.add_argument("--select-pathway-vulns", default="", help="Step4 用户选择的传播节点脆弱点序号,逗号分隔。")
parser.add_argument("--strategy-target", choices=["", "audience", "pathway"], default="", help="策略生成目标类型;Step5 只能 audience,Step6 只能 pathway。")
parser.add_argument("--select-strategy-audiences", default="", help="Step5 用户选择的受众策略生成对象序号/受众ID/名称,逗号分隔。")
parser.add_argument("--select-strategy-pathways", default="", help="Step6 用户选择的传播节点策略生成对象序号/节点ID/账号名,逗号分隔。")
return parser
def main(argv: list[str] | None = None) -> int:
parser = build_arg_parser()
args = parser.parse_args(argv)
state_path = Path(args.state)
state = default_state()
# Load previous state first, then overlay input JSON and finally CLI flags.
# That ordering makes resume safe while still allowing explicit overrides.
if state_path.exists():
loaded = load_json_file(state_path)
if isinstance(loaded, dict):
state.update(loaded)
if args.input:
config = load_json_file(args.input)
if not isinstance(config, dict):
raise FlowError("--input 必须是 JSON 对象")
merge_config_into_state(state, config)
input_headers = config.get("headers")
if isinstance(input_headers, dict):
args.header.extend(f"{k}: {v}" for k, v in input_headers.items() if str(k).lower() != "authorization")
if config.get("login_username") is not None:
args.login_username = str(config.get("login_username") or "")
if config.get("login_password") is not None:
args.login_password = str(config.get("login_password") or "")
if config.get("login_path") is not None:
args.login_path = str(config.get("login_path") or "")
if config.get("auto_login") is False:
args.no_auto_login = True
if config.get("verify_ssl") is not None:
args.verify_ssl = bool(config.get("verify_ssl"))
if config.get("downstream_audiences") is not None:
args.downstream_audiences = as_int(config.get("downstream_audiences")) or args.downstream_audiences
if config.get("downstream_pathways") is not None:
args.downstream_pathways = as_int(config.get("downstream_pathways")) or args.downstream_pathways
if config.get("require_user_assist") is False or config.get("user_assist") is False:
args.no_user_assist = True
selection_map = {
"select_audiences": "select_audiences",
"select_pathways": "select_pathways",
"select_audience_vulns": "select_audience_vulns",
"select_pathway_vulns": "select_pathway_vulns",
"select_strategy_audiences": "select_strategy_audiences",
"select_strategy_pathways": "select_strategy_pathways",
"strategy_target": "strategy_target",
}
for config_key, arg_name in selection_map.items():
if config.get(config_key) is not None:
setattr(args, arg_name, config.get(config_key))
cli_config = {
"api_base": args.api_base,
"taskId": args.task_id,
"actionId": args.action_id,
"mubiao": args.mubiao,
"direction": args.direction,
"platform": args.platform,
}
merge_config_into_state(state, {k: v for k, v in cli_config.items() if v is not None})
api_base = state.get("api_base") or "https://ch1.b.uat.4.cn"
headers = parse_headers(args.header)
# ctx owns runtime knobs; state owns business data and resumable ids.
ctx = FlowContext(
api_base=api_base,
headers=headers,
timeout=args.timeout,
retries=max(args.retries, 0),
dry_run=args.dry_run,
state_path=state_path,
summary_path=Path(args.summary) if args.summary else None,
max_audiences=max(args.max_audiences, 1),
max_pathways=max(args.max_pathways, 1),
max_vulns=max(args.max_vulns, 1),
max_strategies=max(args.max_strategies, 1),
downstream_audiences=max(args.downstream_audiences, 1),
downstream_pathways=max(args.downstream_pathways, 1),
require_user_assist=not args.no_user_assist,
select_audience_refs=split_selection_refs(args.select_audiences),
select_pathway_refs=split_selection_refs(args.select_pathways),
select_audience_vuln_refs=split_selection_refs(args.select_audience_vulns),
select_pathway_vuln_refs=split_selection_refs(args.select_pathway_vulns),
select_strategy_audience_refs=split_selection_refs(args.select_strategy_audiences),
select_strategy_pathway_refs=split_selection_refs(args.select_strategy_pathways),
strategy_target=str(args.strategy_target or ""),
sync_step_status=not args.no_step_status,
model_retries=max(args.model_retries, 0),
verify_ssl=bool(args.verify_ssl),
auto_login=not args.no_auto_login,
login_username=args.login_username,
login_password=args.login_password,
login_path=args.login_path,
)
state["api_base"] = api_base
state["_statePath"] = str(state_path)
state["_summaryPath"] = str(ctx.summary_path) if ctx.summary_path else ""
state["_autoLogin"] = ctx.auto_login
state["_verifySsl"] = ctx.verify_ssl
state["_downstreamLimits"] = {
"audiences": ctx.downstream_audiences,
"pathways": ctx.downstream_pathways,
}
record_event(state, "runner", f"start mode={args.mode} dry_run={ctx.dry_run}")
try:
require_context(state, "taskId", "actionId", "mubiao")
if ctx.auto_login and not ctx.dry_run:
# Always acquire a token through the imported auth package before the first business call.
if not try_auto_login(ctx, state, "runner/initial-login", force=True):
raise FlowError("启动阶段获取 token 失败,已停止执行,避免无 token 调用业务接口")
if args.mode == "full":
run_full(ctx, state)
else:
run_step_with_feedback(ctx, state, args.mode)
except FlowError as exc:
record_error(state, "runner", str(exc), exception=exc)
save_json_file(state_path, state)
if ctx.summary_path:
write_summary(ctx.summary_path, state)
print(f"错误:{exc}", file=sys.stderr)
return 2
except Exception as exc: # noqa: BLE001
record_error(state, "runner", "未捕获异常", response=traceback.format_exc(), exception=exc)
save_json_file(state_path, state)
if ctx.summary_path:
write_summary(ctx.summary_path, state)
print("未捕获异常堆栈:", file=sys.stderr)
print(traceback.format_exc(), file=sys.stderr)
return 1
state["updatedAt"] = utc_now()
save_json_file(state_path, state)
if ctx.summary_path:
write_summary(ctx.summary_path, state)
# Final JSON is intentionally compact; detailed per-step status has already
# been emitted as Chinese step feedback.
final_payload = {
"日志类型": "执行完成",
"是否成功": "是",
"是否等待用户选择": yes_no(has_pending_assist(state)),
"待用户选择": state.get("_pendingUserAssist"),
"执行模式": mode_label(args.mode),
"状态文件": str(state_path),
"摘要文件": str(ctx.summary_path) if ctx.summary_path else "",
"受众数量": len(state.get("topAudiences") or []),
"传播节点数量": len(state.get("topPathways") or []),
"已选择受众数": len(state.get("selectedDownstreamAudiences") or []),
"已选择传播节点数": len(state.get("selectedDownstreamPathways") or []),
"受众策略数量": len(state.get("audienceStrategies") or []),
"传播节点策略数量": len(state.get("pathwayStrategies") or []),
"错误数": len(state.get("errors") or []),
}
print(LOG_PREFIXES["done"] + " " + json.dumps(final_payload, ensure_ascii=False))
return 0
if __name__ == "__main__":
raise SystemExit(main())