268 lines
11 KiB
Python
268 lines
11 KiB
Python
"""Conditional LLMWiki provider configuration.
|
|
|
|
DeerFlow owns only the connection to WeKnora. Database, Redis, document
|
|
parsing, storage and model configuration stay entirely in WeKnora's own
|
|
deployment files. An effective non-empty API URL is the sole provider switch.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from urllib.parse import urlsplit, urlunsplit
|
|
|
|
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
|
|
|
|
|
def normalize_service_url(value: str | None) -> str:
|
|
"""Normalize an optional HTTP(S) service URL and reject embedded secrets."""
|
|
|
|
raw = str(value or "").strip()
|
|
if not raw:
|
|
return ""
|
|
parsed = urlsplit(raw)
|
|
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
|
raise ValueError("WeKnora addresses must be absolute http(s) URLs")
|
|
if parsed.username is not None or parsed.password is not None:
|
|
raise ValueError("WeKnora addresses must not contain credentials")
|
|
if parsed.query or parsed.fragment:
|
|
raise ValueError("WeKnora addresses must not contain query strings or fragments")
|
|
path = parsed.path.rstrip("/")
|
|
return urlunsplit((parsed.scheme, parsed.netloc, path, "", ""))
|
|
|
|
|
|
class WeKnoraConnectionConfig(BaseModel):
|
|
"""The complete DeerFlow-owned connection surface for WeKnora."""
|
|
|
|
api_base_url: str = Field(default="", description="WeKnora app API base URL; non-empty enables WeKnora mode")
|
|
web_base_url: str = Field(default="", description="Optional internal WeKnora administration URL")
|
|
admin_email: str = Field(
|
|
default="",
|
|
description=("Optional WeKnora administrator email used by DeerFlow's server-side WeKnora API client and detail-page iframe proxy. Prefer $ENV references."),
|
|
)
|
|
admin_password: str = Field(
|
|
default="",
|
|
repr=False,
|
|
description=("Optional WeKnora administrator password used by DeerFlow's server-side WeKnora API client and detail-page iframe proxy. Prefer $ENV references."),
|
|
)
|
|
graph: WeKnoraDirectGraphConfig = Field(default_factory=lambda: WeKnoraDirectGraphConfig())
|
|
model_config = ConfigDict(extra="ignore")
|
|
|
|
@field_validator("api_base_url", "web_base_url")
|
|
@classmethod
|
|
def _validate_url(cls, value: str) -> str:
|
|
return normalize_service_url(value)
|
|
|
|
|
|
class WeKnoraDirectGraphConfig(BaseModel):
|
|
"""Backend-only, opt-in Neo4j transactional HTTP writer settings."""
|
|
|
|
direct_write_enabled: bool = False
|
|
neo4j_http_url: str = ""
|
|
username: str = ""
|
|
password: str = Field(default="", repr=False)
|
|
database: str = Field(default="neo4j", pattern=r"^[A-Za-z0-9_.-]+$")
|
|
timeout_seconds: float = Field(default=30, gt=0, le=300)
|
|
model_config = ConfigDict(extra="ignore")
|
|
|
|
@field_validator("neo4j_http_url")
|
|
@classmethod
|
|
def _validate_url(cls, value: str) -> str:
|
|
return normalize_service_url(value)
|
|
|
|
|
|
class LocalWikiEmbeddingConfig(BaseModel):
|
|
enabled: bool = True
|
|
provider: str = "openai"
|
|
model: str = "bge-embedding-m3"
|
|
base_url: str = ""
|
|
api_key: str = Field(default="", repr=False)
|
|
local_model_path: str = ""
|
|
local_cache_dir: str = ""
|
|
local_process_isolation: bool = True
|
|
local_threads: int = Field(default=2, ge=1, le=32)
|
|
dimensions: int | None = Field(
|
|
default=None,
|
|
ge=1,
|
|
le=65536,
|
|
description="Expected response vector dimensions; not sent to the embedding endpoint.",
|
|
)
|
|
batch_size: int = Field(default=32, ge=1, le=512)
|
|
timeout_seconds: float = Field(default=30, gt=0, le=300)
|
|
max_retries: int = Field(default=3, ge=0, le=10)
|
|
model_config = ConfigDict(extra="ignore")
|
|
|
|
@field_validator("provider")
|
|
@classmethod
|
|
def _validate_provider(cls, value: str) -> str:
|
|
normalized = value.strip().lower()
|
|
if normalized not in {"openai", "openai_compatible", "local"}:
|
|
raise ValueError("Wiki embedding provider must be openai, openai_compatible or local")
|
|
return normalized
|
|
|
|
@field_validator("base_url")
|
|
@classmethod
|
|
def _validate_url(cls, value: str) -> str:
|
|
return normalize_service_url(value)
|
|
|
|
|
|
class LocalWikiExternalApiConfig(BaseModel):
|
|
enabled: bool = False
|
|
api_key: str = Field(default="", repr=False)
|
|
previous_api_key: str = Field(default="", repr=False)
|
|
max_query_chars: int = Field(default=1000, ge=1, le=10000)
|
|
max_knowledge_bases: int = Field(default=20, ge=1, le=100)
|
|
max_results: int = Field(default=50, ge=1, le=100)
|
|
default_results: int = Field(default=10, ge=1, le=100)
|
|
include_content_max_chars: int = Field(default=12000, ge=0, le=1_000_000)
|
|
requests_per_minute: int = Field(default=60, ge=1, le=100000)
|
|
model_config = ConfigDict(extra="ignore")
|
|
|
|
@model_validator(mode="after")
|
|
def _validate_enabled_key(self):
|
|
if self.enabled and not self.api_key.strip():
|
|
raise ValueError("llmwiki.local_wiki_index.external_api.api_key is required when external API is enabled")
|
|
if self.default_results > self.max_results:
|
|
raise ValueError("external_api.default_results must not exceed max_results")
|
|
return self
|
|
|
|
|
|
class LocalWikiIndexConfig(BaseModel):
|
|
"""DeerFlow-owned mirror and exact-vector index for WeKnora Wiki pages."""
|
|
|
|
enabled: bool = False
|
|
frontend_base_url: str = Field(
|
|
default="http://127.0.0.1:5174",
|
|
description="DeerFlow frontend base URL used to build directly usable Wiki links in public vector-search results.",
|
|
)
|
|
auto_sync: bool = True
|
|
sync_interval_seconds: int = Field(default=30, ge=10, le=86400)
|
|
sync_page_size: int = Field(default=100, ge=1, le=500)
|
|
sync_concurrency: int = Field(default=3, ge=1, le=32)
|
|
sync_lease_seconds: int = Field(default=600, ge=30, le=86400)
|
|
sync_request_timeout_seconds: float = Field(default=30, gt=0, le=300)
|
|
sync_max_retries: int = Field(default=3, ge=0, le=10)
|
|
sync_retry_backoff_seconds: float = Field(default=5, ge=0, le=300)
|
|
chunk_max_chars: int = Field(default=1000, ge=100, le=20000)
|
|
chunk_overlap_chars: int = Field(default=120, ge=0, le=10000)
|
|
chunk_min_chars: int = Field(default=80, ge=1, le=10000)
|
|
title_prefix_enabled: bool = True
|
|
embedding: LocalWikiEmbeddingConfig = Field(default_factory=LocalWikiEmbeddingConfig)
|
|
engine: str = "numpy_exact"
|
|
strict_vector: bool = True
|
|
top_k_sections: int = Field(default=30, ge=1, le=1000)
|
|
top_k_pages: int = Field(default=8, ge=1, le=100)
|
|
max_sections_per_page: int = Field(default=2, ge=1, le=20)
|
|
min_similarity: float = Field(default=0.35, ge=-1, le=1)
|
|
keyword_boost_enabled: bool = False
|
|
keyword_boost_weight: float = Field(default=0.15, ge=0, le=1)
|
|
cache_max_knowledge_bases: int = Field(default=20, ge=1, le=1000)
|
|
cache_revision_check_seconds: float = Field(default=5, ge=0, le=3600)
|
|
database_batch_size: int = Field(default=50, ge=1, le=500)
|
|
external_api: LocalWikiExternalApiConfig = Field(default_factory=LocalWikiExternalApiConfig)
|
|
model_config = ConfigDict(extra="ignore")
|
|
|
|
@field_validator("frontend_base_url")
|
|
@classmethod
|
|
def _validate_frontend_base_url(cls, value: str) -> str:
|
|
normalized = normalize_service_url(value)
|
|
if not normalized:
|
|
raise ValueError("local Wiki index frontend_base_url is required")
|
|
return normalized
|
|
|
|
@model_validator(mode="before")
|
|
@classmethod
|
|
def _migrate_legacy_flags(cls, value):
|
|
if isinstance(value, dict) and "auto_sync" not in value and "sync_enabled" in value:
|
|
value = {**value, "auto_sync": value.get("sync_enabled")}
|
|
return value
|
|
|
|
@property
|
|
def sync_enabled(self) -> bool:
|
|
"""Backward-compatible status name; automatic work is one explicit switch."""
|
|
|
|
return self.auto_sync
|
|
|
|
@property
|
|
def internal_search_enabled(self) -> bool:
|
|
return self.enabled
|
|
|
|
@property
|
|
def reference_ui_enabled(self) -> bool:
|
|
return self.enabled
|
|
|
|
@model_validator(mode="after")
|
|
def _validate_index(self):
|
|
if self.chunk_overlap_chars >= self.chunk_max_chars:
|
|
raise ValueError("chunk_overlap_chars must be smaller than chunk_max_chars")
|
|
if self.chunk_min_chars > self.chunk_max_chars:
|
|
raise ValueError("chunk_min_chars must not exceed chunk_max_chars")
|
|
if self.top_k_pages > self.top_k_sections:
|
|
raise ValueError("top_k_pages must not exceed top_k_sections")
|
|
if self.engine != "numpy_exact":
|
|
raise ValueError("Only the numpy_exact local Wiki vector engine is currently supported")
|
|
if self.enabled and not self.embedding.enabled:
|
|
raise ValueError("local Wiki index embedding must be enabled when the index is enabled")
|
|
if self.enabled and ((not self.embedding.base_url and self.embedding.provider != "local") or not self.embedding.model.strip()):
|
|
raise ValueError("local Wiki index embedding base_url and model are required when enabled")
|
|
return self
|
|
|
|
|
|
class LlmWikiConfig(BaseModel):
|
|
"""Top-level ``config.yaml`` section for conditional WeKnora mode."""
|
|
|
|
weknora: WeKnoraConnectionConfig = Field(default_factory=WeKnoraConnectionConfig)
|
|
local_wiki_index: LocalWikiIndexConfig = Field(default_factory=LocalWikiIndexConfig)
|
|
model_config = ConfigDict(extra="ignore")
|
|
|
|
|
|
class LlmWikiRuntimeOverride(BaseModel):
|
|
"""Administrator-written URL override stored in system settings.
|
|
|
|
``enabled=False`` means "use config.yaml". When enabled, an empty API URL
|
|
deliberately forces legacy mode even if the source file has an address.
|
|
Secrets are never stored here.
|
|
"""
|
|
|
|
enabled: bool = False
|
|
api_base_url: str = ""
|
|
web_base_url: str = ""
|
|
model_config = ConfigDict(extra="ignore")
|
|
|
|
@field_validator("api_base_url", "web_base_url")
|
|
@classmethod
|
|
def _validate_url(cls, value: str) -> str:
|
|
return normalize_service_url(value)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ResolvedLlmWikiRuntime:
|
|
provider: str
|
|
api_base_url: str
|
|
web_base_url: str
|
|
admin_email: str
|
|
admin_password: str
|
|
override_enabled: bool
|
|
|
|
@property
|
|
def weknora_enabled(self) -> bool:
|
|
return self.provider == "weknora"
|
|
|
|
|
|
def resolve_llmwiki_runtime(
|
|
config: LlmWikiConfig,
|
|
override: LlmWikiRuntimeOverride | None = None,
|
|
) -> ResolvedLlmWikiRuntime:
|
|
"""Resolve the deterministic provider from file config plus URL override."""
|
|
|
|
use_override = bool(override and override.enabled)
|
|
api_base_url = override.api_base_url if use_override and override else config.weknora.api_base_url
|
|
web_base_url = override.web_base_url if use_override and override else config.weknora.web_base_url
|
|
return ResolvedLlmWikiRuntime(
|
|
provider="weknora" if api_base_url else "legacy",
|
|
api_base_url=api_base_url,
|
|
web_base_url=web_base_url,
|
|
admin_email=config.weknora.admin_email,
|
|
admin_password=config.weknora.admin_password,
|
|
override_enabled=use_override,
|
|
)
|