import asyncio import logging import mimetypes import time import zipfile from collections import OrderedDict from datetime import UTC, datetime from pathlib import Path from typing import Literal from urllib.parse import quote from fastapi import APIRouter, HTTPException, Query, Request from fastapi.responses import FileResponse, PlainTextResponse, Response from pydantic import BaseModel from app.gateway.authz import require_permission from app.gateway.path_utils import aresolve_thread_virtual_path from deerflow.config.paths import get_paths from deerflow.runtime.thread_paths import aresolve_path_user_id from deerflow.runtime.user_context import get_effective_user_id logger = logging.getLogger(__name__) router = APIRouter(prefix="/api", tags=["artifacts"]) ACTIVE_CONTENT_MIME_TYPES = { "text/html", "application/xhtml+xml", "image/svg+xml", } EDITABLE_TEXT_SUFFIXES = { ".md", ".markdown", ".txt", ".json", ".yaml", ".yml", ".csv", ".xml", ".log", } _VIRTUAL_USER_DATA_PREFIX = "mnt/user-data" ArtifactLibraryKind = Literal["all", "document", "spreadsheet", "text_data", "archive", "other"] ArtifactLibrarySource = Literal["all", "general_qa", "agent"] _MEDIA_MIME_PREFIXES = ("image/", "audio/", "video/") _MEDIA_SUFFIXES = { ".aac", ".aiff", ".ape", ".avi", ".bmp", ".flac", ".gif", ".heic", ".ico", ".jpeg", ".jpg", ".m4a", ".m4v", ".mkv", ".mov", ".mp3", ".mp4", ".ogg", ".png", ".svg", ".tiff", ".wav", ".webm", ".webp", ".wma", } _DOCUMENT_SUFFIXES = { ".doc", ".docx", ".epub", ".odt", ".pdf", ".ppt", ".pptx", ".rtf", } _SPREADSHEET_SUFFIXES = {".csv", ".ods", ".tsv", ".xls", ".xlsx"} _TEXT_DATA_SUFFIXES = { ".bash", ".c", ".conf", ".cpp", ".css", ".go", ".h", ".html", ".ini", ".java", ".js", ".json", ".jsonl", ".log", ".markdown", ".md", ".mmd", ".py", ".rs", ".sh", ".sql", ".toml", ".ts", ".tsx", ".txt", ".xml", ".yaml", ".yml", } _ARCHIVE_SUFFIXES = {".7z", ".gz", ".rar", ".skill", ".tar", ".tgz", ".zip"} _ARTIFACT_LIBRARY_CACHE_TTL_SECONDS = 15.0 _ARTIFACT_LIBRARY_CACHE_MAX_USERS = 128 _ARTIFACT_LIBRARY_SCAN_CONCURRENCY = 8 _ARTIFACT_LIBRARY_THREAD_BATCH_SIZE = 500 _artifact_library_cache: OrderedDict[str, tuple[float, list[dict]]] = OrderedDict() def coerce_artifact_virtual_path(path: str) -> str: """Normalize an artifacts URL path to a sandbox virtual path. Streaming write previews may request a bare filename such as ``未命名文件.md`` before the model has emitted a real ``/mnt/user-data/outputs/...`` path. Mapping a single path segment into the thread outputs directory turns that into a normal 404 (file not written yet) instead of HTTP 400 ``Path must start with /mnt/user-data``. """ stripped = path.strip().lstrip("/").replace("\\", "/") if not stripped: raise HTTPException(status_code=400, detail="Path must start with /mnt/user-data") if stripped == _VIRTUAL_USER_DATA_PREFIX or stripped.startswith(_VIRTUAL_USER_DATA_PREFIX + "/"): return stripped if "/" in stripped or stripped in {".", ".."}: raise HTTPException(status_code=400, detail="Path must start with /mnt/user-data") return f"{_VIRTUAL_USER_DATA_PREFIX}/outputs/{stripped}" class ArtifactUpdateRequest(BaseModel): content: str class ArtifactLibraryItem(BaseModel): thread_id: str thread_title: str agent_id: str | None = None agent_name: str | None = None thread_type: str | None = None source_type: Literal["general_qa", "agent"] path: str name: str size_bytes: int mime_type: str | None = None modified_at: str file_kind: str class ArtifactLibraryResponse(BaseModel): total: int offset: int limit: int items: list[ArtifactLibraryItem] def _build_content_disposition(disposition_type: str, filename: str) -> str: """Build an RFC 5987 encoded Content-Disposition header value.""" return f"{disposition_type}; filename*=UTF-8''{quote(filename)}" def _build_attachment_headers(filename: str, extra_headers: dict[str, str] | None = None) -> dict[str, str]: headers = {"Content-Disposition": _build_content_disposition("attachment", filename)} if extra_headers: headers.update(extra_headers) return headers def is_text_file_by_content(path: Path, sample_size: int = 8192) -> bool: """Check if file is text by examining content for null bytes.""" try: with open(path, "rb") as f: chunk = f.read(sample_size) # Text files shouldn't contain null bytes return b"\x00" not in chunk except Exception: return False def _extract_file_from_skill_archive(zip_path: Path, internal_path: str) -> bytes | None: """Extract a file from a .skill ZIP archive. Args: zip_path: Path to the .skill file (ZIP archive). internal_path: Path to the file inside the archive (e.g., "SKILL.md"). Returns: The file content as bytes, or None if not found. """ if not zipfile.is_zipfile(zip_path): return None try: with zipfile.ZipFile(zip_path, "r") as zip_ref: # List all files in the archive namelist = zip_ref.namelist() # Try direct path first if internal_path in namelist: return zip_ref.read(internal_path) # Try with any top-level directory prefix (e.g., "skill-name/SKILL.md") for name in namelist: if name.endswith("/" + internal_path) or name == internal_path: return zip_ref.read(name) # Not found return None except (zipfile.BadZipFile, KeyError): return None def collect_outputs_listing(outputs_dir: Path) -> list[dict]: """Walk an outputs directory and return artifact listing entries. Each entry has ``path`` (virtual ``mnt/user-data/outputs/...``), ``name``, ``size_bytes``, ``mime_type``, and ``modified_at`` (ISO 8601 UTC). Symlinks that escape the outputs root are dropped to keep the listing scoped to the thread. The result is sorted by ``modified_at`` descending (newest first). """ if not outputs_dir.exists() or not outputs_dir.is_dir(): return [] outputs_root = outputs_dir.resolve() files: list[dict] = [] for entry in outputs_root.rglob("*"): if not entry.is_file(): continue try: resolved = entry.resolve() resolved.relative_to(outputs_root) except (OSError, ValueError): continue try: relative = resolved.relative_to(outputs_root).as_posix() mime_type, _ = mimetypes.guess_type(resolved.name) stat = resolved.stat() except OSError: # A run may atomically replace or remove a file while this directory # is being scanned. Treat that transient entry as absent instead of # failing the complete thread/library listing. continue files.append( { "path": f"mnt/user-data/outputs/{relative}", "name": resolved.name, "size_bytes": stat.st_size, "mime_type": mime_type, "modified_at": datetime.fromtimestamp(stat.st_mtime, tz=UTC).isoformat(), } ) files.sort(key=lambda f: f["modified_at"], reverse=True) return files def artifact_library_kind(name: str) -> str: """Classify one non-media artifact for the library filters.""" suffix = Path(name).suffix.lower() if suffix in _DOCUMENT_SUFFIXES: return "document" if suffix in _SPREADSHEET_SUFFIXES: return "spreadsheet" if suffix in _TEXT_DATA_SUFFIXES: return "text_data" if suffix in _ARCHIVE_SUFFIXES: return "archive" return "other" def is_artifact_library_file(file: dict) -> bool: """Return whether a generated file belongs in the document-only library.""" mime_type = str(file.get("mime_type") or "").lower() if mime_type.startswith(_MEDIA_MIME_PREFIXES): return False return Path(str(file.get("name") or file.get("path") or "")).suffix.lower() not in _MEDIA_SUFFIXES async def _search_artifact_library_threads(thread_store) -> list[dict]: """Load every normal conversation owned by the current request user.""" rows: list[dict] = [] seen: set[str] = set() offset = 0 while True: batch = await thread_store.search( limit=_ARTIFACT_LIBRARY_THREAD_BATCH_SIZE, offset=offset, exclude_system=True, ) if not batch: break added = 0 for row in batch: thread_id = str(row.get("thread_id") or "") if not thread_id or thread_id in seen: continue seen.add(thread_id) rows.append(row) added += 1 if len(batch) < _ARTIFACT_LIBRARY_THREAD_BATCH_SIZE or added == 0: break offset += len(batch) return rows async def _scan_artifact_library(thread_store) -> list[dict]: """Scan generated outputs for all current-user conversation threads.""" rows = await _search_artifact_library_threads(thread_store) paths = get_paths() semaphore = asyncio.Semaphore(_ARTIFACT_LIBRARY_SCAN_CONCURRENCY) async def scan_thread(row: dict) -> list[dict]: thread_id = str(row.get("thread_id") or "") if not thread_id: return [] async with semaphore: try: path_user_id = await aresolve_path_user_id(thread_id) outputs_dir = paths.sandbox_outputs_dir(thread_id, user_id=path_user_id) files = await asyncio.to_thread(collect_outputs_listing, outputs_dir) except Exception: logger.warning("Failed to scan artifacts for library thread %s", thread_id, exc_info=True) return [] metadata = row.get("metadata") if isinstance(row.get("metadata"), dict) else {} agent_id = metadata.get("agent_id") agent_name = metadata.get("agent_name") thread_type = metadata.get("thread_type") normalized_agent_id = agent_id.strip() if isinstance(agent_id, str) else "" normalized_agent_name = agent_name.strip() if isinstance(agent_name, str) else "" title = str(row.get("display_name") or "新对话") items: list[dict] = [] for file in files: if not is_artifact_library_file(file): continue items.append( { "thread_id": thread_id, "thread_title": title, "agent_id": normalized_agent_id or None, "agent_name": normalized_agent_name or None, "thread_type": thread_type if isinstance(thread_type, str) and thread_type else None, "source_type": "agent" if normalized_agent_id else "general_qa", **file, "file_kind": artifact_library_kind(str(file.get("name") or "")), } ) return items nested = await asyncio.gather(*(scan_thread(row) for row in rows)) items = [item for group in nested for item in group] items.sort( key=lambda item: ( str(item.get("modified_at") or ""), str(item.get("thread_id") or ""), str(item.get("path") or ""), ), reverse=True, ) return items async def _cached_artifact_library(thread_store, *, refresh: bool) -> list[dict]: user_id = str(get_effective_user_id() or "default") now = time.monotonic() cached = _artifact_library_cache.get(user_id) if not refresh and cached and cached[0] > now: _artifact_library_cache.move_to_end(user_id) return cached[1] items = await _scan_artifact_library(thread_store) _artifact_library_cache[user_id] = (now + _ARTIFACT_LIBRARY_CACHE_TTL_SECONDS, items) _artifact_library_cache.move_to_end(user_id) while len(_artifact_library_cache) > _ARTIFACT_LIBRARY_CACHE_MAX_USERS: _artifact_library_cache.popitem(last=False) return items @router.get( "/artifact-library", response_model=ArtifactLibraryResponse, summary="List Generated Files Across Conversations", description="Aggregate non-media files generated in the current user's normal conversations.", ) @require_permission("threads", "read") async def list_artifact_library( request: Request, query: str | None = Query(default=None, max_length=200), file_kind: ArtifactLibraryKind = Query(default="all"), source_type: ArtifactLibrarySource = Query(default="all"), offset: int = Query(default=0, ge=0), limit: int = Query(default=50, ge=1, le=200), refresh: bool = Query(default=False), ) -> ArtifactLibraryResponse: """Return a flat, newest-first library of generated conversation files. The filesystem remains the source of truth: every owned, non-system thread's ``outputs`` directory is scanned, including files written without an explicit ``present_files`` call. Uploads and image/audio/video outputs are intentionally outside this collection. """ from app.gateway.deps import get_agent_store, get_thread_store items = await _cached_artifact_library(get_thread_store(request), refresh=refresh) agent_ids = list(dict.fromkeys(str(item.get("agent_id") or "") for item in items if item.get("agent_id"))) agent_names: dict[str, str] = {} if agent_ids: try: agents = await get_agent_store(request).list_by_ids( agent_ids, str(get_effective_user_id() or "default"), ) agent_names = {str(agent.get("id") or ""): str(agent.get("name") or "").strip() for agent in agents if agent.get("id") and agent.get("name")} except Exception: logger.warning("Failed to resolve artifact library agent names", exc_info=True) items_with_sources = [ { **item, "agent_name": (agent_names.get(str(item.get("agent_id") or "")) or item.get("agent_name") or item.get("agent_id")), } for item in items ] normalized_query = (query or "").strip().casefold() filtered = [ item for item in items_with_sources if (file_kind == "all" or item["file_kind"] == file_kind) and (source_type == "all" or item["source_type"] == source_type) and (not normalized_query or normalized_query in str(item.get("name") or "").casefold() or normalized_query in str(item.get("thread_title") or "").casefold() or normalized_query in str(item.get("agent_name") or "").casefold()) ] page = filtered[offset : offset + limit] return ArtifactLibraryResponse( total=len(filtered), offset=offset, limit=limit, items=[ArtifactLibraryItem.model_validate(item) for item in page], ) @router.get( "/threads/{thread_id}/artifacts", summary="List Artifact Files", description="List all files under the thread's outputs directory (/mnt/user-data/outputs/). Returns an empty list if the thread has no outputs.", ) @require_permission("threads", "read", owner_check=True) async def list_artifacts(thread_id: str, request: Request) -> dict: """列出 thread 在 ``outputs/`` 下生成的产物文件。 圆桌 Step 3（``HighFidelityReport``）对每个**子席位** thread_id 调用本接口， 展示各智能体写入沙箱 ``mnt/user-data/outputs/`` 的文件；协调智能体 thread 通常无独立交付文件。 遍历路径:: .deer-flow/users/{user_id}/threads/{thread_id}/user-data/outputs/ 返回 ``{ "thread_id", "files": [{ path, name, size_bytes, mime_type, modified_at }] }``， 目录不存在时 ``files`` 为空数组（非 404）。 """ outputs_dir = get_paths().sandbox_outputs_dir(thread_id, user_id=await aresolve_path_user_id(thread_id)) return {"thread_id": thread_id, "files": collect_outputs_listing(outputs_dir)} @router.get( "/threads/{thread_id}/artifacts/{path:path}", summary="Get Artifact File", description="Retrieve an artifact file generated by the AI agent. Text and binary files can be viewed inline, while active web content is always downloaded.", ) @require_permission("threads", "read", owner_check=True) async def get_artifact(thread_id: str, path: str, request: Request, download: bool = False) -> Response: """Get an artifact file by its path. The endpoint automatically detects file types and returns appropriate content types. Use the `download` query parameter to force file download for non-active content. Args: thread_id: The thread ID. path: The artifact path with virtual prefix (e.g., mnt/user-data/outputs/file.txt). request: FastAPI request object (automatically injected). Returns: The file content as a FileResponse with appropriate content type: - Active content (HTML/XHTML/SVG): Served as download attachment - Text files: Plain text with proper MIME type - Binary files: Inline display with download option Raises: HTTPException: - 400 if path is invalid or not a file - 403 if access denied (path traversal detected) - 404 if file not found Query Parameters: download (bool): If true, forces attachment download for file types that are otherwise returned inline or as plain text. Active HTML/XHTML/SVG content is always downloaded regardless of this flag. Example: - Get text file inline: `/api/threads/abc123/artifacts/mnt/user-data/outputs/notes.txt` - Download file: `/api/threads/abc123/artifacts/mnt/user-data/outputs/data.csv?download=true` - Active web content such as `.html`, `.xhtml`, and `.svg` artifacts is always downloaded """ path = coerce_artifact_virtual_path(path) # Check if this is a request for a file inside a .skill archive (e.g., xxx.skill/SKILL.md) if ".skill/" in path: # Split the path at ".skill/" to get the ZIP file path and internal path skill_marker = ".skill/" marker_pos = path.find(skill_marker) skill_file_path = path[: marker_pos + len(".skill")] # e.g., "mnt/user-data/outputs/my-skill.skill" internal_path = path[marker_pos + len(skill_marker) :] # e.g., "SKILL.md" actual_skill_path = await aresolve_thread_virtual_path(thread_id, skill_file_path) if not actual_skill_path.exists(): raise HTTPException(status_code=404, detail=f"Skill file not found: {skill_file_path}") if not actual_skill_path.is_file(): raise HTTPException(status_code=400, detail=f"Path is not a file: {skill_file_path}") # Extract the file from the .skill archive content = _extract_file_from_skill_archive(actual_skill_path, internal_path) if content is None: raise HTTPException(status_code=404, detail=f"File '{internal_path}' not found in skill archive") # Determine MIME type based on the internal file mime_type, _ = mimetypes.guess_type(internal_path) # Add cache headers to avoid repeated ZIP extraction (cache for 5 minutes) cache_headers = {"Cache-Control": "private, max-age=300"} download_name = Path(internal_path).name or actual_skill_path.stem if download or mime_type in ACTIVE_CONTENT_MIME_TYPES: return Response(content=content, media_type=mime_type or "application/octet-stream", headers=_build_attachment_headers(download_name, cache_headers)) if mime_type and mime_type.startswith("text/"): return PlainTextResponse(content=content.decode("utf-8"), media_type=mime_type, headers=cache_headers) # Default to plain text for unknown types that look like text try: return PlainTextResponse(content=content.decode("utf-8"), media_type="text/plain", headers=cache_headers) except UnicodeDecodeError: return Response(content=content, media_type=mime_type or "application/octet-stream", headers=cache_headers) actual_path = await aresolve_thread_virtual_path(thread_id, path) logger.info(f"Resolving artifact path: thread_id={thread_id}, requested_path={path}, actual_path={actual_path}") if not actual_path.exists(): raise HTTPException(status_code=404, detail=f"Artifact not found: {path}") if not actual_path.is_file(): raise HTTPException(status_code=400, detail=f"Path is not a file: {path}") mime_type, _ = mimetypes.guess_type(actual_path) if download: return FileResponse(path=actual_path, filename=actual_path.name, media_type=mime_type, headers=_build_attachment_headers(actual_path.name)) # Always force download for active content types to prevent script execution # in the application origin when users open generated artifacts. if mime_type in ACTIVE_CONTENT_MIME_TYPES: return FileResponse(path=actual_path, filename=actual_path.name, media_type=mime_type, headers=_build_attachment_headers(actual_path.name)) if mime_type and mime_type.startswith("text/"): return PlainTextResponse(content=actual_path.read_text(encoding="utf-8"), media_type=mime_type) if is_text_file_by_content(actual_path): return PlainTextResponse(content=actual_path.read_text(encoding="utf-8"), media_type=mime_type) return Response(content=actual_path.read_bytes(), media_type=mime_type, headers={"Content-Disposition": _build_content_disposition("inline", actual_path.name)}) @router.patch( "/threads/{thread_id}/artifacts/{path:path}", summary="Update Artifact File", description="Update a text artifact file in the current thread workspace.", ) @require_permission("threads", "write", owner_check=True) async def update_artifact( thread_id: str, path: str, request: Request, body: ArtifactUpdateRequest, ) -> Response: """Update a text artifact. This endpoint is intentionally limited to normal text artifacts. Files inside .skill archives and active web content remain read-only from the artifact UI. """ if ".skill/" in path: raise HTTPException(status_code=400, detail="Skill archive contents are read-only.") path = coerce_artifact_virtual_path(path) actual_path = await aresolve_thread_virtual_path(thread_id, path) logger.info(f"Updating artifact path: thread_id={thread_id}, requested_path={path}, actual_path={actual_path}") if not actual_path.exists(): raise HTTPException(status_code=404, detail=f"Artifact not found: {path}") if not actual_path.is_file(): raise HTTPException(status_code=400, detail=f"Path is not a file: {path}") mime_type, _ = mimetypes.guess_type(actual_path) suffix = actual_path.suffix.lower() if mime_type in ACTIVE_CONTENT_MIME_TYPES: raise HTTPException(status_code=400, detail="Active web artifacts are read-only.") if suffix not in EDITABLE_TEXT_SUFFIXES and not (mime_type and mime_type.startswith("text/")): raise HTTPException(status_code=400, detail="Only text artifacts can be edited.") actual_path.write_text(body.content, encoding="utf-8") return PlainTextResponse(content=body.content, media_type=mime_type or "text/plain")