chore: 纳入本地 Dify 1.19.0 源码及品牌与免登改动

演示目录只保留可重建的源码。运行密钥、数据卷、依赖和带口令的本地配置不进入仓库。

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
wuzilongandCursor committed 2026-11-18 13:59:25 +08:00
1 parent 60158935f4
commit 6519682885
15831 files changed
+3063428

No files matched your search

+1
View File
@@ -0,0 +1 @@
"""Development tooling for Dify API contracts and checks."""
@@ -0,0 +1,360 @@
"""Local, idle-expiring OpenAPI export cache. Contract bytes never touch the filesystem.
The POSIX server owns a private Unix socket and an empty lifecycle lock. It caches
the document in RAM. A single-threaded snapshot preloads only dependency modules,
never controllers. Each refresh imports controllers in a fresh fork; edits to a
preloaded dependency re-exec the snapshot before exporting. Unsupported platforms
or unsafe snapshots use the ordinary fresh-process exporter.
"""
from __future__ import annotations
import contextlib
import hashlib
import importlib.machinery
import io
import json
import os
import select
import signal
import socket
import subprocess
import sys
import tempfile
import time
import traceback
from pathlib import Path
from typing import Any, Never
API_ROOT = Path(__file__).resolve().parents[1]
SOURCE_ROOTS = (API_ROOT, API_ROOT.parent / "dify-agent/src")
PACKAGE_INPUTS = (API_ROOT / "pyproject.toml", API_ROOT / "uv.lock", API_ROOT.parent / "dify-agent/pyproject.toml")
TIMEOUT = 90
IDLE_TIMEOUT = 120
MAX_BYTES = 64 * 1024 * 1024
def source_imports() -> None:
"""Read editable backend modules from source, preserving installed-library bytecode caches."""
original = importlib.machinery.SourceFileLoader.get_code
def get_code(self: importlib.machinery.SourceFileLoader, fullname: str):
path = Path(self.path)
if any(path.is_relative_to(root) for root in SOURCE_ROOTS) and ".venv" not in path.parts:
return self.source_to_code(self.get_data(self.path), self.path)
return original(self, fullname)
importlib.machinery.SourceFileLoader.get_code = get_code
sys.dont_write_bytecode = True
def source_state() -> dict[str, tuple[int, int, int]]:
"""Match the Console descriptor's inputs, excluding tests and local environments."""
files = [*PACKAGE_INPUTS, Path.cwd() / ".env"]
for root in SOURCE_ROOTS:
for directory, children, names in os.walk(root):
children[:] = [name for name in children if name not in {".venv", ".git", "__pycache__"}]
if Path(directory) == API_ROOT:
children[:] = [name for name in children if name != "tests"]
files.extend(Path(directory) / name for name in names if name.endswith(".py"))
return {str(file): file_stamp(file) for file in files}
def file_stamp(file: Path) -> tuple[int, int, int]:
try:
stat = file.stat()
return stat.st_mtime_ns, stat.st_ctime_ns, stat.st_size
except FileNotFoundError:
return 0, 0, 0
def fresh_export() -> bytes:
result = subprocess.run(
[
sys.executable,
"-B",
"-c",
"import sys;sys.path.insert(0,sys.argv[1]);"
"from dev.cached_console_openapi import source_imports;source_imports();"
"from dev.export_console_openapi import main;main()",
str(API_ROOT),
],
capture_output=True,
timeout=TIMEOUT,
check=False,
)
if result.returncode:
raise RuntimeError(result.stderr.decode(errors="replace")[-16000:])
return result.stdout
def single_threaded() -> bool:
"""Check native threads too: threading.active_count misses C library pools."""
try:
if sys.platform.startswith("linux"):
return len(os.listdir("/proc/self/task")) == 1
if sys.platform == "darwin":
# Darwin -M always prints its thread header, even with -o pid=.
output = subprocess.check_output(["ps", "-M", "-p", str(os.getpid())], timeout=5)
return len(output.strip().splitlines()) == 2
except (OSError, subprocess.SubprocessError):
return False
return False
def preload() -> tuple[bool, dict[str, tuple[int, int, int]]]:
"""Load schema dependencies without retaining route registries or controller classes."""
import urllib.request
getproxies = urllib.request.getproxies
try:
if sys.platform == "darwin":
# macOS SystemConfiguration starts native threads. Resolve its proxy
# settings in a helper, preserving the values without forking those threads.
proxies = json.loads(
subprocess.check_output(
[sys.executable, "-c", "import json,urllib.request;print(json.dumps(urllib.request.getproxies()))"],
timeout=10,
)
)
urllib.request.getproxies = lambda: proxies
sys.path.insert(0, str(API_ROOT))
from dev.generate_swagger_specs import apply_runtime_defaults
apply_runtime_defaults()
# Keep runtime composition (and libraries with native thread pools) out
# of the snapshot. These pure model dependencies account for much of startup.
import dify_agent.protocol.schemas
import core.mcp.types
import graphon.nodes.parameter_extractor.entities
import models
import services.knowledge_fs.product_dto
_ = core.mcp.types, dify_agent.protocol.schemas, graphon.nodes.parameter_extractor.entities
_ = models, services.knowledge_fs.product_dto
safe = not any(name.startswith("controllers.") for name in sys.modules) and single_threaded()
except Exception:
# A cache optimization must not prevent a normal export from working.
safe = False
finally:
urllib.request.getproxies = getproxies
dependencies = {}
for module in tuple(sys.modules.values()):
filename = getattr(module, "__file__", None) # guard-ignore: no-new-getattr -- optional module source metadata
if isinstance(filename, str):
dependencies[str(Path(filename).resolve())] = file_stamp(Path(filename))
for filename in (*PACKAGE_INPUTS, Path.cwd() / ".env"):
dependencies[str(filename)] = file_stamp(filename)
return safe, dependencies
def export_worker(write_fd: int) -> Never:
"""Run the isolated exporter and keep its pipe open through process exit."""
try:
from dev.export_console_openapi import main
output = io.StringIO()
with contextlib.redirect_stdout(output):
main()
payload = b"OK\n" + output.getvalue().encode()
except BaseException:
payload = b"ERROR\n" + traceback.format_exc().encode()
try:
with os.fdopen(write_fd, "wb") as pipe:
pipe.write(payload)
pipe.flush()
# Keep the pipe open through process exit. Closing it first lets
# the parent kill a successful worker while exit instrumentation
# (for example, subprocess coverage) is still flushing its data.
os._exit(0)
finally:
os._exit(1)
def fork_export() -> bytes:
"""Export from a clean controller registry; the parent snapshot is never mutated."""
read_fd, write_fd = os.pipe()
pid = os.fork()
if pid == 0:
os.close(read_fd)
export_worker(write_fd)
os.close(write_fd)
chunks: list[bytes] = []
size = 0
deadline = time.monotonic() + TIMEOUT
try:
while True:
if not select.select([read_fd], [], [], max(0, deadline - time.monotonic()))[0]:
raise TimeoutError("OpenAPI refresh timed out")
chunk = os.read(read_fd, 65536)
if not chunk:
break
size += len(chunk)
if size > MAX_BYTES:
raise ValueError("OpenAPI export exceeds the response limit")
chunks.append(chunk)
result = b"".join(chunks)
status, _, payload = result.partition(b"\n")
if status != b"OK":
raise RuntimeError(payload.decode(errors="replace") or "OpenAPI worker exited without a response")
return payload
finally:
os.close(read_fd)
# A worker can fail after closing its pipe. Never leave it running or unreaped.
with contextlib.suppress(ProcessLookupError):
os.kill(pid, signal.SIGKILL)
os.waitpid(pid, 0)
def socket_path() -> Path:
directory = Path(tempfile.gettempdir()) / f"dify-openapi-{os.getuid()}"
# macOS Unix socket paths have a small length limit.
if len(str(directory)) > 55:
directory = Path("/tmp") / f"dify-openapi-{os.getuid()}"
directory.mkdir(mode=0o700, exist_ok=True)
stat = directory.lstat()
if directory.is_symlink() or stat.st_uid != os.getuid() or stat.st_mode & 0o077:
raise PermissionError("OpenAPI IPC directory must be private to the current user")
environment = {
key: value
for key, value in os.environ.items()
if key not in {"_", "PWD", "OLDPWD", "SHLVL", "UV_RUN_RECURSION_DEPTH"}
}
key = hashlib.sha256(json.dumps([str(API_ROOT), os.getcwd(), sys.executable, environment], sort_keys=True).encode())
key.update(Path(__file__).read_bytes())
return directory / f"{key.hexdigest()[:24]}.sock"
def serve(path: Path, inherited: list[str]) -> None:
import fcntl
if inherited:
listener = socket.socket(fileno=int(inherited[0]))
lock_fd = int(inherited[1])
else:
lock_fd = os.open(path.with_suffix(".lock"), os.O_CREAT | os.O_RDWR, 0o600)
try:
fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
os.close(lock_fd)
return
path.unlink(missing_ok=True)
listener = socket.socket(socket.AF_UNIX)
listener.bind(str(path))
listener.listen(16)
listener.settimeout(IDLE_TIMEOUT)
baseline = source_state()
safe, dependencies = preload()
snapshot_dirty = source_state() != baseline
cached_state: dict[str, tuple[int, int, int]] | None = None
cached = b""
try:
while True:
try:
connection, _ = listener.accept()
except TimeoutError:
return
with connection:
connection.settimeout(TIMEOUT)
try:
request = b""
while not request.endswith(b"\n") and len(request) < 16:
part = connection.recv(16 - len(request))
if not part:
break
request += part
if request == b"stop\n":
connection.sendall(b"OK\n")
return
if request != b"export\n":
continue
current = source_state()
stale = (
snapshot_dirty
or set(current) != set(baseline)
or any(file_stamp(Path(file)) != stamp for file, stamp in dependencies.items())
)
if stale:
# No reload(sys.modules): restart all preloaded dependencies.
connection.sendall(b"RETRY\n")
connection.close()
os.set_inheritable(listener.fileno(), True)
os.set_inheritable(lock_fd, True)
os.execv(
sys.executable,
[sys.executable, __file__, "--serve", str(path), str(listener.fileno()), str(lock_fd)],
)
if current != cached_state:
output = fork_export() if safe and single_threaded() else fresh_export()
# Do not publish a contract assembled while source files changed.
if source_state() != current:
connection.sendall(b"RETRY\n")
continue
payload: Any = json.loads(output)
if not isinstance(payload, dict) or set(payload) != {"console", "fast"}:
raise ValueError("Invalid Console export response")
cached, cached_state = output, current
connection.sendall(b"OK\n" + cached)
except (BrokenPipeError, ConnectionResetError):
pass
except Exception:
with contextlib.suppress(OSError):
connection.sendall(b"ERROR\n" + traceback.format_exc().encode()[-16000:])
finally:
listener.close()
path.unlink(missing_ok=True)
os.close(lock_fd)
def request_export(*, stop: bool = False) -> bytes:
if not hasattr(os, "fork"):
return b"" if stop else fresh_export()
path = socket_path()
deadline = time.monotonic() + TIMEOUT
launched = False
while time.monotonic() < deadline:
try:
with socket.socket(socket.AF_UNIX) as connection:
connection.settimeout(max(0.1, deadline - time.monotonic()))
connection.connect(str(path))
connection.sendall(b"stop\n" if stop else b"export\n")
chunks: list[bytes] = []
size = 0
while chunk := connection.recv(65536):
size += len(chunk)
if size > MAX_BYTES:
raise ValueError("OpenAPI response exceeds the limit")
chunks.append(chunk)
status, _, payload = b"".join(chunks).partition(b"\n")
if status == b"OK":
return payload
if status == b"ERROR":
raise RuntimeError(payload.decode(errors="replace"))
if status != b"RETRY":
raise RuntimeError("Incomplete OpenAPI cache response")
except (FileNotFoundError, ConnectionRefusedError):
if stop:
return b""
if not launched:
subprocess.Popen(
[sys.executable, __file__, "--serve", str(path)],
start_new_session=True,
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
launched = True
time.sleep(0.03)
raise TimeoutError("OpenAPI cache did not respond in time")
if __name__ == "__main__":
# Timestamp-based .pyc files can serve stale code after same-size edits within one second.
source_imports()
if sys.argv[1:2] == ["--serve"]:
serve(Path(sys.argv[2]), sys.argv[3:])
elif sys.argv[1:] == ["--stop"]:
request_export(stop=True)
else:
sys.stdout.buffer.write(request_export())
@@ -0,0 +1,360 @@
"""Enforce focused KnowledgeFS coverage without hiding integration-critical glue.
The primary threshold aggregates statement and branch coverage for every Dify
module owned by the KnowledgeFS integration. Large pre-existing Dify modules
that only contain narrow integration hooks are checked with changed-line
coverage instead, so unrelated legacy code cannot dilute or inflate the gate.
"""
from __future__ import annotations
import argparse
import json
import logging
import os
import re
import subprocess
from dataclasses import dataclass
from pathlib import Path
from typing import TypedDict, cast
WORKSPACE_ROOT = Path(__file__).resolve().parents[2]
NON_CORE_COVERAGE_ALLOWLIST = frozenset(
{
"api/dev/check_knowledge_fs_coverage.py",
"api/dev/generate_knowledge_fs_contract.py",
"api/dev/knowledge_fs_product_contract.py",
"api/migrations/versions/2026_07_21_1200-a4e7c2f91b30_add_knowledge_fs_control_plane.py",
"api/migrations/versions/2026_07_21_1300-b7f2a9d41c60_add_knowledge_fs_cutover.py",
"api/migrations/versions/2026_07_21_1400-c8e31b7d52a4_add_knowledge_fs_cleanup_authorization.py",
"api/migrations/versions/2026_07_21_1500-d4f6e8a1c305_add_knowledge_fs_remote_freeze_evidence.py",
"api/migrations/versions/2026_07_21_1600-e5a7c9b2d416_add_knowledge_fs_cleanup_completion.py",
"api/migrations/versions/2026_08_10_1200-7c1e9a4b2d60_add_knowledge_fs_staged_uploads.py",
"api/migrations/versions/2026_08_13_1200-9d4e6f8a1b2c_add_knowledge_fs_space_tag_bindings.py",
"api/migrations/versions/2026_08_17_1200-4f8b2c7d9e10_add_knowledge_fs_icon_background.py",
"api/migrations/versions/2026_08_17_1200-f3a8c1d7e920_add_knowledge_fs_upgrade_jobs.py",
"api/migrations/versions/2026_08_18_1200-e6b4a2c9d731_limit_active_knowledge_fs_upgrades.py",
"api/migrations/versions/2026_10_21_1300-3edd6cd27f54_limit_active_knowledge_fs_upgrades_on_mysql.py",
}
)
HUNK_HEADER = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@")
logger = logging.getLogger(__name__)
class CoverageSummary(TypedDict):
"""Coverage.py counts required by the aggregate gate."""
covered_lines: int
num_statements: int
covered_branches: int
num_branches: int
class CoverageFile(TypedDict):
"""Per-file coverage data emitted by ``coverage json``."""
executed_lines: list[int]
missing_lines: list[int]
summary: CoverageSummary
class CoverageReport(TypedDict):
"""Relevant top-level shape of a coverage.py JSON report."""
files: dict[str, CoverageFile]
@dataclass(frozen=True, slots=True)
class CoverageTotals:
"""Covered and measurable units for one gate surface."""
covered: int
total: int
@property
def percent(self) -> float:
return 100.0 if self.total == 0 else self.covered * 100 / self.total
class CoverageGateError(RuntimeError):
"""Raised when coverage input is incomplete or below its threshold."""
def main() -> None:
"""Validate focused module coverage and changed integration glue."""
logging.basicConfig(level=logging.INFO, format="%(message)s")
parser = argparse.ArgumentParser()
parser.add_argument("--coverage-json", type=Path, required=True)
parser.add_argument("--glue-manifest", type=Path, required=True)
parser.add_argument("--workspace-root", type=Path, default=WORKSPACE_ROOT)
parser.add_argument("--base", default="")
parser.add_argument("--minimum", type=float, default=90.0)
parser.add_argument("--glue-minimum", type=float, default=90.0)
args = parser.parse_args()
workspace_root = args.workspace_root.resolve()
report = load_coverage_report(args.coverage_json)
core_totals = validate_core_coverage(report, workspace_root=workspace_root, minimum=args.minimum)
base = resolve_diff_base(workspace_root, args.base)
glue_paths = load_glue_coverage_paths(args.glue_manifest, workspace_root=workspace_root)
changed_lines = collect_changed_glue_lines(workspace_root, base, glue_paths=glue_paths)
glue_totals = validate_changed_glue_coverage(
report,
changed_lines=changed_lines,
minimum=args.glue_minimum,
)
logger.info(
"Dify KnowledgeFS coverage passed: core lines+branches %.2f%% (%d/%d); changed glue lines %.2f%% (%d/%d)",
core_totals.percent,
core_totals.covered,
core_totals.total,
glue_totals.percent,
glue_totals.covered,
glue_totals.total,
)
def load_coverage_report(path: Path) -> CoverageReport:
"""Load the detailed JSON report used by both coverage checks."""
if not path.is_file():
raise CoverageGateError(f"coverage JSON does not exist: {path}")
document = json.loads(path.read_text())
if not isinstance(document, dict) or not isinstance(document.get("files"), dict):
raise CoverageGateError(f"coverage JSON has no files object: {path}")
return cast(CoverageReport, document)
def load_glue_coverage_paths(path: Path, *, workspace_root: Path) -> tuple[str, ...]:
"""Load the workflow's authoritative NUL-delimited integration touchpoints."""
if not path.is_file():
raise CoverageGateError(f"KnowledgeFS glue manifest does not exist: {path}")
try:
paths = tuple(item.decode() for item in path.read_bytes().split(b"\0") if item)
except UnicodeDecodeError as error:
raise CoverageGateError(f"KnowledgeFS glue manifest is not UTF-8: {path}") from error
if not paths:
raise CoverageGateError("Dify KnowledgeFS glue coverage target set is empty")
if len(paths) != len(set(paths)):
raise CoverageGateError("Dify KnowledgeFS glue coverage manifest contains duplicate paths")
invalid_paths = [
candidate
for candidate in paths
if not candidate.startswith("api/")
or not candidate.endswith(".py")
or not (workspace_root / candidate).is_file()
]
if invalid_paths:
raise CoverageGateError(f"Dify KnowledgeFS glue coverage paths are invalid: {', '.join(invalid_paths)}")
return paths
def is_core_coverage_path(path: str) -> bool:
"""Return whether a repository-relative path belongs to the focused aggregate."""
if not path.endswith(".py"):
return False
if path in {
"api/commands/knowledge_fs.py",
"api/configs/extra/knowledge_fs_config.py",
"api/controllers/openapi/knowledge_fs.py",
"api/events/event_handlers/sync_knowledge_fs_bindings_when_app_published_workflow_updated.py",
"api/extensions/ext_knowledge_fs_observability.py",
"api/services/dataset_knowledge_fs_upgrade_file_lease.py",
"api/services/dataset_knowledge_fs_upgrade_service.py",
"api/services/knowledge_fs_capability.py",
}:
return True
if path.startswith(
(
"api/controllers/console/knowledge_fs/",
"api/controllers/inner_api/knowledge_fs/",
"api/controllers/service_api/knowledge_fs/",
"api/services/knowledge_fs/",
)
):
return True
filename = path.rsplit("/", maxsplit=1)[-1]
return (
path.startswith("api/models/")
and filename.startswith("knowledge_fs")
or path.startswith("api/repositories/")
and "knowledge_fs" in filename
or path.startswith("api/tasks/")
and "knowledge_fs" in filename
)
def discover_core_coverage_paths(workspace_root: Path) -> tuple[str, ...]:
"""Classify every KnowledgeFS-named production file or fail closed."""
named_paths = discover_knowledge_fs_production_paths(workspace_root)
core_paths = {path for path in named_paths if is_core_coverage_path(path)}
unclassified_paths = set(named_paths) - core_paths - NON_CORE_COVERAGE_ALLOWLIST
if unclassified_paths:
raise CoverageGateError(
"unclassified Dify KnowledgeFS production files must join the core coverage scope or explicit allowlist: "
+ ", ".join(sorted(unclassified_paths))
)
if not core_paths:
raise CoverageGateError("Dify KnowledgeFS core coverage target set is empty")
return tuple(sorted(core_paths))
def discover_knowledge_fs_production_paths(workspace_root: Path) -> tuple[str, ...]:
"""Mirror the workflow's dynamic KnowledgeFS filename discovery."""
api_root = workspace_root / "api"
if not api_root.is_dir():
raise CoverageGateError(f"Dify API directory does not exist: {api_root}")
paths: set[str] = set()
for directory, child_directories, filenames in os.walk(api_root):
current_directory = Path(directory)
if current_directory == api_root:
child_directories[:] = [name for name in child_directories if name not in {".venv", "storage", "tests"}]
child_directories[:] = [name for name in child_directories if name != "__pycache__"]
for filename in filenames:
path = (current_directory / filename).relative_to(workspace_root).as_posix()
if filename.endswith(".py") and "knowledge_fs" in path:
paths.add(path)
if not paths:
raise CoverageGateError("Dify KnowledgeFS production target set is empty")
return tuple(sorted(paths))
def validate_core_coverage(
report: CoverageReport,
*,
workspace_root: Path,
minimum: float,
) -> CoverageTotals:
"""Require the exact combined line-and-branch percentage for all core files."""
paths = discover_core_coverage_paths(workspace_root)
missing_paths = [path for path in paths if path not in report["files"]]
if missing_paths:
raise CoverageGateError(f"coverage report is missing core files: {', '.join(missing_paths)}")
covered = 0
total = 0
for path in paths:
summary = report["files"][path]["summary"]
covered += summary["covered_lines"] + summary["covered_branches"]
total += summary["num_statements"] + summary["num_branches"]
if total == 0:
raise CoverageGateError("Dify KnowledgeFS core coverage has no measurable statements or branches")
totals = CoverageTotals(covered=covered, total=total)
_require_minimum(totals, minimum=minimum, label="Dify KnowledgeFS core line-and-branch coverage")
return totals
def resolve_diff_base(workspace_root: Path, preferred: str) -> str:
"""Resolve an explicit event base, falling back to the previous commit for manual runs."""
base = preferred.strip()
if not base or set(base) == {"0"}:
base = "HEAD^"
result = subprocess.run(
["git", "cat-file", "-e", f"{base}^{{commit}}"],
cwd=workspace_root,
check=False,
capture_output=True,
text=True,
)
if result.returncode != 0:
detail = result.stderr.strip() or "commit is unavailable"
raise CoverageGateError(f"cannot resolve coverage diff base {base}: {detail}")
return base
def collect_changed_glue_lines(
workspace_root: Path,
base: str,
*,
glue_paths: tuple[str, ...],
) -> dict[str, set[int]]:
"""Return added line numbers in the narrow Dify modules touched by this integration."""
result = subprocess.run(
[
"git",
"diff",
"--no-ext-diff",
"--no-color",
"--unified=0",
"--src-prefix=a/",
"--dst-prefix=b/",
base,
"--",
*glue_paths,
],
cwd=workspace_root,
check=False,
capture_output=True,
text=True,
)
if result.returncode != 0:
detail = result.stderr.strip() or "git diff failed"
raise CoverageGateError(f"cannot collect KnowledgeFS glue diff from {base}: {detail}")
return parse_added_lines(result.stdout)
def parse_added_lines(diff: str) -> dict[str, set[int]]:
"""Parse repository paths and added-side line numbers from a zero-context Git diff."""
changed_lines: dict[str, set[int]] = {}
current_path: str | None = None
current_line: int | None = None
for raw_line in diff.splitlines():
if raw_line.startswith("diff --git "):
current_line = None
continue
if raw_line.startswith("+++ "):
candidate = raw_line[4:]
current_path = candidate[2:] if candidate.startswith("b/") else None
if current_path is not None:
changed_lines.setdefault(current_path, set())
current_line = None
continue
if raw_line.startswith("@@ "):
match = HUNK_HEADER.match(raw_line)
current_line = int(match.group(1)) if match is not None else None
continue
if current_path is None or current_line is None:
continue
if raw_line.startswith("+"):
changed_lines[current_path].add(current_line)
current_line += 1
elif raw_line.startswith("-") or raw_line.startswith("\\"):
continue
else:
current_line += 1
return changed_lines
def validate_changed_glue_coverage(
report: CoverageReport,
*,
changed_lines: dict[str, set[int]],
minimum: float,
) -> CoverageTotals:
"""Require added executable glue lines to be exercised by the focused unit suite."""
covered = 0
total = 0
for path, lines in sorted(changed_lines.items()):
if not lines:
continue
file_coverage = report["files"].get(path)
if file_coverage is None:
raise CoverageGateError(f"coverage report is missing changed glue file: {path}")
executed_lines = set(file_coverage["executed_lines"])
executable_lines = executed_lines | set(file_coverage["missing_lines"])
changed_executable_lines = lines & executable_lines
covered += len(changed_executable_lines & executed_lines)
total += len(changed_executable_lines)
totals = CoverageTotals(covered=covered, total=total)
_require_minimum(totals, minimum=minimum, label="Dify KnowledgeFS changed-glue line coverage")
return totals
def _require_minimum(totals: CoverageTotals, *, minimum: float, label: str) -> None:
if totals.percent + 1e-12 < minimum:
raise CoverageGateError(f"{label} is {totals.percent:.2f}%; minimum {minimum:.2f}%")
if __name__ == "__main__":
main()
@@ -0,0 +1,105 @@
"""Export community Console schemas to stdout for the OpenAPI content mapper."""
import ast
import contextlib
import inspect
import json
import sys
import textwrap
from pathlib import Path
from typing import Any
def source_location(target: Any) -> dict[str, Any] | None:
"""Locate the actual class or function definition, skipping decorators."""
try:
target = inspect.unwrap(target)
filename = inspect.getsourcefile(target)
lines, start = inspect.getsourcelines(target)
definition = ast.parse(textwrap.dedent("".join(lines))).body[0]
except (OSError, TypeError, ValueError, SyntaxError):
# Dynamically created or native handlers have no navigable Python source.
return None
if filename and isinstance(definition, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)):
return {
"file": str(Path(filename).resolve()),
"line": start + definition.lineno - 1,
}
return None
def resolve_source_type(source: str) -> Any:
"""Resolve registered identities only, including nested classes and generic origins."""
parts = source.split("[", 1)[0].split(".")
for end in range(len(parts) - 1, 0, -1):
target = sys.modules.get(".".join(parts[:end]))
if target is None:
continue
for part in parts[end:]:
if not inspect.ismodule(target) and not inspect.isclass(target):
return None
target = vars(target).get(part)
return target if inspect.isclass(target) else None
return None
def attach_source_location(document: dict[str, Any], path: str, method: str, handler: Any) -> None:
"""Attach editor-only origins to exported operations, leaving live schemas unchanged."""
operation = document["paths"].get(path, {}).get(method.lower())
if isinstance(operation, dict) and (location := source_location(handler)):
operation["x-source-location"] = location
def attach_api_sources(document: dict[str, Any], api: Any) -> None:
"""Annotate exported RESTX models and operations from their registered Python owners."""
from flask_restx.swagger import extract_path
from controllers.common.schema import registered_schema_sources
schemas = document.get("components", {}).get("schemas", {})
for namespace in api.namespaces:
for name, source in registered_schema_sources(namespace).items():
if name in schemas and (location := source_location(resolve_source_type(source))):
schemas[name]["x-source-location"] = location
for resource in namespace.resources:
cls = resource.resource
for url in api.ns_urls(namespace, resource.urls):
for method in cls.methods:
handler = getattr(cls, method.lower(), None) # guard-ignore: no-new-getattr -- HTTP method
attach_source_location(document, extract_path(url), method, handler)
def export_all_with_sources() -> dict[str, dict[str, Any]]:
"""Export every RESTX surface with editor metadata, without writing spec files."""
from dev.generate_swagger_specs import SPEC_TARGETS, _target_apis, export_specs
documents = export_specs()
apis = _target_apis()
for target in SPEC_TARGETS:
attach_api_sources(documents[target.filename], apis[target.route])
return documents
def main() -> None:
# Backend initialization may log to stdout. Keep the protocol output pure JSON.
with contextlib.redirect_stdout(sys.stderr):
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from dev.generate_fastopenapi_specs import export_fastopenapi_specs
from dev.generate_swagger_specs import export_console_spec
documents = {
"console": export_console_spec(),
"fast": export_fastopenapi_specs()["fastopenapi-console-openapi.json"],
}
from controllers.console import api
from controllers.fastopenapi import console_router
attach_api_sources(documents["console"], api)
for path, method, handler in console_router.get_routes():
attach_source_location(documents["fast"], f"/console/api{path}", method, handler)
json.dump(documents, sys.stdout, sort_keys=True)
sys.stdout.write("\n")
if __name__ == "__main__":
main()
@@ -0,0 +1,646 @@
"""Snapshot the billing and saas OpenAPI specs and generate the typed client in `services/billing_client`.
The snapshot is the generation input, so CI can regenerate and diff without the private dify-saas
checkout. Only operations on `ALLOWED_OPERATIONS`, the schemas they reach, the `Error` body, and the
`<Service>ErrorReason` of each proto service they belong to are snapshotted: dify is public, and every
other dify-saas endpoint stays out of this repository.
The proto service is the operationId prefix before the underscore (`BillingEducation` for
`BillingEducation_VerifyEducationIdentity`). Each one gets its own reason Literal, error classes, and
`ServiceErrors`, so a caller catching `BillingEducationUpstreamError` only matches the reasons that
service can return.
uv run --project api --dev python api/dev/generate_billing_client.py # snapshot + generate
uv run --project api --dev python api/dev/generate_billing_client.py --from-snapshot # generate only
uv run --project api --dev python api/dev/generate_billing_client.py --check # CI drift check
The dify-saas checkout defaults to `../dify-saas` next to the dify repository; override it with
`DIFY_SAAS_REPO`.
Kratos serializes with protojson `EmitUnpopulated: true`, which the snapshot step records as presence:
- Scalars, enums, lists, and maps are required and non-null.
- Message fields are required but nullable, since unset messages are emitted as null. A message field
the upstream schema lists in `required` (`google.api.field_behavior = REQUIRED`) is non-null.
- Proto3 `optional` fields (annotated `nullable` in dify-saas) are synthetic oneofs, which protojson
omits when unset, so they are optional and nullable.
"""
from __future__ import annotations
import argparse
import difflib
import json
import keyword
import logging
import os
import re
import subprocess
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Literal
import yaml
from pydantic import BaseModel
API_ROOT = Path(__file__).resolve().parents[1]
REPO_ROOT = API_ROOT.parent
CLIENT_DIR = API_ROOT / "services" / "billing_client"
GENERATOR = "api/dev/generate_billing_client.py"
logger = logging.getLogger("generate_billing_client")
type Service = Literal["billing", "saas"]
type UpstreamName = Literal["BILLING", "BILLING_QUOTA", "SAAS", "NETWORK_ACCESS"]
type JsonObject = dict[str, Any]
SERVICES: tuple[Service, ...] = ("billing", "saas")
SCHEMA_PREFIXES: dict[Service, str] = {"billing": "dify.saas.api.billing.", "saas": "dify.saas.api.saas."}
# BILLING_API_URL already ends with the API version.
PATH_VERSION_PREFIX = "/v1"
SNAPSHOT_COMMIT_KEY = "x-dify-saas-commit"
HTTP_METHODS = ("get", "post", "put", "delete", "patch")
LINE_LENGTH = 120
BASE_MODEL_ATTRIBUTES = frozenset(dir(BaseModel))
ERROR_SCHEMA = "Error"
ERROR_REASON_SUFFIX = "ErrorReason"
ERROR_CLASS_SUFFIXES = ("UpstreamError", "UpstreamUnavailableError", "UpstreamInvalidResponseError")
@dataclass(frozen=True)
class AllowedOperation:
method: str
path: str
upstream: UpstreamName
# Endpoints dify API calls today. Add an entry here, then rerun this script, to call a new one.
ALLOWED_OPERATIONS: dict[Service, tuple[AllowedOperation, ...]] = {
"billing": (
AllowedOperation("GET", "/v1/subscription/info", "BILLING"),
AllowedOperation("GET", "/v1/subscription/vector-space", "BILLING"),
AllowedOperation("GET", "/v1/subscription/knowledge-rate-limit", "BILLING"),
AllowedOperation("GET", "/v1/subscription/payment-link", "BILLING"),
AllowedOperation("POST", "/v1/subscription/plan/batch", "BILLING"),
AllowedOperation("GET", "/v1/subscription/cleanup/whitelist", "BILLING"),
AllowedOperation("GET", "/v1/model-provider/payment-link", "BILLING"),
AllowedOperation("GET", "/v1/invoices", "BILLING"),
AllowedOperation("GET", "/v1/education/verify", "BILLING"),
AllowedOperation("GET", "/v1/education/status", "BILLING"),
AllowedOperation("POST", "/v1/education", "BILLING"),
AllowedOperation("GET", "/v1/education/autocomplete", "BILLING"),
AllowedOperation("POST", "/v1/compliance/download", "BILLING"),
AllowedOperation("GET", "/v1/quota/info", "BILLING_QUOTA"),
AllowedOperation("POST", "/v1/quota/reserve", "BILLING_QUOTA"),
AllowedOperation("POST", "/v1/quota/commit", "BILLING_QUOTA"),
AllowedOperation("POST", "/v1/quota/release", "BILLING_QUOTA"),
AllowedOperation("GET", "/v1/quota/balance", "BILLING_QUOTA"),
AllowedOperation("POST", "/v1/quota/consume-capped", "BILLING_QUOTA"),
),
"saas": (
AllowedOperation("DELETE", "/v1/account", "SAAS"),
AllowedOperation("GET", "/v1/account/in-freeze", "SAAS"),
AllowedOperation("POST", "/v1/account/delete-feedback", "SAAS"),
AllowedOperation("GET", "/v1/notifications/active", "SAAS"),
AllowedOperation("POST", "/v1/notifications/{notificationId}/dismiss", "SAAS"),
AllowedOperation("POST", "/v1/tenant_utms", "SAAS"),
AllowedOperation("GET", "/v1/tenants/{tenantId}/network-access-groups", "NETWORK_ACCESS"),
AllowedOperation("POST", "/v1/tenants/{tenantId}/network-access-groups", "NETWORK_ACCESS"),
AllowedOperation("GET", "/v1/tenants/{tenantId}/network-access-groups/{groupId}", "NETWORK_ACCESS"),
AllowedOperation("PUT", "/v1/tenants/{tenantId}/network-access-groups/{groupId}", "NETWORK_ACCESS"),
AllowedOperation("DELETE", "/v1/tenants/{tenantId}/network-access-groups/{groupId}", "NETWORK_ACCESS"),
AllowedOperation("GET", "/v1/tenants/{tenantId}/apps/{appId}/network-access-group", "NETWORK_ACCESS"),
AllowedOperation("PUT", "/v1/tenants/{tenantId}/apps/{appId}/network-access-group", "NETWORK_ACCESS"),
AllowedOperation("GET", "/v1/tenants/{tenantId}/apps/{appId}/network-access-group/status", "NETWORK_ACCESS"),
AllowedOperation(
"DELETE", "/v1/tenants/{tenantId}/apps/{appId}/network-access-group-binding", "NETWORK_ACCESS"
),
),
}
class GenerationError(Exception):
pass
def snapshot_path(service: Service) -> Path:
return CLIENT_DIR / f"{service}.openapi.json"
def module_path(service: Service) -> Path:
return CLIENT_DIR / f"{service}.py"
# --- snapshot -------------------------------------------------------------------------------------
def dify_saas_root() -> Path:
override = os.environ.get("DIFY_SAAS_REPO")
return Path(override).resolve() if override else (REPO_ROOT.parent / "dify-saas").resolve()
def _git(root: Path, *args: str) -> str:
return subprocess.run(["git", "-C", str(root), *args], check=True, capture_output=True, text=True).stdout.strip()
def dify_saas_commit(root: Path) -> str:
if _git(root, "status", "--porcelain"):
logger.warning("dify-saas at %s has uncommitted changes; the snapshot will not match its commit", root)
branch = _git(root, "rev-parse", "--abbrev-ref", "HEAD")
if branch != "main":
logger.warning("dify-saas is on %s, not main", branch)
try:
behind = int(_git(root, "rev-list", "--count", "HEAD..origin/main"))
except subprocess.CalledProcessError:
logger.warning("dify-saas has no origin/main to compare against")
else:
if behind:
logger.warning("dify-saas is %d commits behind origin/main (as of its last fetch)", behind)
return _git(root, "rev-parse", "HEAD")
def _rewrite_refs(node: Any, prefix: str) -> Any:
if isinstance(node, dict):
return {
key: (_strip_ref(value, prefix) if key == "$ref" else _rewrite_refs(value, prefix))
for key, value in node.items()
}
if isinstance(node, list):
return [_rewrite_refs(item, prefix) for item in node]
return node
def _strip_ref(ref: str, prefix: str) -> str:
head, _, name = ref.rpartition("/")
return f"{head}/{name.removeprefix(prefix)}"
def _ref_names(node: Any) -> set[str]:
if isinstance(node, dict):
names = {node["$ref"].rpartition("/")[2]} if "$ref" in node else set()
for value in node.values():
names |= _ref_names(value)
return names
if isinstance(node, list):
names: set[str] = set()
for item in node:
names |= _ref_names(item)
return names
return set()
def _annotate_presence(schema: JsonObject) -> JsonObject:
properties: dict[str, JsonObject] = schema.get("properties", {})
always_set = set(schema.get("required", []))
annotated: dict[str, JsonObject] = {}
required: list[str] = []
for name, field in properties.items():
if "$ref" in field and name not in always_set:
# OpenAPI 3.0 ignores siblings of $ref, so nullable refs are wrapped in allOf.
annotated[name] = {"allOf": [{"$ref": field["$ref"]}], "nullable": True}
else:
annotated[name] = field
if "$ref" in field or not field.get("nullable"):
required.append(name)
result = {**schema, "properties": annotated}
if required:
result["required"] = required
return result
def proto_service(operation_id: str) -> str:
prefix, separator, method = operation_id.partition("_")
if not separator or not prefix or not method:
raise GenerationError(f"operationId {operation_id!r} has no `<Service>_` prefix")
return prefix
def error_reason_schema(proto: str) -> str:
return f"{proto}{ERROR_REASON_SUFFIX}"
def build_snapshot(service: Service, source: JsonObject, commit: str) -> JsonObject:
prefix = SCHEMA_PREFIXES[service]
source_paths: dict[str, JsonObject] = source.get("paths", {})
paths: dict[str, JsonObject] = {}
for allowed in ALLOWED_OPERATIONS[service]:
operation = source_paths.get(allowed.path, {}).get(allowed.method.lower())
if operation is None:
raise GenerationError(f"{service}: {allowed.method} {allowed.path} is not in the dify-saas spec")
paths.setdefault(allowed.path, {})[allowed.method.lower()] = _rewrite_refs(operation, prefix)
source_schemas: dict[str, JsonObject] = {
name.removeprefix(prefix): _rewrite_refs(schema, prefix)
for name, schema in source.get("components", {}).get("schemas", {}).items()
}
error_schemas = {ERROR_SCHEMA}
for allowed in ALLOWED_OPERATIONS[service]:
operation_id = paths[allowed.path][allowed.method.lower()].get("operationId", "")
reason_schema = error_reason_schema(proto_service(operation_id))
if reason_schema not in source_schemas:
raise GenerationError(
f"{service}: {allowed.method} {allowed.path} ({operation_id}) has no "
f"{prefix}{reason_schema} schema in the dify-saas spec"
)
error_schemas.add(reason_schema)
if ERROR_SCHEMA not in source_schemas:
raise GenerationError(f"{service}: error schema {prefix}{ERROR_SCHEMA} is not in the dify-saas spec")
reachable: set[str] = set()
pending = _ref_names(paths) | error_schemas
while pending:
name = pending.pop()
if name in reachable:
continue
if name not in source_schemas:
raise GenerationError(f"{service}: schema {name} is referenced but not defined")
reachable.add(name)
pending |= _ref_names(source_schemas[name])
return {
"openapi": source["openapi"],
"info": source.get("info", {}),
SNAPSHOT_COMMIT_KEY: commit,
"paths": paths,
"components": {"schemas": {name: _annotate_presence(source_schemas[name]) for name in sorted(reachable)}},
}
def take_snapshots() -> dict[Service, JsonObject]:
root = dify_saas_root()
if not (root / "pkg" / "apis").is_dir():
raise GenerationError(f"dify-saas checkout not found at {root}; set DIFY_SAAS_REPO")
commit = dify_saas_commit(root)
snapshots: dict[Service, JsonObject] = {}
for service in SERVICES:
source = yaml.safe_load((root / "pkg" / "apis" / service / "openapi.yaml").read_text())
snapshots[service] = build_snapshot(service, source, commit)
return snapshots
def render_snapshot(snapshot: JsonObject) -> str:
return json.dumps(snapshot, indent=2, ensure_ascii=False) + "\n"
def load_snapshot(service: Service) -> JsonObject:
snapshot: JsonObject = json.loads(snapshot_path(service).read_text())
operations = {
(method.upper(), path) for path, item in snapshot["paths"].items() for method in item if method in HTTP_METHODS
}
allowed = {(op.method, op.path) for op in ALLOWED_OPERATIONS[service]}
if operations != allowed:
raise GenerationError(
f"{service} snapshot operations differ from ALLOWED_OPERATIONS: "
f"missing {sorted(allowed - operations)}, unlisted {sorted(operations - allowed)}; rerun the snapshot"
)
return snapshot
# --- generation -----------------------------------------------------------------------------------
def snake_case(name: str) -> str:
words = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", name)
words = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1_\2", words)
return re.sub(r"[^0-9a-zA-Z]+", "_", words).strip("_").lower()
def python_name(name: str) -> str:
candidate = snake_case(name)
if not candidate.isidentifier() or keyword.iskeyword(candidate) or candidate in BASE_MODEL_ATTRIBUTES:
candidate = f"{candidate}_"
return candidate
def operation_function_name(operation_id: str, service: Service) -> str:
tag, _, method = operation_id.partition("_")
tag = tag.removeprefix(service.capitalize())
return python_name(f"{tag}_{method}")
def python_type(schema: JsonObject, context: str) -> str:
if "allOf" in schema:
parts = schema["allOf"]
if len(parts) != 1:
raise GenerationError(f"{context}: only single-element allOf is supported")
return python_type(parts[0], context)
if "$ref" in schema:
return schema["$ref"].rpartition("/")[2]
schema_type = schema.get("type")
if "enum" in schema:
if schema_type != "string":
raise GenerationError(f"{context}: only string enums are supported")
return f"Literal[{', '.join(json.dumps(value) for value in schema['enum'])}]"
if schema_type == "integer" or (schema_type == "string" and schema.get("format") in {"int64", "uint64"}):
return "int"
if schema_type == "string" and schema.get("format") == "date-time":
return "datetime"
if schema_type == "string":
return "str"
if schema_type == "number":
return "float"
if schema_type == "boolean":
return "bool"
if schema_type == "array":
return f"list[{python_type(schema['items'], context)}]"
if schema_type == "object" and isinstance(schema.get("additionalProperties"), dict):
return f"dict[str, {python_type(schema['additionalProperties'], context)}]"
raise GenerationError(f"{context}: unsupported schema {json.dumps(schema)}; give it a named message type")
def field_type(schema: JsonObject, required: bool, context: str) -> str:
annotation = python_type(schema, context)
if schema.get("nullable") or not required:
annotation = f"{annotation} | None"
return annotation
def model_order(schemas: dict[str, JsonObject]) -> list[str]:
ordered: list[str] = []
visiting: set[str] = set()
def visit(name: str) -> None:
if name in ordered:
return
if name in visiting:
raise GenerationError(f"recursive schema {name} is not supported")
visiting.add(name)
for dependency in sorted(_ref_names(schemas[name])):
visit(dependency)
visiting.discard(name)
ordered.append(name)
for name in sorted(schemas):
visit(name)
return ordered
def check_error_body(schema: JsonObject) -> None:
properties: dict[str, JsonObject] = schema.get("properties", {})
expected = {
"code": {"type": "integer", "format": "int32"},
"reason": {"type": "string"},
"message": {"type": "string"},
"metadata": {"type": "object", "additionalProperties": {"type": "string"}},
}
actual = {
name: {key: value for key, value in field.items() if key != "description"} for name, field in properties.items()
}
if actual != expected:
raise GenerationError(f"Error is now {json.dumps(actual)}; update UpstreamErrorBody and the transport")
def render_schema(name: str, schema: JsonObject) -> list[str]:
if name == ERROR_SCHEMA:
check_error_body(schema)
return []
if "enum" in schema:
return [f"type {name} = {python_type(schema, name)}"]
return render_model(name, schema)
def render_model(name: str, schema: JsonObject) -> list[str]:
if schema.get("type") != "object":
raise GenerationError(f"schema {name}: only object schemas can be top-level models")
required = set(schema.get("required", []))
lines = [f"class {name}(BillingModel):"]
properties: dict[str, JsonObject] = schema.get("properties", {})
for wire_name, field in properties.items():
attribute = python_name(wire_name)
annotation = field_type(field, wire_name in required, f"{name}.{wire_name}")
default = "" if wire_name in required else "None"
if attribute != wire_name:
arguments = ", ".join([*([default] if default else []), f"alias={json.dumps(wire_name)}"])
lines.append(f" {attribute}: {annotation} = Field({arguments})")
else:
lines.append(f" {attribute}: {annotation}{f' = {default}' if default else ''}")
if not properties:
lines.append(" pass")
return lines
def _json_schema_ref(content: JsonObject | None, context: str) -> str:
schema = (content or {}).get("application/json", {}).get("schema")
if not schema or "$ref" not in schema:
raise GenerationError(f"{context}: expected an application/json $ref schema")
return schema["$ref"].rpartition("/")[2]
def render_operation(service: Service, allowed: AllowedOperation, operation: JsonObject) -> list[str]:
context = f"{allowed.method} {allowed.path}"
function_name = operation_function_name(operation["operationId"], service)
response = _json_schema_ref(operation.get("responses", {}).get("200", {}).get("content"), context)
arguments: list[str] = []
path_params: list[str] = []
query_params: list[str] = []
template = allowed.path.removeprefix(PATH_VERSION_PREFIX)
for parameter in operation.get("parameters", []):
wire_name = parameter["name"]
argument = python_name(wire_name)
annotation = python_type(parameter["schema"], f"{context} {wire_name}")
arguments.append(f"{argument}: {annotation}")
if parameter["in"] == "path":
template = template.replace(f"{{{wire_name}}}", f"{{{argument}}}")
path_params.append(f"{json.dumps(argument)}: {argument}")
elif parameter["in"] == "query":
query_params.append(f"{json.dumps(wire_name)}: {argument}")
else:
raise GenerationError(f"{context}: unsupported parameter location {parameter['in']}")
body = "None"
if "requestBody" in operation:
arguments.append(f"body: {_json_schema_ref(operation['requestBody'].get('content'), context)}")
body = "body"
if len({argument.partition(":")[0] for argument in arguments}) != len(arguments):
raise GenerationError(f"{context}: argument names collide")
signature = f"def {function_name}({'*, ' + ', '.join(arguments) if arguments else ''}) -> {response}:"
docstring = f' """{allowed.method} {allowed.path} ({operation["operationId"]})."""'
if len(docstring) > LINE_LENGTH:
docstring = f' """{allowed.method} {allowed.path}\n\n {operation["operationId"]}\n """'
return [
signature,
docstring,
" return request(",
f" Upstream.{allowed.upstream},",
f" {json.dumps(allowed.method)},",
f" {json.dumps(template)},",
f" response_model={response},",
f" errors={service_errors_name(proto_service(operation['operationId']))},",
f" path_params={{{', '.join(path_params)}}},",
f" params={{{', '.join(query_params)}}},",
f" body={body},",
" )",
]
def service_errors_name(proto: str) -> str:
return f"{snake_case(proto).upper()}_ERRORS"
def proto_services(service: Service, snapshot: JsonObject) -> list[str]:
protos: list[str] = []
for allowed in ALLOWED_OPERATIONS[service]:
proto = proto_service(snapshot["paths"][allowed.path][allowed.method.lower()]["operationId"])
if proto not in protos:
protos.append(proto)
return protos
def check_error_schemas(service: Service, protos: list[str], schemas: dict[str, JsonObject]) -> None:
expected = {error_reason_schema(proto) for proto in protos}
actual = {name for name in schemas if name.endswith(ERROR_REASON_SUFFIX)}
if actual != expected:
raise GenerationError(
f"{service} snapshot reason schemas differ from its operations' services: "
f"missing {sorted(expected - actual)}, unused {sorted(actual - expected)}; rerun the snapshot"
)
generated = {f"{proto}{suffix}" for proto in protos for suffix in ERROR_CLASS_SUFFIXES}
if collisions := sorted(generated & set(schemas)):
raise GenerationError(f"{service}: generated error classes collide with schemas {collisions}")
def render_errors(proto: str) -> list[str]:
reason = error_reason_schema(proto)
guard = f"_is_{snake_case(proto)}_error_reason"
return f'''def {guard}(value: str) -> TypeIs[{reason}]:
return value in get_args({reason}.__value__)
class {proto}UpstreamError(BillingUpstreamError):
"""A failed {proto} call, with its reason typed by `{reason}`."""
def __init__(
self,
*args: object,
status_code: int | None = None,
body: UpstreamErrorBody | None = None,
reason: {reason} | UnknownErrorReason | None = None,
) -> None:
super().__init__(*args, status_code=status_code, body=body)
self.reason = reason
class {proto}UpstreamUnavailableError({proto}UpstreamError, BillingUpstreamUnavailableError):
pass
class {proto}UpstreamInvalidResponseError({proto}UpstreamError, BillingUpstreamInvalidResponseError):
pass
{service_errors_name(proto)} = ServiceErrors(
is_reason={guard},
unavailable={proto}UpstreamUnavailableError,
invalid_response={proto}UpstreamInvalidResponseError,
)'''.splitlines()
def render_module(service: Service, snapshot: JsonObject) -> str:
schemas: dict[str, JsonObject] = snapshot["components"]["schemas"]
protos = proto_services(service, snapshot)
check_error_schemas(service, protos, schemas)
blocks = [block for name in model_order(schemas) if (block := render_schema(name, schemas[name]))]
blocks.extend(render_errors(proto) for proto in protos)
for allowed in ALLOWED_OPERATIONS[service]:
operation = snapshot["paths"][allowed.path][allowed.method.lower()]
blocks.append(render_operation(service, allowed, operation))
body = "\n\n\n".join("\n".join(block) for block in blocks)
header = [
f"# Code generated by {GENERATOR} from {service}.openapi.json. DO NOT EDIT.",
f"# dify-saas commit: {snapshot[SNAPSHOT_COMMIT_KEY]}",
"",
"from datetime import datetime",
"from typing import Literal, TypeIs, get_args",
"",
"from pydantic import Field",
"",
"from services.billing_client.transport import BillingModel, ServiceErrors, Upstream, request",
"from services.errors.billing import (",
" BillingUpstreamError,",
" BillingUpstreamInvalidResponseError,",
" BillingUpstreamUnavailableError,",
" UnknownErrorReason,",
" UpstreamErrorBody,",
")",
]
return format_python("\n".join(header) + "\n\n\n" + body + "\n", module_path(service))
def format_python(source: str, path: Path) -> str:
ruff = [sys.executable, "-m", "ruff"]
fixed = subprocess.run(
[*ruff, "check", "--fix", "--quiet", "--select", "F401,I", "--stdin-filename", str(path), "-"],
input=source,
capture_output=True,
text=True,
check=True,
cwd=API_ROOT,
).stdout
return subprocess.run(
[*ruff, "format", "--quiet", "--stdin-filename", str(path), "-"],
input=fixed,
capture_output=True,
text=True,
check=True,
cwd=API_ROOT,
).stdout
# --- entry point ----------------------------------------------------------------------------------
def expected_files(snapshots: dict[Service, JsonObject]) -> dict[Path, str]:
files: dict[Path, str] = {}
for service, snapshot in snapshots.items():
files[snapshot_path(service)] = render_snapshot(snapshot)
files[module_path(service)] = render_module(service, snapshot)
return files
def check(files: dict[Path, str]) -> int:
drifted = 0
for path, expected in files.items():
actual = path.read_text() if path.exists() else ""
if actual != expected:
drifted += 1
diff = difflib.unified_diff(
actual.splitlines(keepends=True),
expected.splitlines(keepends=True),
fromfile=str(path.relative_to(REPO_ROOT)),
tofile="regenerated",
)
sys.stderr.writelines(diff)
if drifted:
logger.error("billing client is out of date; regenerate with `%s --from-snapshot`", GENERATOR)
return 1 if drifted else 0
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
mode = parser.add_mutually_exclusive_group()
mode.add_argument("--from-snapshot", action="store_true", help="generate from the committed snapshots")
mode.add_argument("--check", action="store_true", help="fail if regenerating the committed snapshots differs")
args = parser.parse_args()
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
try:
if args.check or args.from_snapshot:
snapshots = {service: load_snapshot(service) for service in SERVICES}
else:
snapshots = take_snapshots()
files = expected_files(snapshots)
except GenerationError as error:
sys.stderr.write(f"error: {error}\n")
return 1
if args.check:
return check(files)
for path, content in files.items():
path.write_text(content)
logger.info("wrote %s", path.relative_to(REPO_ROOT))
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,103 @@
"""Generate FastOpenAPI OpenAPI 3.1 specs without booting the full backend."""
from __future__ import annotations
import argparse
import json
import logging
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Any
API_ROOT = Path(__file__).resolve().parents[1]
if str(API_ROOT) not in sys.path:
sys.path.insert(0, str(API_ROOT))
# Import after script path setup.
from dev.generate_swagger_specs import apply_runtime_defaults # noqa: E402
# Import after script path setup.
from libs.flask_restx_compat import sort_openapi_arrays # noqa: E402
logger = logging.getLogger(__name__)
@dataclass(frozen=True)
class FastOpenApiSpecTarget:
route: str
filename: str
FASTOPENAPI_SPEC_TARGETS: tuple[FastOpenApiSpecTarget, ...] = (
FastOpenApiSpecTarget(route="/fastopenapi/openapi.json", filename="fastopenapi-console-openapi.json"),
)
def create_fastopenapi_spec_app():
"""Build a minimal Flask app that only mounts FastOpenAPI docs routes."""
apply_runtime_defaults()
from app_factory import create_flask_app_with_configs
from extensions import ext_fastopenapi
app = create_flask_app_with_configs()
ext_fastopenapi.init_app(app)
return app
def export_fastopenapi_specs() -> dict[str, dict[str, Any]]:
"""Build FastOpenAPI documents in memory without writing generated files."""
app = create_fastopenapi_spec_app()
client = app.test_client()
documents = {}
for target in FASTOPENAPI_SPEC_TARGETS:
response = client.get(target.route)
if response.status_code != 200:
raise RuntimeError(f"failed to fetch {target.route}: {response.status_code}")
payload = sort_openapi_arrays(response.get_json())
if not isinstance(payload, dict):
raise RuntimeError(f"unexpected response payload for {target.route}")
documents[target.filename] = payload
return documents
def generate_fastopenapi_specs(output_dir: Path) -> list[Path]:
"""Write FastOpenAPI specs to `output_dir` and return the written paths."""
output_dir.mkdir(parents=True, exist_ok=True)
written_paths: list[Path] = []
for filename, payload in export_fastopenapi_specs().items():
output_path = output_dir / filename
output_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
written_paths.append(output_path)
return written_paths
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"-o",
"--output-dir",
type=Path,
default=Path("openapi"),
help="Directory where the OpenAPI JSON files will be written.",
)
return parser.parse_args()
def main() -> int:
args = parse_args()
written_paths = generate_fastopenapi_specs(args.output_dir)
for path in written_paths:
logger.debug(path)
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,660 @@
"""Verify Dify against checked-in contracts from an exact standalone KnowledgeFS commit.
CI uses only api/knowledge-fs-contract: no private source checkout, Node runtime, or network
is needed for --check. Intentional updates export from a clean --knowledge-fs-root checkout,
record its full commit/tree provenance, and pin every artifact and Dify operation manifest.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import subprocess
import sys
import tempfile
from pathlib import Path
from typing import Any, Literal, TypedDict, cast
import jwt
from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey
from jwt.algorithms import RSAAlgorithm
API_ROOT = Path(__file__).resolve().parents[1]
if str(API_ROOT) not in sys.path:
sys.path.insert(0, str(API_ROOT))
# Import after script path setup.
from dev.knowledge_fs_product_contract import ( # noqa: E402
capability_operation_runtime_contracts,
parse_capability_operation_policy,
parse_product_operation_gap_manifest,
parse_product_operation_manifest,
product_operation_runtime_contracts,
validate_product_operation_contracts,
)
WORKSPACE_ROOT = API_ROOT.parent
PIN_RELATIVE_PATH = Path("api/knowledge-fs-contract")
OPENAPI_FILENAME = "knowledge-fs.openapi.json"
CAPABILITY_POLICY_FILENAME = "dify-capability-v2-operations.json"
CAPABILITY_V2_AUTH_MANIFEST_RELATIVE_PATH = Path("contracts/dify-capability-v2-auth-profile.json")
CAPABILITY_V2_AUTH_TEST_VECTOR_RELATIVE_PATH = Path("contracts/dify-capability-v2-test-vector.json")
PROVENANCE_FILENAME = "source-provenance.json"
LOCK_RELATIVE_PATH = Path("api/knowledge-fs-contract.lock.json")
PRODUCT_OPERATIONS_RELATIVE_PATH = Path("api/knowledge-fs-product-operations.json")
PRODUCT_OPERATION_GAPS_RELATIVE_PATH = Path("api/knowledge-fs-product-operation-gaps.json")
OPENAPI_METHODS = ("delete", "get", "head", "options", "patch", "post", "put", "trace")
PROXY_METHODS = frozenset({"delete", "get", "patch", "post", "put"})
LOCK_SCHEMA_VERSION = 6
class ContractDeclaration(TypedDict):
"""KnowledgeFS transport contract declared by one Dify Console registry entry."""
operation_id: str
method: str
path: str
required_scope: str | None
response_kind: str
max_response_bytes: int
request_headers: tuple[str, ...]
response_headers: tuple[str, ...]
response_media_types: tuple[str, ...]
class ContractLock(TypedDict):
"""Content-addressed contract inputs that must move together."""
schemaVersion: int
sourceCommit: str
sourceTree: str
sourceProvenanceSha256: str
capabilityPolicySha256: str
openapiSha256: str
capabilityV2AuthManifestSha256: str
capabilityV2AuthTestVectorSha256: str
productOperationManifestSha256: str
productOperationGapManifestSha256: str
type DeclarationField = Literal[
"method",
"path",
"required_scope",
"response_kind",
"max_response_bytes",
"request_headers",
"response_headers",
"response_media_types",
]
DECLARATION_FIELDS: tuple[DeclarationField, ...] = (
"method",
"path",
"required_scope",
"response_kind",
"max_response_bytes",
"request_headers",
"response_headers",
"response_media_types",
)
def main() -> None:
"""Update an exact source pin or verify the self-contained Dify contract boundary."""
parser = argparse.ArgumentParser()
mode = parser.add_mutually_exclusive_group(required=True)
mode.add_argument("--check", action="store_true")
mode.add_argument("--update-lock", action="store_true")
parser.add_argument("--workspace-root", type=Path, default=WORKSPACE_ROOT)
parser.add_argument("--knowledge-fs-root", type=Path, help="Clean standalone source checkout (updates only)")
args = parser.parse_args()
if args.check and args.knowledge_fs_root is not None:
parser.error("--check reads pinned artifacts only; --knowledge-fs-root is for --update-lock")
if args.update_lock and args.knowledge_fs_root is None:
parser.error("--update-lock requires --knowledge-fs-root pointing to a clean committed checkout")
workspace_root = args.workspace_root.resolve()
pin_root = workspace_root / PIN_RELATIVE_PATH
if args.update_lock:
with tempfile.TemporaryDirectory(prefix="dify-knowledge-fs-contract-") as directory:
exported_root = Path(directory)
export_contract(args.knowledge_fs_root.resolve(), exported_root)
expected_lock = validate_pinned_contract(workspace_root, exported_root)
pin_root.mkdir(parents=True, exist_ok=True)
for path in exported_root.iterdir():
(pin_root / path.name).write_bytes(path.read_bytes())
(workspace_root / LOCK_RELATIVE_PATH).write_text(json.dumps(expected_lock, indent=2) + "\n")
return
expected_lock = validate_pinned_contract(workspace_root, pin_root)
received_lock = parse_contract_lock(load_json_object(workspace_root / LOCK_RELATIVE_PATH))
for field, expected_value in expected_lock.items():
received_value = dict(received_lock)[field]
if received_value != expected_value:
raise RuntimeError(
f"KnowledgeFS contract lock field {field} drifted: "
f"expected {expected_value!r}, received {received_value!r}. "
"Run --update-lock --knowledge-fs-root intentionally after reviewing source and contract changes."
)
def export_contract(knowledge_fs_root: Path, output_root: Path) -> None:
"""Export only a clean committed source tree so provenance identifies the actual input."""
ensure_clean_knowledge_fs_worktree(knowledge_fs_root)
provenance = {
"schemaVersion": 1,
"repository": "https://github.com/langgenius/knowledge-fs",
"commit": run("git", "rev-parse", "HEAD", cwd=knowledge_fs_root).strip(),
"tree": run("git", "rev-parse", "HEAD^{tree}", cwd=knowledge_fs_root).strip(),
}
output_root.mkdir(parents=True, exist_ok=True)
for command, filename in (("openapi:export", OPENAPI_FILENAME), ("capability:export", CAPABILITY_POLICY_FILENAME)):
subprocess.run(
["pnpm", command, "--", "--output", str(output_root / filename)], cwd=knowledge_fs_root, check=True
)
for relative_path in (CAPABILITY_V2_AUTH_MANIFEST_RELATIVE_PATH, CAPABILITY_V2_AUTH_TEST_VECTOR_RELATIVE_PATH):
(output_root / relative_path.name).write_bytes((knowledge_fs_root / relative_path).read_bytes())
# Detect source changes made while the export subprocesses ran.
ensure_clean_knowledge_fs_worktree(knowledge_fs_root)
if run("git", "rev-parse", "HEAD", cwd=knowledge_fs_root).strip() != provenance["commit"]:
raise RuntimeError("KnowledgeFS source commit changed during contract export")
(output_root / PROVENANCE_FILENAME).write_text(json.dumps(provenance, indent=2) + "\n")
def ensure_clean_knowledge_fs_worktree(knowledge_fs_root: Path) -> None:
"""Require the standalone checkout's working tree and index to match its commit."""
if run("git", "status", "--porcelain", "--untracked-files=all", cwd=knowledge_fs_root).strip():
raise RuntimeError("KnowledgeFS source contains staged, unstaged or untracked changes; commit before export")
def validate_pinned_contract(workspace_root: Path, pin_root: Path) -> ContractLock:
"""Validate pinned bytes against executable Python declarations without invoking source tools."""
openapi_path = pin_root / OPENAPI_FILENAME
policy_path = pin_root / CAPABILITY_POLICY_FILENAME
auth_path = pin_root / CAPABILITY_V2_AUTH_MANIFEST_RELATIVE_PATH.name
vector_path = pin_root / CAPABILITY_V2_AUTH_TEST_VECTOR_RELATIVE_PATH.name
provenance_path = pin_root / PROVENANCE_FILENAME
product_path = workspace_root / PRODUCT_OPERATIONS_RELATIVE_PATH
gaps_path = workspace_root / PRODUCT_OPERATION_GAPS_RELATIVE_PATH
for path in (openapi_path, policy_path, auth_path, vector_path, provenance_path, product_path, gaps_path):
if not path.is_file():
raise RuntimeError(f"KnowledgeFS pinned contract input is missing: {path}")
auth_manifest = load_json_object(auth_path)
validate_capability_v2_auth_manifest(auth_manifest)
validate_capability_v2_auth_test_vector(load_json_object(vector_path), auth_manifest)
provenance = load_json_object(provenance_path)
validate_upstream_provenance(provenance)
document = load_json_object(openapi_path)
validate_product_operation_contracts(
capability_operations=capability_operation_runtime_contracts(),
capability_policy=parse_capability_operation_policy(load_json_object(policy_path)),
document=document,
gap_manifest=parse_product_operation_gap_manifest(load_json_object(gaps_path)),
manifest=parse_product_operation_manifest(load_json_object(product_path)),
product_operations=product_operation_runtime_contracts(),
)
validate_declarations(document, console_contract_declarations())
return {
"schemaVersion": LOCK_SCHEMA_VERSION,
"sourceCommit": provenance["commit"],
"sourceTree": provenance["tree"],
"sourceProvenanceSha256": sha256(provenance_path.read_bytes()),
"openapiSha256": sha256(openapi_path.read_bytes()),
"capabilityPolicySha256": sha256(policy_path.read_bytes()),
"capabilityV2AuthManifestSha256": sha256(auth_path.read_bytes()),
"capabilityV2AuthTestVectorSha256": sha256(vector_path.read_bytes()),
"productOperationManifestSha256": sha256(product_path.read_bytes()),
"productOperationGapManifestSha256": sha256(gaps_path.read_bytes()),
}
def load_json_object(path: Path) -> dict[str, Any]:
"""Load a JSON object and reject arrays/scalars at contract boundaries."""
value = json.loads(path.read_text())
if not isinstance(value, dict):
raise ValueError(f"KnowledgeFS contract file must contain a JSON object: {path}")
return cast(dict[str, Any], value)
def parse_contract_lock(value: dict[str, Any]) -> ContractLock:
"""Validate the compact, non-self-referential contract lock schema."""
expected_fields = {
"capabilityV2AuthManifestSha256",
"capabilityV2AuthTestVectorSha256",
"openapiSha256",
"productOperationGapManifestSha256",
"productOperationManifestSha256",
"schemaVersion",
"sourceCommit",
"sourceTree",
"sourceProvenanceSha256",
"capabilityPolicySha256",
}
if set(value) != expected_fields:
raise ValueError(f"KnowledgeFS contract lock fields must be exactly {sorted(expected_fields)}")
if value.get("schemaVersion") != LOCK_SCHEMA_VERSION:
raise ValueError(f"KnowledgeFS contract lock schemaVersion must be {LOCK_SCHEMA_VERSION}")
for field in (
"capabilityV2AuthManifestSha256",
"capabilityV2AuthTestVectorSha256",
"openapiSha256",
"productOperationGapManifestSha256",
"productOperationManifestSha256",
"sourceCommit",
"sourceTree",
"sourceProvenanceSha256",
"capabilityPolicySha256",
):
field_value = value.get(field)
expected_length = 40 if field in {"sourceCommit", "sourceTree"} else 64
if (
not isinstance(field_value, str)
or len(field_value) != expected_length
or any(character not in "0123456789abcdef" for character in field_value)
):
raise ValueError(f"KnowledgeFS contract lock field {field} has an invalid digest")
return cast(ContractLock, value)
def validate_required_product_operations(document: dict[str, Any], required_operation_ids: list[str]) -> None:
"""Fail when the pinned KFS OpenAPI omits an operation required by the Dify product."""
available_operation_ids = [
operation_id
for path_item in document.get("paths", {}).values()
for method in OPENAPI_METHODS
if isinstance(path_item, dict)
for operation in (path_item.get(method),)
if isinstance(operation, dict)
for operation_id in (operation.get("operationId"),)
if isinstance(operation_id, str) and operation_id
]
for operation_id in required_operation_ids:
count = available_operation_ids.count(operation_id)
if count != 1:
raise ValueError(
f"KnowledgeFS OpenAPI required product operation {operation_id} must occur exactly once; found {count}"
)
def validate_capability_v2_auth_manifest(value: dict[str, Any]) -> None:
"""Validate the active production RS256 profile consumed by both Dify and KnowledgeFS."""
expected_fields = {
"active",
"audience",
"callerProfiles",
"claimBindings",
"issuer",
"lifecycle",
"maxTtlSeconds",
"productionReady",
"profileId",
"protectedHeader",
"requiredClaims",
"resourceContract",
"runtimeAssembly",
"schemaVersion",
"signatureAlgorithms",
"tokenKind",
}
if set(value) != expected_fields:
raise ValueError(f"KnowledgeFS Capability v2 auth manifest fields must be exactly {sorted(expected_fields)}")
fixed_values = {
"active": True,
"audience": "knowledge-fs",
"issuer": "dify-control-plane",
"lifecycle": "active",
"maxTtlSeconds": 60,
"productionReady": True,
"profileId": "dify-capability-v2",
"schemaVersion": 3,
"signatureAlgorithms": ["RS256"],
"tokenKind": "jwt",
}
for field, expected in fixed_values.items():
if value.get(field) != expected:
raise ValueError(f"KnowledgeFS Capability v2 auth manifest field {field} must be {expected!r}")
if value.get("protectedHeader") != {
"algorithm": "RS256",
"keyIdClaim": "kid",
"keyIdRequired": True,
"type": "JWT",
}:
raise ValueError("KnowledgeFS Capability v2 protected header contract is invalid")
required_claims = [
"action",
"actor",
"aud",
"authz_revision",
"azp",
"caller_kind",
"cap_ver",
"content_policy_revision",
"content_scope_ids",
"control_space_id",
"exp",
"grant_id",
"iat",
"iss",
"jti",
"namespace_id",
"nbf",
"resource",
"sub",
"trace_id",
]
if value.get("requiredClaims") != required_claims:
raise ValueError("KnowledgeFS Capability v2 required claims are invalid")
if value.get("claimBindings") != {
"action": "action",
"callerKind": "caller_kind",
"controlSpace": "control_space_id",
"namespace": "namespace_id",
"resource": "resource",
"resourceParent": "resource.parent_id",
"subject": "sub",
}:
raise ValueError("KnowledgeFS Capability v2 claim bindings are invalid")
if value.get("resourceContract") != {
"fields": ["id", "parent_id", "type"],
"parentForbiddenFor": ["namespace", "knowledge_space"],
"parentRequiredFor": ["document", "job", "query", "research_task", "source", "upload_session"],
}:
raise ValueError("KnowledgeFS Capability v2 resource contract is invalid")
if value.get("callerProfiles") != {
"agent": {"authorizedParty": "dify-agent", "subjectPrefix": "dify-app:"},
"interactive": {"authorizedParty": "dify-console", "subjectPrefix": "dify-account:"},
"internal_worker": {"authorizedParty": "dify-worker", "subjectPrefix": "dify-worker:"},
"mcp": {"authorizedParty": "dify-mcp", "subjectPrefix": "dify-mcp-session:"},
"service": {"authorizedParty": "dify-service-api", "subjectPrefix": "dify-dataset-api-key:"},
"workflow": {"authorizedParty": "dify-workflow", "subjectPrefix": "dify-app:"},
}:
raise ValueError("KnowledgeFS Capability v2 caller profiles are invalid")
if value.get("runtimeAssembly") != {
"failClosed": True,
"keySelection": "kid",
"maximumPublishedKeys": 3,
"verificationKeySource": "jwks",
}:
raise ValueError("KnowledgeFS Capability v2 runtime assembly is invalid")
def validate_capability_v2_auth_test_vector(
value: dict[str, Any],
manifest: dict[str, Any],
) -> None:
"""Verify the deterministic public-key vector and every security-sensitive binding."""
expected_fields = {
"algorithm",
"audience",
"expectedClaims",
"expectedPrincipal",
"issuer",
"operation",
"profileId",
"protectedHeader",
"publicJwk",
"schemaVersion",
"testOnly",
"token",
"ttlSeconds",
}
if set(value) != expected_fields:
raise ValueError(f"KnowledgeFS Capability v2 test vector fields must be exactly {sorted(expected_fields)}")
if (
value.get("schemaVersion") != 2
or value.get("profileId") != manifest.get("profileId")
or value.get("testOnly") is not True
or value.get("algorithm") != "RS256"
or value.get("issuer") != manifest.get("issuer")
or value.get("audience") != manifest.get("audience")
or value.get("ttlSeconds") != manifest.get("maxTtlSeconds")
):
raise ValueError("KnowledgeFS Capability v2 test vector does not match the active profile")
protected_header = value.get("protectedHeader")
if not isinstance(protected_header, dict) or set(protected_header) != {"alg", "kid", "typ"}:
raise ValueError("KnowledgeFS Capability v2 test vector protected header is invalid")
kid = protected_header.get("kid")
if protected_header.get("alg") != "RS256" or protected_header.get("typ") != "JWT" or not _is_non_blank(kid):
raise ValueError("KnowledgeFS Capability v2 test vector protected header is invalid")
public_jwk = value.get("publicJwk")
if not isinstance(public_jwk, dict) or set(public_jwk) != {"alg", "e", "kid", "kty", "n", "use"}:
raise ValueError("KnowledgeFS Capability v2 test vector public JWK is invalid")
if (
public_jwk.get("alg") != "RS256"
or public_jwk.get("kid") != kid
or public_jwk.get("kty") != "RSA"
or public_jwk.get("use") != "sig"
or not _is_non_blank(public_jwk.get("e"))
or not _is_non_blank(public_jwk.get("n"))
):
raise ValueError("KnowledgeFS Capability v2 test vector public JWK is invalid")
claims = value.get("expectedClaims")
required_claims = manifest.get("requiredClaims")
if not isinstance(claims, dict) or not isinstance(required_claims, list) or set(claims) != set(required_claims):
raise ValueError("KnowledgeFS Capability v2 test vector claims do not match the active profile")
operation = value.get("operation")
if not isinstance(operation, dict) or set(operation) != {"action", "method", "operationId", "requestPath"}:
raise ValueError("KnowledgeFS Capability v2 test vector operation is invalid")
resource = claims.get("resource")
if not isinstance(resource, dict) or set(resource) != {"id", "parent_id", "type"}:
raise ValueError("KnowledgeFS Capability v2 test vector resource is invalid")
expected_operation = {
"action": "documents.read",
"method": "GET",
"operationId": "getDocument",
"requestPath": "/knowledge-spaces/space-contract-vector/documents/document-contract-vector",
}
if operation != expected_operation:
raise ValueError("KnowledgeFS Capability v2 test vector operation binding is invalid")
exact_claims = {
"action": operation["action"],
"aud": value["audience"],
"caller_kind": "interactive",
"cap_ver": 2,
"control_space_id": "control-space-contract-vector",
"iss": value["issuer"],
"namespace_id": "workspace-contract-vector",
"resource": {
"id": "document-contract-vector",
"parent_id": "space-contract-vector",
"type": "document",
},
"sub": "dify-account:account-contract-vector",
}
for field, expected in exact_claims.items():
if claims.get(field) != expected:
raise ValueError(f"KnowledgeFS Capability v2 test vector claim {field} is invalid")
if claims.get("actor") != claims["sub"] or claims.get("azp") != "dify-console":
raise ValueError("KnowledgeFS Capability v2 test vector caller binding is invalid")
issued_at = claims.get("iat")
not_before = claims.get("nbf")
expires_at = claims.get("exp")
if (
not isinstance(issued_at, int)
or isinstance(issued_at, bool)
or not_before != issued_at
or not isinstance(expires_at, int)
or isinstance(expires_at, bool)
or expires_at - issued_at != value["ttlSeconds"]
):
raise ValueError("KnowledgeFS Capability v2 test vector TTL is invalid")
expected_principal = {
"callerKind": claims["caller_kind"],
"subject": {
"scopes": ["knowledge-spaces:read"],
"subjectId": claims["sub"],
"tenantId": claims["namespace_id"],
},
}
if value.get("expectedPrincipal") != expected_principal:
raise ValueError("KnowledgeFS Capability v2 test vector principal is invalid")
token = value.get("token")
if not isinstance(token, str) or not _is_non_blank(token):
raise ValueError("KnowledgeFS Capability v2 test vector token is invalid")
try:
verification_key = RSAAlgorithm.from_jwk(public_jwk)
if not isinstance(verification_key, RSAPublicKey):
raise ValueError("Capability vector verification key is not RSA public material")
header = jwt.get_unverified_header(token)
decoded_claims = jwt.decode(
token,
verification_key,
algorithms=["RS256"],
audience=cast(str, value["audience"]),
issuer=cast(str, value["issuer"]),
options={"verify_exp": False, "verify_iat": False, "verify_nbf": False},
)
except (jwt.PyJWTError, TypeError, ValueError) as exc:
raise ValueError("KnowledgeFS Capability v2 test vector signature is invalid") from exc
if header != protected_header or decoded_claims != claims:
raise ValueError("KnowledgeFS Capability v2 test vector token content drifted")
def _is_non_blank(value: object) -> bool:
return isinstance(value, str) and bool(value.strip()) and value == value.strip()
def validate_upstream_provenance(value: dict[str, Any]) -> None:
"""Validate exact standalone source provenance covered by the schema-6 lock."""
expected_fields = {"commit", "tree", "repository", "schemaVersion"}
if set(value) != expected_fields or value.get("schemaVersion") != 1:
raise ValueError("KnowledgeFS source provenance must use schemaVersion 1 with commit, tree and repository")
if value.get("repository") != "https://github.com/langgenius/knowledge-fs":
raise ValueError("KnowledgeFS source provenance repository does not match the contract owner")
for field in ("commit", "tree"):
digest = value.get(field)
if not isinstance(digest, str) or len(digest) != 40 or any(c not in "0123456789abcdef" for c in digest):
raise ValueError(f"KnowledgeFS source provenance {field} must be a lowercase full Git SHA")
def validate_declarations(document: dict[str, Any], declarations: tuple[ContractDeclaration, ...]) -> None:
"""Validate Dify Console declarations against matching pinned OpenAPI operations."""
operations_by_id: dict[str, list[tuple[str, str, dict[str, Any], dict[str, Any]]]] = {}
for path, path_item in document.get("paths", {}).items():
for method in OPENAPI_METHODS:
operation = path_item.get(method)
if operation is None:
continue
operation_id = operation.get("operationId")
if isinstance(operation_id, str) and operation_id:
operations_by_id.setdefault(operation_id, []).append((method, path, path_item, operation))
declared_ids: set[str] = set()
for declaration in declarations:
operation_id = declaration["operation_id"]
if operation_id in declared_ids:
raise ValueError(f"Dify Console registry has duplicate operationId: {operation_id}")
declared_ids.add(operation_id)
matches = operations_by_id.get(operation_id, [])
if not matches:
raise ValueError(f"KnowledgeFS OpenAPI has no operationId: {operation_id}")
if len(matches) > 1:
raise ValueError(f"KnowledgeFS OpenAPI has duplicate operationId: {operation_id}")
method, path, path_item, operation = matches[0]
if not path.startswith("/"):
raise ValueError(f"KnowledgeFS OpenAPI path must be absolute: {path}")
if method not in PROXY_METHODS:
raise ValueError(f"KnowledgeFS proxy does not support {method.upper()} {path}")
expected: ContractDeclaration = {
"operation_id": operation_id,
"method": method.upper(),
"path": path[1:],
"required_scope": required_scope(document, operation),
"response_kind": response_kind(operation),
"max_response_bytes": required_max_response_bytes(operation),
"request_headers": request_header_names(path_item, operation),
"response_headers": response_header_names(operation),
"response_media_types": response_media_types(operation),
}
for field in DECLARATION_FIELDS:
expected_value = expected[field]
received_value = declaration[field]
if received_value != expected_value:
raise ValueError(
f"KnowledgeFS operation {operation_id} field {field} drifted: "
f"expected {expected_value!r}, received {received_value!r}"
)
def console_contract_declarations() -> tuple[ContractDeclaration, ...]:
"""The P9 backend exposes only typed product controllers; the raw Console proxy is removed."""
return ()
def response_kind(operation: dict[str, Any]) -> str:
media_types = response_media_types(operation)
if "text/event-stream" in media_types:
return "stream"
if "application/octet-stream" in media_types:
return "binary"
return "buffered"
def response_media_types(operation: dict[str, Any]) -> tuple[str, ...]:
media_types: set[str] = set()
for status, response in operation.get("responses", {}).items():
if status == "2XX" or (len(status) == 3 and status.startswith("2") and status.isdigit()):
media_types.update(response.get("content", {}))
return tuple(sorted(media_types))
def required_scope(document: dict[str, Any], operation: dict[str, Any]) -> str | None:
scope = operation.get("x-knowledge-fs-required-scope")
security = operation["security"] if "security" in operation else document.get("security")
if security == []:
if scope is not None:
raise ValueError(f"KnowledgeFS public operation must not declare a required scope: {scope}")
return None
if security != [{"bearerAuth": []}]:
raise ValueError(f"KnowledgeFS operation effective security must be exactly bearerAuth: {security}")
if scope in ("knowledge-spaces:read", "knowledge-spaces:write"):
return scope
raise ValueError(f"KnowledgeFS operation has no supported required scope: {scope}")
def required_max_response_bytes(operation: dict[str, Any]) -> int:
value = operation.get("x-knowledge-fs-max-response-bytes")
if not isinstance(value, int) or isinstance(value, bool) or value <= 0:
raise ValueError(f"KnowledgeFS operation has no valid response byte limit: {value}")
return value
def request_header_names(path_item: dict[str, Any], operation: dict[str, Any]) -> tuple[str, ...]:
names: set[str] = set()
for parameter in [*path_item.get("parameters", []), *operation.get("parameters", [])]:
if "$ref" in parameter:
raise ValueError(f"KnowledgeFS request header references are not supported: {parameter['$ref']}")
if parameter.get("in") == "header":
names.add(parameter["name"].lower())
return tuple(sorted(names))
def response_header_names(operation: dict[str, Any]) -> tuple[str, ...]:
return tuple(
sorted(
{
name.lower()
for response in operation.get("responses", {}).values()
for name in response.get("headers", {})
}
)
)
def sha256(content: bytes) -> str:
return hashlib.sha256(content).hexdigest()
def run(*command: str, cwd: Path) -> str:
return subprocess.run(command, cwd=cwd, check=True, capture_output=True, text=True).stdout
if __name__ == "__main__":
main()
@@ -0,0 +1,445 @@
"""Generate OpenAPI JSON specs and split Markdown API docs.
The Markdown step uses `swagger-markdown`, the same converter family as the
legacy Markdown UI, so CI and local regeneration catch converter-incompatible
OpenAPI output early.
"""
from __future__ import annotations
import argparse
import json
import logging
import re
import subprocess
import sys
import tempfile
from pathlib import Path
API_ROOT = Path(__file__).resolve().parents[1]
if str(API_ROOT) not in sys.path:
sys.path.insert(0, str(API_ROOT))
# Import after script path setup.
from dev.generate_fastopenapi_specs import FASTOPENAPI_SPEC_TARGETS, generate_fastopenapi_specs # noqa: E402
# Import after script path setup.
from dev.generate_swagger_specs import SPEC_TARGETS, generate_specs # noqa: E402
logger = logging.getLogger(__name__)
SWAGGER_MARKDOWN_PACKAGE = "swagger-markdown@3.0.0"
CONSOLE_OPENAPI_FILENAME = "console-openapi.json"
STALE_COMBINED_MARKDOWN_FILENAME = "api-reference.md"
# The inner API is a trusted server-to-server surface, not part of the published API reference.
MARKDOWN_SPEC_TARGETS = tuple(target for target in SPEC_TARGETS if target.filename != "inner-openapi.json")
def _schema_ref_name(schema: object) -> str | None:
if not isinstance(schema, dict):
return None
ref = schema.get("$ref")
if not isinstance(ref, str):
return None
if ref.startswith("#/components/schemas/"):
return ref.removeprefix("#/components/schemas/")
return None
def _markdown_anchor(name: str) -> str:
return name.lower()
def _schema_markdown_type(schema: object) -> str:
if not isinstance(schema, dict):
return ""
ref_name = _schema_ref_name(schema)
if ref_name is not None:
return f"[{ref_name}](#{_markdown_anchor(ref_name)})"
for union_key in ("oneOf", "anyOf"):
variants = schema.get(union_key)
if not isinstance(variants, list):
continue
variant_types = [
variant_type
for variant in variants
if not (isinstance(variant, dict) and variant.get("type") == "null")
for variant_type in [_schema_markdown_type(variant)]
if variant_type
]
if len(variant_types) == 1:
return variant_types[0]
if variant_types:
return "<br>".join(variant_types)
schema_type = schema.get("type")
if schema_type == "array":
item_type = _schema_markdown_type(schema.get("items"))
return f"[ {item_type or 'object'} ]"
if isinstance(schema_type, str):
enum_values = schema.get("enum")
if isinstance(enum_values, list) and enum_values:
rendered_values = ", ".join(json.dumps(value, ensure_ascii=False) for value in enum_values)
return f"{schema_type}, <br>**Available values:** {rendered_values}"
return schema_type
return ""
def _replace_schema_table_type(markdown: str, definition_name: str, row_name: str, type_markdown: str) -> str:
if not type_markdown:
return markdown
lines = markdown.splitlines()
section_header = f"#### {definition_name}"
in_section = False
for index, line in enumerate(lines):
if line == section_header:
in_section = True
continue
if in_section and line.startswith("#### "):
break
if not in_section or not line.startswith(f"| {row_name} |"):
continue
cells = line.split("|")
if len(cells) < 5:
continue
cells[2] = f" {type_markdown} "
lines[index] = "|".join(cells)
break
return "\n".join(lines) + ("\n" if markdown.endswith("\n") else "")
def _has_union_schema(schema: object) -> bool:
if not isinstance(schema, dict):
return False
if isinstance(schema.get("oneOf"), list) or isinstance(schema.get("anyOf"), list):
return True
return _has_union_schema(schema.get("items"))
def _patch_union_schema_markdown(markdown: str, spec_path: Path) -> str:
"""Fill Markdown table cells that `swagger-markdown` leaves blank for unions, including array items."""
spec = json.loads(spec_path.read_text(encoding="utf-8"))
components = spec.get("components")
schemas = components.get("schemas") if isinstance(components, dict) else None
if not isinstance(schemas, dict):
return markdown
for schema_name, schema in schemas.items():
if not isinstance(schema_name, str) or not isinstance(schema, dict):
continue
properties = schema.get("properties")
if isinstance(properties, dict):
for property_name, property_schema in properties.items():
if isinstance(property_name, str) and _has_union_schema(property_schema):
markdown = _replace_schema_table_type(
markdown,
schema_name,
property_name,
_schema_markdown_type(property_schema),
)
union_variants = schema.get("oneOf") or schema.get("anyOf")
if not isinstance(union_variants, list):
continue
markdown = _replace_schema_table_type(
markdown,
schema_name,
schema_name,
_schema_markdown_type(schema),
)
for variant in union_variants:
variant_name = _schema_ref_name(variant)
variant_schema = schemas.get(variant_name) if variant_name is not None else None
if not isinstance(variant_name, str) or not isinstance(variant_schema, dict):
continue
properties = variant_schema.get("properties")
if not isinstance(properties, dict):
continue
for property_name, property_schema in properties.items():
if isinstance(property_name, str):
markdown = _replace_schema_table_type(
markdown,
variant_name,
property_name,
_schema_markdown_type(property_schema),
)
return markdown
def _strip_trailing_line_whitespace(markdown: str) -> str:
"""Remove converter-emitted trailing whitespace without changing line structure."""
return "\n".join(line.rstrip(" \t") for line in markdown.split("\n"))
# Deprecated operations render struck through: ``### ~~[POST] /path~~``.
_OPERATION_HEADING = re.compile(r"^### (?:~~)?\[(?P<method>[A-Z]+)\] (?P<path>\S+?)(?:~~)?$")
_BLANK_MEDIA_TYPE_CELL = re.compile(r"\*\*(?P<media>[^*]+)\*\*: <br>")
def _patch_union_response_markdown(markdown: str, spec_path: Path) -> str:
"""Fill the response Schema cells `swagger-markdown` leaves blank for union bodies.
An error status that documents business errors is an ``anyOf`` of their components and
the open error body; the converter renders such a body as ``**application/json**: <br>``.
Only blank cells are filled, from the operation the enclosing ``### [METHOD] /path`` names.
"""
spec = json.loads(spec_path.read_text(encoding="utf-8"))
paths = spec.get("paths")
if not isinstance(paths, dict):
return markdown
lines = markdown.splitlines()
responses: dict[str, object] = {}
for index, line in enumerate(lines):
heading = _OPERATION_HEADING.match(line)
if heading is not None:
path_item = paths.get(heading["path"])
operation = path_item.get(heading["method"].lower()) if isinstance(path_item, dict) else None
found = operation.get("responses") if isinstance(operation, dict) else None
responses = found if isinstance(found, dict) else {}
continue
if not responses or not line.startswith("|") or "**: <br>" not in line:
continue
cells = re.split(r"(?<!\\)\|", line.strip().strip("|"))
if len(cells) != 3:
continue
response = responses.get(cells[0].strip())
content = response.get("content") if isinstance(response, dict) else None
if not isinstance(content, dict):
continue
def fill(match: re.Match[str], content: dict[str, object] = content) -> str:
media = content.get(match["media"])
schema_type = _schema_markdown_type(media.get("schema")) if isinstance(media, dict) else ""
return f"**{match['media']}**: {schema_type}<br>" if schema_type else match.group(0)
cells[2] = _BLANK_MEDIA_TYPE_CELL.sub(fill, cells[2])
lines[index] = f"|{'|'.join(cells)}|"
return "\n".join(lines) + ("\n" if markdown.endswith("\n") else "")
def _patch_wildcard_media_type_markdown(markdown: str) -> str:
"""Keep swagger-markdown from interpreting the wildcard media type as emphasis."""
return markdown.replace("***/***", "`*/*`")
def _patch_request_body_markdown(markdown: str) -> str:
"""Collapse swagger-markdown 3.0.0's repeated cells for multi-content request bodies.
Its request-body renderer appends the same mutable Schema cell for every
content type, so every extra column repeats the complete set of media types.
Only repair that exact shape; preserve distinct cells and other tables.
"""
lines = markdown.splitlines()
for index, line in enumerate(lines):
if index < 2 or lines[index - 2].strip() != "| Required | Schema |":
continue
cells = [cell.strip() for cell in re.split(r"(?<!\\)\|", line.strip().strip("|"))]
if len(cells) > 2 and cells[0] in {"Yes", "No"} and len(set(cells[1:])) == 1:
lines[index] = f"| {cells[0]} | {cells[1]} |"
return "\n".join(lines) + ("\n" if markdown.endswith("\n") else "")
def _drop_null_values_for_markdown(value: object) -> object:
"""Remove null object members only from the converter's temporary input."""
if isinstance(value, dict):
return {key: _drop_null_values_for_markdown(item) for key, item in value.items() if item is not None}
if isinstance(value, list):
return [_drop_null_values_for_markdown(item) for item in value]
return value
def _convert_spec_to_markdown(spec_path: Path, markdown_path: Path) -> None:
markdown_path.parent.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(prefix=f"{markdown_path.stem}-") as temp_dir:
converter_spec_path = Path(temp_dir) / spec_path.name
converter_payload = json.loads(spec_path.read_text(encoding="utf-8"))
converter_spec_path.write_text(
json.dumps(_drop_null_values_for_markdown(converter_payload)),
encoding="utf-8",
)
temp_markdown_path = Path(temp_dir) / markdown_path.name
result = subprocess.run(
[
"npx",
"--yes",
SWAGGER_MARKDOWN_PACKAGE,
"-i",
str(converter_spec_path.resolve()),
"-o",
str(temp_markdown_path.resolve()),
],
check=False,
capture_output=True,
cwd=temp_dir,
text=True,
)
if result.returncode != 0:
logger.error(
"swagger-markdown failed for %s (exit code %s)\nstdout:\n%s\nstderr:\n%s",
spec_path,
result.returncode,
result.stdout,
result.stderr,
)
raise subprocess.CalledProcessError(
result.returncode,
result.args,
output=result.stdout,
stderr=result.stderr,
)
if not temp_markdown_path.exists():
converter_output = "\n".join(item for item in (result.stdout, result.stderr) if item).strip()
raise RuntimeError(f"swagger-markdown did not write {markdown_path}: {converter_output}")
converted_markdown = _patch_union_schema_markdown(
temp_markdown_path.read_text(encoding="utf-8"),
spec_path,
)
converted_markdown = _patch_union_response_markdown(converted_markdown, spec_path)
converted_markdown = _patch_wildcard_media_type_markdown(converted_markdown)
converted_markdown = _patch_request_body_markdown(converted_markdown)
converted_markdown = _strip_trailing_line_whitespace(converted_markdown)
if not converted_markdown.strip():
raise RuntimeError(f"swagger-markdown wrote an empty document for {markdown_path}")
markdown_path.write_text(converted_markdown, encoding="utf-8")
def _demote_markdown_headings(markdown: str, *, levels: int = 1) -> str:
"""Nest generated Markdown under another Markdown section."""
heading_prefix = "#" * levels
lines = []
for line in markdown.splitlines():
if line.startswith("#"):
lines.append(f"{heading_prefix}{line}")
else:
lines.append(line)
return "\n".join(lines).strip()
def _append_fastopenapi_markdown(console_markdown_path: Path, fastopenapi_markdown_path: Path) -> None:
"""Append FastOpenAPI console docs to the existing console API Markdown."""
console_markdown = console_markdown_path.read_text(encoding="utf-8").rstrip()
fastopenapi_markdown = _demote_markdown_headings(
fastopenapi_markdown_path.read_text(encoding="utf-8"),
levels=2,
)
console_markdown_path.write_text(
"\n\n".join(
[
console_markdown,
"## FastOpenAPI Preview (OpenAPI 3.1)",
fastopenapi_markdown,
]
)
+ "\n",
encoding="utf-8",
)
def generate_markdown_docs(
openapi_dir: Path,
markdown_dir: Path,
*,
keep_swagger_json: bool = False,
) -> list[Path]:
"""Generate intermediate specs, convert them to split Markdown API docs, and return Markdown paths."""
openapi_paths = generate_specs(openapi_dir)
fastopenapi_paths = generate_fastopenapi_specs(openapi_dir)
spec_paths = [*openapi_paths, *fastopenapi_paths]
openapi_paths_by_name = {path.name: path for path in openapi_paths}
fastopenapi_paths_by_name = {path.name: path for path in fastopenapi_paths}
markdown_dir.mkdir(parents=True, exist_ok=True)
written_paths: list[Path] = []
try:
with tempfile.TemporaryDirectory(prefix="dify-api-docs-") as temp_dir:
temp_markdown_dir = Path(temp_dir)
for target in MARKDOWN_SPEC_TARGETS:
openapi_path = openapi_paths_by_name[target.filename]
markdown_path = markdown_dir / f"{openapi_path.stem}.md"
_convert_spec_to_markdown(openapi_path, markdown_path)
written_paths.append(markdown_path)
for target in FASTOPENAPI_SPEC_TARGETS: # type: ignore
fastopenapi_path = fastopenapi_paths_by_name[target.filename]
markdown_path = temp_markdown_dir / f"{fastopenapi_path.stem}.md"
_convert_spec_to_markdown(fastopenapi_path, markdown_path)
console_markdown_path = markdown_dir / f"{Path(CONSOLE_OPENAPI_FILENAME).stem}.md"
_append_fastopenapi_markdown(console_markdown_path, markdown_path)
(markdown_dir / STALE_COMBINED_MARKDOWN_FILENAME).unlink(missing_ok=True)
finally:
if not keep_swagger_json:
for path in spec_paths:
path.unlink(missing_ok=True)
return written_paths
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--swagger-dir",
"--openapi-dir",
dest="openapi_dir",
type=Path,
default=Path("openapi"),
help="Directory where intermediate JSON spec files will be written.",
)
parser.add_argument(
"--markdown-dir",
type=Path,
default=Path("openapi/markdown"),
help="Directory where split Markdown API docs will be written.",
)
parser.add_argument(
"--keep-swagger-json",
action="store_true",
help="Keep intermediate JSON spec files after Markdown generation.",
)
return parser.parse_args()
def main() -> int:
args = parse_args()
written_paths = generate_markdown_docs(
args.openapi_dir,
args.markdown_dir,
keep_swagger_json=args.keep_swagger_json,
)
for path in written_paths:
logger.debug(path)
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,342 @@
"""Generate Flask-RESTX OpenAPI 3 specs without booting the full backend.
This helper intentionally avoids `app_factory.create_app()`. The normal backend
startup eagerly initializes database, Redis, Celery, and storage extensions,
which is unnecessary when the goal is only to serialize the Flask-RESTX
`/openapi.json` documents.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import logging
import os
import sys
from collections.abc import MutableMapping
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Protocol, TypeGuard, cast
from flask import Flask
from flask_restx import Api
from flask_restx.swagger import Swagger
logger = logging.getLogger(__name__)
API_ROOT = Path(__file__).resolve().parents[1]
if str(API_ROOT) not in sys.path:
sys.path.insert(0, str(API_ROOT))
# Import after script path setup.
from libs.flask_restx_compat import finalize_openapi_payload # noqa: E402
@dataclass(frozen=True)
class SpecTarget:
route: str
filename: str
class RestxApi(Protocol):
models: MutableMapping[str, object]
def model(self, name: str, model: dict[object, object]) -> object: ...
SPEC_TARGETS: tuple[SpecTarget, ...] = (
SpecTarget(route="/console/api/openapi.json", filename="console-openapi.json"),
SpecTarget(route="/api/openapi.json", filename="web-openapi.json"),
SpecTarget(route="/v1/openapi.json", filename="service-openapi.json"),
SpecTarget(route="/openapi/v1/openapi.json", filename="openapi-openapi.json"),
# Not consumed by @dify/contracts; enterprise generates its inner API client from this spec.
SpecTarget(route="/inner/api/openapi.json", filename="inner-openapi.json"),
)
def _is_inline_field_map(value: object) -> TypeGuard[dict[object, object]]:
"""Return whether a nested field map is an anonymous inline mapping."""
from flask_restx.model import Model, OrderedModel
return isinstance(value, dict) and not isinstance(value, (Model, OrderedModel))
def _jsonable_schema_value(value: object) -> object:
"""Return a deterministic JSON-serializable representation for schema fingerprints."""
if value is None or isinstance(value, str | int | float | bool):
return value
if isinstance(value, list | tuple):
return [_jsonable_schema_value(item) for item in value]
if isinstance(value, dict):
return {str(key): _jsonable_schema_value(item) for key, item in value.items()}
value_type = type(value)
return f"<{value_type.__module__}.{value_type.__qualname__}>"
def _field_signature(field: object) -> object:
"""Build a stable signature for a Flask-RESTX field object."""
from flask_restx import fields
from flask_restx.model import instance
field_instance = instance(field)
signature: dict[str, object] = {
"class": f"{field_instance.__class__.__module__}.{field_instance.__class__.__qualname__}"
}
if isinstance(field_instance, fields.Nested):
nested = getattr(field_instance, "nested", None)
if _is_inline_field_map(nested):
signature["nested"] = _inline_model_signature(nested)
else:
signature["nested"] = getattr(
nested,
"name",
f"<{type(nested).__module__}.{type(nested).__qualname__}>",
)
elif hasattr(field_instance, "container"):
signature["container"] = _field_signature(field_instance.container)
else:
schema = getattr(field_instance, "__schema__", None)
if isinstance(schema, dict):
signature["schema"] = _jsonable_schema_value(schema)
for attr_name in (
"attribute",
"default",
"description",
"example",
"max",
"max_items",
"min",
"min_items",
"nullable",
"readonly",
"required",
"title",
"unique",
):
if hasattr(field_instance, attr_name):
signature[attr_name] = _jsonable_schema_value(getattr(field_instance, attr_name))
return signature
def _inline_model_signature(nested_fields: dict[object, object]) -> object:
"""Build a stable signature for an anonymous inline model."""
return [
(str(field_name), _field_signature(field))
for field_name, field in sorted(nested_fields.items(), key=lambda item: str(item[0]))
]
def _inline_model_name(nested_fields: dict[object, object]) -> str:
"""Return a stable OpenAPI model name for an anonymous inline field map."""
signature = json.dumps(_inline_model_signature(nested_fields), sort_keys=True, separators=(",", ":"))
digest = hashlib.sha1(signature.encode("utf-8")).hexdigest()[:12]
return f"_AnonymousInlineModel_{digest}"
def apply_runtime_defaults() -> None:
"""Force the small config surface required for OpenAPI generation."""
os.environ.setdefault("SECRET_KEY", "spec-export")
os.environ.setdefault("STORAGE_TYPE", "local")
os.environ.setdefault("STORAGE_LOCAL_PATH", "/tmp/dify-storage")
from configs import dify_config
dify_config.SECRET_KEY = os.environ["SECRET_KEY"]
dify_config.STORAGE_TYPE = "local"
dify_config.STORAGE_LOCAL_PATH = os.environ["STORAGE_LOCAL_PATH"]
dify_config.SWAGGER_UI_ENABLED = True
def create_spec_app() -> Flask:
"""Build a minimal Flask app that only mounts the OpenAPI-producing blueprints."""
apply_runtime_defaults()
from libs.flask_restx_compat import install_swagger_compatibility
install_swagger_compatibility()
app = Flask(__name__)
from controllers.console import bp as console_bp
from controllers.console import console_ns
from controllers.inner_api import bp as inner_api_bp
from controllers.inner_api import inner_api_ns
from controllers.openapi import bp as openapi_bp
from controllers.openapi import openapi_ns
from controllers.service_api import bp as service_api_bp
from controllers.service_api import service_api_ns
from controllers.web import bp as web_bp
from controllers.web import web_ns
app.register_blueprint(console_bp)
app.register_blueprint(web_bp)
app.register_blueprint(service_api_bp)
app.register_blueprint(openapi_bp)
app.register_blueprint(inner_api_bp)
for namespace in (console_ns, web_ns, service_api_ns, openapi_ns, inner_api_ns):
for api in namespace.apis:
_materialize_inline_model_definitions(api)
return app
def _target_apis() -> dict[str, Api]:
"""Return the API instance that owns each exported OpenAPI target."""
from controllers.console import api as console_api
from controllers.inner_api import api as inner_api
from controllers.openapi import api as openapi_api
from controllers.service_api import api as service_api
from controllers.web import api as web_api
return {
"/console/api/openapi.json": console_api,
"/api/openapi.json": web_api,
"/v1/openapi.json": service_api,
"/openapi/v1/openapi.json": openapi_api,
"/inner/api/openapi.json": inner_api,
}
def _materialize_inline_model_definitions(api: RestxApi) -> None:
"""Convert inline `fields.Nested({...})` maps into named API models."""
from flask_restx import fields
from flask_restx.model import Model, OrderedModel, instance
inline_models: dict[int, dict[object, object]] = {}
inline_model_names: dict[int, str] = {}
def collect_field(field: object) -> None:
field_instance = instance(field)
if isinstance(field_instance, fields.Nested):
nested = getattr(field_instance, "nested", None)
if _is_inline_field_map(nested) and id(nested) not in inline_models:
inline_models[id(nested)] = nested
for nested_field in nested.values():
collect_field(nested_field)
container = getattr(field_instance, "container", None)
if container is not None:
collect_field(container)
for model in list(api.models.values()):
if isinstance(model, (Model, OrderedModel)):
for field in model.values():
collect_field(field)
for nested_fields in sorted(inline_models.values(), key=_inline_model_name):
anonymous_name = _inline_model_name(nested_fields)
inline_model_names[id(nested_fields)] = anonymous_name
if anonymous_name not in api.models:
api.model(anonymous_name, nested_fields)
def model_name_for(nested_fields: dict[object, object]) -> str:
anonymous_name = inline_model_names.get(id(nested_fields))
if anonymous_name is None:
anonymous_name = _inline_model_name(nested_fields)
inline_model_names[id(nested_fields)] = anonymous_name
if anonymous_name not in api.models:
api.model(anonymous_name, nested_fields)
return anonymous_name
def materialize_field(field: object) -> None:
field_instance = instance(field)
if isinstance(field_instance, fields.Nested):
nested = getattr(field_instance, "nested", None)
if _is_inline_field_map(nested):
field_instance.model = api.models[model_name_for(nested)] # type: ignore[attr-defined]
container = getattr(field_instance, "container", None)
if container is not None:
materialize_field(container)
index = 0
while index < len(api.models):
model = list(api.models.values())[index]
index += 1
if isinstance(model, (Model, OrderedModel)):
for field in model.values():
materialize_field(field)
def export_console_spec() -> dict[str, Any]:
"""Build only Console; editor refreshes do not need the other four API surfaces."""
apply_runtime_defaults()
from libs.flask_restx_compat import install_swagger_compatibility
install_swagger_compatibility()
from controllers.console import api, bp
app = Flask(__name__)
app.register_blueprint(bp)
_materialize_inline_model_definitions(cast(RestxApi, api))
with app.test_request_context("/console/api/openapi.json"):
return finalize_openapi_payload(Swagger(api).as_dict())
def export_specs() -> dict[str, dict[str, Any]]:
"""Build OpenAPI documents in memory without writing generated files."""
app = create_spec_app()
target_apis = _target_apis()
documents = {}
for target in SPEC_TARGETS:
# Build directly from the API instance instead of depending on the
# optional HTTP spec route. ExternalApi decides whether to register
# that route when controller modules are first imported, which may
# happen before this exporter forces its generation-only defaults.
with app.test_request_context(target.route):
payload = Swagger(target_apis[target.route]).as_dict()
payload = finalize_openapi_payload(payload)
documents[target.filename] = payload
return documents
def generate_specs(output_dir: Path) -> list[Path]:
"""Write all OpenAPI specs to `output_dir` and return the written paths."""
output_dir.mkdir(parents=True, exist_ok=True)
written_paths: list[Path] = []
for filename, payload in export_specs().items():
output_path = output_dir / filename
output_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
written_paths.append(output_path)
return written_paths
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"-o",
"--output-dir",
type=Path,
default=Path("openapi"),
help="Directory where the OpenAPI JSON files will be written.",
)
return parser.parse_args()
def main() -> int:
args = parse_args()
written_paths = generate_specs(args.output_dir)
for path in written_paths:
logger.debug(path)
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,494 @@
"""Cross-service KnowledgeFS product-operation contract validation.
The checked product manifest is the reviewable boundary, while the Dify product registry, Dify
Capability issuer registry, exported KnowledgeFS OpenAPI, and exported TypeScript request-guard
policy remain executable sources of truth. A product operation must appear exactly once as ready
or as an explicit gap; internal KFS-only operations require a named exclusion.
"""
from __future__ import annotations
from collections.abc import Iterable, Mapping
from typing import Any, Literal, NamedTuple, TypedDict, cast
class ProductOperationLimits(TypedDict):
kfsMaxResponseBytes: int
productMaxRequestBytes: int
productMaxResponseBytes: int
class ProductOperationStream(TypedDict):
kfsResponseKind: str
productKind: str
class ProductOperationManifestEntry(TypedDict):
action: str
kfsOperationId: str
limits: ProductOperationLimits
method: str
path: str
productOperationId: str
resource: str
stream: ProductOperationStream
transport: str
type ProductOperationManifestField = Literal[
"action",
"kfsOperationId",
"limits",
"method",
"path",
"productOperationId",
"resource",
"stream",
"transport",
]
PRODUCT_OPERATION_MANIFEST_FIELDS: tuple[ProductOperationManifestField, ...] = (
"action",
"kfsOperationId",
"limits",
"method",
"path",
"productOperationId",
"resource",
"stream",
"transport",
)
class ProductOperationManifest(TypedDict):
operations: list[ProductOperationManifestEntry]
schemaVersion: int
class ProductOperationGapEntry(ProductOperationManifestEntry):
reason: str
reasonCode: str
replacementProductOperationIds: list[str]
class InternalKfsOperationExclusion(TypedDict):
kfsOperationId: str
reason: str
reasonCode: str
class ProductOperationGapManifest(TypedDict):
gaps: list[ProductOperationGapEntry]
internalKfsOperationExclusions: list[InternalKfsOperationExclusion]
schemaVersion: int
class ProductOperationRuntimeContract(NamedTuple):
action: str
kfs_operation_id: str
max_request_bytes: int
max_response_bytes: int
method: str
path: str
product_operation_id: str
ready: bool
resource: str
stream_kind: str
transport: str
class CapabilityOperationRuntimeContract(NamedTuple):
action: str
allowed_caller_kinds: tuple[str, ...]
method: str
operation_id: str
path: str
resource: str
class CapabilityOperationPolicy(NamedTuple):
action: str
allowed_caller_kinds: tuple[str, ...]
method: str
operation_id: str
parent_resource_binding: dict[str, object] | None
path: str
resource: str
resource_binding: dict[str, object]
def product_operation_runtime_contracts() -> tuple[ProductOperationRuntimeContract, ...]:
"""Read the complete typed Dify product registry without normalizing away gaps."""
from services.knowledge_fs.product_operations import (
KNOWLEDGE_FS_PRODUCT_OPERATIONS,
is_product_operation_ready,
)
contracts: list[ProductOperationRuntimeContract] = []
for product_operation_id, operation in KNOWLEDGE_FS_PRODUCT_OPERATIONS.items():
if operation.capability_operation_id is None or operation.kfs_path is None or operation.action is None:
raise ValueError(f"Dify product operation {product_operation_id} has no Capability contract")
contracts.append(
ProductOperationRuntimeContract(
action=operation.action,
kfs_operation_id=operation.capability_operation_id,
max_request_bytes=operation.max_request_bytes,
max_response_bytes=operation.max_response_bytes,
method=operation.method,
path=operation.kfs_path,
product_operation_id=product_operation_id,
ready=is_product_operation_ready(product_operation_id),
resource=operation.resource_resolver,
stream_kind=operation.stream_kind,
transport=operation.transport,
)
)
return tuple(contracts)
def capability_operation_runtime_contracts() -> tuple[CapabilityOperationRuntimeContract, ...]:
"""Read the Python issuer policy used to mint each operation Capability."""
from services.knowledge_fs_capability import KNOWLEDGE_FS_CAPABILITY_OPERATIONS
return tuple(
CapabilityOperationRuntimeContract(
action=operation.action,
allowed_caller_kinds=operation.allowed_caller_kinds,
method=operation.method,
operation_id=operation_id,
path=operation.path,
resource=operation.resource_type,
)
for operation_id, operation in KNOWLEDGE_FS_CAPABILITY_OPERATIONS.items()
)
def parse_product_operation_manifest(value: dict[str, Any]) -> ProductOperationManifest:
"""Parse the ready product-operation inventory with a closed schema."""
if set(value) != {"operations", "schemaVersion"} or value.get("schemaVersion") != 2:
raise ValueError("KnowledgeFS product operation manifest must use schemaVersion 2")
operations = value.get("operations")
if not isinstance(operations, list) or not operations:
raise ValueError("KnowledgeFS product operation manifest operations must be a non-empty list")
return {
"operations": [_parse_product_entry(entry, gap=False) for entry in operations],
"schemaVersion": 2,
}
def parse_product_operation_gap_manifest(value: dict[str, Any]) -> ProductOperationGapManifest:
"""Parse explicit product gaps and KFS-only operation exclusions with a closed schema."""
expected_fields = {"gaps", "internalKfsOperationExclusions", "schemaVersion"}
if set(value) != expected_fields or value.get("schemaVersion") != 2:
raise ValueError("KnowledgeFS product operation gap manifest must use schemaVersion 2")
gaps = value.get("gaps")
exclusions = value.get("internalKfsOperationExclusions")
if not isinstance(gaps, list) or not isinstance(exclusions, list):
raise ValueError("KnowledgeFS product operation gap manifest lists are invalid")
parsed_exclusions: list[InternalKfsOperationExclusion] = []
for raw in exclusions:
if not isinstance(raw, dict) or set(raw) != {"kfsOperationId", "reason", "reasonCode"}:
raise ValueError("KnowledgeFS internal KFS operation exclusion fields are invalid")
for field in ("kfsOperationId", "reason", "reasonCode"):
_required_string(raw.get(field), f"internal exclusion {field}")
parsed_exclusions.append(cast(InternalKfsOperationExclusion, raw))
return {
"gaps": [cast(ProductOperationGapEntry, _parse_product_entry(entry, gap=True)) for entry in gaps],
"internalKfsOperationExclusions": parsed_exclusions,
"schemaVersion": 2,
}
def parse_capability_operation_policy(value: dict[str, Any]) -> tuple[CapabilityOperationPolicy, ...]:
"""Parse the runtime-exported TypeScript request-guard registry."""
if set(value) != {"operations", "schemaVersion"} or value.get("schemaVersion") != 1:
raise ValueError("KnowledgeFS Capability operation policy must use schemaVersion 1")
operations = value.get("operations")
if not isinstance(operations, list) or not operations:
raise ValueError("KnowledgeFS Capability operation policy must contain operations")
parsed: list[CapabilityOperationPolicy] = []
for raw in operations:
expected_fields = {
"action",
"allowedCallerKinds",
"method",
"operationId",
"parentResourceBinding",
"path",
"resourceBinding",
"resourceType",
}
if not isinstance(raw, dict) or set(raw) != expected_fields:
raise ValueError("KnowledgeFS Capability operation policy fields are invalid")
callers = raw.get("allowedCallerKinds")
resource_binding = raw.get("resourceBinding")
parent_binding = raw.get("parentResourceBinding")
if (
not isinstance(callers, list)
or not callers
or not all(isinstance(caller, str) and caller for caller in callers)
or not isinstance(resource_binding, dict)
or not resource_binding
or (parent_binding is not None and not isinstance(parent_binding, dict))
):
raise ValueError("KnowledgeFS Capability operation request binding is invalid")
for field in ("action", "method", "operationId", "path", "resourceType"):
_required_string(raw.get(field), f"Capability operation {field}")
parsed.append(
CapabilityOperationPolicy(
action=cast(str, raw["action"]),
allowed_caller_kinds=tuple(callers),
method=cast(str, raw["method"]),
operation_id=cast(str, raw["operationId"]),
parent_resource_binding=cast(dict[str, object] | None, parent_binding),
path=cast(str, raw["path"]),
resource=cast(str, raw["resourceType"]),
resource_binding=cast(dict[str, object], resource_binding),
)
)
_unique((operation.operation_id for operation in parsed), "Capability operation policy")
return tuple(parsed)
def validate_product_operation_contracts(
*,
capability_operations: tuple[CapabilityOperationRuntimeContract, ...],
capability_policy: tuple[CapabilityOperationPolicy, ...],
document: dict[str, Any],
gap_manifest: ProductOperationGapManifest,
manifest: ProductOperationManifest,
product_operations: tuple[ProductOperationRuntimeContract, ...],
) -> None:
"""Require exact agreement across manifests, Dify registries, KFS guard policy, and OpenAPI."""
ready_entries = {entry["productOperationId"]: entry for entry in manifest["operations"]}
gap_entries = {entry["productOperationId"]: entry for entry in gap_manifest["gaps"]}
if len(ready_entries) != len(manifest["operations"]):
raise ValueError("KnowledgeFS product operation manifest contains duplicate product ids")
if len(gap_entries) != len(gap_manifest["gaps"]):
raise ValueError("KnowledgeFS product operation gap manifest contains duplicate product ids")
if set(ready_entries) & set(gap_entries):
raise ValueError("KnowledgeFS product operation cannot be both ready and a gap")
runtime_by_product = {operation.product_operation_id: operation for operation in product_operations}
if set(ready_entries) | set(gap_entries) != set(runtime_by_product):
missing = sorted(set(runtime_by_product) - set(ready_entries) - set(gap_entries))
extra = sorted((set(ready_entries) | set(gap_entries)) - set(runtime_by_product))
raise ValueError(f"KnowledgeFS product operation completeness drifted: missing={missing}, extra={extra}")
python_capabilities = {operation.operation_id: operation for operation in capability_operations}
guard_capabilities = {operation.operation_id: operation for operation in capability_policy}
openapi_operations = _openapi_operations(document)
for product_operation_id, runtime in runtime_by_product.items():
entry = ready_entries.get(product_operation_id) or gap_entries[product_operation_id]
if runtime.ready != (product_operation_id in ready_entries):
state = "ready" if runtime.ready else "gap"
raise ValueError(f"Dify product operation {product_operation_id} must be declared as {state}")
_validate_product_entry(entry, runtime, openapi_operations, python_capabilities, guard_capabilities)
if product_operation_id in gap_entries:
_required_string(gap_entries[product_operation_id]["reason"], "product gap reason")
_required_string(gap_entries[product_operation_id]["reasonCode"], "product gap reasonCode")
replacements = gap_entries[product_operation_id]["replacementProductOperationIds"]
if not replacements or any(replacement not in ready_entries for replacement in replacements):
raise ValueError(f"KnowledgeFS product gap {product_operation_id} has invalid replacements")
exclusions = gap_manifest["internalKfsOperationExclusions"]
exclusion_by_id = {entry["kfsOperationId"]: entry for entry in exclusions}
if len(exclusion_by_id) != len(exclusions):
raise ValueError("KnowledgeFS internal KFS operation exclusions contain duplicate ids")
lifecycle_operation_ids = {
"activateDifyWorkspaceIntegration",
"freezeDifyWorkspaceIntegration",
}
if set(exclusion_by_id) != lifecycle_operation_ids:
raise ValueError("KnowledgeFS Dify integration lifecycle operations must be explicit internal exclusions")
product_capability_ids = {operation.kfs_operation_id for operation in product_operations}
for operation_id in exclusion_by_id:
if operation_id in product_capability_ids:
raise ValueError(f"Internal KFS operation {operation_id} cannot also be a product operation")
_validate_capability_alignment(
operation_id,
openapi_operations,
python_capabilities,
guard_capabilities,
)
python_operation = python_capabilities[operation_id]
if python_operation.allowed_caller_kinds != ("internal_worker",) or python_operation.resource != "namespace":
raise ValueError("Dify integration lifecycle operations must remain internal-worker namespace-only")
def _parse_product_entry(raw: object, *, gap: bool) -> ProductOperationManifestEntry:
base_fields = {
"action",
"kfsOperationId",
"limits",
"method",
"path",
"productOperationId",
"resource",
"stream",
"transport",
}
expected_fields = base_fields | ({"reason", "reasonCode", "replacementProductOperationIds"} if gap else set())
if not isinstance(raw, dict) or set(raw) != expected_fields:
raise ValueError("KnowledgeFS product operation entry fields are invalid")
for field in ("action", "kfsOperationId", "method", "path", "productOperationId", "resource", "transport"):
_required_string(raw.get(field), f"product operation {field}")
limits = raw.get("limits")
stream = raw.get("stream")
if not isinstance(limits, dict) or set(limits) != {
"kfsMaxResponseBytes",
"productMaxRequestBytes",
"productMaxResponseBytes",
}:
raise ValueError("KnowledgeFS product operation limits are invalid")
if not all(isinstance(limit, int) and not isinstance(limit, bool) and limit >= 0 for limit in limits.values()):
raise ValueError("KnowledgeFS product operation limits must be nonnegative integers")
if not isinstance(stream, dict) or set(stream) != {"kfsResponseKind", "productKind"}:
raise ValueError("KnowledgeFS product operation stream contract is invalid")
_required_string(stream.get("kfsResponseKind"), "KFS response kind")
_required_string(stream.get("productKind"), "product stream kind")
if gap:
_required_string(raw.get("reason"), "product gap reason")
_required_string(raw.get("reasonCode"), "product gap reasonCode")
replacements = raw.get("replacementProductOperationIds")
if not isinstance(replacements, list) or not all(isinstance(item, str) and item for item in replacements):
raise ValueError("KnowledgeFS product operation gap replacements are invalid")
return cast(ProductOperationManifestEntry, raw)
def _validate_product_entry(
entry: ProductOperationManifestEntry,
runtime: ProductOperationRuntimeContract,
openapi_operations: Mapping[str, tuple[str, str, dict[str, Any]]],
python_capabilities: Mapping[str, CapabilityOperationRuntimeContract],
guard_capabilities: Mapping[str, CapabilityOperationPolicy],
) -> None:
_validate_capability_alignment(
runtime.kfs_operation_id,
openapi_operations,
python_capabilities,
guard_capabilities,
)
_, _, openapi_operation = openapi_operations[runtime.kfs_operation_id]
expected: ProductOperationManifestEntry = {
"action": runtime.action,
"kfsOperationId": runtime.kfs_operation_id,
"limits": {
"kfsMaxResponseBytes": _openapi_max_response_bytes(openapi_operation),
"productMaxRequestBytes": runtime.max_request_bytes,
"productMaxResponseBytes": runtime.max_response_bytes,
},
"method": runtime.method,
"path": runtime.path,
"productOperationId": runtime.product_operation_id,
"resource": runtime.resource,
"stream": {
"kfsResponseKind": _openapi_response_kind(openapi_operation),
"productKind": runtime.stream_kind,
},
"transport": runtime.transport,
}
for field in PRODUCT_OPERATION_MANIFEST_FIELDS:
expected_value = expected[field]
if entry[field] != expected_value:
raise ValueError(
f"KnowledgeFS product operation {runtime.product_operation_id} field {field} drifted: "
f"expected {expected_value!r}, received {entry[field]!r}"
)
def _validate_capability_alignment(
operation_id: str,
openapi_operations: Mapping[str, tuple[str, str, dict[str, Any]]],
python_capabilities: Mapping[str, CapabilityOperationRuntimeContract],
guard_capabilities: Mapping[str, CapabilityOperationPolicy],
) -> None:
python_operation = python_capabilities.get(operation_id)
guard_operation = guard_capabilities.get(operation_id)
openapi_operation = openapi_operations.get(operation_id)
if python_operation is None or guard_operation is None or openapi_operation is None:
raise ValueError(
f"KnowledgeFS operation {operation_id} must exist in Python issuer, TypeScript guard, and OpenAPI"
)
method, path, _ = openapi_operation
expected = (
python_operation.action,
python_operation.method,
python_operation.path,
python_operation.resource,
)
received_guard = (guard_operation.action, guard_operation.method, guard_operation.path, guard_operation.resource)
if received_guard != expected:
raise ValueError(f"KnowledgeFS operation {operation_id} TypeScript guard drifted")
if (method, path) != (python_operation.method, python_operation.path):
raise ValueError(f"KnowledgeFS operation {operation_id} OpenAPI method/path drifted")
if guard_operation.allowed_caller_kinds != python_operation.allowed_caller_kinds:
raise ValueError(f"KnowledgeFS operation {operation_id} caller policy drifted")
def _openapi_operations(document: dict[str, Any]) -> dict[str, tuple[str, str, dict[str, Any]]]:
operations: dict[str, tuple[str, str, dict[str, Any]]] = {}
duplicates: set[str] = set()
for path, path_item in document.get("paths", {}).items():
if not isinstance(path, str) or not isinstance(path_item, dict):
continue
for method in ("delete", "get", "patch", "post", "put"):
operation = path_item.get(method)
if not isinstance(operation, dict):
continue
operation_id = operation.get("operationId")
if not isinstance(operation_id, str) or not operation_id:
continue
if operation_id in operations:
duplicates.add(operation_id)
operations[operation_id] = (method.upper(), path, operation)
if duplicates:
raise ValueError(f"KnowledgeFS OpenAPI contains duplicate operation ids: {sorted(duplicates)}")
return operations
def _openapi_response_kind(operation: dict[str, Any]) -> Literal["binary", "buffered", "stream"]:
media_types = {
media_type
for status, response in operation.get("responses", {}).items()
if status == "2XX" or (len(status) == 3 and status.startswith("2") and status.isdigit())
for media_type in response.get("content", {})
}
if "text/event-stream" in media_types:
return "stream"
if "application/octet-stream" in media_types:
return "binary"
return "buffered"
def _openapi_max_response_bytes(operation: dict[str, Any]) -> int:
value = operation.get("x-knowledge-fs-max-response-bytes")
if not isinstance(value, int) or isinstance(value, bool) or value <= 0:
raise ValueError("KnowledgeFS product OpenAPI operation has no positive response byte limit")
return value
def _required_string(value: object, field: str) -> str:
if not isinstance(value, str) or not value.strip() or value != value.strip():
raise ValueError(f"KnowledgeFS {field} must be a non-empty trimmed string")
return value
def _unique(values: Iterable[str], label: str) -> None:
sequence = tuple(values)
if len(sequence) != len(set(sequence)):
raise ValueError(f"KnowledgeFS {label} contains duplicate ids")
__all__ = [
"CapabilityOperationPolicy",
"CapabilityOperationRuntimeContract",
"ProductOperationGapManifest",
"ProductOperationManifest",
"ProductOperationRuntimeContract",
"capability_operation_runtime_contracts",
"parse_capability_operation_policy",
"parse_product_operation_gap_manifest",
"parse_product_operation_manifest",
"product_operation_runtime_contracts",
"validate_product_operation_contracts",
]
@@ -0,0 +1,757 @@
"""Lint Flask-RESTX response docs against statically visible response serializers.
This checker intentionally stays conservative. It only reports a hard schema
mismatch when both sides are statically known for the same 2xx status code:
a documented ``@ns.response(..., Model)`` and an actual ``dump_response(Model, ...)``,
``Model(...).model_dump()``, or ``Model.model_validate(...).model_dump()`` return.
Raw dictionaries, raw lists, ``None`` responses, streaming helpers, missing
response schemas, and returns with non-literal status codes are classified as
unknown. Unknown details are hidden by default to keep routine output focused;
pass ``--include-unknown`` when triaging them. The one intentional non-schema
mismatch is a known body/schema on a no-body status such as 204, 205, or 304.
"""
from __future__ import annotations
import argparse
import ast
import json
import sys
from collections import Counter, defaultdict
from collections.abc import Iterable, Sequence
from dataclasses import asdict, dataclass, field
from http import HTTPStatus
from pathlib import Path
from typing import Any, Literal
HTTP_METHODS = {"delete", "get", "head", "options", "patch", "post", "put"}
NO_BODY_STATUSES = {HTTPStatus.NO_CONTENT.value, HTTPStatus.RESET_CONTENT.value, HTTPStatus.NOT_MODIFIED.value}
DEFAULT_CONTROLLER_DIRS = ("controllers/console", "controllers/service_api", "controllers/web")
IGNORE_COMMENT_MARKERS = ("response-contract:ignore",)
type Classification = Literal["valid", "mismatch", "unknown", "refactorable"]
type ActualKind = Literal[
"empty",
"model",
"model_dump_variable",
"none",
"raw_dict",
"raw_list",
"raw_value",
"unknown",
]
type MethodNode = ast.FunctionDef | ast.AsyncFunctionDef
type ModelValueSource = Literal["constructor", "model_validate"]
HTTP_STATUS_NAMES = {status.name: status.value for status in HTTPStatus}
HTTP_STATUS_NAMES.update({f"HTTP_{status.value}_{status.name}": status.value for status in HTTPStatus})
@dataclass(frozen=True)
class DocumentedResponse:
status: int
model: str | None
line: int
@dataclass(frozen=True)
class ActualResponse:
status: int | None
kind: ActualKind
model: str | None
line: int
@dataclass(frozen=True)
class ContractCheck:
classification: Classification
file: str
class_name: str
method: str
route: str
line: int
reason: str
documented: dict[int, str | None]
actual: list[ActualResponse]
@dataclass(frozen=True)
class ContractCheckContext:
"""Stable route-method context shared by every classification result."""
file: str
class_name: str
method: str
route: str
line: int
documented: dict[int, str | None]
def build(
self, classification: Classification, reason: str, actual_responses: Sequence[ActualResponse]
) -> ContractCheck:
return ContractCheck(
classification=classification,
file=self.file,
class_name=self.class_name,
method=self.method,
route=self.route,
line=self.line,
reason=reason,
documented=self.documented,
actual=list(actual_responses),
)
def mismatch(self, reason: str, documented: DocumentedResponse, actual: ActualResponse) -> ContractCheck:
return self.build("mismatch", f"{reason} (doc line {documented.line}, return line {actual.line})", [actual])
@dataclass
class VariableAssignmentSummary:
"""Track whether a local name is safe to treat as one specific response model."""
known_models: set[str] = field(default_factory=set)
known_sources: set[ModelValueSource] = field(default_factory=set)
has_unknown_assignment: bool = False
def add_known(self, model: str, source: ModelValueSource) -> None:
self.known_models.add(model)
self.known_sources.add(source)
def add_unknown(self) -> None:
self.has_unknown_assignment = True
def single_known_model(self) -> tuple[str, ModelValueSource] | None:
if self.has_unknown_assignment or len(self.known_models) != 1:
return None
model = next(iter(self.known_models))
source: ModelValueSource = "constructor" if self.known_sources == {"constructor"} else "model_validate"
return model, source
def dotted_name(node: ast.AST) -> str | None:
match node:
case ast.Name():
return node.id
case ast.Attribute():
parent = dotted_name(node.value)
if parent:
return f"{parent}.{node.attr}"
return node.attr
return None
def leaf_name(node: ast.AST) -> str | None:
name = dotted_name(node)
if name is None:
return None
return name.rsplit(".", 1)[-1]
def int_constant(node: ast.AST | None) -> int | None:
if isinstance(node, ast.Constant) and isinstance(node.value, int):
return node.value
if isinstance(node, ast.Name):
return HTTP_STATUS_NAMES.get(node.id)
if isinstance(node, ast.Attribute):
return HTTP_STATUS_NAMES.get(node.attr)
return None
def string_constant(node: ast.AST | None) -> str | None:
if isinstance(node, ast.Constant) and isinstance(node.value, str):
return node.value
return None
def keyword_value(call: ast.Call, *names: str) -> ast.AST | None:
for keyword in call.keywords:
if keyword.arg in names:
return keyword.value
return None
def is_probable_model_name(name: str) -> bool:
return bool(name) and name[0].isupper()
def model_name_from_schema_expr(node: ast.AST | None) -> str | None:
if node is None:
return None
if isinstance(node, ast.Subscript):
value_name = dotted_name(node.value)
if value_name and value_name.endswith(".models"):
# register_response_schema_models stores schemas by model name; both
# ns.models[Model.__name__] and ns.models["Model"] appear in controllers.
key = node.slice
if isinstance(key, ast.Attribute) and key.attr == "__name__":
return leaf_name(key.value)
return string_constant(key)
if isinstance(node, ast.Call):
func_name = dotted_name(node.func)
if func_name and func_name.endswith(".model"):
return string_constant(node.args[0] if node.args else keyword_value(node, "name"))
if isinstance(node, ast.Name):
return node.id
return None
def documented_response_from_decorator(decorator: ast.expr) -> DocumentedResponse | None:
if not isinstance(decorator, ast.Call):
return None
func_name = dotted_name(decorator.func)
if not func_name or not func_name.endswith(".response"):
return None
status_expr = decorator.args[0] if decorator.args else keyword_value(decorator, "code", "status")
status = int_constant(status_expr)
if status is None:
return None
schema_expr: ast.AST | None = decorator.args[2] if len(decorator.args) >= 3 else None
schema_expr = keyword_value(decorator, "model", "schema") or schema_expr
return DocumentedResponse(
status=status,
model=model_name_from_schema_expr(schema_expr),
line=decorator.lineno,
)
def route_from_decorator(decorator: ast.expr) -> str | None:
if not isinstance(decorator, ast.Call):
return None
func_name = dotted_name(decorator.func)
if not func_name or not func_name.endswith(".route"):
return None
return string_constant(decorator.args[0] if decorator.args else keyword_value(decorator, "route", "path"))
def routes_from_decorators(decorators: Iterable[ast.expr]) -> list[str]:
return [route for decorator in decorators if (route := route_from_decorator(decorator))]
def response_docs_from_decorators(decorators: Iterable[ast.expr]) -> dict[int, DocumentedResponse]:
docs: dict[int, DocumentedResponse] = {}
for decorator in decorators:
response = documented_response_from_decorator(decorator)
if response and 200 <= response.status < 300:
docs[response.status] = response
return docs
def model_name_from_model_validate_call(node: ast.AST) -> str | None:
if not isinstance(node, ast.Call):
return None
if isinstance(node.func, ast.Attribute) and node.func.attr == "model_validate":
return leaf_name(node.func.value)
return None
def model_value_from_model_validate_call(node: ast.AST) -> tuple[str, ModelValueSource] | None:
if model_name := model_name_from_model_validate_call(node):
return model_name, "model_validate"
return None
def model_name_from_constructor_call(node: ast.AST) -> str | None:
if not isinstance(node, ast.Call):
return None
if isinstance(node.func, ast.Name) and is_probable_model_name(node.func.id):
return node.func.id
return None
def model_value_from_constructor_call(node: ast.AST) -> tuple[str, ModelValueSource] | None:
if model_name := model_name_from_constructor_call(node):
return model_name, "constructor"
return None
def model_name_from_model_dump(node: ast.AST) -> str | None:
if not isinstance(node, ast.Call) or not isinstance(node.func, ast.Attribute) or node.func.attr != "model_dump":
return None
dumped_value = node.func.value
if isinstance(dumped_value, ast.Call):
return model_name_from_model_validate_call(dumped_value) or model_name_from_constructor_call(dumped_value)
return None
def model_name_from_model_value(node: ast.AST) -> str | None:
return model_name_from_model_validate_call(node) or model_name_from_constructor_call(node)
def model_value_from_model_value(node: ast.AST) -> tuple[str, ModelValueSource] | None:
return model_value_from_model_validate_call(node) or model_value_from_constructor_call(node)
def model_name_from_dump_response(node: ast.AST) -> str | None:
if not isinstance(node, ast.Call):
return None
func_name = dotted_name(node.func)
if func_name != "dump_response" and not (func_name and func_name.endswith(".dump_response")):
return None
model_expr = node.args[0] if node.args else keyword_value(node, "model", "schema", "response_model")
if isinstance(model_expr, ast.Name):
return model_expr.id
return None
def actual_kind_from_expr(
expr: ast.AST | None, variable_models: dict[str, tuple[str, ModelValueSource]] | None = None
) -> tuple[ActualKind, str | None]:
if expr is None:
return "none", None
dump_response_model = model_name_from_dump_response(expr)
if dump_response_model:
return "model", dump_response_model
if isinstance(expr, ast.Call) and isinstance(expr.func, ast.Attribute) and expr.func.attr == "model_dump":
dumped_value = expr.func.value
if isinstance(dumped_value, ast.Name) and variable_models:
model_assignment = variable_models.get(dumped_value.id)
if model_assignment:
model_name, source = model_assignment
if source == "constructor":
return "model", model_name
# A variable dump from model_validate can match today, but it
# bypasses dump_response and is easier to drift; keep it visible
# as refactorable.
return "model_dump_variable", model_name
model_dump_model = model_name_from_model_dump(expr)
if model_dump_model:
return "model", model_dump_model
if isinstance(expr, ast.Constant):
if expr.value is None:
return "none", None
if expr.value == "":
return "empty", None
return "raw_value", None
if isinstance(expr, ast.Dict):
return "raw_dict", None
if isinstance(expr, ast.List):
return "raw_list", None
return "unknown", None
def actual_response_from_return(
return_node: ast.Return, variable_models: dict[str, tuple[str, ModelValueSource]]
) -> ActualResponse:
status: int | None = 200
body_expr = return_node.value
if isinstance(return_node.value, ast.Tuple) and return_node.value.elts:
body_expr = return_node.value.elts[0]
if len(return_node.value.elts) >= 2:
# Dynamic statuses are not safe to coerce to 200; classify them as unknown.
status = int_constant(return_node.value.elts[1])
kind, model = actual_kind_from_expr(body_expr, variable_models)
return ActualResponse(status=status, kind=kind, model=model, line=return_node.lineno)
def iter_method_nodes(method: MethodNode) -> Iterable[ast.AST]:
"""Yield method body nodes while ignoring nested function/class scopes."""
stack: list[ast.AST] = list(reversed(method.body))
while stack:
node = stack.pop()
yield node
if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.Lambda | ast.ClassDef):
continue
stack.extend(reversed(list(ast.iter_child_nodes(node))))
def target_names(target: ast.AST) -> Iterable[str]:
match target:
case ast.Name():
yield target.id
case ast.Tuple() | ast.List():
for item in target.elts:
yield from target_names(item)
def record_assignment(
assignments: defaultdict[str, VariableAssignmentSummary],
targets: Iterable[str],
model_assignment: tuple[str, ModelValueSource] | None,
) -> None:
for target in targets:
if model_assignment is None:
# Once a name receives an unknown value, later model_dump() calls on it
# are no longer a reliable signal for the returned schema.
assignments[target].add_unknown()
else:
model_name, source = model_assignment
assignments[target].add_known(model_name, source)
def variable_model_assignments_for_method(method: MethodNode) -> dict[str, tuple[str, ModelValueSource]]:
"""Infer local variables that are unambiguously assigned one response model."""
assignments: defaultdict[str, VariableAssignmentSummary] = defaultdict(VariableAssignmentSummary)
for node in iter_method_nodes(method):
match node:
case ast.Assign(targets=targets, value=value):
record_assignment(
assignments,
(name for target in targets for name in target_names(target)),
model_value_from_model_value(value),
)
case ast.AnnAssign(target=target, value=value) if value is not None:
record_assignment(assignments, target_names(target), model_value_from_model_value(value))
case ast.AugAssign(target=target) | ast.For(target=target) | ast.AsyncFor(target=target):
# Mutation and loop targets overwrite prior values with runtime-dependent data.
record_assignment(assignments, target_names(target), None)
case ast.With(items=items) | ast.AsyncWith(items=items):
for item in items:
if item.optional_vars is not None:
record_assignment(assignments, target_names(item.optional_vars), None)
case ast.ExceptHandler(name=name) if name:
assignments[name].add_unknown()
case ast.NamedExpr(target=target, value=value):
record_assignment(assignments, target_names(target), model_value_from_model_value(value))
return {
name: assignment
for name, summary in assignments.items()
if (assignment := summary.single_known_model()) is not None
}
def actual_responses_for_method(method: MethodNode) -> list[ActualResponse]:
"""Extract statically visible 2xx returns from one controller method.
The analysis is deliberately shallow and conservative:
1. Walk only the method's own body, skipping nested functions/classes.
2. Infer local variables that are assigned exactly one recognizable response
model, so ``response.model_dump()`` can still be connected to its schema.
3. Treat any later unknown assignment, mutation target, loop target, context
manager binding, or exception binding as invalidating that variable.
4. For each top-level return path, split Flask-style ``(body, status)``
tuples, classify the body expression, and keep non-literal statuses as
``None`` so the classifier reports them as unknown instead of assuming 200.
5. Drop non-2xx literal statuses, since response contracts here only compare
successful response schemas.
"""
variable_models = variable_model_assignments_for_method(method)
responses: list[ActualResponse] = []
for node in iter_method_nodes(method):
if isinstance(node, ast.Return):
responses.append(actual_response_from_return(node, variable_models))
return [response for response in responses if response.status is None or 200 <= response.status < 300]
def display_path(file_path: Path, repo_root: Path) -> str:
try:
return str(file_path.relative_to(repo_root))
except ValueError:
return str(file_path)
def classify_method(
*,
actual_responses: Sequence[ActualResponse],
class_name: str,
documented_responses: dict[int, DocumentedResponse],
file_path: Path,
method: MethodNode,
repo_root: Path,
route: str,
) -> ContractCheck:
documented_summary = {status: response.model for status, response in sorted(documented_responses.items())}
context = ContractCheckContext(
file=display_path(file_path, repo_root),
class_name=class_name,
method=method.name,
route=route,
line=method.lineno,
documented=documented_summary,
)
if not actual_responses:
return context.build("unknown", "no statically visible 2xx return", [])
unknown_reasons: list[str] = []
refactorable_reasons: list[str] = []
for actual in actual_responses:
if actual.status is None:
unknown_reasons.append(f"return line {actual.line} has non-literal or unsupported status")
continue
documented = documented_responses.get(actual.status)
if actual.status in NO_BODY_STATUSES:
# No-body statuses are contract violations even when the schema names
# would otherwise match, because clients should not expect a payload.
if documented is not None and documented.model is not None:
return context.mismatch(
f"status {actual.status} is a no-body response but documents {documented.model}",
documented,
actual,
)
if actual.kind in {"model", "model_dump_variable", "raw_dict", "raw_list", "raw_value"}:
no_body_doc = DocumentedResponse(status=actual.status, model=None, line=method.lineno)
return context.mismatch(
f"status {actual.status} is a no-body response but returns {actual.kind}",
no_body_doc,
actual,
)
if actual.kind == "unknown":
unknown_reasons.append(f"status {actual.status} returns unknown body expression")
continue
if documented is None:
unknown_reasons.append(f"status {actual.status} has no @response doc")
continue
if documented.model is None:
unknown_reasons.append(f"status {actual.status} response doc has no schema model")
continue
if actual.kind == "model_dump_variable" and actual.model is not None:
if documented.model != actual.model:
return context.mismatch(
f"status {actual.status} documents {documented.model} but returns {actual.model}",
documented,
actual,
)
# The schema matches, but this path still deserves cleanup because
# dump_response is the contract-aware serialization helper.
refactorable_reasons.append(
f"status {actual.status} returns {actual.model}.model_dump() through a variable; prefer dump_response"
)
continue
if actual.kind != "model" or actual.model is None:
unknown_reasons.append(f"status {actual.status} returns {actual.kind}")
continue
if documented.model != actual.model:
return context.mismatch(
f"status {actual.status} documents {documented.model} but returns {actual.model}",
documented,
actual,
)
if unknown_reasons:
# Unknown beats refactorable: if any return path is ambiguous, do not
# imply the endpoint is merely a cleanup candidate.
return context.build("unknown", "; ".join(sorted(set(unknown_reasons))), actual_responses)
if refactorable_reasons:
return context.build("refactorable", "; ".join(sorted(set(refactorable_reasons))), actual_responses)
return context.build(
"valid",
"documented response schema matches statically visible return schema",
actual_responses,
)
def iter_controller_files(paths: Iterable[Path]) -> Iterable[Path]:
for path in paths:
if path.is_file() and path.suffix == ".py":
yield path
elif path.is_dir():
yield from sorted(child for child in path.rglob("*.py") if child.is_file())
def node_start_lineno(node: ast.ClassDef | MethodNode) -> int:
decorator_lines = [decorator.lineno for decorator in node.decorator_list]
if decorator_lines:
return min(decorator_lines)
return node.lineno
def line_has_ignore_marker(line: str) -> bool:
_, marker, comment = line.partition("#")
if not marker:
return False
normalized = comment.lower()
return any(ignore_marker in normalized for ignore_marker in IGNORE_COMMENT_MARKERS)
def node_ignore_scan_end_lineno(node: ast.ClassDef | MethodNode) -> int:
if isinstance(node, ast.ClassDef):
return node.lineno
return node.end_lineno or node.lineno
def node_has_ignore_comment(lines: Sequence[str], node: ast.ClassDef | MethodNode) -> bool:
start = node_start_lineno(node)
end = node_ignore_scan_end_lineno(node)
if any(line_has_ignore_marker(line) for line in lines[start - 1 : end]):
return True
line_index = start - 2
while line_index >= 0:
stripped = lines[line_index].strip()
if not stripped:
line_index -= 1
continue
if not stripped.startswith("#"):
break
if line_has_ignore_marker(lines[line_index]):
return True
line_index -= 1
return False
def checks_for_file(file_path: Path, repo_root: Path) -> list[ContractCheck]:
source = file_path.read_text(encoding="utf-8")
lines = source.splitlines()
module = ast.parse(source, filename=str(file_path))
checks: list[ContractCheck] = []
for node in module.body:
if not isinstance(node, ast.ClassDef):
continue
if node_has_ignore_comment(lines, node):
continue
class_routes = routes_from_decorators(node.decorator_list)
class_documented = response_docs_from_decorators(node.decorator_list)
for item in node.body:
if not isinstance(item, ast.FunctionDef | ast.AsyncFunctionDef) or item.name not in HTTP_METHODS:
continue
if node_has_ignore_comment(lines, item):
continue
routes = routes_from_decorators(item.decorator_list) or class_routes
if not routes:
continue
documented = {**class_documented, **response_docs_from_decorators(item.decorator_list)}
# Method-level @response decorators override class-level defaults for
# the same status code, matching Flask-RESTX's common controller style.
actual = actual_responses_for_method(item)
for route in routes:
checks.append(
classify_method(
actual_responses=actual,
class_name=node.name,
documented_responses=documented,
file_path=file_path,
method=item,
repo_root=repo_root,
route=route,
)
)
return checks
def as_jsonable(check: ContractCheck) -> dict[str, Any]:
data = asdict(check)
# pyrefly: ignore [bad-assignment]
data["documented"] = {str(status): model for status, model in check.documented.items()}
return data
def print_text_report(checks: Sequence[ContractCheck], *, include_unknown: bool, include_valid: bool) -> None:
counts = Counter(check.classification for check in checks)
sys.stdout.write(
"Response contract lint: "
f"{counts['valid']} valid, "
f"{counts['mismatch']} mismatch, "
f"{counts['refactorable']} refactorable, "
f"{counts['unknown']} unknown\n"
)
for classification in ("mismatch", "refactorable", "unknown", "valid"):
filtered = [check for check in checks if check.classification == classification]
if classification == "unknown" and not include_unknown:
continue
if classification == "valid" and not include_valid:
continue
if not filtered:
continue
sys.stdout.write(f"\n{classification.upper()}:\n")
for check in filtered:
location = f"{check.file}:{check.line} {check.class_name}.{check.method.upper()} {check.route}"
sys.stdout.write(f"- {location}: {check.reason}\n")
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"paths",
nargs="*",
help="Files or directories to lint. Defaults to Flask controller directories.",
)
parser.add_argument("--include-unknown", action="store_true", help="Print unknown route methods in output.")
parser.add_argument("--include-valid", action="store_true", help="Print valid route methods in text output.")
parser.add_argument("--json", action="store_true", help="Emit machine-readable JSON.")
parser.add_argument(
"--fail-on-mismatch",
action="store_true",
help="Treat mismatched response contracts as failures. By default this linter is report-only.",
)
parser.add_argument(
"--fail-on-unknown",
action="store_true",
help="Treat unknown route methods as failures. By default this linter is report-only.",
)
return parser.parse_args()
def main() -> int:
args = parse_args()
api_root = Path(__file__).resolve().parents[1]
repo_root = api_root.parent
raw_paths = args.paths or list(DEFAULT_CONTROLLER_DIRS)
paths = [path if path.is_absolute() else api_root / path for path in map(Path, raw_paths)]
checks: list[ContractCheck] = []
for file_path in iter_controller_files(paths):
checks.extend(checks_for_file(file_path.resolve(), repo_root))
checks.sort(key=lambda check: (check.classification, check.file, check.line, check.method))
if args.json:
grouped = defaultdict(list)
for check in checks:
if check.classification == "unknown" and not args.include_unknown:
continue
grouped[check.classification].append(as_jsonable(check))
sys.stdout.write(f"{json.dumps(grouped, indent=2, sort_keys=True)}\n")
else:
print_text_report(
checks,
include_unknown=bool(args.include_unknown),
include_valid=bool(args.include_valid),
)
has_mismatch = any(check.classification == "mismatch" for check in checks)
has_unknown = any(check.classification == "unknown" for check in checks)
return int((bool(args.fail_on_mismatch) and has_mismatch) or (bool(args.fail_on_unknown) and has_unknown))
if __name__ == "__main__":
raise SystemExit(main())
+57
View File
@@ -0,0 +1,57 @@
# API unit-test sharding
The workflow plans shards before pytest collection. Files with recorded durations
are sorted longest first and assigned to the least-loaded shard. Unseen files are
assigned round robin. File discovery always comes from the current checkout, so
stale history cannot omit new tests or reintroduce deleted files.
Like PyTorch's `tools/testing/test_selections.py`, oversized files become logical
parts. Our default is 60 summed testcase seconds per part, capped at the number of
CI shards. This is worker work, including setup/call/teardown, not wall time; the
threshold corresponds roughly to 30 seconds of work on two workers. Each case's
SHA-256 hash assigns it to one part, including new parametrized cases. Only files
assigned to a runner are collected; shared oversized files are collected on each
assigned runner. All workers use the same frozen plan. Ordinary xdist scheduling
continues inside each runner.
Successful coordinator reports are uploaded as two small timing artifacts. The
public [dify-test-infra](https://github.com/langgenius/dify-test-infra) repository
collects complete observations from successful CI on merged Dify PRs daily at
03:05 UTC. It averages up to five recent samples and publishes `stats/test-times.json`
on its `generated-stats` branch. Metadata records the source runs. No timing data
or generated history is committed to Dify itself.
Every PR, including fork PRs, downloads that same public baseline without a token.
Unmerged PR observations do not update the shared baseline automatically.
Keeping the producer in the statistics repository avoids needing a cross-repository write token in Dify or additional
main-branch test runs.
A single planning job downloads the baseline once and publishes both the input
and assignment plan as run artifacts. All shards therefore use the same snapshot,
even if statistics are updated during a run. Downloads have bounded timeouts and
retries; missing or invalid JSON falls back to round-robin allocation. Successful
unit runs continue publishing observations for the external updater; failed or
incomplete runs are not accepted as baseline samples.
This trades an extra planning job for consistent assignments. Timing noise,
shared fixtures, collection, and an indivisible slow case can still produce
imbalance. Logical splitting can duplicate expensive module/session setup. Compare both
shard test steps and the end-to-end API completion time when evaluating a change.
Local examples (from repository root):
```sh
uv run --project api python api/dev/pytest_sharding.py \
--shard-index 1 --shard-total 2 --durations /tmp/durations.json \
--write-plan /tmp/plan.json --ignore api/tests/unit_tests/controllers \
api/tests/unit_tests api/providers/vdb/*/tests/unit_tests api/providers/trace/*/tests/unit_tests
uv run --project api python api/dev/pytest_sharding.py \
--shard-index 1 --shard-total 2 --plan /tmp/plan.json api > /tmp/files.txt
uv run --project api pytest --file-shard-plan /tmp/plan.json \
--shard-index 1 --shard-total 2 --write-test-durations /tmp/durations-1.json \
@/tmp/files.txt
```
Repeat the last two commands for shard 2. Merge their file durations by addition
before using them as the next history snapshot. Keep generated files outside the
checkout locally.
+117
View File
@@ -0,0 +1,117 @@
"""Select pytest files before collection so workers do not import other shards."""
import argparse
import hashlib
import json
import math
import sys
from collections.abc import Sequence
from pathlib import Path
def load_durations(path: Path) -> dict[str, float]:
"""History is optional and never controls which files are discovered."""
try:
data = json.loads(path.read_text())
except (OSError, ValueError):
return {}
if not isinstance(data, dict):
return {}
return {
name: float(value)
for name, value in data.items()
if isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) and value >= 0
}
def build_plan(
files: Sequence[Path], durations: dict[str, float], total: int, threshold: float
) -> dict[str, list[int]]:
"""Longest estimated work first; split oversized files into logical parts.
Each list maps a file's hash partitions to one-based CI shard numbers.
All CI shards must consume the same plan. Unknown files use round robin.
Estimates are summed worker seconds, not predicted job wall time.
"""
if total < 1 or not math.isfinite(threshold) or threshold <= 0:
raise ValueError("shard-total and split threshold must be positive")
plan: dict[str, list[int]] = {}
tasks: list[tuple[float, str, int]] = []
unknown: list[str] = []
for path in sorted(set(files)):
name = path.as_posix()
duration = durations.get(name)
if duration is None:
unknown.append(name)
continue
parts = min(total, max(1, math.ceil(duration / threshold)))
plan[name] = [0] * parts
tasks.extend((duration / parts, name, part) for part in range(parts))
loads = [0.0] * total
for duration, name, part in sorted(tasks, key=lambda task: (-task[0], task[1], task[2])):
target = min(range(total), key=lambda index: (loads[index], index))
plan[name][part] = target + 1
loads[target] += duration
for index, name in enumerate(unknown):
plan[name] = [index % total + 1]
return plan
def case_shard(nodeid: str, targets: Sequence[int]) -> int:
"""Stable across processes, including newly added parametrized cases."""
digest = hashlib.sha256(nodeid.encode()).digest()
return targets[int.from_bytes(digest[:8], "big") % len(targets)]
def select_test_files(
roots: Sequence[Path], *, shard_index: int, shard_total: int, ignored: Sequence[Path] = ()
) -> list[Path]:
"""Return a deterministic file shard using pytest's default filename patterns."""
if shard_total < 1 or not 1 <= shard_index <= shard_total:
raise ValueError("shard-index must be between 1 and shard-total")
files = sorted(
{
path
for root in roots
for path in root.rglob("*.py")
if (path.name.startswith("test_") or path.name.endswith("_test.py"))
and not any(path.is_relative_to(directory) for directory in ignored)
}
)
return files[shard_index - 1 :: shard_total]
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--shard-index", type=int, required=True)
parser.add_argument("--shard-total", type=int, required=True)
parser.add_argument("--ignore", type=Path, action="append", default=[])
parser.add_argument("--durations", type=Path)
parser.add_argument("--write-plan", type=Path)
parser.add_argument("--plan", type=Path)
parser.add_argument("--split-threshold", type=float, default=60, help="Summed testcase seconds per logical part.")
parser.add_argument("roots", type=Path, nargs="+")
args = parser.parse_args()
if args.write_plan:
files = select_test_files(args.roots, shard_index=1, shard_total=1, ignored=args.ignore)
durations = load_durations(args.durations) if args.durations else {}
plan = build_plan(files, durations, args.shard_total, args.split_threshold)
args.write_plan.write_text(json.dumps(plan, sort_keys=True) + "\n")
sys.stderr.write(
f"Planned {len(files)} files; {sum(name in durations for name in plan)} have historical durations.\n"
)
return
if args.plan:
plan = json.loads(args.plan.read_text())
files = [Path(name) for name, targets in sorted(plan.items()) if args.shard_index in targets]
else:
files = select_test_files(
args.roots, shard_index=args.shard_index, shard_total=args.shard_total, ignored=args.ignore
)
if not files:
parser.error("No test files selected; refusing to fall back to pytest's default discovery")
sys.stdout.write("\n".join(path.as_posix() for path in files) + "\n")
if __name__ == "__main__":
main()