chore: 纳入本地 Dify 1.19.0 源码及品牌与免登改动
演示目录只保留可重建的源码。运行密钥、数据卷、依赖和带口令的本地配置不进入仓库。 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
60158935f4
commit
6519682885
15831 files changed
+3063428
No files matched your search
@@ -0,0 +1 @@
|
||||
"""Development tooling for Dify API contracts and checks."""
|
||||
@@ -0,0 +1,360 @@
|
||||
"""Local, idle-expiring OpenAPI export cache. Contract bytes never touch the filesystem.
|
||||
|
||||
The POSIX server owns a private Unix socket and an empty lifecycle lock. It caches
|
||||
the document in RAM. A single-threaded snapshot preloads only dependency modules,
|
||||
never controllers. Each refresh imports controllers in a fresh fork; edits to a
|
||||
preloaded dependency re-exec the snapshot before exporting. Unsupported platforms
|
||||
or unsafe snapshots use the ordinary fresh-process exporter.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import hashlib
|
||||
import importlib.machinery
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import select
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Any, Never
|
||||
|
||||
API_ROOT = Path(__file__).resolve().parents[1]
|
||||
SOURCE_ROOTS = (API_ROOT, API_ROOT.parent / "dify-agent/src")
|
||||
PACKAGE_INPUTS = (API_ROOT / "pyproject.toml", API_ROOT / "uv.lock", API_ROOT.parent / "dify-agent/pyproject.toml")
|
||||
TIMEOUT = 90
|
||||
IDLE_TIMEOUT = 120
|
||||
MAX_BYTES = 64 * 1024 * 1024
|
||||
|
||||
|
||||
def source_imports() -> None:
|
||||
"""Read editable backend modules from source, preserving installed-library bytecode caches."""
|
||||
original = importlib.machinery.SourceFileLoader.get_code
|
||||
|
||||
def get_code(self: importlib.machinery.SourceFileLoader, fullname: str):
|
||||
path = Path(self.path)
|
||||
if any(path.is_relative_to(root) for root in SOURCE_ROOTS) and ".venv" not in path.parts:
|
||||
return self.source_to_code(self.get_data(self.path), self.path)
|
||||
return original(self, fullname)
|
||||
|
||||
importlib.machinery.SourceFileLoader.get_code = get_code
|
||||
sys.dont_write_bytecode = True
|
||||
|
||||
|
||||
def source_state() -> dict[str, tuple[int, int, int]]:
|
||||
"""Match the Console descriptor's inputs, excluding tests and local environments."""
|
||||
files = [*PACKAGE_INPUTS, Path.cwd() / ".env"]
|
||||
for root in SOURCE_ROOTS:
|
||||
for directory, children, names in os.walk(root):
|
||||
children[:] = [name for name in children if name not in {".venv", ".git", "__pycache__"}]
|
||||
if Path(directory) == API_ROOT:
|
||||
children[:] = [name for name in children if name != "tests"]
|
||||
files.extend(Path(directory) / name for name in names if name.endswith(".py"))
|
||||
return {str(file): file_stamp(file) for file in files}
|
||||
|
||||
|
||||
def file_stamp(file: Path) -> tuple[int, int, int]:
|
||||
try:
|
||||
stat = file.stat()
|
||||
return stat.st_mtime_ns, stat.st_ctime_ns, stat.st_size
|
||||
except FileNotFoundError:
|
||||
return 0, 0, 0
|
||||
|
||||
|
||||
def fresh_export() -> bytes:
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-B",
|
||||
"-c",
|
||||
"import sys;sys.path.insert(0,sys.argv[1]);"
|
||||
"from dev.cached_console_openapi import source_imports;source_imports();"
|
||||
"from dev.export_console_openapi import main;main()",
|
||||
str(API_ROOT),
|
||||
],
|
||||
capture_output=True,
|
||||
timeout=TIMEOUT,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode:
|
||||
raise RuntimeError(result.stderr.decode(errors="replace")[-16000:])
|
||||
return result.stdout
|
||||
|
||||
|
||||
def single_threaded() -> bool:
|
||||
"""Check native threads too: threading.active_count misses C library pools."""
|
||||
try:
|
||||
if sys.platform.startswith("linux"):
|
||||
return len(os.listdir("/proc/self/task")) == 1
|
||||
if sys.platform == "darwin":
|
||||
# Darwin -M always prints its thread header, even with -o pid=.
|
||||
output = subprocess.check_output(["ps", "-M", "-p", str(os.getpid())], timeout=5)
|
||||
return len(output.strip().splitlines()) == 2
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
def preload() -> tuple[bool, dict[str, tuple[int, int, int]]]:
|
||||
"""Load schema dependencies without retaining route registries or controller classes."""
|
||||
import urllib.request
|
||||
|
||||
getproxies = urllib.request.getproxies
|
||||
try:
|
||||
if sys.platform == "darwin":
|
||||
# macOS SystemConfiguration starts native threads. Resolve its proxy
|
||||
# settings in a helper, preserving the values without forking those threads.
|
||||
proxies = json.loads(
|
||||
subprocess.check_output(
|
||||
[sys.executable, "-c", "import json,urllib.request;print(json.dumps(urllib.request.getproxies()))"],
|
||||
timeout=10,
|
||||
)
|
||||
)
|
||||
urllib.request.getproxies = lambda: proxies
|
||||
sys.path.insert(0, str(API_ROOT))
|
||||
from dev.generate_swagger_specs import apply_runtime_defaults
|
||||
|
||||
apply_runtime_defaults()
|
||||
# Keep runtime composition (and libraries with native thread pools) out
|
||||
# of the snapshot. These pure model dependencies account for much of startup.
|
||||
import dify_agent.protocol.schemas
|
||||
|
||||
import core.mcp.types
|
||||
import graphon.nodes.parameter_extractor.entities
|
||||
import models
|
||||
import services.knowledge_fs.product_dto
|
||||
|
||||
_ = core.mcp.types, dify_agent.protocol.schemas, graphon.nodes.parameter_extractor.entities
|
||||
_ = models, services.knowledge_fs.product_dto
|
||||
safe = not any(name.startswith("controllers.") for name in sys.modules) and single_threaded()
|
||||
except Exception:
|
||||
# A cache optimization must not prevent a normal export from working.
|
||||
safe = False
|
||||
finally:
|
||||
urllib.request.getproxies = getproxies
|
||||
dependencies = {}
|
||||
for module in tuple(sys.modules.values()):
|
||||
filename = getattr(module, "__file__", None) # guard-ignore: no-new-getattr -- optional module source metadata
|
||||
if isinstance(filename, str):
|
||||
dependencies[str(Path(filename).resolve())] = file_stamp(Path(filename))
|
||||
for filename in (*PACKAGE_INPUTS, Path.cwd() / ".env"):
|
||||
dependencies[str(filename)] = file_stamp(filename)
|
||||
return safe, dependencies
|
||||
|
||||
|
||||
def export_worker(write_fd: int) -> Never:
|
||||
"""Run the isolated exporter and keep its pipe open through process exit."""
|
||||
try:
|
||||
from dev.export_console_openapi import main
|
||||
|
||||
output = io.StringIO()
|
||||
with contextlib.redirect_stdout(output):
|
||||
main()
|
||||
payload = b"OK\n" + output.getvalue().encode()
|
||||
except BaseException:
|
||||
payload = b"ERROR\n" + traceback.format_exc().encode()
|
||||
try:
|
||||
with os.fdopen(write_fd, "wb") as pipe:
|
||||
pipe.write(payload)
|
||||
pipe.flush()
|
||||
# Keep the pipe open through process exit. Closing it first lets
|
||||
# the parent kill a successful worker while exit instrumentation
|
||||
# (for example, subprocess coverage) is still flushing its data.
|
||||
os._exit(0)
|
||||
finally:
|
||||
os._exit(1)
|
||||
|
||||
|
||||
def fork_export() -> bytes:
|
||||
"""Export from a clean controller registry; the parent snapshot is never mutated."""
|
||||
read_fd, write_fd = os.pipe()
|
||||
pid = os.fork()
|
||||
if pid == 0:
|
||||
os.close(read_fd)
|
||||
export_worker(write_fd)
|
||||
os.close(write_fd)
|
||||
chunks: list[bytes] = []
|
||||
size = 0
|
||||
deadline = time.monotonic() + TIMEOUT
|
||||
try:
|
||||
while True:
|
||||
if not select.select([read_fd], [], [], max(0, deadline - time.monotonic()))[0]:
|
||||
raise TimeoutError("OpenAPI refresh timed out")
|
||||
chunk = os.read(read_fd, 65536)
|
||||
if not chunk:
|
||||
break
|
||||
size += len(chunk)
|
||||
if size > MAX_BYTES:
|
||||
raise ValueError("OpenAPI export exceeds the response limit")
|
||||
chunks.append(chunk)
|
||||
result = b"".join(chunks)
|
||||
status, _, payload = result.partition(b"\n")
|
||||
if status != b"OK":
|
||||
raise RuntimeError(payload.decode(errors="replace") or "OpenAPI worker exited without a response")
|
||||
return payload
|
||||
finally:
|
||||
os.close(read_fd)
|
||||
# A worker can fail after closing its pipe. Never leave it running or unreaped.
|
||||
with contextlib.suppress(ProcessLookupError):
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
os.waitpid(pid, 0)
|
||||
|
||||
|
||||
def socket_path() -> Path:
|
||||
directory = Path(tempfile.gettempdir()) / f"dify-openapi-{os.getuid()}"
|
||||
# macOS Unix socket paths have a small length limit.
|
||||
if len(str(directory)) > 55:
|
||||
directory = Path("/tmp") / f"dify-openapi-{os.getuid()}"
|
||||
directory.mkdir(mode=0o700, exist_ok=True)
|
||||
stat = directory.lstat()
|
||||
if directory.is_symlink() or stat.st_uid != os.getuid() or stat.st_mode & 0o077:
|
||||
raise PermissionError("OpenAPI IPC directory must be private to the current user")
|
||||
environment = {
|
||||
key: value
|
||||
for key, value in os.environ.items()
|
||||
if key not in {"_", "PWD", "OLDPWD", "SHLVL", "UV_RUN_RECURSION_DEPTH"}
|
||||
}
|
||||
key = hashlib.sha256(json.dumps([str(API_ROOT), os.getcwd(), sys.executable, environment], sort_keys=True).encode())
|
||||
key.update(Path(__file__).read_bytes())
|
||||
return directory / f"{key.hexdigest()[:24]}.sock"
|
||||
|
||||
|
||||
def serve(path: Path, inherited: list[str]) -> None:
|
||||
import fcntl
|
||||
|
||||
if inherited:
|
||||
listener = socket.socket(fileno=int(inherited[0]))
|
||||
lock_fd = int(inherited[1])
|
||||
else:
|
||||
lock_fd = os.open(path.with_suffix(".lock"), os.O_CREAT | os.O_RDWR, 0o600)
|
||||
try:
|
||||
fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
os.close(lock_fd)
|
||||
return
|
||||
path.unlink(missing_ok=True)
|
||||
listener = socket.socket(socket.AF_UNIX)
|
||||
listener.bind(str(path))
|
||||
listener.listen(16)
|
||||
listener.settimeout(IDLE_TIMEOUT)
|
||||
baseline = source_state()
|
||||
safe, dependencies = preload()
|
||||
snapshot_dirty = source_state() != baseline
|
||||
cached_state: dict[str, tuple[int, int, int]] | None = None
|
||||
cached = b""
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
connection, _ = listener.accept()
|
||||
except TimeoutError:
|
||||
return
|
||||
with connection:
|
||||
connection.settimeout(TIMEOUT)
|
||||
try:
|
||||
request = b""
|
||||
while not request.endswith(b"\n") and len(request) < 16:
|
||||
part = connection.recv(16 - len(request))
|
||||
if not part:
|
||||
break
|
||||
request += part
|
||||
if request == b"stop\n":
|
||||
connection.sendall(b"OK\n")
|
||||
return
|
||||
if request != b"export\n":
|
||||
continue
|
||||
current = source_state()
|
||||
stale = (
|
||||
snapshot_dirty
|
||||
or set(current) != set(baseline)
|
||||
or any(file_stamp(Path(file)) != stamp for file, stamp in dependencies.items())
|
||||
)
|
||||
if stale:
|
||||
# No reload(sys.modules): restart all preloaded dependencies.
|
||||
connection.sendall(b"RETRY\n")
|
||||
connection.close()
|
||||
os.set_inheritable(listener.fileno(), True)
|
||||
os.set_inheritable(lock_fd, True)
|
||||
os.execv(
|
||||
sys.executable,
|
||||
[sys.executable, __file__, "--serve", str(path), str(listener.fileno()), str(lock_fd)],
|
||||
)
|
||||
if current != cached_state:
|
||||
output = fork_export() if safe and single_threaded() else fresh_export()
|
||||
# Do not publish a contract assembled while source files changed.
|
||||
if source_state() != current:
|
||||
connection.sendall(b"RETRY\n")
|
||||
continue
|
||||
payload: Any = json.loads(output)
|
||||
if not isinstance(payload, dict) or set(payload) != {"console", "fast"}:
|
||||
raise ValueError("Invalid Console export response")
|
||||
cached, cached_state = output, current
|
||||
connection.sendall(b"OK\n" + cached)
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
pass
|
||||
except Exception:
|
||||
with contextlib.suppress(OSError):
|
||||
connection.sendall(b"ERROR\n" + traceback.format_exc().encode()[-16000:])
|
||||
finally:
|
||||
listener.close()
|
||||
path.unlink(missing_ok=True)
|
||||
os.close(lock_fd)
|
||||
|
||||
|
||||
def request_export(*, stop: bool = False) -> bytes:
|
||||
if not hasattr(os, "fork"):
|
||||
return b"" if stop else fresh_export()
|
||||
path = socket_path()
|
||||
deadline = time.monotonic() + TIMEOUT
|
||||
launched = False
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with socket.socket(socket.AF_UNIX) as connection:
|
||||
connection.settimeout(max(0.1, deadline - time.monotonic()))
|
||||
connection.connect(str(path))
|
||||
connection.sendall(b"stop\n" if stop else b"export\n")
|
||||
chunks: list[bytes] = []
|
||||
size = 0
|
||||
while chunk := connection.recv(65536):
|
||||
size += len(chunk)
|
||||
if size > MAX_BYTES:
|
||||
raise ValueError("OpenAPI response exceeds the limit")
|
||||
chunks.append(chunk)
|
||||
status, _, payload = b"".join(chunks).partition(b"\n")
|
||||
if status == b"OK":
|
||||
return payload
|
||||
if status == b"ERROR":
|
||||
raise RuntimeError(payload.decode(errors="replace"))
|
||||
if status != b"RETRY":
|
||||
raise RuntimeError("Incomplete OpenAPI cache response")
|
||||
except (FileNotFoundError, ConnectionRefusedError):
|
||||
if stop:
|
||||
return b""
|
||||
if not launched:
|
||||
subprocess.Popen(
|
||||
[sys.executable, __file__, "--serve", str(path)],
|
||||
start_new_session=True,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
launched = True
|
||||
time.sleep(0.03)
|
||||
raise TimeoutError("OpenAPI cache did not respond in time")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Timestamp-based .pyc files can serve stale code after same-size edits within one second.
|
||||
source_imports()
|
||||
if sys.argv[1:2] == ["--serve"]:
|
||||
serve(Path(sys.argv[2]), sys.argv[3:])
|
||||
elif sys.argv[1:] == ["--stop"]:
|
||||
request_export(stop=True)
|
||||
else:
|
||||
sys.stdout.buffer.write(request_export())
|
||||
@@ -0,0 +1,360 @@
|
||||
"""Enforce focused KnowledgeFS coverage without hiding integration-critical glue.
|
||||
|
||||
The primary threshold aggregates statement and branch coverage for every Dify
|
||||
module owned by the KnowledgeFS integration. Large pre-existing Dify modules
|
||||
that only contain narrow integration hooks are checked with changed-line
|
||||
coverage instead, so unrelated legacy code cannot dilute or inflate the gate.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import TypedDict, cast
|
||||
|
||||
WORKSPACE_ROOT = Path(__file__).resolve().parents[2]
|
||||
NON_CORE_COVERAGE_ALLOWLIST = frozenset(
|
||||
{
|
||||
"api/dev/check_knowledge_fs_coverage.py",
|
||||
"api/dev/generate_knowledge_fs_contract.py",
|
||||
"api/dev/knowledge_fs_product_contract.py",
|
||||
"api/migrations/versions/2026_07_21_1200-a4e7c2f91b30_add_knowledge_fs_control_plane.py",
|
||||
"api/migrations/versions/2026_07_21_1300-b7f2a9d41c60_add_knowledge_fs_cutover.py",
|
||||
"api/migrations/versions/2026_07_21_1400-c8e31b7d52a4_add_knowledge_fs_cleanup_authorization.py",
|
||||
"api/migrations/versions/2026_07_21_1500-d4f6e8a1c305_add_knowledge_fs_remote_freeze_evidence.py",
|
||||
"api/migrations/versions/2026_07_21_1600-e5a7c9b2d416_add_knowledge_fs_cleanup_completion.py",
|
||||
"api/migrations/versions/2026_08_10_1200-7c1e9a4b2d60_add_knowledge_fs_staged_uploads.py",
|
||||
"api/migrations/versions/2026_08_13_1200-9d4e6f8a1b2c_add_knowledge_fs_space_tag_bindings.py",
|
||||
"api/migrations/versions/2026_08_17_1200-4f8b2c7d9e10_add_knowledge_fs_icon_background.py",
|
||||
"api/migrations/versions/2026_08_17_1200-f3a8c1d7e920_add_knowledge_fs_upgrade_jobs.py",
|
||||
"api/migrations/versions/2026_08_18_1200-e6b4a2c9d731_limit_active_knowledge_fs_upgrades.py",
|
||||
"api/migrations/versions/2026_10_21_1300-3edd6cd27f54_limit_active_knowledge_fs_upgrades_on_mysql.py",
|
||||
}
|
||||
)
|
||||
HUNK_HEADER = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CoverageSummary(TypedDict):
|
||||
"""Coverage.py counts required by the aggregate gate."""
|
||||
|
||||
covered_lines: int
|
||||
num_statements: int
|
||||
covered_branches: int
|
||||
num_branches: int
|
||||
|
||||
|
||||
class CoverageFile(TypedDict):
|
||||
"""Per-file coverage data emitted by ``coverage json``."""
|
||||
|
||||
executed_lines: list[int]
|
||||
missing_lines: list[int]
|
||||
summary: CoverageSummary
|
||||
|
||||
|
||||
class CoverageReport(TypedDict):
|
||||
"""Relevant top-level shape of a coverage.py JSON report."""
|
||||
|
||||
files: dict[str, CoverageFile]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CoverageTotals:
|
||||
"""Covered and measurable units for one gate surface."""
|
||||
|
||||
covered: int
|
||||
total: int
|
||||
|
||||
@property
|
||||
def percent(self) -> float:
|
||||
return 100.0 if self.total == 0 else self.covered * 100 / self.total
|
||||
|
||||
|
||||
class CoverageGateError(RuntimeError):
|
||||
"""Raised when coverage input is incomplete or below its threshold."""
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Validate focused module coverage and changed integration glue."""
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--coverage-json", type=Path, required=True)
|
||||
parser.add_argument("--glue-manifest", type=Path, required=True)
|
||||
parser.add_argument("--workspace-root", type=Path, default=WORKSPACE_ROOT)
|
||||
parser.add_argument("--base", default="")
|
||||
parser.add_argument("--minimum", type=float, default=90.0)
|
||||
parser.add_argument("--glue-minimum", type=float, default=90.0)
|
||||
args = parser.parse_args()
|
||||
|
||||
workspace_root = args.workspace_root.resolve()
|
||||
report = load_coverage_report(args.coverage_json)
|
||||
core_totals = validate_core_coverage(report, workspace_root=workspace_root, minimum=args.minimum)
|
||||
base = resolve_diff_base(workspace_root, args.base)
|
||||
glue_paths = load_glue_coverage_paths(args.glue_manifest, workspace_root=workspace_root)
|
||||
changed_lines = collect_changed_glue_lines(workspace_root, base, glue_paths=glue_paths)
|
||||
glue_totals = validate_changed_glue_coverage(
|
||||
report,
|
||||
changed_lines=changed_lines,
|
||||
minimum=args.glue_minimum,
|
||||
)
|
||||
logger.info(
|
||||
"Dify KnowledgeFS coverage passed: core lines+branches %.2f%% (%d/%d); changed glue lines %.2f%% (%d/%d)",
|
||||
core_totals.percent,
|
||||
core_totals.covered,
|
||||
core_totals.total,
|
||||
glue_totals.percent,
|
||||
glue_totals.covered,
|
||||
glue_totals.total,
|
||||
)
|
||||
|
||||
|
||||
def load_coverage_report(path: Path) -> CoverageReport:
|
||||
"""Load the detailed JSON report used by both coverage checks."""
|
||||
if not path.is_file():
|
||||
raise CoverageGateError(f"coverage JSON does not exist: {path}")
|
||||
document = json.loads(path.read_text())
|
||||
if not isinstance(document, dict) or not isinstance(document.get("files"), dict):
|
||||
raise CoverageGateError(f"coverage JSON has no files object: {path}")
|
||||
return cast(CoverageReport, document)
|
||||
|
||||
|
||||
def load_glue_coverage_paths(path: Path, *, workspace_root: Path) -> tuple[str, ...]:
|
||||
"""Load the workflow's authoritative NUL-delimited integration touchpoints."""
|
||||
if not path.is_file():
|
||||
raise CoverageGateError(f"KnowledgeFS glue manifest does not exist: {path}")
|
||||
try:
|
||||
paths = tuple(item.decode() for item in path.read_bytes().split(b"\0") if item)
|
||||
except UnicodeDecodeError as error:
|
||||
raise CoverageGateError(f"KnowledgeFS glue manifest is not UTF-8: {path}") from error
|
||||
if not paths:
|
||||
raise CoverageGateError("Dify KnowledgeFS glue coverage target set is empty")
|
||||
if len(paths) != len(set(paths)):
|
||||
raise CoverageGateError("Dify KnowledgeFS glue coverage manifest contains duplicate paths")
|
||||
invalid_paths = [
|
||||
candidate
|
||||
for candidate in paths
|
||||
if not candidate.startswith("api/")
|
||||
or not candidate.endswith(".py")
|
||||
or not (workspace_root / candidate).is_file()
|
||||
]
|
||||
if invalid_paths:
|
||||
raise CoverageGateError(f"Dify KnowledgeFS glue coverage paths are invalid: {', '.join(invalid_paths)}")
|
||||
return paths
|
||||
|
||||
|
||||
def is_core_coverage_path(path: str) -> bool:
|
||||
"""Return whether a repository-relative path belongs to the focused aggregate."""
|
||||
if not path.endswith(".py"):
|
||||
return False
|
||||
if path in {
|
||||
"api/commands/knowledge_fs.py",
|
||||
"api/configs/extra/knowledge_fs_config.py",
|
||||
"api/controllers/openapi/knowledge_fs.py",
|
||||
"api/events/event_handlers/sync_knowledge_fs_bindings_when_app_published_workflow_updated.py",
|
||||
"api/extensions/ext_knowledge_fs_observability.py",
|
||||
"api/services/dataset_knowledge_fs_upgrade_file_lease.py",
|
||||
"api/services/dataset_knowledge_fs_upgrade_service.py",
|
||||
"api/services/knowledge_fs_capability.py",
|
||||
}:
|
||||
return True
|
||||
if path.startswith(
|
||||
(
|
||||
"api/controllers/console/knowledge_fs/",
|
||||
"api/controllers/inner_api/knowledge_fs/",
|
||||
"api/controllers/service_api/knowledge_fs/",
|
||||
"api/services/knowledge_fs/",
|
||||
)
|
||||
):
|
||||
return True
|
||||
filename = path.rsplit("/", maxsplit=1)[-1]
|
||||
return (
|
||||
path.startswith("api/models/")
|
||||
and filename.startswith("knowledge_fs")
|
||||
or path.startswith("api/repositories/")
|
||||
and "knowledge_fs" in filename
|
||||
or path.startswith("api/tasks/")
|
||||
and "knowledge_fs" in filename
|
||||
)
|
||||
|
||||
|
||||
def discover_core_coverage_paths(workspace_root: Path) -> tuple[str, ...]:
|
||||
"""Classify every KnowledgeFS-named production file or fail closed."""
|
||||
named_paths = discover_knowledge_fs_production_paths(workspace_root)
|
||||
core_paths = {path for path in named_paths if is_core_coverage_path(path)}
|
||||
unclassified_paths = set(named_paths) - core_paths - NON_CORE_COVERAGE_ALLOWLIST
|
||||
if unclassified_paths:
|
||||
raise CoverageGateError(
|
||||
"unclassified Dify KnowledgeFS production files must join the core coverage scope or explicit allowlist: "
|
||||
+ ", ".join(sorted(unclassified_paths))
|
||||
)
|
||||
if not core_paths:
|
||||
raise CoverageGateError("Dify KnowledgeFS core coverage target set is empty")
|
||||
return tuple(sorted(core_paths))
|
||||
|
||||
|
||||
def discover_knowledge_fs_production_paths(workspace_root: Path) -> tuple[str, ...]:
|
||||
"""Mirror the workflow's dynamic KnowledgeFS filename discovery."""
|
||||
api_root = workspace_root / "api"
|
||||
if not api_root.is_dir():
|
||||
raise CoverageGateError(f"Dify API directory does not exist: {api_root}")
|
||||
paths: set[str] = set()
|
||||
for directory, child_directories, filenames in os.walk(api_root):
|
||||
current_directory = Path(directory)
|
||||
if current_directory == api_root:
|
||||
child_directories[:] = [name for name in child_directories if name not in {".venv", "storage", "tests"}]
|
||||
child_directories[:] = [name for name in child_directories if name != "__pycache__"]
|
||||
for filename in filenames:
|
||||
path = (current_directory / filename).relative_to(workspace_root).as_posix()
|
||||
if filename.endswith(".py") and "knowledge_fs" in path:
|
||||
paths.add(path)
|
||||
if not paths:
|
||||
raise CoverageGateError("Dify KnowledgeFS production target set is empty")
|
||||
return tuple(sorted(paths))
|
||||
|
||||
|
||||
def validate_core_coverage(
|
||||
report: CoverageReport,
|
||||
*,
|
||||
workspace_root: Path,
|
||||
minimum: float,
|
||||
) -> CoverageTotals:
|
||||
"""Require the exact combined line-and-branch percentage for all core files."""
|
||||
paths = discover_core_coverage_paths(workspace_root)
|
||||
missing_paths = [path for path in paths if path not in report["files"]]
|
||||
if missing_paths:
|
||||
raise CoverageGateError(f"coverage report is missing core files: {', '.join(missing_paths)}")
|
||||
|
||||
covered = 0
|
||||
total = 0
|
||||
for path in paths:
|
||||
summary = report["files"][path]["summary"]
|
||||
covered += summary["covered_lines"] + summary["covered_branches"]
|
||||
total += summary["num_statements"] + summary["num_branches"]
|
||||
if total == 0:
|
||||
raise CoverageGateError("Dify KnowledgeFS core coverage has no measurable statements or branches")
|
||||
|
||||
totals = CoverageTotals(covered=covered, total=total)
|
||||
_require_minimum(totals, minimum=minimum, label="Dify KnowledgeFS core line-and-branch coverage")
|
||||
return totals
|
||||
|
||||
|
||||
def resolve_diff_base(workspace_root: Path, preferred: str) -> str:
|
||||
"""Resolve an explicit event base, falling back to the previous commit for manual runs."""
|
||||
base = preferred.strip()
|
||||
if not base or set(base) == {"0"}:
|
||||
base = "HEAD^"
|
||||
result = subprocess.run(
|
||||
["git", "cat-file", "-e", f"{base}^{{commit}}"],
|
||||
cwd=workspace_root,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
detail = result.stderr.strip() or "commit is unavailable"
|
||||
raise CoverageGateError(f"cannot resolve coverage diff base {base}: {detail}")
|
||||
return base
|
||||
|
||||
|
||||
def collect_changed_glue_lines(
|
||||
workspace_root: Path,
|
||||
base: str,
|
||||
*,
|
||||
glue_paths: tuple[str, ...],
|
||||
) -> dict[str, set[int]]:
|
||||
"""Return added line numbers in the narrow Dify modules touched by this integration."""
|
||||
result = subprocess.run(
|
||||
[
|
||||
"git",
|
||||
"diff",
|
||||
"--no-ext-diff",
|
||||
"--no-color",
|
||||
"--unified=0",
|
||||
"--src-prefix=a/",
|
||||
"--dst-prefix=b/",
|
||||
base,
|
||||
"--",
|
||||
*glue_paths,
|
||||
],
|
||||
cwd=workspace_root,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
detail = result.stderr.strip() or "git diff failed"
|
||||
raise CoverageGateError(f"cannot collect KnowledgeFS glue diff from {base}: {detail}")
|
||||
return parse_added_lines(result.stdout)
|
||||
|
||||
|
||||
def parse_added_lines(diff: str) -> dict[str, set[int]]:
|
||||
"""Parse repository paths and added-side line numbers from a zero-context Git diff."""
|
||||
changed_lines: dict[str, set[int]] = {}
|
||||
current_path: str | None = None
|
||||
current_line: int | None = None
|
||||
for raw_line in diff.splitlines():
|
||||
if raw_line.startswith("diff --git "):
|
||||
current_line = None
|
||||
continue
|
||||
if raw_line.startswith("+++ "):
|
||||
candidate = raw_line[4:]
|
||||
current_path = candidate[2:] if candidate.startswith("b/") else None
|
||||
if current_path is not None:
|
||||
changed_lines.setdefault(current_path, set())
|
||||
current_line = None
|
||||
continue
|
||||
if raw_line.startswith("@@ "):
|
||||
match = HUNK_HEADER.match(raw_line)
|
||||
current_line = int(match.group(1)) if match is not None else None
|
||||
continue
|
||||
if current_path is None or current_line is None:
|
||||
continue
|
||||
if raw_line.startswith("+"):
|
||||
changed_lines[current_path].add(current_line)
|
||||
current_line += 1
|
||||
elif raw_line.startswith("-") or raw_line.startswith("\\"):
|
||||
continue
|
||||
else:
|
||||
current_line += 1
|
||||
return changed_lines
|
||||
|
||||
|
||||
def validate_changed_glue_coverage(
|
||||
report: CoverageReport,
|
||||
*,
|
||||
changed_lines: dict[str, set[int]],
|
||||
minimum: float,
|
||||
) -> CoverageTotals:
|
||||
"""Require added executable glue lines to be exercised by the focused unit suite."""
|
||||
covered = 0
|
||||
total = 0
|
||||
for path, lines in sorted(changed_lines.items()):
|
||||
if not lines:
|
||||
continue
|
||||
file_coverage = report["files"].get(path)
|
||||
if file_coverage is None:
|
||||
raise CoverageGateError(f"coverage report is missing changed glue file: {path}")
|
||||
executed_lines = set(file_coverage["executed_lines"])
|
||||
executable_lines = executed_lines | set(file_coverage["missing_lines"])
|
||||
changed_executable_lines = lines & executable_lines
|
||||
covered += len(changed_executable_lines & executed_lines)
|
||||
total += len(changed_executable_lines)
|
||||
|
||||
totals = CoverageTotals(covered=covered, total=total)
|
||||
_require_minimum(totals, minimum=minimum, label="Dify KnowledgeFS changed-glue line coverage")
|
||||
return totals
|
||||
|
||||
|
||||
def _require_minimum(totals: CoverageTotals, *, minimum: float, label: str) -> None:
|
||||
if totals.percent + 1e-12 < minimum:
|
||||
raise CoverageGateError(f"{label} is {totals.percent:.2f}%; minimum {minimum:.2f}%")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,105 @@
|
||||
"""Export community Console schemas to stdout for the OpenAPI content mapper."""
|
||||
|
||||
import ast
|
||||
import contextlib
|
||||
import inspect
|
||||
import json
|
||||
import sys
|
||||
import textwrap
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def source_location(target: Any) -> dict[str, Any] | None:
|
||||
"""Locate the actual class or function definition, skipping decorators."""
|
||||
try:
|
||||
target = inspect.unwrap(target)
|
||||
filename = inspect.getsourcefile(target)
|
||||
lines, start = inspect.getsourcelines(target)
|
||||
definition = ast.parse(textwrap.dedent("".join(lines))).body[0]
|
||||
except (OSError, TypeError, ValueError, SyntaxError):
|
||||
# Dynamically created or native handlers have no navigable Python source.
|
||||
return None
|
||||
if filename and isinstance(definition, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
return {
|
||||
"file": str(Path(filename).resolve()),
|
||||
"line": start + definition.lineno - 1,
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def resolve_source_type(source: str) -> Any:
|
||||
"""Resolve registered identities only, including nested classes and generic origins."""
|
||||
parts = source.split("[", 1)[0].split(".")
|
||||
for end in range(len(parts) - 1, 0, -1):
|
||||
target = sys.modules.get(".".join(parts[:end]))
|
||||
if target is None:
|
||||
continue
|
||||
for part in parts[end:]:
|
||||
if not inspect.ismodule(target) and not inspect.isclass(target):
|
||||
return None
|
||||
target = vars(target).get(part)
|
||||
return target if inspect.isclass(target) else None
|
||||
return None
|
||||
|
||||
|
||||
def attach_source_location(document: dict[str, Any], path: str, method: str, handler: Any) -> None:
|
||||
"""Attach editor-only origins to exported operations, leaving live schemas unchanged."""
|
||||
operation = document["paths"].get(path, {}).get(method.lower())
|
||||
if isinstance(operation, dict) and (location := source_location(handler)):
|
||||
operation["x-source-location"] = location
|
||||
|
||||
|
||||
def attach_api_sources(document: dict[str, Any], api: Any) -> None:
|
||||
"""Annotate exported RESTX models and operations from their registered Python owners."""
|
||||
from flask_restx.swagger import extract_path
|
||||
|
||||
from controllers.common.schema import registered_schema_sources
|
||||
|
||||
schemas = document.get("components", {}).get("schemas", {})
|
||||
for namespace in api.namespaces:
|
||||
for name, source in registered_schema_sources(namespace).items():
|
||||
if name in schemas and (location := source_location(resolve_source_type(source))):
|
||||
schemas[name]["x-source-location"] = location
|
||||
for resource in namespace.resources:
|
||||
cls = resource.resource
|
||||
for url in api.ns_urls(namespace, resource.urls):
|
||||
for method in cls.methods:
|
||||
handler = getattr(cls, method.lower(), None) # guard-ignore: no-new-getattr -- HTTP method
|
||||
attach_source_location(document, extract_path(url), method, handler)
|
||||
|
||||
|
||||
def export_all_with_sources() -> dict[str, dict[str, Any]]:
|
||||
"""Export every RESTX surface with editor metadata, without writing spec files."""
|
||||
from dev.generate_swagger_specs import SPEC_TARGETS, _target_apis, export_specs
|
||||
|
||||
documents = export_specs()
|
||||
apis = _target_apis()
|
||||
for target in SPEC_TARGETS:
|
||||
attach_api_sources(documents[target.filename], apis[target.route])
|
||||
return documents
|
||||
|
||||
|
||||
def main() -> None:
|
||||
# Backend initialization may log to stdout. Keep the protocol output pure JSON.
|
||||
with contextlib.redirect_stdout(sys.stderr):
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from dev.generate_fastopenapi_specs import export_fastopenapi_specs
|
||||
from dev.generate_swagger_specs import export_console_spec
|
||||
|
||||
documents = {
|
||||
"console": export_console_spec(),
|
||||
"fast": export_fastopenapi_specs()["fastopenapi-console-openapi.json"],
|
||||
}
|
||||
from controllers.console import api
|
||||
from controllers.fastopenapi import console_router
|
||||
|
||||
attach_api_sources(documents["console"], api)
|
||||
for path, method, handler in console_router.get_routes():
|
||||
attach_source_location(documents["fast"], f"/console/api{path}", method, handler)
|
||||
json.dump(documents, sys.stdout, sort_keys=True)
|
||||
sys.stdout.write("\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,646 @@
|
||||
"""Snapshot the billing and saas OpenAPI specs and generate the typed client in `services/billing_client`.
|
||||
|
||||
The snapshot is the generation input, so CI can regenerate and diff without the private dify-saas
|
||||
checkout. Only operations on `ALLOWED_OPERATIONS`, the schemas they reach, the `Error` body, and the
|
||||
`<Service>ErrorReason` of each proto service they belong to are snapshotted: dify is public, and every
|
||||
other dify-saas endpoint stays out of this repository.
|
||||
|
||||
The proto service is the operationId prefix before the underscore (`BillingEducation` for
|
||||
`BillingEducation_VerifyEducationIdentity`). Each one gets its own reason Literal, error classes, and
|
||||
`ServiceErrors`, so a caller catching `BillingEducationUpstreamError` only matches the reasons that
|
||||
service can return.
|
||||
|
||||
uv run --project api --dev python api/dev/generate_billing_client.py # snapshot + generate
|
||||
uv run --project api --dev python api/dev/generate_billing_client.py --from-snapshot # generate only
|
||||
uv run --project api --dev python api/dev/generate_billing_client.py --check # CI drift check
|
||||
|
||||
The dify-saas checkout defaults to `../dify-saas` next to the dify repository; override it with
|
||||
`DIFY_SAAS_REPO`.
|
||||
|
||||
Kratos serializes with protojson `EmitUnpopulated: true`, which the snapshot step records as presence:
|
||||
- Scalars, enums, lists, and maps are required and non-null.
|
||||
- Message fields are required but nullable, since unset messages are emitted as null. A message field
|
||||
the upstream schema lists in `required` (`google.api.field_behavior = REQUIRED`) is non-null.
|
||||
- Proto3 `optional` fields (annotated `nullable` in dify-saas) are synthetic oneofs, which protojson
|
||||
omits when unset, so they are optional and nullable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import difflib
|
||||
import json
|
||||
import keyword
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
import yaml
|
||||
from pydantic import BaseModel
|
||||
|
||||
API_ROOT = Path(__file__).resolve().parents[1]
|
||||
REPO_ROOT = API_ROOT.parent
|
||||
CLIENT_DIR = API_ROOT / "services" / "billing_client"
|
||||
GENERATOR = "api/dev/generate_billing_client.py"
|
||||
|
||||
logger = logging.getLogger("generate_billing_client")
|
||||
|
||||
type Service = Literal["billing", "saas"]
|
||||
type UpstreamName = Literal["BILLING", "BILLING_QUOTA", "SAAS", "NETWORK_ACCESS"]
|
||||
type JsonObject = dict[str, Any]
|
||||
|
||||
SERVICES: tuple[Service, ...] = ("billing", "saas")
|
||||
SCHEMA_PREFIXES: dict[Service, str] = {"billing": "dify.saas.api.billing.", "saas": "dify.saas.api.saas."}
|
||||
# BILLING_API_URL already ends with the API version.
|
||||
PATH_VERSION_PREFIX = "/v1"
|
||||
SNAPSHOT_COMMIT_KEY = "x-dify-saas-commit"
|
||||
HTTP_METHODS = ("get", "post", "put", "delete", "patch")
|
||||
LINE_LENGTH = 120
|
||||
BASE_MODEL_ATTRIBUTES = frozenset(dir(BaseModel))
|
||||
ERROR_SCHEMA = "Error"
|
||||
ERROR_REASON_SUFFIX = "ErrorReason"
|
||||
ERROR_CLASS_SUFFIXES = ("UpstreamError", "UpstreamUnavailableError", "UpstreamInvalidResponseError")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AllowedOperation:
|
||||
method: str
|
||||
path: str
|
||||
upstream: UpstreamName
|
||||
|
||||
|
||||
# Endpoints dify API calls today. Add an entry here, then rerun this script, to call a new one.
|
||||
ALLOWED_OPERATIONS: dict[Service, tuple[AllowedOperation, ...]] = {
|
||||
"billing": (
|
||||
AllowedOperation("GET", "/v1/subscription/info", "BILLING"),
|
||||
AllowedOperation("GET", "/v1/subscription/vector-space", "BILLING"),
|
||||
AllowedOperation("GET", "/v1/subscription/knowledge-rate-limit", "BILLING"),
|
||||
AllowedOperation("GET", "/v1/subscription/payment-link", "BILLING"),
|
||||
AllowedOperation("POST", "/v1/subscription/plan/batch", "BILLING"),
|
||||
AllowedOperation("GET", "/v1/subscription/cleanup/whitelist", "BILLING"),
|
||||
AllowedOperation("GET", "/v1/model-provider/payment-link", "BILLING"),
|
||||
AllowedOperation("GET", "/v1/invoices", "BILLING"),
|
||||
AllowedOperation("GET", "/v1/education/verify", "BILLING"),
|
||||
AllowedOperation("GET", "/v1/education/status", "BILLING"),
|
||||
AllowedOperation("POST", "/v1/education", "BILLING"),
|
||||
AllowedOperation("GET", "/v1/education/autocomplete", "BILLING"),
|
||||
AllowedOperation("POST", "/v1/compliance/download", "BILLING"),
|
||||
AllowedOperation("GET", "/v1/quota/info", "BILLING_QUOTA"),
|
||||
AllowedOperation("POST", "/v1/quota/reserve", "BILLING_QUOTA"),
|
||||
AllowedOperation("POST", "/v1/quota/commit", "BILLING_QUOTA"),
|
||||
AllowedOperation("POST", "/v1/quota/release", "BILLING_QUOTA"),
|
||||
AllowedOperation("GET", "/v1/quota/balance", "BILLING_QUOTA"),
|
||||
AllowedOperation("POST", "/v1/quota/consume-capped", "BILLING_QUOTA"),
|
||||
),
|
||||
"saas": (
|
||||
AllowedOperation("DELETE", "/v1/account", "SAAS"),
|
||||
AllowedOperation("GET", "/v1/account/in-freeze", "SAAS"),
|
||||
AllowedOperation("POST", "/v1/account/delete-feedback", "SAAS"),
|
||||
AllowedOperation("GET", "/v1/notifications/active", "SAAS"),
|
||||
AllowedOperation("POST", "/v1/notifications/{notificationId}/dismiss", "SAAS"),
|
||||
AllowedOperation("POST", "/v1/tenant_utms", "SAAS"),
|
||||
AllowedOperation("GET", "/v1/tenants/{tenantId}/network-access-groups", "NETWORK_ACCESS"),
|
||||
AllowedOperation("POST", "/v1/tenants/{tenantId}/network-access-groups", "NETWORK_ACCESS"),
|
||||
AllowedOperation("GET", "/v1/tenants/{tenantId}/network-access-groups/{groupId}", "NETWORK_ACCESS"),
|
||||
AllowedOperation("PUT", "/v1/tenants/{tenantId}/network-access-groups/{groupId}", "NETWORK_ACCESS"),
|
||||
AllowedOperation("DELETE", "/v1/tenants/{tenantId}/network-access-groups/{groupId}", "NETWORK_ACCESS"),
|
||||
AllowedOperation("GET", "/v1/tenants/{tenantId}/apps/{appId}/network-access-group", "NETWORK_ACCESS"),
|
||||
AllowedOperation("PUT", "/v1/tenants/{tenantId}/apps/{appId}/network-access-group", "NETWORK_ACCESS"),
|
||||
AllowedOperation("GET", "/v1/tenants/{tenantId}/apps/{appId}/network-access-group/status", "NETWORK_ACCESS"),
|
||||
AllowedOperation(
|
||||
"DELETE", "/v1/tenants/{tenantId}/apps/{appId}/network-access-group-binding", "NETWORK_ACCESS"
|
||||
),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
class GenerationError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def snapshot_path(service: Service) -> Path:
|
||||
return CLIENT_DIR / f"{service}.openapi.json"
|
||||
|
||||
|
||||
def module_path(service: Service) -> Path:
|
||||
return CLIENT_DIR / f"{service}.py"
|
||||
|
||||
|
||||
# --- snapshot -------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def dify_saas_root() -> Path:
|
||||
override = os.environ.get("DIFY_SAAS_REPO")
|
||||
return Path(override).resolve() if override else (REPO_ROOT.parent / "dify-saas").resolve()
|
||||
|
||||
|
||||
def _git(root: Path, *args: str) -> str:
|
||||
return subprocess.run(["git", "-C", str(root), *args], check=True, capture_output=True, text=True).stdout.strip()
|
||||
|
||||
|
||||
def dify_saas_commit(root: Path) -> str:
|
||||
if _git(root, "status", "--porcelain"):
|
||||
logger.warning("dify-saas at %s has uncommitted changes; the snapshot will not match its commit", root)
|
||||
branch = _git(root, "rev-parse", "--abbrev-ref", "HEAD")
|
||||
if branch != "main":
|
||||
logger.warning("dify-saas is on %s, not main", branch)
|
||||
try:
|
||||
behind = int(_git(root, "rev-list", "--count", "HEAD..origin/main"))
|
||||
except subprocess.CalledProcessError:
|
||||
logger.warning("dify-saas has no origin/main to compare against")
|
||||
else:
|
||||
if behind:
|
||||
logger.warning("dify-saas is %d commits behind origin/main (as of its last fetch)", behind)
|
||||
return _git(root, "rev-parse", "HEAD")
|
||||
|
||||
|
||||
def _rewrite_refs(node: Any, prefix: str) -> Any:
|
||||
if isinstance(node, dict):
|
||||
return {
|
||||
key: (_strip_ref(value, prefix) if key == "$ref" else _rewrite_refs(value, prefix))
|
||||
for key, value in node.items()
|
||||
}
|
||||
if isinstance(node, list):
|
||||
return [_rewrite_refs(item, prefix) for item in node]
|
||||
return node
|
||||
|
||||
|
||||
def _strip_ref(ref: str, prefix: str) -> str:
|
||||
head, _, name = ref.rpartition("/")
|
||||
return f"{head}/{name.removeprefix(prefix)}"
|
||||
|
||||
|
||||
def _ref_names(node: Any) -> set[str]:
|
||||
if isinstance(node, dict):
|
||||
names = {node["$ref"].rpartition("/")[2]} if "$ref" in node else set()
|
||||
for value in node.values():
|
||||
names |= _ref_names(value)
|
||||
return names
|
||||
if isinstance(node, list):
|
||||
names: set[str] = set()
|
||||
for item in node:
|
||||
names |= _ref_names(item)
|
||||
return names
|
||||
return set()
|
||||
|
||||
|
||||
def _annotate_presence(schema: JsonObject) -> JsonObject:
|
||||
properties: dict[str, JsonObject] = schema.get("properties", {})
|
||||
always_set = set(schema.get("required", []))
|
||||
annotated: dict[str, JsonObject] = {}
|
||||
required: list[str] = []
|
||||
for name, field in properties.items():
|
||||
if "$ref" in field and name not in always_set:
|
||||
# OpenAPI 3.0 ignores siblings of $ref, so nullable refs are wrapped in allOf.
|
||||
annotated[name] = {"allOf": [{"$ref": field["$ref"]}], "nullable": True}
|
||||
else:
|
||||
annotated[name] = field
|
||||
if "$ref" in field or not field.get("nullable"):
|
||||
required.append(name)
|
||||
result = {**schema, "properties": annotated}
|
||||
if required:
|
||||
result["required"] = required
|
||||
return result
|
||||
|
||||
|
||||
def proto_service(operation_id: str) -> str:
|
||||
prefix, separator, method = operation_id.partition("_")
|
||||
if not separator or not prefix or not method:
|
||||
raise GenerationError(f"operationId {operation_id!r} has no `<Service>_` prefix")
|
||||
return prefix
|
||||
|
||||
|
||||
def error_reason_schema(proto: str) -> str:
|
||||
return f"{proto}{ERROR_REASON_SUFFIX}"
|
||||
|
||||
|
||||
def build_snapshot(service: Service, source: JsonObject, commit: str) -> JsonObject:
|
||||
prefix = SCHEMA_PREFIXES[service]
|
||||
source_paths: dict[str, JsonObject] = source.get("paths", {})
|
||||
paths: dict[str, JsonObject] = {}
|
||||
for allowed in ALLOWED_OPERATIONS[service]:
|
||||
operation = source_paths.get(allowed.path, {}).get(allowed.method.lower())
|
||||
if operation is None:
|
||||
raise GenerationError(f"{service}: {allowed.method} {allowed.path} is not in the dify-saas spec")
|
||||
paths.setdefault(allowed.path, {})[allowed.method.lower()] = _rewrite_refs(operation, prefix)
|
||||
|
||||
source_schemas: dict[str, JsonObject] = {
|
||||
name.removeprefix(prefix): _rewrite_refs(schema, prefix)
|
||||
for name, schema in source.get("components", {}).get("schemas", {}).items()
|
||||
}
|
||||
error_schemas = {ERROR_SCHEMA}
|
||||
for allowed in ALLOWED_OPERATIONS[service]:
|
||||
operation_id = paths[allowed.path][allowed.method.lower()].get("operationId", "")
|
||||
reason_schema = error_reason_schema(proto_service(operation_id))
|
||||
if reason_schema not in source_schemas:
|
||||
raise GenerationError(
|
||||
f"{service}: {allowed.method} {allowed.path} ({operation_id}) has no "
|
||||
f"{prefix}{reason_schema} schema in the dify-saas spec"
|
||||
)
|
||||
error_schemas.add(reason_schema)
|
||||
if ERROR_SCHEMA not in source_schemas:
|
||||
raise GenerationError(f"{service}: error schema {prefix}{ERROR_SCHEMA} is not in the dify-saas spec")
|
||||
reachable: set[str] = set()
|
||||
pending = _ref_names(paths) | error_schemas
|
||||
while pending:
|
||||
name = pending.pop()
|
||||
if name in reachable:
|
||||
continue
|
||||
if name not in source_schemas:
|
||||
raise GenerationError(f"{service}: schema {name} is referenced but not defined")
|
||||
reachable.add(name)
|
||||
pending |= _ref_names(source_schemas[name])
|
||||
|
||||
return {
|
||||
"openapi": source["openapi"],
|
||||
"info": source.get("info", {}),
|
||||
SNAPSHOT_COMMIT_KEY: commit,
|
||||
"paths": paths,
|
||||
"components": {"schemas": {name: _annotate_presence(source_schemas[name]) for name in sorted(reachable)}},
|
||||
}
|
||||
|
||||
|
||||
def take_snapshots() -> dict[Service, JsonObject]:
|
||||
root = dify_saas_root()
|
||||
if not (root / "pkg" / "apis").is_dir():
|
||||
raise GenerationError(f"dify-saas checkout not found at {root}; set DIFY_SAAS_REPO")
|
||||
commit = dify_saas_commit(root)
|
||||
snapshots: dict[Service, JsonObject] = {}
|
||||
for service in SERVICES:
|
||||
source = yaml.safe_load((root / "pkg" / "apis" / service / "openapi.yaml").read_text())
|
||||
snapshots[service] = build_snapshot(service, source, commit)
|
||||
return snapshots
|
||||
|
||||
|
||||
def render_snapshot(snapshot: JsonObject) -> str:
|
||||
return json.dumps(snapshot, indent=2, ensure_ascii=False) + "\n"
|
||||
|
||||
|
||||
def load_snapshot(service: Service) -> JsonObject:
|
||||
snapshot: JsonObject = json.loads(snapshot_path(service).read_text())
|
||||
operations = {
|
||||
(method.upper(), path) for path, item in snapshot["paths"].items() for method in item if method in HTTP_METHODS
|
||||
}
|
||||
allowed = {(op.method, op.path) for op in ALLOWED_OPERATIONS[service]}
|
||||
if operations != allowed:
|
||||
raise GenerationError(
|
||||
f"{service} snapshot operations differ from ALLOWED_OPERATIONS: "
|
||||
f"missing {sorted(allowed - operations)}, unlisted {sorted(operations - allowed)}; rerun the snapshot"
|
||||
)
|
||||
return snapshot
|
||||
|
||||
|
||||
# --- generation -----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def snake_case(name: str) -> str:
|
||||
words = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", name)
|
||||
words = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1_\2", words)
|
||||
return re.sub(r"[^0-9a-zA-Z]+", "_", words).strip("_").lower()
|
||||
|
||||
|
||||
def python_name(name: str) -> str:
|
||||
candidate = snake_case(name)
|
||||
if not candidate.isidentifier() or keyword.iskeyword(candidate) or candidate in BASE_MODEL_ATTRIBUTES:
|
||||
candidate = f"{candidate}_"
|
||||
return candidate
|
||||
|
||||
|
||||
def operation_function_name(operation_id: str, service: Service) -> str:
|
||||
tag, _, method = operation_id.partition("_")
|
||||
tag = tag.removeprefix(service.capitalize())
|
||||
return python_name(f"{tag}_{method}")
|
||||
|
||||
|
||||
def python_type(schema: JsonObject, context: str) -> str:
|
||||
if "allOf" in schema:
|
||||
parts = schema["allOf"]
|
||||
if len(parts) != 1:
|
||||
raise GenerationError(f"{context}: only single-element allOf is supported")
|
||||
return python_type(parts[0], context)
|
||||
if "$ref" in schema:
|
||||
return schema["$ref"].rpartition("/")[2]
|
||||
schema_type = schema.get("type")
|
||||
if "enum" in schema:
|
||||
if schema_type != "string":
|
||||
raise GenerationError(f"{context}: only string enums are supported")
|
||||
return f"Literal[{', '.join(json.dumps(value) for value in schema['enum'])}]"
|
||||
if schema_type == "integer" or (schema_type == "string" and schema.get("format") in {"int64", "uint64"}):
|
||||
return "int"
|
||||
if schema_type == "string" and schema.get("format") == "date-time":
|
||||
return "datetime"
|
||||
if schema_type == "string":
|
||||
return "str"
|
||||
if schema_type == "number":
|
||||
return "float"
|
||||
if schema_type == "boolean":
|
||||
return "bool"
|
||||
if schema_type == "array":
|
||||
return f"list[{python_type(schema['items'], context)}]"
|
||||
if schema_type == "object" and isinstance(schema.get("additionalProperties"), dict):
|
||||
return f"dict[str, {python_type(schema['additionalProperties'], context)}]"
|
||||
raise GenerationError(f"{context}: unsupported schema {json.dumps(schema)}; give it a named message type")
|
||||
|
||||
|
||||
def field_type(schema: JsonObject, required: bool, context: str) -> str:
|
||||
annotation = python_type(schema, context)
|
||||
if schema.get("nullable") or not required:
|
||||
annotation = f"{annotation} | None"
|
||||
return annotation
|
||||
|
||||
|
||||
def model_order(schemas: dict[str, JsonObject]) -> list[str]:
|
||||
ordered: list[str] = []
|
||||
visiting: set[str] = set()
|
||||
|
||||
def visit(name: str) -> None:
|
||||
if name in ordered:
|
||||
return
|
||||
if name in visiting:
|
||||
raise GenerationError(f"recursive schema {name} is not supported")
|
||||
visiting.add(name)
|
||||
for dependency in sorted(_ref_names(schemas[name])):
|
||||
visit(dependency)
|
||||
visiting.discard(name)
|
||||
ordered.append(name)
|
||||
|
||||
for name in sorted(schemas):
|
||||
visit(name)
|
||||
return ordered
|
||||
|
||||
|
||||
def check_error_body(schema: JsonObject) -> None:
|
||||
properties: dict[str, JsonObject] = schema.get("properties", {})
|
||||
expected = {
|
||||
"code": {"type": "integer", "format": "int32"},
|
||||
"reason": {"type": "string"},
|
||||
"message": {"type": "string"},
|
||||
"metadata": {"type": "object", "additionalProperties": {"type": "string"}},
|
||||
}
|
||||
actual = {
|
||||
name: {key: value for key, value in field.items() if key != "description"} for name, field in properties.items()
|
||||
}
|
||||
if actual != expected:
|
||||
raise GenerationError(f"Error is now {json.dumps(actual)}; update UpstreamErrorBody and the transport")
|
||||
|
||||
|
||||
def render_schema(name: str, schema: JsonObject) -> list[str]:
|
||||
if name == ERROR_SCHEMA:
|
||||
check_error_body(schema)
|
||||
return []
|
||||
if "enum" in schema:
|
||||
return [f"type {name} = {python_type(schema, name)}"]
|
||||
return render_model(name, schema)
|
||||
|
||||
|
||||
def render_model(name: str, schema: JsonObject) -> list[str]:
|
||||
if schema.get("type") != "object":
|
||||
raise GenerationError(f"schema {name}: only object schemas can be top-level models")
|
||||
required = set(schema.get("required", []))
|
||||
lines = [f"class {name}(BillingModel):"]
|
||||
properties: dict[str, JsonObject] = schema.get("properties", {})
|
||||
for wire_name, field in properties.items():
|
||||
attribute = python_name(wire_name)
|
||||
annotation = field_type(field, wire_name in required, f"{name}.{wire_name}")
|
||||
default = "" if wire_name in required else "None"
|
||||
if attribute != wire_name:
|
||||
arguments = ", ".join([*([default] if default else []), f"alias={json.dumps(wire_name)}"])
|
||||
lines.append(f" {attribute}: {annotation} = Field({arguments})")
|
||||
else:
|
||||
lines.append(f" {attribute}: {annotation}{f' = {default}' if default else ''}")
|
||||
if not properties:
|
||||
lines.append(" pass")
|
||||
return lines
|
||||
|
||||
|
||||
def _json_schema_ref(content: JsonObject | None, context: str) -> str:
|
||||
schema = (content or {}).get("application/json", {}).get("schema")
|
||||
if not schema or "$ref" not in schema:
|
||||
raise GenerationError(f"{context}: expected an application/json $ref schema")
|
||||
return schema["$ref"].rpartition("/")[2]
|
||||
|
||||
|
||||
def render_operation(service: Service, allowed: AllowedOperation, operation: JsonObject) -> list[str]:
|
||||
context = f"{allowed.method} {allowed.path}"
|
||||
function_name = operation_function_name(operation["operationId"], service)
|
||||
response = _json_schema_ref(operation.get("responses", {}).get("200", {}).get("content"), context)
|
||||
|
||||
arguments: list[str] = []
|
||||
path_params: list[str] = []
|
||||
query_params: list[str] = []
|
||||
template = allowed.path.removeprefix(PATH_VERSION_PREFIX)
|
||||
for parameter in operation.get("parameters", []):
|
||||
wire_name = parameter["name"]
|
||||
argument = python_name(wire_name)
|
||||
annotation = python_type(parameter["schema"], f"{context} {wire_name}")
|
||||
arguments.append(f"{argument}: {annotation}")
|
||||
if parameter["in"] == "path":
|
||||
template = template.replace(f"{{{wire_name}}}", f"{{{argument}}}")
|
||||
path_params.append(f"{json.dumps(argument)}: {argument}")
|
||||
elif parameter["in"] == "query":
|
||||
query_params.append(f"{json.dumps(wire_name)}: {argument}")
|
||||
else:
|
||||
raise GenerationError(f"{context}: unsupported parameter location {parameter['in']}")
|
||||
body = "None"
|
||||
if "requestBody" in operation:
|
||||
arguments.append(f"body: {_json_schema_ref(operation['requestBody'].get('content'), context)}")
|
||||
body = "body"
|
||||
if len({argument.partition(":")[0] for argument in arguments}) != len(arguments):
|
||||
raise GenerationError(f"{context}: argument names collide")
|
||||
|
||||
signature = f"def {function_name}({'*, ' + ', '.join(arguments) if arguments else ''}) -> {response}:"
|
||||
docstring = f' """{allowed.method} {allowed.path} ({operation["operationId"]})."""'
|
||||
if len(docstring) > LINE_LENGTH:
|
||||
docstring = f' """{allowed.method} {allowed.path}\n\n {operation["operationId"]}\n """'
|
||||
return [
|
||||
signature,
|
||||
docstring,
|
||||
" return request(",
|
||||
f" Upstream.{allowed.upstream},",
|
||||
f" {json.dumps(allowed.method)},",
|
||||
f" {json.dumps(template)},",
|
||||
f" response_model={response},",
|
||||
f" errors={service_errors_name(proto_service(operation['operationId']))},",
|
||||
f" path_params={{{', '.join(path_params)}}},",
|
||||
f" params={{{', '.join(query_params)}}},",
|
||||
f" body={body},",
|
||||
" )",
|
||||
]
|
||||
|
||||
|
||||
def service_errors_name(proto: str) -> str:
|
||||
return f"{snake_case(proto).upper()}_ERRORS"
|
||||
|
||||
|
||||
def proto_services(service: Service, snapshot: JsonObject) -> list[str]:
|
||||
protos: list[str] = []
|
||||
for allowed in ALLOWED_OPERATIONS[service]:
|
||||
proto = proto_service(snapshot["paths"][allowed.path][allowed.method.lower()]["operationId"])
|
||||
if proto not in protos:
|
||||
protos.append(proto)
|
||||
return protos
|
||||
|
||||
|
||||
def check_error_schemas(service: Service, protos: list[str], schemas: dict[str, JsonObject]) -> None:
|
||||
expected = {error_reason_schema(proto) for proto in protos}
|
||||
actual = {name for name in schemas if name.endswith(ERROR_REASON_SUFFIX)}
|
||||
if actual != expected:
|
||||
raise GenerationError(
|
||||
f"{service} snapshot reason schemas differ from its operations' services: "
|
||||
f"missing {sorted(expected - actual)}, unused {sorted(actual - expected)}; rerun the snapshot"
|
||||
)
|
||||
generated = {f"{proto}{suffix}" for proto in protos for suffix in ERROR_CLASS_SUFFIXES}
|
||||
if collisions := sorted(generated & set(schemas)):
|
||||
raise GenerationError(f"{service}: generated error classes collide with schemas {collisions}")
|
||||
|
||||
|
||||
def render_errors(proto: str) -> list[str]:
|
||||
reason = error_reason_schema(proto)
|
||||
guard = f"_is_{snake_case(proto)}_error_reason"
|
||||
return f'''def {guard}(value: str) -> TypeIs[{reason}]:
|
||||
return value in get_args({reason}.__value__)
|
||||
|
||||
|
||||
class {proto}UpstreamError(BillingUpstreamError):
|
||||
"""A failed {proto} call, with its reason typed by `{reason}`."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*args: object,
|
||||
status_code: int | None = None,
|
||||
body: UpstreamErrorBody | None = None,
|
||||
reason: {reason} | UnknownErrorReason | None = None,
|
||||
) -> None:
|
||||
super().__init__(*args, status_code=status_code, body=body)
|
||||
self.reason = reason
|
||||
|
||||
|
||||
class {proto}UpstreamUnavailableError({proto}UpstreamError, BillingUpstreamUnavailableError):
|
||||
pass
|
||||
|
||||
|
||||
class {proto}UpstreamInvalidResponseError({proto}UpstreamError, BillingUpstreamInvalidResponseError):
|
||||
pass
|
||||
|
||||
|
||||
{service_errors_name(proto)} = ServiceErrors(
|
||||
is_reason={guard},
|
||||
unavailable={proto}UpstreamUnavailableError,
|
||||
invalid_response={proto}UpstreamInvalidResponseError,
|
||||
)'''.splitlines()
|
||||
|
||||
|
||||
def render_module(service: Service, snapshot: JsonObject) -> str:
|
||||
schemas: dict[str, JsonObject] = snapshot["components"]["schemas"]
|
||||
protos = proto_services(service, snapshot)
|
||||
check_error_schemas(service, protos, schemas)
|
||||
blocks = [block for name in model_order(schemas) if (block := render_schema(name, schemas[name]))]
|
||||
blocks.extend(render_errors(proto) for proto in protos)
|
||||
for allowed in ALLOWED_OPERATIONS[service]:
|
||||
operation = snapshot["paths"][allowed.path][allowed.method.lower()]
|
||||
blocks.append(render_operation(service, allowed, operation))
|
||||
|
||||
body = "\n\n\n".join("\n".join(block) for block in blocks)
|
||||
header = [
|
||||
f"# Code generated by {GENERATOR} from {service}.openapi.json. DO NOT EDIT.",
|
||||
f"# dify-saas commit: {snapshot[SNAPSHOT_COMMIT_KEY]}",
|
||||
"",
|
||||
"from datetime import datetime",
|
||||
"from typing import Literal, TypeIs, get_args",
|
||||
"",
|
||||
"from pydantic import Field",
|
||||
"",
|
||||
"from services.billing_client.transport import BillingModel, ServiceErrors, Upstream, request",
|
||||
"from services.errors.billing import (",
|
||||
" BillingUpstreamError,",
|
||||
" BillingUpstreamInvalidResponseError,",
|
||||
" BillingUpstreamUnavailableError,",
|
||||
" UnknownErrorReason,",
|
||||
" UpstreamErrorBody,",
|
||||
")",
|
||||
]
|
||||
return format_python("\n".join(header) + "\n\n\n" + body + "\n", module_path(service))
|
||||
|
||||
|
||||
def format_python(source: str, path: Path) -> str:
|
||||
ruff = [sys.executable, "-m", "ruff"]
|
||||
fixed = subprocess.run(
|
||||
[*ruff, "check", "--fix", "--quiet", "--select", "F401,I", "--stdin-filename", str(path), "-"],
|
||||
input=source,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
cwd=API_ROOT,
|
||||
).stdout
|
||||
return subprocess.run(
|
||||
[*ruff, "format", "--quiet", "--stdin-filename", str(path), "-"],
|
||||
input=fixed,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
cwd=API_ROOT,
|
||||
).stdout
|
||||
|
||||
|
||||
# --- entry point ----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def expected_files(snapshots: dict[Service, JsonObject]) -> dict[Path, str]:
|
||||
files: dict[Path, str] = {}
|
||||
for service, snapshot in snapshots.items():
|
||||
files[snapshot_path(service)] = render_snapshot(snapshot)
|
||||
files[module_path(service)] = render_module(service, snapshot)
|
||||
return files
|
||||
|
||||
|
||||
def check(files: dict[Path, str]) -> int:
|
||||
drifted = 0
|
||||
for path, expected in files.items():
|
||||
actual = path.read_text() if path.exists() else ""
|
||||
if actual != expected:
|
||||
drifted += 1
|
||||
diff = difflib.unified_diff(
|
||||
actual.splitlines(keepends=True),
|
||||
expected.splitlines(keepends=True),
|
||||
fromfile=str(path.relative_to(REPO_ROOT)),
|
||||
tofile="regenerated",
|
||||
)
|
||||
sys.stderr.writelines(diff)
|
||||
if drifted:
|
||||
logger.error("billing client is out of date; regenerate with `%s --from-snapshot`", GENERATOR)
|
||||
return 1 if drifted else 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
mode = parser.add_mutually_exclusive_group()
|
||||
mode.add_argument("--from-snapshot", action="store_true", help="generate from the committed snapshots")
|
||||
mode.add_argument("--check", action="store_true", help="fail if regenerating the committed snapshots differs")
|
||||
args = parser.parse_args()
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
|
||||
|
||||
try:
|
||||
if args.check or args.from_snapshot:
|
||||
snapshots = {service: load_snapshot(service) for service in SERVICES}
|
||||
else:
|
||||
snapshots = take_snapshots()
|
||||
files = expected_files(snapshots)
|
||||
except GenerationError as error:
|
||||
sys.stderr.write(f"error: {error}\n")
|
||||
return 1
|
||||
|
||||
if args.check:
|
||||
return check(files)
|
||||
for path, content in files.items():
|
||||
path.write_text(content)
|
||||
logger.info("wrote %s", path.relative_to(REPO_ROOT))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Generate FastOpenAPI OpenAPI 3.1 specs without booting the full backend."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
API_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(API_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(API_ROOT))
|
||||
|
||||
# Import after script path setup.
|
||||
from dev.generate_swagger_specs import apply_runtime_defaults # noqa: E402
|
||||
|
||||
# Import after script path setup.
|
||||
from libs.flask_restx_compat import sort_openapi_arrays # noqa: E402
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FastOpenApiSpecTarget:
|
||||
route: str
|
||||
filename: str
|
||||
|
||||
|
||||
FASTOPENAPI_SPEC_TARGETS: tuple[FastOpenApiSpecTarget, ...] = (
|
||||
FastOpenApiSpecTarget(route="/fastopenapi/openapi.json", filename="fastopenapi-console-openapi.json"),
|
||||
)
|
||||
|
||||
|
||||
def create_fastopenapi_spec_app():
|
||||
"""Build a minimal Flask app that only mounts FastOpenAPI docs routes."""
|
||||
|
||||
apply_runtime_defaults()
|
||||
|
||||
from app_factory import create_flask_app_with_configs
|
||||
from extensions import ext_fastopenapi
|
||||
|
||||
app = create_flask_app_with_configs()
|
||||
ext_fastopenapi.init_app(app)
|
||||
return app
|
||||
|
||||
|
||||
def export_fastopenapi_specs() -> dict[str, dict[str, Any]]:
|
||||
"""Build FastOpenAPI documents in memory without writing generated files."""
|
||||
app = create_fastopenapi_spec_app()
|
||||
client = app.test_client()
|
||||
|
||||
documents = {}
|
||||
for target in FASTOPENAPI_SPEC_TARGETS:
|
||||
response = client.get(target.route)
|
||||
if response.status_code != 200:
|
||||
raise RuntimeError(f"failed to fetch {target.route}: {response.status_code}")
|
||||
|
||||
payload = sort_openapi_arrays(response.get_json())
|
||||
if not isinstance(payload, dict):
|
||||
raise RuntimeError(f"unexpected response payload for {target.route}")
|
||||
documents[target.filename] = payload
|
||||
return documents
|
||||
|
||||
|
||||
def generate_fastopenapi_specs(output_dir: Path) -> list[Path]:
|
||||
"""Write FastOpenAPI specs to `output_dir` and return the written paths."""
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
written_paths: list[Path] = []
|
||||
for filename, payload in export_fastopenapi_specs().items():
|
||||
output_path = output_dir / filename
|
||||
output_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
written_paths.append(output_path)
|
||||
|
||||
return written_paths
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--output-dir",
|
||||
type=Path,
|
||||
default=Path("openapi"),
|
||||
help="Directory where the OpenAPI JSON files will be written.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
written_paths = generate_fastopenapi_specs(args.output_dir)
|
||||
|
||||
for path in written_paths:
|
||||
logger.debug(path)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,660 @@
|
||||
"""Verify Dify against checked-in contracts from an exact standalone KnowledgeFS commit.
|
||||
|
||||
CI uses only api/knowledge-fs-contract: no private source checkout, Node runtime, or network
|
||||
is needed for --check. Intentional updates export from a clean --knowledge-fs-root checkout,
|
||||
record its full commit/tree provenance, and pin every artifact and Dify operation manifest.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal, TypedDict, cast
|
||||
|
||||
import jwt
|
||||
from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey
|
||||
from jwt.algorithms import RSAAlgorithm
|
||||
|
||||
API_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(API_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(API_ROOT))
|
||||
|
||||
# Import after script path setup.
|
||||
from dev.knowledge_fs_product_contract import ( # noqa: E402
|
||||
capability_operation_runtime_contracts,
|
||||
parse_capability_operation_policy,
|
||||
parse_product_operation_gap_manifest,
|
||||
parse_product_operation_manifest,
|
||||
product_operation_runtime_contracts,
|
||||
validate_product_operation_contracts,
|
||||
)
|
||||
|
||||
WORKSPACE_ROOT = API_ROOT.parent
|
||||
PIN_RELATIVE_PATH = Path("api/knowledge-fs-contract")
|
||||
OPENAPI_FILENAME = "knowledge-fs.openapi.json"
|
||||
CAPABILITY_POLICY_FILENAME = "dify-capability-v2-operations.json"
|
||||
CAPABILITY_V2_AUTH_MANIFEST_RELATIVE_PATH = Path("contracts/dify-capability-v2-auth-profile.json")
|
||||
CAPABILITY_V2_AUTH_TEST_VECTOR_RELATIVE_PATH = Path("contracts/dify-capability-v2-test-vector.json")
|
||||
PROVENANCE_FILENAME = "source-provenance.json"
|
||||
LOCK_RELATIVE_PATH = Path("api/knowledge-fs-contract.lock.json")
|
||||
PRODUCT_OPERATIONS_RELATIVE_PATH = Path("api/knowledge-fs-product-operations.json")
|
||||
PRODUCT_OPERATION_GAPS_RELATIVE_PATH = Path("api/knowledge-fs-product-operation-gaps.json")
|
||||
OPENAPI_METHODS = ("delete", "get", "head", "options", "patch", "post", "put", "trace")
|
||||
PROXY_METHODS = frozenset({"delete", "get", "patch", "post", "put"})
|
||||
LOCK_SCHEMA_VERSION = 6
|
||||
|
||||
|
||||
class ContractDeclaration(TypedDict):
|
||||
"""KnowledgeFS transport contract declared by one Dify Console registry entry."""
|
||||
|
||||
operation_id: str
|
||||
method: str
|
||||
path: str
|
||||
required_scope: str | None
|
||||
response_kind: str
|
||||
max_response_bytes: int
|
||||
request_headers: tuple[str, ...]
|
||||
response_headers: tuple[str, ...]
|
||||
response_media_types: tuple[str, ...]
|
||||
|
||||
|
||||
class ContractLock(TypedDict):
|
||||
"""Content-addressed contract inputs that must move together."""
|
||||
|
||||
schemaVersion: int
|
||||
sourceCommit: str
|
||||
sourceTree: str
|
||||
sourceProvenanceSha256: str
|
||||
capabilityPolicySha256: str
|
||||
openapiSha256: str
|
||||
capabilityV2AuthManifestSha256: str
|
||||
capabilityV2AuthTestVectorSha256: str
|
||||
productOperationManifestSha256: str
|
||||
productOperationGapManifestSha256: str
|
||||
|
||||
|
||||
type DeclarationField = Literal[
|
||||
"method",
|
||||
"path",
|
||||
"required_scope",
|
||||
"response_kind",
|
||||
"max_response_bytes",
|
||||
"request_headers",
|
||||
"response_headers",
|
||||
"response_media_types",
|
||||
]
|
||||
|
||||
DECLARATION_FIELDS: tuple[DeclarationField, ...] = (
|
||||
"method",
|
||||
"path",
|
||||
"required_scope",
|
||||
"response_kind",
|
||||
"max_response_bytes",
|
||||
"request_headers",
|
||||
"response_headers",
|
||||
"response_media_types",
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Update an exact source pin or verify the self-contained Dify contract boundary."""
|
||||
parser = argparse.ArgumentParser()
|
||||
mode = parser.add_mutually_exclusive_group(required=True)
|
||||
mode.add_argument("--check", action="store_true")
|
||||
mode.add_argument("--update-lock", action="store_true")
|
||||
parser.add_argument("--workspace-root", type=Path, default=WORKSPACE_ROOT)
|
||||
parser.add_argument("--knowledge-fs-root", type=Path, help="Clean standalone source checkout (updates only)")
|
||||
args = parser.parse_args()
|
||||
if args.check and args.knowledge_fs_root is not None:
|
||||
parser.error("--check reads pinned artifacts only; --knowledge-fs-root is for --update-lock")
|
||||
if args.update_lock and args.knowledge_fs_root is None:
|
||||
parser.error("--update-lock requires --knowledge-fs-root pointing to a clean committed checkout")
|
||||
|
||||
workspace_root = args.workspace_root.resolve()
|
||||
pin_root = workspace_root / PIN_RELATIVE_PATH
|
||||
if args.update_lock:
|
||||
with tempfile.TemporaryDirectory(prefix="dify-knowledge-fs-contract-") as directory:
|
||||
exported_root = Path(directory)
|
||||
export_contract(args.knowledge_fs_root.resolve(), exported_root)
|
||||
expected_lock = validate_pinned_contract(workspace_root, exported_root)
|
||||
pin_root.mkdir(parents=True, exist_ok=True)
|
||||
for path in exported_root.iterdir():
|
||||
(pin_root / path.name).write_bytes(path.read_bytes())
|
||||
(workspace_root / LOCK_RELATIVE_PATH).write_text(json.dumps(expected_lock, indent=2) + "\n")
|
||||
return
|
||||
|
||||
expected_lock = validate_pinned_contract(workspace_root, pin_root)
|
||||
received_lock = parse_contract_lock(load_json_object(workspace_root / LOCK_RELATIVE_PATH))
|
||||
for field, expected_value in expected_lock.items():
|
||||
received_value = dict(received_lock)[field]
|
||||
if received_value != expected_value:
|
||||
raise RuntimeError(
|
||||
f"KnowledgeFS contract lock field {field} drifted: "
|
||||
f"expected {expected_value!r}, received {received_value!r}. "
|
||||
"Run --update-lock --knowledge-fs-root intentionally after reviewing source and contract changes."
|
||||
)
|
||||
|
||||
|
||||
def export_contract(knowledge_fs_root: Path, output_root: Path) -> None:
|
||||
"""Export only a clean committed source tree so provenance identifies the actual input."""
|
||||
ensure_clean_knowledge_fs_worktree(knowledge_fs_root)
|
||||
provenance = {
|
||||
"schemaVersion": 1,
|
||||
"repository": "https://github.com/langgenius/knowledge-fs",
|
||||
"commit": run("git", "rev-parse", "HEAD", cwd=knowledge_fs_root).strip(),
|
||||
"tree": run("git", "rev-parse", "HEAD^{tree}", cwd=knowledge_fs_root).strip(),
|
||||
}
|
||||
output_root.mkdir(parents=True, exist_ok=True)
|
||||
for command, filename in (("openapi:export", OPENAPI_FILENAME), ("capability:export", CAPABILITY_POLICY_FILENAME)):
|
||||
subprocess.run(
|
||||
["pnpm", command, "--", "--output", str(output_root / filename)], cwd=knowledge_fs_root, check=True
|
||||
)
|
||||
for relative_path in (CAPABILITY_V2_AUTH_MANIFEST_RELATIVE_PATH, CAPABILITY_V2_AUTH_TEST_VECTOR_RELATIVE_PATH):
|
||||
(output_root / relative_path.name).write_bytes((knowledge_fs_root / relative_path).read_bytes())
|
||||
# Detect source changes made while the export subprocesses ran.
|
||||
ensure_clean_knowledge_fs_worktree(knowledge_fs_root)
|
||||
if run("git", "rev-parse", "HEAD", cwd=knowledge_fs_root).strip() != provenance["commit"]:
|
||||
raise RuntimeError("KnowledgeFS source commit changed during contract export")
|
||||
(output_root / PROVENANCE_FILENAME).write_text(json.dumps(provenance, indent=2) + "\n")
|
||||
|
||||
|
||||
def ensure_clean_knowledge_fs_worktree(knowledge_fs_root: Path) -> None:
|
||||
"""Require the standalone checkout's working tree and index to match its commit."""
|
||||
if run("git", "status", "--porcelain", "--untracked-files=all", cwd=knowledge_fs_root).strip():
|
||||
raise RuntimeError("KnowledgeFS source contains staged, unstaged or untracked changes; commit before export")
|
||||
|
||||
|
||||
def validate_pinned_contract(workspace_root: Path, pin_root: Path) -> ContractLock:
|
||||
"""Validate pinned bytes against executable Python declarations without invoking source tools."""
|
||||
openapi_path = pin_root / OPENAPI_FILENAME
|
||||
policy_path = pin_root / CAPABILITY_POLICY_FILENAME
|
||||
auth_path = pin_root / CAPABILITY_V2_AUTH_MANIFEST_RELATIVE_PATH.name
|
||||
vector_path = pin_root / CAPABILITY_V2_AUTH_TEST_VECTOR_RELATIVE_PATH.name
|
||||
provenance_path = pin_root / PROVENANCE_FILENAME
|
||||
product_path = workspace_root / PRODUCT_OPERATIONS_RELATIVE_PATH
|
||||
gaps_path = workspace_root / PRODUCT_OPERATION_GAPS_RELATIVE_PATH
|
||||
for path in (openapi_path, policy_path, auth_path, vector_path, provenance_path, product_path, gaps_path):
|
||||
if not path.is_file():
|
||||
raise RuntimeError(f"KnowledgeFS pinned contract input is missing: {path}")
|
||||
auth_manifest = load_json_object(auth_path)
|
||||
validate_capability_v2_auth_manifest(auth_manifest)
|
||||
validate_capability_v2_auth_test_vector(load_json_object(vector_path), auth_manifest)
|
||||
provenance = load_json_object(provenance_path)
|
||||
validate_upstream_provenance(provenance)
|
||||
document = load_json_object(openapi_path)
|
||||
validate_product_operation_contracts(
|
||||
capability_operations=capability_operation_runtime_contracts(),
|
||||
capability_policy=parse_capability_operation_policy(load_json_object(policy_path)),
|
||||
document=document,
|
||||
gap_manifest=parse_product_operation_gap_manifest(load_json_object(gaps_path)),
|
||||
manifest=parse_product_operation_manifest(load_json_object(product_path)),
|
||||
product_operations=product_operation_runtime_contracts(),
|
||||
)
|
||||
validate_declarations(document, console_contract_declarations())
|
||||
return {
|
||||
"schemaVersion": LOCK_SCHEMA_VERSION,
|
||||
"sourceCommit": provenance["commit"],
|
||||
"sourceTree": provenance["tree"],
|
||||
"sourceProvenanceSha256": sha256(provenance_path.read_bytes()),
|
||||
"openapiSha256": sha256(openapi_path.read_bytes()),
|
||||
"capabilityPolicySha256": sha256(policy_path.read_bytes()),
|
||||
"capabilityV2AuthManifestSha256": sha256(auth_path.read_bytes()),
|
||||
"capabilityV2AuthTestVectorSha256": sha256(vector_path.read_bytes()),
|
||||
"productOperationManifestSha256": sha256(product_path.read_bytes()),
|
||||
"productOperationGapManifestSha256": sha256(gaps_path.read_bytes()),
|
||||
}
|
||||
|
||||
|
||||
def load_json_object(path: Path) -> dict[str, Any]:
|
||||
"""Load a JSON object and reject arrays/scalars at contract boundaries."""
|
||||
value = json.loads(path.read_text())
|
||||
if not isinstance(value, dict):
|
||||
raise ValueError(f"KnowledgeFS contract file must contain a JSON object: {path}")
|
||||
return cast(dict[str, Any], value)
|
||||
|
||||
|
||||
def parse_contract_lock(value: dict[str, Any]) -> ContractLock:
|
||||
"""Validate the compact, non-self-referential contract lock schema."""
|
||||
expected_fields = {
|
||||
"capabilityV2AuthManifestSha256",
|
||||
"capabilityV2AuthTestVectorSha256",
|
||||
"openapiSha256",
|
||||
"productOperationGapManifestSha256",
|
||||
"productOperationManifestSha256",
|
||||
"schemaVersion",
|
||||
"sourceCommit",
|
||||
"sourceTree",
|
||||
"sourceProvenanceSha256",
|
||||
"capabilityPolicySha256",
|
||||
}
|
||||
if set(value) != expected_fields:
|
||||
raise ValueError(f"KnowledgeFS contract lock fields must be exactly {sorted(expected_fields)}")
|
||||
if value.get("schemaVersion") != LOCK_SCHEMA_VERSION:
|
||||
raise ValueError(f"KnowledgeFS contract lock schemaVersion must be {LOCK_SCHEMA_VERSION}")
|
||||
for field in (
|
||||
"capabilityV2AuthManifestSha256",
|
||||
"capabilityV2AuthTestVectorSha256",
|
||||
"openapiSha256",
|
||||
"productOperationGapManifestSha256",
|
||||
"productOperationManifestSha256",
|
||||
"sourceCommit",
|
||||
"sourceTree",
|
||||
"sourceProvenanceSha256",
|
||||
"capabilityPolicySha256",
|
||||
):
|
||||
field_value = value.get(field)
|
||||
expected_length = 40 if field in {"sourceCommit", "sourceTree"} else 64
|
||||
if (
|
||||
not isinstance(field_value, str)
|
||||
or len(field_value) != expected_length
|
||||
or any(character not in "0123456789abcdef" for character in field_value)
|
||||
):
|
||||
raise ValueError(f"KnowledgeFS contract lock field {field} has an invalid digest")
|
||||
return cast(ContractLock, value)
|
||||
|
||||
|
||||
def validate_required_product_operations(document: dict[str, Any], required_operation_ids: list[str]) -> None:
|
||||
"""Fail when the pinned KFS OpenAPI omits an operation required by the Dify product."""
|
||||
available_operation_ids = [
|
||||
operation_id
|
||||
for path_item in document.get("paths", {}).values()
|
||||
for method in OPENAPI_METHODS
|
||||
if isinstance(path_item, dict)
|
||||
for operation in (path_item.get(method),)
|
||||
if isinstance(operation, dict)
|
||||
for operation_id in (operation.get("operationId"),)
|
||||
if isinstance(operation_id, str) and operation_id
|
||||
]
|
||||
for operation_id in required_operation_ids:
|
||||
count = available_operation_ids.count(operation_id)
|
||||
if count != 1:
|
||||
raise ValueError(
|
||||
f"KnowledgeFS OpenAPI required product operation {operation_id} must occur exactly once; found {count}"
|
||||
)
|
||||
|
||||
|
||||
def validate_capability_v2_auth_manifest(value: dict[str, Any]) -> None:
|
||||
"""Validate the active production RS256 profile consumed by both Dify and KnowledgeFS."""
|
||||
expected_fields = {
|
||||
"active",
|
||||
"audience",
|
||||
"callerProfiles",
|
||||
"claimBindings",
|
||||
"issuer",
|
||||
"lifecycle",
|
||||
"maxTtlSeconds",
|
||||
"productionReady",
|
||||
"profileId",
|
||||
"protectedHeader",
|
||||
"requiredClaims",
|
||||
"resourceContract",
|
||||
"runtimeAssembly",
|
||||
"schemaVersion",
|
||||
"signatureAlgorithms",
|
||||
"tokenKind",
|
||||
}
|
||||
if set(value) != expected_fields:
|
||||
raise ValueError(f"KnowledgeFS Capability v2 auth manifest fields must be exactly {sorted(expected_fields)}")
|
||||
fixed_values = {
|
||||
"active": True,
|
||||
"audience": "knowledge-fs",
|
||||
"issuer": "dify-control-plane",
|
||||
"lifecycle": "active",
|
||||
"maxTtlSeconds": 60,
|
||||
"productionReady": True,
|
||||
"profileId": "dify-capability-v2",
|
||||
"schemaVersion": 3,
|
||||
"signatureAlgorithms": ["RS256"],
|
||||
"tokenKind": "jwt",
|
||||
}
|
||||
for field, expected in fixed_values.items():
|
||||
if value.get(field) != expected:
|
||||
raise ValueError(f"KnowledgeFS Capability v2 auth manifest field {field} must be {expected!r}")
|
||||
if value.get("protectedHeader") != {
|
||||
"algorithm": "RS256",
|
||||
"keyIdClaim": "kid",
|
||||
"keyIdRequired": True,
|
||||
"type": "JWT",
|
||||
}:
|
||||
raise ValueError("KnowledgeFS Capability v2 protected header contract is invalid")
|
||||
required_claims = [
|
||||
"action",
|
||||
"actor",
|
||||
"aud",
|
||||
"authz_revision",
|
||||
"azp",
|
||||
"caller_kind",
|
||||
"cap_ver",
|
||||
"content_policy_revision",
|
||||
"content_scope_ids",
|
||||
"control_space_id",
|
||||
"exp",
|
||||
"grant_id",
|
||||
"iat",
|
||||
"iss",
|
||||
"jti",
|
||||
"namespace_id",
|
||||
"nbf",
|
||||
"resource",
|
||||
"sub",
|
||||
"trace_id",
|
||||
]
|
||||
if value.get("requiredClaims") != required_claims:
|
||||
raise ValueError("KnowledgeFS Capability v2 required claims are invalid")
|
||||
if value.get("claimBindings") != {
|
||||
"action": "action",
|
||||
"callerKind": "caller_kind",
|
||||
"controlSpace": "control_space_id",
|
||||
"namespace": "namespace_id",
|
||||
"resource": "resource",
|
||||
"resourceParent": "resource.parent_id",
|
||||
"subject": "sub",
|
||||
}:
|
||||
raise ValueError("KnowledgeFS Capability v2 claim bindings are invalid")
|
||||
if value.get("resourceContract") != {
|
||||
"fields": ["id", "parent_id", "type"],
|
||||
"parentForbiddenFor": ["namespace", "knowledge_space"],
|
||||
"parentRequiredFor": ["document", "job", "query", "research_task", "source", "upload_session"],
|
||||
}:
|
||||
raise ValueError("KnowledgeFS Capability v2 resource contract is invalid")
|
||||
if value.get("callerProfiles") != {
|
||||
"agent": {"authorizedParty": "dify-agent", "subjectPrefix": "dify-app:"},
|
||||
"interactive": {"authorizedParty": "dify-console", "subjectPrefix": "dify-account:"},
|
||||
"internal_worker": {"authorizedParty": "dify-worker", "subjectPrefix": "dify-worker:"},
|
||||
"mcp": {"authorizedParty": "dify-mcp", "subjectPrefix": "dify-mcp-session:"},
|
||||
"service": {"authorizedParty": "dify-service-api", "subjectPrefix": "dify-dataset-api-key:"},
|
||||
"workflow": {"authorizedParty": "dify-workflow", "subjectPrefix": "dify-app:"},
|
||||
}:
|
||||
raise ValueError("KnowledgeFS Capability v2 caller profiles are invalid")
|
||||
if value.get("runtimeAssembly") != {
|
||||
"failClosed": True,
|
||||
"keySelection": "kid",
|
||||
"maximumPublishedKeys": 3,
|
||||
"verificationKeySource": "jwks",
|
||||
}:
|
||||
raise ValueError("KnowledgeFS Capability v2 runtime assembly is invalid")
|
||||
|
||||
|
||||
def validate_capability_v2_auth_test_vector(
|
||||
value: dict[str, Any],
|
||||
manifest: dict[str, Any],
|
||||
) -> None:
|
||||
"""Verify the deterministic public-key vector and every security-sensitive binding."""
|
||||
expected_fields = {
|
||||
"algorithm",
|
||||
"audience",
|
||||
"expectedClaims",
|
||||
"expectedPrincipal",
|
||||
"issuer",
|
||||
"operation",
|
||||
"profileId",
|
||||
"protectedHeader",
|
||||
"publicJwk",
|
||||
"schemaVersion",
|
||||
"testOnly",
|
||||
"token",
|
||||
"ttlSeconds",
|
||||
}
|
||||
if set(value) != expected_fields:
|
||||
raise ValueError(f"KnowledgeFS Capability v2 test vector fields must be exactly {sorted(expected_fields)}")
|
||||
if (
|
||||
value.get("schemaVersion") != 2
|
||||
or value.get("profileId") != manifest.get("profileId")
|
||||
or value.get("testOnly") is not True
|
||||
or value.get("algorithm") != "RS256"
|
||||
or value.get("issuer") != manifest.get("issuer")
|
||||
or value.get("audience") != manifest.get("audience")
|
||||
or value.get("ttlSeconds") != manifest.get("maxTtlSeconds")
|
||||
):
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector does not match the active profile")
|
||||
protected_header = value.get("protectedHeader")
|
||||
if not isinstance(protected_header, dict) or set(protected_header) != {"alg", "kid", "typ"}:
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector protected header is invalid")
|
||||
kid = protected_header.get("kid")
|
||||
if protected_header.get("alg") != "RS256" or protected_header.get("typ") != "JWT" or not _is_non_blank(kid):
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector protected header is invalid")
|
||||
public_jwk = value.get("publicJwk")
|
||||
if not isinstance(public_jwk, dict) or set(public_jwk) != {"alg", "e", "kid", "kty", "n", "use"}:
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector public JWK is invalid")
|
||||
if (
|
||||
public_jwk.get("alg") != "RS256"
|
||||
or public_jwk.get("kid") != kid
|
||||
or public_jwk.get("kty") != "RSA"
|
||||
or public_jwk.get("use") != "sig"
|
||||
or not _is_non_blank(public_jwk.get("e"))
|
||||
or not _is_non_blank(public_jwk.get("n"))
|
||||
):
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector public JWK is invalid")
|
||||
|
||||
claims = value.get("expectedClaims")
|
||||
required_claims = manifest.get("requiredClaims")
|
||||
if not isinstance(claims, dict) or not isinstance(required_claims, list) or set(claims) != set(required_claims):
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector claims do not match the active profile")
|
||||
operation = value.get("operation")
|
||||
if not isinstance(operation, dict) or set(operation) != {"action", "method", "operationId", "requestPath"}:
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector operation is invalid")
|
||||
resource = claims.get("resource")
|
||||
if not isinstance(resource, dict) or set(resource) != {"id", "parent_id", "type"}:
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector resource is invalid")
|
||||
expected_operation = {
|
||||
"action": "documents.read",
|
||||
"method": "GET",
|
||||
"operationId": "getDocument",
|
||||
"requestPath": "/knowledge-spaces/space-contract-vector/documents/document-contract-vector",
|
||||
}
|
||||
if operation != expected_operation:
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector operation binding is invalid")
|
||||
exact_claims = {
|
||||
"action": operation["action"],
|
||||
"aud": value["audience"],
|
||||
"caller_kind": "interactive",
|
||||
"cap_ver": 2,
|
||||
"control_space_id": "control-space-contract-vector",
|
||||
"iss": value["issuer"],
|
||||
"namespace_id": "workspace-contract-vector",
|
||||
"resource": {
|
||||
"id": "document-contract-vector",
|
||||
"parent_id": "space-contract-vector",
|
||||
"type": "document",
|
||||
},
|
||||
"sub": "dify-account:account-contract-vector",
|
||||
}
|
||||
for field, expected in exact_claims.items():
|
||||
if claims.get(field) != expected:
|
||||
raise ValueError(f"KnowledgeFS Capability v2 test vector claim {field} is invalid")
|
||||
if claims.get("actor") != claims["sub"] or claims.get("azp") != "dify-console":
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector caller binding is invalid")
|
||||
issued_at = claims.get("iat")
|
||||
not_before = claims.get("nbf")
|
||||
expires_at = claims.get("exp")
|
||||
if (
|
||||
not isinstance(issued_at, int)
|
||||
or isinstance(issued_at, bool)
|
||||
or not_before != issued_at
|
||||
or not isinstance(expires_at, int)
|
||||
or isinstance(expires_at, bool)
|
||||
or expires_at - issued_at != value["ttlSeconds"]
|
||||
):
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector TTL is invalid")
|
||||
expected_principal = {
|
||||
"callerKind": claims["caller_kind"],
|
||||
"subject": {
|
||||
"scopes": ["knowledge-spaces:read"],
|
||||
"subjectId": claims["sub"],
|
||||
"tenantId": claims["namespace_id"],
|
||||
},
|
||||
}
|
||||
if value.get("expectedPrincipal") != expected_principal:
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector principal is invalid")
|
||||
|
||||
token = value.get("token")
|
||||
if not isinstance(token, str) or not _is_non_blank(token):
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector token is invalid")
|
||||
try:
|
||||
verification_key = RSAAlgorithm.from_jwk(public_jwk)
|
||||
if not isinstance(verification_key, RSAPublicKey):
|
||||
raise ValueError("Capability vector verification key is not RSA public material")
|
||||
header = jwt.get_unverified_header(token)
|
||||
decoded_claims = jwt.decode(
|
||||
token,
|
||||
verification_key,
|
||||
algorithms=["RS256"],
|
||||
audience=cast(str, value["audience"]),
|
||||
issuer=cast(str, value["issuer"]),
|
||||
options={"verify_exp": False, "verify_iat": False, "verify_nbf": False},
|
||||
)
|
||||
except (jwt.PyJWTError, TypeError, ValueError) as exc:
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector signature is invalid") from exc
|
||||
if header != protected_header or decoded_claims != claims:
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector token content drifted")
|
||||
|
||||
|
||||
def _is_non_blank(value: object) -> bool:
|
||||
return isinstance(value, str) and bool(value.strip()) and value == value.strip()
|
||||
|
||||
|
||||
def validate_upstream_provenance(value: dict[str, Any]) -> None:
|
||||
"""Validate exact standalone source provenance covered by the schema-6 lock."""
|
||||
expected_fields = {"commit", "tree", "repository", "schemaVersion"}
|
||||
if set(value) != expected_fields or value.get("schemaVersion") != 1:
|
||||
raise ValueError("KnowledgeFS source provenance must use schemaVersion 1 with commit, tree and repository")
|
||||
if value.get("repository") != "https://github.com/langgenius/knowledge-fs":
|
||||
raise ValueError("KnowledgeFS source provenance repository does not match the contract owner")
|
||||
for field in ("commit", "tree"):
|
||||
digest = value.get(field)
|
||||
if not isinstance(digest, str) or len(digest) != 40 or any(c not in "0123456789abcdef" for c in digest):
|
||||
raise ValueError(f"KnowledgeFS source provenance {field} must be a lowercase full Git SHA")
|
||||
|
||||
|
||||
def validate_declarations(document: dict[str, Any], declarations: tuple[ContractDeclaration, ...]) -> None:
|
||||
"""Validate Dify Console declarations against matching pinned OpenAPI operations."""
|
||||
operations_by_id: dict[str, list[tuple[str, str, dict[str, Any], dict[str, Any]]]] = {}
|
||||
for path, path_item in document.get("paths", {}).items():
|
||||
for method in OPENAPI_METHODS:
|
||||
operation = path_item.get(method)
|
||||
if operation is None:
|
||||
continue
|
||||
operation_id = operation.get("operationId")
|
||||
if isinstance(operation_id, str) and operation_id:
|
||||
operations_by_id.setdefault(operation_id, []).append((method, path, path_item, operation))
|
||||
|
||||
declared_ids: set[str] = set()
|
||||
for declaration in declarations:
|
||||
operation_id = declaration["operation_id"]
|
||||
if operation_id in declared_ids:
|
||||
raise ValueError(f"Dify Console registry has duplicate operationId: {operation_id}")
|
||||
declared_ids.add(operation_id)
|
||||
|
||||
matches = operations_by_id.get(operation_id, [])
|
||||
if not matches:
|
||||
raise ValueError(f"KnowledgeFS OpenAPI has no operationId: {operation_id}")
|
||||
if len(matches) > 1:
|
||||
raise ValueError(f"KnowledgeFS OpenAPI has duplicate operationId: {operation_id}")
|
||||
|
||||
method, path, path_item, operation = matches[0]
|
||||
if not path.startswith("/"):
|
||||
raise ValueError(f"KnowledgeFS OpenAPI path must be absolute: {path}")
|
||||
if method not in PROXY_METHODS:
|
||||
raise ValueError(f"KnowledgeFS proxy does not support {method.upper()} {path}")
|
||||
expected: ContractDeclaration = {
|
||||
"operation_id": operation_id,
|
||||
"method": method.upper(),
|
||||
"path": path[1:],
|
||||
"required_scope": required_scope(document, operation),
|
||||
"response_kind": response_kind(operation),
|
||||
"max_response_bytes": required_max_response_bytes(operation),
|
||||
"request_headers": request_header_names(path_item, operation),
|
||||
"response_headers": response_header_names(operation),
|
||||
"response_media_types": response_media_types(operation),
|
||||
}
|
||||
for field in DECLARATION_FIELDS:
|
||||
expected_value = expected[field]
|
||||
received_value = declaration[field]
|
||||
if received_value != expected_value:
|
||||
raise ValueError(
|
||||
f"KnowledgeFS operation {operation_id} field {field} drifted: "
|
||||
f"expected {expected_value!r}, received {received_value!r}"
|
||||
)
|
||||
|
||||
|
||||
def console_contract_declarations() -> tuple[ContractDeclaration, ...]:
|
||||
"""The P9 backend exposes only typed product controllers; the raw Console proxy is removed."""
|
||||
|
||||
return ()
|
||||
|
||||
|
||||
def response_kind(operation: dict[str, Any]) -> str:
|
||||
media_types = response_media_types(operation)
|
||||
if "text/event-stream" in media_types:
|
||||
return "stream"
|
||||
if "application/octet-stream" in media_types:
|
||||
return "binary"
|
||||
return "buffered"
|
||||
|
||||
|
||||
def response_media_types(operation: dict[str, Any]) -> tuple[str, ...]:
|
||||
media_types: set[str] = set()
|
||||
for status, response in operation.get("responses", {}).items():
|
||||
if status == "2XX" or (len(status) == 3 and status.startswith("2") and status.isdigit()):
|
||||
media_types.update(response.get("content", {}))
|
||||
return tuple(sorted(media_types))
|
||||
|
||||
|
||||
def required_scope(document: dict[str, Any], operation: dict[str, Any]) -> str | None:
|
||||
scope = operation.get("x-knowledge-fs-required-scope")
|
||||
security = operation["security"] if "security" in operation else document.get("security")
|
||||
if security == []:
|
||||
if scope is not None:
|
||||
raise ValueError(f"KnowledgeFS public operation must not declare a required scope: {scope}")
|
||||
return None
|
||||
if security != [{"bearerAuth": []}]:
|
||||
raise ValueError(f"KnowledgeFS operation effective security must be exactly bearerAuth: {security}")
|
||||
if scope in ("knowledge-spaces:read", "knowledge-spaces:write"):
|
||||
return scope
|
||||
raise ValueError(f"KnowledgeFS operation has no supported required scope: {scope}")
|
||||
|
||||
|
||||
def required_max_response_bytes(operation: dict[str, Any]) -> int:
|
||||
value = operation.get("x-knowledge-fs-max-response-bytes")
|
||||
if not isinstance(value, int) or isinstance(value, bool) or value <= 0:
|
||||
raise ValueError(f"KnowledgeFS operation has no valid response byte limit: {value}")
|
||||
return value
|
||||
|
||||
|
||||
def request_header_names(path_item: dict[str, Any], operation: dict[str, Any]) -> tuple[str, ...]:
|
||||
names: set[str] = set()
|
||||
for parameter in [*path_item.get("parameters", []), *operation.get("parameters", [])]:
|
||||
if "$ref" in parameter:
|
||||
raise ValueError(f"KnowledgeFS request header references are not supported: {parameter['$ref']}")
|
||||
if parameter.get("in") == "header":
|
||||
names.add(parameter["name"].lower())
|
||||
return tuple(sorted(names))
|
||||
|
||||
|
||||
def response_header_names(operation: dict[str, Any]) -> tuple[str, ...]:
|
||||
return tuple(
|
||||
sorted(
|
||||
{
|
||||
name.lower()
|
||||
for response in operation.get("responses", {}).values()
|
||||
for name in response.get("headers", {})
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def sha256(content: bytes) -> str:
|
||||
return hashlib.sha256(content).hexdigest()
|
||||
|
||||
|
||||
def run(*command: str, cwd: Path) -> str:
|
||||
return subprocess.run(command, cwd=cwd, check=True, capture_output=True, text=True).stdout
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,445 @@
|
||||
"""Generate OpenAPI JSON specs and split Markdown API docs.
|
||||
|
||||
The Markdown step uses `swagger-markdown`, the same converter family as the
|
||||
legacy Markdown UI, so CI and local regeneration catch converter-incompatible
|
||||
OpenAPI output early.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
API_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(API_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(API_ROOT))
|
||||
|
||||
# Import after script path setup.
|
||||
from dev.generate_fastopenapi_specs import FASTOPENAPI_SPEC_TARGETS, generate_fastopenapi_specs # noqa: E402
|
||||
|
||||
# Import after script path setup.
|
||||
from dev.generate_swagger_specs import SPEC_TARGETS, generate_specs # noqa: E402
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SWAGGER_MARKDOWN_PACKAGE = "swagger-markdown@3.0.0"
|
||||
CONSOLE_OPENAPI_FILENAME = "console-openapi.json"
|
||||
STALE_COMBINED_MARKDOWN_FILENAME = "api-reference.md"
|
||||
# The inner API is a trusted server-to-server surface, not part of the published API reference.
|
||||
MARKDOWN_SPEC_TARGETS = tuple(target for target in SPEC_TARGETS if target.filename != "inner-openapi.json")
|
||||
|
||||
|
||||
def _schema_ref_name(schema: object) -> str | None:
|
||||
if not isinstance(schema, dict):
|
||||
return None
|
||||
|
||||
ref = schema.get("$ref")
|
||||
if not isinstance(ref, str):
|
||||
return None
|
||||
|
||||
if ref.startswith("#/components/schemas/"):
|
||||
return ref.removeprefix("#/components/schemas/")
|
||||
return None
|
||||
|
||||
|
||||
def _markdown_anchor(name: str) -> str:
|
||||
return name.lower()
|
||||
|
||||
|
||||
def _schema_markdown_type(schema: object) -> str:
|
||||
if not isinstance(schema, dict):
|
||||
return ""
|
||||
|
||||
ref_name = _schema_ref_name(schema)
|
||||
if ref_name is not None:
|
||||
return f"[{ref_name}](#{_markdown_anchor(ref_name)})"
|
||||
|
||||
for union_key in ("oneOf", "anyOf"):
|
||||
variants = schema.get(union_key)
|
||||
if not isinstance(variants, list):
|
||||
continue
|
||||
|
||||
variant_types = [
|
||||
variant_type
|
||||
for variant in variants
|
||||
if not (isinstance(variant, dict) and variant.get("type") == "null")
|
||||
for variant_type in [_schema_markdown_type(variant)]
|
||||
if variant_type
|
||||
]
|
||||
if len(variant_types) == 1:
|
||||
return variant_types[0]
|
||||
if variant_types:
|
||||
return "<br>".join(variant_types)
|
||||
|
||||
schema_type = schema.get("type")
|
||||
if schema_type == "array":
|
||||
item_type = _schema_markdown_type(schema.get("items"))
|
||||
return f"[ {item_type or 'object'} ]"
|
||||
if isinstance(schema_type, str):
|
||||
enum_values = schema.get("enum")
|
||||
if isinstance(enum_values, list) and enum_values:
|
||||
rendered_values = ", ".join(json.dumps(value, ensure_ascii=False) for value in enum_values)
|
||||
return f"{schema_type}, <br>**Available values:** {rendered_values}"
|
||||
return schema_type
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
def _replace_schema_table_type(markdown: str, definition_name: str, row_name: str, type_markdown: str) -> str:
|
||||
if not type_markdown:
|
||||
return markdown
|
||||
|
||||
lines = markdown.splitlines()
|
||||
section_header = f"#### {definition_name}"
|
||||
in_section = False
|
||||
|
||||
for index, line in enumerate(lines):
|
||||
if line == section_header:
|
||||
in_section = True
|
||||
continue
|
||||
if in_section and line.startswith("#### "):
|
||||
break
|
||||
if not in_section or not line.startswith(f"| {row_name} |"):
|
||||
continue
|
||||
|
||||
cells = line.split("|")
|
||||
if len(cells) < 5:
|
||||
continue
|
||||
cells[2] = f" {type_markdown} "
|
||||
lines[index] = "|".join(cells)
|
||||
break
|
||||
|
||||
return "\n".join(lines) + ("\n" if markdown.endswith("\n") else "")
|
||||
|
||||
|
||||
def _has_union_schema(schema: object) -> bool:
|
||||
if not isinstance(schema, dict):
|
||||
return False
|
||||
if isinstance(schema.get("oneOf"), list) or isinstance(schema.get("anyOf"), list):
|
||||
return True
|
||||
return _has_union_schema(schema.get("items"))
|
||||
|
||||
|
||||
def _patch_union_schema_markdown(markdown: str, spec_path: Path) -> str:
|
||||
"""Fill Markdown table cells that `swagger-markdown` leaves blank for unions, including array items."""
|
||||
|
||||
spec = json.loads(spec_path.read_text(encoding="utf-8"))
|
||||
components = spec.get("components")
|
||||
schemas = components.get("schemas") if isinstance(components, dict) else None
|
||||
if not isinstance(schemas, dict):
|
||||
return markdown
|
||||
|
||||
for schema_name, schema in schemas.items():
|
||||
if not isinstance(schema_name, str) or not isinstance(schema, dict):
|
||||
continue
|
||||
|
||||
properties = schema.get("properties")
|
||||
if isinstance(properties, dict):
|
||||
for property_name, property_schema in properties.items():
|
||||
if isinstance(property_name, str) and _has_union_schema(property_schema):
|
||||
markdown = _replace_schema_table_type(
|
||||
markdown,
|
||||
schema_name,
|
||||
property_name,
|
||||
_schema_markdown_type(property_schema),
|
||||
)
|
||||
|
||||
union_variants = schema.get("oneOf") or schema.get("anyOf")
|
||||
if not isinstance(union_variants, list):
|
||||
continue
|
||||
|
||||
markdown = _replace_schema_table_type(
|
||||
markdown,
|
||||
schema_name,
|
||||
schema_name,
|
||||
_schema_markdown_type(schema),
|
||||
)
|
||||
|
||||
for variant in union_variants:
|
||||
variant_name = _schema_ref_name(variant)
|
||||
variant_schema = schemas.get(variant_name) if variant_name is not None else None
|
||||
if not isinstance(variant_name, str) or not isinstance(variant_schema, dict):
|
||||
continue
|
||||
properties = variant_schema.get("properties")
|
||||
if not isinstance(properties, dict):
|
||||
continue
|
||||
for property_name, property_schema in properties.items():
|
||||
if isinstance(property_name, str):
|
||||
markdown = _replace_schema_table_type(
|
||||
markdown,
|
||||
variant_name,
|
||||
property_name,
|
||||
_schema_markdown_type(property_schema),
|
||||
)
|
||||
|
||||
return markdown
|
||||
|
||||
|
||||
def _strip_trailing_line_whitespace(markdown: str) -> str:
|
||||
"""Remove converter-emitted trailing whitespace without changing line structure."""
|
||||
|
||||
return "\n".join(line.rstrip(" \t") for line in markdown.split("\n"))
|
||||
|
||||
|
||||
# Deprecated operations render struck through: ``### ~~[POST] /path~~``.
|
||||
_OPERATION_HEADING = re.compile(r"^### (?:~~)?\[(?P<method>[A-Z]+)\] (?P<path>\S+?)(?:~~)?$")
|
||||
_BLANK_MEDIA_TYPE_CELL = re.compile(r"\*\*(?P<media>[^*]+)\*\*: <br>")
|
||||
|
||||
|
||||
def _patch_union_response_markdown(markdown: str, spec_path: Path) -> str:
|
||||
"""Fill the response Schema cells `swagger-markdown` leaves blank for union bodies.
|
||||
|
||||
An error status that documents business errors is an ``anyOf`` of their components and
|
||||
the open error body; the converter renders such a body as ``**application/json**: <br>``.
|
||||
Only blank cells are filled, from the operation the enclosing ``### [METHOD] /path`` names.
|
||||
"""
|
||||
|
||||
spec = json.loads(spec_path.read_text(encoding="utf-8"))
|
||||
paths = spec.get("paths")
|
||||
if not isinstance(paths, dict):
|
||||
return markdown
|
||||
|
||||
lines = markdown.splitlines()
|
||||
responses: dict[str, object] = {}
|
||||
for index, line in enumerate(lines):
|
||||
heading = _OPERATION_HEADING.match(line)
|
||||
if heading is not None:
|
||||
path_item = paths.get(heading["path"])
|
||||
operation = path_item.get(heading["method"].lower()) if isinstance(path_item, dict) else None
|
||||
found = operation.get("responses") if isinstance(operation, dict) else None
|
||||
responses = found if isinstance(found, dict) else {}
|
||||
continue
|
||||
if not responses or not line.startswith("|") or "**: <br>" not in line:
|
||||
continue
|
||||
cells = re.split(r"(?<!\\)\|", line.strip().strip("|"))
|
||||
if len(cells) != 3:
|
||||
continue
|
||||
response = responses.get(cells[0].strip())
|
||||
content = response.get("content") if isinstance(response, dict) else None
|
||||
if not isinstance(content, dict):
|
||||
continue
|
||||
|
||||
def fill(match: re.Match[str], content: dict[str, object] = content) -> str:
|
||||
media = content.get(match["media"])
|
||||
schema_type = _schema_markdown_type(media.get("schema")) if isinstance(media, dict) else ""
|
||||
return f"**{match['media']}**: {schema_type}<br>" if schema_type else match.group(0)
|
||||
|
||||
cells[2] = _BLANK_MEDIA_TYPE_CELL.sub(fill, cells[2])
|
||||
lines[index] = f"|{'|'.join(cells)}|"
|
||||
return "\n".join(lines) + ("\n" if markdown.endswith("\n") else "")
|
||||
|
||||
|
||||
def _patch_wildcard_media_type_markdown(markdown: str) -> str:
|
||||
"""Keep swagger-markdown from interpreting the wildcard media type as emphasis."""
|
||||
|
||||
return markdown.replace("***/***", "`*/*`")
|
||||
|
||||
|
||||
def _patch_request_body_markdown(markdown: str) -> str:
|
||||
"""Collapse swagger-markdown 3.0.0's repeated cells for multi-content request bodies.
|
||||
|
||||
Its request-body renderer appends the same mutable Schema cell for every
|
||||
content type, so every extra column repeats the complete set of media types.
|
||||
Only repair that exact shape; preserve distinct cells and other tables.
|
||||
"""
|
||||
lines = markdown.splitlines()
|
||||
for index, line in enumerate(lines):
|
||||
if index < 2 or lines[index - 2].strip() != "| Required | Schema |":
|
||||
continue
|
||||
cells = [cell.strip() for cell in re.split(r"(?<!\\)\|", line.strip().strip("|"))]
|
||||
if len(cells) > 2 and cells[0] in {"Yes", "No"} and len(set(cells[1:])) == 1:
|
||||
lines[index] = f"| {cells[0]} | {cells[1]} |"
|
||||
return "\n".join(lines) + ("\n" if markdown.endswith("\n") else "")
|
||||
|
||||
|
||||
def _drop_null_values_for_markdown(value: object) -> object:
|
||||
"""Remove null object members only from the converter's temporary input."""
|
||||
|
||||
if isinstance(value, dict):
|
||||
return {key: _drop_null_values_for_markdown(item) for key, item in value.items() if item is not None}
|
||||
if isinstance(value, list):
|
||||
return [_drop_null_values_for_markdown(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def _convert_spec_to_markdown(spec_path: Path, markdown_path: Path) -> None:
|
||||
markdown_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(prefix=f"{markdown_path.stem}-") as temp_dir:
|
||||
converter_spec_path = Path(temp_dir) / spec_path.name
|
||||
converter_payload = json.loads(spec_path.read_text(encoding="utf-8"))
|
||||
converter_spec_path.write_text(
|
||||
json.dumps(_drop_null_values_for_markdown(converter_payload)),
|
||||
encoding="utf-8",
|
||||
)
|
||||
temp_markdown_path = Path(temp_dir) / markdown_path.name
|
||||
result = subprocess.run(
|
||||
[
|
||||
"npx",
|
||||
"--yes",
|
||||
SWAGGER_MARKDOWN_PACKAGE,
|
||||
"-i",
|
||||
str(converter_spec_path.resolve()),
|
||||
"-o",
|
||||
str(temp_markdown_path.resolve()),
|
||||
],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
cwd=temp_dir,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
logger.error(
|
||||
"swagger-markdown failed for %s (exit code %s)\nstdout:\n%s\nstderr:\n%s",
|
||||
spec_path,
|
||||
result.returncode,
|
||||
result.stdout,
|
||||
result.stderr,
|
||||
)
|
||||
raise subprocess.CalledProcessError(
|
||||
result.returncode,
|
||||
result.args,
|
||||
output=result.stdout,
|
||||
stderr=result.stderr,
|
||||
)
|
||||
if not temp_markdown_path.exists():
|
||||
converter_output = "\n".join(item for item in (result.stdout, result.stderr) if item).strip()
|
||||
raise RuntimeError(f"swagger-markdown did not write {markdown_path}: {converter_output}")
|
||||
|
||||
converted_markdown = _patch_union_schema_markdown(
|
||||
temp_markdown_path.read_text(encoding="utf-8"),
|
||||
spec_path,
|
||||
)
|
||||
converted_markdown = _patch_union_response_markdown(converted_markdown, spec_path)
|
||||
converted_markdown = _patch_wildcard_media_type_markdown(converted_markdown)
|
||||
converted_markdown = _patch_request_body_markdown(converted_markdown)
|
||||
converted_markdown = _strip_trailing_line_whitespace(converted_markdown)
|
||||
if not converted_markdown.strip():
|
||||
raise RuntimeError(f"swagger-markdown wrote an empty document for {markdown_path}")
|
||||
|
||||
markdown_path.write_text(converted_markdown, encoding="utf-8")
|
||||
|
||||
|
||||
def _demote_markdown_headings(markdown: str, *, levels: int = 1) -> str:
|
||||
"""Nest generated Markdown under another Markdown section."""
|
||||
|
||||
heading_prefix = "#" * levels
|
||||
lines = []
|
||||
for line in markdown.splitlines():
|
||||
if line.startswith("#"):
|
||||
lines.append(f"{heading_prefix}{line}")
|
||||
else:
|
||||
lines.append(line)
|
||||
return "\n".join(lines).strip()
|
||||
|
||||
|
||||
def _append_fastopenapi_markdown(console_markdown_path: Path, fastopenapi_markdown_path: Path) -> None:
|
||||
"""Append FastOpenAPI console docs to the existing console API Markdown."""
|
||||
|
||||
console_markdown = console_markdown_path.read_text(encoding="utf-8").rstrip()
|
||||
fastopenapi_markdown = _demote_markdown_headings(
|
||||
fastopenapi_markdown_path.read_text(encoding="utf-8"),
|
||||
levels=2,
|
||||
)
|
||||
console_markdown_path.write_text(
|
||||
"\n\n".join(
|
||||
[
|
||||
console_markdown,
|
||||
"## FastOpenAPI Preview (OpenAPI 3.1)",
|
||||
fastopenapi_markdown,
|
||||
]
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def generate_markdown_docs(
|
||||
openapi_dir: Path,
|
||||
markdown_dir: Path,
|
||||
*,
|
||||
keep_swagger_json: bool = False,
|
||||
) -> list[Path]:
|
||||
"""Generate intermediate specs, convert them to split Markdown API docs, and return Markdown paths."""
|
||||
|
||||
openapi_paths = generate_specs(openapi_dir)
|
||||
fastopenapi_paths = generate_fastopenapi_specs(openapi_dir)
|
||||
spec_paths = [*openapi_paths, *fastopenapi_paths]
|
||||
openapi_paths_by_name = {path.name: path for path in openapi_paths}
|
||||
fastopenapi_paths_by_name = {path.name: path for path in fastopenapi_paths}
|
||||
|
||||
markdown_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
written_paths: list[Path] = []
|
||||
try:
|
||||
with tempfile.TemporaryDirectory(prefix="dify-api-docs-") as temp_dir:
|
||||
temp_markdown_dir = Path(temp_dir)
|
||||
|
||||
for target in MARKDOWN_SPEC_TARGETS:
|
||||
openapi_path = openapi_paths_by_name[target.filename]
|
||||
markdown_path = markdown_dir / f"{openapi_path.stem}.md"
|
||||
_convert_spec_to_markdown(openapi_path, markdown_path)
|
||||
written_paths.append(markdown_path)
|
||||
|
||||
for target in FASTOPENAPI_SPEC_TARGETS: # type: ignore
|
||||
fastopenapi_path = fastopenapi_paths_by_name[target.filename]
|
||||
markdown_path = temp_markdown_dir / f"{fastopenapi_path.stem}.md"
|
||||
_convert_spec_to_markdown(fastopenapi_path, markdown_path)
|
||||
|
||||
console_markdown_path = markdown_dir / f"{Path(CONSOLE_OPENAPI_FILENAME).stem}.md"
|
||||
_append_fastopenapi_markdown(console_markdown_path, markdown_path)
|
||||
|
||||
(markdown_dir / STALE_COMBINED_MARKDOWN_FILENAME).unlink(missing_ok=True)
|
||||
finally:
|
||||
if not keep_swagger_json:
|
||||
for path in spec_paths:
|
||||
path.unlink(missing_ok=True)
|
||||
|
||||
return written_paths
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--swagger-dir",
|
||||
"--openapi-dir",
|
||||
dest="openapi_dir",
|
||||
type=Path,
|
||||
default=Path("openapi"),
|
||||
help="Directory where intermediate JSON spec files will be written.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--markdown-dir",
|
||||
type=Path,
|
||||
default=Path("openapi/markdown"),
|
||||
help="Directory where split Markdown API docs will be written.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--keep-swagger-json",
|
||||
action="store_true",
|
||||
help="Keep intermediate JSON spec files after Markdown generation.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
written_paths = generate_markdown_docs(
|
||||
args.openapi_dir,
|
||||
args.markdown_dir,
|
||||
keep_swagger_json=args.keep_swagger_json,
|
||||
)
|
||||
|
||||
for path in written_paths:
|
||||
logger.debug(path)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,342 @@
|
||||
"""Generate Flask-RESTX OpenAPI 3 specs without booting the full backend.
|
||||
|
||||
This helper intentionally avoids `app_factory.create_app()`. The normal backend
|
||||
startup eagerly initializes database, Redis, Celery, and storage extensions,
|
||||
which is unnecessary when the goal is only to serialize the Flask-RESTX
|
||||
`/openapi.json` documents.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from collections.abc import MutableMapping
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Protocol, TypeGuard, cast
|
||||
|
||||
from flask import Flask
|
||||
from flask_restx import Api
|
||||
from flask_restx.swagger import Swagger
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
API_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(API_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(API_ROOT))
|
||||
|
||||
# Import after script path setup.
|
||||
from libs.flask_restx_compat import finalize_openapi_payload # noqa: E402
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SpecTarget:
|
||||
route: str
|
||||
filename: str
|
||||
|
||||
|
||||
class RestxApi(Protocol):
|
||||
models: MutableMapping[str, object]
|
||||
|
||||
def model(self, name: str, model: dict[object, object]) -> object: ...
|
||||
|
||||
|
||||
SPEC_TARGETS: tuple[SpecTarget, ...] = (
|
||||
SpecTarget(route="/console/api/openapi.json", filename="console-openapi.json"),
|
||||
SpecTarget(route="/api/openapi.json", filename="web-openapi.json"),
|
||||
SpecTarget(route="/v1/openapi.json", filename="service-openapi.json"),
|
||||
SpecTarget(route="/openapi/v1/openapi.json", filename="openapi-openapi.json"),
|
||||
# Not consumed by @dify/contracts; enterprise generates its inner API client from this spec.
|
||||
SpecTarget(route="/inner/api/openapi.json", filename="inner-openapi.json"),
|
||||
)
|
||||
|
||||
|
||||
def _is_inline_field_map(value: object) -> TypeGuard[dict[object, object]]:
|
||||
"""Return whether a nested field map is an anonymous inline mapping."""
|
||||
|
||||
from flask_restx.model import Model, OrderedModel
|
||||
|
||||
return isinstance(value, dict) and not isinstance(value, (Model, OrderedModel))
|
||||
|
||||
|
||||
def _jsonable_schema_value(value: object) -> object:
|
||||
"""Return a deterministic JSON-serializable representation for schema fingerprints."""
|
||||
|
||||
if value is None or isinstance(value, str | int | float | bool):
|
||||
return value
|
||||
if isinstance(value, list | tuple):
|
||||
return [_jsonable_schema_value(item) for item in value]
|
||||
if isinstance(value, dict):
|
||||
return {str(key): _jsonable_schema_value(item) for key, item in value.items()}
|
||||
value_type = type(value)
|
||||
return f"<{value_type.__module__}.{value_type.__qualname__}>"
|
||||
|
||||
|
||||
def _field_signature(field: object) -> object:
|
||||
"""Build a stable signature for a Flask-RESTX field object."""
|
||||
|
||||
from flask_restx import fields
|
||||
from flask_restx.model import instance
|
||||
|
||||
field_instance = instance(field)
|
||||
signature: dict[str, object] = {
|
||||
"class": f"{field_instance.__class__.__module__}.{field_instance.__class__.__qualname__}"
|
||||
}
|
||||
|
||||
if isinstance(field_instance, fields.Nested):
|
||||
nested = getattr(field_instance, "nested", None)
|
||||
if _is_inline_field_map(nested):
|
||||
signature["nested"] = _inline_model_signature(nested)
|
||||
else:
|
||||
signature["nested"] = getattr(
|
||||
nested,
|
||||
"name",
|
||||
f"<{type(nested).__module__}.{type(nested).__qualname__}>",
|
||||
)
|
||||
elif hasattr(field_instance, "container"):
|
||||
signature["container"] = _field_signature(field_instance.container)
|
||||
else:
|
||||
schema = getattr(field_instance, "__schema__", None)
|
||||
if isinstance(schema, dict):
|
||||
signature["schema"] = _jsonable_schema_value(schema)
|
||||
|
||||
for attr_name in (
|
||||
"attribute",
|
||||
"default",
|
||||
"description",
|
||||
"example",
|
||||
"max",
|
||||
"max_items",
|
||||
"min",
|
||||
"min_items",
|
||||
"nullable",
|
||||
"readonly",
|
||||
"required",
|
||||
"title",
|
||||
"unique",
|
||||
):
|
||||
if hasattr(field_instance, attr_name):
|
||||
signature[attr_name] = _jsonable_schema_value(getattr(field_instance, attr_name))
|
||||
|
||||
return signature
|
||||
|
||||
|
||||
def _inline_model_signature(nested_fields: dict[object, object]) -> object:
|
||||
"""Build a stable signature for an anonymous inline model."""
|
||||
|
||||
return [
|
||||
(str(field_name), _field_signature(field))
|
||||
for field_name, field in sorted(nested_fields.items(), key=lambda item: str(item[0]))
|
||||
]
|
||||
|
||||
|
||||
def _inline_model_name(nested_fields: dict[object, object]) -> str:
|
||||
"""Return a stable OpenAPI model name for an anonymous inline field map."""
|
||||
|
||||
signature = json.dumps(_inline_model_signature(nested_fields), sort_keys=True, separators=(",", ":"))
|
||||
digest = hashlib.sha1(signature.encode("utf-8")).hexdigest()[:12]
|
||||
return f"_AnonymousInlineModel_{digest}"
|
||||
|
||||
|
||||
def apply_runtime_defaults() -> None:
|
||||
"""Force the small config surface required for OpenAPI generation."""
|
||||
|
||||
os.environ.setdefault("SECRET_KEY", "spec-export")
|
||||
os.environ.setdefault("STORAGE_TYPE", "local")
|
||||
os.environ.setdefault("STORAGE_LOCAL_PATH", "/tmp/dify-storage")
|
||||
|
||||
from configs import dify_config
|
||||
|
||||
dify_config.SECRET_KEY = os.environ["SECRET_KEY"]
|
||||
dify_config.STORAGE_TYPE = "local"
|
||||
dify_config.STORAGE_LOCAL_PATH = os.environ["STORAGE_LOCAL_PATH"]
|
||||
dify_config.SWAGGER_UI_ENABLED = True
|
||||
|
||||
|
||||
def create_spec_app() -> Flask:
|
||||
"""Build a minimal Flask app that only mounts the OpenAPI-producing blueprints."""
|
||||
|
||||
apply_runtime_defaults()
|
||||
|
||||
from libs.flask_restx_compat import install_swagger_compatibility
|
||||
|
||||
install_swagger_compatibility()
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
from controllers.console import bp as console_bp
|
||||
from controllers.console import console_ns
|
||||
from controllers.inner_api import bp as inner_api_bp
|
||||
from controllers.inner_api import inner_api_ns
|
||||
from controllers.openapi import bp as openapi_bp
|
||||
from controllers.openapi import openapi_ns
|
||||
from controllers.service_api import bp as service_api_bp
|
||||
from controllers.service_api import service_api_ns
|
||||
from controllers.web import bp as web_bp
|
||||
from controllers.web import web_ns
|
||||
|
||||
app.register_blueprint(console_bp)
|
||||
app.register_blueprint(web_bp)
|
||||
app.register_blueprint(service_api_bp)
|
||||
app.register_blueprint(openapi_bp)
|
||||
app.register_blueprint(inner_api_bp)
|
||||
|
||||
for namespace in (console_ns, web_ns, service_api_ns, openapi_ns, inner_api_ns):
|
||||
for api in namespace.apis:
|
||||
_materialize_inline_model_definitions(api)
|
||||
|
||||
return app
|
||||
|
||||
|
||||
def _target_apis() -> dict[str, Api]:
|
||||
"""Return the API instance that owns each exported OpenAPI target."""
|
||||
|
||||
from controllers.console import api as console_api
|
||||
from controllers.inner_api import api as inner_api
|
||||
from controllers.openapi import api as openapi_api
|
||||
from controllers.service_api import api as service_api
|
||||
from controllers.web import api as web_api
|
||||
|
||||
return {
|
||||
"/console/api/openapi.json": console_api,
|
||||
"/api/openapi.json": web_api,
|
||||
"/v1/openapi.json": service_api,
|
||||
"/openapi/v1/openapi.json": openapi_api,
|
||||
"/inner/api/openapi.json": inner_api,
|
||||
}
|
||||
|
||||
|
||||
def _materialize_inline_model_definitions(api: RestxApi) -> None:
|
||||
"""Convert inline `fields.Nested({...})` maps into named API models."""
|
||||
|
||||
from flask_restx import fields
|
||||
from flask_restx.model import Model, OrderedModel, instance
|
||||
|
||||
inline_models: dict[int, dict[object, object]] = {}
|
||||
inline_model_names: dict[int, str] = {}
|
||||
|
||||
def collect_field(field: object) -> None:
|
||||
field_instance = instance(field)
|
||||
if isinstance(field_instance, fields.Nested):
|
||||
nested = getattr(field_instance, "nested", None)
|
||||
if _is_inline_field_map(nested) and id(nested) not in inline_models:
|
||||
inline_models[id(nested)] = nested
|
||||
for nested_field in nested.values():
|
||||
collect_field(nested_field)
|
||||
|
||||
container = getattr(field_instance, "container", None)
|
||||
if container is not None:
|
||||
collect_field(container)
|
||||
|
||||
for model in list(api.models.values()):
|
||||
if isinstance(model, (Model, OrderedModel)):
|
||||
for field in model.values():
|
||||
collect_field(field)
|
||||
|
||||
for nested_fields in sorted(inline_models.values(), key=_inline_model_name):
|
||||
anonymous_name = _inline_model_name(nested_fields)
|
||||
inline_model_names[id(nested_fields)] = anonymous_name
|
||||
if anonymous_name not in api.models:
|
||||
api.model(anonymous_name, nested_fields)
|
||||
|
||||
def model_name_for(nested_fields: dict[object, object]) -> str:
|
||||
anonymous_name = inline_model_names.get(id(nested_fields))
|
||||
if anonymous_name is None:
|
||||
anonymous_name = _inline_model_name(nested_fields)
|
||||
inline_model_names[id(nested_fields)] = anonymous_name
|
||||
if anonymous_name not in api.models:
|
||||
api.model(anonymous_name, nested_fields)
|
||||
return anonymous_name
|
||||
|
||||
def materialize_field(field: object) -> None:
|
||||
field_instance = instance(field)
|
||||
if isinstance(field_instance, fields.Nested):
|
||||
nested = getattr(field_instance, "nested", None)
|
||||
if _is_inline_field_map(nested):
|
||||
field_instance.model = api.models[model_name_for(nested)] # type: ignore[attr-defined]
|
||||
|
||||
container = getattr(field_instance, "container", None)
|
||||
if container is not None:
|
||||
materialize_field(container)
|
||||
|
||||
index = 0
|
||||
while index < len(api.models):
|
||||
model = list(api.models.values())[index]
|
||||
index += 1
|
||||
if isinstance(model, (Model, OrderedModel)):
|
||||
for field in model.values():
|
||||
materialize_field(field)
|
||||
|
||||
|
||||
def export_console_spec() -> dict[str, Any]:
|
||||
"""Build only Console; editor refreshes do not need the other four API surfaces."""
|
||||
apply_runtime_defaults()
|
||||
from libs.flask_restx_compat import install_swagger_compatibility
|
||||
|
||||
install_swagger_compatibility()
|
||||
from controllers.console import api, bp
|
||||
|
||||
app = Flask(__name__)
|
||||
app.register_blueprint(bp)
|
||||
_materialize_inline_model_definitions(cast(RestxApi, api))
|
||||
with app.test_request_context("/console/api/openapi.json"):
|
||||
return finalize_openapi_payload(Swagger(api).as_dict())
|
||||
|
||||
|
||||
def export_specs() -> dict[str, dict[str, Any]]:
|
||||
"""Build OpenAPI documents in memory without writing generated files."""
|
||||
app = create_spec_app()
|
||||
target_apis = _target_apis()
|
||||
documents = {}
|
||||
for target in SPEC_TARGETS:
|
||||
# Build directly from the API instance instead of depending on the
|
||||
# optional HTTP spec route. ExternalApi decides whether to register
|
||||
# that route when controller modules are first imported, which may
|
||||
# happen before this exporter forces its generation-only defaults.
|
||||
with app.test_request_context(target.route):
|
||||
payload = Swagger(target_apis[target.route]).as_dict()
|
||||
payload = finalize_openapi_payload(payload)
|
||||
documents[target.filename] = payload
|
||||
return documents
|
||||
|
||||
|
||||
def generate_specs(output_dir: Path) -> list[Path]:
|
||||
"""Write all OpenAPI specs to `output_dir` and return the written paths."""
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
written_paths: list[Path] = []
|
||||
for filename, payload in export_specs().items():
|
||||
output_path = output_dir / filename
|
||||
output_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
written_paths.append(output_path)
|
||||
|
||||
return written_paths
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--output-dir",
|
||||
type=Path,
|
||||
default=Path("openapi"),
|
||||
help="Directory where the OpenAPI JSON files will be written.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
written_paths = generate_specs(args.output_dir)
|
||||
|
||||
for path in written_paths:
|
||||
logger.debug(path)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,494 @@
|
||||
"""Cross-service KnowledgeFS product-operation contract validation.
|
||||
|
||||
The checked product manifest is the reviewable boundary, while the Dify product registry, Dify
|
||||
Capability issuer registry, exported KnowledgeFS OpenAPI, and exported TypeScript request-guard
|
||||
policy remain executable sources of truth. A product operation must appear exactly once as ready
|
||||
or as an explicit gap; internal KFS-only operations require a named exclusion.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable, Mapping
|
||||
from typing import Any, Literal, NamedTuple, TypedDict, cast
|
||||
|
||||
|
||||
class ProductOperationLimits(TypedDict):
|
||||
kfsMaxResponseBytes: int
|
||||
productMaxRequestBytes: int
|
||||
productMaxResponseBytes: int
|
||||
|
||||
|
||||
class ProductOperationStream(TypedDict):
|
||||
kfsResponseKind: str
|
||||
productKind: str
|
||||
|
||||
|
||||
class ProductOperationManifestEntry(TypedDict):
|
||||
action: str
|
||||
kfsOperationId: str
|
||||
limits: ProductOperationLimits
|
||||
method: str
|
||||
path: str
|
||||
productOperationId: str
|
||||
resource: str
|
||||
stream: ProductOperationStream
|
||||
transport: str
|
||||
|
||||
|
||||
type ProductOperationManifestField = Literal[
|
||||
"action",
|
||||
"kfsOperationId",
|
||||
"limits",
|
||||
"method",
|
||||
"path",
|
||||
"productOperationId",
|
||||
"resource",
|
||||
"stream",
|
||||
"transport",
|
||||
]
|
||||
|
||||
PRODUCT_OPERATION_MANIFEST_FIELDS: tuple[ProductOperationManifestField, ...] = (
|
||||
"action",
|
||||
"kfsOperationId",
|
||||
"limits",
|
||||
"method",
|
||||
"path",
|
||||
"productOperationId",
|
||||
"resource",
|
||||
"stream",
|
||||
"transport",
|
||||
)
|
||||
|
||||
|
||||
class ProductOperationManifest(TypedDict):
|
||||
operations: list[ProductOperationManifestEntry]
|
||||
schemaVersion: int
|
||||
|
||||
|
||||
class ProductOperationGapEntry(ProductOperationManifestEntry):
|
||||
reason: str
|
||||
reasonCode: str
|
||||
replacementProductOperationIds: list[str]
|
||||
|
||||
|
||||
class InternalKfsOperationExclusion(TypedDict):
|
||||
kfsOperationId: str
|
||||
reason: str
|
||||
reasonCode: str
|
||||
|
||||
|
||||
class ProductOperationGapManifest(TypedDict):
|
||||
gaps: list[ProductOperationGapEntry]
|
||||
internalKfsOperationExclusions: list[InternalKfsOperationExclusion]
|
||||
schemaVersion: int
|
||||
|
||||
|
||||
class ProductOperationRuntimeContract(NamedTuple):
|
||||
action: str
|
||||
kfs_operation_id: str
|
||||
max_request_bytes: int
|
||||
max_response_bytes: int
|
||||
method: str
|
||||
path: str
|
||||
product_operation_id: str
|
||||
ready: bool
|
||||
resource: str
|
||||
stream_kind: str
|
||||
transport: str
|
||||
|
||||
|
||||
class CapabilityOperationRuntimeContract(NamedTuple):
|
||||
action: str
|
||||
allowed_caller_kinds: tuple[str, ...]
|
||||
method: str
|
||||
operation_id: str
|
||||
path: str
|
||||
resource: str
|
||||
|
||||
|
||||
class CapabilityOperationPolicy(NamedTuple):
|
||||
action: str
|
||||
allowed_caller_kinds: tuple[str, ...]
|
||||
method: str
|
||||
operation_id: str
|
||||
parent_resource_binding: dict[str, object] | None
|
||||
path: str
|
||||
resource: str
|
||||
resource_binding: dict[str, object]
|
||||
|
||||
|
||||
def product_operation_runtime_contracts() -> tuple[ProductOperationRuntimeContract, ...]:
|
||||
"""Read the complete typed Dify product registry without normalizing away gaps."""
|
||||
from services.knowledge_fs.product_operations import (
|
||||
KNOWLEDGE_FS_PRODUCT_OPERATIONS,
|
||||
is_product_operation_ready,
|
||||
)
|
||||
|
||||
contracts: list[ProductOperationRuntimeContract] = []
|
||||
for product_operation_id, operation in KNOWLEDGE_FS_PRODUCT_OPERATIONS.items():
|
||||
if operation.capability_operation_id is None or operation.kfs_path is None or operation.action is None:
|
||||
raise ValueError(f"Dify product operation {product_operation_id} has no Capability contract")
|
||||
contracts.append(
|
||||
ProductOperationRuntimeContract(
|
||||
action=operation.action,
|
||||
kfs_operation_id=operation.capability_operation_id,
|
||||
max_request_bytes=operation.max_request_bytes,
|
||||
max_response_bytes=operation.max_response_bytes,
|
||||
method=operation.method,
|
||||
path=operation.kfs_path,
|
||||
product_operation_id=product_operation_id,
|
||||
ready=is_product_operation_ready(product_operation_id),
|
||||
resource=operation.resource_resolver,
|
||||
stream_kind=operation.stream_kind,
|
||||
transport=operation.transport,
|
||||
)
|
||||
)
|
||||
return tuple(contracts)
|
||||
|
||||
|
||||
def capability_operation_runtime_contracts() -> tuple[CapabilityOperationRuntimeContract, ...]:
|
||||
"""Read the Python issuer policy used to mint each operation Capability."""
|
||||
from services.knowledge_fs_capability import KNOWLEDGE_FS_CAPABILITY_OPERATIONS
|
||||
|
||||
return tuple(
|
||||
CapabilityOperationRuntimeContract(
|
||||
action=operation.action,
|
||||
allowed_caller_kinds=operation.allowed_caller_kinds,
|
||||
method=operation.method,
|
||||
operation_id=operation_id,
|
||||
path=operation.path,
|
||||
resource=operation.resource_type,
|
||||
)
|
||||
for operation_id, operation in KNOWLEDGE_FS_CAPABILITY_OPERATIONS.items()
|
||||
)
|
||||
|
||||
|
||||
def parse_product_operation_manifest(value: dict[str, Any]) -> ProductOperationManifest:
|
||||
"""Parse the ready product-operation inventory with a closed schema."""
|
||||
if set(value) != {"operations", "schemaVersion"} or value.get("schemaVersion") != 2:
|
||||
raise ValueError("KnowledgeFS product operation manifest must use schemaVersion 2")
|
||||
operations = value.get("operations")
|
||||
if not isinstance(operations, list) or not operations:
|
||||
raise ValueError("KnowledgeFS product operation manifest operations must be a non-empty list")
|
||||
return {
|
||||
"operations": [_parse_product_entry(entry, gap=False) for entry in operations],
|
||||
"schemaVersion": 2,
|
||||
}
|
||||
|
||||
|
||||
def parse_product_operation_gap_manifest(value: dict[str, Any]) -> ProductOperationGapManifest:
|
||||
"""Parse explicit product gaps and KFS-only operation exclusions with a closed schema."""
|
||||
expected_fields = {"gaps", "internalKfsOperationExclusions", "schemaVersion"}
|
||||
if set(value) != expected_fields or value.get("schemaVersion") != 2:
|
||||
raise ValueError("KnowledgeFS product operation gap manifest must use schemaVersion 2")
|
||||
gaps = value.get("gaps")
|
||||
exclusions = value.get("internalKfsOperationExclusions")
|
||||
if not isinstance(gaps, list) or not isinstance(exclusions, list):
|
||||
raise ValueError("KnowledgeFS product operation gap manifest lists are invalid")
|
||||
parsed_exclusions: list[InternalKfsOperationExclusion] = []
|
||||
for raw in exclusions:
|
||||
if not isinstance(raw, dict) or set(raw) != {"kfsOperationId", "reason", "reasonCode"}:
|
||||
raise ValueError("KnowledgeFS internal KFS operation exclusion fields are invalid")
|
||||
for field in ("kfsOperationId", "reason", "reasonCode"):
|
||||
_required_string(raw.get(field), f"internal exclusion {field}")
|
||||
parsed_exclusions.append(cast(InternalKfsOperationExclusion, raw))
|
||||
return {
|
||||
"gaps": [cast(ProductOperationGapEntry, _parse_product_entry(entry, gap=True)) for entry in gaps],
|
||||
"internalKfsOperationExclusions": parsed_exclusions,
|
||||
"schemaVersion": 2,
|
||||
}
|
||||
|
||||
|
||||
def parse_capability_operation_policy(value: dict[str, Any]) -> tuple[CapabilityOperationPolicy, ...]:
|
||||
"""Parse the runtime-exported TypeScript request-guard registry."""
|
||||
if set(value) != {"operations", "schemaVersion"} or value.get("schemaVersion") != 1:
|
||||
raise ValueError("KnowledgeFS Capability operation policy must use schemaVersion 1")
|
||||
operations = value.get("operations")
|
||||
if not isinstance(operations, list) or not operations:
|
||||
raise ValueError("KnowledgeFS Capability operation policy must contain operations")
|
||||
parsed: list[CapabilityOperationPolicy] = []
|
||||
for raw in operations:
|
||||
expected_fields = {
|
||||
"action",
|
||||
"allowedCallerKinds",
|
||||
"method",
|
||||
"operationId",
|
||||
"parentResourceBinding",
|
||||
"path",
|
||||
"resourceBinding",
|
||||
"resourceType",
|
||||
}
|
||||
if not isinstance(raw, dict) or set(raw) != expected_fields:
|
||||
raise ValueError("KnowledgeFS Capability operation policy fields are invalid")
|
||||
callers = raw.get("allowedCallerKinds")
|
||||
resource_binding = raw.get("resourceBinding")
|
||||
parent_binding = raw.get("parentResourceBinding")
|
||||
if (
|
||||
not isinstance(callers, list)
|
||||
or not callers
|
||||
or not all(isinstance(caller, str) and caller for caller in callers)
|
||||
or not isinstance(resource_binding, dict)
|
||||
or not resource_binding
|
||||
or (parent_binding is not None and not isinstance(parent_binding, dict))
|
||||
):
|
||||
raise ValueError("KnowledgeFS Capability operation request binding is invalid")
|
||||
for field in ("action", "method", "operationId", "path", "resourceType"):
|
||||
_required_string(raw.get(field), f"Capability operation {field}")
|
||||
parsed.append(
|
||||
CapabilityOperationPolicy(
|
||||
action=cast(str, raw["action"]),
|
||||
allowed_caller_kinds=tuple(callers),
|
||||
method=cast(str, raw["method"]),
|
||||
operation_id=cast(str, raw["operationId"]),
|
||||
parent_resource_binding=cast(dict[str, object] | None, parent_binding),
|
||||
path=cast(str, raw["path"]),
|
||||
resource=cast(str, raw["resourceType"]),
|
||||
resource_binding=cast(dict[str, object], resource_binding),
|
||||
)
|
||||
)
|
||||
_unique((operation.operation_id for operation in parsed), "Capability operation policy")
|
||||
return tuple(parsed)
|
||||
|
||||
|
||||
def validate_product_operation_contracts(
|
||||
*,
|
||||
capability_operations: tuple[CapabilityOperationRuntimeContract, ...],
|
||||
capability_policy: tuple[CapabilityOperationPolicy, ...],
|
||||
document: dict[str, Any],
|
||||
gap_manifest: ProductOperationGapManifest,
|
||||
manifest: ProductOperationManifest,
|
||||
product_operations: tuple[ProductOperationRuntimeContract, ...],
|
||||
) -> None:
|
||||
"""Require exact agreement across manifests, Dify registries, KFS guard policy, and OpenAPI."""
|
||||
ready_entries = {entry["productOperationId"]: entry for entry in manifest["operations"]}
|
||||
gap_entries = {entry["productOperationId"]: entry for entry in gap_manifest["gaps"]}
|
||||
if len(ready_entries) != len(manifest["operations"]):
|
||||
raise ValueError("KnowledgeFS product operation manifest contains duplicate product ids")
|
||||
if len(gap_entries) != len(gap_manifest["gaps"]):
|
||||
raise ValueError("KnowledgeFS product operation gap manifest contains duplicate product ids")
|
||||
if set(ready_entries) & set(gap_entries):
|
||||
raise ValueError("KnowledgeFS product operation cannot be both ready and a gap")
|
||||
runtime_by_product = {operation.product_operation_id: operation for operation in product_operations}
|
||||
if set(ready_entries) | set(gap_entries) != set(runtime_by_product):
|
||||
missing = sorted(set(runtime_by_product) - set(ready_entries) - set(gap_entries))
|
||||
extra = sorted((set(ready_entries) | set(gap_entries)) - set(runtime_by_product))
|
||||
raise ValueError(f"KnowledgeFS product operation completeness drifted: missing={missing}, extra={extra}")
|
||||
|
||||
python_capabilities = {operation.operation_id: operation for operation in capability_operations}
|
||||
guard_capabilities = {operation.operation_id: operation for operation in capability_policy}
|
||||
openapi_operations = _openapi_operations(document)
|
||||
for product_operation_id, runtime in runtime_by_product.items():
|
||||
entry = ready_entries.get(product_operation_id) or gap_entries[product_operation_id]
|
||||
if runtime.ready != (product_operation_id in ready_entries):
|
||||
state = "ready" if runtime.ready else "gap"
|
||||
raise ValueError(f"Dify product operation {product_operation_id} must be declared as {state}")
|
||||
_validate_product_entry(entry, runtime, openapi_operations, python_capabilities, guard_capabilities)
|
||||
if product_operation_id in gap_entries:
|
||||
_required_string(gap_entries[product_operation_id]["reason"], "product gap reason")
|
||||
_required_string(gap_entries[product_operation_id]["reasonCode"], "product gap reasonCode")
|
||||
replacements = gap_entries[product_operation_id]["replacementProductOperationIds"]
|
||||
if not replacements or any(replacement not in ready_entries for replacement in replacements):
|
||||
raise ValueError(f"KnowledgeFS product gap {product_operation_id} has invalid replacements")
|
||||
|
||||
exclusions = gap_manifest["internalKfsOperationExclusions"]
|
||||
exclusion_by_id = {entry["kfsOperationId"]: entry for entry in exclusions}
|
||||
if len(exclusion_by_id) != len(exclusions):
|
||||
raise ValueError("KnowledgeFS internal KFS operation exclusions contain duplicate ids")
|
||||
lifecycle_operation_ids = {
|
||||
"activateDifyWorkspaceIntegration",
|
||||
"freezeDifyWorkspaceIntegration",
|
||||
}
|
||||
if set(exclusion_by_id) != lifecycle_operation_ids:
|
||||
raise ValueError("KnowledgeFS Dify integration lifecycle operations must be explicit internal exclusions")
|
||||
product_capability_ids = {operation.kfs_operation_id for operation in product_operations}
|
||||
for operation_id in exclusion_by_id:
|
||||
if operation_id in product_capability_ids:
|
||||
raise ValueError(f"Internal KFS operation {operation_id} cannot also be a product operation")
|
||||
_validate_capability_alignment(
|
||||
operation_id,
|
||||
openapi_operations,
|
||||
python_capabilities,
|
||||
guard_capabilities,
|
||||
)
|
||||
python_operation = python_capabilities[operation_id]
|
||||
if python_operation.allowed_caller_kinds != ("internal_worker",) or python_operation.resource != "namespace":
|
||||
raise ValueError("Dify integration lifecycle operations must remain internal-worker namespace-only")
|
||||
|
||||
|
||||
def _parse_product_entry(raw: object, *, gap: bool) -> ProductOperationManifestEntry:
|
||||
base_fields = {
|
||||
"action",
|
||||
"kfsOperationId",
|
||||
"limits",
|
||||
"method",
|
||||
"path",
|
||||
"productOperationId",
|
||||
"resource",
|
||||
"stream",
|
||||
"transport",
|
||||
}
|
||||
expected_fields = base_fields | ({"reason", "reasonCode", "replacementProductOperationIds"} if gap else set())
|
||||
if not isinstance(raw, dict) or set(raw) != expected_fields:
|
||||
raise ValueError("KnowledgeFS product operation entry fields are invalid")
|
||||
for field in ("action", "kfsOperationId", "method", "path", "productOperationId", "resource", "transport"):
|
||||
_required_string(raw.get(field), f"product operation {field}")
|
||||
limits = raw.get("limits")
|
||||
stream = raw.get("stream")
|
||||
if not isinstance(limits, dict) or set(limits) != {
|
||||
"kfsMaxResponseBytes",
|
||||
"productMaxRequestBytes",
|
||||
"productMaxResponseBytes",
|
||||
}:
|
||||
raise ValueError("KnowledgeFS product operation limits are invalid")
|
||||
if not all(isinstance(limit, int) and not isinstance(limit, bool) and limit >= 0 for limit in limits.values()):
|
||||
raise ValueError("KnowledgeFS product operation limits must be nonnegative integers")
|
||||
if not isinstance(stream, dict) or set(stream) != {"kfsResponseKind", "productKind"}:
|
||||
raise ValueError("KnowledgeFS product operation stream contract is invalid")
|
||||
_required_string(stream.get("kfsResponseKind"), "KFS response kind")
|
||||
_required_string(stream.get("productKind"), "product stream kind")
|
||||
if gap:
|
||||
_required_string(raw.get("reason"), "product gap reason")
|
||||
_required_string(raw.get("reasonCode"), "product gap reasonCode")
|
||||
replacements = raw.get("replacementProductOperationIds")
|
||||
if not isinstance(replacements, list) or not all(isinstance(item, str) and item for item in replacements):
|
||||
raise ValueError("KnowledgeFS product operation gap replacements are invalid")
|
||||
return cast(ProductOperationManifestEntry, raw)
|
||||
|
||||
|
||||
def _validate_product_entry(
|
||||
entry: ProductOperationManifestEntry,
|
||||
runtime: ProductOperationRuntimeContract,
|
||||
openapi_operations: Mapping[str, tuple[str, str, dict[str, Any]]],
|
||||
python_capabilities: Mapping[str, CapabilityOperationRuntimeContract],
|
||||
guard_capabilities: Mapping[str, CapabilityOperationPolicy],
|
||||
) -> None:
|
||||
_validate_capability_alignment(
|
||||
runtime.kfs_operation_id,
|
||||
openapi_operations,
|
||||
python_capabilities,
|
||||
guard_capabilities,
|
||||
)
|
||||
_, _, openapi_operation = openapi_operations[runtime.kfs_operation_id]
|
||||
expected: ProductOperationManifestEntry = {
|
||||
"action": runtime.action,
|
||||
"kfsOperationId": runtime.kfs_operation_id,
|
||||
"limits": {
|
||||
"kfsMaxResponseBytes": _openapi_max_response_bytes(openapi_operation),
|
||||
"productMaxRequestBytes": runtime.max_request_bytes,
|
||||
"productMaxResponseBytes": runtime.max_response_bytes,
|
||||
},
|
||||
"method": runtime.method,
|
||||
"path": runtime.path,
|
||||
"productOperationId": runtime.product_operation_id,
|
||||
"resource": runtime.resource,
|
||||
"stream": {
|
||||
"kfsResponseKind": _openapi_response_kind(openapi_operation),
|
||||
"productKind": runtime.stream_kind,
|
||||
},
|
||||
"transport": runtime.transport,
|
||||
}
|
||||
for field in PRODUCT_OPERATION_MANIFEST_FIELDS:
|
||||
expected_value = expected[field]
|
||||
if entry[field] != expected_value:
|
||||
raise ValueError(
|
||||
f"KnowledgeFS product operation {runtime.product_operation_id} field {field} drifted: "
|
||||
f"expected {expected_value!r}, received {entry[field]!r}"
|
||||
)
|
||||
|
||||
|
||||
def _validate_capability_alignment(
|
||||
operation_id: str,
|
||||
openapi_operations: Mapping[str, tuple[str, str, dict[str, Any]]],
|
||||
python_capabilities: Mapping[str, CapabilityOperationRuntimeContract],
|
||||
guard_capabilities: Mapping[str, CapabilityOperationPolicy],
|
||||
) -> None:
|
||||
python_operation = python_capabilities.get(operation_id)
|
||||
guard_operation = guard_capabilities.get(operation_id)
|
||||
openapi_operation = openapi_operations.get(operation_id)
|
||||
if python_operation is None or guard_operation is None or openapi_operation is None:
|
||||
raise ValueError(
|
||||
f"KnowledgeFS operation {operation_id} must exist in Python issuer, TypeScript guard, and OpenAPI"
|
||||
)
|
||||
method, path, _ = openapi_operation
|
||||
expected = (
|
||||
python_operation.action,
|
||||
python_operation.method,
|
||||
python_operation.path,
|
||||
python_operation.resource,
|
||||
)
|
||||
received_guard = (guard_operation.action, guard_operation.method, guard_operation.path, guard_operation.resource)
|
||||
if received_guard != expected:
|
||||
raise ValueError(f"KnowledgeFS operation {operation_id} TypeScript guard drifted")
|
||||
if (method, path) != (python_operation.method, python_operation.path):
|
||||
raise ValueError(f"KnowledgeFS operation {operation_id} OpenAPI method/path drifted")
|
||||
if guard_operation.allowed_caller_kinds != python_operation.allowed_caller_kinds:
|
||||
raise ValueError(f"KnowledgeFS operation {operation_id} caller policy drifted")
|
||||
|
||||
|
||||
def _openapi_operations(document: dict[str, Any]) -> dict[str, tuple[str, str, dict[str, Any]]]:
|
||||
operations: dict[str, tuple[str, str, dict[str, Any]]] = {}
|
||||
duplicates: set[str] = set()
|
||||
for path, path_item in document.get("paths", {}).items():
|
||||
if not isinstance(path, str) or not isinstance(path_item, dict):
|
||||
continue
|
||||
for method in ("delete", "get", "patch", "post", "put"):
|
||||
operation = path_item.get(method)
|
||||
if not isinstance(operation, dict):
|
||||
continue
|
||||
operation_id = operation.get("operationId")
|
||||
if not isinstance(operation_id, str) or not operation_id:
|
||||
continue
|
||||
if operation_id in operations:
|
||||
duplicates.add(operation_id)
|
||||
operations[operation_id] = (method.upper(), path, operation)
|
||||
if duplicates:
|
||||
raise ValueError(f"KnowledgeFS OpenAPI contains duplicate operation ids: {sorted(duplicates)}")
|
||||
return operations
|
||||
|
||||
|
||||
def _openapi_response_kind(operation: dict[str, Any]) -> Literal["binary", "buffered", "stream"]:
|
||||
media_types = {
|
||||
media_type
|
||||
for status, response in operation.get("responses", {}).items()
|
||||
if status == "2XX" or (len(status) == 3 and status.startswith("2") and status.isdigit())
|
||||
for media_type in response.get("content", {})
|
||||
}
|
||||
if "text/event-stream" in media_types:
|
||||
return "stream"
|
||||
if "application/octet-stream" in media_types:
|
||||
return "binary"
|
||||
return "buffered"
|
||||
|
||||
|
||||
def _openapi_max_response_bytes(operation: dict[str, Any]) -> int:
|
||||
value = operation.get("x-knowledge-fs-max-response-bytes")
|
||||
if not isinstance(value, int) or isinstance(value, bool) or value <= 0:
|
||||
raise ValueError("KnowledgeFS product OpenAPI operation has no positive response byte limit")
|
||||
return value
|
||||
|
||||
|
||||
def _required_string(value: object, field: str) -> str:
|
||||
if not isinstance(value, str) or not value.strip() or value != value.strip():
|
||||
raise ValueError(f"KnowledgeFS {field} must be a non-empty trimmed string")
|
||||
return value
|
||||
|
||||
|
||||
def _unique(values: Iterable[str], label: str) -> None:
|
||||
sequence = tuple(values)
|
||||
if len(sequence) != len(set(sequence)):
|
||||
raise ValueError(f"KnowledgeFS {label} contains duplicate ids")
|
||||
|
||||
|
||||
__all__ = [
|
||||
"CapabilityOperationPolicy",
|
||||
"CapabilityOperationRuntimeContract",
|
||||
"ProductOperationGapManifest",
|
||||
"ProductOperationManifest",
|
||||
"ProductOperationRuntimeContract",
|
||||
"capability_operation_runtime_contracts",
|
||||
"parse_capability_operation_policy",
|
||||
"parse_product_operation_gap_manifest",
|
||||
"parse_product_operation_manifest",
|
||||
"product_operation_runtime_contracts",
|
||||
"validate_product_operation_contracts",
|
||||
]
|
||||
@@ -0,0 +1,757 @@
|
||||
"""Lint Flask-RESTX response docs against statically visible response serializers.
|
||||
|
||||
This checker intentionally stays conservative. It only reports a hard schema
|
||||
mismatch when both sides are statically known for the same 2xx status code:
|
||||
a documented ``@ns.response(..., Model)`` and an actual ``dump_response(Model, ...)``,
|
||||
``Model(...).model_dump()``, or ``Model.model_validate(...).model_dump()`` return.
|
||||
|
||||
Raw dictionaries, raw lists, ``None`` responses, streaming helpers, missing
|
||||
response schemas, and returns with non-literal status codes are classified as
|
||||
unknown. Unknown details are hidden by default to keep routine output focused;
|
||||
pass ``--include-unknown`` when triaging them. The one intentional non-schema
|
||||
mismatch is a known body/schema on a no-body status such as 204, 205, or 304.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import json
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from http import HTTPStatus
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
HTTP_METHODS = {"delete", "get", "head", "options", "patch", "post", "put"}
|
||||
NO_BODY_STATUSES = {HTTPStatus.NO_CONTENT.value, HTTPStatus.RESET_CONTENT.value, HTTPStatus.NOT_MODIFIED.value}
|
||||
DEFAULT_CONTROLLER_DIRS = ("controllers/console", "controllers/service_api", "controllers/web")
|
||||
IGNORE_COMMENT_MARKERS = ("response-contract:ignore",)
|
||||
|
||||
type Classification = Literal["valid", "mismatch", "unknown", "refactorable"]
|
||||
type ActualKind = Literal[
|
||||
"empty",
|
||||
"model",
|
||||
"model_dump_variable",
|
||||
"none",
|
||||
"raw_dict",
|
||||
"raw_list",
|
||||
"raw_value",
|
||||
"unknown",
|
||||
]
|
||||
type MethodNode = ast.FunctionDef | ast.AsyncFunctionDef
|
||||
type ModelValueSource = Literal["constructor", "model_validate"]
|
||||
|
||||
HTTP_STATUS_NAMES = {status.name: status.value for status in HTTPStatus}
|
||||
HTTP_STATUS_NAMES.update({f"HTTP_{status.value}_{status.name}": status.value for status in HTTPStatus})
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DocumentedResponse:
|
||||
status: int
|
||||
model: str | None
|
||||
line: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ActualResponse:
|
||||
status: int | None
|
||||
kind: ActualKind
|
||||
model: str | None
|
||||
line: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ContractCheck:
|
||||
classification: Classification
|
||||
file: str
|
||||
class_name: str
|
||||
method: str
|
||||
route: str
|
||||
line: int
|
||||
reason: str
|
||||
documented: dict[int, str | None]
|
||||
actual: list[ActualResponse]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ContractCheckContext:
|
||||
"""Stable route-method context shared by every classification result."""
|
||||
|
||||
file: str
|
||||
class_name: str
|
||||
method: str
|
||||
route: str
|
||||
line: int
|
||||
documented: dict[int, str | None]
|
||||
|
||||
def build(
|
||||
self, classification: Classification, reason: str, actual_responses: Sequence[ActualResponse]
|
||||
) -> ContractCheck:
|
||||
return ContractCheck(
|
||||
classification=classification,
|
||||
file=self.file,
|
||||
class_name=self.class_name,
|
||||
method=self.method,
|
||||
route=self.route,
|
||||
line=self.line,
|
||||
reason=reason,
|
||||
documented=self.documented,
|
||||
actual=list(actual_responses),
|
||||
)
|
||||
|
||||
def mismatch(self, reason: str, documented: DocumentedResponse, actual: ActualResponse) -> ContractCheck:
|
||||
return self.build("mismatch", f"{reason} (doc line {documented.line}, return line {actual.line})", [actual])
|
||||
|
||||
|
||||
@dataclass
|
||||
class VariableAssignmentSummary:
|
||||
"""Track whether a local name is safe to treat as one specific response model."""
|
||||
|
||||
known_models: set[str] = field(default_factory=set)
|
||||
known_sources: set[ModelValueSource] = field(default_factory=set)
|
||||
has_unknown_assignment: bool = False
|
||||
|
||||
def add_known(self, model: str, source: ModelValueSource) -> None:
|
||||
self.known_models.add(model)
|
||||
self.known_sources.add(source)
|
||||
|
||||
def add_unknown(self) -> None:
|
||||
self.has_unknown_assignment = True
|
||||
|
||||
def single_known_model(self) -> tuple[str, ModelValueSource] | None:
|
||||
if self.has_unknown_assignment or len(self.known_models) != 1:
|
||||
return None
|
||||
model = next(iter(self.known_models))
|
||||
source: ModelValueSource = "constructor" if self.known_sources == {"constructor"} else "model_validate"
|
||||
return model, source
|
||||
|
||||
|
||||
def dotted_name(node: ast.AST) -> str | None:
|
||||
match node:
|
||||
case ast.Name():
|
||||
return node.id
|
||||
case ast.Attribute():
|
||||
parent = dotted_name(node.value)
|
||||
if parent:
|
||||
return f"{parent}.{node.attr}"
|
||||
return node.attr
|
||||
return None
|
||||
|
||||
|
||||
def leaf_name(node: ast.AST) -> str | None:
|
||||
name = dotted_name(node)
|
||||
if name is None:
|
||||
return None
|
||||
return name.rsplit(".", 1)[-1]
|
||||
|
||||
|
||||
def int_constant(node: ast.AST | None) -> int | None:
|
||||
if isinstance(node, ast.Constant) and isinstance(node.value, int):
|
||||
return node.value
|
||||
if isinstance(node, ast.Name):
|
||||
return HTTP_STATUS_NAMES.get(node.id)
|
||||
if isinstance(node, ast.Attribute):
|
||||
return HTTP_STATUS_NAMES.get(node.attr)
|
||||
return None
|
||||
|
||||
|
||||
def string_constant(node: ast.AST | None) -> str | None:
|
||||
if isinstance(node, ast.Constant) and isinstance(node.value, str):
|
||||
return node.value
|
||||
return None
|
||||
|
||||
|
||||
def keyword_value(call: ast.Call, *names: str) -> ast.AST | None:
|
||||
for keyword in call.keywords:
|
||||
if keyword.arg in names:
|
||||
return keyword.value
|
||||
return None
|
||||
|
||||
|
||||
def is_probable_model_name(name: str) -> bool:
|
||||
return bool(name) and name[0].isupper()
|
||||
|
||||
|
||||
def model_name_from_schema_expr(node: ast.AST | None) -> str | None:
|
||||
if node is None:
|
||||
return None
|
||||
|
||||
if isinstance(node, ast.Subscript):
|
||||
value_name = dotted_name(node.value)
|
||||
if value_name and value_name.endswith(".models"):
|
||||
# register_response_schema_models stores schemas by model name; both
|
||||
# ns.models[Model.__name__] and ns.models["Model"] appear in controllers.
|
||||
key = node.slice
|
||||
if isinstance(key, ast.Attribute) and key.attr == "__name__":
|
||||
return leaf_name(key.value)
|
||||
return string_constant(key)
|
||||
|
||||
if isinstance(node, ast.Call):
|
||||
func_name = dotted_name(node.func)
|
||||
if func_name and func_name.endswith(".model"):
|
||||
return string_constant(node.args[0] if node.args else keyword_value(node, "name"))
|
||||
|
||||
if isinstance(node, ast.Name):
|
||||
return node.id
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def documented_response_from_decorator(decorator: ast.expr) -> DocumentedResponse | None:
|
||||
if not isinstance(decorator, ast.Call):
|
||||
return None
|
||||
|
||||
func_name = dotted_name(decorator.func)
|
||||
if not func_name or not func_name.endswith(".response"):
|
||||
return None
|
||||
|
||||
status_expr = decorator.args[0] if decorator.args else keyword_value(decorator, "code", "status")
|
||||
status = int_constant(status_expr)
|
||||
if status is None:
|
||||
return None
|
||||
|
||||
schema_expr: ast.AST | None = decorator.args[2] if len(decorator.args) >= 3 else None
|
||||
schema_expr = keyword_value(decorator, "model", "schema") or schema_expr
|
||||
|
||||
return DocumentedResponse(
|
||||
status=status,
|
||||
model=model_name_from_schema_expr(schema_expr),
|
||||
line=decorator.lineno,
|
||||
)
|
||||
|
||||
|
||||
def route_from_decorator(decorator: ast.expr) -> str | None:
|
||||
if not isinstance(decorator, ast.Call):
|
||||
return None
|
||||
|
||||
func_name = dotted_name(decorator.func)
|
||||
if not func_name or not func_name.endswith(".route"):
|
||||
return None
|
||||
|
||||
return string_constant(decorator.args[0] if decorator.args else keyword_value(decorator, "route", "path"))
|
||||
|
||||
|
||||
def routes_from_decorators(decorators: Iterable[ast.expr]) -> list[str]:
|
||||
return [route for decorator in decorators if (route := route_from_decorator(decorator))]
|
||||
|
||||
|
||||
def response_docs_from_decorators(decorators: Iterable[ast.expr]) -> dict[int, DocumentedResponse]:
|
||||
docs: dict[int, DocumentedResponse] = {}
|
||||
for decorator in decorators:
|
||||
response = documented_response_from_decorator(decorator)
|
||||
if response and 200 <= response.status < 300:
|
||||
docs[response.status] = response
|
||||
return docs
|
||||
|
||||
|
||||
def model_name_from_model_validate_call(node: ast.AST) -> str | None:
|
||||
if not isinstance(node, ast.Call):
|
||||
return None
|
||||
if isinstance(node.func, ast.Attribute) and node.func.attr == "model_validate":
|
||||
return leaf_name(node.func.value)
|
||||
return None
|
||||
|
||||
|
||||
def model_value_from_model_validate_call(node: ast.AST) -> tuple[str, ModelValueSource] | None:
|
||||
if model_name := model_name_from_model_validate_call(node):
|
||||
return model_name, "model_validate"
|
||||
return None
|
||||
|
||||
|
||||
def model_name_from_constructor_call(node: ast.AST) -> str | None:
|
||||
if not isinstance(node, ast.Call):
|
||||
return None
|
||||
if isinstance(node.func, ast.Name) and is_probable_model_name(node.func.id):
|
||||
return node.func.id
|
||||
return None
|
||||
|
||||
|
||||
def model_value_from_constructor_call(node: ast.AST) -> tuple[str, ModelValueSource] | None:
|
||||
if model_name := model_name_from_constructor_call(node):
|
||||
return model_name, "constructor"
|
||||
return None
|
||||
|
||||
|
||||
def model_name_from_model_dump(node: ast.AST) -> str | None:
|
||||
if not isinstance(node, ast.Call) or not isinstance(node.func, ast.Attribute) or node.func.attr != "model_dump":
|
||||
return None
|
||||
|
||||
dumped_value = node.func.value
|
||||
if isinstance(dumped_value, ast.Call):
|
||||
return model_name_from_model_validate_call(dumped_value) or model_name_from_constructor_call(dumped_value)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def model_name_from_model_value(node: ast.AST) -> str | None:
|
||||
return model_name_from_model_validate_call(node) or model_name_from_constructor_call(node)
|
||||
|
||||
|
||||
def model_value_from_model_value(node: ast.AST) -> tuple[str, ModelValueSource] | None:
|
||||
return model_value_from_model_validate_call(node) or model_value_from_constructor_call(node)
|
||||
|
||||
|
||||
def model_name_from_dump_response(node: ast.AST) -> str | None:
|
||||
if not isinstance(node, ast.Call):
|
||||
return None
|
||||
|
||||
func_name = dotted_name(node.func)
|
||||
if func_name != "dump_response" and not (func_name and func_name.endswith(".dump_response")):
|
||||
return None
|
||||
|
||||
model_expr = node.args[0] if node.args else keyword_value(node, "model", "schema", "response_model")
|
||||
if isinstance(model_expr, ast.Name):
|
||||
return model_expr.id
|
||||
return None
|
||||
|
||||
|
||||
def actual_kind_from_expr(
|
||||
expr: ast.AST | None, variable_models: dict[str, tuple[str, ModelValueSource]] | None = None
|
||||
) -> tuple[ActualKind, str | None]:
|
||||
if expr is None:
|
||||
return "none", None
|
||||
|
||||
dump_response_model = model_name_from_dump_response(expr)
|
||||
if dump_response_model:
|
||||
return "model", dump_response_model
|
||||
|
||||
if isinstance(expr, ast.Call) and isinstance(expr.func, ast.Attribute) and expr.func.attr == "model_dump":
|
||||
dumped_value = expr.func.value
|
||||
if isinstance(dumped_value, ast.Name) and variable_models:
|
||||
model_assignment = variable_models.get(dumped_value.id)
|
||||
if model_assignment:
|
||||
model_name, source = model_assignment
|
||||
if source == "constructor":
|
||||
return "model", model_name
|
||||
# A variable dump from model_validate can match today, but it
|
||||
# bypasses dump_response and is easier to drift; keep it visible
|
||||
# as refactorable.
|
||||
return "model_dump_variable", model_name
|
||||
|
||||
model_dump_model = model_name_from_model_dump(expr)
|
||||
if model_dump_model:
|
||||
return "model", model_dump_model
|
||||
|
||||
if isinstance(expr, ast.Constant):
|
||||
if expr.value is None:
|
||||
return "none", None
|
||||
if expr.value == "":
|
||||
return "empty", None
|
||||
return "raw_value", None
|
||||
|
||||
if isinstance(expr, ast.Dict):
|
||||
return "raw_dict", None
|
||||
|
||||
if isinstance(expr, ast.List):
|
||||
return "raw_list", None
|
||||
|
||||
return "unknown", None
|
||||
|
||||
|
||||
def actual_response_from_return(
|
||||
return_node: ast.Return, variable_models: dict[str, tuple[str, ModelValueSource]]
|
||||
) -> ActualResponse:
|
||||
status: int | None = 200
|
||||
body_expr = return_node.value
|
||||
|
||||
if isinstance(return_node.value, ast.Tuple) and return_node.value.elts:
|
||||
body_expr = return_node.value.elts[0]
|
||||
if len(return_node.value.elts) >= 2:
|
||||
# Dynamic statuses are not safe to coerce to 200; classify them as unknown.
|
||||
status = int_constant(return_node.value.elts[1])
|
||||
|
||||
kind, model = actual_kind_from_expr(body_expr, variable_models)
|
||||
return ActualResponse(status=status, kind=kind, model=model, line=return_node.lineno)
|
||||
|
||||
|
||||
def iter_method_nodes(method: MethodNode) -> Iterable[ast.AST]:
|
||||
"""Yield method body nodes while ignoring nested function/class scopes."""
|
||||
|
||||
stack: list[ast.AST] = list(reversed(method.body))
|
||||
while stack:
|
||||
node = stack.pop()
|
||||
yield node
|
||||
|
||||
if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.Lambda | ast.ClassDef):
|
||||
continue
|
||||
|
||||
stack.extend(reversed(list(ast.iter_child_nodes(node))))
|
||||
|
||||
|
||||
def target_names(target: ast.AST) -> Iterable[str]:
|
||||
match target:
|
||||
case ast.Name():
|
||||
yield target.id
|
||||
case ast.Tuple() | ast.List():
|
||||
for item in target.elts:
|
||||
yield from target_names(item)
|
||||
|
||||
|
||||
def record_assignment(
|
||||
assignments: defaultdict[str, VariableAssignmentSummary],
|
||||
targets: Iterable[str],
|
||||
model_assignment: tuple[str, ModelValueSource] | None,
|
||||
) -> None:
|
||||
for target in targets:
|
||||
if model_assignment is None:
|
||||
# Once a name receives an unknown value, later model_dump() calls on it
|
||||
# are no longer a reliable signal for the returned schema.
|
||||
assignments[target].add_unknown()
|
||||
else:
|
||||
model_name, source = model_assignment
|
||||
assignments[target].add_known(model_name, source)
|
||||
|
||||
|
||||
def variable_model_assignments_for_method(method: MethodNode) -> dict[str, tuple[str, ModelValueSource]]:
|
||||
"""Infer local variables that are unambiguously assigned one response model."""
|
||||
|
||||
assignments: defaultdict[str, VariableAssignmentSummary] = defaultdict(VariableAssignmentSummary)
|
||||
|
||||
for node in iter_method_nodes(method):
|
||||
match node:
|
||||
case ast.Assign(targets=targets, value=value):
|
||||
record_assignment(
|
||||
assignments,
|
||||
(name for target in targets for name in target_names(target)),
|
||||
model_value_from_model_value(value),
|
||||
)
|
||||
case ast.AnnAssign(target=target, value=value) if value is not None:
|
||||
record_assignment(assignments, target_names(target), model_value_from_model_value(value))
|
||||
case ast.AugAssign(target=target) | ast.For(target=target) | ast.AsyncFor(target=target):
|
||||
# Mutation and loop targets overwrite prior values with runtime-dependent data.
|
||||
record_assignment(assignments, target_names(target), None)
|
||||
case ast.With(items=items) | ast.AsyncWith(items=items):
|
||||
for item in items:
|
||||
if item.optional_vars is not None:
|
||||
record_assignment(assignments, target_names(item.optional_vars), None)
|
||||
case ast.ExceptHandler(name=name) if name:
|
||||
assignments[name].add_unknown()
|
||||
case ast.NamedExpr(target=target, value=value):
|
||||
record_assignment(assignments, target_names(target), model_value_from_model_value(value))
|
||||
|
||||
return {
|
||||
name: assignment
|
||||
for name, summary in assignments.items()
|
||||
if (assignment := summary.single_known_model()) is not None
|
||||
}
|
||||
|
||||
|
||||
def actual_responses_for_method(method: MethodNode) -> list[ActualResponse]:
|
||||
"""Extract statically visible 2xx returns from one controller method.
|
||||
|
||||
The analysis is deliberately shallow and conservative:
|
||||
|
||||
1. Walk only the method's own body, skipping nested functions/classes.
|
||||
2. Infer local variables that are assigned exactly one recognizable response
|
||||
model, so ``response.model_dump()`` can still be connected to its schema.
|
||||
3. Treat any later unknown assignment, mutation target, loop target, context
|
||||
manager binding, or exception binding as invalidating that variable.
|
||||
4. For each top-level return path, split Flask-style ``(body, status)``
|
||||
tuples, classify the body expression, and keep non-literal statuses as
|
||||
``None`` so the classifier reports them as unknown instead of assuming 200.
|
||||
5. Drop non-2xx literal statuses, since response contracts here only compare
|
||||
successful response schemas.
|
||||
"""
|
||||
|
||||
variable_models = variable_model_assignments_for_method(method)
|
||||
responses: list[ActualResponse] = []
|
||||
for node in iter_method_nodes(method):
|
||||
if isinstance(node, ast.Return):
|
||||
responses.append(actual_response_from_return(node, variable_models))
|
||||
return [response for response in responses if response.status is None or 200 <= response.status < 300]
|
||||
|
||||
|
||||
def display_path(file_path: Path, repo_root: Path) -> str:
|
||||
try:
|
||||
return str(file_path.relative_to(repo_root))
|
||||
except ValueError:
|
||||
return str(file_path)
|
||||
|
||||
|
||||
def classify_method(
|
||||
*,
|
||||
actual_responses: Sequence[ActualResponse],
|
||||
class_name: str,
|
||||
documented_responses: dict[int, DocumentedResponse],
|
||||
file_path: Path,
|
||||
method: MethodNode,
|
||||
repo_root: Path,
|
||||
route: str,
|
||||
) -> ContractCheck:
|
||||
documented_summary = {status: response.model for status, response in sorted(documented_responses.items())}
|
||||
context = ContractCheckContext(
|
||||
file=display_path(file_path, repo_root),
|
||||
class_name=class_name,
|
||||
method=method.name,
|
||||
route=route,
|
||||
line=method.lineno,
|
||||
documented=documented_summary,
|
||||
)
|
||||
|
||||
if not actual_responses:
|
||||
return context.build("unknown", "no statically visible 2xx return", [])
|
||||
|
||||
unknown_reasons: list[str] = []
|
||||
refactorable_reasons: list[str] = []
|
||||
|
||||
for actual in actual_responses:
|
||||
if actual.status is None:
|
||||
unknown_reasons.append(f"return line {actual.line} has non-literal or unsupported status")
|
||||
continue
|
||||
|
||||
documented = documented_responses.get(actual.status)
|
||||
|
||||
if actual.status in NO_BODY_STATUSES:
|
||||
# No-body statuses are contract violations even when the schema names
|
||||
# would otherwise match, because clients should not expect a payload.
|
||||
if documented is not None and documented.model is not None:
|
||||
return context.mismatch(
|
||||
f"status {actual.status} is a no-body response but documents {documented.model}",
|
||||
documented,
|
||||
actual,
|
||||
)
|
||||
if actual.kind in {"model", "model_dump_variable", "raw_dict", "raw_list", "raw_value"}:
|
||||
no_body_doc = DocumentedResponse(status=actual.status, model=None, line=method.lineno)
|
||||
return context.mismatch(
|
||||
f"status {actual.status} is a no-body response but returns {actual.kind}",
|
||||
no_body_doc,
|
||||
actual,
|
||||
)
|
||||
if actual.kind == "unknown":
|
||||
unknown_reasons.append(f"status {actual.status} returns unknown body expression")
|
||||
continue
|
||||
|
||||
if documented is None:
|
||||
unknown_reasons.append(f"status {actual.status} has no @response doc")
|
||||
continue
|
||||
|
||||
if documented.model is None:
|
||||
unknown_reasons.append(f"status {actual.status} response doc has no schema model")
|
||||
continue
|
||||
|
||||
if actual.kind == "model_dump_variable" and actual.model is not None:
|
||||
if documented.model != actual.model:
|
||||
return context.mismatch(
|
||||
f"status {actual.status} documents {documented.model} but returns {actual.model}",
|
||||
documented,
|
||||
actual,
|
||||
)
|
||||
# The schema matches, but this path still deserves cleanup because
|
||||
# dump_response is the contract-aware serialization helper.
|
||||
refactorable_reasons.append(
|
||||
f"status {actual.status} returns {actual.model}.model_dump() through a variable; prefer dump_response"
|
||||
)
|
||||
continue
|
||||
|
||||
if actual.kind != "model" or actual.model is None:
|
||||
unknown_reasons.append(f"status {actual.status} returns {actual.kind}")
|
||||
continue
|
||||
|
||||
if documented.model != actual.model:
|
||||
return context.mismatch(
|
||||
f"status {actual.status} documents {documented.model} but returns {actual.model}",
|
||||
documented,
|
||||
actual,
|
||||
)
|
||||
|
||||
if unknown_reasons:
|
||||
# Unknown beats refactorable: if any return path is ambiguous, do not
|
||||
# imply the endpoint is merely a cleanup candidate.
|
||||
return context.build("unknown", "; ".join(sorted(set(unknown_reasons))), actual_responses)
|
||||
|
||||
if refactorable_reasons:
|
||||
return context.build("refactorable", "; ".join(sorted(set(refactorable_reasons))), actual_responses)
|
||||
|
||||
return context.build(
|
||||
"valid",
|
||||
"documented response schema matches statically visible return schema",
|
||||
actual_responses,
|
||||
)
|
||||
|
||||
|
||||
def iter_controller_files(paths: Iterable[Path]) -> Iterable[Path]:
|
||||
for path in paths:
|
||||
if path.is_file() and path.suffix == ".py":
|
||||
yield path
|
||||
elif path.is_dir():
|
||||
yield from sorted(child for child in path.rglob("*.py") if child.is_file())
|
||||
|
||||
|
||||
def node_start_lineno(node: ast.ClassDef | MethodNode) -> int:
|
||||
decorator_lines = [decorator.lineno for decorator in node.decorator_list]
|
||||
if decorator_lines:
|
||||
return min(decorator_lines)
|
||||
return node.lineno
|
||||
|
||||
|
||||
def line_has_ignore_marker(line: str) -> bool:
|
||||
_, marker, comment = line.partition("#")
|
||||
if not marker:
|
||||
return False
|
||||
normalized = comment.lower()
|
||||
return any(ignore_marker in normalized for ignore_marker in IGNORE_COMMENT_MARKERS)
|
||||
|
||||
|
||||
def node_ignore_scan_end_lineno(node: ast.ClassDef | MethodNode) -> int:
|
||||
if isinstance(node, ast.ClassDef):
|
||||
return node.lineno
|
||||
return node.end_lineno or node.lineno
|
||||
|
||||
|
||||
def node_has_ignore_comment(lines: Sequence[str], node: ast.ClassDef | MethodNode) -> bool:
|
||||
start = node_start_lineno(node)
|
||||
end = node_ignore_scan_end_lineno(node)
|
||||
if any(line_has_ignore_marker(line) for line in lines[start - 1 : end]):
|
||||
return True
|
||||
|
||||
line_index = start - 2
|
||||
while line_index >= 0:
|
||||
stripped = lines[line_index].strip()
|
||||
if not stripped:
|
||||
line_index -= 1
|
||||
continue
|
||||
if not stripped.startswith("#"):
|
||||
break
|
||||
if line_has_ignore_marker(lines[line_index]):
|
||||
return True
|
||||
line_index -= 1
|
||||
return False
|
||||
|
||||
|
||||
def checks_for_file(file_path: Path, repo_root: Path) -> list[ContractCheck]:
|
||||
source = file_path.read_text(encoding="utf-8")
|
||||
lines = source.splitlines()
|
||||
module = ast.parse(source, filename=str(file_path))
|
||||
checks: list[ContractCheck] = []
|
||||
|
||||
for node in module.body:
|
||||
if not isinstance(node, ast.ClassDef):
|
||||
continue
|
||||
if node_has_ignore_comment(lines, node):
|
||||
continue
|
||||
|
||||
class_routes = routes_from_decorators(node.decorator_list)
|
||||
class_documented = response_docs_from_decorators(node.decorator_list)
|
||||
|
||||
for item in node.body:
|
||||
if not isinstance(item, ast.FunctionDef | ast.AsyncFunctionDef) or item.name not in HTTP_METHODS:
|
||||
continue
|
||||
if node_has_ignore_comment(lines, item):
|
||||
continue
|
||||
|
||||
routes = routes_from_decorators(item.decorator_list) or class_routes
|
||||
if not routes:
|
||||
continue
|
||||
|
||||
documented = {**class_documented, **response_docs_from_decorators(item.decorator_list)}
|
||||
# Method-level @response decorators override class-level defaults for
|
||||
# the same status code, matching Flask-RESTX's common controller style.
|
||||
actual = actual_responses_for_method(item)
|
||||
for route in routes:
|
||||
checks.append(
|
||||
classify_method(
|
||||
actual_responses=actual,
|
||||
class_name=node.name,
|
||||
documented_responses=documented,
|
||||
file_path=file_path,
|
||||
method=item,
|
||||
repo_root=repo_root,
|
||||
route=route,
|
||||
)
|
||||
)
|
||||
|
||||
return checks
|
||||
|
||||
|
||||
def as_jsonable(check: ContractCheck) -> dict[str, Any]:
|
||||
data = asdict(check)
|
||||
# pyrefly: ignore [bad-assignment]
|
||||
data["documented"] = {str(status): model for status, model in check.documented.items()}
|
||||
return data
|
||||
|
||||
|
||||
def print_text_report(checks: Sequence[ContractCheck], *, include_unknown: bool, include_valid: bool) -> None:
|
||||
counts = Counter(check.classification for check in checks)
|
||||
sys.stdout.write(
|
||||
"Response contract lint: "
|
||||
f"{counts['valid']} valid, "
|
||||
f"{counts['mismatch']} mismatch, "
|
||||
f"{counts['refactorable']} refactorable, "
|
||||
f"{counts['unknown']} unknown\n"
|
||||
)
|
||||
|
||||
for classification in ("mismatch", "refactorable", "unknown", "valid"):
|
||||
filtered = [check for check in checks if check.classification == classification]
|
||||
if classification == "unknown" and not include_unknown:
|
||||
continue
|
||||
if classification == "valid" and not include_valid:
|
||||
continue
|
||||
if not filtered:
|
||||
continue
|
||||
|
||||
sys.stdout.write(f"\n{classification.upper()}:\n")
|
||||
for check in filtered:
|
||||
location = f"{check.file}:{check.line} {check.class_name}.{check.method.upper()} {check.route}"
|
||||
sys.stdout.write(f"- {location}: {check.reason}\n")
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"paths",
|
||||
nargs="*",
|
||||
help="Files or directories to lint. Defaults to Flask controller directories.",
|
||||
)
|
||||
parser.add_argument("--include-unknown", action="store_true", help="Print unknown route methods in output.")
|
||||
parser.add_argument("--include-valid", action="store_true", help="Print valid route methods in text output.")
|
||||
parser.add_argument("--json", action="store_true", help="Emit machine-readable JSON.")
|
||||
parser.add_argument(
|
||||
"--fail-on-mismatch",
|
||||
action="store_true",
|
||||
help="Treat mismatched response contracts as failures. By default this linter is report-only.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--fail-on-unknown",
|
||||
action="store_true",
|
||||
help="Treat unknown route methods as failures. By default this linter is report-only.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
api_root = Path(__file__).resolve().parents[1]
|
||||
repo_root = api_root.parent
|
||||
raw_paths = args.paths or list(DEFAULT_CONTROLLER_DIRS)
|
||||
paths = [path if path.is_absolute() else api_root / path for path in map(Path, raw_paths)]
|
||||
|
||||
checks: list[ContractCheck] = []
|
||||
for file_path in iter_controller_files(paths):
|
||||
checks.extend(checks_for_file(file_path.resolve(), repo_root))
|
||||
|
||||
checks.sort(key=lambda check: (check.classification, check.file, check.line, check.method))
|
||||
|
||||
if args.json:
|
||||
grouped = defaultdict(list)
|
||||
for check in checks:
|
||||
if check.classification == "unknown" and not args.include_unknown:
|
||||
continue
|
||||
grouped[check.classification].append(as_jsonable(check))
|
||||
sys.stdout.write(f"{json.dumps(grouped, indent=2, sort_keys=True)}\n")
|
||||
else:
|
||||
print_text_report(
|
||||
checks,
|
||||
include_unknown=bool(args.include_unknown),
|
||||
include_valid=bool(args.include_valid),
|
||||
)
|
||||
|
||||
has_mismatch = any(check.classification == "mismatch" for check in checks)
|
||||
has_unknown = any(check.classification == "unknown" for check in checks)
|
||||
return int((bool(args.fail_on_mismatch) and has_mismatch) or (bool(args.fail_on_unknown) and has_unknown))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,57 @@
|
||||
# API unit-test sharding
|
||||
|
||||
The workflow plans shards before pytest collection. Files with recorded durations
|
||||
are sorted longest first and assigned to the least-loaded shard. Unseen files are
|
||||
assigned round robin. File discovery always comes from the current checkout, so
|
||||
stale history cannot omit new tests or reintroduce deleted files.
|
||||
|
||||
Like PyTorch's `tools/testing/test_selections.py`, oversized files become logical
|
||||
parts. Our default is 60 summed testcase seconds per part, capped at the number of
|
||||
CI shards. This is worker work, including setup/call/teardown, not wall time; the
|
||||
threshold corresponds roughly to 30 seconds of work on two workers. Each case's
|
||||
SHA-256 hash assigns it to one part, including new parametrized cases. Only files
|
||||
assigned to a runner are collected; shared oversized files are collected on each
|
||||
assigned runner. All workers use the same frozen plan. Ordinary xdist scheduling
|
||||
continues inside each runner.
|
||||
|
||||
Successful coordinator reports are uploaded as two small timing artifacts. The
|
||||
public [dify-test-infra](https://github.com/langgenius/dify-test-infra) repository
|
||||
collects complete observations from successful CI on merged Dify PRs daily at
|
||||
03:05 UTC. It averages up to five recent samples and publishes `stats/test-times.json`
|
||||
on its `generated-stats` branch. Metadata records the source runs. No timing data
|
||||
or generated history is committed to Dify itself.
|
||||
|
||||
Every PR, including fork PRs, downloads that same public baseline without a token.
|
||||
Unmerged PR observations do not update the shared baseline automatically.
|
||||
Keeping the producer in the statistics repository avoids needing a cross-repository write token in Dify or additional
|
||||
main-branch test runs.
|
||||
|
||||
A single planning job downloads the baseline once and publishes both the input
|
||||
and assignment plan as run artifacts. All shards therefore use the same snapshot,
|
||||
even if statistics are updated during a run. Downloads have bounded timeouts and
|
||||
retries; missing or invalid JSON falls back to round-robin allocation. Successful
|
||||
unit runs continue publishing observations for the external updater; failed or
|
||||
incomplete runs are not accepted as baseline samples.
|
||||
|
||||
This trades an extra planning job for consistent assignments. Timing noise,
|
||||
shared fixtures, collection, and an indivisible slow case can still produce
|
||||
imbalance. Logical splitting can duplicate expensive module/session setup. Compare both
|
||||
shard test steps and the end-to-end API completion time when evaluating a change.
|
||||
|
||||
Local examples (from repository root):
|
||||
|
||||
```sh
|
||||
uv run --project api python api/dev/pytest_sharding.py \
|
||||
--shard-index 1 --shard-total 2 --durations /tmp/durations.json \
|
||||
--write-plan /tmp/plan.json --ignore api/tests/unit_tests/controllers \
|
||||
api/tests/unit_tests api/providers/vdb/*/tests/unit_tests api/providers/trace/*/tests/unit_tests
|
||||
uv run --project api python api/dev/pytest_sharding.py \
|
||||
--shard-index 1 --shard-total 2 --plan /tmp/plan.json api > /tmp/files.txt
|
||||
uv run --project api pytest --file-shard-plan /tmp/plan.json \
|
||||
--shard-index 1 --shard-total 2 --write-test-durations /tmp/durations-1.json \
|
||||
@/tmp/files.txt
|
||||
```
|
||||
|
||||
Repeat the last two commands for shard 2. Merge their file durations by addition
|
||||
before using them as the next history snapshot. Keep generated files outside the
|
||||
checkout locally.
|
||||
@@ -0,0 +1,117 @@
|
||||
"""Select pytest files before collection so workers do not import other shards."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
from collections.abc import Sequence
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def load_durations(path: Path) -> dict[str, float]:
|
||||
"""History is optional and never controls which files are discovered."""
|
||||
try:
|
||||
data = json.loads(path.read_text())
|
||||
except (OSError, ValueError):
|
||||
return {}
|
||||
if not isinstance(data, dict):
|
||||
return {}
|
||||
return {
|
||||
name: float(value)
|
||||
for name, value in data.items()
|
||||
if isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) and value >= 0
|
||||
}
|
||||
|
||||
|
||||
def build_plan(
|
||||
files: Sequence[Path], durations: dict[str, float], total: int, threshold: float
|
||||
) -> dict[str, list[int]]:
|
||||
"""Longest estimated work first; split oversized files into logical parts.
|
||||
|
||||
Each list maps a file's hash partitions to one-based CI shard numbers.
|
||||
All CI shards must consume the same plan. Unknown files use round robin.
|
||||
Estimates are summed worker seconds, not predicted job wall time.
|
||||
"""
|
||||
if total < 1 or not math.isfinite(threshold) or threshold <= 0:
|
||||
raise ValueError("shard-total and split threshold must be positive")
|
||||
plan: dict[str, list[int]] = {}
|
||||
tasks: list[tuple[float, str, int]] = []
|
||||
unknown: list[str] = []
|
||||
for path in sorted(set(files)):
|
||||
name = path.as_posix()
|
||||
duration = durations.get(name)
|
||||
if duration is None:
|
||||
unknown.append(name)
|
||||
continue
|
||||
parts = min(total, max(1, math.ceil(duration / threshold)))
|
||||
plan[name] = [0] * parts
|
||||
tasks.extend((duration / parts, name, part) for part in range(parts))
|
||||
loads = [0.0] * total
|
||||
for duration, name, part in sorted(tasks, key=lambda task: (-task[0], task[1], task[2])):
|
||||
target = min(range(total), key=lambda index: (loads[index], index))
|
||||
plan[name][part] = target + 1
|
||||
loads[target] += duration
|
||||
for index, name in enumerate(unknown):
|
||||
plan[name] = [index % total + 1]
|
||||
return plan
|
||||
|
||||
|
||||
def case_shard(nodeid: str, targets: Sequence[int]) -> int:
|
||||
"""Stable across processes, including newly added parametrized cases."""
|
||||
digest = hashlib.sha256(nodeid.encode()).digest()
|
||||
return targets[int.from_bytes(digest[:8], "big") % len(targets)]
|
||||
|
||||
|
||||
def select_test_files(
|
||||
roots: Sequence[Path], *, shard_index: int, shard_total: int, ignored: Sequence[Path] = ()
|
||||
) -> list[Path]:
|
||||
"""Return a deterministic file shard using pytest's default filename patterns."""
|
||||
if shard_total < 1 or not 1 <= shard_index <= shard_total:
|
||||
raise ValueError("shard-index must be between 1 and shard-total")
|
||||
files = sorted(
|
||||
{
|
||||
path
|
||||
for root in roots
|
||||
for path in root.rglob("*.py")
|
||||
if (path.name.startswith("test_") or path.name.endswith("_test.py"))
|
||||
and not any(path.is_relative_to(directory) for directory in ignored)
|
||||
}
|
||||
)
|
||||
return files[shard_index - 1 :: shard_total]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--shard-index", type=int, required=True)
|
||||
parser.add_argument("--shard-total", type=int, required=True)
|
||||
parser.add_argument("--ignore", type=Path, action="append", default=[])
|
||||
parser.add_argument("--durations", type=Path)
|
||||
parser.add_argument("--write-plan", type=Path)
|
||||
parser.add_argument("--plan", type=Path)
|
||||
parser.add_argument("--split-threshold", type=float, default=60, help="Summed testcase seconds per logical part.")
|
||||
parser.add_argument("roots", type=Path, nargs="+")
|
||||
args = parser.parse_args()
|
||||
if args.write_plan:
|
||||
files = select_test_files(args.roots, shard_index=1, shard_total=1, ignored=args.ignore)
|
||||
durations = load_durations(args.durations) if args.durations else {}
|
||||
plan = build_plan(files, durations, args.shard_total, args.split_threshold)
|
||||
args.write_plan.write_text(json.dumps(plan, sort_keys=True) + "\n")
|
||||
sys.stderr.write(
|
||||
f"Planned {len(files)} files; {sum(name in durations for name in plan)} have historical durations.\n"
|
||||
)
|
||||
return
|
||||
if args.plan:
|
||||
plan = json.loads(args.plan.read_text())
|
||||
files = [Path(name) for name, targets in sorted(plan.items()) if args.shard_index in targets]
|
||||
else:
|
||||
files = select_test_files(
|
||||
args.roots, shard_index=args.shard_index, shard_total=args.shard_total, ignored=args.ignore
|
||||
)
|
||||
if not files:
|
||||
parser.error("No test files selected; refusing to fall back to pytest's default discovery")
|
||||
sys.stdout.write("\n".join(path.as_posix() for path in files) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user