chore: 纳入本地 Dify 1.19.0 源码及品牌与免登改动
演示目录只保留可重建的源码。运行密钥、数据卷、依赖和带口令的本地配置不进入仓库。 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
60158935f4
commit
6519682885
15831 files changed
+3063428
No files matched your search
@@ -0,0 +1,376 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Shared helpers for baseline-aware ast-grep CI guards."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import textwrap
|
||||
from collections import Counter
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
SCAN_ROOT = Path.cwd()
|
||||
RULES_ROOT = REPO_ROOT / "scripts" / "ast_grep_rules"
|
||||
HUNK_PATTERN = re.compile(
|
||||
r"^@@ -(?P<old_start>\d+)(?:,(?P<old_count>\d+))? \+(?P<new_start>\d+)(?:,(?P<new_count>\d+))? @@"
|
||||
)
|
||||
|
||||
PathPredicate = Callable[[str], bool]
|
||||
MatchPredicate = Callable[["Match"], bool]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Hunk:
|
||||
old_start: int
|
||||
old_count: int
|
||||
new_start: int
|
||||
new_count: int
|
||||
|
||||
def contains_old(self, line_number: int) -> bool:
|
||||
return self._contains(line_number, self.old_start, self.old_count)
|
||||
|
||||
def contains_new(self, line_number: int) -> bool:
|
||||
return self._contains(line_number, self.new_start, self.new_count)
|
||||
|
||||
@staticmethod
|
||||
def _contains(line_number: int, start: int, count: int) -> bool:
|
||||
if count == 0:
|
||||
return False
|
||||
return start <= line_number < start + count
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Match:
|
||||
line_number: int
|
||||
source_line: str
|
||||
text: str
|
||||
meta_variables: dict[str, str]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Violation:
|
||||
path: str
|
||||
line_number: int
|
||||
message: str
|
||||
|
||||
|
||||
def rule_path(filename: str) -> Path:
|
||||
return RULES_ROOT / filename
|
||||
|
||||
|
||||
def parse_args(description: str | None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=description)
|
||||
diff_source_group = parser.add_mutually_exclusive_group(required=True)
|
||||
diff_source_group.add_argument(
|
||||
"--staged",
|
||||
action="store_true",
|
||||
help="Inspect staged changes against HEAD, for pre-commit style usage.",
|
||||
)
|
||||
diff_source_group.add_argument(
|
||||
"--base-rev",
|
||||
help="Inspect changes between the provided git revision and HEAD.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def resolve_ast_grep_command() -> list[str]:
|
||||
if shutil.which("ast-grep"):
|
||||
return ["ast-grep"]
|
||||
if shutil.which("uvx"):
|
||||
return ["uvx", "--from", "ast-grep-cli", "ast-grep"]
|
||||
raise RuntimeError("ast-grep executable not found")
|
||||
|
||||
|
||||
def git_output(*args: str, allow_missing: bool = False) -> str:
|
||||
completed = subprocess.run(
|
||||
["git", *args],
|
||||
cwd=SCAN_ROOT,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
if completed.returncode == 0:
|
||||
return completed.stdout
|
||||
if allow_missing:
|
||||
return ""
|
||||
raise RuntimeError(completed.stderr.strip() or completed.stdout.strip() or "git command failed")
|
||||
|
||||
|
||||
def collect_diff_text(args: argparse.Namespace) -> str:
|
||||
if args.staged:
|
||||
return git_output(
|
||||
"diff",
|
||||
"--cached",
|
||||
"--unified=0",
|
||||
"--no-renames",
|
||||
"--diff-filter=AMD",
|
||||
"--no-ext-diff",
|
||||
)
|
||||
return git_output(
|
||||
"diff",
|
||||
"--unified=0",
|
||||
"--no-renames",
|
||||
"--diff-filter=AMD",
|
||||
"--no-ext-diff",
|
||||
f"{args.base_rev}..HEAD",
|
||||
)
|
||||
|
||||
|
||||
def parse_changed_hunks(diff_text: str) -> dict[str, list[Hunk]]:
|
||||
changed_hunks: dict[str, list[Hunk]] = {}
|
||||
current_path: str | None = None
|
||||
|
||||
for line in diff_text.splitlines():
|
||||
if line.startswith("diff --git "):
|
||||
parts = line.split()
|
||||
current_path = parts[3][2:]
|
||||
changed_hunks.setdefault(current_path, [])
|
||||
continue
|
||||
|
||||
if current_path is None:
|
||||
continue
|
||||
|
||||
hunk_match = HUNK_PATTERN.match(line)
|
||||
if not hunk_match:
|
||||
continue
|
||||
|
||||
changed_hunks[current_path].append(
|
||||
Hunk(
|
||||
old_start=int(hunk_match.group("old_start")),
|
||||
old_count=parse_count(hunk_match.group("old_count")),
|
||||
new_start=int(hunk_match.group("new_start")),
|
||||
new_count=parse_count(hunk_match.group("new_count")),
|
||||
)
|
||||
)
|
||||
|
||||
return {path: hunks for path, hunks in changed_hunks.items() if hunks}
|
||||
|
||||
|
||||
def parse_count(value: str | None) -> int:
|
||||
if value is None:
|
||||
return 1
|
||||
return int(value)
|
||||
|
||||
|
||||
def is_python_source_path(path: str) -> bool:
|
||||
return Path(path).suffix in {".py", ".pyi"}
|
||||
|
||||
|
||||
def load_file_versions(path: str, args: argparse.Namespace) -> tuple[str, str]:
|
||||
if args.staged:
|
||||
return (
|
||||
git_output("show", f"HEAD:{path}", allow_missing=True),
|
||||
git_output("show", f":{path}", allow_missing=True),
|
||||
)
|
||||
|
||||
return (
|
||||
git_output("show", f"{args.base_rev}:{path}", allow_missing=True),
|
||||
git_output("show", f"HEAD:{path}", allow_missing=True),
|
||||
)
|
||||
|
||||
|
||||
def run_ast_grep(source: str, *, rule: Path) -> list[Match]:
|
||||
if not source.strip():
|
||||
return []
|
||||
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".py", encoding="utf-8") as temp_file:
|
||||
temp_file.write(source)
|
||||
temp_file.flush()
|
||||
completed = subprocess.run(
|
||||
[
|
||||
*resolve_ast_grep_command(),
|
||||
"scan",
|
||||
"--rule",
|
||||
str(rule),
|
||||
"--json=compact",
|
||||
temp_file.name,
|
||||
],
|
||||
cwd=REPO_ROOT,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
if completed.returncode not in (0, 1):
|
||||
raise RuntimeError(completed.stderr.strip() or completed.stdout.strip() or "ast-grep command failed")
|
||||
|
||||
if not completed.stdout.strip():
|
||||
return []
|
||||
|
||||
lines = source.splitlines()
|
||||
raw_matches = json.loads(completed.stdout)
|
||||
matches: list[Match] = []
|
||||
for raw_match in raw_matches:
|
||||
line_number = int(raw_match["range"]["start"]["line"]) + 1
|
||||
source_line = lines[line_number - 1] if line_number - 1 < len(lines) else ""
|
||||
matches.append(
|
||||
Match(
|
||||
line_number=line_number,
|
||||
source_line=source_line,
|
||||
text=extract_match_text(raw_match, fallback=source_line),
|
||||
meta_variables=extract_meta_variables(raw_match),
|
||||
)
|
||||
)
|
||||
return matches
|
||||
|
||||
|
||||
def extract_match_text(raw_match: dict[str, Any], *, fallback: str) -> str:
|
||||
text = raw_match.get("text")
|
||||
if isinstance(text, str) and text:
|
||||
return text
|
||||
|
||||
lines = raw_match.get("lines")
|
||||
if isinstance(lines, str) and lines:
|
||||
return lines
|
||||
|
||||
return fallback
|
||||
|
||||
|
||||
def extract_meta_variables(raw_match: dict[str, Any]) -> dict[str, str]:
|
||||
meta_variables = raw_match.get("metaVariables")
|
||||
if not isinstance(meta_variables, dict):
|
||||
return {}
|
||||
|
||||
single_variables = meta_variables.get("single")
|
||||
if not isinstance(single_variables, dict):
|
||||
return {}
|
||||
|
||||
result: dict[str, str] = {}
|
||||
for name, value in single_variables.items():
|
||||
if not isinstance(name, str) or not isinstance(value, dict):
|
||||
continue
|
||||
text = value.get("text")
|
||||
if isinstance(text, str):
|
||||
result[name] = text
|
||||
return result
|
||||
|
||||
|
||||
def has_reasoned_guard_ignore(source_line: str, rule_id: str) -> bool:
|
||||
pattern = re.compile(rf"# guard-ignore: {re.escape(rule_id)} -- (?P<reason>\S.*)\s*$")
|
||||
match = pattern.search(source_line)
|
||||
if not match:
|
||||
return False
|
||||
reason = match.group("reason")
|
||||
return bool(reason.strip())
|
||||
|
||||
|
||||
def match_key(match: Match) -> str:
|
||||
"""Ignore formatting when comparing moved Python syntax, retaining literal values."""
|
||||
return ast.dump(ast.parse(textwrap.dedent(match.text)))
|
||||
|
||||
|
||||
def filter_moved_matches(
|
||||
added: list[tuple[str, Match]],
|
||||
removed: Counter[str],
|
||||
) -> list[tuple[str, Match]]:
|
||||
unmatched = []
|
||||
for path, match in added:
|
||||
key = match_key(match)
|
||||
if removed[key]:
|
||||
removed[key] -= 1
|
||||
else:
|
||||
unmatched.append((path, match))
|
||||
return unmatched
|
||||
|
||||
|
||||
def collect_hunk_changes(
|
||||
old_matches: list[Match],
|
||||
new_matches: list[Match],
|
||||
hunk: Hunk,
|
||||
*,
|
||||
is_reportable_match: MatchPredicate,
|
||||
) -> tuple[Counter[str], list[Match]]:
|
||||
old_in_hunk = [
|
||||
match for match in old_matches if hunk.contains_old(match.line_number) and is_reportable_match(match)
|
||||
]
|
||||
new_in_hunk = [
|
||||
match for match in new_matches if hunk.contains_new(match.line_number) and is_reportable_match(match)
|
||||
]
|
||||
existing = Counter(match_key(match) for match in old_in_hunk)
|
||||
added = []
|
||||
for match in new_in_hunk:
|
||||
key = match_key(match)
|
||||
if existing[key]:
|
||||
existing[key] -= 1
|
||||
else:
|
||||
added.append(match)
|
||||
# Preserve the existing allowance for replacing legacy calls within a hunk.
|
||||
remaining = list(existing.elements())
|
||||
replacements = min(len(remaining), len(added))
|
||||
return Counter(remaining[replacements:]), added[replacements:]
|
||||
|
||||
|
||||
def find_net_new_violations(
|
||||
changed_hunks: dict[str, list[Hunk]],
|
||||
args: argparse.Namespace,
|
||||
*,
|
||||
rule: Path,
|
||||
is_scanned_path: PathPredicate,
|
||||
is_reportable_match: MatchPredicate,
|
||||
violation_message: str,
|
||||
) -> list[Violation]:
|
||||
removed: Counter[str] = Counter()
|
||||
added: list[tuple[str, Match]] = []
|
||||
for path, hunks in changed_hunks.items():
|
||||
if not is_scanned_path(path):
|
||||
continue
|
||||
old_source, new_source = load_file_versions(path, args)
|
||||
old_matches = run_ast_grep(old_source, rule=rule)
|
||||
new_matches = run_ast_grep(new_source, rule=rule)
|
||||
for hunk in hunks:
|
||||
removed_in_hunk, added_in_hunk = collect_hunk_changes(
|
||||
old_matches,
|
||||
new_matches,
|
||||
hunk,
|
||||
is_reportable_match=is_reportable_match,
|
||||
)
|
||||
removed.update(removed_in_hunk)
|
||||
added.extend((path, match) for match in added_in_hunk)
|
||||
violations = [
|
||||
Violation(path, match.line_number, violation_message) for path, match in filter_moved_matches(added, removed)
|
||||
]
|
||||
return sorted(violations, key=lambda item: (item.path, item.line_number))
|
||||
|
||||
|
||||
def print_violations(violations: list[Violation]) -> None:
|
||||
for violation in violations:
|
||||
print(f"{violation.path}:{violation.line_number}: {violation.message}", file=sys.stderr)
|
||||
|
||||
|
||||
def run_guard(
|
||||
*,
|
||||
description: str | None,
|
||||
rule: Path,
|
||||
is_scanned_path: PathPredicate,
|
||||
is_reportable_match: MatchPredicate,
|
||||
violation_message: str,
|
||||
) -> int:
|
||||
try:
|
||||
args = parse_args(description)
|
||||
changed_hunks = parse_changed_hunks(collect_diff_text(args))
|
||||
violations = find_net_new_violations(
|
||||
changed_hunks,
|
||||
args,
|
||||
rule=rule,
|
||||
is_scanned_path=is_scanned_path,
|
||||
is_reportable_match=is_reportable_match,
|
||||
violation_message=violation_message,
|
||||
)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), file=sys.stderr)
|
||||
return 2
|
||||
|
||||
if violations:
|
||||
print_violations(violations)
|
||||
return 1
|
||||
return 0
|
||||
@@ -0,0 +1,93 @@
|
||||
id: no-new-base-model
|
||||
language: python
|
||||
severity: error
|
||||
message: New SQLAlchemy models must inherit from TypeBase instead of Base.
|
||||
utils:
|
||||
project-import:
|
||||
kind: import_from_statement
|
||||
has:
|
||||
field: module_name
|
||||
regex: '^(models\.base|api\.models\.base|\.+base)$'
|
||||
project-base:
|
||||
any:
|
||||
- all:
|
||||
- pattern: Base
|
||||
- inside:
|
||||
kind: module
|
||||
stopBy: end
|
||||
has:
|
||||
stopBy: end
|
||||
all:
|
||||
- matches: project-import
|
||||
- has:
|
||||
stopBy: neighbor
|
||||
all:
|
||||
- kind: dotted_name
|
||||
- regex: '^Base$'
|
||||
- regex: '^(api\.)?models\.base\.Base$'
|
||||
- all:
|
||||
- pattern: $BASE
|
||||
- inside:
|
||||
kind: module
|
||||
stopBy: end
|
||||
has:
|
||||
stopBy: end
|
||||
all:
|
||||
- pattern:
|
||||
context: from models.base import Base as $BASE
|
||||
selector: aliased_import
|
||||
- inside:
|
||||
stopBy: neighbor
|
||||
matches: project-import
|
||||
- all:
|
||||
- pattern: $MODULE.Base
|
||||
- inside:
|
||||
kind: module
|
||||
stopBy: end
|
||||
has:
|
||||
stopBy: end
|
||||
any:
|
||||
- all:
|
||||
- pattern:
|
||||
context: import models.base as $MODULE
|
||||
selector: aliased_import
|
||||
- has:
|
||||
field: name
|
||||
regex: '^(api\.)?models\.base$'
|
||||
- all:
|
||||
- pattern:
|
||||
context: from models import base as $MODULE
|
||||
selector: aliased_import
|
||||
- inside:
|
||||
kind: import_from_statement
|
||||
stopBy: neighbor
|
||||
has:
|
||||
field: module_name
|
||||
regex: '^(api\.)?models$'
|
||||
- all:
|
||||
- pattern: base.Base
|
||||
- inside:
|
||||
kind: module
|
||||
stopBy: end
|
||||
has:
|
||||
stopBy: end
|
||||
all:
|
||||
- kind: import_from_statement
|
||||
- has:
|
||||
field: module_name
|
||||
regex: '^(api\.)?models$'
|
||||
- has:
|
||||
stopBy: neighbor
|
||||
all:
|
||||
- kind: dotted_name
|
||||
- regex: '^base$'
|
||||
rule:
|
||||
all:
|
||||
- kind: class_definition
|
||||
- has:
|
||||
field: name
|
||||
pattern: $NAME
|
||||
- has:
|
||||
field: superclasses
|
||||
has:
|
||||
matches: project-base
|
||||
@@ -0,0 +1,84 @@
|
||||
id: no-new-cast
|
||||
language: python
|
||||
severity: error
|
||||
message: Use type narrowing or validate values instead of typing.cast().
|
||||
utils:
|
||||
typing-import:
|
||||
kind: import_from_statement
|
||||
has:
|
||||
field: module_name
|
||||
regex: '^(typing|typing_extensions)$'
|
||||
cast-reference:
|
||||
any:
|
||||
- pattern: typing.cast
|
||||
- pattern: typing_extensions.cast
|
||||
- all:
|
||||
- pattern: $MODULE.cast
|
||||
- inside:
|
||||
kind: module
|
||||
stopBy: end
|
||||
has:
|
||||
stopBy: end
|
||||
all:
|
||||
- any:
|
||||
- pattern:
|
||||
context: import typing as $MODULE
|
||||
selector: aliased_import
|
||||
- pattern:
|
||||
context: import typing_extensions as $MODULE
|
||||
selector: aliased_import
|
||||
- has:
|
||||
field: name
|
||||
regex: '^(typing|typing_extensions)$'
|
||||
- all:
|
||||
- kind: identifier
|
||||
- pattern: cast
|
||||
- inside:
|
||||
kind: module
|
||||
stopBy: end
|
||||
has:
|
||||
stopBy: end
|
||||
all:
|
||||
- matches: typing-import
|
||||
- has:
|
||||
stopBy: neighbor
|
||||
all:
|
||||
- kind: dotted_name
|
||||
- regex: '^cast$'
|
||||
- all:
|
||||
- kind: identifier
|
||||
- pattern: $IMPORTED
|
||||
- inside:
|
||||
kind: module
|
||||
stopBy: end
|
||||
has:
|
||||
stopBy: end
|
||||
all:
|
||||
- pattern:
|
||||
context: from typing import cast as $IMPORTED
|
||||
selector: aliased_import
|
||||
- inside:
|
||||
matches: typing-import
|
||||
stopBy: neighbor
|
||||
rule:
|
||||
all:
|
||||
- kind: call
|
||||
- any:
|
||||
- has:
|
||||
field: function
|
||||
matches: cast-reference
|
||||
- all:
|
||||
- has:
|
||||
field: function
|
||||
kind: identifier
|
||||
pattern: $ALIAS
|
||||
- inside:
|
||||
kind: module
|
||||
stopBy: end
|
||||
has:
|
||||
stopBy: end
|
||||
all:
|
||||
- pattern: $ALIAS = $ORIGINAL
|
||||
- has:
|
||||
field: right
|
||||
matches: cast-reference
|
||||
@@ -0,0 +1,56 @@
|
||||
id: no-new-constructor-bypass
|
||||
language: python
|
||||
severity: error
|
||||
message: Use the class constructor to preserve initialization and validation.
|
||||
utils:
|
||||
allocation-in-new:
|
||||
all:
|
||||
- any:
|
||||
- pattern: $OWNER.__new__($CLS)
|
||||
- pattern: $OWNER.__new__($CLS, $$$TAIL)
|
||||
- inside:
|
||||
all:
|
||||
- kind: function_definition
|
||||
- has:
|
||||
field: name
|
||||
regex: '^__new__$'
|
||||
- has:
|
||||
field: parameters
|
||||
has:
|
||||
all:
|
||||
- nthChild: 1
|
||||
- any:
|
||||
- all:
|
||||
- kind: typed_parameter
|
||||
- has:
|
||||
pattern: $CLS
|
||||
stopBy: neighbor
|
||||
- pattern: $CLS
|
||||
stopBy: neighbor
|
||||
- inside:
|
||||
kind: class_definition
|
||||
stopBy:
|
||||
any:
|
||||
- kind: function_definition
|
||||
- kind: lambda
|
||||
- kind: class_definition
|
||||
stopBy:
|
||||
any:
|
||||
- kind: function_definition
|
||||
- kind: lambda
|
||||
- kind: class_definition
|
||||
rule:
|
||||
any:
|
||||
- all:
|
||||
- pattern: $OWNER.__new__($$$ARGS)
|
||||
- not:
|
||||
matches: allocation-in-new
|
||||
- all:
|
||||
- pattern: $OWNER.__new__
|
||||
- not:
|
||||
inside:
|
||||
kind: call
|
||||
has:
|
||||
field: function
|
||||
pattern: $OWNER.__new__
|
||||
stopBy: neighbor
|
||||
@@ -0,0 +1,54 @@
|
||||
id: no-new-controller-sqlalchemy
|
||||
language: python
|
||||
rule:
|
||||
any:
|
||||
- pattern: db.session.add($$$ARGS)
|
||||
- pattern: db.session.add_all($$$ARGS)
|
||||
- pattern: db.session.begin($$$ARGS)
|
||||
- pattern: db.session.commit($$$ARGS)
|
||||
- pattern: db.session.delete($$$ARGS)
|
||||
- pattern: db.session.execute($$$ARGS)
|
||||
- pattern: db.session.flush($$$ARGS)
|
||||
- pattern: db.session.get($MODEL, $$$ARGS)
|
||||
- pattern: db.session.merge($$$ARGS)
|
||||
- pattern: db.session.refresh($$$ARGS)
|
||||
- pattern: db.session.rollback($$$ARGS)
|
||||
- pattern: db.session.scalar($$$ARGS)
|
||||
- pattern: db.session.scalars($$$ARGS)
|
||||
- pattern: session.add($$$ARGS)
|
||||
- pattern: session.add_all($$$ARGS)
|
||||
- pattern: session.begin($$$ARGS)
|
||||
- pattern: session.commit($$$ARGS)
|
||||
- pattern: session.delete($$$ARGS)
|
||||
- pattern: session.execute($$$ARGS)
|
||||
- pattern: session.flush($$$ARGS)
|
||||
- pattern: session.get($MODEL, $$$ARGS)
|
||||
- pattern: session.merge($$$ARGS)
|
||||
- pattern: session.refresh($$$ARGS)
|
||||
- pattern: session.rollback($$$ARGS)
|
||||
- pattern: session.scalar($$$ARGS)
|
||||
- pattern: session.scalars($$$ARGS)
|
||||
- pattern: Session($$$ARGS)
|
||||
- pattern: sessionmaker($$$ARGS)
|
||||
- pattern: select($$$ARGS)
|
||||
- pattern: insert($$$ARGS)
|
||||
- pattern: update($$$ARGS)
|
||||
- pattern: delete($$$ARGS)
|
||||
- pattern: text($$$ARGS)
|
||||
- pattern: sa.select($$$ARGS)
|
||||
- pattern: sa.insert($$$ARGS)
|
||||
- pattern: sa.update($$$ARGS)
|
||||
- pattern: sa.delete($$$ARGS)
|
||||
- pattern: sa.text($$$ARGS)
|
||||
- pattern: sqlalchemy.select($$$ARGS)
|
||||
- pattern: sqlalchemy.insert($$$ARGS)
|
||||
- pattern: sqlalchemy.update($$$ARGS)
|
||||
- pattern: sqlalchemy.delete($$$ARGS)
|
||||
- pattern: sqlalchemy.text($$$ARGS)
|
||||
- pattern: db.select($$$ARGS)
|
||||
- pattern: db.insert($$$ARGS)
|
||||
- pattern: db.update($$$ARGS)
|
||||
- pattern: db.delete($$$ARGS)
|
||||
- pattern: db.text($$$ARGS)
|
||||
message: net-new direct SQLAlchemy call in controller code
|
||||
severity: error
|
||||
@@ -0,0 +1,12 @@
|
||||
id: no-new-getattr
|
||||
language: python
|
||||
rule:
|
||||
any:
|
||||
- pattern: getattr($OBJ, $NAME)
|
||||
- pattern: getattr($OBJ, $NAME, $$$REST)
|
||||
- pattern: builtins.getattr($OBJ, $NAME)
|
||||
- pattern: builtins.getattr($OBJ, $NAME, $$$REST)
|
||||
- pattern: __builtins__.getattr($OBJ, $NAME)
|
||||
- pattern: __builtins__.getattr($OBJ, $NAME, $$$REST)
|
||||
message: net-new getattr() in added code
|
||||
severity: error
|
||||
@@ -0,0 +1,61 @@
|
||||
id: no-new-mock
|
||||
language: python
|
||||
severity: error
|
||||
message: Use a real instance instead of adding a Mock, MagicMock, or SimpleNamespace constructor.
|
||||
rule:
|
||||
all:
|
||||
- pattern: $CALLEE($$$ARGS)
|
||||
- any:
|
||||
- has:
|
||||
field: function
|
||||
stopBy: end
|
||||
regex: '(^|\.)(Mock|MagicMock|SimpleNamespace)$'
|
||||
- inside:
|
||||
kind: module
|
||||
stopBy: end
|
||||
has:
|
||||
stopBy: end
|
||||
all:
|
||||
- pattern:
|
||||
context: from package import $ORIGINAL as $CALLEE
|
||||
selector: aliased_import
|
||||
- any:
|
||||
- all:
|
||||
- has:
|
||||
field: name
|
||||
stopBy: end
|
||||
regex: '^(Mock|MagicMock)$'
|
||||
- inside:
|
||||
kind: import_from_statement
|
||||
stopBy: end
|
||||
has:
|
||||
field: module_name
|
||||
stopBy: end
|
||||
regex: '^unittest\.mock$'
|
||||
- all:
|
||||
- has:
|
||||
field: name
|
||||
stopBy: end
|
||||
regex: '^SimpleNamespace$'
|
||||
- inside:
|
||||
kind: import_from_statement
|
||||
stopBy: end
|
||||
has:
|
||||
field: module_name
|
||||
stopBy: end
|
||||
regex: '^types$'
|
||||
- all:
|
||||
- has:
|
||||
field: function
|
||||
kind: identifier
|
||||
- inside:
|
||||
kind: module
|
||||
stopBy: end
|
||||
has:
|
||||
stopBy: end
|
||||
all:
|
||||
- pattern: $CALLEE = $ORIGINAL
|
||||
- has:
|
||||
field: right
|
||||
stopBy: end
|
||||
regex: '(^|\.)(Mock|MagicMock|SimpleNamespace)$'
|
||||
@@ -0,0 +1,11 @@
|
||||
id: no-new-session-mock
|
||||
language: python
|
||||
rule:
|
||||
any:
|
||||
- pattern: $SESSION = MagicMock($$$ARGS)
|
||||
- pattern: $SESSION = Mock($$$ARGS)
|
||||
constraints:
|
||||
SESSION:
|
||||
regex: ^(session|db_session|mock_session|read_session|write_session)$
|
||||
message: net-new session mock assignment in test code
|
||||
severity: error
|
||||
@@ -0,0 +1,98 @@
|
||||
id: no-new-stub-subclass
|
||||
language: python
|
||||
severity: error
|
||||
message: Instantiate the production runtime class; do not bypass its constructor in a test subclass.
|
||||
utils:
|
||||
runtime-base:
|
||||
any:
|
||||
- regex: '(^|\.)(ModelInstance|ModelManager|DatasetRetrieval|TraceQueueManager|MessageBasedAppQueueManager|Thread)$'
|
||||
- all:
|
||||
- pattern: $BASE
|
||||
- inside:
|
||||
kind: module
|
||||
stopBy: end
|
||||
has:
|
||||
stopBy: end
|
||||
all:
|
||||
- pattern:
|
||||
context: from package import $ORIGINAL as $BASE
|
||||
selector: aliased_import
|
||||
- has:
|
||||
field: name
|
||||
stopBy: end
|
||||
regex: '^(ModelInstance|ModelManager|DatasetRetrieval|TraceQueueManager|MessageBasedAppQueueManager|Thread)$'
|
||||
initializer:
|
||||
kind: function_definition
|
||||
has:
|
||||
field: name
|
||||
stopBy: end
|
||||
regex: ^__init__$
|
||||
delegates:
|
||||
has:
|
||||
# Nested definitions do not initialize the containing object.
|
||||
stopBy:
|
||||
any:
|
||||
- kind: function_definition
|
||||
- kind: class_definition
|
||||
- kind: lambda
|
||||
any:
|
||||
- pattern: super($$$ARGS).__init__($$$PARAMS)
|
||||
- pattern: $BASE.__init__(self, $$$PARAMS)
|
||||
dataclass-decorator:
|
||||
kind: decorator
|
||||
regex: '^@(dataclasses\.)?dataclass(\(|$)'
|
||||
not:
|
||||
has:
|
||||
stopBy: end
|
||||
pattern:
|
||||
context: dataclass(init=False)
|
||||
selector: keyword_argument
|
||||
rule:
|
||||
all:
|
||||
- kind: class_definition
|
||||
- has:
|
||||
field: name
|
||||
stopBy: neighbor
|
||||
pattern: $NAME
|
||||
- has:
|
||||
field: superclasses
|
||||
stopBy: neighbor
|
||||
has:
|
||||
stopBy: neighbor
|
||||
all:
|
||||
- pattern: $BASE
|
||||
- matches: runtime-base
|
||||
- any:
|
||||
- has:
|
||||
field: body
|
||||
stopBy: neighbor
|
||||
has:
|
||||
# Only methods of this class, not constructors of nested classes.
|
||||
stopBy: neighbor
|
||||
all:
|
||||
- matches: initializer
|
||||
- not:
|
||||
matches: delegates
|
||||
- all:
|
||||
- inside:
|
||||
# A nested class must not inherit its outer class's decorator.
|
||||
stopBy: neighbor
|
||||
kind: decorated_definition
|
||||
has:
|
||||
stopBy: neighbor
|
||||
matches: dataclass-decorator
|
||||
- not:
|
||||
has:
|
||||
field: body
|
||||
stopBy: neighbor
|
||||
has:
|
||||
stopBy: neighbor
|
||||
any:
|
||||
- matches: initializer
|
||||
- all:
|
||||
- kind: function_definition
|
||||
- has:
|
||||
field: name
|
||||
stopBy: end
|
||||
regex: ^__post_init__$
|
||||
- matches: delegates
|
||||
@@ -0,0 +1,63 @@
|
||||
id: no-spec-mock
|
||||
language: python
|
||||
severity: error
|
||||
message: Use a real instance or a concrete test fake instead of a spec-based mock.
|
||||
rule:
|
||||
all:
|
||||
- pattern: $CALLEE($$$ARGS)
|
||||
# Inspect direct arguments only; nested calls may have unrelated spec keywords.
|
||||
- has:
|
||||
field: arguments
|
||||
has:
|
||||
any:
|
||||
- all:
|
||||
- kind: keyword_argument
|
||||
- has:
|
||||
field: name
|
||||
regex: ^spec(_set)?$
|
||||
- all:
|
||||
- kind: dictionary_splat
|
||||
- has:
|
||||
kind: dictionary
|
||||
has:
|
||||
kind: pair
|
||||
has:
|
||||
field: key
|
||||
regex: ^['"]spec(_set)?['"]$
|
||||
- not:
|
||||
any:
|
||||
- kind: keyword_argument
|
||||
- kind: dictionary_splat
|
||||
- kind: comment
|
||||
- regex: '^[(),]$'
|
||||
- any:
|
||||
- has:
|
||||
field: function
|
||||
regex: '(^|\.)(Mock|MagicMock|AsyncMock|NonCallableMock|NonCallableMagicMock)$'
|
||||
- inside:
|
||||
kind: module
|
||||
stopBy: end
|
||||
has:
|
||||
stopBy: end
|
||||
all:
|
||||
- pattern:
|
||||
context: from unittest.mock import $ORIGINAL as $CALLEE
|
||||
selector: aliased_import
|
||||
- has:
|
||||
field: name
|
||||
regex: '^(Mock|MagicMock|AsyncMock|NonCallableMock|NonCallableMagicMock)$'
|
||||
- inside:
|
||||
kind: import_from_statement
|
||||
has:
|
||||
field: module_name
|
||||
regex: '^unittest\.mock$'
|
||||
- inside:
|
||||
kind: module
|
||||
stopBy: end
|
||||
has:
|
||||
stopBy: end
|
||||
all:
|
||||
- pattern: $CALLEE = $ORIGINAL
|
||||
- has:
|
||||
field: right
|
||||
regex: '(^|\.)(Mock|MagicMock|AsyncMock|NonCallableMock|NonCallableMagicMock)$'
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Require TypeBase for new SQLAlchemy model inheritance.
|
||||
|
||||
Existing Base subclasses may be edited or migrated. New classes that inherit
|
||||
from models.base.Base fail, including import aliases and qualified bases.
|
||||
Class names preserve the baseline even when surrounding lines change. Renames
|
||||
preserve the baseline through Git's rename detection.
|
||||
|
||||
CI compares committed revisions with --base-rev. Local checks inspect the
|
||||
working tree and untracked Python files against the merge base with main.
|
||||
Use --staged to inspect only staged changes against HEAD.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
from ast_grep_guard import (
|
||||
Violation,
|
||||
git_output,
|
||||
print_violations,
|
||||
rule_path,
|
||||
run_ast_grep,
|
||||
)
|
||||
from check_no_new_test_doubles import changed_file_versions
|
||||
|
||||
|
||||
def working_file_versions() -> dict[str, tuple[str, str]]:
|
||||
baseline = "HEAD"
|
||||
for revision in ("upstream/main", "origin/main", "main"):
|
||||
merge_base = git_output("merge-base", "HEAD", revision, allow_missing=True).strip()
|
||||
if merge_base:
|
||||
baseline = merge_base
|
||||
break
|
||||
fields = iter(
|
||||
git_output(
|
||||
"diff",
|
||||
"--name-status",
|
||||
"-z",
|
||||
"--find-renames",
|
||||
"--diff-filter=AMR",
|
||||
baseline,
|
||||
"--",
|
||||
"api",
|
||||
)
|
||||
.rstrip("\0")
|
||||
.split("\0")
|
||||
)
|
||||
paths: dict[str, str] = {}
|
||||
for status in fields:
|
||||
if not status:
|
||||
continue
|
||||
old_path = next(fields)
|
||||
path = next(fields) if status.startswith("R") else old_path
|
||||
paths[path] = old_path
|
||||
for path in git_output("ls-files", "--others", "--exclude-standard", "-z", "api").rstrip("\0").split("\0"):
|
||||
paths[path] = path
|
||||
versions = {}
|
||||
for path in sorted(paths):
|
||||
source = Path(path)
|
||||
if source.suffix not in {".py", ".pyi"} or not source.is_file():
|
||||
continue
|
||||
versions[path] = (
|
||||
git_output("show", f"{baseline}:{paths[path]}", allow_missing=True),
|
||||
source.read_text(),
|
||||
)
|
||||
return versions
|
||||
|
||||
|
||||
def find_new_base_models(versions: dict[str, tuple[str, str]]) -> list[Violation]:
|
||||
rule = rule_path("no_new_base_model.yml")
|
||||
violations = []
|
||||
for path, (old_source, new_source) in versions.items():
|
||||
if not Path(path).is_relative_to("api"):
|
||||
continue
|
||||
existing = Counter(match.meta_variables["NAME"] for match in run_ast_grep(old_source, rule=rule))
|
||||
for match in run_ast_grep(new_source, rule=rule):
|
||||
name = match.meta_variables["NAME"]
|
||||
if existing[name]:
|
||||
existing[name] -= 1
|
||||
else:
|
||||
violations.append(
|
||||
Violation(
|
||||
path,
|
||||
match.line_number,
|
||||
f"no-new-base-model: {name} must inherit from TypeBase.",
|
||||
)
|
||||
)
|
||||
return violations
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
source = parser.add_mutually_exclusive_group()
|
||||
source.add_argument("--base-rev", help="Compare this revision with committed HEAD.")
|
||||
source.add_argument("--staged", action="store_true", help="Compare staged changes with HEAD.")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
versions = changed_file_versions(args) if args.base_rev or args.staged else working_file_versions()
|
||||
violations = find_new_base_models(versions)
|
||||
except (OSError, RuntimeError) as error:
|
||||
sys.stderr.write(f"{error}\n")
|
||||
return 2
|
||||
print_violations(violations)
|
||||
return int(bool(violations))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Reject new typing.cast and typing_extensions.cast calls in API Python files.
|
||||
|
||||
Preconditions: Run from the repository root with Git and ast-grep available.
|
||||
Use --staged for index contents or --base-rev for committed revisions.
|
||||
Without either option, inspect local API changes against the main merge base.
|
||||
|
||||
Postconditions: Identical existing calls retain their Git baseline.
|
||||
Copies and changed calls fail. Import aliases and direct assignment aliases
|
||||
are included. Other libraries' cast functions are outside this rule's scope.
|
||||
This syntactic rule does not resolve arbitrary assignments or shadowed names.
|
||||
Exit codes are 0 for success, 1 for violations, and 2 for check failures.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
from ast_grep_guard import (
|
||||
Violation,
|
||||
filter_moved_matches,
|
||||
match_key,
|
||||
print_violations,
|
||||
rule_path,
|
||||
run_ast_grep,
|
||||
)
|
||||
from check_no_new_base_model import working_file_versions
|
||||
from check_no_new_test_doubles import changed_file_versions
|
||||
|
||||
|
||||
def find_new_cast_calls(
|
||||
versions: dict[str, tuple[str, str]],
|
||||
) -> list[Violation]:
|
||||
"""Compare cast calls in complete source versions.
|
||||
|
||||
Preconditions: Each key is a repository-relative path with old and new source.
|
||||
Postconditions: Return API violations without changing files or source versions.
|
||||
Each removed occurrence offsets one identical addition. Scan failures raise.
|
||||
"""
|
||||
rule = rule_path("no_new_cast.yml")
|
||||
removed: Counter[str] = Counter()
|
||||
added = []
|
||||
for path, (old_source, new_source) in versions.items():
|
||||
if Path(path).is_relative_to("api"):
|
||||
existing = Counter(match_key(match) for match in run_ast_grep(old_source, rule=rule))
|
||||
for match in run_ast_grep(new_source, rule=rule):
|
||||
key = match_key(match)
|
||||
if existing[key]:
|
||||
existing[key] -= 1
|
||||
else:
|
||||
added.append((path, match))
|
||||
removed.update(existing)
|
||||
return [
|
||||
Violation(
|
||||
path,
|
||||
match.line_number,
|
||||
"no-new-cast: use type narrowing or validate values instead of typing.cast().",
|
||||
)
|
||||
for path, match in filter_moved_matches(added, removed)
|
||||
]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Precondition: Git and ast-grep are available in the repository root.
|
||||
|
||||
Postconditions: Report violations without changing repository files.
|
||||
Return 0 for success, 1 for violations, or 2 for check failures.
|
||||
"""
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
source = parser.add_mutually_exclusive_group()
|
||||
source.add_argument("--base-rev", help="Compare this revision with committed HEAD.")
|
||||
source.add_argument("--staged", action="store_true", help="Compare staged changes with HEAD.")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
versions = changed_file_versions(args) if args.base_rev or args.staged else working_file_versions()
|
||||
violations = find_new_cast_calls(versions)
|
||||
except (OSError, RuntimeError) as error:
|
||||
sys.stderr.write(f"{error}\n")
|
||||
return 2
|
||||
print_violations(sorted(violations, key=lambda item: (item.path, item.line_number)))
|
||||
return int(bool(violations))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Reject new __new__ calls and references that bypass constructor checks.
|
||||
|
||||
Preconditions: Run from the repository root with Git and ast-grep available.
|
||||
Use --staged for index contents or --base-rev for committed revisions.
|
||||
Without either option, inspect local API changes against the main merge base.
|
||||
|
||||
Postconditions: Existing matching occurrences retain their Git baseline.
|
||||
Identical moves retain that baseline. Copies and changed calls fail.
|
||||
Allocation for the first parameter of a class's __new__ method is allowed.
|
||||
Nested functions and lambdas do not inherit that allocation permission.
|
||||
This syntactic rule does not resolve dynamic attribute access.
|
||||
Exit codes are 0 for success, 1 for violations, and 2 for check failures.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
from ast_grep_guard import (
|
||||
Violation,
|
||||
filter_moved_matches,
|
||||
match_key,
|
||||
print_violations,
|
||||
rule_path,
|
||||
run_ast_grep,
|
||||
)
|
||||
from check_no_new_base_model import working_file_versions
|
||||
from check_no_new_test_doubles import changed_file_versions
|
||||
|
||||
|
||||
def find_new_constructor_bypasses(
|
||||
versions: dict[str, tuple[str, str]],
|
||||
) -> list[Violation]:
|
||||
"""Compare constructor access in complete source versions.
|
||||
|
||||
Preconditions: Each key is a repository-relative path with old and new source.
|
||||
Postconditions: Return API violations without changing files or source versions.
|
||||
Each removed occurrence offsets one identical addition. Scan failures raise.
|
||||
"""
|
||||
rule = rule_path("no_new_constructor_bypass.yml")
|
||||
removed: Counter[str] = Counter()
|
||||
added = []
|
||||
for path, (old_source, new_source) in versions.items():
|
||||
if Path(path).is_relative_to("api"):
|
||||
existing = Counter(match_key(match) for match in run_ast_grep(old_source, rule=rule))
|
||||
for match in run_ast_grep(new_source, rule=rule):
|
||||
key = match_key(match)
|
||||
if existing[key]:
|
||||
existing[key] -= 1
|
||||
else:
|
||||
added.append((path, match))
|
||||
removed.update(existing)
|
||||
return [
|
||||
Violation(
|
||||
path,
|
||||
match.line_number,
|
||||
"no-new-constructor-bypass: use the class constructor to preserve initialization and validation.",
|
||||
)
|
||||
for path, match in filter_moved_matches(added, removed)
|
||||
]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Precondition: Git and ast-grep are available in the repository root.
|
||||
|
||||
Postconditions: Report violations without changing repository files.
|
||||
Return 0 for success, 1 for violations, or 2 for check failures.
|
||||
"""
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
source = parser.add_mutually_exclusive_group()
|
||||
source.add_argument("--base-rev", help="Compare this revision with committed HEAD.")
|
||||
source.add_argument("--staged", action="store_true", help="Compare staged changes with HEAD.")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
versions = changed_file_versions(args) if args.base_rev or args.staged else working_file_versions()
|
||||
violations = find_new_constructor_bypasses(versions)
|
||||
except (OSError, RuntimeError) as error:
|
||||
sys.stderr.write(f"{error}\n")
|
||||
return 2
|
||||
print_violations(sorted(violations, key=lambda item: (item.path, item.line_number)))
|
||||
return int(bool(violations))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,61 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Block net-new direct SQLAlchemy calls in controller code.
|
||||
|
||||
Allowing flush() and commit() to allow controllers to control
|
||||
the transaction border.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from ast_grep_guard import Match, has_reasoned_guard_ignore, rule_path, run_guard
|
||||
|
||||
|
||||
RULE_ID = "no-new-controller-sqlalchemy"
|
||||
RULE_PATH = rule_path("no_new_controller_sqlalchemy.yml")
|
||||
CONTROLLER_ROOT = Path("api/controllers")
|
||||
FLASK_SESSION_GET_PATTERN = re.compile(r"^session\.get\(\s*['\"]")
|
||||
ALLOWED_SESSION_METHODS = frozenset({"commit", "flush"})
|
||||
VIOLATION_MESSAGE = "no-new-controller-sqlalchemy net-new direct SQLAlchemy call in controller code"
|
||||
|
||||
|
||||
def is_controller_python_source_path(path: str) -> bool:
|
||||
source_path = Path(path)
|
||||
return source_path.suffix in {".py", ".pyi"} and source_path.is_relative_to(CONTROLLER_ROOT)
|
||||
|
||||
|
||||
def is_allowed_session_boundary(match: Match) -> bool:
|
||||
method = match.meta_variables.get("METHOD") or match.meta_variables.get("SESSION_METHOD")
|
||||
if method in ALLOWED_SESSION_METHODS:
|
||||
return True
|
||||
|
||||
stripped_text = match.text.strip()
|
||||
return stripped_text.startswith(("db.session.commit(", "db.session.flush(", "session.commit(", "session.flush("))
|
||||
|
||||
|
||||
def is_flask_session_get(match: Match) -> bool:
|
||||
return bool(FLASK_SESSION_GET_PATTERN.match(match.text.strip()))
|
||||
|
||||
|
||||
def is_suppressed(match: Match) -> bool:
|
||||
return has_reasoned_guard_ignore(match.source_line, RULE_ID)
|
||||
|
||||
|
||||
def is_reportable_match(match: Match) -> bool:
|
||||
return not is_allowed_session_boundary(match) and not is_flask_session_get(match) and not is_suppressed(match)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
return run_guard(
|
||||
description=__doc__,
|
||||
rule=RULE_PATH,
|
||||
is_scanned_path=is_controller_python_source_path,
|
||||
is_reportable_match=is_reportable_match,
|
||||
violation_message=VIOLATION_MESSAGE,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,29 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Block net-new getattr() usage in staged changes or against an explicit base revision."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from ast_grep_guard import Match, has_reasoned_guard_ignore, is_python_source_path, rule_path, run_guard
|
||||
|
||||
|
||||
RULE_ID = "no-new-getattr"
|
||||
RULE_PATH = rule_path("no_new_getattr.yml")
|
||||
VIOLATION_MESSAGE = "no-new-getattr net-new getattr() in added code"
|
||||
|
||||
|
||||
def is_reportable_match(match: Match) -> bool:
|
||||
return not has_reasoned_guard_ignore(match.source_line, RULE_ID)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
return run_guard(
|
||||
description=__doc__,
|
||||
rule=RULE_PATH,
|
||||
is_scanned_path=is_python_source_path,
|
||||
is_reportable_match=is_reportable_match,
|
||||
violation_message=VIOLATION_MESSAGE,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Block net-new session mock assignments in unit tests.
|
||||
|
||||
This catches net-new top-level session mocks like
|
||||
`session = MagicMock()` or `db_session = Mock()` added to
|
||||
`tests/unit_tests/**`. Pre-existing matches are ignored via the
|
||||
shared baseline diff (`scripts/ast_grep_guard.py`), so historical
|
||||
mocks that haven't been cleaned up yet do not block the gate.
|
||||
|
||||
A `# guard-ignore: no-new-session-mock -- <reason>` line on the same
|
||||
source line suppresses a violation with a human-readable reason.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from ast_grep_guard import Match, has_reasoned_guard_ignore, is_python_source_path, rule_path, run_guard
|
||||
|
||||
|
||||
RULE_ID = "no-new-session-mock"
|
||||
RULE_PATH = rule_path("no_new_session_mock.yml")
|
||||
TEST_ROOT = Path("api/tests/unit_tests")
|
||||
VIOLATION_MESSAGE = "no-new-session-mock net-new session mock assignment in test code"
|
||||
|
||||
|
||||
def is_test_source_path(path: str) -> bool:
|
||||
if not is_python_source_path(path):
|
||||
return False
|
||||
return Path(path).is_relative_to(TEST_ROOT)
|
||||
|
||||
|
||||
def is_reportable_match(match: Match) -> bool:
|
||||
return not has_reasoned_guard_ignore(match.source_line, RULE_ID)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
return run_guard(
|
||||
description=__doc__,
|
||||
rule=RULE_PATH,
|
||||
is_scanned_path=is_test_source_path,
|
||||
is_reportable_match=is_reportable_match,
|
||||
violation_message=VIOLATION_MESSAGE,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,182 @@
|
||||
"""Reject net-new mock constructors and constructor-bypassing runtime subclasses.
|
||||
|
||||
Mock, MagicMock and SimpleNamespace calls are checked in all changed Python
|
||||
sources, including common qualified names, import aliases and direct factory
|
||||
aliases. AsyncMock and method patching are outside this rule's scope.
|
||||
|
||||
The subclass rule covers the audited ModelInstance, ModelManager,
|
||||
DatasetRetrieval, TraceQueueManager, MessageBasedAppQueueManager and Thread
|
||||
base names (including qualified names and import aliases) in api/tests/. It
|
||||
checks explicit __init__ overrides and dataclass-generated initializers. This
|
||||
is a syntactic guard, not general Python inheritance or execution analysis.
|
||||
Protocol and Pydantic model implementations are not targeted.
|
||||
|
||||
Both rules retain the provided Git baseline. Identical mock calls are counted
|
||||
per file, with removed occurrences available to match code moved between files:
|
||||
an additional copy or a different constructor call is rejected.
|
||||
Git-detected renames retain the source file's baseline, including paths with
|
||||
spaces. Subclass matches are compared by class name within each changed file,
|
||||
not only by the line of the class header:
|
||||
removing a super call or adding a dataclass decorator must still be detected.
|
||||
An existing offending class can be edited or removed without adding another.
|
||||
For a justified subclass exception only, add
|
||||
`# guard-ignore: no-new-stub-subclass -- <reason>` on the class header.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import sys
|
||||
import textwrap
|
||||
from collections import Counter
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
|
||||
from ast_grep_guard import (
|
||||
Match,
|
||||
Violation,
|
||||
filter_moved_matches,
|
||||
git_output,
|
||||
has_reasoned_guard_ignore,
|
||||
is_python_source_path,
|
||||
match_key,
|
||||
parse_args,
|
||||
print_violations,
|
||||
rule_path,
|
||||
run_ast_grep,
|
||||
)
|
||||
|
||||
STUB_RULE_ID = "no-new-stub-subclass"
|
||||
|
||||
|
||||
def changed_file_versions(args: argparse.Namespace) -> dict[str, tuple[str, str]]:
|
||||
"""Read Git blobs, preserving rename baselines and NUL-delimited filenames."""
|
||||
revisions = ["--cached"] if args.staged else [f"{args.base_rev}..HEAD"]
|
||||
status_text = git_output(
|
||||
"diff",
|
||||
"--name-status",
|
||||
"-z",
|
||||
"--find-renames",
|
||||
"--diff-filter=AMRD",
|
||||
"--no-ext-diff",
|
||||
*revisions,
|
||||
)
|
||||
if not status_text:
|
||||
return {}
|
||||
fields = iter(status_text.rstrip("\0").split("\0"))
|
||||
old_revision = "HEAD" if args.staged else args.base_rev
|
||||
new_revision = "" if args.staged else "HEAD"
|
||||
versions: dict[str, tuple[str, str]] = {}
|
||||
for status in fields:
|
||||
try:
|
||||
old_path = next(fields)
|
||||
path = next(fields) if status.startswith("R") else old_path
|
||||
except StopIteration as exc:
|
||||
raise RuntimeError("Incomplete Git name-status output") from exc
|
||||
if not is_python_source_path(path):
|
||||
continue
|
||||
old_source = "" if status == "A" else git_output("show", f"{old_revision}:{old_path}")
|
||||
new_source = "" if status == "D" else git_output("show", f"{new_revision}:{path}")
|
||||
versions[path] = (old_source, new_source)
|
||||
return versions
|
||||
|
||||
|
||||
def mock_matches(source: str) -> list[Match]:
|
||||
"""Keep import renames from turning existing constructors into new calls."""
|
||||
bindings: dict[str, str] = {}
|
||||
for node in ast.parse(source).body:
|
||||
if isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
bindings[alias.asname or alias.name.split(".")[0]] = (
|
||||
alias.name if alias.asname else alias.name.split(".")[0]
|
||||
)
|
||||
elif isinstance(node, ast.ImportFrom):
|
||||
for alias in node.names:
|
||||
bindings[alias.asname or alias.name] = alias.name
|
||||
matches = []
|
||||
for match in run_ast_grep(source, rule=rule_path("no_new_mock.yml")):
|
||||
tree = ast.parse(textwrap.dedent(match.text))
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Name) and node.id in bindings:
|
||||
node.id = bindings[node.id]
|
||||
matches.append(replace(match, text=ast.unparse(tree)))
|
||||
return matches
|
||||
|
||||
|
||||
def find_new_mock_calls(changed: dict[str, tuple[str, str]]) -> list[Violation]:
|
||||
"""Keep identical baseline calls, but catch alias changes outside their hunks."""
|
||||
removed: Counter[str] = Counter()
|
||||
added: list[tuple[str, Match]] = []
|
||||
for path, (old_source, new_source) in changed.items():
|
||||
if old_source == new_source:
|
||||
continue
|
||||
old_calls = Counter(match_key(match) for match in mock_matches(old_source))
|
||||
for match in mock_matches(new_source):
|
||||
key = match_key(match)
|
||||
if old_calls[key]:
|
||||
old_calls[key] -= 1
|
||||
continue
|
||||
added.append((path, match))
|
||||
removed.update(old_calls)
|
||||
return [
|
||||
Violation(
|
||||
path,
|
||||
match.line_number,
|
||||
"no-new-mock: instantiate a real class instead of Mock, MagicMock or SimpleNamespace",
|
||||
)
|
||||
for path, match in filter_moved_matches(added, removed)
|
||||
]
|
||||
|
||||
|
||||
def is_reportable_subclass(match: Match) -> bool:
|
||||
return not has_reasoned_guard_ignore(match.source_line, STUB_RULE_ID)
|
||||
|
||||
|
||||
def find_new_stub_subclasses(changed: dict[str, tuple[str, str]]) -> list[Violation]:
|
||||
removed: Counter[str] = Counter()
|
||||
added: list[tuple[str, Match]] = []
|
||||
for path, (old_source, new_source) in changed.items():
|
||||
if not Path(path).is_relative_to("api/tests"):
|
||||
continue
|
||||
old_matches = run_ast_grep(old_source, rule=rule_path("no_new_stub_subclass.yml"))
|
||||
new_matches = run_ast_grep(new_source, rule=rule_path("no_new_stub_subclass.yml"))
|
||||
old_names: dict[str, list[Match]] = {}
|
||||
for match in old_matches:
|
||||
if is_reportable_subclass(match):
|
||||
old_names.setdefault(match.meta_variables["NAME"], []).append(match)
|
||||
for match in new_matches:
|
||||
if not is_reportable_subclass(match):
|
||||
continue
|
||||
name = match.meta_variables["NAME"]
|
||||
if old_names.get(name):
|
||||
old_names[name].pop()
|
||||
continue
|
||||
added.append((path, match))
|
||||
removed.update(match_key(match) for matches in old_names.values() for match in matches)
|
||||
return [
|
||||
Violation(
|
||||
path,
|
||||
match.line_number,
|
||||
f"{STUB_RULE_ID}: {match.meta_variables['NAME']} bypasses a runtime constructor; "
|
||||
"instantiate the real class",
|
||||
)
|
||||
for path, match in filter_moved_matches(added, removed)
|
||||
]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
args = parse_args(__doc__)
|
||||
changed = changed_file_versions(args)
|
||||
violations = find_new_mock_calls(changed)
|
||||
violations.extend(find_new_stub_subclasses(changed))
|
||||
except (OSError, RuntimeError) as exc:
|
||||
print(str(exc), file=sys.stderr)
|
||||
return 2
|
||||
print_violations(sorted(violations, key=lambda item: (item.path, item.line_number)))
|
||||
return 1 if violations else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Reject spec-based mock constructors in the API, including unchanged files.
|
||||
|
||||
Use real model/SDK instances or concrete test fakes. The rule covers explicit
|
||||
spec/spec_set keywords, positional specs, qualified constructors, import aliases,
|
||||
and direct constructor aliases. It does not prohibit ordinary method mocks or
|
||||
create_autospec. Optional paths allow a focused local scan; CI scans all of api/.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from ast_grep_guard import REPO_ROOT, resolve_ast_grep_command, rule_path
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("paths", nargs="*", type=Path, help="Files or directories to scan (default: api/).")
|
||||
args = parser.parse_args()
|
||||
paths = args.paths or [REPO_ROOT / "api"]
|
||||
try:
|
||||
for path in paths:
|
||||
if not path.exists():
|
||||
raise RuntimeError(f"Scan path does not exist: {path}")
|
||||
return subprocess.run(
|
||||
[
|
||||
*resolve_ast_grep_command(),
|
||||
"scan",
|
||||
"--rule",
|
||||
str(rule_path("no_spec_mock.yml")),
|
||||
"--report-style",
|
||||
"short",
|
||||
*(str(path) for path in paths),
|
||||
],
|
||||
check=False,
|
||||
).returncode
|
||||
except (OSError, RuntimeError) as exc:
|
||||
sys.stderr.write(f"{exc}\n")
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,223 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Report direct SQLAlchemy usage in controller code.
|
||||
|
||||
This is the full-inventory companion to ``check_no_new_controller_sqlalchemy.py``.
|
||||
It scans the current tree instead of a git diff, so migration work can inspect
|
||||
existing controller usage without turning historical debt into a CI failure.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
|
||||
import check_no_new_controller_sqlalchemy as controller_sqlalchemy_guard
|
||||
from ast_grep_guard import Match, REPO_ROOT, run_ast_grep
|
||||
|
||||
|
||||
DEFAULT_CONTROLLER_DIR = REPO_ROOT / "api" / "controllers"
|
||||
MAX_TEXT_LENGTH = 180
|
||||
SESSION_METHOD_PATTERN = re.compile(r"^(?:db\.session|session)\.(?P<method>[A-Za-z_][A-Za-z0-9_]*)\(")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Finding:
|
||||
classification: str
|
||||
category: str
|
||||
file: str
|
||||
line: int
|
||||
text: str
|
||||
source_line: str
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"paths",
|
||||
nargs="*",
|
||||
help="Files or directories to scan. Defaults to api/controllers.",
|
||||
)
|
||||
parser.add_argument("--include-allowed", action="store_true", help="Print allowed flush()/commit() findings.")
|
||||
parser.add_argument("--include-suppressed", action="store_true", help="Print reasoned guard suppressions.")
|
||||
parser.add_argument("--summary-only", action="store_true", help="Print counts without per-finding details.")
|
||||
parser.add_argument("--json", action="store_true", help="Emit machine-readable JSON.")
|
||||
parser.add_argument(
|
||||
"--fail-on-findings",
|
||||
action="store_true",
|
||||
help="Treat reportable direct SQLAlchemy usage as a failure. By default this scanner is report-only.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def iter_python_files(paths: Iterable[Path]) -> Iterable[Path]:
|
||||
for path in paths:
|
||||
if path.is_file() and path.suffix in {".py", ".pyi"}:
|
||||
yield path
|
||||
elif path.is_dir():
|
||||
yield from sorted(child for child in path.rglob("*.py") if child.is_file())
|
||||
|
||||
|
||||
def display_path(path: Path) -> str:
|
||||
try:
|
||||
return str(path.relative_to(REPO_ROOT))
|
||||
except ValueError:
|
||||
return str(path)
|
||||
|
||||
|
||||
def compact_text(text: str) -> str:
|
||||
compact = " ".join(text.split())
|
||||
if len(compact) <= MAX_TEXT_LENGTH:
|
||||
return compact
|
||||
return f"{compact[: MAX_TEXT_LENGTH - 3]}..."
|
||||
|
||||
|
||||
def category_for_match(match: Match) -> str:
|
||||
text = match.text.strip()
|
||||
method_match = SESSION_METHOD_PATTERN.match(text)
|
||||
method = method_match.group("method") if method_match else None
|
||||
|
||||
if text.startswith("db.session."):
|
||||
return f"scoped-session.{method or 'call'}"
|
||||
if text.startswith("session."):
|
||||
return f"explicit-session.{method or 'call'}"
|
||||
if text.startswith(("Session(", "sessionmaker(")):
|
||||
return "session-factory"
|
||||
if text.startswith(("sa.", "sqlalchemy.", "db.")):
|
||||
return "sql-expression-qualified"
|
||||
return "sql-expression"
|
||||
|
||||
|
||||
def classification_for_match(match: Match) -> str:
|
||||
if controller_sqlalchemy_guard.is_allowed_session_boundary(match):
|
||||
return "allowed"
|
||||
if controller_sqlalchemy_guard.is_suppressed(match):
|
||||
return "suppressed"
|
||||
if controller_sqlalchemy_guard.is_flask_session_get(match):
|
||||
return "ignored"
|
||||
return "reportable"
|
||||
|
||||
|
||||
def findings_for_file(path: Path) -> list[Finding]:
|
||||
source = path.read_text(encoding="utf-8")
|
||||
findings: list[Finding] = []
|
||||
for match in run_ast_grep(source, rule=controller_sqlalchemy_guard.RULE_PATH):
|
||||
findings.append(
|
||||
Finding(
|
||||
classification=classification_for_match(match),
|
||||
category=category_for_match(match),
|
||||
file=display_path(path),
|
||||
line=match.line_number,
|
||||
text=compact_text(match.text),
|
||||
source_line=match.source_line.strip(),
|
||||
)
|
||||
)
|
||||
return findings
|
||||
|
||||
|
||||
def print_text_report(
|
||||
findings: Sequence[Finding],
|
||||
*,
|
||||
include_allowed: bool,
|
||||
include_suppressed: bool,
|
||||
summary_only: bool,
|
||||
) -> None:
|
||||
counts = Counter(finding.classification for finding in findings)
|
||||
reportable = [finding for finding in findings if finding.classification == "reportable"]
|
||||
category_counts = Counter(finding.category for finding in reportable)
|
||||
file_counts = Counter(finding.file for finding in reportable)
|
||||
sys.stdout.write(
|
||||
"Controller SQLAlchemy scan: "
|
||||
f"{counts['reportable']} reportable, "
|
||||
f"{counts['allowed']} allowed, "
|
||||
f"{counts['suppressed']} suppressed, "
|
||||
f"{counts['ignored']} ignored\n"
|
||||
)
|
||||
|
||||
if category_counts:
|
||||
sys.stdout.write("\nREPORTABLE BY CATEGORY:\n")
|
||||
for category, count in sorted(category_counts.items()):
|
||||
sys.stdout.write(f"- {category}: {count}\n")
|
||||
|
||||
if file_counts:
|
||||
sys.stdout.write("\nREPORTABLE BY FILE:\n")
|
||||
for file, count in sorted(file_counts.items()):
|
||||
sys.stdout.write(f"- {file}: {count}\n")
|
||||
|
||||
if summary_only:
|
||||
return
|
||||
|
||||
print_findings("REPORTABLE", reportable)
|
||||
if include_allowed:
|
||||
print_findings("ALLOWED", [finding for finding in findings if finding.classification == "allowed"])
|
||||
if include_suppressed:
|
||||
print_findings("SUPPRESSED", [finding for finding in findings if finding.classification == "suppressed"])
|
||||
|
||||
|
||||
def print_findings(title: str, findings: Sequence[Finding]) -> None:
|
||||
if not findings:
|
||||
return
|
||||
|
||||
sys.stdout.write(f"\n{title}:\n")
|
||||
for finding in findings:
|
||||
sys.stdout.write(f"- {finding.file}:{finding.line} [{finding.category}] {finding.text}\n")
|
||||
|
||||
|
||||
def jsonable_findings(
|
||||
findings: Sequence[Finding],
|
||||
*,
|
||||
include_allowed: bool,
|
||||
include_suppressed: bool,
|
||||
summary_only: bool,
|
||||
) -> dict[str, object]:
|
||||
visible = [
|
||||
finding
|
||||
for finding in findings
|
||||
if finding.classification == "reportable"
|
||||
or (include_allowed and finding.classification == "allowed")
|
||||
or (include_suppressed and finding.classification == "suppressed")
|
||||
]
|
||||
counts = Counter(finding.classification for finding in findings)
|
||||
return {
|
||||
"summary": dict(sorted(counts.items())),
|
||||
"findings": [] if summary_only else [asdict(finding) for finding in visible],
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
raw_paths = args.paths or [str(DEFAULT_CONTROLLER_DIR)]
|
||||
paths = [path if path.is_absolute() else REPO_ROOT / path for path in map(Path, raw_paths)]
|
||||
|
||||
findings: list[Finding] = []
|
||||
for path in iter_python_files(paths):
|
||||
findings.extend(findings_for_file(path.resolve()))
|
||||
findings.sort(key=lambda finding: (finding.file, finding.line, finding.category, finding.text))
|
||||
|
||||
if args.json:
|
||||
payload = jsonable_findings(
|
||||
findings,
|
||||
include_allowed=bool(args.include_allowed),
|
||||
include_suppressed=bool(args.include_suppressed),
|
||||
summary_only=bool(args.summary_only),
|
||||
)
|
||||
sys.stdout.write(f"{json.dumps(payload, indent=2, sort_keys=True)}\n")
|
||||
else:
|
||||
print_text_report(
|
||||
findings,
|
||||
include_allowed=bool(args.include_allowed),
|
||||
include_suppressed=bool(args.include_suppressed),
|
||||
summary_only=bool(args.summary_only),
|
||||
)
|
||||
|
||||
has_reportable = any(finding.classification == "reportable" for finding in findings)
|
||||
return int(bool(args.fail_on_findings) and has_reportable)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,98 @@
|
||||
# Dify SSE stress tests
|
||||
|
||||
This Locust suite exercises streaming `POST /v1/workflows/run` requests against a local Dify instance with a mock OpenAI server. It measures connection counts, event throughput, time to first event (TTFE), stream duration, inter-event latency, and failures.
|
||||
|
||||
Run the following commands from the repository root. Keep long-running services in separate terminals.
|
||||
|
||||
## Prepare the runtime
|
||||
|
||||
Complete the [backend setup](../../api/README.md), including middleware, migrations, the plugin daemon, and workers. For measurements, run the API with Gunicorn instead of the development server:
|
||||
|
||||
```bash
|
||||
uv run --project api gunicorn --chdir api \
|
||||
--bind 0.0.0.0:5001 --workers 4 --worker-class gevent \
|
||||
--timeout 120 --keep-alive 5 app:app
|
||||
```
|
||||
|
||||
The worker count is a starting configuration; tune it for the machine and record it with the results.
|
||||
|
||||
Start the mock provider before running setup:
|
||||
|
||||
```bash
|
||||
uv run --project api python scripts/stress-test/setup/mock_openai_server.py
|
||||
```
|
||||
|
||||
Then provision the benchmark app, plugin configuration, and API key:
|
||||
|
||||
```bash
|
||||
STRESS_TEST_ADMIN_EMAIL='your-admin@example.com' \
|
||||
STRESS_TEST_ADMIN_USERNAME='dify' \
|
||||
STRESS_TEST_ADMIN_PASSWORD='your-password' \
|
||||
uv run --project api python scripts/stress-test/setup_all.py
|
||||
```
|
||||
|
||||
Setup creates the first admin on an uninitialized instance or signs in with the supplied account on an existing instance. `STRESS_TEST_ADMIN_USERNAME` is used only for initial setup. It writes credentials and the app API key to `scripts/stress-test/setup/config/stress_test_state.json`; keep that local file private.
|
||||
|
||||
## Run
|
||||
|
||||
```bash
|
||||
./scripts/stress-test/run_locust_stress_test.sh
|
||||
```
|
||||
|
||||
The runner checks the API on port 5001, the mock provider on port 5004, and the saved API key. It prompts for headless mode or the Web UI at <http://localhost:8089>. Locust runs in its own environment through `uvx --from locust`.
|
||||
|
||||
For headless runs, the wrapper reads `users`, `spawn-rate`, and `run-time` from [`locust.conf`](locust.conf). To set options directly, invoke Locust from the repository root:
|
||||
|
||||
```bash
|
||||
uvx --from locust locust -f scripts/stress-test/sse_benchmark.py \
|
||||
--host http://localhost:5001 --users 50 --spawn-rate 5 \
|
||||
--run-time 1m --headless
|
||||
```
|
||||
|
||||
For the Web UI:
|
||||
|
||||
```bash
|
||||
uvx --from locust locust -f scripts/stress-test/sse_benchmark.py \
|
||||
--host http://localhost:5001 --web-port 8089
|
||||
```
|
||||
|
||||
The wrapper fixes the target to localhost; use the direct invocation to change the host. Direct invocations must set their own options or explicitly load `scripts/stress-test/locust.conf` with `--config`.
|
||||
|
||||
## Workload configuration
|
||||
|
||||
[`sse_benchmark.py`](sse_benchmark.py) reads these environment variables:
|
||||
|
||||
| Variable | Default | Purpose |
|
||||
| --- | --- | --- |
|
||||
| `WORKFLOW_PATH` | `/v1/workflows/run` | Request path |
|
||||
| `CONNECT_TIMEOUT` | `10` | Connection timeout in seconds |
|
||||
| `READ_TIMEOUT` | `60` | Stream read timeout in seconds |
|
||||
| `TERMINAL_EVENTS` | `workflow_finished,error` | Comma-separated terminal event names |
|
||||
| `QUESTIONS_FILE` | Empty | Text file with one nonempty question per line; otherwise uses built-in questions |
|
||||
| `WAIT_TIME` | `0` | `0` sends the next request immediately; any other value selects a random 1–3 second wait |
|
||||
|
||||
Requests pass the selected question as `inputs.question`. Keep this aligned with the benchmark workflow. A custom question file avoids editing the benchmark source.
|
||||
|
||||
## Reports and interpretation
|
||||
|
||||
Headless wrapper runs create `scripts/stress-test/reports/YYYYMMDD_HHMMSS/` containing:
|
||||
|
||||
- `locust_summary.txt`: console output.
|
||||
- `locust_report.html`: Locust report.
|
||||
- `locust_stats.csv` and `locust_stats_history.csv`: request statistics.
|
||||
- `sse_metrics_YYYYMMDD_HHMMSS.json`: custom SSE metrics.
|
||||
|
||||
Direct runs write custom SSE JSON under `scripts/stress-test/reports/`; request CSV or HTML explicitly with Locust options. Live SSE metrics update every five seconds.
|
||||
|
||||
Compare results with the same workflow, mock response behavior, hardware, worker configuration, and load profile. TTFE measures arrival of the first SSE event, not necessarily the first model token. Use the custom stream-duration metrics to assess stream completion separately from the standard Locust request timings. This mock-provider workload does not establish real-model latency or a universal production capacity threshold.
|
||||
|
||||
For failures, inspect API, worker, plugin-daemon, and mock-server logs. Check timeouts, CPU, memory, database connections, and client resource limits before changing concurrency or server settings.
|
||||
|
||||
## Maintenance
|
||||
|
||||
- Setup and fixture provisioning: `setup_all.py` and `setup/`.
|
||||
- Load defaults: `locust.conf`.
|
||||
- Stream parsing and metrics: `sse_benchmark.py`.
|
||||
- Service checks and report collection: `run_locust_stress_test.sh`.
|
||||
|
||||
`cleanup.py` removes local setup state and reports, not the resources created in Dify. Preserve reports you need before running it; it asks for confirmation only in an interactive terminal.
|
||||
+88
@@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from common import Logger
|
||||
|
||||
|
||||
def cleanup() -> None:
|
||||
"""Clean up all configuration files and reports created during setup and stress testing."""
|
||||
|
||||
log = Logger("Cleanup")
|
||||
log.header("Stress Test Cleanup")
|
||||
|
||||
config_dir = Path(__file__).parent / "setup" / "config"
|
||||
reports_dir = Path(__file__).parent / "reports"
|
||||
|
||||
dirs_to_clean = []
|
||||
if config_dir.exists():
|
||||
dirs_to_clean.append(config_dir)
|
||||
if reports_dir.exists():
|
||||
dirs_to_clean.append(reports_dir)
|
||||
|
||||
if not dirs_to_clean:
|
||||
log.success("No directories to clean. Everything is already clean.")
|
||||
return
|
||||
|
||||
log.info("Cleaning up stress test data...")
|
||||
log.info("This will remove:")
|
||||
for dir_path in dirs_to_clean:
|
||||
log.list_item(str(dir_path))
|
||||
|
||||
# List files that will be deleted
|
||||
log.separator()
|
||||
if config_dir.exists():
|
||||
config_files = list(config_dir.glob("*.json"))
|
||||
if config_files:
|
||||
log.info("Config files to be removed:")
|
||||
for file in config_files:
|
||||
log.list_item(file.name)
|
||||
|
||||
if reports_dir.exists():
|
||||
report_files = list(reports_dir.glob("*"))
|
||||
if report_files:
|
||||
log.info("Report files to be removed:")
|
||||
for file in report_files:
|
||||
log.list_item(file.name)
|
||||
|
||||
# Ask for confirmation if running interactively
|
||||
if sys.stdin.isatty():
|
||||
log.separator()
|
||||
log.warning("This action cannot be undone!")
|
||||
confirmation = input("Are you sure you want to remove all config and report files? (yes/no): ")
|
||||
|
||||
if confirmation.lower() not in ["yes", "y"]:
|
||||
log.error("Cleanup cancelled.")
|
||||
return
|
||||
|
||||
try:
|
||||
# Remove directories and all their contents
|
||||
for dir_path in dirs_to_clean:
|
||||
shutil.rmtree(dir_path)
|
||||
log.success(f"{dir_path.name} directory removed successfully!")
|
||||
|
||||
log.separator()
|
||||
log.info("To run the setup again, execute:")
|
||||
log.list_item("python setup_all.py")
|
||||
log.info("Or run scripts individually in this order:")
|
||||
log.list_item("python setup/mock_openai_server.py (in a separate terminal)")
|
||||
log.list_item("python setup/setup_admin.py")
|
||||
log.list_item("python setup/login_admin.py")
|
||||
log.list_item("python setup/install_openai_plugin.py")
|
||||
log.list_item("python setup/configure_openai_plugin.py")
|
||||
log.list_item("python setup/import_workflow_app.py")
|
||||
log.list_item("python setup/create_api_key.py")
|
||||
log.list_item("python setup/publish_workflow.py")
|
||||
log.list_item("python setup/run_workflow.py")
|
||||
|
||||
except PermissionError as e:
|
||||
log.error(f"Permission denied: {e}")
|
||||
log.info("Try running with appropriate permissions.")
|
||||
except Exception as e:
|
||||
log.error(f"An error occurred during cleanup: {e}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cleanup()
|
||||
@@ -0,0 +1,6 @@
|
||||
"""Common utilities for Dify benchmark suite."""
|
||||
|
||||
from .config_helper import config_helper
|
||||
from .logger_helper import Logger, ProgressLogger
|
||||
|
||||
__all__ = ["Logger", "ProgressLogger", "config_helper"]
|
||||
@@ -0,0 +1,313 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import NotRequired, TypedDict
|
||||
|
||||
|
||||
class AdminConfig(TypedDict):
|
||||
"""Configuration for admin section."""
|
||||
|
||||
email: str
|
||||
username: str
|
||||
password: str
|
||||
|
||||
|
||||
class AuthConfig(TypedDict):
|
||||
"""Configuration for authentication section."""
|
||||
|
||||
access_token: str
|
||||
refresh_token: NotRequired[str]
|
||||
csrf_token: NotRequired[str]
|
||||
expires_at: NotRequired[int]
|
||||
|
||||
|
||||
class AppConfig(TypedDict):
|
||||
"""Configuration for app section."""
|
||||
|
||||
app_id: str
|
||||
app_name: NotRequired[str]
|
||||
description: NotRequired[str]
|
||||
|
||||
|
||||
class ApiKeyConfig(TypedDict):
|
||||
"""Configuration for API key section."""
|
||||
|
||||
token: str
|
||||
key_name: NotRequired[str]
|
||||
expires_at: NotRequired[int]
|
||||
|
||||
|
||||
class StressTestState(TypedDict):
|
||||
"""Complete stress test state structure."""
|
||||
|
||||
admin: NotRequired[AdminConfig]
|
||||
auth: NotRequired[AuthConfig]
|
||||
app: NotRequired[AppConfig]
|
||||
api_key: NotRequired[ApiKeyConfig]
|
||||
|
||||
|
||||
class ConfigHelper:
|
||||
_LEGACY_SECTION_MAP = {
|
||||
"admin_config": "admin",
|
||||
"token_config": "auth",
|
||||
"app_config": "app",
|
||||
"api_key_config": "api_key",
|
||||
}
|
||||
|
||||
"""Helper class for reading and writing configuration files."""
|
||||
|
||||
def __init__(self, base_dir: Path | None = None):
|
||||
"""Initialize ConfigHelper with base directory.
|
||||
|
||||
Args:
|
||||
base_dir: Base directory for config files. If None, uses setup/config
|
||||
"""
|
||||
if base_dir is None:
|
||||
# Default to config directory in setup folder
|
||||
base_dir = Path(__file__).parent.parent / "setup" / "config"
|
||||
self.base_dir = base_dir
|
||||
self.state_file = "stress_test_state.json"
|
||||
|
||||
def ensure_config_dir(self) -> None:
|
||||
"""Ensure the config directory exists."""
|
||||
self.base_dir.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
def get_config_path(self, filename: str) -> Path:
|
||||
"""Get the full path for a config file.
|
||||
|
||||
Args:
|
||||
filename: Name of the config file (e.g., 'admin_config.json')
|
||||
|
||||
Returns:
|
||||
Full path to the config file
|
||||
"""
|
||||
if not filename.endswith(".json"):
|
||||
filename += ".json"
|
||||
return self.base_dir / filename
|
||||
|
||||
def read_config[T](self, filename: str) -> T | None:
|
||||
"""Read a configuration file with generic return type.
|
||||
|
||||
DEPRECATED: Use read_state() or get_state_section() for new code.
|
||||
This method provides backward compatibility.
|
||||
|
||||
Args:
|
||||
filename: Name of the config file to read
|
||||
|
||||
Returns:
|
||||
Configuration data of type T, or None if file doesn't exist
|
||||
"""
|
||||
# Provide backward compatibility for old config names
|
||||
if filename in self._LEGACY_SECTION_MAP:
|
||||
section_data = self.get_state_section(self._LEGACY_SECTION_MAP[filename])
|
||||
return section_data # type: ignore
|
||||
|
||||
config_path = self.get_config_path(filename)
|
||||
|
||||
if not config_path.exists():
|
||||
return None
|
||||
|
||||
try:
|
||||
with open(config_path) as f:
|
||||
return json.load(f) # type: ignore
|
||||
except (OSError, json.JSONDecodeError) as e:
|
||||
print(f"❌ Error reading {filename}: {e}")
|
||||
return None
|
||||
|
||||
def write_config[T](self, filename: str, data: T) -> bool:
|
||||
"""Write data to a configuration file.
|
||||
|
||||
DEPRECATED: Use write_state() or update_state_section() for new code.
|
||||
This method provides backward compatibility.
|
||||
|
||||
Args:
|
||||
filename: Name of the config file to write
|
||||
data: Data to save (must be JSON serializable)
|
||||
|
||||
Returns:
|
||||
True if successful, False otherwise
|
||||
"""
|
||||
# Provide backward compatibility for old config names
|
||||
if filename in self._LEGACY_SECTION_MAP:
|
||||
return self.update_state_section(
|
||||
self._LEGACY_SECTION_MAP[filename],
|
||||
data, # type: ignore
|
||||
)
|
||||
|
||||
self.ensure_config_dir()
|
||||
config_path = self.get_config_path(filename)
|
||||
|
||||
try:
|
||||
with open(config_path, "w") as f:
|
||||
json.dump(data, f, indent=2) # type: ignore
|
||||
return True
|
||||
except OSError as e:
|
||||
print(f"❌ Error writing {filename}: {e}")
|
||||
return False
|
||||
|
||||
def config_exists(self, filename: str) -> bool:
|
||||
"""Check if a config file exists.
|
||||
|
||||
Args:
|
||||
filename: Name of the config file to check
|
||||
|
||||
Returns:
|
||||
True if file exists, False otherwise
|
||||
"""
|
||||
return self.get_config_path(filename).exists()
|
||||
|
||||
def delete_config(self, filename: str) -> bool:
|
||||
"""Delete a configuration file.
|
||||
|
||||
Args:
|
||||
filename: Name of the config file to delete
|
||||
|
||||
Returns:
|
||||
True if successful, False otherwise
|
||||
"""
|
||||
config_path = self.get_config_path(filename)
|
||||
|
||||
if not config_path.exists():
|
||||
return True # Already doesn't exist
|
||||
|
||||
try:
|
||||
config_path.unlink()
|
||||
return True
|
||||
except OSError as e:
|
||||
print(f"❌ Error deleting {filename}: {e}")
|
||||
return False
|
||||
|
||||
def read_state(self) -> StressTestState | None:
|
||||
"""Read the entire stress test state.
|
||||
|
||||
Returns:
|
||||
Dictionary containing all state data, or None if file doesn't exist
|
||||
"""
|
||||
state_path = self.get_config_path(self.state_file)
|
||||
if not state_path.exists():
|
||||
return None
|
||||
|
||||
try:
|
||||
with open(state_path) as f:
|
||||
data = json.load(f)
|
||||
# Validate basic structure
|
||||
if not isinstance(data, dict):
|
||||
print(f"❌ Invalid state format in {self.state_file}")
|
||||
return None
|
||||
return data # type: ignore
|
||||
except (OSError, json.JSONDecodeError) as e:
|
||||
print(f"❌ Error reading {self.state_file}: {e}")
|
||||
return None
|
||||
|
||||
def write_state(self, data: StressTestState) -> bool:
|
||||
"""Write the entire stress test state.
|
||||
|
||||
Args:
|
||||
data: Dictionary containing all state data to save
|
||||
|
||||
Returns:
|
||||
True if successful, False otherwise
|
||||
"""
|
||||
self.ensure_config_dir()
|
||||
state_path = self.get_config_path(self.state_file)
|
||||
|
||||
try:
|
||||
with open(state_path, "w") as f:
|
||||
json.dump(data, f, indent=2)
|
||||
return True
|
||||
except OSError as e:
|
||||
print(f"❌ Error writing {self.state_file}: {e}")
|
||||
return False
|
||||
|
||||
def update_state_section[T](self, section: str, data: T) -> bool:
|
||||
"""Update a specific section of the stress test state.
|
||||
|
||||
Args:
|
||||
section: Name of the section to update (e.g., 'admin', 'auth', 'app', 'api_key')
|
||||
data: Section data to save
|
||||
|
||||
Returns:
|
||||
True if successful, False otherwise
|
||||
"""
|
||||
state = self.read_state() or {}
|
||||
state[section] = data # type: ignore
|
||||
return self.write_state(state) # type: ignore
|
||||
|
||||
def get_state_section[T](self, section: str) -> T | None:
|
||||
"""Get a specific section from the stress test state.
|
||||
|
||||
Args:
|
||||
section: Name of the section to get (e.g., 'admin', 'auth', 'app', 'api_key')
|
||||
|
||||
Returns:
|
||||
Section data of type T, or None if not found
|
||||
"""
|
||||
state = self.read_state()
|
||||
if state:
|
||||
return state.get(section) # type: ignore
|
||||
return None
|
||||
|
||||
def get_token(self) -> str | None:
|
||||
"""Get the access token from auth section.
|
||||
|
||||
Returns:
|
||||
Access token string or None if not found
|
||||
"""
|
||||
auth = self.get_state_section("auth")
|
||||
if auth:
|
||||
return auth.get("access_token")
|
||||
return None
|
||||
|
||||
def get_csrf_token(self) -> str | None:
|
||||
"""Get the CSRF token from auth section."""
|
||||
auth = self.get_state_section("auth")
|
||||
if auth:
|
||||
return auth.get("csrf_token")
|
||||
return None
|
||||
|
||||
def get_app_id(self) -> str | None:
|
||||
"""Get the app ID from app section.
|
||||
|
||||
Returns:
|
||||
App ID string or None if not found
|
||||
"""
|
||||
app = self.get_state_section("app")
|
||||
if app:
|
||||
return app.get("app_id")
|
||||
return None
|
||||
|
||||
def get_api_key(self) -> str | None:
|
||||
"""Get the API key token from api_key section.
|
||||
|
||||
Returns:
|
||||
API key token string or None if not found
|
||||
"""
|
||||
api_key = self.get_state_section("api_key")
|
||||
if api_key:
|
||||
return api_key.get("token")
|
||||
return None
|
||||
|
||||
def console_auth_headers(self) -> dict[str, str]:
|
||||
access_token = self.get_token()
|
||||
csrf_token = self.get_csrf_token()
|
||||
headers: dict[str, str] = {}
|
||||
if access_token:
|
||||
headers["authorization"] = f"Bearer {access_token}"
|
||||
if csrf_token:
|
||||
headers["X-CSRF-Token"] = csrf_token
|
||||
return headers
|
||||
|
||||
def console_auth_cookies(self) -> dict[str, str]:
|
||||
access_token = self.get_token()
|
||||
csrf_token = self.get_csrf_token()
|
||||
cookies = {"locale": "en-US"}
|
||||
if access_token:
|
||||
cookies["access_token"] = access_token
|
||||
if csrf_token:
|
||||
cookies["csrf_token"] = csrf_token
|
||||
return cookies
|
||||
|
||||
|
||||
# Create a default instance for convenience
|
||||
config_helper = ConfigHelper()
|
||||
@@ -0,0 +1,218 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import sys
|
||||
import time
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class LogLevel(Enum):
|
||||
"""Log levels with associated colors and symbols."""
|
||||
|
||||
DEBUG = ("🔍", "\033[90m") # Gray
|
||||
INFO = ("ℹ️ ", "\033[94m") # Blue
|
||||
SUCCESS = ("✅", "\033[92m") # Green
|
||||
WARNING = ("⚠️ ", "\033[93m") # Yellow
|
||||
ERROR = ("❌", "\033[91m") # Red
|
||||
STEP = ("🚀", "\033[96m") # Cyan
|
||||
PROGRESS = ("📋", "\033[95m") # Magenta
|
||||
|
||||
|
||||
class Logger:
|
||||
"""Logger class for formatted console output."""
|
||||
|
||||
def __init__(self, name: str | None = None, use_colors: bool = True):
|
||||
"""Initialize logger.
|
||||
|
||||
Args:
|
||||
name: Optional name for the logger (e.g., script name)
|
||||
use_colors: Whether to use ANSI color codes
|
||||
"""
|
||||
self.name = name
|
||||
self.use_colors = use_colors and sys.stdout.isatty()
|
||||
self._reset_color = "\033[0m" if self.use_colors else ""
|
||||
|
||||
def _format_message(self, level: LogLevel, message: str, indent: int = 0) -> str:
|
||||
"""Format a log message with level, color, and indentation.
|
||||
|
||||
Args:
|
||||
level: Log level
|
||||
message: Message to log
|
||||
indent: Number of spaces to indent
|
||||
|
||||
Returns:
|
||||
Formatted message string
|
||||
"""
|
||||
symbol, color = level.value
|
||||
color = color if self.use_colors else ""
|
||||
reset = self._reset_color
|
||||
|
||||
prefix = " " * indent
|
||||
|
||||
if self.name and level in [LogLevel.STEP, LogLevel.ERROR]:
|
||||
return f"{prefix}{color}{symbol} [{self.name}] {message}{reset}"
|
||||
else:
|
||||
return f"{prefix}{color}{symbol} {message}{reset}"
|
||||
|
||||
def debug(self, message: str, indent: int = 0) -> None:
|
||||
"""Log debug message."""
|
||||
print(self._format_message(LogLevel.DEBUG, message, indent))
|
||||
|
||||
def info(self, message: str, indent: int = 0) -> None:
|
||||
"""Log info message."""
|
||||
print(self._format_message(LogLevel.INFO, message, indent))
|
||||
|
||||
def success(self, message: str, indent: int = 0) -> None:
|
||||
"""Log success message."""
|
||||
print(self._format_message(LogLevel.SUCCESS, message, indent))
|
||||
|
||||
def warning(self, message: str, indent: int = 0) -> None:
|
||||
"""Log warning message."""
|
||||
print(self._format_message(LogLevel.WARNING, message, indent))
|
||||
|
||||
def error(self, message: str, indent: int = 0) -> None:
|
||||
"""Log error message."""
|
||||
print(self._format_message(LogLevel.ERROR, message, indent), file=sys.stderr)
|
||||
|
||||
def step(self, message: str, indent: int = 0) -> None:
|
||||
"""Log a step in a process."""
|
||||
print(self._format_message(LogLevel.STEP, message, indent))
|
||||
|
||||
def progress(self, message: str, indent: int = 0) -> None:
|
||||
"""Log progress information."""
|
||||
print(self._format_message(LogLevel.PROGRESS, message, indent))
|
||||
|
||||
def separator(self, char: str = "-", length: int = 60) -> None:
|
||||
"""Print a separator line."""
|
||||
print(char * length)
|
||||
|
||||
def header(self, title: str, width: int = 60) -> None:
|
||||
"""Print a formatted header."""
|
||||
if self.use_colors:
|
||||
print(f"\n\033[1m{'=' * width}\033[0m") # Bold
|
||||
print(f"\033[1m{title.center(width)}\033[0m")
|
||||
print(f"\033[1m{'=' * width}\033[0m\n")
|
||||
else:
|
||||
print(f"\n{'=' * width}")
|
||||
print(title.center(width))
|
||||
print(f"{'=' * width}\n")
|
||||
|
||||
def box(self, title: str, width: int = 60) -> None:
|
||||
"""Print a title in a box."""
|
||||
border = "═" * (width - 2)
|
||||
if self.use_colors:
|
||||
print(f"\033[1m╔{border}╗\033[0m")
|
||||
print(f"\033[1m║{title.center(width - 2)}║\033[0m")
|
||||
print(f"\033[1m╚{border}╝\033[0m")
|
||||
else:
|
||||
print(f"╔{border}╗")
|
||||
print(f"║{title.center(width - 2)}║")
|
||||
print(f"╚{border}╝")
|
||||
|
||||
def list_item(self, item: str, indent: int = 2) -> None:
|
||||
"""Print a list item."""
|
||||
prefix = " " * indent
|
||||
print(f"{prefix}• {item}")
|
||||
|
||||
def key_value(self, key: str, value: str, indent: int = 2) -> None:
|
||||
"""Print a key-value pair."""
|
||||
prefix = " " * indent
|
||||
if self.use_colors:
|
||||
print(f"{prefix}\033[1m{key}:\033[0m {value}")
|
||||
else:
|
||||
print(f"{prefix}{key}: {value}")
|
||||
|
||||
def spinner_start(self, message: str) -> None:
|
||||
"""Start a spinner (simple implementation)."""
|
||||
sys.stdout.write(f"\r{message}... ")
|
||||
sys.stdout.flush()
|
||||
|
||||
def spinner_stop(self, success: bool = True, message: str | None = None) -> None:
|
||||
"""Stop the spinner and show result."""
|
||||
if success:
|
||||
symbol = "✅" if message else "Done"
|
||||
sys.stdout.write(f"\r{symbol} {message or ''}\n")
|
||||
else:
|
||||
symbol = "❌" if message else "Failed"
|
||||
sys.stdout.write(f"\r{symbol} {message or ''}\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
class ProgressLogger:
|
||||
"""Logger for tracking progress through multiple steps."""
|
||||
|
||||
def __init__(self, total_steps: int, logger: Logger | None = None):
|
||||
"""Initialize progress logger.
|
||||
|
||||
Args:
|
||||
total_steps: Total number of steps
|
||||
logger: Logger instance to use (creates new if None)
|
||||
"""
|
||||
self.total_steps = total_steps
|
||||
self.current_step = 0
|
||||
self.logger = logger or Logger()
|
||||
self.start_time = time.time()
|
||||
|
||||
def next_step(self, description: str) -> None:
|
||||
"""Move to next step and log it."""
|
||||
self.current_step += 1
|
||||
elapsed = time.time() - self.start_time
|
||||
|
||||
if self.logger.use_colors:
|
||||
progress_bar = self._create_progress_bar()
|
||||
print(f"\n\033[1m[Step {self.current_step}/{self.total_steps}]\033[0m {progress_bar}")
|
||||
self.logger.step(f"{description} (Elapsed: {elapsed:.1f}s)")
|
||||
else:
|
||||
print(f"\n[Step {self.current_step}/{self.total_steps}]")
|
||||
self.logger.step(f"{description} (Elapsed: {elapsed:.1f}s)")
|
||||
|
||||
def _create_progress_bar(self, width: int = 20) -> str:
|
||||
"""Create a simple progress bar."""
|
||||
filled = int(width * self.current_step / self.total_steps)
|
||||
bar = "█" * filled + "░" * (width - filled)
|
||||
percentage = int(100 * self.current_step / self.total_steps)
|
||||
return f"[{bar}] {percentage}%"
|
||||
|
||||
def complete(self) -> None:
|
||||
"""Mark progress as complete."""
|
||||
elapsed = time.time() - self.start_time
|
||||
self.logger.success(f"All steps completed! Total time: {elapsed:.1f}s")
|
||||
|
||||
|
||||
# Create default logger instance
|
||||
logger = Logger()
|
||||
|
||||
|
||||
# Convenience functions using default logger
|
||||
def debug(message: str, indent: int = 0) -> None:
|
||||
"""Log debug message using default logger."""
|
||||
logger.debug(message, indent)
|
||||
|
||||
|
||||
def info(message: str, indent: int = 0) -> None:
|
||||
"""Log info message using default logger."""
|
||||
logger.info(message, indent)
|
||||
|
||||
|
||||
def success(message: str, indent: int = 0) -> None:
|
||||
"""Log success message using default logger."""
|
||||
logger.success(message, indent)
|
||||
|
||||
|
||||
def warning(message: str, indent: int = 0) -> None:
|
||||
"""Log warning message using default logger."""
|
||||
logger.warning(message, indent)
|
||||
|
||||
|
||||
def error(message: str, indent: int = 0) -> None:
|
||||
"""Log error message using default logger."""
|
||||
logger.error(message, indent)
|
||||
|
||||
|
||||
def step(message: str, indent: int = 0) -> None:
|
||||
"""Log step using default logger."""
|
||||
logger.step(message, indent)
|
||||
|
||||
|
||||
def progress(message: str, indent: int = 0) -> None:
|
||||
"""Log progress using default logger."""
|
||||
logger.progress(message, indent)
|
||||
@@ -0,0 +1,37 @@
|
||||
# Locust configuration file for Dify SSE benchmark
|
||||
|
||||
# Target host
|
||||
host = http://localhost:5001
|
||||
|
||||
# Number of users to simulate
|
||||
users = 10
|
||||
|
||||
# Spawn rate (users per second)
|
||||
spawn-rate = 2
|
||||
|
||||
# Run time (use format like 30s, 5m, 1h)
|
||||
run-time = 1m
|
||||
|
||||
# Locustfile to use
|
||||
locustfile = scripts/stress-test/sse_benchmark.py
|
||||
|
||||
# Headless mode (no web UI)
|
||||
headless = true
|
||||
|
||||
# Print stats in the console
|
||||
print-stats = true
|
||||
|
||||
# Only print summary stats
|
||||
only-summary = false
|
||||
|
||||
# Reset statistics after ramp-up
|
||||
reset-stats = false
|
||||
|
||||
# Log level
|
||||
loglevel = INFO
|
||||
|
||||
# CSV output (uncomment to enable)
|
||||
# csv = reports/locust_results
|
||||
|
||||
# HTML report (uncomment to enable)
|
||||
# html = reports/locust_report.html
|
||||
@@ -0,0 +1,234 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Run Dify SSE Stress Test using Locust
|
||||
|
||||
set -e
|
||||
|
||||
# Get the directory where this script is located
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
# Go to project root first, then to script dir
|
||||
PROJECT_ROOT="$( cd "${SCRIPT_DIR}/../.." && pwd )"
|
||||
cd "${PROJECT_ROOT}"
|
||||
STRESS_TEST_DIR="scripts/stress-test"
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
CYAN='\033[0;36m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Configuration
|
||||
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
|
||||
START_EPOCH=$(date +%s)
|
||||
REPORT_ROOT="${STRESS_TEST_DIR}/reports"
|
||||
REPORT_DIR="${REPORT_ROOT}/${TIMESTAMP}"
|
||||
CSV_PREFIX="${REPORT_DIR}/locust"
|
||||
HTML_REPORT="${REPORT_DIR}/locust_report.html"
|
||||
SUMMARY_REPORT="${REPORT_DIR}/locust_summary.txt"
|
||||
|
||||
# Create reports directory if it doesn't exist
|
||||
mkdir -p "${REPORT_DIR}"
|
||||
|
||||
echo -e "${BLUE}╔════════════════════════════════════════════════════════════════╗${NC}"
|
||||
echo -e "${BLUE}║ DIFY SSE WORKFLOW STRESS TEST (LOCUST) ║${NC}"
|
||||
echo -e "${BLUE}╚════════════════════════════════════════════════════════════════╝${NC}"
|
||||
echo
|
||||
|
||||
# Check if services are running
|
||||
echo -e "${YELLOW}Checking services...${NC}"
|
||||
|
||||
# Check Dify API
|
||||
if curl -s -f http://localhost:5001/health > /dev/null 2>&1; then
|
||||
echo -e "${GREEN}✓ Dify API is running${NC}"
|
||||
|
||||
# Warn if running in debug mode (check for werkzeug in process)
|
||||
if ps aux | grep -v grep | grep -q "werkzeug.*5001\|flask.*run.*5001"; then
|
||||
echo -e "${YELLOW}⚠ WARNING: API appears to be running in debug mode (Flask development server)${NC}"
|
||||
echo -e "${YELLOW} This will give inaccurate benchmark results!${NC}"
|
||||
echo -e "${YELLOW} For accurate benchmarking, restart with Gunicorn:${NC}"
|
||||
echo -e "${CYAN} cd api && uv run gunicorn --bind 0.0.0.0:5001 --workers 4 --worker-class gevent app:app${NC}"
|
||||
echo
|
||||
echo -n "Continue anyway? (not recommended) [y/N]: "
|
||||
read -t 10 continue_debug || continue_debug="n"
|
||||
if [ "$continue_debug" != "y" ] && [ "$continue_debug" != "Y" ]; then
|
||||
echo -e "${RED}Benchmark cancelled. Please restart API with Gunicorn.${NC}"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
else
|
||||
echo -e "${RED}✗ Dify API is not running on port 5001${NC}"
|
||||
echo -e "${YELLOW} Start it with Gunicorn for accurate benchmarking:${NC}"
|
||||
echo -e "${CYAN} cd api && uv run gunicorn --bind 0.0.0.0:5001 --workers 4 --worker-class gevent app:app${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check Mock OpenAI server
|
||||
if curl -s -f http://localhost:5004/v1/models > /dev/null 2>&1; then
|
||||
echo -e "${GREEN}✓ Mock OpenAI server is running${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Mock OpenAI server is not running on port 5004${NC}"
|
||||
echo -e "${YELLOW} Start it with: python scripts/stress-test/setup/mock_openai_server.py${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check API token exists
|
||||
if [ ! -f "${STRESS_TEST_DIR}/setup/config/stress_test_state.json" ]; then
|
||||
echo -e "${RED}✗ Stress test configuration not found${NC}"
|
||||
echo -e "${YELLOW} Run setup first: python scripts/stress-test/setup_all.py${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
API_TOKEN=$(python3 -c "import json; state = json.load(open('${STRESS_TEST_DIR}/setup/config/stress_test_state.json')); print(state.get('api_key', {}).get('token', ''))" 2>/dev/null)
|
||||
if [ -z "$API_TOKEN" ]; then
|
||||
echo -e "${RED}✗ Failed to read API token from stress test state${NC}"
|
||||
exit 1
|
||||
fi
|
||||
echo -e "${GREEN}✓ API token found: ${API_TOKEN:0:10}...${NC}"
|
||||
|
||||
echo
|
||||
echo -e "${CYAN}═══════════════════════════════════════════════════════════════${NC}"
|
||||
echo -e "${CYAN} STRESS TEST PARAMETERS ${NC}"
|
||||
echo -e "${CYAN}═══════════════════════════════════════════════════════════════${NC}"
|
||||
|
||||
# Parse configuration
|
||||
USERS=$(grep "^users" ${STRESS_TEST_DIR}/locust.conf | cut -d'=' -f2 | tr -d ' ')
|
||||
SPAWN_RATE=$(grep "^spawn-rate" ${STRESS_TEST_DIR}/locust.conf | cut -d'=' -f2 | tr -d ' ')
|
||||
RUN_TIME=$(grep "^run-time" ${STRESS_TEST_DIR}/locust.conf | cut -d'=' -f2 | tr -d ' ')
|
||||
|
||||
echo -e " ${YELLOW}Users:${NC} $USERS concurrent users"
|
||||
echo -e " ${YELLOW}Spawn Rate:${NC} $SPAWN_RATE users/second"
|
||||
echo -e " ${YELLOW}Duration:${NC} $RUN_TIME"
|
||||
echo -e " ${YELLOW}Mode:${NC} SSE Streaming"
|
||||
echo
|
||||
|
||||
# Ask user for run mode
|
||||
echo -e "${YELLOW}Select run mode:${NC}"
|
||||
echo " 1) Headless (CLI only) - Default"
|
||||
echo " 2) Web UI (http://localhost:8089)"
|
||||
echo -n "Choice [1]: "
|
||||
read -t 10 choice || choice="1"
|
||||
echo
|
||||
|
||||
# Use SSE stress test script
|
||||
LOCUST_SCRIPT="${STRESS_TEST_DIR}/sse_benchmark.py"
|
||||
LOCUST_RUN=(uvx --from locust locust)
|
||||
|
||||
# Prepare Locust command
|
||||
if [ "$choice" = "2" ]; then
|
||||
echo -e "${BLUE}Starting Locust with Web UI...${NC}"
|
||||
echo -e "${YELLOW}Access the web interface at: ${CYAN}http://localhost:8089${NC}"
|
||||
echo
|
||||
|
||||
# Run with web UI
|
||||
"${LOCUST_RUN[@]}" \
|
||||
-f ${LOCUST_SCRIPT} \
|
||||
--host http://localhost:5001 \
|
||||
--web-port 8089
|
||||
else
|
||||
echo -e "${BLUE}Starting stress test in headless mode...${NC}"
|
||||
echo
|
||||
|
||||
# Run in headless mode with CSV output
|
||||
"${LOCUST_RUN[@]}" \
|
||||
-f ${LOCUST_SCRIPT} \
|
||||
--host http://localhost:5001 \
|
||||
--users $USERS \
|
||||
--spawn-rate $SPAWN_RATE \
|
||||
--run-time $RUN_TIME \
|
||||
--headless \
|
||||
--print-stats \
|
||||
--csv=$CSV_PREFIX \
|
||||
--html=$HTML_REPORT \
|
||||
2>&1 | tee $SUMMARY_REPORT
|
||||
SSE_METRICS_REPORT=$(python3 - <<EOF
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
|
||||
reports = [
|
||||
p for p in Path("${REPORT_ROOT}").glob("sse_metrics_*.json")
|
||||
if p.stat().st_mtime >= ${START_EPOCH}
|
||||
]
|
||||
if reports:
|
||||
source = max(reports, key=lambda p: p.stat().st_mtime)
|
||||
target = Path("${REPORT_DIR}") / source.name
|
||||
if source != target:
|
||||
shutil.move(str(source), str(target))
|
||||
print(target)
|
||||
EOF
|
||||
)
|
||||
|
||||
echo
|
||||
echo -e "${GREEN}═══════════════════════════════════════════════════════════════${NC}"
|
||||
echo -e "${GREEN} STRESS TEST COMPLETE ${NC}"
|
||||
echo -e "${GREEN}═══════════════════════════════════════════════════════════════${NC}"
|
||||
echo
|
||||
echo -e "${BLUE}Reports generated:${NC}"
|
||||
echo -e " ${YELLOW}Summary:${NC} $SUMMARY_REPORT"
|
||||
echo -e " ${YELLOW}HTML Report:${NC} $HTML_REPORT"
|
||||
echo -e " ${YELLOW}CSV Stats:${NC} ${CSV_PREFIX}_stats.csv"
|
||||
echo -e " ${YELLOW}CSV History:${NC} ${CSV_PREFIX}_stats_history.csv"
|
||||
if [ -n "$SSE_METRICS_REPORT" ]; then
|
||||
echo -e " ${YELLOW}SSE Metrics:${NC} $SSE_METRICS_REPORT"
|
||||
else
|
||||
echo -e " ${YELLOW}SSE Metrics:${NC} not found"
|
||||
fi
|
||||
echo
|
||||
echo -e "${CYAN}View HTML report:${NC}"
|
||||
echo " open $HTML_REPORT # macOS"
|
||||
echo " xdg-open $HTML_REPORT # Linux"
|
||||
echo
|
||||
|
||||
# Parse and display key metrics
|
||||
echo -e "${CYAN}═══════════════════════════════════════════════════════════════${NC}"
|
||||
echo -e "${CYAN} KEY METRICS ${NC}"
|
||||
echo -e "${CYAN}═══════════════════════════════════════════════════════════════${NC}"
|
||||
|
||||
if [ -f "${CSV_PREFIX}_stats.csv" ]; then
|
||||
python3 - <<EOF
|
||||
import csv
|
||||
import sys
|
||||
|
||||
csv_file = "${CSV_PREFIX}_stats.csv"
|
||||
|
||||
try:
|
||||
with open(csv_file, 'r') as f:
|
||||
reader = csv.DictReader(f)
|
||||
rows = list(reader)
|
||||
|
||||
# Find the aggregated row
|
||||
for row in rows:
|
||||
if row.get('Name') == 'Aggregated':
|
||||
print(f" Total Requests: {row.get('Request Count', 'N/A')}")
|
||||
print(f" Failure Rate: {row.get('Failure Count', '0')} failures")
|
||||
print(f" Median Response: {row.get('Median Response Time', 'N/A')} ms")
|
||||
print(f" 95%ile Response: {row.get('95%', 'N/A')} ms")
|
||||
print(f" 99%ile Response: {row.get('99%', 'N/A')} ms")
|
||||
print(f" RPS: {row.get('Requests/s', 'N/A')}")
|
||||
break
|
||||
|
||||
print()
|
||||
print("SSE Streaming Metrics:")
|
||||
import json
|
||||
sse_metrics_report = "${SSE_METRICS_REPORT}"
|
||||
if sse_metrics_report:
|
||||
with open(sse_metrics_report, 'r') as metrics_file:
|
||||
metrics = json.load(metrics_file).get("metrics", {})
|
||||
print(f" Total Connections: {metrics.get('total_connections', 'N/A')}")
|
||||
print(f" Total Events: {metrics.get('total_events', 'N/A')}")
|
||||
print(f" Connection Rate: {metrics.get('overall_conn_rate', 0):.2f} conn/s")
|
||||
print(f" Event Throughput: {metrics.get('overall_event_rate', 0):.2f} events/s")
|
||||
print(f" TTFE: {metrics.get('ttfe_p50', 0):.1f} ms p50 / {metrics.get('ttfe_p95', 0):.1f} ms p95")
|
||||
print(f" Stream Duration: {metrics.get('stream_duration_p50', 0):.1f} ms p50 / {metrics.get('stream_duration_p95', 0):.1f} ms p95")
|
||||
print(f" Inter-event Latency: {metrics.get('inter_event_latency_p50', 0):.1f} ms p50 / {metrics.get('inter_event_latency_p95', 0):.1f} ms p95")
|
||||
else:
|
||||
print(" No SSE metrics JSON report found for this run")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Could not parse metrics: {e}")
|
||||
EOF
|
||||
fi
|
||||
|
||||
echo -e "${CYAN}═══════════════════════════════════════════════════════════════${NC}"
|
||||
fi
|
||||
@@ -0,0 +1,104 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.append(str(Path(__file__).parent.parent))
|
||||
|
||||
import httpx
|
||||
from common import Logger, config_helper
|
||||
|
||||
|
||||
def configure_openai_plugin() -> bool:
|
||||
"""Configure OpenAI plugin with mock server credentials."""
|
||||
|
||||
log = Logger("ConfigPlugin")
|
||||
log.header("Configuring OpenAI Plugin")
|
||||
|
||||
# Read token from config
|
||||
access_token = config_helper.get_token()
|
||||
if not access_token:
|
||||
log.error("No access token found in config")
|
||||
log.info("Please run login_admin.py first to get access token")
|
||||
return False
|
||||
|
||||
log.step("Configuring OpenAI plugin with mock server...")
|
||||
|
||||
# API endpoint for plugin configuration
|
||||
base_url = "http://localhost:5001"
|
||||
config_endpoint = f"{base_url}/console/api/workspaces/current/model-providers/langgenius/openai/openai/credentials"
|
||||
|
||||
# Configuration payload with mock server
|
||||
config_payload = {
|
||||
"credentials": {
|
||||
"openai_api_key": "apikey",
|
||||
"openai_organization": None,
|
||||
"openai_api_base": "http://host.docker.internal:5004",
|
||||
}
|
||||
}
|
||||
|
||||
headers = {
|
||||
"Accept": "*/*",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
"DNT": "1",
|
||||
"Origin": "http://localhost:3000",
|
||||
"Pragma": "no-cache",
|
||||
"Referer": "http://localhost:3000/",
|
||||
"Sec-Fetch-Dest": "empty",
|
||||
"Sec-Fetch-Mode": "cors",
|
||||
"Sec-Fetch-Site": "same-site",
|
||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
|
||||
**config_helper.console_auth_headers(),
|
||||
"content-type": "application/json",
|
||||
"sec-ch-ua": '"Not;A=Brand";v="99", "Google Chrome";v="139", "Chromium";v="139"',
|
||||
"sec-ch-ua-mobile": "?0",
|
||||
"sec-ch-ua-platform": '"macOS"',
|
||||
}
|
||||
|
||||
cookies = config_helper.console_auth_cookies()
|
||||
|
||||
try:
|
||||
# Make the configuration request
|
||||
with httpx.Client() as client:
|
||||
response = client.post(
|
||||
config_endpoint,
|
||||
json=config_payload,
|
||||
headers=headers,
|
||||
cookies=cookies,
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
log.success("OpenAI plugin configured successfully!")
|
||||
log.key_value("API Base", config_payload["credentials"]["openai_api_base"])
|
||||
log.key_value("API Key", config_payload["credentials"]["openai_api_key"])
|
||||
return True
|
||||
|
||||
elif response.status_code == 201:
|
||||
log.success("OpenAI plugin credentials created successfully!")
|
||||
log.key_value("API Base", config_payload["credentials"]["openai_api_base"])
|
||||
log.key_value("API Key", config_payload["credentials"]["openai_api_key"])
|
||||
return True
|
||||
|
||||
elif response.status_code == 401:
|
||||
log.error("Configuration failed: Unauthorized")
|
||||
log.info("Token may have expired. Please run login_admin.py again")
|
||||
return False
|
||||
else:
|
||||
log.error(f"Configuration failed with status code: {response.status_code}")
|
||||
log.debug(f"Response: {response.text}")
|
||||
return False
|
||||
|
||||
except httpx.ConnectError:
|
||||
log.error("Could not connect to Dify API at http://localhost:5001")
|
||||
log.info("Make sure the API server is running with: ./dev/start-api")
|
||||
return False
|
||||
except Exception as e:
|
||||
log.error(f"An error occurred: {e}")
|
||||
return False
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if not configure_openai_plugin():
|
||||
sys.exit(1)
|
||||
+121
@@ -0,0 +1,121 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.append(str(Path(__file__).parent.parent))
|
||||
|
||||
import json
|
||||
|
||||
import httpx
|
||||
from common import Logger, config_helper
|
||||
|
||||
|
||||
def create_api_key() -> bool:
|
||||
"""Create API key for the imported app."""
|
||||
|
||||
log = Logger("CreateAPIKey")
|
||||
log.header("Creating API Key")
|
||||
|
||||
# Read token from config
|
||||
access_token = config_helper.get_token()
|
||||
if not access_token:
|
||||
log.error("No access token found in config")
|
||||
return False
|
||||
|
||||
# Read app_id from config
|
||||
app_id = config_helper.get_app_id()
|
||||
if not app_id:
|
||||
log.error("No app_id found in config")
|
||||
log.info("Please run import_workflow_app.py first to import the app")
|
||||
return False
|
||||
|
||||
log.step(f"Creating API key for app: {app_id}")
|
||||
|
||||
# API endpoint for creating API key
|
||||
base_url = "http://localhost:5001"
|
||||
api_key_endpoint = f"{base_url}/console/api/apps/{app_id}/api-keys"
|
||||
|
||||
headers = {
|
||||
"Accept": "*/*",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
"Content-Length": "0",
|
||||
"DNT": "1",
|
||||
"Origin": "http://localhost:3000",
|
||||
"Pragma": "no-cache",
|
||||
"Referer": "http://localhost:3000/",
|
||||
"Sec-Fetch-Dest": "empty",
|
||||
"Sec-Fetch-Mode": "cors",
|
||||
"Sec-Fetch-Site": "same-site",
|
||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
|
||||
**config_helper.console_auth_headers(),
|
||||
"content-type": "application/json",
|
||||
"sec-ch-ua": '"Not;A=Brand";v="99", "Google Chrome";v="139", "Chromium";v="139"',
|
||||
"sec-ch-ua-mobile": "?0",
|
||||
"sec-ch-ua-platform": '"macOS"',
|
||||
}
|
||||
|
||||
cookies = config_helper.console_auth_cookies()
|
||||
|
||||
try:
|
||||
# Make the API key creation request
|
||||
with httpx.Client() as client:
|
||||
response = client.post(
|
||||
api_key_endpoint,
|
||||
headers=headers,
|
||||
cookies=cookies,
|
||||
)
|
||||
|
||||
if response.status_code == 200 or response.status_code == 201:
|
||||
response_data = response.json()
|
||||
|
||||
api_key_id = response_data.get("id")
|
||||
api_key_token = response_data.get("token")
|
||||
|
||||
if api_key_token:
|
||||
log.success("API key created successfully!")
|
||||
log.key_value("Key ID", api_key_id)
|
||||
log.key_value("Token", api_key_token)
|
||||
log.key_value("Type", response_data.get("type"))
|
||||
|
||||
# Save API key to config
|
||||
api_key_config = {
|
||||
"id": api_key_id,
|
||||
"token": api_key_token,
|
||||
"type": response_data.get("type"),
|
||||
"app_id": app_id,
|
||||
"created_at": response_data.get("created_at"),
|
||||
}
|
||||
|
||||
if config_helper.write_config("api_key_config", api_key_config):
|
||||
log.info(f"API key saved to: {config_helper.get_config_path('benchmark_state')}")
|
||||
return True
|
||||
return False
|
||||
else:
|
||||
log.error("No API token received")
|
||||
log.debug(f"Response: {json.dumps(response_data, indent=2)}")
|
||||
return False
|
||||
|
||||
elif response.status_code == 401:
|
||||
log.error("API key creation failed: Unauthorized")
|
||||
log.info("Token may have expired. Please run login_admin.py again")
|
||||
return False
|
||||
else:
|
||||
log.error(f"API key creation failed with status code: {response.status_code}")
|
||||
log.debug(f"Response: {response.text}")
|
||||
return False
|
||||
|
||||
except httpx.ConnectError:
|
||||
log.error("Could not connect to Dify API at http://localhost:5001")
|
||||
log.info("Make sure the API server is running with: ./dev/start-api")
|
||||
return False
|
||||
except Exception as e:
|
||||
log.error(f"An error occurred: {e}")
|
||||
return False
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if not create_api_key():
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,176 @@
|
||||
app:
|
||||
description: ''
|
||||
icon: 🤖
|
||||
icon_background: '#FFEAD5'
|
||||
mode: workflow
|
||||
name: workflow_llm
|
||||
use_icon_as_answer_icon: false
|
||||
dependencies:
|
||||
- current_identifier: null
|
||||
type: marketplace
|
||||
value:
|
||||
marketplace_plugin_unique_identifier: langgenius/openai:0.2.5@373362a028986aae53a7baf73a7f11991ba3c22c69eaf97d6cde048cfd4a9f98
|
||||
kind: app
|
||||
version: 0.4.0
|
||||
workflow:
|
||||
conversation_variables: []
|
||||
environment_variables: []
|
||||
features:
|
||||
file_upload:
|
||||
allowed_file_extensions:
|
||||
- .JPG
|
||||
- .JPEG
|
||||
- .PNG
|
||||
- .GIF
|
||||
- .WEBP
|
||||
- .SVG
|
||||
allowed_file_types:
|
||||
- image
|
||||
allowed_file_upload_methods:
|
||||
- local_file
|
||||
- remote_url
|
||||
enabled: false
|
||||
fileUploadConfig:
|
||||
audio_file_size_limit: 50
|
||||
batch_count_limit: 5
|
||||
file_size_limit: 15
|
||||
image_file_size_limit: 10
|
||||
video_file_size_limit: 100
|
||||
workflow_file_upload_limit: 10
|
||||
image:
|
||||
enabled: false
|
||||
number_limits: 3
|
||||
transfer_methods:
|
||||
- local_file
|
||||
- remote_url
|
||||
number_limits: 3
|
||||
opening_statement: ''
|
||||
retriever_resource:
|
||||
enabled: true
|
||||
sensitive_word_avoidance:
|
||||
enabled: false
|
||||
speech_to_text:
|
||||
enabled: false
|
||||
suggested_questions: []
|
||||
suggested_questions_after_answer:
|
||||
enabled: false
|
||||
text_to_speech:
|
||||
enabled: false
|
||||
language: ''
|
||||
voice: ''
|
||||
graph:
|
||||
edges:
|
||||
- data:
|
||||
isInIteration: false
|
||||
isInLoop: false
|
||||
sourceType: start
|
||||
targetType: llm
|
||||
id: 1757611990947-source-1757611992921-target
|
||||
source: '1757611990947'
|
||||
sourceHandle: source
|
||||
target: '1757611992921'
|
||||
targetHandle: target
|
||||
type: custom
|
||||
zIndex: 0
|
||||
- data:
|
||||
isInIteration: false
|
||||
isInLoop: false
|
||||
sourceType: llm
|
||||
targetType: end
|
||||
id: 1757611992921-source-1757611996447-target
|
||||
source: '1757611992921'
|
||||
sourceHandle: source
|
||||
target: '1757611996447'
|
||||
targetHandle: target
|
||||
type: custom
|
||||
zIndex: 0
|
||||
nodes:
|
||||
- data:
|
||||
desc: ''
|
||||
selected: false
|
||||
title: Start
|
||||
type: start
|
||||
variables:
|
||||
- label: question
|
||||
max_length: null
|
||||
options: []
|
||||
required: true
|
||||
type: text-input
|
||||
variable: question
|
||||
height: 90
|
||||
id: '1757611990947'
|
||||
position:
|
||||
x: 30
|
||||
y: 245
|
||||
positionAbsolute:
|
||||
x: 30
|
||||
y: 245
|
||||
selected: false
|
||||
sourcePosition: right
|
||||
targetPosition: left
|
||||
type: custom
|
||||
width: 244
|
||||
- data:
|
||||
context:
|
||||
enabled: false
|
||||
variable_selector: []
|
||||
desc: ''
|
||||
model:
|
||||
completion_params:
|
||||
temperature: 0.7
|
||||
mode: chat
|
||||
name: gpt-4o
|
||||
provider: langgenius/openai/openai
|
||||
prompt_template:
|
||||
- id: c165fcb6-f1f0-42f2-abab-e81982434deb
|
||||
role: system
|
||||
text: ''
|
||||
- role: user
|
||||
text: '{{#1757611990947.question#}}'
|
||||
selected: false
|
||||
title: LLM
|
||||
type: llm
|
||||
variables: []
|
||||
vision:
|
||||
enabled: false
|
||||
height: 90
|
||||
id: '1757611992921'
|
||||
position:
|
||||
x: 334
|
||||
y: 245
|
||||
positionAbsolute:
|
||||
x: 334
|
||||
y: 245
|
||||
selected: false
|
||||
sourcePosition: right
|
||||
targetPosition: left
|
||||
type: custom
|
||||
width: 244
|
||||
- data:
|
||||
desc: ''
|
||||
outputs:
|
||||
- value_selector:
|
||||
- '1757611992921'
|
||||
- text
|
||||
value_type: string
|
||||
variable: answer
|
||||
selected: false
|
||||
title: End
|
||||
type: end
|
||||
height: 90
|
||||
id: '1757611996447'
|
||||
position:
|
||||
x: 638
|
||||
y: 245
|
||||
positionAbsolute:
|
||||
x: 638
|
||||
y: 245
|
||||
selected: true
|
||||
sourcePosition: right
|
||||
targetPosition: left
|
||||
type: custom
|
||||
width: 244
|
||||
viewport:
|
||||
x: 0
|
||||
y: 0
|
||||
zoom: 0.7
|
||||
@@ -0,0 +1,138 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.append(str(Path(__file__).parent.parent))
|
||||
|
||||
import json
|
||||
|
||||
import httpx
|
||||
from common import Logger, config_helper # type: ignore[import]
|
||||
|
||||
|
||||
def is_successful_import_response(response_data: dict[str, object]) -> bool:
|
||||
return response_data.get("status") != "failed" and bool(response_data.get("app_id"))
|
||||
|
||||
|
||||
def import_workflow_app() -> bool:
|
||||
"""Import workflow app from DSL file and save app_id."""
|
||||
|
||||
log = Logger("ImportApp")
|
||||
log.header("Importing Workflow Application")
|
||||
|
||||
# Read token from config
|
||||
access_token = config_helper.get_token()
|
||||
if not access_token:
|
||||
log.error("No access token found in config")
|
||||
log.info("Please run login_admin.py first to get access token")
|
||||
return False
|
||||
|
||||
# Read workflow DSL file
|
||||
dsl_path = Path(__file__).parent / "dsl" / "workflow_llm.yml"
|
||||
|
||||
if not dsl_path.exists():
|
||||
log.error(f"DSL file not found: {dsl_path}")
|
||||
return False
|
||||
|
||||
with open(dsl_path) as f:
|
||||
yaml_content = f.read()
|
||||
|
||||
log.step("Importing workflow app from DSL...")
|
||||
log.key_value("DSL file", dsl_path.name)
|
||||
|
||||
# API endpoint for app import
|
||||
base_url = "http://localhost:5001"
|
||||
import_endpoint = f"{base_url}/console/api/apps/imports"
|
||||
|
||||
# Import payload
|
||||
import_payload = {"mode": "yaml-content", "yaml_content": yaml_content}
|
||||
|
||||
headers = {
|
||||
"Accept": "*/*",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
"DNT": "1",
|
||||
"Origin": "http://localhost:3000",
|
||||
"Pragma": "no-cache",
|
||||
"Referer": "http://localhost:3000/",
|
||||
"Sec-Fetch-Dest": "empty",
|
||||
"Sec-Fetch-Mode": "cors",
|
||||
"Sec-Fetch-Site": "same-site",
|
||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
|
||||
**config_helper.console_auth_headers(),
|
||||
"content-type": "application/json",
|
||||
"sec-ch-ua": '"Not;A=Brand";v="99", "Google Chrome";v="139", "Chromium";v="139"',
|
||||
"sec-ch-ua-mobile": "?0",
|
||||
"sec-ch-ua-platform": '"macOS"',
|
||||
}
|
||||
|
||||
cookies = config_helper.console_auth_cookies()
|
||||
|
||||
try:
|
||||
# Make the import request
|
||||
with httpx.Client() as client:
|
||||
response = client.post(
|
||||
import_endpoint,
|
||||
json=import_payload,
|
||||
headers=headers,
|
||||
cookies=cookies,
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
response_data = response.json()
|
||||
|
||||
if is_successful_import_response(response_data):
|
||||
app_id = response_data.get("app_id")
|
||||
if response_data.get("status") != "completed":
|
||||
log.warning(f"Import status: {response_data.get('status')}")
|
||||
log.debug(f"Response: {json.dumps(response_data, indent=2)}")
|
||||
log.success("Workflow app imported successfully!")
|
||||
log.key_value("App ID", app_id)
|
||||
log.key_value("App Mode", response_data.get("app_mode"))
|
||||
log.key_value("DSL Version", response_data.get("imported_dsl_version"))
|
||||
|
||||
# Save app_id to config
|
||||
app_config = {
|
||||
"app_id": app_id,
|
||||
"app_mode": response_data.get("app_mode"),
|
||||
"app_name": "workflow_llm",
|
||||
"dsl_version": response_data.get("imported_dsl_version"),
|
||||
}
|
||||
|
||||
if config_helper.write_config("app_config", app_config):
|
||||
log.info(f"App config saved to: {config_helper.get_config_path('benchmark_state')}")
|
||||
return True
|
||||
return False
|
||||
|
||||
elif response_data.get("status") == "failed":
|
||||
log.error("Import failed")
|
||||
log.error(f"Error: {response_data.get('error')}")
|
||||
return False
|
||||
else:
|
||||
log.error("Import response did not include app_id")
|
||||
log.debug(f"Response: {json.dumps(response_data, indent=2)}")
|
||||
return False
|
||||
|
||||
elif response.status_code == 401:
|
||||
log.error("Import failed: Unauthorized")
|
||||
log.info("Token may have expired. Please run login_admin.py again")
|
||||
return False
|
||||
else:
|
||||
log.error(f"Import failed with status code: {response.status_code}")
|
||||
log.debug(f"Response: {response.text}")
|
||||
return False
|
||||
|
||||
except httpx.ConnectError:
|
||||
log.error("Could not connect to Dify API at http://localhost:5001")
|
||||
log.info("Make sure the API server is running with: ./dev/start-api")
|
||||
return False
|
||||
except Exception as e:
|
||||
log.error(f"An error occurred: {e}")
|
||||
return False
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if not import_workflow_app():
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,173 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.append(str(Path(__file__).parent.parent))
|
||||
|
||||
import time
|
||||
|
||||
import httpx
|
||||
from common import Logger, config_helper
|
||||
|
||||
|
||||
def is_non_blocking_install_response(response_data: dict[str, object]) -> bool:
|
||||
return not response_data.get("code")
|
||||
|
||||
|
||||
def install_openai_plugin() -> bool:
|
||||
"""Install OpenAI plugin using saved access token."""
|
||||
|
||||
log = Logger("InstallPlugin")
|
||||
log.header("Installing OpenAI Plugin")
|
||||
|
||||
# Read token from config
|
||||
access_token = config_helper.get_token()
|
||||
if not access_token:
|
||||
log.error("No access token found in config")
|
||||
log.info("Please run login_admin.py first to get access token")
|
||||
return False
|
||||
|
||||
log.step("Installing OpenAI plugin...")
|
||||
|
||||
# API endpoint for plugin installation
|
||||
base_url = "http://localhost:5001"
|
||||
install_endpoint = f"{base_url}/console/api/workspaces/current/plugin/install/marketplace"
|
||||
|
||||
# Plugin identifier
|
||||
plugin_payload = {
|
||||
"plugin_unique_identifiers": [
|
||||
"langgenius/openai:0.2.5@373362a028986aae53a7baf73a7f11991ba3c22c69eaf97d6cde048cfd4a9f98"
|
||||
]
|
||||
}
|
||||
|
||||
headers = {
|
||||
"Accept": "*/*",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
"DNT": "1",
|
||||
"Origin": "http://localhost:3000",
|
||||
"Pragma": "no-cache",
|
||||
"Referer": "http://localhost:3000/",
|
||||
"Sec-Fetch-Dest": "empty",
|
||||
"Sec-Fetch-Mode": "cors",
|
||||
"Sec-Fetch-Site": "same-site",
|
||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
|
||||
**config_helper.console_auth_headers(),
|
||||
"content-type": "application/json",
|
||||
"sec-ch-ua": '"Not;A=Brand";v="99", "Google Chrome";v="139", "Chromium";v="139"',
|
||||
"sec-ch-ua-mobile": "?0",
|
||||
"sec-ch-ua-platform": '"macOS"',
|
||||
}
|
||||
|
||||
cookies = config_helper.console_auth_cookies()
|
||||
|
||||
try:
|
||||
# Make the installation request
|
||||
with httpx.Client() as client:
|
||||
response = client.post(
|
||||
install_endpoint,
|
||||
json=plugin_payload,
|
||||
headers=headers,
|
||||
cookies=cookies,
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
response_data = response.json()
|
||||
task_id = response_data.get("task_id")
|
||||
|
||||
if not task_id:
|
||||
if is_non_blocking_install_response(response_data):
|
||||
log.warning("No installation task returned; plugin may already be installed")
|
||||
log.debug(f"Response: {response.text}")
|
||||
return True
|
||||
log.error("No task ID received from installation request")
|
||||
log.debug(f"Response: {response.text}")
|
||||
return False
|
||||
|
||||
log.progress(f"Installation task created: {task_id}")
|
||||
log.info("Polling for task completion...")
|
||||
|
||||
# Poll for task completion
|
||||
task_endpoint = f"{base_url}/console/api/workspaces/current/plugin/tasks/{task_id}"
|
||||
|
||||
max_attempts = 30 # 30 attempts with 2 second delay = 60 seconds max
|
||||
attempt = 0
|
||||
|
||||
log.spinner_start("Installing plugin")
|
||||
|
||||
while attempt < max_attempts:
|
||||
attempt += 1
|
||||
time.sleep(2) # Wait 2 seconds between polls
|
||||
|
||||
task_response = client.get(
|
||||
task_endpoint,
|
||||
headers=headers,
|
||||
cookies=cookies,
|
||||
)
|
||||
|
||||
if task_response.status_code != 200:
|
||||
log.spinner_stop(
|
||||
success=False,
|
||||
message=f"Failed to get task status: {task_response.status_code}",
|
||||
)
|
||||
return False
|
||||
|
||||
task_data = task_response.json()
|
||||
task_info = task_data.get("task", {})
|
||||
status = task_info.get("status")
|
||||
|
||||
if status == "success":
|
||||
log.spinner_stop(success=True, message="Plugin installed!")
|
||||
log.success("OpenAI plugin installed successfully!")
|
||||
|
||||
# Display plugin info
|
||||
plugins = task_info.get("plugins", [])
|
||||
if plugins:
|
||||
plugin_info = plugins[0]
|
||||
log.key_value("Plugin ID", plugin_info.get("plugin_id"))
|
||||
log.key_value("Message", plugin_info.get("message"))
|
||||
return True
|
||||
|
||||
elif status == "failed":
|
||||
log.spinner_stop(success=False, message="Installation failed")
|
||||
log.error("Plugin installation failed")
|
||||
plugins = task_info.get("plugins", [])
|
||||
if plugins:
|
||||
for plugin in plugins:
|
||||
log.list_item(f"{plugin.get('plugin_id')}: {plugin.get('message')}")
|
||||
return False
|
||||
|
||||
# Continue polling if status is "pending" or other
|
||||
|
||||
else:
|
||||
log.spinner_stop(success=False, message="Installation timed out")
|
||||
log.error("Installation timed out after 60 seconds")
|
||||
return False
|
||||
|
||||
elif response.status_code == 401:
|
||||
log.error("Installation failed: Unauthorized")
|
||||
log.info("Token may have expired. Please run login_admin.py again")
|
||||
return False
|
||||
elif response.status_code == 409:
|
||||
log.warning("Plugin may already be installed")
|
||||
log.debug(f"Response: {response.text}")
|
||||
return True
|
||||
else:
|
||||
log.error(f"Installation failed with status code: {response.status_code}")
|
||||
log.debug(f"Response: {response.text}")
|
||||
return False
|
||||
|
||||
except httpx.ConnectError:
|
||||
log.error("Could not connect to Dify API at http://localhost:5001")
|
||||
log.info("Make sure the API server is running with: ./dev/start-api")
|
||||
return False
|
||||
except Exception as e:
|
||||
log.error(f"An error occurred: {e}")
|
||||
return False
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if not install_openai_plugin():
|
||||
sys.exit(1)
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.append(str(Path(__file__).parent.parent))
|
||||
|
||||
import base64
|
||||
import json
|
||||
|
||||
import httpx
|
||||
from common import Logger, config_helper
|
||||
|
||||
|
||||
def encode_sensitive_field(value: str) -> str:
|
||||
"""Encode fields the same way the web client does before login."""
|
||||
return base64.b64encode(value.encode("utf-8")).decode()
|
||||
|
||||
|
||||
def login_admin() -> bool:
|
||||
"""Login with admin account and save access token."""
|
||||
|
||||
log = Logger("Login")
|
||||
log.header("Admin Login")
|
||||
|
||||
# Read admin credentials from config
|
||||
admin_config = config_helper.read_config("admin_config")
|
||||
|
||||
if not admin_config:
|
||||
log.error("Admin config not found")
|
||||
log.info("Please run setup_admin.py first to create the admin account")
|
||||
return False
|
||||
|
||||
log.info(f"Logging in with email: {admin_config['email']}")
|
||||
|
||||
# API login endpoint
|
||||
base_url = "http://localhost:5001"
|
||||
login_endpoint = f"{base_url}/console/api/login"
|
||||
|
||||
# Prepare login payload
|
||||
login_payload = {
|
||||
"email": admin_config["email"],
|
||||
"password": encode_sensitive_field(admin_config["password"]),
|
||||
"remember_me": True,
|
||||
}
|
||||
|
||||
try:
|
||||
# Make the login request
|
||||
with httpx.Client() as client:
|
||||
response = client.post(
|
||||
login_endpoint,
|
||||
json=login_payload,
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
log.success("Login successful!")
|
||||
|
||||
response_data = response.json()
|
||||
|
||||
# Check if login was successful
|
||||
if response_data.get("result") != "success":
|
||||
log.error(f"Login failed: {response_data}")
|
||||
return False
|
||||
|
||||
access_token = response.cookies.get("access_token", "")
|
||||
refresh_token = response.cookies.get("refresh_token", "")
|
||||
csrf_token = response.cookies.get("csrf_token", "")
|
||||
|
||||
if not access_token:
|
||||
log.error("No access token found in response")
|
||||
log.debug(f"Full response: {json.dumps(response_data, indent=2)}")
|
||||
return False
|
||||
|
||||
# Save token to config file
|
||||
token_config = {
|
||||
"email": admin_config["email"],
|
||||
"access_token": access_token,
|
||||
"refresh_token": refresh_token,
|
||||
"csrf_token": csrf_token,
|
||||
}
|
||||
|
||||
# Save token config
|
||||
if config_helper.write_config("token_config", token_config):
|
||||
log.info(f"Token saved to: {config_helper.get_config_path('benchmark_state')}")
|
||||
|
||||
# Show truncated token for verification
|
||||
token_display = f"{access_token[:20]}..." if len(access_token) > 20 else "Token saved"
|
||||
log.key_value("Access token", token_display)
|
||||
return True
|
||||
|
||||
elif response.status_code == 401:
|
||||
log.error("Login failed: Invalid credentials")
|
||||
log.debug(f"Response: {response.text}")
|
||||
return False
|
||||
else:
|
||||
log.error(f"Login failed with status code: {response.status_code}")
|
||||
log.debug(f"Response: {response.text}")
|
||||
return False
|
||||
|
||||
except httpx.ConnectError:
|
||||
log.error("Could not connect to Dify API at http://localhost:5001")
|
||||
log.info("Make sure the API server is running with: ./dev/start-api")
|
||||
return False
|
||||
except Exception as e:
|
||||
log.error(f"An error occurred: {e}")
|
||||
return False
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if not login_admin():
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,204 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
|
||||
from flask import Flask, Response, jsonify, request
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
# Mock models list
|
||||
MODELS = [
|
||||
{
|
||||
"id": "gpt-3.5-turbo",
|
||||
"object": "model",
|
||||
"created": 1677649963,
|
||||
"owned_by": "openai",
|
||||
},
|
||||
{"id": "gpt-4", "object": "model", "created": 1687882411, "owned_by": "openai"},
|
||||
{
|
||||
"id": "text-embedding-ada-002",
|
||||
"object": "model",
|
||||
"created": 1671217299,
|
||||
"owned_by": "openai-internal",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@app.route("/v1/models", methods=["GET"])
|
||||
def list_models() -> Response:
|
||||
"""List available models."""
|
||||
return jsonify({"object": "list", "data": MODELS})
|
||||
|
||||
|
||||
@app.route("/v1/chat/completions", methods=["POST"])
|
||||
def chat_completions() -> Response:
|
||||
"""Handle chat completions."""
|
||||
data = request.json or {}
|
||||
model = data.get("model", "gpt-3.5-turbo")
|
||||
messages = data.get("messages", [])
|
||||
stream = data.get("stream", False)
|
||||
|
||||
# Generate mock response
|
||||
response_content = "This is a mock response from the OpenAI server."
|
||||
if messages:
|
||||
last_message = messages[-1].get("content", "")
|
||||
response_content = f"Mock response to: {last_message[:100]}..."
|
||||
|
||||
if stream:
|
||||
# Streaming response
|
||||
def generate() -> Iterator[str]:
|
||||
# Send initial chunk
|
||||
chunk = {
|
||||
"id": f"chatcmpl-{uuid.uuid4().hex[:8]}",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": int(time.time()),
|
||||
"model": model,
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {"role": "assistant", "content": ""},
|
||||
"finish_reason": None,
|
||||
}
|
||||
],
|
||||
}
|
||||
yield f"data: {json.dumps(chunk)}\n\n"
|
||||
|
||||
# Send content in chunks
|
||||
words = response_content.split()
|
||||
for word in words:
|
||||
chunk = {
|
||||
"id": f"chatcmpl-{uuid.uuid4().hex[:8]}",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": int(time.time()),
|
||||
"model": model,
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {"content": word + " "},
|
||||
"finish_reason": None,
|
||||
}
|
||||
],
|
||||
}
|
||||
yield f"data: {json.dumps(chunk)}\n\n"
|
||||
time.sleep(0.05) # Simulate streaming delay
|
||||
|
||||
# Send final chunk
|
||||
chunk = {
|
||||
"id": f"chatcmpl-{uuid.uuid4().hex[:8]}",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": int(time.time()),
|
||||
"model": model,
|
||||
"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
|
||||
}
|
||||
yield f"data: {json.dumps(chunk)}\n\n"
|
||||
yield "data: [DONE]\n\n"
|
||||
|
||||
return Response(generate(), mimetype="text/event-stream")
|
||||
else:
|
||||
# Non-streaming response
|
||||
return jsonify(
|
||||
{
|
||||
"id": f"chatcmpl-{uuid.uuid4().hex[:8]}",
|
||||
"object": "chat.completion",
|
||||
"created": int(time.time()),
|
||||
"model": model,
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": response_content},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"prompt_tokens": len(str(messages)),
|
||||
"completion_tokens": len(response_content.split()),
|
||||
"total_tokens": len(str(messages)) + len(response_content.split()),
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@app.route("/v1/completions", methods=["POST"])
|
||||
def completions() -> Response:
|
||||
"""Handle text completions."""
|
||||
data = request.json or {}
|
||||
model = data.get("model", "gpt-3.5-turbo-instruct")
|
||||
prompt = data.get("prompt", "")
|
||||
|
||||
response_text = f"Mock completion for prompt: {prompt[:100]}..."
|
||||
|
||||
return jsonify(
|
||||
{
|
||||
"id": f"cmpl-{uuid.uuid4().hex[:8]}",
|
||||
"object": "text_completion",
|
||||
"created": int(time.time()),
|
||||
"model": model,
|
||||
"choices": [
|
||||
{
|
||||
"text": response_text,
|
||||
"index": 0,
|
||||
"logprobs": None,
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"prompt_tokens": len(prompt.split()),
|
||||
"completion_tokens": len(response_text.split()),
|
||||
"total_tokens": len(prompt.split()) + len(response_text.split()),
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@app.route("/v1/embeddings", methods=["POST"])
|
||||
def embeddings() -> Response:
|
||||
"""Handle embeddings requests."""
|
||||
data = request.json or {}
|
||||
model = data.get("model", "text-embedding-ada-002")
|
||||
input_text = data.get("input", "")
|
||||
|
||||
# Generate mock embedding (1536 dimensions for ada-002)
|
||||
mock_embedding = [0.1] * 1536
|
||||
|
||||
return jsonify(
|
||||
{
|
||||
"object": "list",
|
||||
"data": [{"object": "embedding", "embedding": mock_embedding, "index": 0}],
|
||||
"model": model,
|
||||
"usage": {
|
||||
"prompt_tokens": len(input_text.split()),
|
||||
"total_tokens": len(input_text.split()),
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@app.route("/v1/models/<model_id>", methods=["GET"])
|
||||
def get_model(model_id: str) -> Response | tuple[Response, int]:
|
||||
"""Get specific model details."""
|
||||
for model in MODELS:
|
||||
if model["id"] == model_id:
|
||||
return jsonify(model)
|
||||
|
||||
return jsonify({"error": "Model not found"}), 404
|
||||
|
||||
|
||||
@app.route("/health", methods=["GET"])
|
||||
def health() -> Response:
|
||||
"""Health check endpoint."""
|
||||
return jsonify({"status": "healthy"})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("🚀 Starting Mock OpenAI Server on http://localhost:5004")
|
||||
print("Available endpoints:")
|
||||
print(" - GET /v1/models")
|
||||
print(" - POST /v1/chat/completions")
|
||||
print(" - POST /v1/completions")
|
||||
print(" - POST /v1/embeddings")
|
||||
print(" - GET /v1/models/<model_id>")
|
||||
print(" - GET /health")
|
||||
app.run(host="0.0.0.0", port=5004, debug=True)
|
||||
@@ -0,0 +1,112 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.append(str(Path(__file__).parent.parent))
|
||||
|
||||
import json
|
||||
|
||||
import httpx
|
||||
from common import Logger, config_helper
|
||||
|
||||
|
||||
def publish_workflow() -> bool:
|
||||
"""Publish the imported workflow app."""
|
||||
|
||||
log = Logger("PublishWorkflow")
|
||||
log.header("Publishing Workflow")
|
||||
|
||||
# Read token from config
|
||||
access_token = config_helper.get_token()
|
||||
if not access_token:
|
||||
log.error("No access token found in config")
|
||||
return False
|
||||
|
||||
# Read app_id from config
|
||||
app_id = config_helper.get_app_id()
|
||||
if not app_id:
|
||||
log.error("No app_id found in config")
|
||||
return False
|
||||
|
||||
log.step(f"Publishing workflow for app: {app_id}")
|
||||
|
||||
# API endpoint for publishing workflow
|
||||
base_url = "http://localhost:5001"
|
||||
publish_endpoint = f"{base_url}/console/api/apps/{app_id}/workflows/publish"
|
||||
|
||||
# Publish payload
|
||||
publish_payload = {"marked_name": "", "marked_comment": ""}
|
||||
|
||||
headers = {
|
||||
"Accept": "*/*",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
"DNT": "1",
|
||||
"Origin": "http://localhost:3000",
|
||||
"Pragma": "no-cache",
|
||||
"Referer": "http://localhost:3000/",
|
||||
"Sec-Fetch-Dest": "empty",
|
||||
"Sec-Fetch-Mode": "cors",
|
||||
"Sec-Fetch-Site": "same-site",
|
||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
|
||||
**config_helper.console_auth_headers(),
|
||||
"content-type": "application/json",
|
||||
"sec-ch-ua": '"Not;A=Brand";v="99", "Google Chrome";v="139", "Chromium";v="139"',
|
||||
"sec-ch-ua-mobile": "?0",
|
||||
"sec-ch-ua-platform": '"macOS"',
|
||||
}
|
||||
|
||||
cookies = config_helper.console_auth_cookies()
|
||||
|
||||
try:
|
||||
# Make the publish request
|
||||
with httpx.Client() as client:
|
||||
response = client.post(
|
||||
publish_endpoint,
|
||||
json=publish_payload,
|
||||
headers=headers,
|
||||
cookies=cookies,
|
||||
)
|
||||
|
||||
if response.status_code == 200 or response.status_code == 201:
|
||||
log.success("Workflow published successfully!")
|
||||
log.key_value("App ID", app_id)
|
||||
|
||||
# Try to parse response if it has JSON content
|
||||
if response.text:
|
||||
try:
|
||||
response_data = response.json()
|
||||
if response_data:
|
||||
log.debug(f"Response: {json.dumps(response_data, indent=2)}")
|
||||
except json.JSONDecodeError:
|
||||
# Response might be empty or non-JSON
|
||||
pass
|
||||
return True
|
||||
|
||||
elif response.status_code == 401:
|
||||
log.error("Workflow publish failed: Unauthorized")
|
||||
log.info("Token may have expired. Please run login_admin.py again")
|
||||
return False
|
||||
elif response.status_code == 404:
|
||||
log.error("Workflow publish failed: App not found")
|
||||
log.info("Make sure the app was imported successfully")
|
||||
return False
|
||||
else:
|
||||
log.error(f"Workflow publish failed with status code: {response.status_code}")
|
||||
log.debug(f"Response: {response.text}")
|
||||
return False
|
||||
|
||||
except httpx.ConnectError:
|
||||
log.error("Could not connect to Dify API at http://localhost:5001")
|
||||
log.info("Make sure the API server is running with: ./dev/start-api")
|
||||
return False
|
||||
except Exception as e:
|
||||
log.error(f"An error occurred: {e}")
|
||||
return False
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if not publish_workflow():
|
||||
sys.exit(1)
|
||||
+161
@@ -0,0 +1,161 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.append(str(Path(__file__).parent.parent))
|
||||
|
||||
import json
|
||||
|
||||
import httpx
|
||||
from common import Logger, config_helper
|
||||
|
||||
|
||||
def run_workflow(question: str = "fake question", streaming: bool = True) -> None:
|
||||
"""Run the workflow app with a question."""
|
||||
|
||||
log = Logger("RunWorkflow")
|
||||
log.header("Running Workflow")
|
||||
|
||||
# Read API key from config
|
||||
api_token = config_helper.get_api_key()
|
||||
if not api_token:
|
||||
log.error("No API token found in config")
|
||||
log.info("Please run create_api_key.py first to create an API key")
|
||||
return
|
||||
|
||||
log.key_value("Question", question)
|
||||
log.key_value("Mode", "Streaming" if streaming else "Blocking")
|
||||
log.separator()
|
||||
|
||||
# API endpoint for running workflow
|
||||
base_url = "http://localhost:5001"
|
||||
run_endpoint = f"{base_url}/v1/workflows/run"
|
||||
|
||||
# Run payload
|
||||
run_payload = {
|
||||
"inputs": {"question": question},
|
||||
"user": "default user",
|
||||
"response_mode": "streaming" if streaming else "blocking",
|
||||
}
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {api_token}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
try:
|
||||
# Make the run request
|
||||
with httpx.Client(timeout=30.0) as client:
|
||||
if streaming:
|
||||
# Handle streaming response
|
||||
with client.stream(
|
||||
"POST",
|
||||
run_endpoint,
|
||||
json=run_payload,
|
||||
headers=headers,
|
||||
) as response:
|
||||
if response.status_code == 200:
|
||||
log.success("Workflow started successfully!")
|
||||
log.separator()
|
||||
log.step("Streaming response:")
|
||||
|
||||
for line in response.iter_lines():
|
||||
if line.startswith("data: "):
|
||||
data_str = line[6:] # Remove "data: " prefix
|
||||
if data_str == "[DONE]":
|
||||
log.success("Workflow completed!")
|
||||
break
|
||||
try:
|
||||
data = json.loads(data_str)
|
||||
event = data.get("event")
|
||||
|
||||
if event == "workflow_started":
|
||||
log.progress(f"Workflow started: {data.get('data', {}).get('id')}")
|
||||
elif event == "node_started":
|
||||
node_data = data.get("data", {})
|
||||
log.progress(
|
||||
f"Node started: {node_data.get('node_type')} - {node_data.get('title')}"
|
||||
)
|
||||
elif event == "node_finished":
|
||||
node_data = data.get("data", {})
|
||||
log.progress(
|
||||
f"Node finished: {node_data.get('node_type')} - {node_data.get('title')}"
|
||||
)
|
||||
|
||||
# Print output if it's the LLM node
|
||||
outputs = node_data.get("outputs", {})
|
||||
if outputs.get("text"):
|
||||
log.separator()
|
||||
log.info("💬 LLM Response:")
|
||||
log.info(outputs.get("text"), indent=2)
|
||||
log.separator()
|
||||
|
||||
elif event == "workflow_finished":
|
||||
workflow_data = data.get("data", {})
|
||||
outputs = workflow_data.get("outputs", {})
|
||||
if outputs.get("answer"):
|
||||
log.separator()
|
||||
log.info("📤 Final Answer:")
|
||||
log.info(outputs.get("answer"), indent=2)
|
||||
log.separator()
|
||||
log.key_value(
|
||||
"Total tokens",
|
||||
str(workflow_data.get("total_tokens", 0)),
|
||||
)
|
||||
log.key_value(
|
||||
"Total steps",
|
||||
str(workflow_data.get("total_steps", 0)),
|
||||
)
|
||||
|
||||
elif event == "error":
|
||||
log.error(f"Error: {data.get('message')}")
|
||||
|
||||
except json.JSONDecodeError:
|
||||
# Some lines might not be JSON
|
||||
pass
|
||||
else:
|
||||
log.error(f"Workflow run failed with status code: {response.status_code}")
|
||||
log.debug(f"Response: {response.text}")
|
||||
else:
|
||||
# Handle blocking response
|
||||
response = client.post(
|
||||
run_endpoint,
|
||||
json=run_payload,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
log.success("Workflow completed successfully!")
|
||||
response_data = response.json()
|
||||
|
||||
log.separator()
|
||||
log.debug(f"Full response: {json.dumps(response_data, indent=2)}")
|
||||
|
||||
# Extract the answer if available
|
||||
outputs = response_data.get("data", {}).get("outputs", {})
|
||||
if outputs.get("answer"):
|
||||
log.separator()
|
||||
log.info("📤 Final Answer:")
|
||||
log.info(outputs.get("answer"), indent=2)
|
||||
else:
|
||||
log.error(f"Workflow run failed with status code: {response.status_code}")
|
||||
log.debug(f"Response: {response.text}")
|
||||
|
||||
except httpx.ConnectError:
|
||||
log.error("Could not connect to Dify API at http://localhost:5001")
|
||||
log.info("Make sure the API server is running with: ./dev/start-api")
|
||||
except httpx.TimeoutException:
|
||||
log.error("Request timed out")
|
||||
except Exception as e:
|
||||
log.error(f"An error occurred: {e}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Allow passing question as command line argument
|
||||
if len(sys.argv) > 1:
|
||||
question = " ".join(sys.argv[1:])
|
||||
else:
|
||||
question = "What is the capital of France?"
|
||||
|
||||
run_workflow(question=question, streaming=True)
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.append(str(Path(__file__).parent.parent))
|
||||
|
||||
import os
|
||||
|
||||
import httpx
|
||||
from common import Logger, config_helper
|
||||
|
||||
|
||||
def build_admin_config() -> dict[str, str]:
|
||||
return {
|
||||
"email": os.getenv("STRESS_TEST_ADMIN_EMAIL", "test@dify.ai"),
|
||||
"username": os.getenv("STRESS_TEST_ADMIN_USERNAME", "dify"),
|
||||
"password": os.getenv("STRESS_TEST_ADMIN_PASSWORD", "password123"),
|
||||
}
|
||||
|
||||
|
||||
def setup_admin_account() -> bool:
|
||||
"""Setup Dify API with an admin account."""
|
||||
|
||||
log = Logger("SetupAdmin")
|
||||
log.header("Setting up Admin Account")
|
||||
|
||||
admin_config = build_admin_config()
|
||||
|
||||
# Save credentials to config file
|
||||
if config_helper.write_config("admin_config", admin_config):
|
||||
log.info(f"Admin credentials saved to: {config_helper.get_config_path('benchmark_state')}")
|
||||
|
||||
# API setup endpoint
|
||||
base_url = "http://localhost:5001"
|
||||
setup_endpoint = f"{base_url}/console/api/setup"
|
||||
|
||||
# Prepare setup payload
|
||||
setup_payload = {
|
||||
"email": admin_config["email"],
|
||||
"name": admin_config["username"],
|
||||
"password": admin_config["password"],
|
||||
}
|
||||
|
||||
log.step("Configuring Dify with admin account...")
|
||||
|
||||
try:
|
||||
# Make the setup request
|
||||
with httpx.Client() as client:
|
||||
response = client.post(
|
||||
setup_endpoint,
|
||||
json=setup_payload,
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
|
||||
if response.status_code == 201:
|
||||
log.success("Admin account created successfully!")
|
||||
log.key_value("Email", admin_config["email"])
|
||||
log.key_value("Username", admin_config["username"])
|
||||
return True
|
||||
|
||||
elif response.status_code in {400, 403}:
|
||||
log.warning("Setup may have already been completed")
|
||||
log.debug(f"Response: {response.text}")
|
||||
return True
|
||||
else:
|
||||
log.error(f"Setup failed with status code: {response.status_code}")
|
||||
log.debug(f"Response: {response.text}")
|
||||
return False
|
||||
|
||||
except httpx.ConnectError:
|
||||
log.error("Could not connect to Dify API at http://localhost:5001")
|
||||
log.info("Make sure the API server is running with: ./dev/start-api")
|
||||
return False
|
||||
except Exception as e:
|
||||
log.error(f"An error occurred: {e}")
|
||||
return False
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if not setup_admin_account():
|
||||
sys.exit(1)
|
||||
+219
@@ -0,0 +1,219 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
from common import Logger, ProgressLogger, config_helper
|
||||
|
||||
|
||||
def build_admin_config() -> dict[str, str]:
|
||||
return {
|
||||
"email": os.getenv("STRESS_TEST_ADMIN_EMAIL", "test@dify.ai"),
|
||||
"username": os.getenv("STRESS_TEST_ADMIN_USERNAME", "dify"),
|
||||
"password": os.getenv("STRESS_TEST_ADMIN_PASSWORD", "password123"),
|
||||
}
|
||||
|
||||
|
||||
def get_setup_step(base_url: str = "http://localhost:5001") -> str | None:
|
||||
try:
|
||||
response = httpx.get(f"{base_url}/console/api/setup", timeout=5)
|
||||
if response.status_code == 200:
|
||||
return response.json().get("step")
|
||||
except (httpx.HTTPError, ValueError):
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def confirm(prompt: str) -> bool:
|
||||
answer = input(f"\n{prompt} [Y/n]: ").strip().lower()
|
||||
return answer in ("", "y", "yes")
|
||||
|
||||
|
||||
def confirm_admin_credentials(log: Logger) -> bool:
|
||||
admin_config = build_admin_config()
|
||||
setup_step = get_setup_step()
|
||||
config_helper.write_config("admin_config", admin_config)
|
||||
|
||||
if setup_step == "finished":
|
||||
log.warning("Dify is already initialized; setup will use the existing admin account to log in.")
|
||||
log.key_value("Admin email", admin_config["email"])
|
||||
log.info("Set STRESS_TEST_ADMIN_EMAIL and STRESS_TEST_ADMIN_PASSWORD if this is not the right account.")
|
||||
return confirm("Continue with this admin login?")
|
||||
|
||||
log.info("Dify is not initialized; setup will create the first admin account with:")
|
||||
log.key_value("Admin email", admin_config["email"])
|
||||
log.key_value("Admin username", admin_config["username"])
|
||||
log.key_value("Admin password", admin_config["password"])
|
||||
log.info("Set STRESS_TEST_ADMIN_EMAIL, STRESS_TEST_ADMIN_USERNAME, or STRESS_TEST_ADMIN_PASSWORD to override.")
|
||||
return confirm("Create/use this admin account?")
|
||||
|
||||
|
||||
def run_script(script_name: str, description: str) -> bool:
|
||||
"""Run a Python script and return success status."""
|
||||
script_path = Path(__file__).parent / "setup" / script_name
|
||||
|
||||
if not script_path.exists():
|
||||
print(f"❌ Script not found: {script_path}")
|
||||
return False
|
||||
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"🚀 {description}")
|
||||
print(f" Running: {script_name}")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(script_path)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
# Print output
|
||||
if result.stdout:
|
||||
print(result.stdout)
|
||||
if result.stderr:
|
||||
print(result.stderr, file=sys.stderr)
|
||||
|
||||
if result.returncode != 0:
|
||||
print(f"❌ Script failed with exit code: {result.returncode}")
|
||||
return False
|
||||
|
||||
print(f"✅ {script_name} completed successfully")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error running {script_name}: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def check_port(host: str, port: int, service_name: str) -> bool:
|
||||
"""Check if a service is running on the specified port."""
|
||||
try:
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.settimeout(2)
|
||||
result = sock.connect_ex((host, port))
|
||||
sock.close()
|
||||
|
||||
if result == 0:
|
||||
Logger().success(f"{service_name} is running on port {port}")
|
||||
return True
|
||||
else:
|
||||
Logger().error(f"{service_name} is not accessible on port {port}")
|
||||
return False
|
||||
except Exception as e:
|
||||
Logger().error(f"Error checking {service_name}: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Run all setup scripts in order."""
|
||||
|
||||
log = Logger("Setup")
|
||||
log.box("Dify Stress Test Setup - Full Installation")
|
||||
|
||||
# Check if required services are running
|
||||
log.step("Checking required services...")
|
||||
log.separator()
|
||||
|
||||
dify_running = check_port("localhost", 5001, "Dify API server")
|
||||
if not dify_running:
|
||||
log.info("To start Dify API server:")
|
||||
log.list_item("Run: ./dev/start-api")
|
||||
|
||||
mock_running = check_port("localhost", 5004, "Mock OpenAI server")
|
||||
if not mock_running:
|
||||
log.info("To start Mock OpenAI server:")
|
||||
log.list_item("Run: python scripts/stress-test/setup/mock_openai_server.py")
|
||||
|
||||
if not dify_running or not mock_running:
|
||||
print("\n⚠️ Both services must be running before proceeding.")
|
||||
if confirm("Would you like to check again?"):
|
||||
return main() # Recursively call main to check again
|
||||
else:
|
||||
print("❌ Setup cancelled. Please start the required services and try again.")
|
||||
sys.exit(1)
|
||||
|
||||
log.success("All required services are running!")
|
||||
if not confirm_admin_credentials(log):
|
||||
print("❌ Setup cancelled. Please set the admin environment variables and try again.")
|
||||
sys.exit(1)
|
||||
|
||||
# Define setup steps
|
||||
setup_step = get_setup_step()
|
||||
setup_steps = [
|
||||
("login_admin.py", "Logging in and getting access token"),
|
||||
("install_openai_plugin.py", "Installing OpenAI plugin"),
|
||||
("configure_openai_plugin.py", "Configuring OpenAI plugin with mock server"),
|
||||
("import_workflow_app.py", "Importing workflow application"),
|
||||
("create_api_key.py", "Creating API key for the app"),
|
||||
("publish_workflow.py", "Publishing the workflow"),
|
||||
]
|
||||
if setup_step != "finished":
|
||||
setup_steps.insert(0, ("setup_admin.py", "Creating admin account"))
|
||||
|
||||
# Create progress logger
|
||||
progress = ProgressLogger(len(setup_steps), log)
|
||||
failed_step = None
|
||||
|
||||
for script, description in setup_steps:
|
||||
progress.next_step(description)
|
||||
success = run_script(script, description)
|
||||
|
||||
if not success:
|
||||
failed_step = script
|
||||
break
|
||||
|
||||
# Small delay between steps
|
||||
time.sleep(1)
|
||||
|
||||
log.separator()
|
||||
|
||||
if failed_step:
|
||||
log.error(f"Setup failed at: {failed_step}")
|
||||
log.separator()
|
||||
log.info("Troubleshooting:")
|
||||
if failed_step == "login_admin.py":
|
||||
if get_setup_step() == "finished":
|
||||
log.list_item(
|
||||
"Dify is already initialized; set STRESS_TEST_ADMIN_EMAIL and "
|
||||
"STRESS_TEST_ADMIN_PASSWORD to an existing admin account."
|
||||
)
|
||||
else:
|
||||
admin_config = build_admin_config()
|
||||
log.list_item(
|
||||
"Dify is not initialized; setup creates the first admin with "
|
||||
f"{admin_config['email']} / {admin_config['username']} unless overridden by environment variables."
|
||||
)
|
||||
log.list_item("Check if the Dify API server is running (./dev/start-api)")
|
||||
log.list_item("Check if the mock OpenAI server is running (port 5004)")
|
||||
log.list_item("Review the error messages above")
|
||||
log.list_item("Run cleanup.py and try again")
|
||||
sys.exit(1)
|
||||
else:
|
||||
progress.complete()
|
||||
log.separator()
|
||||
log.success("Setup completed successfully!")
|
||||
log.info("Next steps:")
|
||||
log.list_item("Test the workflow:")
|
||||
log.info(
|
||||
' python scripts/stress-test/setup/run_workflow.py "Your question here"',
|
||||
indent=4,
|
||||
)
|
||||
log.list_item("To clean up and start over:")
|
||||
log.info(" python scripts/stress-test/cleanup.py", indent=4)
|
||||
|
||||
# Optionally run a test
|
||||
log.separator()
|
||||
if confirm("Would you like to run a test workflow now?"):
|
||||
log.step("Running test workflow...")
|
||||
run_script("run_workflow.py", "Testing workflow with default question")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,750 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
SSE (Server-Sent Events) Stress Test for Dify Workflow API
|
||||
|
||||
This script stress tests the streaming performance of Dify's workflow execution API,
|
||||
measuring key metrics like connection rate, event throughput, and time to first event (TTFE).
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import statistics
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from collections import deque
|
||||
from dataclasses import asdict, dataclass
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Literal, TypeAlias, TypedDict
|
||||
|
||||
import requests.exceptions
|
||||
from locust import HttpUser, between, constant, events, task
|
||||
|
||||
# Add the stress-test directory to path to import common modules
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from common.config_helper import ConfigHelper # type: ignore[import-not-found]
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Configuration from environment
|
||||
WORKFLOW_PATH = os.getenv("WORKFLOW_PATH", "/v1/workflows/run")
|
||||
CONNECT_TIMEOUT = float(os.getenv("CONNECT_TIMEOUT", "10"))
|
||||
READ_TIMEOUT = float(os.getenv("READ_TIMEOUT", "60"))
|
||||
TERMINAL_EVENTS = [e.strip() for e in os.getenv("TERMINAL_EVENTS", "workflow_finished,error").split(",") if e.strip()]
|
||||
QUESTIONS_FILE = os.getenv("QUESTIONS_FILE", "")
|
||||
|
||||
|
||||
# Type definitions
|
||||
ErrorType: TypeAlias = Literal[
|
||||
"connection_error",
|
||||
"timeout",
|
||||
"invalid_json",
|
||||
"http_4xx",
|
||||
"http_5xx",
|
||||
"early_termination",
|
||||
"invalid_response",
|
||||
]
|
||||
|
||||
|
||||
class ErrorCounts(TypedDict):
|
||||
"""Error count tracking"""
|
||||
|
||||
connection_error: int
|
||||
timeout: int
|
||||
invalid_json: int
|
||||
http_4xx: int
|
||||
http_5xx: int
|
||||
early_termination: int
|
||||
invalid_response: int
|
||||
|
||||
|
||||
class SSEEvent(TypedDict):
|
||||
"""Server-Sent Event structure"""
|
||||
|
||||
data: str
|
||||
event: str
|
||||
id: str | None
|
||||
|
||||
|
||||
class WorkflowInputs(TypedDict):
|
||||
"""Workflow input structure"""
|
||||
|
||||
question: str
|
||||
|
||||
|
||||
class WorkflowRequestData(TypedDict):
|
||||
"""Workflow request payload"""
|
||||
|
||||
inputs: WorkflowInputs
|
||||
response_mode: Literal["streaming"]
|
||||
user: str
|
||||
|
||||
|
||||
class ParsedEventData(TypedDict, total=False):
|
||||
"""Parsed event data from SSE stream"""
|
||||
|
||||
event: str
|
||||
task_id: str
|
||||
workflow_run_id: str
|
||||
data: object # For dynamic content
|
||||
created_at: int
|
||||
|
||||
|
||||
class LocustStats(TypedDict):
|
||||
"""Locust statistics structure"""
|
||||
|
||||
total_requests: int
|
||||
total_failures: int
|
||||
avg_response_time: float
|
||||
min_response_time: float
|
||||
max_response_time: float
|
||||
|
||||
|
||||
class ReportData(TypedDict):
|
||||
"""JSON report structure"""
|
||||
|
||||
timestamp: str
|
||||
duration_seconds: float
|
||||
metrics: dict[str, object] # Metrics as dict for JSON serialization
|
||||
locust_stats: LocustStats | None
|
||||
|
||||
|
||||
@dataclass
|
||||
class StreamMetrics:
|
||||
"""Metrics for a single stream"""
|
||||
|
||||
stream_duration: float
|
||||
events_count: int
|
||||
bytes_received: int
|
||||
ttfe: float
|
||||
inter_event_times: list[float]
|
||||
|
||||
|
||||
@dataclass
|
||||
class MetricsSnapshot:
|
||||
"""Snapshot of current metrics state"""
|
||||
|
||||
active_connections: int
|
||||
total_connections: int
|
||||
total_events: int
|
||||
connection_rate: float
|
||||
event_rate: float
|
||||
overall_conn_rate: float
|
||||
overall_event_rate: float
|
||||
ttfe_avg: float
|
||||
ttfe_min: float
|
||||
ttfe_max: float
|
||||
ttfe_p50: float
|
||||
ttfe_p95: float
|
||||
ttfe_samples: int
|
||||
ttfe_total_samples: int # Total TTFE samples collected (not limited by window)
|
||||
error_counts: ErrorCounts
|
||||
stream_duration_avg: float
|
||||
stream_duration_p50: float
|
||||
stream_duration_p95: float
|
||||
events_per_stream_avg: float
|
||||
inter_event_latency_avg: float
|
||||
inter_event_latency_p50: float
|
||||
inter_event_latency_p95: float
|
||||
|
||||
|
||||
class MetricsTracker:
|
||||
def __init__(self) -> None:
|
||||
self.lock = threading.Lock()
|
||||
self.active_connections = 0
|
||||
self.total_connections = 0
|
||||
self.total_events = 0
|
||||
self.start_time = time.time()
|
||||
|
||||
# Enhanced metrics with memory limits
|
||||
self.max_samples = 10000 # Prevent unbounded growth
|
||||
self.ttfe_samples: deque[float] = deque(maxlen=self.max_samples)
|
||||
self.ttfe_total_count = 0 # Track total TTFE samples collected
|
||||
|
||||
# For rate calculations - no maxlen to avoid artificial limits
|
||||
self.connection_times: deque[float] = deque()
|
||||
self.event_times: deque[float] = deque()
|
||||
self.last_stats_time = time.time()
|
||||
self.last_total_connections = 0
|
||||
self.last_total_events = 0
|
||||
self.stream_metrics: deque[StreamMetrics] = deque(maxlen=self.max_samples)
|
||||
self.error_counts: ErrorCounts = ErrorCounts(
|
||||
connection_error=0,
|
||||
timeout=0,
|
||||
invalid_json=0,
|
||||
http_4xx=0,
|
||||
http_5xx=0,
|
||||
early_termination=0,
|
||||
invalid_response=0,
|
||||
)
|
||||
|
||||
def connection_started(self) -> None:
|
||||
with self.lock:
|
||||
self.active_connections += 1
|
||||
self.total_connections += 1
|
||||
self.connection_times.append(time.time())
|
||||
|
||||
def connection_ended(self) -> None:
|
||||
with self.lock:
|
||||
self.active_connections -= 1
|
||||
|
||||
def event_received(self) -> None:
|
||||
with self.lock:
|
||||
self.total_events += 1
|
||||
self.event_times.append(time.time())
|
||||
|
||||
def record_ttfe(self, ttfe_ms: float) -> None:
|
||||
with self.lock:
|
||||
self.ttfe_samples.append(ttfe_ms) # deque handles maxlen
|
||||
self.ttfe_total_count += 1 # Increment total counter
|
||||
|
||||
def record_stream_metrics(self, metrics: StreamMetrics) -> None:
|
||||
with self.lock:
|
||||
self.stream_metrics.append(metrics) # deque handles maxlen
|
||||
|
||||
def record_error(self, error_type: ErrorType) -> None:
|
||||
with self.lock:
|
||||
self.error_counts[error_type] += 1
|
||||
|
||||
def get_stats(self) -> MetricsSnapshot:
|
||||
with self.lock:
|
||||
current_time = time.time()
|
||||
time_window = 10.0 # 10 second window for rate calculation
|
||||
|
||||
# Clean up old timestamps outside the window
|
||||
cutoff_time = current_time - time_window
|
||||
while self.connection_times and self.connection_times[0] < cutoff_time:
|
||||
self.connection_times.popleft()
|
||||
while self.event_times and self.event_times[0] < cutoff_time:
|
||||
self.event_times.popleft()
|
||||
|
||||
# Calculate rates based on actual window or elapsed time
|
||||
window_duration = min(time_window, current_time - self.start_time)
|
||||
if window_duration > 0:
|
||||
conn_rate = len(self.connection_times) / window_duration
|
||||
event_rate = len(self.event_times) / window_duration
|
||||
else:
|
||||
conn_rate = 0
|
||||
event_rate = 0
|
||||
|
||||
# Calculate TTFE statistics
|
||||
if self.ttfe_samples:
|
||||
avg_ttfe = statistics.mean(self.ttfe_samples)
|
||||
min_ttfe = min(self.ttfe_samples)
|
||||
max_ttfe = max(self.ttfe_samples)
|
||||
p50_ttfe = statistics.median(self.ttfe_samples)
|
||||
if len(self.ttfe_samples) >= 2:
|
||||
quantiles = statistics.quantiles(self.ttfe_samples, n=20, method="inclusive")
|
||||
p95_ttfe = quantiles[18] # 19th of 19 quantiles = 95th percentile
|
||||
else:
|
||||
p95_ttfe = max_ttfe
|
||||
else:
|
||||
avg_ttfe = min_ttfe = max_ttfe = p50_ttfe = p95_ttfe = 0
|
||||
|
||||
# Calculate stream metrics
|
||||
if self.stream_metrics:
|
||||
durations = [m.stream_duration for m in self.stream_metrics]
|
||||
events_per_stream = [m.events_count for m in self.stream_metrics]
|
||||
stream_duration_avg = statistics.mean(durations)
|
||||
stream_duration_p50 = statistics.median(durations)
|
||||
stream_duration_p95 = (
|
||||
statistics.quantiles(durations, n=20, method="inclusive")[18]
|
||||
if len(durations) >= 2
|
||||
else max(durations)
|
||||
if durations
|
||||
else 0
|
||||
)
|
||||
events_per_stream_avg = statistics.mean(events_per_stream) if events_per_stream else 0
|
||||
|
||||
# Calculate inter-event latency statistics
|
||||
all_inter_event_times = []
|
||||
for m in self.stream_metrics:
|
||||
all_inter_event_times.extend(m.inter_event_times)
|
||||
|
||||
if all_inter_event_times:
|
||||
inter_event_latency_avg = statistics.mean(all_inter_event_times)
|
||||
inter_event_latency_p50 = statistics.median(all_inter_event_times)
|
||||
inter_event_latency_p95 = (
|
||||
statistics.quantiles(all_inter_event_times, n=20, method="inclusive")[18]
|
||||
if len(all_inter_event_times) >= 2
|
||||
else max(all_inter_event_times)
|
||||
)
|
||||
else:
|
||||
inter_event_latency_avg = inter_event_latency_p50 = inter_event_latency_p95 = 0
|
||||
else:
|
||||
stream_duration_avg = stream_duration_p50 = stream_duration_p95 = events_per_stream_avg = 0
|
||||
inter_event_latency_avg = inter_event_latency_p50 = inter_event_latency_p95 = 0
|
||||
|
||||
# Also calculate overall average rates
|
||||
total_elapsed = current_time - self.start_time
|
||||
overall_conn_rate = self.total_connections / total_elapsed if total_elapsed > 0 else 0
|
||||
overall_event_rate = self.total_events / total_elapsed if total_elapsed > 0 else 0
|
||||
|
||||
return MetricsSnapshot(
|
||||
active_connections=self.active_connections,
|
||||
total_connections=self.total_connections,
|
||||
total_events=self.total_events,
|
||||
connection_rate=conn_rate,
|
||||
event_rate=event_rate,
|
||||
overall_conn_rate=overall_conn_rate,
|
||||
overall_event_rate=overall_event_rate,
|
||||
ttfe_avg=avg_ttfe,
|
||||
ttfe_min=min_ttfe,
|
||||
ttfe_max=max_ttfe,
|
||||
ttfe_p50=p50_ttfe,
|
||||
ttfe_p95=p95_ttfe,
|
||||
ttfe_samples=len(self.ttfe_samples),
|
||||
ttfe_total_samples=self.ttfe_total_count, # Return total count
|
||||
error_counts=ErrorCounts(**self.error_counts),
|
||||
stream_duration_avg=stream_duration_avg,
|
||||
stream_duration_p50=stream_duration_p50,
|
||||
stream_duration_p95=stream_duration_p95,
|
||||
events_per_stream_avg=events_per_stream_avg,
|
||||
inter_event_latency_avg=inter_event_latency_avg,
|
||||
inter_event_latency_p50=inter_event_latency_p50,
|
||||
inter_event_latency_p95=inter_event_latency_p95,
|
||||
)
|
||||
|
||||
|
||||
# Global metrics instance
|
||||
metrics = MetricsTracker()
|
||||
|
||||
|
||||
class SSEParser:
|
||||
"""Parser for Server-Sent Events according to W3C spec"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.data_buffer: list[str] = []
|
||||
self.event_type: str | None = None
|
||||
self.event_id: str | None = None
|
||||
|
||||
def parse_line(self, line: str) -> SSEEvent | None:
|
||||
"""Parse a single SSE line and return event if complete"""
|
||||
# Empty line signals end of event
|
||||
if not line:
|
||||
if self.data_buffer:
|
||||
event = SSEEvent(
|
||||
data="\n".join(self.data_buffer),
|
||||
event=self.event_type or "message",
|
||||
id=self.event_id,
|
||||
)
|
||||
self.data_buffer = []
|
||||
self.event_type = None
|
||||
self.event_id = None
|
||||
return event
|
||||
return None
|
||||
|
||||
# Comment line
|
||||
if line.startswith(":"):
|
||||
return None
|
||||
|
||||
# Parse field
|
||||
if ":" in line:
|
||||
field, value = line.split(":", 1)
|
||||
value = value.lstrip()
|
||||
|
||||
if field == "data":
|
||||
self.data_buffer.append(value)
|
||||
elif field == "event":
|
||||
self.event_type = value
|
||||
elif field == "id":
|
||||
self.event_id = value
|
||||
|
||||
return None
|
||||
|
||||
|
||||
# Note: SSEClient removed - we'll handle SSE parsing directly in the task for better Locust integration
|
||||
|
||||
|
||||
class DifyWorkflowUser(HttpUser):
|
||||
"""Locust user for testing Dify workflow SSE endpoints"""
|
||||
|
||||
# Use constant wait for streaming workloads
|
||||
wait_time = constant(0) if os.getenv("WAIT_TIME", "0") == "0" else between(1, 3)
|
||||
|
||||
def __init__(self, *args: object, **kwargs: object) -> None:
|
||||
super().__init__(*args, **kwargs) # type: ignore[arg-type]
|
||||
|
||||
# Load API configuration
|
||||
config_helper = ConfigHelper()
|
||||
self.api_token = config_helper.get_api_key()
|
||||
|
||||
if not self.api_token:
|
||||
raise ValueError("API key not found. Please run setup_all.py first.")
|
||||
|
||||
# Load questions from file or use defaults
|
||||
if QUESTIONS_FILE and os.path.exists(QUESTIONS_FILE):
|
||||
with open(QUESTIONS_FILE) as f:
|
||||
self.questions = [line.strip() for line in f if line.strip()]
|
||||
else:
|
||||
self.questions = [
|
||||
"What is artificial intelligence?",
|
||||
"Explain quantum computing",
|
||||
"What is machine learning?",
|
||||
"How do neural networks work?",
|
||||
"What is renewable energy?",
|
||||
]
|
||||
|
||||
self.user_counter = 0
|
||||
|
||||
def on_start(self) -> None:
|
||||
"""Called when a user starts"""
|
||||
self.user_counter = 0
|
||||
|
||||
@task
|
||||
def test_workflow_stream(self) -> None:
|
||||
"""Test workflow SSE streaming endpoint"""
|
||||
|
||||
question = random.choice(self.questions)
|
||||
self.user_counter += 1
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.api_token}",
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "text/event-stream",
|
||||
"Cache-Control": "no-cache",
|
||||
}
|
||||
|
||||
data = WorkflowRequestData(
|
||||
inputs=WorkflowInputs(question=question),
|
||||
response_mode="streaming",
|
||||
user=f"user_{self.user_counter}",
|
||||
)
|
||||
|
||||
start_time = time.time()
|
||||
first_event_time = None
|
||||
event_count = 0
|
||||
inter_event_times: list[float] = []
|
||||
last_event_time = None
|
||||
ttfe = 0
|
||||
request_success = False
|
||||
bytes_received = 0
|
||||
|
||||
metrics.connection_started()
|
||||
|
||||
# Use catch_response context manager directly
|
||||
with self.client.request(
|
||||
method="POST",
|
||||
url=WORKFLOW_PATH,
|
||||
headers=headers,
|
||||
json=data,
|
||||
stream=True,
|
||||
catch_response=True,
|
||||
timeout=(CONNECT_TIMEOUT, READ_TIMEOUT),
|
||||
name="/v1/workflows/run", # Name for Locust stats
|
||||
) as response:
|
||||
try:
|
||||
# Validate response
|
||||
if response.status_code >= 400:
|
||||
error_type: ErrorType = "http_4xx" if response.status_code < 500 else "http_5xx"
|
||||
metrics.record_error(error_type)
|
||||
response.failure(f"HTTP {response.status_code}")
|
||||
return
|
||||
|
||||
content_type = response.headers.get("Content-Type", "")
|
||||
if "text/event-stream" not in content_type and "application/json" not in content_type:
|
||||
logger.error(f"Expected text/event-stream, got: {content_type}")
|
||||
metrics.record_error("invalid_response")
|
||||
response.failure(f"Invalid content type: {content_type}")
|
||||
return
|
||||
|
||||
# Parse SSE events
|
||||
parser = SSEParser()
|
||||
|
||||
for line in response.iter_lines(decode_unicode=True):
|
||||
# Check if runner is stopping
|
||||
if getattr(self.environment.runner, "state", "") in (
|
||||
"stopping",
|
||||
"stopped",
|
||||
):
|
||||
logger.debug("Runner stopping, breaking streaming loop")
|
||||
break
|
||||
|
||||
if line is not None:
|
||||
bytes_received += len(line.encode("utf-8"))
|
||||
|
||||
# Parse SSE line
|
||||
event = parser.parse_line(line if line is not None else "")
|
||||
if event:
|
||||
event_count += 1
|
||||
current_time = time.time()
|
||||
metrics.event_received()
|
||||
|
||||
# Track inter-event timing
|
||||
if last_event_time:
|
||||
inter_event_times.append((current_time - last_event_time) * 1000)
|
||||
last_event_time = current_time
|
||||
|
||||
if first_event_time is None:
|
||||
first_event_time = current_time
|
||||
ttfe = (first_event_time - start_time) * 1000
|
||||
metrics.record_ttfe(ttfe)
|
||||
|
||||
try:
|
||||
# Parse event data
|
||||
event_data = event.get("data", "")
|
||||
if event_data:
|
||||
if event_data == "[DONE]":
|
||||
logger.debug("Received [DONE] sentinel")
|
||||
request_success = True
|
||||
break
|
||||
|
||||
try:
|
||||
parsed_event: ParsedEventData = json.loads(event_data)
|
||||
# Check for terminal events
|
||||
if parsed_event.get("event") in TERMINAL_EVENTS:
|
||||
logger.debug(f"Received terminal event: {parsed_event.get('event')}")
|
||||
request_success = True
|
||||
break
|
||||
except json.JSONDecodeError as e:
|
||||
logger.debug(f"JSON decode error: {e} for data: {event_data[:100]}")
|
||||
metrics.record_error("invalid_json")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing event: {e}")
|
||||
|
||||
# Mark success only if terminal condition was met or events were received
|
||||
if request_success:
|
||||
response.success()
|
||||
elif event_count > 0:
|
||||
# Got events but no proper terminal condition
|
||||
metrics.record_error("early_termination")
|
||||
response.failure("Stream ended without terminal event")
|
||||
else:
|
||||
response.failure("No events received")
|
||||
|
||||
except (
|
||||
requests.exceptions.ConnectTimeout,
|
||||
requests.exceptions.ReadTimeout,
|
||||
) as e:
|
||||
metrics.record_error("timeout")
|
||||
response.failure(f"Timeout: {e}")
|
||||
except (
|
||||
requests.exceptions.ConnectionError,
|
||||
requests.exceptions.RequestException,
|
||||
) as e:
|
||||
metrics.record_error("connection_error")
|
||||
response.failure(f"Connection error: {e}")
|
||||
except Exception as e:
|
||||
response.failure(str(e))
|
||||
raise
|
||||
finally:
|
||||
metrics.connection_ended()
|
||||
|
||||
# Record stream metrics
|
||||
if event_count > 0:
|
||||
stream_duration = (time.time() - start_time) * 1000
|
||||
stream_metrics = StreamMetrics(
|
||||
stream_duration=stream_duration,
|
||||
events_count=event_count,
|
||||
bytes_received=bytes_received,
|
||||
ttfe=ttfe,
|
||||
inter_event_times=inter_event_times,
|
||||
)
|
||||
metrics.record_stream_metrics(stream_metrics)
|
||||
logger.debug(
|
||||
f"Stream completed: {event_count} events, {stream_duration:.1f}ms, success={request_success}"
|
||||
)
|
||||
else:
|
||||
logger.warning("No events received in stream")
|
||||
|
||||
|
||||
# Event handlers
|
||||
@events.test_start.add_listener # type: ignore[misc]
|
||||
def on_test_start(environment: object, **kwargs: object) -> None:
|
||||
logger.info("=" * 80)
|
||||
logger.info(" " * 25 + "DIFY SSE BENCHMARK - REAL-TIME METRICS")
|
||||
logger.info("=" * 80)
|
||||
logger.info(f"Started at: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
logger.info("=" * 80)
|
||||
|
||||
# Periodic stats reporting
|
||||
def report_stats() -> None:
|
||||
if not hasattr(environment, "runner"):
|
||||
return
|
||||
runner = environment.runner
|
||||
while hasattr(runner, "state") and runner.state not in ["stopped", "stopping"]:
|
||||
time.sleep(5) # Report every 5 seconds
|
||||
if hasattr(runner, "state") and runner.state == "running":
|
||||
stats = metrics.get_stats()
|
||||
|
||||
# Only log on master node in distributed mode
|
||||
is_master = (
|
||||
not getattr(environment.runner, "worker_id", None) if hasattr(environment, "runner") else True
|
||||
)
|
||||
if is_master:
|
||||
# Clear previous lines and show updated stats
|
||||
logger.info("\n" + "=" * 80)
|
||||
logger.info(
|
||||
f"{'METRIC':<25} {'CURRENT':>15} {'RATE (10s)':>15} {'AVG (overall)':>15} {'TOTAL':>12}"
|
||||
)
|
||||
logger.info("-" * 80)
|
||||
|
||||
# Active SSE Connections
|
||||
logger.info(
|
||||
f"{'Active SSE Connections':<25} {stats.active_connections:>15,d} {'-':>15} {'-':>12} {'-':>12}"
|
||||
)
|
||||
|
||||
# New Connection Rate
|
||||
logger.info(
|
||||
f"{'New Connections':<25} {'-':>15} {stats.connection_rate:>13.2f}/s {stats.overall_conn_rate:>13.2f}/s {stats.total_connections:>12,d}"
|
||||
)
|
||||
|
||||
# Event Throughput
|
||||
logger.info(
|
||||
f"{'Event Throughput':<25} {'-':>15} {stats.event_rate:>13.2f}/s {stats.overall_event_rate:>13.2f}/s {stats.total_events:>12,d}"
|
||||
)
|
||||
|
||||
logger.info("-" * 80)
|
||||
logger.info(
|
||||
f"{'TIME TO FIRST EVENT':<25} {'AVG':>15} {'P50':>10} {'P95':>10} {'MIN':>10} {'MAX':>10}"
|
||||
)
|
||||
logger.info(
|
||||
f"{'(TTFE in ms)':<25} {stats.ttfe_avg:>15.1f} {stats.ttfe_p50:>10.1f} {stats.ttfe_p95:>10.1f} {stats.ttfe_min:>10.1f} {stats.ttfe_max:>10.1f}"
|
||||
)
|
||||
logger.info(
|
||||
f"{'Window Samples':<25} {stats.ttfe_samples:>15,d} (last {min(10000, stats.ttfe_total_samples):,d} samples)"
|
||||
)
|
||||
logger.info(f"{'Total Samples':<25} {stats.ttfe_total_samples:>15,d}")
|
||||
|
||||
# Inter-event latency
|
||||
if stats.inter_event_latency_avg > 0:
|
||||
logger.info("-" * 80)
|
||||
logger.info(f"{'INTER-EVENT LATENCY':<25} {'AVG':>15} {'P50':>10} {'P95':>10}")
|
||||
logger.info(
|
||||
f"{'(ms between events)':<25} {stats.inter_event_latency_avg:>15.1f} {stats.inter_event_latency_p50:>10.1f} {stats.inter_event_latency_p95:>10.1f}"
|
||||
)
|
||||
|
||||
# Error stats
|
||||
if any(stats.error_counts.values()):
|
||||
logger.info("-" * 80)
|
||||
logger.info(f"{'ERROR TYPE':<25} {'COUNT':>15}")
|
||||
for error_type, count in stats.error_counts.items():
|
||||
if isinstance(count, int) and count > 0:
|
||||
logger.info(f"{error_type:<25} {count:>15,d}")
|
||||
|
||||
logger.info("=" * 80)
|
||||
|
||||
# Show Locust stats summary
|
||||
if hasattr(environment, "stats") and hasattr(environment.stats, "total"):
|
||||
total = environment.stats.total
|
||||
if hasattr(total, "num_requests") and total.num_requests > 0:
|
||||
logger.info(
|
||||
f"{'LOCUST STATS':<25} {'Requests':>12} {'Fails':>8} {'Avg (ms)':>12} {'Min':>8} {'Max':>8}"
|
||||
)
|
||||
logger.info("-" * 80)
|
||||
logger.info(
|
||||
f"{'Aggregated':<25} {total.num_requests:>12,d} "
|
||||
f"{total.num_failures:>8,d} "
|
||||
f"{total.avg_response_time:>12.1f} "
|
||||
f"{total.min_response_time:>8.0f} "
|
||||
f"{total.max_response_time:>8.0f}"
|
||||
)
|
||||
logger.info("=" * 80)
|
||||
|
||||
threading.Thread(target=report_stats, daemon=True).start()
|
||||
|
||||
|
||||
@events.test_stop.add_listener # type: ignore[misc]
|
||||
def on_test_stop(environment: object, **kwargs: object) -> None:
|
||||
stats = metrics.get_stats()
|
||||
test_duration = time.time() - metrics.start_time
|
||||
|
||||
# Log final results
|
||||
logger.info("\n" + "=" * 80)
|
||||
logger.info(" " * 30 + "FINAL BENCHMARK RESULTS")
|
||||
logger.info("=" * 80)
|
||||
logger.info(f"Test Duration: {test_duration:.1f} seconds")
|
||||
logger.info("-" * 80)
|
||||
|
||||
logger.info("")
|
||||
logger.info("CONNECTIONS")
|
||||
logger.info(f" {'Total Connections:':<30} {stats.total_connections:>10,d}")
|
||||
logger.info(f" {'Final Active:':<30} {stats.active_connections:>10,d}")
|
||||
logger.info(f" {'Average Rate:':<30} {stats.overall_conn_rate:>10.2f} conn/s")
|
||||
|
||||
logger.info("")
|
||||
logger.info("EVENTS")
|
||||
logger.info(f" {'Total Events Received:':<30} {stats.total_events:>10,d}")
|
||||
logger.info(f" {'Average Throughput:':<30} {stats.overall_event_rate:>10.2f} events/s")
|
||||
logger.info(f" {'Final Rate (10s window):':<30} {stats.event_rate:>10.2f} events/s")
|
||||
|
||||
logger.info("")
|
||||
logger.info("STREAM METRICS")
|
||||
logger.info(f" {'Avg Stream Duration:':<30} {stats.stream_duration_avg:>10.1f} ms")
|
||||
logger.info(f" {'P50 Stream Duration:':<30} {stats.stream_duration_p50:>10.1f} ms")
|
||||
logger.info(f" {'P95 Stream Duration:':<30} {stats.stream_duration_p95:>10.1f} ms")
|
||||
logger.info(f" {'Avg Events per Stream:':<30} {stats.events_per_stream_avg:>10.1f}")
|
||||
|
||||
logger.info("")
|
||||
logger.info("INTER-EVENT LATENCY")
|
||||
logger.info(f" {'Average:':<30} {stats.inter_event_latency_avg:>10.1f} ms")
|
||||
logger.info(f" {'Median (P50):':<30} {stats.inter_event_latency_p50:>10.1f} ms")
|
||||
logger.info(f" {'95th Percentile:':<30} {stats.inter_event_latency_p95:>10.1f} ms")
|
||||
|
||||
logger.info("")
|
||||
logger.info("TIME TO FIRST EVENT (ms)")
|
||||
logger.info(f" {'Average:':<30} {stats.ttfe_avg:>10.1f} ms")
|
||||
logger.info(f" {'Median (P50):':<30} {stats.ttfe_p50:>10.1f} ms")
|
||||
logger.info(f" {'95th Percentile:':<30} {stats.ttfe_p95:>10.1f} ms")
|
||||
logger.info(f" {'Minimum:':<30} {stats.ttfe_min:>10.1f} ms")
|
||||
logger.info(f" {'Maximum:':<30} {stats.ttfe_max:>10.1f} ms")
|
||||
logger.info(
|
||||
f" {'Window Samples:':<30} {stats.ttfe_samples:>10,d} (last {min(10000, stats.ttfe_total_samples):,d})"
|
||||
)
|
||||
logger.info(f" {'Total Samples:':<30} {stats.ttfe_total_samples:>10,d}")
|
||||
|
||||
# Error summary
|
||||
if any(stats.error_counts.values()):
|
||||
logger.info("")
|
||||
logger.info("ERRORS")
|
||||
for error_type, count in stats.error_counts.items():
|
||||
if isinstance(count, int) and count > 0:
|
||||
logger.info(f" {error_type:<30} {count:>10,d}")
|
||||
|
||||
logger.info("=" * 80 + "\n")
|
||||
|
||||
# Export machine-readable report (only on master node)
|
||||
is_master = not getattr(environment.runner, "worker_id", None) if hasattr(environment, "runner") else True
|
||||
if is_master:
|
||||
export_json_report(stats, test_duration, environment)
|
||||
|
||||
|
||||
def export_json_report(stats: MetricsSnapshot, duration: float, environment: object) -> None:
|
||||
"""Export metrics to JSON file for CI/CD analysis"""
|
||||
|
||||
reports_dir = Path(__file__).parent / "reports"
|
||||
reports_dir.mkdir(exist_ok=True)
|
||||
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
report_file = reports_dir / f"sse_metrics_{timestamp}.json"
|
||||
|
||||
# Access environment.stats.total attributes safely
|
||||
locust_stats: LocustStats | None = None
|
||||
if hasattr(environment, "stats") and hasattr(environment.stats, "total"):
|
||||
total = environment.stats.total
|
||||
if hasattr(total, "num_requests") and total.num_requests > 0:
|
||||
locust_stats = LocustStats(
|
||||
total_requests=total.num_requests,
|
||||
total_failures=total.num_failures,
|
||||
avg_response_time=total.avg_response_time,
|
||||
min_response_time=total.min_response_time,
|
||||
max_response_time=total.max_response_time,
|
||||
)
|
||||
|
||||
report_data = ReportData(
|
||||
timestamp=datetime.now().isoformat(),
|
||||
duration_seconds=duration,
|
||||
metrics=asdict(stats), # type: ignore[arg-type]
|
||||
locust_stats=locust_stats,
|
||||
)
|
||||
|
||||
with open(report_file, "w") as f:
|
||||
json.dump(report_data, f, indent=2)
|
||||
|
||||
logger.info(f"Exported metrics to {report_file}")
|
||||
@@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import base64
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from common.config_helper import ConfigHelper
|
||||
|
||||
|
||||
def _load_setup_module(name: str):
|
||||
module_path = Path(__file__).parent / "setup" / f"{name}.py"
|
||||
spec = importlib.util.spec_from_file_location(f"stress_test_{name}", module_path)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def _load_stress_test_module(name: str):
|
||||
module_path = Path(__file__).parent / f"{name}.py"
|
||||
spec = importlib.util.spec_from_file_location(f"stress_test_{name}", module_path)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_config_helper_getters_read_state_sections(tmp_path):
|
||||
helper = ConfigHelper(base_dir=tmp_path)
|
||||
helper.write_state(
|
||||
{
|
||||
"auth": {"access_token": "console-token", "csrf_token": "csrf-token"},
|
||||
"app": {"app_id": "app-id"},
|
||||
"api_key": {"token": "app-token"},
|
||||
}
|
||||
)
|
||||
|
||||
assert helper.get_token() == "console-token"
|
||||
assert helper.get_csrf_token() == "csrf-token"
|
||||
assert helper.get_app_id() == "app-id"
|
||||
assert helper.get_api_key() == "app-token"
|
||||
assert helper.console_auth_headers() == {
|
||||
"authorization": "Bearer console-token",
|
||||
"X-CSRF-Token": "csrf-token",
|
||||
}
|
||||
assert helper.console_auth_cookies() == {
|
||||
"locale": "en-US",
|
||||
"access_token": "console-token",
|
||||
"csrf_token": "csrf-token",
|
||||
}
|
||||
|
||||
|
||||
def test_login_admin_encodes_password_like_web_client():
|
||||
login_admin = _load_setup_module("login_admin")
|
||||
|
||||
encoded = login_admin.encode_sensitive_field("password123")
|
||||
|
||||
assert encoded == base64.b64encode(b"password123").decode()
|
||||
|
||||
|
||||
def test_setup_admin_reads_credentials_from_environment(monkeypatch):
|
||||
setup_admin = _load_setup_module("setup_admin")
|
||||
monkeypatch.setenv("STRESS_TEST_ADMIN_EMAIL", "real-admin@example.com")
|
||||
monkeypatch.setenv("STRESS_TEST_ADMIN_USERNAME", "real-admin")
|
||||
monkeypatch.setenv("STRESS_TEST_ADMIN_PASSWORD", "secret")
|
||||
|
||||
assert setup_admin.build_admin_config() == {
|
||||
"email": "real-admin@example.com",
|
||||
"username": "real-admin",
|
||||
"password": "secret",
|
||||
}
|
||||
|
||||
|
||||
def test_setup_all_reads_credentials_from_environment(monkeypatch):
|
||||
setup_all = _load_stress_test_module("setup_all")
|
||||
monkeypatch.setenv("STRESS_TEST_ADMIN_EMAIL", "real-admin@example.com")
|
||||
monkeypatch.setenv("STRESS_TEST_ADMIN_USERNAME", "real-admin")
|
||||
monkeypatch.setenv("STRESS_TEST_ADMIN_PASSWORD", "secret")
|
||||
|
||||
assert setup_all.build_admin_config() == {
|
||||
"email": "real-admin@example.com",
|
||||
"username": "real-admin",
|
||||
"password": "secret",
|
||||
}
|
||||
|
||||
|
||||
def test_setup_all_confirm_defaults_to_yes(monkeypatch):
|
||||
setup_all = _load_stress_test_module("setup_all")
|
||||
monkeypatch.setattr("builtins.input", lambda prompt: "")
|
||||
|
||||
assert setup_all.confirm("Continue?") is True
|
||||
|
||||
|
||||
def test_create_api_key_fails_without_app_id(tmp_path):
|
||||
create_api_key = _load_setup_module("create_api_key")
|
||||
create_api_key.config_helper.base_dir = tmp_path
|
||||
create_api_key.config_helper.write_state({"auth": {"access_token": "token"}})
|
||||
|
||||
assert create_api_key.create_api_key() is False
|
||||
|
||||
|
||||
def test_plugin_install_response_without_task_is_non_blocking():
|
||||
install_openai_plugin = _load_setup_module("install_openai_plugin")
|
||||
|
||||
assert install_openai_plugin.is_non_blocking_install_response({"ok": True}) is True
|
||||
assert install_openai_plugin.is_non_blocking_install_response({}) is True
|
||||
assert install_openai_plugin.is_non_blocking_install_response({"code": "plugin_error"}) is False
|
||||
|
||||
|
||||
def test_import_response_with_warnings_and_app_id_is_success():
|
||||
import_workflow_app = _load_setup_module("import_workflow_app")
|
||||
|
||||
assert import_workflow_app.is_successful_import_response({"status": "completed-with-warnings", "app_id": "app-id"})
|
||||
assert not import_workflow_app.is_successful_import_response({"status": "failed", "app_id": "app-id"})
|
||||
assert not import_workflow_app.is_successful_import_response({"status": "completed"})
|
||||
Reference in new issue
Block a user