chore: 纳入本地 Dify 1.19.0 源码及品牌与免登改动

演示目录只保留可重建的源码。运行密钥、数据卷、依赖和带口令的本地配置不进入仓库。

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
wuzilongandCursor committed 2026-11-18 13:59:25 +08:00
1 parent 60158935f4
commit 6519682885
15831 files changed
+3063428

No files matched your search

+376
View File
@@ -0,0 +1,376 @@
#!/usr/bin/env python3
"""Shared helpers for baseline-aware ast-grep CI guards."""
from __future__ import annotations
import argparse
import ast
import json
import re
import shutil
import subprocess
import sys
import tempfile
import textwrap
from collections import Counter
from collections.abc import Callable
from dataclasses import dataclass
from pathlib import Path
from typing import Any
REPO_ROOT = Path(__file__).resolve().parents[1]
SCAN_ROOT = Path.cwd()
RULES_ROOT = REPO_ROOT / "scripts" / "ast_grep_rules"
HUNK_PATTERN = re.compile(
r"^@@ -(?P<old_start>\d+)(?:,(?P<old_count>\d+))? \+(?P<new_start>\d+)(?:,(?P<new_count>\d+))? @@"
)
PathPredicate = Callable[[str], bool]
MatchPredicate = Callable[["Match"], bool]
@dataclass(frozen=True)
class Hunk:
old_start: int
old_count: int
new_start: int
new_count: int
def contains_old(self, line_number: int) -> bool:
return self._contains(line_number, self.old_start, self.old_count)
def contains_new(self, line_number: int) -> bool:
return self._contains(line_number, self.new_start, self.new_count)
@staticmethod
def _contains(line_number: int, start: int, count: int) -> bool:
if count == 0:
return False
return start <= line_number < start + count
@dataclass(frozen=True)
class Match:
line_number: int
source_line: str
text: str
meta_variables: dict[str, str]
@dataclass(frozen=True)
class Violation:
path: str
line_number: int
message: str
def rule_path(filename: str) -> Path:
return RULES_ROOT / filename
def parse_args(description: str | None) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=description)
diff_source_group = parser.add_mutually_exclusive_group(required=True)
diff_source_group.add_argument(
"--staged",
action="store_true",
help="Inspect staged changes against HEAD, for pre-commit style usage.",
)
diff_source_group.add_argument(
"--base-rev",
help="Inspect changes between the provided git revision and HEAD.",
)
return parser.parse_args()
def resolve_ast_grep_command() -> list[str]:
if shutil.which("ast-grep"):
return ["ast-grep"]
if shutil.which("uvx"):
return ["uvx", "--from", "ast-grep-cli", "ast-grep"]
raise RuntimeError("ast-grep executable not found")
def git_output(*args: str, allow_missing: bool = False) -> str:
completed = subprocess.run(
["git", *args],
cwd=SCAN_ROOT,
text=True,
capture_output=True,
check=False,
)
if completed.returncode == 0:
return completed.stdout
if allow_missing:
return ""
raise RuntimeError(completed.stderr.strip() or completed.stdout.strip() or "git command failed")
def collect_diff_text(args: argparse.Namespace) -> str:
if args.staged:
return git_output(
"diff",
"--cached",
"--unified=0",
"--no-renames",
"--diff-filter=AMD",
"--no-ext-diff",
)
return git_output(
"diff",
"--unified=0",
"--no-renames",
"--diff-filter=AMD",
"--no-ext-diff",
f"{args.base_rev}..HEAD",
)
def parse_changed_hunks(diff_text: str) -> dict[str, list[Hunk]]:
changed_hunks: dict[str, list[Hunk]] = {}
current_path: str | None = None
for line in diff_text.splitlines():
if line.startswith("diff --git "):
parts = line.split()
current_path = parts[3][2:]
changed_hunks.setdefault(current_path, [])
continue
if current_path is None:
continue
hunk_match = HUNK_PATTERN.match(line)
if not hunk_match:
continue
changed_hunks[current_path].append(
Hunk(
old_start=int(hunk_match.group("old_start")),
old_count=parse_count(hunk_match.group("old_count")),
new_start=int(hunk_match.group("new_start")),
new_count=parse_count(hunk_match.group("new_count")),
)
)
return {path: hunks for path, hunks in changed_hunks.items() if hunks}
def parse_count(value: str | None) -> int:
if value is None:
return 1
return int(value)
def is_python_source_path(path: str) -> bool:
return Path(path).suffix in {".py", ".pyi"}
def load_file_versions(path: str, args: argparse.Namespace) -> tuple[str, str]:
if args.staged:
return (
git_output("show", f"HEAD:{path}", allow_missing=True),
git_output("show", f":{path}", allow_missing=True),
)
return (
git_output("show", f"{args.base_rev}:{path}", allow_missing=True),
git_output("show", f"HEAD:{path}", allow_missing=True),
)
def run_ast_grep(source: str, *, rule: Path) -> list[Match]:
if not source.strip():
return []
with tempfile.NamedTemporaryFile(mode="w", suffix=".py", encoding="utf-8") as temp_file:
temp_file.write(source)
temp_file.flush()
completed = subprocess.run(
[
*resolve_ast_grep_command(),
"scan",
"--rule",
str(rule),
"--json=compact",
temp_file.name,
],
cwd=REPO_ROOT,
text=True,
capture_output=True,
check=False,
)
if completed.returncode not in (0, 1):
raise RuntimeError(completed.stderr.strip() or completed.stdout.strip() or "ast-grep command failed")
if not completed.stdout.strip():
return []
lines = source.splitlines()
raw_matches = json.loads(completed.stdout)
matches: list[Match] = []
for raw_match in raw_matches:
line_number = int(raw_match["range"]["start"]["line"]) + 1
source_line = lines[line_number - 1] if line_number - 1 < len(lines) else ""
matches.append(
Match(
line_number=line_number,
source_line=source_line,
text=extract_match_text(raw_match, fallback=source_line),
meta_variables=extract_meta_variables(raw_match),
)
)
return matches
def extract_match_text(raw_match: dict[str, Any], *, fallback: str) -> str:
text = raw_match.get("text")
if isinstance(text, str) and text:
return text
lines = raw_match.get("lines")
if isinstance(lines, str) and lines:
return lines
return fallback
def extract_meta_variables(raw_match: dict[str, Any]) -> dict[str, str]:
meta_variables = raw_match.get("metaVariables")
if not isinstance(meta_variables, dict):
return {}
single_variables = meta_variables.get("single")
if not isinstance(single_variables, dict):
return {}
result: dict[str, str] = {}
for name, value in single_variables.items():
if not isinstance(name, str) or not isinstance(value, dict):
continue
text = value.get("text")
if isinstance(text, str):
result[name] = text
return result
def has_reasoned_guard_ignore(source_line: str, rule_id: str) -> bool:
pattern = re.compile(rf"# guard-ignore: {re.escape(rule_id)} -- (?P<reason>\S.*)\s*$")
match = pattern.search(source_line)
if not match:
return False
reason = match.group("reason")
return bool(reason.strip())
def match_key(match: Match) -> str:
"""Ignore formatting when comparing moved Python syntax, retaining literal values."""
return ast.dump(ast.parse(textwrap.dedent(match.text)))
def filter_moved_matches(
added: list[tuple[str, Match]],
removed: Counter[str],
) -> list[tuple[str, Match]]:
unmatched = []
for path, match in added:
key = match_key(match)
if removed[key]:
removed[key] -= 1
else:
unmatched.append((path, match))
return unmatched
def collect_hunk_changes(
old_matches: list[Match],
new_matches: list[Match],
hunk: Hunk,
*,
is_reportable_match: MatchPredicate,
) -> tuple[Counter[str], list[Match]]:
old_in_hunk = [
match for match in old_matches if hunk.contains_old(match.line_number) and is_reportable_match(match)
]
new_in_hunk = [
match for match in new_matches if hunk.contains_new(match.line_number) and is_reportable_match(match)
]
existing = Counter(match_key(match) for match in old_in_hunk)
added = []
for match in new_in_hunk:
key = match_key(match)
if existing[key]:
existing[key] -= 1
else:
added.append(match)
# Preserve the existing allowance for replacing legacy calls within a hunk.
remaining = list(existing.elements())
replacements = min(len(remaining), len(added))
return Counter(remaining[replacements:]), added[replacements:]
def find_net_new_violations(
changed_hunks: dict[str, list[Hunk]],
args: argparse.Namespace,
*,
rule: Path,
is_scanned_path: PathPredicate,
is_reportable_match: MatchPredicate,
violation_message: str,
) -> list[Violation]:
removed: Counter[str] = Counter()
added: list[tuple[str, Match]] = []
for path, hunks in changed_hunks.items():
if not is_scanned_path(path):
continue
old_source, new_source = load_file_versions(path, args)
old_matches = run_ast_grep(old_source, rule=rule)
new_matches = run_ast_grep(new_source, rule=rule)
for hunk in hunks:
removed_in_hunk, added_in_hunk = collect_hunk_changes(
old_matches,
new_matches,
hunk,
is_reportable_match=is_reportable_match,
)
removed.update(removed_in_hunk)
added.extend((path, match) for match in added_in_hunk)
violations = [
Violation(path, match.line_number, violation_message) for path, match in filter_moved_matches(added, removed)
]
return sorted(violations, key=lambda item: (item.path, item.line_number))
def print_violations(violations: list[Violation]) -> None:
for violation in violations:
print(f"{violation.path}:{violation.line_number}: {violation.message}", file=sys.stderr)
def run_guard(
*,
description: str | None,
rule: Path,
is_scanned_path: PathPredicate,
is_reportable_match: MatchPredicate,
violation_message: str,
) -> int:
try:
args = parse_args(description)
changed_hunks = parse_changed_hunks(collect_diff_text(args))
violations = find_net_new_violations(
changed_hunks,
args,
rule=rule,
is_scanned_path=is_scanned_path,
is_reportable_match=is_reportable_match,
violation_message=violation_message,
)
except RuntimeError as exc:
print(str(exc), file=sys.stderr)
return 2
if violations:
print_violations(violations)
return 1
return 0
@@ -0,0 +1,93 @@
id: no-new-base-model
language: python
severity: error
message: New SQLAlchemy models must inherit from TypeBase instead of Base.
utils:
project-import:
kind: import_from_statement
has:
field: module_name
regex: '^(models\.base|api\.models\.base|\.+base)$'
project-base:
any:
- all:
- pattern: Base
- inside:
kind: module
stopBy: end
has:
stopBy: end
all:
- matches: project-import
- has:
stopBy: neighbor
all:
- kind: dotted_name
- regex: '^Base$'
- regex: '^(api\.)?models\.base\.Base$'
- all:
- pattern: $BASE
- inside:
kind: module
stopBy: end
has:
stopBy: end
all:
- pattern:
context: from models.base import Base as $BASE
selector: aliased_import
- inside:
stopBy: neighbor
matches: project-import
- all:
- pattern: $MODULE.Base
- inside:
kind: module
stopBy: end
has:
stopBy: end
any:
- all:
- pattern:
context: import models.base as $MODULE
selector: aliased_import
- has:
field: name
regex: '^(api\.)?models\.base$'
- all:
- pattern:
context: from models import base as $MODULE
selector: aliased_import
- inside:
kind: import_from_statement
stopBy: neighbor
has:
field: module_name
regex: '^(api\.)?models$'
- all:
- pattern: base.Base
- inside:
kind: module
stopBy: end
has:
stopBy: end
all:
- kind: import_from_statement
- has:
field: module_name
regex: '^(api\.)?models$'
- has:
stopBy: neighbor
all:
- kind: dotted_name
- regex: '^base$'
rule:
all:
- kind: class_definition
- has:
field: name
pattern: $NAME
- has:
field: superclasses
has:
matches: project-base
@@ -0,0 +1,84 @@
id: no-new-cast
language: python
severity: error
message: Use type narrowing or validate values instead of typing.cast().
utils:
typing-import:
kind: import_from_statement
has:
field: module_name
regex: '^(typing|typing_extensions)$'
cast-reference:
any:
- pattern: typing.cast
- pattern: typing_extensions.cast
- all:
- pattern: $MODULE.cast
- inside:
kind: module
stopBy: end
has:
stopBy: end
all:
- any:
- pattern:
context: import typing as $MODULE
selector: aliased_import
- pattern:
context: import typing_extensions as $MODULE
selector: aliased_import
- has:
field: name
regex: '^(typing|typing_extensions)$'
- all:
- kind: identifier
- pattern: cast
- inside:
kind: module
stopBy: end
has:
stopBy: end
all:
- matches: typing-import
- has:
stopBy: neighbor
all:
- kind: dotted_name
- regex: '^cast$'
- all:
- kind: identifier
- pattern: $IMPORTED
- inside:
kind: module
stopBy: end
has:
stopBy: end
all:
- pattern:
context: from typing import cast as $IMPORTED
selector: aliased_import
- inside:
matches: typing-import
stopBy: neighbor
rule:
all:
- kind: call
- any:
- has:
field: function
matches: cast-reference
- all:
- has:
field: function
kind: identifier
pattern: $ALIAS
- inside:
kind: module
stopBy: end
has:
stopBy: end
all:
- pattern: $ALIAS = $ORIGINAL
- has:
field: right
matches: cast-reference
@@ -0,0 +1,56 @@
id: no-new-constructor-bypass
language: python
severity: error
message: Use the class constructor to preserve initialization and validation.
utils:
allocation-in-new:
all:
- any:
- pattern: $OWNER.__new__($CLS)
- pattern: $OWNER.__new__($CLS, $$$TAIL)
- inside:
all:
- kind: function_definition
- has:
field: name
regex: '^__new__$'
- has:
field: parameters
has:
all:
- nthChild: 1
- any:
- all:
- kind: typed_parameter
- has:
pattern: $CLS
stopBy: neighbor
- pattern: $CLS
stopBy: neighbor
- inside:
kind: class_definition
stopBy:
any:
- kind: function_definition
- kind: lambda
- kind: class_definition
stopBy:
any:
- kind: function_definition
- kind: lambda
- kind: class_definition
rule:
any:
- all:
- pattern: $OWNER.__new__($$$ARGS)
- not:
matches: allocation-in-new
- all:
- pattern: $OWNER.__new__
- not:
inside:
kind: call
has:
field: function
pattern: $OWNER.__new__
stopBy: neighbor
@@ -0,0 +1,54 @@
id: no-new-controller-sqlalchemy
language: python
rule:
any:
- pattern: db.session.add($$$ARGS)
- pattern: db.session.add_all($$$ARGS)
- pattern: db.session.begin($$$ARGS)
- pattern: db.session.commit($$$ARGS)
- pattern: db.session.delete($$$ARGS)
- pattern: db.session.execute($$$ARGS)
- pattern: db.session.flush($$$ARGS)
- pattern: db.session.get($MODEL, $$$ARGS)
- pattern: db.session.merge($$$ARGS)
- pattern: db.session.refresh($$$ARGS)
- pattern: db.session.rollback($$$ARGS)
- pattern: db.session.scalar($$$ARGS)
- pattern: db.session.scalars($$$ARGS)
- pattern: session.add($$$ARGS)
- pattern: session.add_all($$$ARGS)
- pattern: session.begin($$$ARGS)
- pattern: session.commit($$$ARGS)
- pattern: session.delete($$$ARGS)
- pattern: session.execute($$$ARGS)
- pattern: session.flush($$$ARGS)
- pattern: session.get($MODEL, $$$ARGS)
- pattern: session.merge($$$ARGS)
- pattern: session.refresh($$$ARGS)
- pattern: session.rollback($$$ARGS)
- pattern: session.scalar($$$ARGS)
- pattern: session.scalars($$$ARGS)
- pattern: Session($$$ARGS)
- pattern: sessionmaker($$$ARGS)
- pattern: select($$$ARGS)
- pattern: insert($$$ARGS)
- pattern: update($$$ARGS)
- pattern: delete($$$ARGS)
- pattern: text($$$ARGS)
- pattern: sa.select($$$ARGS)
- pattern: sa.insert($$$ARGS)
- pattern: sa.update($$$ARGS)
- pattern: sa.delete($$$ARGS)
- pattern: sa.text($$$ARGS)
- pattern: sqlalchemy.select($$$ARGS)
- pattern: sqlalchemy.insert($$$ARGS)
- pattern: sqlalchemy.update($$$ARGS)
- pattern: sqlalchemy.delete($$$ARGS)
- pattern: sqlalchemy.text($$$ARGS)
- pattern: db.select($$$ARGS)
- pattern: db.insert($$$ARGS)
- pattern: db.update($$$ARGS)
- pattern: db.delete($$$ARGS)
- pattern: db.text($$$ARGS)
message: net-new direct SQLAlchemy call in controller code
severity: error
@@ -0,0 +1,12 @@
id: no-new-getattr
language: python
rule:
any:
- pattern: getattr($OBJ, $NAME)
- pattern: getattr($OBJ, $NAME, $$$REST)
- pattern: builtins.getattr($OBJ, $NAME)
- pattern: builtins.getattr($OBJ, $NAME, $$$REST)
- pattern: __builtins__.getattr($OBJ, $NAME)
- pattern: __builtins__.getattr($OBJ, $NAME, $$$REST)
message: net-new getattr() in added code
severity: error
@@ -0,0 +1,61 @@
id: no-new-mock
language: python
severity: error
message: Use a real instance instead of adding a Mock, MagicMock, or SimpleNamespace constructor.
rule:
all:
- pattern: $CALLEE($$$ARGS)
- any:
- has:
field: function
stopBy: end
regex: '(^|\.)(Mock|MagicMock|SimpleNamespace)$'
- inside:
kind: module
stopBy: end
has:
stopBy: end
all:
- pattern:
context: from package import $ORIGINAL as $CALLEE
selector: aliased_import
- any:
- all:
- has:
field: name
stopBy: end
regex: '^(Mock|MagicMock)$'
- inside:
kind: import_from_statement
stopBy: end
has:
field: module_name
stopBy: end
regex: '^unittest\.mock$'
- all:
- has:
field: name
stopBy: end
regex: '^SimpleNamespace$'
- inside:
kind: import_from_statement
stopBy: end
has:
field: module_name
stopBy: end
regex: '^types$'
- all:
- has:
field: function
kind: identifier
- inside:
kind: module
stopBy: end
has:
stopBy: end
all:
- pattern: $CALLEE = $ORIGINAL
- has:
field: right
stopBy: end
regex: '(^|\.)(Mock|MagicMock|SimpleNamespace)$'
@@ -0,0 +1,11 @@
id: no-new-session-mock
language: python
rule:
any:
- pattern: $SESSION = MagicMock($$$ARGS)
- pattern: $SESSION = Mock($$$ARGS)
constraints:
SESSION:
regex: ^(session|db_session|mock_session|read_session|write_session)$
message: net-new session mock assignment in test code
severity: error
@@ -0,0 +1,98 @@
id: no-new-stub-subclass
language: python
severity: error
message: Instantiate the production runtime class; do not bypass its constructor in a test subclass.
utils:
runtime-base:
any:
- regex: '(^|\.)(ModelInstance|ModelManager|DatasetRetrieval|TraceQueueManager|MessageBasedAppQueueManager|Thread)$'
- all:
- pattern: $BASE
- inside:
kind: module
stopBy: end
has:
stopBy: end
all:
- pattern:
context: from package import $ORIGINAL as $BASE
selector: aliased_import
- has:
field: name
stopBy: end
regex: '^(ModelInstance|ModelManager|DatasetRetrieval|TraceQueueManager|MessageBasedAppQueueManager|Thread)$'
initializer:
kind: function_definition
has:
field: name
stopBy: end
regex: ^__init__$
delegates:
has:
# Nested definitions do not initialize the containing object.
stopBy:
any:
- kind: function_definition
- kind: class_definition
- kind: lambda
any:
- pattern: super($$$ARGS).__init__($$$PARAMS)
- pattern: $BASE.__init__(self, $$$PARAMS)
dataclass-decorator:
kind: decorator
regex: '^@(dataclasses\.)?dataclass(\(|$)'
not:
has:
stopBy: end
pattern:
context: dataclass(init=False)
selector: keyword_argument
rule:
all:
- kind: class_definition
- has:
field: name
stopBy: neighbor
pattern: $NAME
- has:
field: superclasses
stopBy: neighbor
has:
stopBy: neighbor
all:
- pattern: $BASE
- matches: runtime-base
- any:
- has:
field: body
stopBy: neighbor
has:
# Only methods of this class, not constructors of nested classes.
stopBy: neighbor
all:
- matches: initializer
- not:
matches: delegates
- all:
- inside:
# A nested class must not inherit its outer class's decorator.
stopBy: neighbor
kind: decorated_definition
has:
stopBy: neighbor
matches: dataclass-decorator
- not:
has:
field: body
stopBy: neighbor
has:
stopBy: neighbor
any:
- matches: initializer
- all:
- kind: function_definition
- has:
field: name
stopBy: end
regex: ^__post_init__$
- matches: delegates
@@ -0,0 +1,63 @@
id: no-spec-mock
language: python
severity: error
message: Use a real instance or a concrete test fake instead of a spec-based mock.
rule:
all:
- pattern: $CALLEE($$$ARGS)
# Inspect direct arguments only; nested calls may have unrelated spec keywords.
- has:
field: arguments
has:
any:
- all:
- kind: keyword_argument
- has:
field: name
regex: ^spec(_set)?$
- all:
- kind: dictionary_splat
- has:
kind: dictionary
has:
kind: pair
has:
field: key
regex: ^['"]spec(_set)?['"]$
- not:
any:
- kind: keyword_argument
- kind: dictionary_splat
- kind: comment
- regex: '^[(),]$'
- any:
- has:
field: function
regex: '(^|\.)(Mock|MagicMock|AsyncMock|NonCallableMock|NonCallableMagicMock)$'
- inside:
kind: module
stopBy: end
has:
stopBy: end
all:
- pattern:
context: from unittest.mock import $ORIGINAL as $CALLEE
selector: aliased_import
- has:
field: name
regex: '^(Mock|MagicMock|AsyncMock|NonCallableMock|NonCallableMagicMock)$'
- inside:
kind: import_from_statement
has:
field: module_name
regex: '^unittest\.mock$'
- inside:
kind: module
stopBy: end
has:
stopBy: end
all:
- pattern: $CALLEE = $ORIGINAL
- has:
field: right
regex: '(^|\.)(Mock|MagicMock|AsyncMock|NonCallableMock|NonCallableMagicMock)$'
+112
View File
@@ -0,0 +1,112 @@
#!/usr/bin/env python3
"""Require TypeBase for new SQLAlchemy model inheritance.
Existing Base subclasses may be edited or migrated. New classes that inherit
from models.base.Base fail, including import aliases and qualified bases.
Class names preserve the baseline even when surrounding lines change. Renames
preserve the baseline through Git's rename detection.
CI compares committed revisions with --base-rev. Local checks inspect the
working tree and untracked Python files against the merge base with main.
Use --staged to inspect only staged changes against HEAD.
"""
from __future__ import annotations
import argparse
import sys
from collections import Counter
from pathlib import Path
from ast_grep_guard import (
Violation,
git_output,
print_violations,
rule_path,
run_ast_grep,
)
from check_no_new_test_doubles import changed_file_versions
def working_file_versions() -> dict[str, tuple[str, str]]:
baseline = "HEAD"
for revision in ("upstream/main", "origin/main", "main"):
merge_base = git_output("merge-base", "HEAD", revision, allow_missing=True).strip()
if merge_base:
baseline = merge_base
break
fields = iter(
git_output(
"diff",
"--name-status",
"-z",
"--find-renames",
"--diff-filter=AMR",
baseline,
"--",
"api",
)
.rstrip("\0")
.split("\0")
)
paths: dict[str, str] = {}
for status in fields:
if not status:
continue
old_path = next(fields)
path = next(fields) if status.startswith("R") else old_path
paths[path] = old_path
for path in git_output("ls-files", "--others", "--exclude-standard", "-z", "api").rstrip("\0").split("\0"):
paths[path] = path
versions = {}
for path in sorted(paths):
source = Path(path)
if source.suffix not in {".py", ".pyi"} or not source.is_file():
continue
versions[path] = (
git_output("show", f"{baseline}:{paths[path]}", allow_missing=True),
source.read_text(),
)
return versions
def find_new_base_models(versions: dict[str, tuple[str, str]]) -> list[Violation]:
rule = rule_path("no_new_base_model.yml")
violations = []
for path, (old_source, new_source) in versions.items():
if not Path(path).is_relative_to("api"):
continue
existing = Counter(match.meta_variables["NAME"] for match in run_ast_grep(old_source, rule=rule))
for match in run_ast_grep(new_source, rule=rule):
name = match.meta_variables["NAME"]
if existing[name]:
existing[name] -= 1
else:
violations.append(
Violation(
path,
match.line_number,
f"no-new-base-model: {name} must inherit from TypeBase.",
)
)
return violations
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
source = parser.add_mutually_exclusive_group()
source.add_argument("--base-rev", help="Compare this revision with committed HEAD.")
source.add_argument("--staged", action="store_true", help="Compare staged changes with HEAD.")
args = parser.parse_args()
try:
versions = changed_file_versions(args) if args.base_rev or args.staged else working_file_versions()
violations = find_new_base_models(versions)
except (OSError, RuntimeError) as error:
sys.stderr.write(f"{error}\n")
return 2
print_violations(violations)
return int(bool(violations))
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,87 @@
"""Reject new typing.cast and typing_extensions.cast calls in API Python files.
Preconditions: Run from the repository root with Git and ast-grep available.
Use --staged for index contents or --base-rev for committed revisions.
Without either option, inspect local API changes against the main merge base.
Postconditions: Identical existing calls retain their Git baseline.
Copies and changed calls fail. Import aliases and direct assignment aliases
are included. Other libraries' cast functions are outside this rule's scope.
This syntactic rule does not resolve arbitrary assignments or shadowed names.
Exit codes are 0 for success, 1 for violations, and 2 for check failures.
"""
from __future__ import annotations
import argparse
import sys
from collections import Counter
from pathlib import Path
from ast_grep_guard import (
Violation,
filter_moved_matches,
match_key,
print_violations,
rule_path,
run_ast_grep,
)
from check_no_new_base_model import working_file_versions
from check_no_new_test_doubles import changed_file_versions
def find_new_cast_calls(
versions: dict[str, tuple[str, str]],
) -> list[Violation]:
"""Compare cast calls in complete source versions.
Preconditions: Each key is a repository-relative path with old and new source.
Postconditions: Return API violations without changing files or source versions.
Each removed occurrence offsets one identical addition. Scan failures raise.
"""
rule = rule_path("no_new_cast.yml")
removed: Counter[str] = Counter()
added = []
for path, (old_source, new_source) in versions.items():
if Path(path).is_relative_to("api"):
existing = Counter(match_key(match) for match in run_ast_grep(old_source, rule=rule))
for match in run_ast_grep(new_source, rule=rule):
key = match_key(match)
if existing[key]:
existing[key] -= 1
else:
added.append((path, match))
removed.update(existing)
return [
Violation(
path,
match.line_number,
"no-new-cast: use type narrowing or validate values instead of typing.cast().",
)
for path, match in filter_moved_matches(added, removed)
]
def main() -> int:
"""Precondition: Git and ast-grep are available in the repository root.
Postconditions: Report violations without changing repository files.
Return 0 for success, 1 for violations, or 2 for check failures.
"""
parser = argparse.ArgumentParser(description=__doc__)
source = parser.add_mutually_exclusive_group()
source.add_argument("--base-rev", help="Compare this revision with committed HEAD.")
source.add_argument("--staged", action="store_true", help="Compare staged changes with HEAD.")
args = parser.parse_args()
try:
versions = changed_file_versions(args) if args.base_rev or args.staged else working_file_versions()
violations = find_new_cast_calls(versions)
except (OSError, RuntimeError) as error:
sys.stderr.write(f"{error}\n")
return 2
print_violations(sorted(violations, key=lambda item: (item.path, item.line_number)))
return int(bool(violations))
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,88 @@
"""Reject new __new__ calls and references that bypass constructor checks.
Preconditions: Run from the repository root with Git and ast-grep available.
Use --staged for index contents or --base-rev for committed revisions.
Without either option, inspect local API changes against the main merge base.
Postconditions: Existing matching occurrences retain their Git baseline.
Identical moves retain that baseline. Copies and changed calls fail.
Allocation for the first parameter of a class's __new__ method is allowed.
Nested functions and lambdas do not inherit that allocation permission.
This syntactic rule does not resolve dynamic attribute access.
Exit codes are 0 for success, 1 for violations, and 2 for check failures.
"""
from __future__ import annotations
import argparse
import sys
from collections import Counter
from pathlib import Path
from ast_grep_guard import (
Violation,
filter_moved_matches,
match_key,
print_violations,
rule_path,
run_ast_grep,
)
from check_no_new_base_model import working_file_versions
from check_no_new_test_doubles import changed_file_versions
def find_new_constructor_bypasses(
versions: dict[str, tuple[str, str]],
) -> list[Violation]:
"""Compare constructor access in complete source versions.
Preconditions: Each key is a repository-relative path with old and new source.
Postconditions: Return API violations without changing files or source versions.
Each removed occurrence offsets one identical addition. Scan failures raise.
"""
rule = rule_path("no_new_constructor_bypass.yml")
removed: Counter[str] = Counter()
added = []
for path, (old_source, new_source) in versions.items():
if Path(path).is_relative_to("api"):
existing = Counter(match_key(match) for match in run_ast_grep(old_source, rule=rule))
for match in run_ast_grep(new_source, rule=rule):
key = match_key(match)
if existing[key]:
existing[key] -= 1
else:
added.append((path, match))
removed.update(existing)
return [
Violation(
path,
match.line_number,
"no-new-constructor-bypass: use the class constructor to preserve initialization and validation.",
)
for path, match in filter_moved_matches(added, removed)
]
def main() -> int:
"""Precondition: Git and ast-grep are available in the repository root.
Postconditions: Report violations without changing repository files.
Return 0 for success, 1 for violations, or 2 for check failures.
"""
parser = argparse.ArgumentParser(description=__doc__)
source = parser.add_mutually_exclusive_group()
source.add_argument("--base-rev", help="Compare this revision with committed HEAD.")
source.add_argument("--staged", action="store_true", help="Compare staged changes with HEAD.")
args = parser.parse_args()
try:
versions = changed_file_versions(args) if args.base_rev or args.staged else working_file_versions()
violations = find_new_constructor_bypasses(versions)
except (OSError, RuntimeError) as error:
sys.stderr.write(f"{error}\n")
return 2
print_violations(sorted(violations, key=lambda item: (item.path, item.line_number)))
return int(bool(violations))
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,61 @@
#!/usr/bin/env python3
"""Block net-new direct SQLAlchemy calls in controller code.
Allowing flush() and commit() to allow controllers to control
the transaction border.
"""
from __future__ import annotations
import re
from pathlib import Path
from ast_grep_guard import Match, has_reasoned_guard_ignore, rule_path, run_guard
RULE_ID = "no-new-controller-sqlalchemy"
RULE_PATH = rule_path("no_new_controller_sqlalchemy.yml")
CONTROLLER_ROOT = Path("api/controllers")
FLASK_SESSION_GET_PATTERN = re.compile(r"^session\.get\(\s*['\"]")
ALLOWED_SESSION_METHODS = frozenset({"commit", "flush"})
VIOLATION_MESSAGE = "no-new-controller-sqlalchemy net-new direct SQLAlchemy call in controller code"
def is_controller_python_source_path(path: str) -> bool:
source_path = Path(path)
return source_path.suffix in {".py", ".pyi"} and source_path.is_relative_to(CONTROLLER_ROOT)
def is_allowed_session_boundary(match: Match) -> bool:
method = match.meta_variables.get("METHOD") or match.meta_variables.get("SESSION_METHOD")
if method in ALLOWED_SESSION_METHODS:
return True
stripped_text = match.text.strip()
return stripped_text.startswith(("db.session.commit(", "db.session.flush(", "session.commit(", "session.flush("))
def is_flask_session_get(match: Match) -> bool:
return bool(FLASK_SESSION_GET_PATTERN.match(match.text.strip()))
def is_suppressed(match: Match) -> bool:
return has_reasoned_guard_ignore(match.source_line, RULE_ID)
def is_reportable_match(match: Match) -> bool:
return not is_allowed_session_boundary(match) and not is_flask_session_get(match) and not is_suppressed(match)
def main() -> int:
return run_guard(
description=__doc__,
rule=RULE_PATH,
is_scanned_path=is_controller_python_source_path,
is_reportable_match=is_reportable_match,
violation_message=VIOLATION_MESSAGE,
)
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,29 @@
#!/usr/bin/env python3
"""Block net-new getattr() usage in staged changes or against an explicit base revision."""
from __future__ import annotations
from ast_grep_guard import Match, has_reasoned_guard_ignore, is_python_source_path, rule_path, run_guard
RULE_ID = "no-new-getattr"
RULE_PATH = rule_path("no_new_getattr.yml")
VIOLATION_MESSAGE = "no-new-getattr net-new getattr() in added code"
def is_reportable_match(match: Match) -> bool:
return not has_reasoned_guard_ignore(match.source_line, RULE_ID)
def main() -> int:
return run_guard(
description=__doc__,
rule=RULE_PATH,
is_scanned_path=is_python_source_path,
is_reportable_match=is_reportable_match,
violation_message=VIOLATION_MESSAGE,
)
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,48 @@
#!/usr/bin/env python3
"""Block net-new session mock assignments in unit tests.
This catches net-new top-level session mocks like
`session = MagicMock()` or `db_session = Mock()` added to
`tests/unit_tests/**`. Pre-existing matches are ignored via the
shared baseline diff (`scripts/ast_grep_guard.py`), so historical
mocks that haven't been cleaned up yet do not block the gate.
A `# guard-ignore: no-new-session-mock -- <reason>` line on the same
source line suppresses a violation with a human-readable reason.
"""
from __future__ import annotations
from pathlib import Path
from ast_grep_guard import Match, has_reasoned_guard_ignore, is_python_source_path, rule_path, run_guard
RULE_ID = "no-new-session-mock"
RULE_PATH = rule_path("no_new_session_mock.yml")
TEST_ROOT = Path("api/tests/unit_tests")
VIOLATION_MESSAGE = "no-new-session-mock net-new session mock assignment in test code"
def is_test_source_path(path: str) -> bool:
if not is_python_source_path(path):
return False
return Path(path).is_relative_to(TEST_ROOT)
def is_reportable_match(match: Match) -> bool:
return not has_reasoned_guard_ignore(match.source_line, RULE_ID)
def main() -> int:
return run_guard(
description=__doc__,
rule=RULE_PATH,
is_scanned_path=is_test_source_path,
is_reportable_match=is_reportable_match,
violation_message=VIOLATION_MESSAGE,
)
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,182 @@
"""Reject net-new mock constructors and constructor-bypassing runtime subclasses.
Mock, MagicMock and SimpleNamespace calls are checked in all changed Python
sources, including common qualified names, import aliases and direct factory
aliases. AsyncMock and method patching are outside this rule's scope.
The subclass rule covers the audited ModelInstance, ModelManager,
DatasetRetrieval, TraceQueueManager, MessageBasedAppQueueManager and Thread
base names (including qualified names and import aliases) in api/tests/. It
checks explicit __init__ overrides and dataclass-generated initializers. This
is a syntactic guard, not general Python inheritance or execution analysis.
Protocol and Pydantic model implementations are not targeted.
Both rules retain the provided Git baseline. Identical mock calls are counted
per file, with removed occurrences available to match code moved between files:
an additional copy or a different constructor call is rejected.
Git-detected renames retain the source file's baseline, including paths with
spaces. Subclass matches are compared by class name within each changed file,
not only by the line of the class header:
removing a super call or adding a dataclass decorator must still be detected.
An existing offending class can be edited or removed without adding another.
For a justified subclass exception only, add
`# guard-ignore: no-new-stub-subclass -- <reason>` on the class header.
"""
from __future__ import annotations
import argparse
import ast
import sys
import textwrap
from collections import Counter
from dataclasses import replace
from pathlib import Path
from ast_grep_guard import (
Match,
Violation,
filter_moved_matches,
git_output,
has_reasoned_guard_ignore,
is_python_source_path,
match_key,
parse_args,
print_violations,
rule_path,
run_ast_grep,
)
STUB_RULE_ID = "no-new-stub-subclass"
def changed_file_versions(args: argparse.Namespace) -> dict[str, tuple[str, str]]:
"""Read Git blobs, preserving rename baselines and NUL-delimited filenames."""
revisions = ["--cached"] if args.staged else [f"{args.base_rev}..HEAD"]
status_text = git_output(
"diff",
"--name-status",
"-z",
"--find-renames",
"--diff-filter=AMRD",
"--no-ext-diff",
*revisions,
)
if not status_text:
return {}
fields = iter(status_text.rstrip("\0").split("\0"))
old_revision = "HEAD" if args.staged else args.base_rev
new_revision = "" if args.staged else "HEAD"
versions: dict[str, tuple[str, str]] = {}
for status in fields:
try:
old_path = next(fields)
path = next(fields) if status.startswith("R") else old_path
except StopIteration as exc:
raise RuntimeError("Incomplete Git name-status output") from exc
if not is_python_source_path(path):
continue
old_source = "" if status == "A" else git_output("show", f"{old_revision}:{old_path}")
new_source = "" if status == "D" else git_output("show", f"{new_revision}:{path}")
versions[path] = (old_source, new_source)
return versions
def mock_matches(source: str) -> list[Match]:
"""Keep import renames from turning existing constructors into new calls."""
bindings: dict[str, str] = {}
for node in ast.parse(source).body:
if isinstance(node, ast.Import):
for alias in node.names:
bindings[alias.asname or alias.name.split(".")[0]] = (
alias.name if alias.asname else alias.name.split(".")[0]
)
elif isinstance(node, ast.ImportFrom):
for alias in node.names:
bindings[alias.asname or alias.name] = alias.name
matches = []
for match in run_ast_grep(source, rule=rule_path("no_new_mock.yml")):
tree = ast.parse(textwrap.dedent(match.text))
for node in ast.walk(tree):
if isinstance(node, ast.Name) and node.id in bindings:
node.id = bindings[node.id]
matches.append(replace(match, text=ast.unparse(tree)))
return matches
def find_new_mock_calls(changed: dict[str, tuple[str, str]]) -> list[Violation]:
"""Keep identical baseline calls, but catch alias changes outside their hunks."""
removed: Counter[str] = Counter()
added: list[tuple[str, Match]] = []
for path, (old_source, new_source) in changed.items():
if old_source == new_source:
continue
old_calls = Counter(match_key(match) for match in mock_matches(old_source))
for match in mock_matches(new_source):
key = match_key(match)
if old_calls[key]:
old_calls[key] -= 1
continue
added.append((path, match))
removed.update(old_calls)
return [
Violation(
path,
match.line_number,
"no-new-mock: instantiate a real class instead of Mock, MagicMock or SimpleNamespace",
)
for path, match in filter_moved_matches(added, removed)
]
def is_reportable_subclass(match: Match) -> bool:
return not has_reasoned_guard_ignore(match.source_line, STUB_RULE_ID)
def find_new_stub_subclasses(changed: dict[str, tuple[str, str]]) -> list[Violation]:
removed: Counter[str] = Counter()
added: list[tuple[str, Match]] = []
for path, (old_source, new_source) in changed.items():
if not Path(path).is_relative_to("api/tests"):
continue
old_matches = run_ast_grep(old_source, rule=rule_path("no_new_stub_subclass.yml"))
new_matches = run_ast_grep(new_source, rule=rule_path("no_new_stub_subclass.yml"))
old_names: dict[str, list[Match]] = {}
for match in old_matches:
if is_reportable_subclass(match):
old_names.setdefault(match.meta_variables["NAME"], []).append(match)
for match in new_matches:
if not is_reportable_subclass(match):
continue
name = match.meta_variables["NAME"]
if old_names.get(name):
old_names[name].pop()
continue
added.append((path, match))
removed.update(match_key(match) for matches in old_names.values() for match in matches)
return [
Violation(
path,
match.line_number,
f"{STUB_RULE_ID}: {match.meta_variables['NAME']} bypasses a runtime constructor; "
"instantiate the real class",
)
for path, match in filter_moved_matches(added, removed)
]
def main() -> int:
try:
args = parse_args(__doc__)
changed = changed_file_versions(args)
violations = find_new_mock_calls(changed)
violations.extend(find_new_stub_subclasses(changed))
except (OSError, RuntimeError) as exc:
print(str(exc), file=sys.stderr)
return 2
print_violations(sorted(violations, key=lambda item: (item.path, item.line_number)))
return 1 if violations else 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,47 @@
#!/usr/bin/env python3
"""Reject spec-based mock constructors in the API, including unchanged files.
Use real model/SDK instances or concrete test fakes. The rule covers explicit
spec/spec_set keywords, positional specs, qualified constructors, import aliases,
and direct constructor aliases. It does not prohibit ordinary method mocks or
create_autospec. Optional paths allow a focused local scan; CI scans all of api/.
"""
from __future__ import annotations
import argparse
import subprocess
import sys
from pathlib import Path
from ast_grep_guard import REPO_ROOT, resolve_ast_grep_command, rule_path
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("paths", nargs="*", type=Path, help="Files or directories to scan (default: api/).")
args = parser.parse_args()
paths = args.paths or [REPO_ROOT / "api"]
try:
for path in paths:
if not path.exists():
raise RuntimeError(f"Scan path does not exist: {path}")
return subprocess.run(
[
*resolve_ast_grep_command(),
"scan",
"--rule",
str(rule_path("no_spec_mock.yml")),
"--report-style",
"short",
*(str(path) for path in paths),
],
check=False,
).returncode
except (OSError, RuntimeError) as exc:
sys.stderr.write(f"{exc}\n")
return 2
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,223 @@
#!/usr/bin/env python3
"""Report direct SQLAlchemy usage in controller code.
This is the full-inventory companion to ``check_no_new_controller_sqlalchemy.py``.
It scans the current tree instead of a git diff, so migration work can inspect
existing controller usage without turning historical debt into a CI failure.
"""
from __future__ import annotations
import argparse
import json
import re
import sys
from collections import Counter
from collections.abc import Iterable, Sequence
from dataclasses import asdict, dataclass
from pathlib import Path
import check_no_new_controller_sqlalchemy as controller_sqlalchemy_guard
from ast_grep_guard import Match, REPO_ROOT, run_ast_grep
DEFAULT_CONTROLLER_DIR = REPO_ROOT / "api" / "controllers"
MAX_TEXT_LENGTH = 180
SESSION_METHOD_PATTERN = re.compile(r"^(?:db\.session|session)\.(?P<method>[A-Za-z_][A-Za-z0-9_]*)\(")
@dataclass(frozen=True)
class Finding:
classification: str
category: str
file: str
line: int
text: str
source_line: str
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"paths",
nargs="*",
help="Files or directories to scan. Defaults to api/controllers.",
)
parser.add_argument("--include-allowed", action="store_true", help="Print allowed flush()/commit() findings.")
parser.add_argument("--include-suppressed", action="store_true", help="Print reasoned guard suppressions.")
parser.add_argument("--summary-only", action="store_true", help="Print counts without per-finding details.")
parser.add_argument("--json", action="store_true", help="Emit machine-readable JSON.")
parser.add_argument(
"--fail-on-findings",
action="store_true",
help="Treat reportable direct SQLAlchemy usage as a failure. By default this scanner is report-only.",
)
return parser.parse_args()
def iter_python_files(paths: Iterable[Path]) -> Iterable[Path]:
for path in paths:
if path.is_file() and path.suffix in {".py", ".pyi"}:
yield path
elif path.is_dir():
yield from sorted(child for child in path.rglob("*.py") if child.is_file())
def display_path(path: Path) -> str:
try:
return str(path.relative_to(REPO_ROOT))
except ValueError:
return str(path)
def compact_text(text: str) -> str:
compact = " ".join(text.split())
if len(compact) <= MAX_TEXT_LENGTH:
return compact
return f"{compact[: MAX_TEXT_LENGTH - 3]}..."
def category_for_match(match: Match) -> str:
text = match.text.strip()
method_match = SESSION_METHOD_PATTERN.match(text)
method = method_match.group("method") if method_match else None
if text.startswith("db.session."):
return f"scoped-session.{method or 'call'}"
if text.startswith("session."):
return f"explicit-session.{method or 'call'}"
if text.startswith(("Session(", "sessionmaker(")):
return "session-factory"
if text.startswith(("sa.", "sqlalchemy.", "db.")):
return "sql-expression-qualified"
return "sql-expression"
def classification_for_match(match: Match) -> str:
if controller_sqlalchemy_guard.is_allowed_session_boundary(match):
return "allowed"
if controller_sqlalchemy_guard.is_suppressed(match):
return "suppressed"
if controller_sqlalchemy_guard.is_flask_session_get(match):
return "ignored"
return "reportable"
def findings_for_file(path: Path) -> list[Finding]:
source = path.read_text(encoding="utf-8")
findings: list[Finding] = []
for match in run_ast_grep(source, rule=controller_sqlalchemy_guard.RULE_PATH):
findings.append(
Finding(
classification=classification_for_match(match),
category=category_for_match(match),
file=display_path(path),
line=match.line_number,
text=compact_text(match.text),
source_line=match.source_line.strip(),
)
)
return findings
def print_text_report(
findings: Sequence[Finding],
*,
include_allowed: bool,
include_suppressed: bool,
summary_only: bool,
) -> None:
counts = Counter(finding.classification for finding in findings)
reportable = [finding for finding in findings if finding.classification == "reportable"]
category_counts = Counter(finding.category for finding in reportable)
file_counts = Counter(finding.file for finding in reportable)
sys.stdout.write(
"Controller SQLAlchemy scan: "
f"{counts['reportable']} reportable, "
f"{counts['allowed']} allowed, "
f"{counts['suppressed']} suppressed, "
f"{counts['ignored']} ignored\n"
)
if category_counts:
sys.stdout.write("\nREPORTABLE BY CATEGORY:\n")
for category, count in sorted(category_counts.items()):
sys.stdout.write(f"- {category}: {count}\n")
if file_counts:
sys.stdout.write("\nREPORTABLE BY FILE:\n")
for file, count in sorted(file_counts.items()):
sys.stdout.write(f"- {file}: {count}\n")
if summary_only:
return
print_findings("REPORTABLE", reportable)
if include_allowed:
print_findings("ALLOWED", [finding for finding in findings if finding.classification == "allowed"])
if include_suppressed:
print_findings("SUPPRESSED", [finding for finding in findings if finding.classification == "suppressed"])
def print_findings(title: str, findings: Sequence[Finding]) -> None:
if not findings:
return
sys.stdout.write(f"\n{title}:\n")
for finding in findings:
sys.stdout.write(f"- {finding.file}:{finding.line} [{finding.category}] {finding.text}\n")
def jsonable_findings(
findings: Sequence[Finding],
*,
include_allowed: bool,
include_suppressed: bool,
summary_only: bool,
) -> dict[str, object]:
visible = [
finding
for finding in findings
if finding.classification == "reportable"
or (include_allowed and finding.classification == "allowed")
or (include_suppressed and finding.classification == "suppressed")
]
counts = Counter(finding.classification for finding in findings)
return {
"summary": dict(sorted(counts.items())),
"findings": [] if summary_only else [asdict(finding) for finding in visible],
}
def main() -> int:
args = parse_args()
raw_paths = args.paths or [str(DEFAULT_CONTROLLER_DIR)]
paths = [path if path.is_absolute() else REPO_ROOT / path for path in map(Path, raw_paths)]
findings: list[Finding] = []
for path in iter_python_files(paths):
findings.extend(findings_for_file(path.resolve()))
findings.sort(key=lambda finding: (finding.file, finding.line, finding.category, finding.text))
if args.json:
payload = jsonable_findings(
findings,
include_allowed=bool(args.include_allowed),
include_suppressed=bool(args.include_suppressed),
summary_only=bool(args.summary_only),
)
sys.stdout.write(f"{json.dumps(payload, indent=2, sort_keys=True)}\n")
else:
print_text_report(
findings,
include_allowed=bool(args.include_allowed),
include_suppressed=bool(args.include_suppressed),
summary_only=bool(args.summary_only),
)
has_reportable = any(finding.classification == "reportable" for finding in findings)
return int(bool(args.fail_on_findings) and has_reportable)
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,98 @@
# Dify SSE stress tests
This Locust suite exercises streaming `POST /v1/workflows/run` requests against a local Dify instance with a mock OpenAI server. It measures connection counts, event throughput, time to first event (TTFE), stream duration, inter-event latency, and failures.
Run the following commands from the repository root. Keep long-running services in separate terminals.
## Prepare the runtime
Complete the [backend setup](../../api/README.md), including middleware, migrations, the plugin daemon, and workers. For measurements, run the API with Gunicorn instead of the development server:
```bash
uv run --project api gunicorn --chdir api \
--bind 0.0.0.0:5001 --workers 4 --worker-class gevent \
--timeout 120 --keep-alive 5 app:app
```
The worker count is a starting configuration; tune it for the machine and record it with the results.
Start the mock provider before running setup:
```bash
uv run --project api python scripts/stress-test/setup/mock_openai_server.py
```
Then provision the benchmark app, plugin configuration, and API key:
```bash
STRESS_TEST_ADMIN_EMAIL='your-admin@example.com' \
STRESS_TEST_ADMIN_USERNAME='dify' \
STRESS_TEST_ADMIN_PASSWORD='your-password' \
uv run --project api python scripts/stress-test/setup_all.py
```
Setup creates the first admin on an uninitialized instance or signs in with the supplied account on an existing instance. `STRESS_TEST_ADMIN_USERNAME` is used only for initial setup. It writes credentials and the app API key to `scripts/stress-test/setup/config/stress_test_state.json`; keep that local file private.
## Run
```bash
./scripts/stress-test/run_locust_stress_test.sh
```
The runner checks the API on port 5001, the mock provider on port 5004, and the saved API key. It prompts for headless mode or the Web UI at <http://localhost:8089>. Locust runs in its own environment through `uvx --from locust`.
For headless runs, the wrapper reads `users`, `spawn-rate`, and `run-time` from [`locust.conf`](locust.conf). To set options directly, invoke Locust from the repository root:
```bash
uvx --from locust locust -f scripts/stress-test/sse_benchmark.py \
--host http://localhost:5001 --users 50 --spawn-rate 5 \
--run-time 1m --headless
```
For the Web UI:
```bash
uvx --from locust locust -f scripts/stress-test/sse_benchmark.py \
--host http://localhost:5001 --web-port 8089
```
The wrapper fixes the target to localhost; use the direct invocation to change the host. Direct invocations must set their own options or explicitly load `scripts/stress-test/locust.conf` with `--config`.
## Workload configuration
[`sse_benchmark.py`](sse_benchmark.py) reads these environment variables:
| Variable | Default | Purpose |
| --- | --- | --- |
| `WORKFLOW_PATH` | `/v1/workflows/run` | Request path |
| `CONNECT_TIMEOUT` | `10` | Connection timeout in seconds |
| `READ_TIMEOUT` | `60` | Stream read timeout in seconds |
| `TERMINAL_EVENTS` | `workflow_finished,error` | Comma-separated terminal event names |
| `QUESTIONS_FILE` | Empty | Text file with one nonempty question per line; otherwise uses built-in questions |
| `WAIT_TIME` | `0` | `0` sends the next request immediately; any other value selects a random 1–3 second wait |
Requests pass the selected question as `inputs.question`. Keep this aligned with the benchmark workflow. A custom question file avoids editing the benchmark source.
## Reports and interpretation
Headless wrapper runs create `scripts/stress-test/reports/YYYYMMDD_HHMMSS/` containing:
- `locust_summary.txt`: console output.
- `locust_report.html`: Locust report.
- `locust_stats.csv` and `locust_stats_history.csv`: request statistics.
- `sse_metrics_YYYYMMDD_HHMMSS.json`: custom SSE metrics.
Direct runs write custom SSE JSON under `scripts/stress-test/reports/`; request CSV or HTML explicitly with Locust options. Live SSE metrics update every five seconds.
Compare results with the same workflow, mock response behavior, hardware, worker configuration, and load profile. TTFE measures arrival of the first SSE event, not necessarily the first model token. Use the custom stream-duration metrics to assess stream completion separately from the standard Locust request timings. This mock-provider workload does not establish real-model latency or a universal production capacity threshold.
For failures, inspect API, worker, plugin-daemon, and mock-server logs. Check timeouts, CPU, memory, database connections, and client resource limits before changing concurrency or server settings.
## Maintenance
- Setup and fixture provisioning: `setup_all.py` and `setup/`.
- Load defaults: `locust.conf`.
- Stream parsing and metrics: `sse_benchmark.py`.
- Service checks and report collection: `run_locust_stress_test.sh`.
`cleanup.py` removes local setup state and reports, not the resources created in Dify. Preserve reports you need before running it; it asks for confirmation only in an interactive terminal.
+88
View File
@@ -0,0 +1,88 @@
#!/usr/bin/env python3
import shutil
import sys
from pathlib import Path
from common import Logger
def cleanup() -> None:
"""Clean up all configuration files and reports created during setup and stress testing."""
log = Logger("Cleanup")
log.header("Stress Test Cleanup")
config_dir = Path(__file__).parent / "setup" / "config"
reports_dir = Path(__file__).parent / "reports"
dirs_to_clean = []
if config_dir.exists():
dirs_to_clean.append(config_dir)
if reports_dir.exists():
dirs_to_clean.append(reports_dir)
if not dirs_to_clean:
log.success("No directories to clean. Everything is already clean.")
return
log.info("Cleaning up stress test data...")
log.info("This will remove:")
for dir_path in dirs_to_clean:
log.list_item(str(dir_path))
# List files that will be deleted
log.separator()
if config_dir.exists():
config_files = list(config_dir.glob("*.json"))
if config_files:
log.info("Config files to be removed:")
for file in config_files:
log.list_item(file.name)
if reports_dir.exists():
report_files = list(reports_dir.glob("*"))
if report_files:
log.info("Report files to be removed:")
for file in report_files:
log.list_item(file.name)
# Ask for confirmation if running interactively
if sys.stdin.isatty():
log.separator()
log.warning("This action cannot be undone!")
confirmation = input("Are you sure you want to remove all config and report files? (yes/no): ")
if confirmation.lower() not in ["yes", "y"]:
log.error("Cleanup cancelled.")
return
try:
# Remove directories and all their contents
for dir_path in dirs_to_clean:
shutil.rmtree(dir_path)
log.success(f"{dir_path.name} directory removed successfully!")
log.separator()
log.info("To run the setup again, execute:")
log.list_item("python setup_all.py")
log.info("Or run scripts individually in this order:")
log.list_item("python setup/mock_openai_server.py (in a separate terminal)")
log.list_item("python setup/setup_admin.py")
log.list_item("python setup/login_admin.py")
log.list_item("python setup/install_openai_plugin.py")
log.list_item("python setup/configure_openai_plugin.py")
log.list_item("python setup/import_workflow_app.py")
log.list_item("python setup/create_api_key.py")
log.list_item("python setup/publish_workflow.py")
log.list_item("python setup/run_workflow.py")
except PermissionError as e:
log.error(f"Permission denied: {e}")
log.info("Try running with appropriate permissions.")
except Exception as e:
log.error(f"An error occurred during cleanup: {e}")
if __name__ == "__main__":
cleanup()
@@ -0,0 +1,6 @@
"""Common utilities for Dify benchmark suite."""
from .config_helper import config_helper
from .logger_helper import Logger, ProgressLogger
__all__ = ["Logger", "ProgressLogger", "config_helper"]
@@ -0,0 +1,313 @@
#!/usr/bin/env python3
import json
from pathlib import Path
from typing import NotRequired, TypedDict
class AdminConfig(TypedDict):
"""Configuration for admin section."""
email: str
username: str
password: str
class AuthConfig(TypedDict):
"""Configuration for authentication section."""
access_token: str
refresh_token: NotRequired[str]
csrf_token: NotRequired[str]
expires_at: NotRequired[int]
class AppConfig(TypedDict):
"""Configuration for app section."""
app_id: str
app_name: NotRequired[str]
description: NotRequired[str]
class ApiKeyConfig(TypedDict):
"""Configuration for API key section."""
token: str
key_name: NotRequired[str]
expires_at: NotRequired[int]
class StressTestState(TypedDict):
"""Complete stress test state structure."""
admin: NotRequired[AdminConfig]
auth: NotRequired[AuthConfig]
app: NotRequired[AppConfig]
api_key: NotRequired[ApiKeyConfig]
class ConfigHelper:
_LEGACY_SECTION_MAP = {
"admin_config": "admin",
"token_config": "auth",
"app_config": "app",
"api_key_config": "api_key",
}
"""Helper class for reading and writing configuration files."""
def __init__(self, base_dir: Path | None = None):
"""Initialize ConfigHelper with base directory.
Args:
base_dir: Base directory for config files. If None, uses setup/config
"""
if base_dir is None:
# Default to config directory in setup folder
base_dir = Path(__file__).parent.parent / "setup" / "config"
self.base_dir = base_dir
self.state_file = "stress_test_state.json"
def ensure_config_dir(self) -> None:
"""Ensure the config directory exists."""
self.base_dir.mkdir(exist_ok=True, parents=True)
def get_config_path(self, filename: str) -> Path:
"""Get the full path for a config file.
Args:
filename: Name of the config file (e.g., 'admin_config.json')
Returns:
Full path to the config file
"""
if not filename.endswith(".json"):
filename += ".json"
return self.base_dir / filename
def read_config[T](self, filename: str) -> T | None:
"""Read a configuration file with generic return type.
DEPRECATED: Use read_state() or get_state_section() for new code.
This method provides backward compatibility.
Args:
filename: Name of the config file to read
Returns:
Configuration data of type T, or None if file doesn't exist
"""
# Provide backward compatibility for old config names
if filename in self._LEGACY_SECTION_MAP:
section_data = self.get_state_section(self._LEGACY_SECTION_MAP[filename])
return section_data # type: ignore
config_path = self.get_config_path(filename)
if not config_path.exists():
return None
try:
with open(config_path) as f:
return json.load(f) # type: ignore
except (OSError, json.JSONDecodeError) as e:
print(f"❌ Error reading {filename}: {e}")
return None
def write_config[T](self, filename: str, data: T) -> bool:
"""Write data to a configuration file.
DEPRECATED: Use write_state() or update_state_section() for new code.
This method provides backward compatibility.
Args:
filename: Name of the config file to write
data: Data to save (must be JSON serializable)
Returns:
True if successful, False otherwise
"""
# Provide backward compatibility for old config names
if filename in self._LEGACY_SECTION_MAP:
return self.update_state_section(
self._LEGACY_SECTION_MAP[filename],
data, # type: ignore
)
self.ensure_config_dir()
config_path = self.get_config_path(filename)
try:
with open(config_path, "w") as f:
json.dump(data, f, indent=2) # type: ignore
return True
except OSError as e:
print(f"❌ Error writing {filename}: {e}")
return False
def config_exists(self, filename: str) -> bool:
"""Check if a config file exists.
Args:
filename: Name of the config file to check
Returns:
True if file exists, False otherwise
"""
return self.get_config_path(filename).exists()
def delete_config(self, filename: str) -> bool:
"""Delete a configuration file.
Args:
filename: Name of the config file to delete
Returns:
True if successful, False otherwise
"""
config_path = self.get_config_path(filename)
if not config_path.exists():
return True # Already doesn't exist
try:
config_path.unlink()
return True
except OSError as e:
print(f"❌ Error deleting {filename}: {e}")
return False
def read_state(self) -> StressTestState | None:
"""Read the entire stress test state.
Returns:
Dictionary containing all state data, or None if file doesn't exist
"""
state_path = self.get_config_path(self.state_file)
if not state_path.exists():
return None
try:
with open(state_path) as f:
data = json.load(f)
# Validate basic structure
if not isinstance(data, dict):
print(f"❌ Invalid state format in {self.state_file}")
return None
return data # type: ignore
except (OSError, json.JSONDecodeError) as e:
print(f"❌ Error reading {self.state_file}: {e}")
return None
def write_state(self, data: StressTestState) -> bool:
"""Write the entire stress test state.
Args:
data: Dictionary containing all state data to save
Returns:
True if successful, False otherwise
"""
self.ensure_config_dir()
state_path = self.get_config_path(self.state_file)
try:
with open(state_path, "w") as f:
json.dump(data, f, indent=2)
return True
except OSError as e:
print(f"❌ Error writing {self.state_file}: {e}")
return False
def update_state_section[T](self, section: str, data: T) -> bool:
"""Update a specific section of the stress test state.
Args:
section: Name of the section to update (e.g., 'admin', 'auth', 'app', 'api_key')
data: Section data to save
Returns:
True if successful, False otherwise
"""
state = self.read_state() or {}
state[section] = data # type: ignore
return self.write_state(state) # type: ignore
def get_state_section[T](self, section: str) -> T | None:
"""Get a specific section from the stress test state.
Args:
section: Name of the section to get (e.g., 'admin', 'auth', 'app', 'api_key')
Returns:
Section data of type T, or None if not found
"""
state = self.read_state()
if state:
return state.get(section) # type: ignore
return None
def get_token(self) -> str | None:
"""Get the access token from auth section.
Returns:
Access token string or None if not found
"""
auth = self.get_state_section("auth")
if auth:
return auth.get("access_token")
return None
def get_csrf_token(self) -> str | None:
"""Get the CSRF token from auth section."""
auth = self.get_state_section("auth")
if auth:
return auth.get("csrf_token")
return None
def get_app_id(self) -> str | None:
"""Get the app ID from app section.
Returns:
App ID string or None if not found
"""
app = self.get_state_section("app")
if app:
return app.get("app_id")
return None
def get_api_key(self) -> str | None:
"""Get the API key token from api_key section.
Returns:
API key token string or None if not found
"""
api_key = self.get_state_section("api_key")
if api_key:
return api_key.get("token")
return None
def console_auth_headers(self) -> dict[str, str]:
access_token = self.get_token()
csrf_token = self.get_csrf_token()
headers: dict[str, str] = {}
if access_token:
headers["authorization"] = f"Bearer {access_token}"
if csrf_token:
headers["X-CSRF-Token"] = csrf_token
return headers
def console_auth_cookies(self) -> dict[str, str]:
access_token = self.get_token()
csrf_token = self.get_csrf_token()
cookies = {"locale": "en-US"}
if access_token:
cookies["access_token"] = access_token
if csrf_token:
cookies["csrf_token"] = csrf_token
return cookies
# Create a default instance for convenience
config_helper = ConfigHelper()
@@ -0,0 +1,218 @@
#!/usr/bin/env python3
import sys
import time
from enum import Enum
class LogLevel(Enum):
"""Log levels with associated colors and symbols."""
DEBUG = ("🔍", "\033[90m") # Gray
INFO = ("ℹ️ ", "\033[94m") # Blue
SUCCESS = ("✅", "\033[92m") # Green
WARNING = ("⚠️ ", "\033[93m") # Yellow
ERROR = ("❌", "\033[91m") # Red
STEP = ("🚀", "\033[96m") # Cyan
PROGRESS = ("📋", "\033[95m") # Magenta
class Logger:
"""Logger class for formatted console output."""
def __init__(self, name: str | None = None, use_colors: bool = True):
"""Initialize logger.
Args:
name: Optional name for the logger (e.g., script name)
use_colors: Whether to use ANSI color codes
"""
self.name = name
self.use_colors = use_colors and sys.stdout.isatty()
self._reset_color = "\033[0m" if self.use_colors else ""
def _format_message(self, level: LogLevel, message: str, indent: int = 0) -> str:
"""Format a log message with level, color, and indentation.
Args:
level: Log level
message: Message to log
indent: Number of spaces to indent
Returns:
Formatted message string
"""
symbol, color = level.value
color = color if self.use_colors else ""
reset = self._reset_color
prefix = " " * indent
if self.name and level in [LogLevel.STEP, LogLevel.ERROR]:
return f"{prefix}{color}{symbol} [{self.name}] {message}{reset}"
else:
return f"{prefix}{color}{symbol} {message}{reset}"
def debug(self, message: str, indent: int = 0) -> None:
"""Log debug message."""
print(self._format_message(LogLevel.DEBUG, message, indent))
def info(self, message: str, indent: int = 0) -> None:
"""Log info message."""
print(self._format_message(LogLevel.INFO, message, indent))
def success(self, message: str, indent: int = 0) -> None:
"""Log success message."""
print(self._format_message(LogLevel.SUCCESS, message, indent))
def warning(self, message: str, indent: int = 0) -> None:
"""Log warning message."""
print(self._format_message(LogLevel.WARNING, message, indent))
def error(self, message: str, indent: int = 0) -> None:
"""Log error message."""
print(self._format_message(LogLevel.ERROR, message, indent), file=sys.stderr)
def step(self, message: str, indent: int = 0) -> None:
"""Log a step in a process."""
print(self._format_message(LogLevel.STEP, message, indent))
def progress(self, message: str, indent: int = 0) -> None:
"""Log progress information."""
print(self._format_message(LogLevel.PROGRESS, message, indent))
def separator(self, char: str = "-", length: int = 60) -> None:
"""Print a separator line."""
print(char * length)
def header(self, title: str, width: int = 60) -> None:
"""Print a formatted header."""
if self.use_colors:
print(f"\n\033[1m{'=' * width}\033[0m") # Bold
print(f"\033[1m{title.center(width)}\033[0m")
print(f"\033[1m{'=' * width}\033[0m\n")
else:
print(f"\n{'=' * width}")
print(title.center(width))
print(f"{'=' * width}\n")
def box(self, title: str, width: int = 60) -> None:
"""Print a title in a box."""
border = "═" * (width - 2)
if self.use_colors:
print(f"\033[1m╔{border}╗\033[0m")
print(f"\033[1m║{title.center(width - 2)}║\033[0m")
print(f"\033[1m╚{border}╝\033[0m")
else:
print(f"╔{border}╗")
print(f"║{title.center(width - 2)}║")
print(f"╚{border}╝")
def list_item(self, item: str, indent: int = 2) -> None:
"""Print a list item."""
prefix = " " * indent
print(f"{prefix}• {item}")
def key_value(self, key: str, value: str, indent: int = 2) -> None:
"""Print a key-value pair."""
prefix = " " * indent
if self.use_colors:
print(f"{prefix}\033[1m{key}:\033[0m {value}")
else:
print(f"{prefix}{key}: {value}")
def spinner_start(self, message: str) -> None:
"""Start a spinner (simple implementation)."""
sys.stdout.write(f"\r{message}... ")
sys.stdout.flush()
def spinner_stop(self, success: bool = True, message: str | None = None) -> None:
"""Stop the spinner and show result."""
if success:
symbol = "✅" if message else "Done"
sys.stdout.write(f"\r{symbol} {message or ''}\n")
else:
symbol = "❌" if message else "Failed"
sys.stdout.write(f"\r{symbol} {message or ''}\n")
sys.stdout.flush()
class ProgressLogger:
"""Logger for tracking progress through multiple steps."""
def __init__(self, total_steps: int, logger: Logger | None = None):
"""Initialize progress logger.
Args:
total_steps: Total number of steps
logger: Logger instance to use (creates new if None)
"""
self.total_steps = total_steps
self.current_step = 0
self.logger = logger or Logger()
self.start_time = time.time()
def next_step(self, description: str) -> None:
"""Move to next step and log it."""
self.current_step += 1
elapsed = time.time() - self.start_time
if self.logger.use_colors:
progress_bar = self._create_progress_bar()
print(f"\n\033[1m[Step {self.current_step}/{self.total_steps}]\033[0m {progress_bar}")
self.logger.step(f"{description} (Elapsed: {elapsed:.1f}s)")
else:
print(f"\n[Step {self.current_step}/{self.total_steps}]")
self.logger.step(f"{description} (Elapsed: {elapsed:.1f}s)")
def _create_progress_bar(self, width: int = 20) -> str:
"""Create a simple progress bar."""
filled = int(width * self.current_step / self.total_steps)
bar = "█" * filled + "░" * (width - filled)
percentage = int(100 * self.current_step / self.total_steps)
return f"[{bar}] {percentage}%"
def complete(self) -> None:
"""Mark progress as complete."""
elapsed = time.time() - self.start_time
self.logger.success(f"All steps completed! Total time: {elapsed:.1f}s")
# Create default logger instance
logger = Logger()
# Convenience functions using default logger
def debug(message: str, indent: int = 0) -> None:
"""Log debug message using default logger."""
logger.debug(message, indent)
def info(message: str, indent: int = 0) -> None:
"""Log info message using default logger."""
logger.info(message, indent)
def success(message: str, indent: int = 0) -> None:
"""Log success message using default logger."""
logger.success(message, indent)
def warning(message: str, indent: int = 0) -> None:
"""Log warning message using default logger."""
logger.warning(message, indent)
def error(message: str, indent: int = 0) -> None:
"""Log error message using default logger."""
logger.error(message, indent)
def step(message: str, indent: int = 0) -> None:
"""Log step using default logger."""
logger.step(message, indent)
def progress(message: str, indent: int = 0) -> None:
"""Log progress using default logger."""
logger.progress(message, indent)
@@ -0,0 +1,37 @@
# Locust configuration file for Dify SSE benchmark
# Target host
host = http://localhost:5001
# Number of users to simulate
users = 10
# Spawn rate (users per second)
spawn-rate = 2
# Run time (use format like 30s, 5m, 1h)
run-time = 1m
# Locustfile to use
locustfile = scripts/stress-test/sse_benchmark.py
# Headless mode (no web UI)
headless = true
# Print stats in the console
print-stats = true
# Only print summary stats
only-summary = false
# Reset statistics after ramp-up
reset-stats = false
# Log level
loglevel = INFO
# CSV output (uncomment to enable)
# csv = reports/locust_results
# HTML report (uncomment to enable)
# html = reports/locust_report.html
@@ -0,0 +1,234 @@
#!/bin/bash
# Run Dify SSE Stress Test using Locust
set -e
# Get the directory where this script is located
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# Go to project root first, then to script dir
PROJECT_ROOT="$( cd "${SCRIPT_DIR}/../.." && pwd )"
cd "${PROJECT_ROOT}"
STRESS_TEST_DIR="scripts/stress-test"
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
NC='\033[0m' # No Color
# Configuration
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
START_EPOCH=$(date +%s)
REPORT_ROOT="${STRESS_TEST_DIR}/reports"
REPORT_DIR="${REPORT_ROOT}/${TIMESTAMP}"
CSV_PREFIX="${REPORT_DIR}/locust"
HTML_REPORT="${REPORT_DIR}/locust_report.html"
SUMMARY_REPORT="${REPORT_DIR}/locust_summary.txt"
# Create reports directory if it doesn't exist
mkdir -p "${REPORT_DIR}"
echo -e "${BLUE}╔════════════════════════════════════════════════════════════════╗${NC}"
echo -e "${BLUE}║ DIFY SSE WORKFLOW STRESS TEST (LOCUST) ║${NC}"
echo -e "${BLUE}╚════════════════════════════════════════════════════════════════╝${NC}"
echo
# Check if services are running
echo -e "${YELLOW}Checking services...${NC}"
# Check Dify API
if curl -s -f http://localhost:5001/health > /dev/null 2>&1; then
echo -e "${GREEN}✓ Dify API is running${NC}"
# Warn if running in debug mode (check for werkzeug in process)
if ps aux | grep -v grep | grep -q "werkzeug.*5001\|flask.*run.*5001"; then
echo -e "${YELLOW}⚠ WARNING: API appears to be running in debug mode (Flask development server)${NC}"
echo -e "${YELLOW} This will give inaccurate benchmark results!${NC}"
echo -e "${YELLOW} For accurate benchmarking, restart with Gunicorn:${NC}"
echo -e "${CYAN} cd api && uv run gunicorn --bind 0.0.0.0:5001 --workers 4 --worker-class gevent app:app${NC}"
echo
echo -n "Continue anyway? (not recommended) [y/N]: "
read -t 10 continue_debug || continue_debug="n"
if [ "$continue_debug" != "y" ] && [ "$continue_debug" != "Y" ]; then
echo -e "${RED}Benchmark cancelled. Please restart API with Gunicorn.${NC}"
exit 1
fi
fi
else
echo -e "${RED}✗ Dify API is not running on port 5001${NC}"
echo -e "${YELLOW} Start it with Gunicorn for accurate benchmarking:${NC}"
echo -e "${CYAN} cd api && uv run gunicorn --bind 0.0.0.0:5001 --workers 4 --worker-class gevent app:app${NC}"
exit 1
fi
# Check Mock OpenAI server
if curl -s -f http://localhost:5004/v1/models > /dev/null 2>&1; then
echo -e "${GREEN}✓ Mock OpenAI server is running${NC}"
else
echo -e "${RED}✗ Mock OpenAI server is not running on port 5004${NC}"
echo -e "${YELLOW} Start it with: python scripts/stress-test/setup/mock_openai_server.py${NC}"
exit 1
fi
# Check API token exists
if [ ! -f "${STRESS_TEST_DIR}/setup/config/stress_test_state.json" ]; then
echo -e "${RED}✗ Stress test configuration not found${NC}"
echo -e "${YELLOW} Run setup first: python scripts/stress-test/setup_all.py${NC}"
exit 1
fi
API_TOKEN=$(python3 -c "import json; state = json.load(open('${STRESS_TEST_DIR}/setup/config/stress_test_state.json')); print(state.get('api_key', {}).get('token', ''))" 2>/dev/null)
if [ -z "$API_TOKEN" ]; then
echo -e "${RED}✗ Failed to read API token from stress test state${NC}"
exit 1
fi
echo -e "${GREEN}✓ API token found: ${API_TOKEN:0:10}...${NC}"
echo
echo -e "${CYAN}═══════════════════════════════════════════════════════════════${NC}"
echo -e "${CYAN} STRESS TEST PARAMETERS ${NC}"
echo -e "${CYAN}═══════════════════════════════════════════════════════════════${NC}"
# Parse configuration
USERS=$(grep "^users" ${STRESS_TEST_DIR}/locust.conf | cut -d'=' -f2 | tr -d ' ')
SPAWN_RATE=$(grep "^spawn-rate" ${STRESS_TEST_DIR}/locust.conf | cut -d'=' -f2 | tr -d ' ')
RUN_TIME=$(grep "^run-time" ${STRESS_TEST_DIR}/locust.conf | cut -d'=' -f2 | tr -d ' ')
echo -e " ${YELLOW}Users:${NC} $USERS concurrent users"
echo -e " ${YELLOW}Spawn Rate:${NC} $SPAWN_RATE users/second"
echo -e " ${YELLOW}Duration:${NC} $RUN_TIME"
echo -e " ${YELLOW}Mode:${NC} SSE Streaming"
echo
# Ask user for run mode
echo -e "${YELLOW}Select run mode:${NC}"
echo " 1) Headless (CLI only) - Default"
echo " 2) Web UI (http://localhost:8089)"
echo -n "Choice [1]: "
read -t 10 choice || choice="1"
echo
# Use SSE stress test script
LOCUST_SCRIPT="${STRESS_TEST_DIR}/sse_benchmark.py"
LOCUST_RUN=(uvx --from locust locust)
# Prepare Locust command
if [ "$choice" = "2" ]; then
echo -e "${BLUE}Starting Locust with Web UI...${NC}"
echo -e "${YELLOW}Access the web interface at: ${CYAN}http://localhost:8089${NC}"
echo
# Run with web UI
"${LOCUST_RUN[@]}" \
-f ${LOCUST_SCRIPT} \
--host http://localhost:5001 \
--web-port 8089
else
echo -e "${BLUE}Starting stress test in headless mode...${NC}"
echo
# Run in headless mode with CSV output
"${LOCUST_RUN[@]}" \
-f ${LOCUST_SCRIPT} \
--host http://localhost:5001 \
--users $USERS \
--spawn-rate $SPAWN_RATE \
--run-time $RUN_TIME \
--headless \
--print-stats \
--csv=$CSV_PREFIX \
--html=$HTML_REPORT \
2>&1 | tee $SUMMARY_REPORT
SSE_METRICS_REPORT=$(python3 - <<EOF
from pathlib import Path
import shutil
reports = [
p for p in Path("${REPORT_ROOT}").glob("sse_metrics_*.json")
if p.stat().st_mtime >= ${START_EPOCH}
]
if reports:
source = max(reports, key=lambda p: p.stat().st_mtime)
target = Path("${REPORT_DIR}") / source.name
if source != target:
shutil.move(str(source), str(target))
print(target)
EOF
)
echo
echo -e "${GREEN}═══════════════════════════════════════════════════════════════${NC}"
echo -e "${GREEN} STRESS TEST COMPLETE ${NC}"
echo -e "${GREEN}═══════════════════════════════════════════════════════════════${NC}"
echo
echo -e "${BLUE}Reports generated:${NC}"
echo -e " ${YELLOW}Summary:${NC} $SUMMARY_REPORT"
echo -e " ${YELLOW}HTML Report:${NC} $HTML_REPORT"
echo -e " ${YELLOW}CSV Stats:${NC} ${CSV_PREFIX}_stats.csv"
echo -e " ${YELLOW}CSV History:${NC} ${CSV_PREFIX}_stats_history.csv"
if [ -n "$SSE_METRICS_REPORT" ]; then
echo -e " ${YELLOW}SSE Metrics:${NC} $SSE_METRICS_REPORT"
else
echo -e " ${YELLOW}SSE Metrics:${NC} not found"
fi
echo
echo -e "${CYAN}View HTML report:${NC}"
echo " open $HTML_REPORT # macOS"
echo " xdg-open $HTML_REPORT # Linux"
echo
# Parse and display key metrics
echo -e "${CYAN}═══════════════════════════════════════════════════════════════${NC}"
echo -e "${CYAN} KEY METRICS ${NC}"
echo -e "${CYAN}═══════════════════════════════════════════════════════════════${NC}"
if [ -f "${CSV_PREFIX}_stats.csv" ]; then
python3 - <<EOF
import csv
import sys
csv_file = "${CSV_PREFIX}_stats.csv"
try:
with open(csv_file, 'r') as f:
reader = csv.DictReader(f)
rows = list(reader)
# Find the aggregated row
for row in rows:
if row.get('Name') == 'Aggregated':
print(f" Total Requests: {row.get('Request Count', 'N/A')}")
print(f" Failure Rate: {row.get('Failure Count', '0')} failures")
print(f" Median Response: {row.get('Median Response Time', 'N/A')} ms")
print(f" 95%ile Response: {row.get('95%', 'N/A')} ms")
print(f" 99%ile Response: {row.get('99%', 'N/A')} ms")
print(f" RPS: {row.get('Requests/s', 'N/A')}")
break
print()
print("SSE Streaming Metrics:")
import json
sse_metrics_report = "${SSE_METRICS_REPORT}"
if sse_metrics_report:
with open(sse_metrics_report, 'r') as metrics_file:
metrics = json.load(metrics_file).get("metrics", {})
print(f" Total Connections: {metrics.get('total_connections', 'N/A')}")
print(f" Total Events: {metrics.get('total_events', 'N/A')}")
print(f" Connection Rate: {metrics.get('overall_conn_rate', 0):.2f} conn/s")
print(f" Event Throughput: {metrics.get('overall_event_rate', 0):.2f} events/s")
print(f" TTFE: {metrics.get('ttfe_p50', 0):.1f} ms p50 / {metrics.get('ttfe_p95', 0):.1f} ms p95")
print(f" Stream Duration: {metrics.get('stream_duration_p50', 0):.1f} ms p50 / {metrics.get('stream_duration_p95', 0):.1f} ms p95")
print(f" Inter-event Latency: {metrics.get('inter_event_latency_p50', 0):.1f} ms p50 / {metrics.get('inter_event_latency_p95', 0):.1f} ms p95")
else:
print(" No SSE metrics JSON report found for this run")
except Exception as e:
print(f"Could not parse metrics: {e}")
EOF
fi
echo -e "${CYAN}═══════════════════════════════════════════════════════════════${NC}"
fi
@@ -0,0 +1,104 @@
#!/usr/bin/env python3
import sys
from pathlib import Path
sys.path.append(str(Path(__file__).parent.parent))
import httpx
from common import Logger, config_helper
def configure_openai_plugin() -> bool:
"""Configure OpenAI plugin with mock server credentials."""
log = Logger("ConfigPlugin")
log.header("Configuring OpenAI Plugin")
# Read token from config
access_token = config_helper.get_token()
if not access_token:
log.error("No access token found in config")
log.info("Please run login_admin.py first to get access token")
return False
log.step("Configuring OpenAI plugin with mock server...")
# API endpoint for plugin configuration
base_url = "http://localhost:5001"
config_endpoint = f"{base_url}/console/api/workspaces/current/model-providers/langgenius/openai/openai/credentials"
# Configuration payload with mock server
config_payload = {
"credentials": {
"openai_api_key": "apikey",
"openai_organization": None,
"openai_api_base": "http://host.docker.internal:5004",
}
}
headers = {
"Accept": "*/*",
"Accept-Language": "en-US,en;q=0.9",
"Cache-Control": "no-cache",
"Connection": "keep-alive",
"DNT": "1",
"Origin": "http://localhost:3000",
"Pragma": "no-cache",
"Referer": "http://localhost:3000/",
"Sec-Fetch-Dest": "empty",
"Sec-Fetch-Mode": "cors",
"Sec-Fetch-Site": "same-site",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
**config_helper.console_auth_headers(),
"content-type": "application/json",
"sec-ch-ua": '"Not;A=Brand";v="99", "Google Chrome";v="139", "Chromium";v="139"',
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": '"macOS"',
}
cookies = config_helper.console_auth_cookies()
try:
# Make the configuration request
with httpx.Client() as client:
response = client.post(
config_endpoint,
json=config_payload,
headers=headers,
cookies=cookies,
)
if response.status_code == 200:
log.success("OpenAI plugin configured successfully!")
log.key_value("API Base", config_payload["credentials"]["openai_api_base"])
log.key_value("API Key", config_payload["credentials"]["openai_api_key"])
return True
elif response.status_code == 201:
log.success("OpenAI plugin credentials created successfully!")
log.key_value("API Base", config_payload["credentials"]["openai_api_base"])
log.key_value("API Key", config_payload["credentials"]["openai_api_key"])
return True
elif response.status_code == 401:
log.error("Configuration failed: Unauthorized")
log.info("Token may have expired. Please run login_admin.py again")
return False
else:
log.error(f"Configuration failed with status code: {response.status_code}")
log.debug(f"Response: {response.text}")
return False
except httpx.ConnectError:
log.error("Could not connect to Dify API at http://localhost:5001")
log.info("Make sure the API server is running with: ./dev/start-api")
return False
except Exception as e:
log.error(f"An error occurred: {e}")
return False
if __name__ == "__main__":
if not configure_openai_plugin():
sys.exit(1)
+121
View File
@@ -0,0 +1,121 @@
#!/usr/bin/env python3
import sys
from pathlib import Path
sys.path.append(str(Path(__file__).parent.parent))
import json
import httpx
from common import Logger, config_helper
def create_api_key() -> bool:
"""Create API key for the imported app."""
log = Logger("CreateAPIKey")
log.header("Creating API Key")
# Read token from config
access_token = config_helper.get_token()
if not access_token:
log.error("No access token found in config")
return False
# Read app_id from config
app_id = config_helper.get_app_id()
if not app_id:
log.error("No app_id found in config")
log.info("Please run import_workflow_app.py first to import the app")
return False
log.step(f"Creating API key for app: {app_id}")
# API endpoint for creating API key
base_url = "http://localhost:5001"
api_key_endpoint = f"{base_url}/console/api/apps/{app_id}/api-keys"
headers = {
"Accept": "*/*",
"Accept-Language": "en-US,en;q=0.9",
"Cache-Control": "no-cache",
"Connection": "keep-alive",
"Content-Length": "0",
"DNT": "1",
"Origin": "http://localhost:3000",
"Pragma": "no-cache",
"Referer": "http://localhost:3000/",
"Sec-Fetch-Dest": "empty",
"Sec-Fetch-Mode": "cors",
"Sec-Fetch-Site": "same-site",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
**config_helper.console_auth_headers(),
"content-type": "application/json",
"sec-ch-ua": '"Not;A=Brand";v="99", "Google Chrome";v="139", "Chromium";v="139"',
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": '"macOS"',
}
cookies = config_helper.console_auth_cookies()
try:
# Make the API key creation request
with httpx.Client() as client:
response = client.post(
api_key_endpoint,
headers=headers,
cookies=cookies,
)
if response.status_code == 200 or response.status_code == 201:
response_data = response.json()
api_key_id = response_data.get("id")
api_key_token = response_data.get("token")
if api_key_token:
log.success("API key created successfully!")
log.key_value("Key ID", api_key_id)
log.key_value("Token", api_key_token)
log.key_value("Type", response_data.get("type"))
# Save API key to config
api_key_config = {
"id": api_key_id,
"token": api_key_token,
"type": response_data.get("type"),
"app_id": app_id,
"created_at": response_data.get("created_at"),
}
if config_helper.write_config("api_key_config", api_key_config):
log.info(f"API key saved to: {config_helper.get_config_path('benchmark_state')}")
return True
return False
else:
log.error("No API token received")
log.debug(f"Response: {json.dumps(response_data, indent=2)}")
return False
elif response.status_code == 401:
log.error("API key creation failed: Unauthorized")
log.info("Token may have expired. Please run login_admin.py again")
return False
else:
log.error(f"API key creation failed with status code: {response.status_code}")
log.debug(f"Response: {response.text}")
return False
except httpx.ConnectError:
log.error("Could not connect to Dify API at http://localhost:5001")
log.info("Make sure the API server is running with: ./dev/start-api")
return False
except Exception as e:
log.error(f"An error occurred: {e}")
return False
if __name__ == "__main__":
if not create_api_key():
sys.exit(1)
@@ -0,0 +1,176 @@
app:
description: ''
icon: 🤖
icon_background: '#FFEAD5'
mode: workflow
name: workflow_llm
use_icon_as_answer_icon: false
dependencies:
- current_identifier: null
type: marketplace
value:
marketplace_plugin_unique_identifier: langgenius/openai:0.2.5@373362a028986aae53a7baf73a7f11991ba3c22c69eaf97d6cde048cfd4a9f98
kind: app
version: 0.4.0
workflow:
conversation_variables: []
environment_variables: []
features:
file_upload:
allowed_file_extensions:
- .JPG
- .JPEG
- .PNG
- .GIF
- .WEBP
- .SVG
allowed_file_types:
- image
allowed_file_upload_methods:
- local_file
- remote_url
enabled: false
fileUploadConfig:
audio_file_size_limit: 50
batch_count_limit: 5
file_size_limit: 15
image_file_size_limit: 10
video_file_size_limit: 100
workflow_file_upload_limit: 10
image:
enabled: false
number_limits: 3
transfer_methods:
- local_file
- remote_url
number_limits: 3
opening_statement: ''
retriever_resource:
enabled: true
sensitive_word_avoidance:
enabled: false
speech_to_text:
enabled: false
suggested_questions: []
suggested_questions_after_answer:
enabled: false
text_to_speech:
enabled: false
language: ''
voice: ''
graph:
edges:
- data:
isInIteration: false
isInLoop: false
sourceType: start
targetType: llm
id: 1757611990947-source-1757611992921-target
source: '1757611990947'
sourceHandle: source
target: '1757611992921'
targetHandle: target
type: custom
zIndex: 0
- data:
isInIteration: false
isInLoop: false
sourceType: llm
targetType: end
id: 1757611992921-source-1757611996447-target
source: '1757611992921'
sourceHandle: source
target: '1757611996447'
targetHandle: target
type: custom
zIndex: 0
nodes:
- data:
desc: ''
selected: false
title: Start
type: start
variables:
- label: question
max_length: null
options: []
required: true
type: text-input
variable: question
height: 90
id: '1757611990947'
position:
x: 30
y: 245
positionAbsolute:
x: 30
y: 245
selected: false
sourcePosition: right
targetPosition: left
type: custom
width: 244
- data:
context:
enabled: false
variable_selector: []
desc: ''
model:
completion_params:
temperature: 0.7
mode: chat
name: gpt-4o
provider: langgenius/openai/openai
prompt_template:
- id: c165fcb6-f1f0-42f2-abab-e81982434deb
role: system
text: ''
- role: user
text: '{{#1757611990947.question#}}'
selected: false
title: LLM
type: llm
variables: []
vision:
enabled: false
height: 90
id: '1757611992921'
position:
x: 334
y: 245
positionAbsolute:
x: 334
y: 245
selected: false
sourcePosition: right
targetPosition: left
type: custom
width: 244
- data:
desc: ''
outputs:
- value_selector:
- '1757611992921'
- text
value_type: string
variable: answer
selected: false
title: End
type: end
height: 90
id: '1757611996447'
position:
x: 638
y: 245
positionAbsolute:
x: 638
y: 245
selected: true
sourcePosition: right
targetPosition: left
type: custom
width: 244
viewport:
x: 0
y: 0
zoom: 0.7
@@ -0,0 +1,138 @@
#!/usr/bin/env python3
import sys
from pathlib import Path
sys.path.append(str(Path(__file__).parent.parent))
import json
import httpx
from common import Logger, config_helper # type: ignore[import]
def is_successful_import_response(response_data: dict[str, object]) -> bool:
return response_data.get("status") != "failed" and bool(response_data.get("app_id"))
def import_workflow_app() -> bool:
"""Import workflow app from DSL file and save app_id."""
log = Logger("ImportApp")
log.header("Importing Workflow Application")
# Read token from config
access_token = config_helper.get_token()
if not access_token:
log.error("No access token found in config")
log.info("Please run login_admin.py first to get access token")
return False
# Read workflow DSL file
dsl_path = Path(__file__).parent / "dsl" / "workflow_llm.yml"
if not dsl_path.exists():
log.error(f"DSL file not found: {dsl_path}")
return False
with open(dsl_path) as f:
yaml_content = f.read()
log.step("Importing workflow app from DSL...")
log.key_value("DSL file", dsl_path.name)
# API endpoint for app import
base_url = "http://localhost:5001"
import_endpoint = f"{base_url}/console/api/apps/imports"
# Import payload
import_payload = {"mode": "yaml-content", "yaml_content": yaml_content}
headers = {
"Accept": "*/*",
"Accept-Language": "en-US,en;q=0.9",
"Cache-Control": "no-cache",
"Connection": "keep-alive",
"DNT": "1",
"Origin": "http://localhost:3000",
"Pragma": "no-cache",
"Referer": "http://localhost:3000/",
"Sec-Fetch-Dest": "empty",
"Sec-Fetch-Mode": "cors",
"Sec-Fetch-Site": "same-site",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
**config_helper.console_auth_headers(),
"content-type": "application/json",
"sec-ch-ua": '"Not;A=Brand";v="99", "Google Chrome";v="139", "Chromium";v="139"',
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": '"macOS"',
}
cookies = config_helper.console_auth_cookies()
try:
# Make the import request
with httpx.Client() as client:
response = client.post(
import_endpoint,
json=import_payload,
headers=headers,
cookies=cookies,
)
if response.status_code == 200:
response_data = response.json()
if is_successful_import_response(response_data):
app_id = response_data.get("app_id")
if response_data.get("status") != "completed":
log.warning(f"Import status: {response_data.get('status')}")
log.debug(f"Response: {json.dumps(response_data, indent=2)}")
log.success("Workflow app imported successfully!")
log.key_value("App ID", app_id)
log.key_value("App Mode", response_data.get("app_mode"))
log.key_value("DSL Version", response_data.get("imported_dsl_version"))
# Save app_id to config
app_config = {
"app_id": app_id,
"app_mode": response_data.get("app_mode"),
"app_name": "workflow_llm",
"dsl_version": response_data.get("imported_dsl_version"),
}
if config_helper.write_config("app_config", app_config):
log.info(f"App config saved to: {config_helper.get_config_path('benchmark_state')}")
return True
return False
elif response_data.get("status") == "failed":
log.error("Import failed")
log.error(f"Error: {response_data.get('error')}")
return False
else:
log.error("Import response did not include app_id")
log.debug(f"Response: {json.dumps(response_data, indent=2)}")
return False
elif response.status_code == 401:
log.error("Import failed: Unauthorized")
log.info("Token may have expired. Please run login_admin.py again")
return False
else:
log.error(f"Import failed with status code: {response.status_code}")
log.debug(f"Response: {response.text}")
return False
except httpx.ConnectError:
log.error("Could not connect to Dify API at http://localhost:5001")
log.info("Make sure the API server is running with: ./dev/start-api")
return False
except Exception as e:
log.error(f"An error occurred: {e}")
return False
if __name__ == "__main__":
if not import_workflow_app():
sys.exit(1)
@@ -0,0 +1,173 @@
#!/usr/bin/env python3
import sys
from pathlib import Path
sys.path.append(str(Path(__file__).parent.parent))
import time
import httpx
from common import Logger, config_helper
def is_non_blocking_install_response(response_data: dict[str, object]) -> bool:
return not response_data.get("code")
def install_openai_plugin() -> bool:
"""Install OpenAI plugin using saved access token."""
log = Logger("InstallPlugin")
log.header("Installing OpenAI Plugin")
# Read token from config
access_token = config_helper.get_token()
if not access_token:
log.error("No access token found in config")
log.info("Please run login_admin.py first to get access token")
return False
log.step("Installing OpenAI plugin...")
# API endpoint for plugin installation
base_url = "http://localhost:5001"
install_endpoint = f"{base_url}/console/api/workspaces/current/plugin/install/marketplace"
# Plugin identifier
plugin_payload = {
"plugin_unique_identifiers": [
"langgenius/openai:0.2.5@373362a028986aae53a7baf73a7f11991ba3c22c69eaf97d6cde048cfd4a9f98"
]
}
headers = {
"Accept": "*/*",
"Accept-Language": "en-US,en;q=0.9",
"Cache-Control": "no-cache",
"Connection": "keep-alive",
"DNT": "1",
"Origin": "http://localhost:3000",
"Pragma": "no-cache",
"Referer": "http://localhost:3000/",
"Sec-Fetch-Dest": "empty",
"Sec-Fetch-Mode": "cors",
"Sec-Fetch-Site": "same-site",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
**config_helper.console_auth_headers(),
"content-type": "application/json",
"sec-ch-ua": '"Not;A=Brand";v="99", "Google Chrome";v="139", "Chromium";v="139"',
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": '"macOS"',
}
cookies = config_helper.console_auth_cookies()
try:
# Make the installation request
with httpx.Client() as client:
response = client.post(
install_endpoint,
json=plugin_payload,
headers=headers,
cookies=cookies,
)
if response.status_code == 200:
response_data = response.json()
task_id = response_data.get("task_id")
if not task_id:
if is_non_blocking_install_response(response_data):
log.warning("No installation task returned; plugin may already be installed")
log.debug(f"Response: {response.text}")
return True
log.error("No task ID received from installation request")
log.debug(f"Response: {response.text}")
return False
log.progress(f"Installation task created: {task_id}")
log.info("Polling for task completion...")
# Poll for task completion
task_endpoint = f"{base_url}/console/api/workspaces/current/plugin/tasks/{task_id}"
max_attempts = 30 # 30 attempts with 2 second delay = 60 seconds max
attempt = 0
log.spinner_start("Installing plugin")
while attempt < max_attempts:
attempt += 1
time.sleep(2) # Wait 2 seconds between polls
task_response = client.get(
task_endpoint,
headers=headers,
cookies=cookies,
)
if task_response.status_code != 200:
log.spinner_stop(
success=False,
message=f"Failed to get task status: {task_response.status_code}",
)
return False
task_data = task_response.json()
task_info = task_data.get("task", {})
status = task_info.get("status")
if status == "success":
log.spinner_stop(success=True, message="Plugin installed!")
log.success("OpenAI plugin installed successfully!")
# Display plugin info
plugins = task_info.get("plugins", [])
if plugins:
plugin_info = plugins[0]
log.key_value("Plugin ID", plugin_info.get("plugin_id"))
log.key_value("Message", plugin_info.get("message"))
return True
elif status == "failed":
log.spinner_stop(success=False, message="Installation failed")
log.error("Plugin installation failed")
plugins = task_info.get("plugins", [])
if plugins:
for plugin in plugins:
log.list_item(f"{plugin.get('plugin_id')}: {plugin.get('message')}")
return False
# Continue polling if status is "pending" or other
else:
log.spinner_stop(success=False, message="Installation timed out")
log.error("Installation timed out after 60 seconds")
return False
elif response.status_code == 401:
log.error("Installation failed: Unauthorized")
log.info("Token may have expired. Please run login_admin.py again")
return False
elif response.status_code == 409:
log.warning("Plugin may already be installed")
log.debug(f"Response: {response.text}")
return True
else:
log.error(f"Installation failed with status code: {response.status_code}")
log.debug(f"Response: {response.text}")
return False
except httpx.ConnectError:
log.error("Could not connect to Dify API at http://localhost:5001")
log.info("Make sure the API server is running with: ./dev/start-api")
return False
except Exception as e:
log.error(f"An error occurred: {e}")
return False
if __name__ == "__main__":
if not install_openai_plugin():
sys.exit(1)
+112
View File
@@ -0,0 +1,112 @@
#!/usr/bin/env python3
import sys
from pathlib import Path
sys.path.append(str(Path(__file__).parent.parent))
import base64
import json
import httpx
from common import Logger, config_helper
def encode_sensitive_field(value: str) -> str:
"""Encode fields the same way the web client does before login."""
return base64.b64encode(value.encode("utf-8")).decode()
def login_admin() -> bool:
"""Login with admin account and save access token."""
log = Logger("Login")
log.header("Admin Login")
# Read admin credentials from config
admin_config = config_helper.read_config("admin_config")
if not admin_config:
log.error("Admin config not found")
log.info("Please run setup_admin.py first to create the admin account")
return False
log.info(f"Logging in with email: {admin_config['email']}")
# API login endpoint
base_url = "http://localhost:5001"
login_endpoint = f"{base_url}/console/api/login"
# Prepare login payload
login_payload = {
"email": admin_config["email"],
"password": encode_sensitive_field(admin_config["password"]),
"remember_me": True,
}
try:
# Make the login request
with httpx.Client() as client:
response = client.post(
login_endpoint,
json=login_payload,
headers={"Content-Type": "application/json"},
)
if response.status_code == 200:
log.success("Login successful!")
response_data = response.json()
# Check if login was successful
if response_data.get("result") != "success":
log.error(f"Login failed: {response_data}")
return False
access_token = response.cookies.get("access_token", "")
refresh_token = response.cookies.get("refresh_token", "")
csrf_token = response.cookies.get("csrf_token", "")
if not access_token:
log.error("No access token found in response")
log.debug(f"Full response: {json.dumps(response_data, indent=2)}")
return False
# Save token to config file
token_config = {
"email": admin_config["email"],
"access_token": access_token,
"refresh_token": refresh_token,
"csrf_token": csrf_token,
}
# Save token config
if config_helper.write_config("token_config", token_config):
log.info(f"Token saved to: {config_helper.get_config_path('benchmark_state')}")
# Show truncated token for verification
token_display = f"{access_token[:20]}..." if len(access_token) > 20 else "Token saved"
log.key_value("Access token", token_display)
return True
elif response.status_code == 401:
log.error("Login failed: Invalid credentials")
log.debug(f"Response: {response.text}")
return False
else:
log.error(f"Login failed with status code: {response.status_code}")
log.debug(f"Response: {response.text}")
return False
except httpx.ConnectError:
log.error("Could not connect to Dify API at http://localhost:5001")
log.info("Make sure the API server is running with: ./dev/start-api")
return False
except Exception as e:
log.error(f"An error occurred: {e}")
return False
if __name__ == "__main__":
if not login_admin():
sys.exit(1)
@@ -0,0 +1,204 @@
#!/usr/bin/env python3
import json
import time
import uuid
from collections.abc import Iterator
from flask import Flask, Response, jsonify, request
app = Flask(__name__)
# Mock models list
MODELS = [
{
"id": "gpt-3.5-turbo",
"object": "model",
"created": 1677649963,
"owned_by": "openai",
},
{"id": "gpt-4", "object": "model", "created": 1687882411, "owned_by": "openai"},
{
"id": "text-embedding-ada-002",
"object": "model",
"created": 1671217299,
"owned_by": "openai-internal",
},
]
@app.route("/v1/models", methods=["GET"])
def list_models() -> Response:
"""List available models."""
return jsonify({"object": "list", "data": MODELS})
@app.route("/v1/chat/completions", methods=["POST"])
def chat_completions() -> Response:
"""Handle chat completions."""
data = request.json or {}
model = data.get("model", "gpt-3.5-turbo")
messages = data.get("messages", [])
stream = data.get("stream", False)
# Generate mock response
response_content = "This is a mock response from the OpenAI server."
if messages:
last_message = messages[-1].get("content", "")
response_content = f"Mock response to: {last_message[:100]}..."
if stream:
# Streaming response
def generate() -> Iterator[str]:
# Send initial chunk
chunk = {
"id": f"chatcmpl-{uuid.uuid4().hex[:8]}",
"object": "chat.completion.chunk",
"created": int(time.time()),
"model": model,
"choices": [
{
"index": 0,
"delta": {"role": "assistant", "content": ""},
"finish_reason": None,
}
],
}
yield f"data: {json.dumps(chunk)}\n\n"
# Send content in chunks
words = response_content.split()
for word in words:
chunk = {
"id": f"chatcmpl-{uuid.uuid4().hex[:8]}",
"object": "chat.completion.chunk",
"created": int(time.time()),
"model": model,
"choices": [
{
"index": 0,
"delta": {"content": word + " "},
"finish_reason": None,
}
],
}
yield f"data: {json.dumps(chunk)}\n\n"
time.sleep(0.05) # Simulate streaming delay
# Send final chunk
chunk = {
"id": f"chatcmpl-{uuid.uuid4().hex[:8]}",
"object": "chat.completion.chunk",
"created": int(time.time()),
"model": model,
"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
}
yield f"data: {json.dumps(chunk)}\n\n"
yield "data: [DONE]\n\n"
return Response(generate(), mimetype="text/event-stream")
else:
# Non-streaming response
return jsonify(
{
"id": f"chatcmpl-{uuid.uuid4().hex[:8]}",
"object": "chat.completion",
"created": int(time.time()),
"model": model,
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": response_content},
"finish_reason": "stop",
}
],
"usage": {
"prompt_tokens": len(str(messages)),
"completion_tokens": len(response_content.split()),
"total_tokens": len(str(messages)) + len(response_content.split()),
},
}
)
@app.route("/v1/completions", methods=["POST"])
def completions() -> Response:
"""Handle text completions."""
data = request.json or {}
model = data.get("model", "gpt-3.5-turbo-instruct")
prompt = data.get("prompt", "")
response_text = f"Mock completion for prompt: {prompt[:100]}..."
return jsonify(
{
"id": f"cmpl-{uuid.uuid4().hex[:8]}",
"object": "text_completion",
"created": int(time.time()),
"model": model,
"choices": [
{
"text": response_text,
"index": 0,
"logprobs": None,
"finish_reason": "stop",
}
],
"usage": {
"prompt_tokens": len(prompt.split()),
"completion_tokens": len(response_text.split()),
"total_tokens": len(prompt.split()) + len(response_text.split()),
},
}
)
@app.route("/v1/embeddings", methods=["POST"])
def embeddings() -> Response:
"""Handle embeddings requests."""
data = request.json or {}
model = data.get("model", "text-embedding-ada-002")
input_text = data.get("input", "")
# Generate mock embedding (1536 dimensions for ada-002)
mock_embedding = [0.1] * 1536
return jsonify(
{
"object": "list",
"data": [{"object": "embedding", "embedding": mock_embedding, "index": 0}],
"model": model,
"usage": {
"prompt_tokens": len(input_text.split()),
"total_tokens": len(input_text.split()),
},
}
)
@app.route("/v1/models/<model_id>", methods=["GET"])
def get_model(model_id: str) -> Response | tuple[Response, int]:
"""Get specific model details."""
for model in MODELS:
if model["id"] == model_id:
return jsonify(model)
return jsonify({"error": "Model not found"}), 404
@app.route("/health", methods=["GET"])
def health() -> Response:
"""Health check endpoint."""
return jsonify({"status": "healthy"})
if __name__ == "__main__":
print("🚀 Starting Mock OpenAI Server on http://localhost:5004")
print("Available endpoints:")
print(" - GET /v1/models")
print(" - POST /v1/chat/completions")
print(" - POST /v1/completions")
print(" - POST /v1/embeddings")
print(" - GET /v1/models/<model_id>")
print(" - GET /health")
app.run(host="0.0.0.0", port=5004, debug=True)
@@ -0,0 +1,112 @@
#!/usr/bin/env python3
import sys
from pathlib import Path
sys.path.append(str(Path(__file__).parent.parent))
import json
import httpx
from common import Logger, config_helper
def publish_workflow() -> bool:
"""Publish the imported workflow app."""
log = Logger("PublishWorkflow")
log.header("Publishing Workflow")
# Read token from config
access_token = config_helper.get_token()
if not access_token:
log.error("No access token found in config")
return False
# Read app_id from config
app_id = config_helper.get_app_id()
if not app_id:
log.error("No app_id found in config")
return False
log.step(f"Publishing workflow for app: {app_id}")
# API endpoint for publishing workflow
base_url = "http://localhost:5001"
publish_endpoint = f"{base_url}/console/api/apps/{app_id}/workflows/publish"
# Publish payload
publish_payload = {"marked_name": "", "marked_comment": ""}
headers = {
"Accept": "*/*",
"Accept-Language": "en-US,en;q=0.9",
"Cache-Control": "no-cache",
"Connection": "keep-alive",
"DNT": "1",
"Origin": "http://localhost:3000",
"Pragma": "no-cache",
"Referer": "http://localhost:3000/",
"Sec-Fetch-Dest": "empty",
"Sec-Fetch-Mode": "cors",
"Sec-Fetch-Site": "same-site",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
**config_helper.console_auth_headers(),
"content-type": "application/json",
"sec-ch-ua": '"Not;A=Brand";v="99", "Google Chrome";v="139", "Chromium";v="139"',
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": '"macOS"',
}
cookies = config_helper.console_auth_cookies()
try:
# Make the publish request
with httpx.Client() as client:
response = client.post(
publish_endpoint,
json=publish_payload,
headers=headers,
cookies=cookies,
)
if response.status_code == 200 or response.status_code == 201:
log.success("Workflow published successfully!")
log.key_value("App ID", app_id)
# Try to parse response if it has JSON content
if response.text:
try:
response_data = response.json()
if response_data:
log.debug(f"Response: {json.dumps(response_data, indent=2)}")
except json.JSONDecodeError:
# Response might be empty or non-JSON
pass
return True
elif response.status_code == 401:
log.error("Workflow publish failed: Unauthorized")
log.info("Token may have expired. Please run login_admin.py again")
return False
elif response.status_code == 404:
log.error("Workflow publish failed: App not found")
log.info("Make sure the app was imported successfully")
return False
else:
log.error(f"Workflow publish failed with status code: {response.status_code}")
log.debug(f"Response: {response.text}")
return False
except httpx.ConnectError:
log.error("Could not connect to Dify API at http://localhost:5001")
log.info("Make sure the API server is running with: ./dev/start-api")
return False
except Exception as e:
log.error(f"An error occurred: {e}")
return False
if __name__ == "__main__":
if not publish_workflow():
sys.exit(1)
+161
View File
@@ -0,0 +1,161 @@
#!/usr/bin/env python3
import sys
from pathlib import Path
sys.path.append(str(Path(__file__).parent.parent))
import json
import httpx
from common import Logger, config_helper
def run_workflow(question: str = "fake question", streaming: bool = True) -> None:
"""Run the workflow app with a question."""
log = Logger("RunWorkflow")
log.header("Running Workflow")
# Read API key from config
api_token = config_helper.get_api_key()
if not api_token:
log.error("No API token found in config")
log.info("Please run create_api_key.py first to create an API key")
return
log.key_value("Question", question)
log.key_value("Mode", "Streaming" if streaming else "Blocking")
log.separator()
# API endpoint for running workflow
base_url = "http://localhost:5001"
run_endpoint = f"{base_url}/v1/workflows/run"
# Run payload
run_payload = {
"inputs": {"question": question},
"user": "default user",
"response_mode": "streaming" if streaming else "blocking",
}
headers = {
"Authorization": f"Bearer {api_token}",
"Content-Type": "application/json",
}
try:
# Make the run request
with httpx.Client(timeout=30.0) as client:
if streaming:
# Handle streaming response
with client.stream(
"POST",
run_endpoint,
json=run_payload,
headers=headers,
) as response:
if response.status_code == 200:
log.success("Workflow started successfully!")
log.separator()
log.step("Streaming response:")
for line in response.iter_lines():
if line.startswith("data: "):
data_str = line[6:] # Remove "data: " prefix
if data_str == "[DONE]":
log.success("Workflow completed!")
break
try:
data = json.loads(data_str)
event = data.get("event")
if event == "workflow_started":
log.progress(f"Workflow started: {data.get('data', {}).get('id')}")
elif event == "node_started":
node_data = data.get("data", {})
log.progress(
f"Node started: {node_data.get('node_type')} - {node_data.get('title')}"
)
elif event == "node_finished":
node_data = data.get("data", {})
log.progress(
f"Node finished: {node_data.get('node_type')} - {node_data.get('title')}"
)
# Print output if it's the LLM node
outputs = node_data.get("outputs", {})
if outputs.get("text"):
log.separator()
log.info("💬 LLM Response:")
log.info(outputs.get("text"), indent=2)
log.separator()
elif event == "workflow_finished":
workflow_data = data.get("data", {})
outputs = workflow_data.get("outputs", {})
if outputs.get("answer"):
log.separator()
log.info("📤 Final Answer:")
log.info(outputs.get("answer"), indent=2)
log.separator()
log.key_value(
"Total tokens",
str(workflow_data.get("total_tokens", 0)),
)
log.key_value(
"Total steps",
str(workflow_data.get("total_steps", 0)),
)
elif event == "error":
log.error(f"Error: {data.get('message')}")
except json.JSONDecodeError:
# Some lines might not be JSON
pass
else:
log.error(f"Workflow run failed with status code: {response.status_code}")
log.debug(f"Response: {response.text}")
else:
# Handle blocking response
response = client.post(
run_endpoint,
json=run_payload,
headers=headers,
)
if response.status_code == 200:
log.success("Workflow completed successfully!")
response_data = response.json()
log.separator()
log.debug(f"Full response: {json.dumps(response_data, indent=2)}")
# Extract the answer if available
outputs = response_data.get("data", {}).get("outputs", {})
if outputs.get("answer"):
log.separator()
log.info("📤 Final Answer:")
log.info(outputs.get("answer"), indent=2)
else:
log.error(f"Workflow run failed with status code: {response.status_code}")
log.debug(f"Response: {response.text}")
except httpx.ConnectError:
log.error("Could not connect to Dify API at http://localhost:5001")
log.info("Make sure the API server is running with: ./dev/start-api")
except httpx.TimeoutException:
log.error("Request timed out")
except Exception as e:
log.error(f"An error occurred: {e}")
if __name__ == "__main__":
# Allow passing question as command line argument
if len(sys.argv) > 1:
question = " ".join(sys.argv[1:])
else:
question = "What is the capital of France?"
run_workflow(question=question, streaming=True)
+82
View File
@@ -0,0 +1,82 @@
#!/usr/bin/env python3
import sys
from pathlib import Path
sys.path.append(str(Path(__file__).parent.parent))
import os
import httpx
from common import Logger, config_helper
def build_admin_config() -> dict[str, str]:
return {
"email": os.getenv("STRESS_TEST_ADMIN_EMAIL", "test@dify.ai"),
"username": os.getenv("STRESS_TEST_ADMIN_USERNAME", "dify"),
"password": os.getenv("STRESS_TEST_ADMIN_PASSWORD", "password123"),
}
def setup_admin_account() -> bool:
"""Setup Dify API with an admin account."""
log = Logger("SetupAdmin")
log.header("Setting up Admin Account")
admin_config = build_admin_config()
# Save credentials to config file
if config_helper.write_config("admin_config", admin_config):
log.info(f"Admin credentials saved to: {config_helper.get_config_path('benchmark_state')}")
# API setup endpoint
base_url = "http://localhost:5001"
setup_endpoint = f"{base_url}/console/api/setup"
# Prepare setup payload
setup_payload = {
"email": admin_config["email"],
"name": admin_config["username"],
"password": admin_config["password"],
}
log.step("Configuring Dify with admin account...")
try:
# Make the setup request
with httpx.Client() as client:
response = client.post(
setup_endpoint,
json=setup_payload,
headers={"Content-Type": "application/json"},
)
if response.status_code == 201:
log.success("Admin account created successfully!")
log.key_value("Email", admin_config["email"])
log.key_value("Username", admin_config["username"])
return True
elif response.status_code in {400, 403}:
log.warning("Setup may have already been completed")
log.debug(f"Response: {response.text}")
return True
else:
log.error(f"Setup failed with status code: {response.status_code}")
log.debug(f"Response: {response.text}")
return False
except httpx.ConnectError:
log.error("Could not connect to Dify API at http://localhost:5001")
log.info("Make sure the API server is running with: ./dev/start-api")
return False
except Exception as e:
log.error(f"An error occurred: {e}")
return False
if __name__ == "__main__":
if not setup_admin_account():
sys.exit(1)
+219
View File
@@ -0,0 +1,219 @@
#!/usr/bin/env python3
import os
import socket
import subprocess
import sys
import time
from pathlib import Path
import httpx
from common import Logger, ProgressLogger, config_helper
def build_admin_config() -> dict[str, str]:
return {
"email": os.getenv("STRESS_TEST_ADMIN_EMAIL", "test@dify.ai"),
"username": os.getenv("STRESS_TEST_ADMIN_USERNAME", "dify"),
"password": os.getenv("STRESS_TEST_ADMIN_PASSWORD", "password123"),
}
def get_setup_step(base_url: str = "http://localhost:5001") -> str | None:
try:
response = httpx.get(f"{base_url}/console/api/setup", timeout=5)
if response.status_code == 200:
return response.json().get("step")
except (httpx.HTTPError, ValueError):
return None
return None
def confirm(prompt: str) -> bool:
answer = input(f"\n{prompt} [Y/n]: ").strip().lower()
return answer in ("", "y", "yes")
def confirm_admin_credentials(log: Logger) -> bool:
admin_config = build_admin_config()
setup_step = get_setup_step()
config_helper.write_config("admin_config", admin_config)
if setup_step == "finished":
log.warning("Dify is already initialized; setup will use the existing admin account to log in.")
log.key_value("Admin email", admin_config["email"])
log.info("Set STRESS_TEST_ADMIN_EMAIL and STRESS_TEST_ADMIN_PASSWORD if this is not the right account.")
return confirm("Continue with this admin login?")
log.info("Dify is not initialized; setup will create the first admin account with:")
log.key_value("Admin email", admin_config["email"])
log.key_value("Admin username", admin_config["username"])
log.key_value("Admin password", admin_config["password"])
log.info("Set STRESS_TEST_ADMIN_EMAIL, STRESS_TEST_ADMIN_USERNAME, or STRESS_TEST_ADMIN_PASSWORD to override.")
return confirm("Create/use this admin account?")
def run_script(script_name: str, description: str) -> bool:
"""Run a Python script and return success status."""
script_path = Path(__file__).parent / "setup" / script_name
if not script_path.exists():
print(f"❌ Script not found: {script_path}")
return False
print(f"\n{'=' * 60}")
print(f"🚀 {description}")
print(f" Running: {script_name}")
print(f"{'=' * 60}")
try:
result = subprocess.run(
[sys.executable, str(script_path)],
capture_output=True,
text=True,
check=False,
)
# Print output
if result.stdout:
print(result.stdout)
if result.stderr:
print(result.stderr, file=sys.stderr)
if result.returncode != 0:
print(f"❌ Script failed with exit code: {result.returncode}")
return False
print(f"✅ {script_name} completed successfully")
return True
except Exception as e:
print(f"❌ Error running {script_name}: {e}")
return False
def check_port(host: str, port: int, service_name: str) -> bool:
"""Check if a service is running on the specified port."""
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(2)
result = sock.connect_ex((host, port))
sock.close()
if result == 0:
Logger().success(f"{service_name} is running on port {port}")
return True
else:
Logger().error(f"{service_name} is not accessible on port {port}")
return False
except Exception as e:
Logger().error(f"Error checking {service_name}: {e}")
return False
def main() -> None:
"""Run all setup scripts in order."""
log = Logger("Setup")
log.box("Dify Stress Test Setup - Full Installation")
# Check if required services are running
log.step("Checking required services...")
log.separator()
dify_running = check_port("localhost", 5001, "Dify API server")
if not dify_running:
log.info("To start Dify API server:")
log.list_item("Run: ./dev/start-api")
mock_running = check_port("localhost", 5004, "Mock OpenAI server")
if not mock_running:
log.info("To start Mock OpenAI server:")
log.list_item("Run: python scripts/stress-test/setup/mock_openai_server.py")
if not dify_running or not mock_running:
print("\n⚠️ Both services must be running before proceeding.")
if confirm("Would you like to check again?"):
return main() # Recursively call main to check again
else:
print("❌ Setup cancelled. Please start the required services and try again.")
sys.exit(1)
log.success("All required services are running!")
if not confirm_admin_credentials(log):
print("❌ Setup cancelled. Please set the admin environment variables and try again.")
sys.exit(1)
# Define setup steps
setup_step = get_setup_step()
setup_steps = [
("login_admin.py", "Logging in and getting access token"),
("install_openai_plugin.py", "Installing OpenAI plugin"),
("configure_openai_plugin.py", "Configuring OpenAI plugin with mock server"),
("import_workflow_app.py", "Importing workflow application"),
("create_api_key.py", "Creating API key for the app"),
("publish_workflow.py", "Publishing the workflow"),
]
if setup_step != "finished":
setup_steps.insert(0, ("setup_admin.py", "Creating admin account"))
# Create progress logger
progress = ProgressLogger(len(setup_steps), log)
failed_step = None
for script, description in setup_steps:
progress.next_step(description)
success = run_script(script, description)
if not success:
failed_step = script
break
# Small delay between steps
time.sleep(1)
log.separator()
if failed_step:
log.error(f"Setup failed at: {failed_step}")
log.separator()
log.info("Troubleshooting:")
if failed_step == "login_admin.py":
if get_setup_step() == "finished":
log.list_item(
"Dify is already initialized; set STRESS_TEST_ADMIN_EMAIL and "
"STRESS_TEST_ADMIN_PASSWORD to an existing admin account."
)
else:
admin_config = build_admin_config()
log.list_item(
"Dify is not initialized; setup creates the first admin with "
f"{admin_config['email']} / {admin_config['username']} unless overridden by environment variables."
)
log.list_item("Check if the Dify API server is running (./dev/start-api)")
log.list_item("Check if the mock OpenAI server is running (port 5004)")
log.list_item("Review the error messages above")
log.list_item("Run cleanup.py and try again")
sys.exit(1)
else:
progress.complete()
log.separator()
log.success("Setup completed successfully!")
log.info("Next steps:")
log.list_item("Test the workflow:")
log.info(
' python scripts/stress-test/setup/run_workflow.py "Your question here"',
indent=4,
)
log.list_item("To clean up and start over:")
log.info(" python scripts/stress-test/cleanup.py", indent=4)
# Optionally run a test
log.separator()
if confirm("Would you like to run a test workflow now?"):
log.step("Running test workflow...")
run_script("run_workflow.py", "Testing workflow with default question")
if __name__ == "__main__":
main()
@@ -0,0 +1,750 @@
#!/usr/bin/env python3
"""
SSE (Server-Sent Events) Stress Test for Dify Workflow API
This script stress tests the streaming performance of Dify's workflow execution API,
measuring key metrics like connection rate, event throughput, and time to first event (TTFE).
"""
import json
import logging
import os
import random
import statistics
import sys
import threading
import time
from collections import deque
from dataclasses import asdict, dataclass
from datetime import datetime
from pathlib import Path
from typing import Literal, TypeAlias, TypedDict
import requests.exceptions
from locust import HttpUser, between, constant, events, task
# Add the stress-test directory to path to import common modules
sys.path.insert(0, str(Path(__file__).parent))
from common.config_helper import ConfigHelper # type: ignore[import-not-found]
# Configure logging
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
logger = logging.getLogger(__name__)
# Configuration from environment
WORKFLOW_PATH = os.getenv("WORKFLOW_PATH", "/v1/workflows/run")
CONNECT_TIMEOUT = float(os.getenv("CONNECT_TIMEOUT", "10"))
READ_TIMEOUT = float(os.getenv("READ_TIMEOUT", "60"))
TERMINAL_EVENTS = [e.strip() for e in os.getenv("TERMINAL_EVENTS", "workflow_finished,error").split(",") if e.strip()]
QUESTIONS_FILE = os.getenv("QUESTIONS_FILE", "")
# Type definitions
ErrorType: TypeAlias = Literal[
"connection_error",
"timeout",
"invalid_json",
"http_4xx",
"http_5xx",
"early_termination",
"invalid_response",
]
class ErrorCounts(TypedDict):
"""Error count tracking"""
connection_error: int
timeout: int
invalid_json: int
http_4xx: int
http_5xx: int
early_termination: int
invalid_response: int
class SSEEvent(TypedDict):
"""Server-Sent Event structure"""
data: str
event: str
id: str | None
class WorkflowInputs(TypedDict):
"""Workflow input structure"""
question: str
class WorkflowRequestData(TypedDict):
"""Workflow request payload"""
inputs: WorkflowInputs
response_mode: Literal["streaming"]
user: str
class ParsedEventData(TypedDict, total=False):
"""Parsed event data from SSE stream"""
event: str
task_id: str
workflow_run_id: str
data: object # For dynamic content
created_at: int
class LocustStats(TypedDict):
"""Locust statistics structure"""
total_requests: int
total_failures: int
avg_response_time: float
min_response_time: float
max_response_time: float
class ReportData(TypedDict):
"""JSON report structure"""
timestamp: str
duration_seconds: float
metrics: dict[str, object] # Metrics as dict for JSON serialization
locust_stats: LocustStats | None
@dataclass
class StreamMetrics:
"""Metrics for a single stream"""
stream_duration: float
events_count: int
bytes_received: int
ttfe: float
inter_event_times: list[float]
@dataclass
class MetricsSnapshot:
"""Snapshot of current metrics state"""
active_connections: int
total_connections: int
total_events: int
connection_rate: float
event_rate: float
overall_conn_rate: float
overall_event_rate: float
ttfe_avg: float
ttfe_min: float
ttfe_max: float
ttfe_p50: float
ttfe_p95: float
ttfe_samples: int
ttfe_total_samples: int # Total TTFE samples collected (not limited by window)
error_counts: ErrorCounts
stream_duration_avg: float
stream_duration_p50: float
stream_duration_p95: float
events_per_stream_avg: float
inter_event_latency_avg: float
inter_event_latency_p50: float
inter_event_latency_p95: float
class MetricsTracker:
def __init__(self) -> None:
self.lock = threading.Lock()
self.active_connections = 0
self.total_connections = 0
self.total_events = 0
self.start_time = time.time()
# Enhanced metrics with memory limits
self.max_samples = 10000 # Prevent unbounded growth
self.ttfe_samples: deque[float] = deque(maxlen=self.max_samples)
self.ttfe_total_count = 0 # Track total TTFE samples collected
# For rate calculations - no maxlen to avoid artificial limits
self.connection_times: deque[float] = deque()
self.event_times: deque[float] = deque()
self.last_stats_time = time.time()
self.last_total_connections = 0
self.last_total_events = 0
self.stream_metrics: deque[StreamMetrics] = deque(maxlen=self.max_samples)
self.error_counts: ErrorCounts = ErrorCounts(
connection_error=0,
timeout=0,
invalid_json=0,
http_4xx=0,
http_5xx=0,
early_termination=0,
invalid_response=0,
)
def connection_started(self) -> None:
with self.lock:
self.active_connections += 1
self.total_connections += 1
self.connection_times.append(time.time())
def connection_ended(self) -> None:
with self.lock:
self.active_connections -= 1
def event_received(self) -> None:
with self.lock:
self.total_events += 1
self.event_times.append(time.time())
def record_ttfe(self, ttfe_ms: float) -> None:
with self.lock:
self.ttfe_samples.append(ttfe_ms) # deque handles maxlen
self.ttfe_total_count += 1 # Increment total counter
def record_stream_metrics(self, metrics: StreamMetrics) -> None:
with self.lock:
self.stream_metrics.append(metrics) # deque handles maxlen
def record_error(self, error_type: ErrorType) -> None:
with self.lock:
self.error_counts[error_type] += 1
def get_stats(self) -> MetricsSnapshot:
with self.lock:
current_time = time.time()
time_window = 10.0 # 10 second window for rate calculation
# Clean up old timestamps outside the window
cutoff_time = current_time - time_window
while self.connection_times and self.connection_times[0] < cutoff_time:
self.connection_times.popleft()
while self.event_times and self.event_times[0] < cutoff_time:
self.event_times.popleft()
# Calculate rates based on actual window or elapsed time
window_duration = min(time_window, current_time - self.start_time)
if window_duration > 0:
conn_rate = len(self.connection_times) / window_duration
event_rate = len(self.event_times) / window_duration
else:
conn_rate = 0
event_rate = 0
# Calculate TTFE statistics
if self.ttfe_samples:
avg_ttfe = statistics.mean(self.ttfe_samples)
min_ttfe = min(self.ttfe_samples)
max_ttfe = max(self.ttfe_samples)
p50_ttfe = statistics.median(self.ttfe_samples)
if len(self.ttfe_samples) >= 2:
quantiles = statistics.quantiles(self.ttfe_samples, n=20, method="inclusive")
p95_ttfe = quantiles[18] # 19th of 19 quantiles = 95th percentile
else:
p95_ttfe = max_ttfe
else:
avg_ttfe = min_ttfe = max_ttfe = p50_ttfe = p95_ttfe = 0
# Calculate stream metrics
if self.stream_metrics:
durations = [m.stream_duration for m in self.stream_metrics]
events_per_stream = [m.events_count for m in self.stream_metrics]
stream_duration_avg = statistics.mean(durations)
stream_duration_p50 = statistics.median(durations)
stream_duration_p95 = (
statistics.quantiles(durations, n=20, method="inclusive")[18]
if len(durations) >= 2
else max(durations)
if durations
else 0
)
events_per_stream_avg = statistics.mean(events_per_stream) if events_per_stream else 0
# Calculate inter-event latency statistics
all_inter_event_times = []
for m in self.stream_metrics:
all_inter_event_times.extend(m.inter_event_times)
if all_inter_event_times:
inter_event_latency_avg = statistics.mean(all_inter_event_times)
inter_event_latency_p50 = statistics.median(all_inter_event_times)
inter_event_latency_p95 = (
statistics.quantiles(all_inter_event_times, n=20, method="inclusive")[18]
if len(all_inter_event_times) >= 2
else max(all_inter_event_times)
)
else:
inter_event_latency_avg = inter_event_latency_p50 = inter_event_latency_p95 = 0
else:
stream_duration_avg = stream_duration_p50 = stream_duration_p95 = events_per_stream_avg = 0
inter_event_latency_avg = inter_event_latency_p50 = inter_event_latency_p95 = 0
# Also calculate overall average rates
total_elapsed = current_time - self.start_time
overall_conn_rate = self.total_connections / total_elapsed if total_elapsed > 0 else 0
overall_event_rate = self.total_events / total_elapsed if total_elapsed > 0 else 0
return MetricsSnapshot(
active_connections=self.active_connections,
total_connections=self.total_connections,
total_events=self.total_events,
connection_rate=conn_rate,
event_rate=event_rate,
overall_conn_rate=overall_conn_rate,
overall_event_rate=overall_event_rate,
ttfe_avg=avg_ttfe,
ttfe_min=min_ttfe,
ttfe_max=max_ttfe,
ttfe_p50=p50_ttfe,
ttfe_p95=p95_ttfe,
ttfe_samples=len(self.ttfe_samples),
ttfe_total_samples=self.ttfe_total_count, # Return total count
error_counts=ErrorCounts(**self.error_counts),
stream_duration_avg=stream_duration_avg,
stream_duration_p50=stream_duration_p50,
stream_duration_p95=stream_duration_p95,
events_per_stream_avg=events_per_stream_avg,
inter_event_latency_avg=inter_event_latency_avg,
inter_event_latency_p50=inter_event_latency_p50,
inter_event_latency_p95=inter_event_latency_p95,
)
# Global metrics instance
metrics = MetricsTracker()
class SSEParser:
"""Parser for Server-Sent Events according to W3C spec"""
def __init__(self) -> None:
self.data_buffer: list[str] = []
self.event_type: str | None = None
self.event_id: str | None = None
def parse_line(self, line: str) -> SSEEvent | None:
"""Parse a single SSE line and return event if complete"""
# Empty line signals end of event
if not line:
if self.data_buffer:
event = SSEEvent(
data="\n".join(self.data_buffer),
event=self.event_type or "message",
id=self.event_id,
)
self.data_buffer = []
self.event_type = None
self.event_id = None
return event
return None
# Comment line
if line.startswith(":"):
return None
# Parse field
if ":" in line:
field, value = line.split(":", 1)
value = value.lstrip()
if field == "data":
self.data_buffer.append(value)
elif field == "event":
self.event_type = value
elif field == "id":
self.event_id = value
return None
# Note: SSEClient removed - we'll handle SSE parsing directly in the task for better Locust integration
class DifyWorkflowUser(HttpUser):
"""Locust user for testing Dify workflow SSE endpoints"""
# Use constant wait for streaming workloads
wait_time = constant(0) if os.getenv("WAIT_TIME", "0") == "0" else between(1, 3)
def __init__(self, *args: object, **kwargs: object) -> None:
super().__init__(*args, **kwargs) # type: ignore[arg-type]
# Load API configuration
config_helper = ConfigHelper()
self.api_token = config_helper.get_api_key()
if not self.api_token:
raise ValueError("API key not found. Please run setup_all.py first.")
# Load questions from file or use defaults
if QUESTIONS_FILE and os.path.exists(QUESTIONS_FILE):
with open(QUESTIONS_FILE) as f:
self.questions = [line.strip() for line in f if line.strip()]
else:
self.questions = [
"What is artificial intelligence?",
"Explain quantum computing",
"What is machine learning?",
"How do neural networks work?",
"What is renewable energy?",
]
self.user_counter = 0
def on_start(self) -> None:
"""Called when a user starts"""
self.user_counter = 0
@task
def test_workflow_stream(self) -> None:
"""Test workflow SSE streaming endpoint"""
question = random.choice(self.questions)
self.user_counter += 1
headers = {
"Authorization": f"Bearer {self.api_token}",
"Content-Type": "application/json",
"Accept": "text/event-stream",
"Cache-Control": "no-cache",
}
data = WorkflowRequestData(
inputs=WorkflowInputs(question=question),
response_mode="streaming",
user=f"user_{self.user_counter}",
)
start_time = time.time()
first_event_time = None
event_count = 0
inter_event_times: list[float] = []
last_event_time = None
ttfe = 0
request_success = False
bytes_received = 0
metrics.connection_started()
# Use catch_response context manager directly
with self.client.request(
method="POST",
url=WORKFLOW_PATH,
headers=headers,
json=data,
stream=True,
catch_response=True,
timeout=(CONNECT_TIMEOUT, READ_TIMEOUT),
name="/v1/workflows/run", # Name for Locust stats
) as response:
try:
# Validate response
if response.status_code >= 400:
error_type: ErrorType = "http_4xx" if response.status_code < 500 else "http_5xx"
metrics.record_error(error_type)
response.failure(f"HTTP {response.status_code}")
return
content_type = response.headers.get("Content-Type", "")
if "text/event-stream" not in content_type and "application/json" not in content_type:
logger.error(f"Expected text/event-stream, got: {content_type}")
metrics.record_error("invalid_response")
response.failure(f"Invalid content type: {content_type}")
return
# Parse SSE events
parser = SSEParser()
for line in response.iter_lines(decode_unicode=True):
# Check if runner is stopping
if getattr(self.environment.runner, "state", "") in (
"stopping",
"stopped",
):
logger.debug("Runner stopping, breaking streaming loop")
break
if line is not None:
bytes_received += len(line.encode("utf-8"))
# Parse SSE line
event = parser.parse_line(line if line is not None else "")
if event:
event_count += 1
current_time = time.time()
metrics.event_received()
# Track inter-event timing
if last_event_time:
inter_event_times.append((current_time - last_event_time) * 1000)
last_event_time = current_time
if first_event_time is None:
first_event_time = current_time
ttfe = (first_event_time - start_time) * 1000
metrics.record_ttfe(ttfe)
try:
# Parse event data
event_data = event.get("data", "")
if event_data:
if event_data == "[DONE]":
logger.debug("Received [DONE] sentinel")
request_success = True
break
try:
parsed_event: ParsedEventData = json.loads(event_data)
# Check for terminal events
if parsed_event.get("event") in TERMINAL_EVENTS:
logger.debug(f"Received terminal event: {parsed_event.get('event')}")
request_success = True
break
except json.JSONDecodeError as e:
logger.debug(f"JSON decode error: {e} for data: {event_data[:100]}")
metrics.record_error("invalid_json")
except Exception as e:
logger.error(f"Error processing event: {e}")
# Mark success only if terminal condition was met or events were received
if request_success:
response.success()
elif event_count > 0:
# Got events but no proper terminal condition
metrics.record_error("early_termination")
response.failure("Stream ended without terminal event")
else:
response.failure("No events received")
except (
requests.exceptions.ConnectTimeout,
requests.exceptions.ReadTimeout,
) as e:
metrics.record_error("timeout")
response.failure(f"Timeout: {e}")
except (
requests.exceptions.ConnectionError,
requests.exceptions.RequestException,
) as e:
metrics.record_error("connection_error")
response.failure(f"Connection error: {e}")
except Exception as e:
response.failure(str(e))
raise
finally:
metrics.connection_ended()
# Record stream metrics
if event_count > 0:
stream_duration = (time.time() - start_time) * 1000
stream_metrics = StreamMetrics(
stream_duration=stream_duration,
events_count=event_count,
bytes_received=bytes_received,
ttfe=ttfe,
inter_event_times=inter_event_times,
)
metrics.record_stream_metrics(stream_metrics)
logger.debug(
f"Stream completed: {event_count} events, {stream_duration:.1f}ms, success={request_success}"
)
else:
logger.warning("No events received in stream")
# Event handlers
@events.test_start.add_listener # type: ignore[misc]
def on_test_start(environment: object, **kwargs: object) -> None:
logger.info("=" * 80)
logger.info(" " * 25 + "DIFY SSE BENCHMARK - REAL-TIME METRICS")
logger.info("=" * 80)
logger.info(f"Started at: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
logger.info("=" * 80)
# Periodic stats reporting
def report_stats() -> None:
if not hasattr(environment, "runner"):
return
runner = environment.runner
while hasattr(runner, "state") and runner.state not in ["stopped", "stopping"]:
time.sleep(5) # Report every 5 seconds
if hasattr(runner, "state") and runner.state == "running":
stats = metrics.get_stats()
# Only log on master node in distributed mode
is_master = (
not getattr(environment.runner, "worker_id", None) if hasattr(environment, "runner") else True
)
if is_master:
# Clear previous lines and show updated stats
logger.info("\n" + "=" * 80)
logger.info(
f"{'METRIC':<25} {'CURRENT':>15} {'RATE (10s)':>15} {'AVG (overall)':>15} {'TOTAL':>12}"
)
logger.info("-" * 80)
# Active SSE Connections
logger.info(
f"{'Active SSE Connections':<25} {stats.active_connections:>15,d} {'-':>15} {'-':>12} {'-':>12}"
)
# New Connection Rate
logger.info(
f"{'New Connections':<25} {'-':>15} {stats.connection_rate:>13.2f}/s {stats.overall_conn_rate:>13.2f}/s {stats.total_connections:>12,d}"
)
# Event Throughput
logger.info(
f"{'Event Throughput':<25} {'-':>15} {stats.event_rate:>13.2f}/s {stats.overall_event_rate:>13.2f}/s {stats.total_events:>12,d}"
)
logger.info("-" * 80)
logger.info(
f"{'TIME TO FIRST EVENT':<25} {'AVG':>15} {'P50':>10} {'P95':>10} {'MIN':>10} {'MAX':>10}"
)
logger.info(
f"{'(TTFE in ms)':<25} {stats.ttfe_avg:>15.1f} {stats.ttfe_p50:>10.1f} {stats.ttfe_p95:>10.1f} {stats.ttfe_min:>10.1f} {stats.ttfe_max:>10.1f}"
)
logger.info(
f"{'Window Samples':<25} {stats.ttfe_samples:>15,d} (last {min(10000, stats.ttfe_total_samples):,d} samples)"
)
logger.info(f"{'Total Samples':<25} {stats.ttfe_total_samples:>15,d}")
# Inter-event latency
if stats.inter_event_latency_avg > 0:
logger.info("-" * 80)
logger.info(f"{'INTER-EVENT LATENCY':<25} {'AVG':>15} {'P50':>10} {'P95':>10}")
logger.info(
f"{'(ms between events)':<25} {stats.inter_event_latency_avg:>15.1f} {stats.inter_event_latency_p50:>10.1f} {stats.inter_event_latency_p95:>10.1f}"
)
# Error stats
if any(stats.error_counts.values()):
logger.info("-" * 80)
logger.info(f"{'ERROR TYPE':<25} {'COUNT':>15}")
for error_type, count in stats.error_counts.items():
if isinstance(count, int) and count > 0:
logger.info(f"{error_type:<25} {count:>15,d}")
logger.info("=" * 80)
# Show Locust stats summary
if hasattr(environment, "stats") and hasattr(environment.stats, "total"):
total = environment.stats.total
if hasattr(total, "num_requests") and total.num_requests > 0:
logger.info(
f"{'LOCUST STATS':<25} {'Requests':>12} {'Fails':>8} {'Avg (ms)':>12} {'Min':>8} {'Max':>8}"
)
logger.info("-" * 80)
logger.info(
f"{'Aggregated':<25} {total.num_requests:>12,d} "
f"{total.num_failures:>8,d} "
f"{total.avg_response_time:>12.1f} "
f"{total.min_response_time:>8.0f} "
f"{total.max_response_time:>8.0f}"
)
logger.info("=" * 80)
threading.Thread(target=report_stats, daemon=True).start()
@events.test_stop.add_listener # type: ignore[misc]
def on_test_stop(environment: object, **kwargs: object) -> None:
stats = metrics.get_stats()
test_duration = time.time() - metrics.start_time
# Log final results
logger.info("\n" + "=" * 80)
logger.info(" " * 30 + "FINAL BENCHMARK RESULTS")
logger.info("=" * 80)
logger.info(f"Test Duration: {test_duration:.1f} seconds")
logger.info("-" * 80)
logger.info("")
logger.info("CONNECTIONS")
logger.info(f" {'Total Connections:':<30} {stats.total_connections:>10,d}")
logger.info(f" {'Final Active:':<30} {stats.active_connections:>10,d}")
logger.info(f" {'Average Rate:':<30} {stats.overall_conn_rate:>10.2f} conn/s")
logger.info("")
logger.info("EVENTS")
logger.info(f" {'Total Events Received:':<30} {stats.total_events:>10,d}")
logger.info(f" {'Average Throughput:':<30} {stats.overall_event_rate:>10.2f} events/s")
logger.info(f" {'Final Rate (10s window):':<30} {stats.event_rate:>10.2f} events/s")
logger.info("")
logger.info("STREAM METRICS")
logger.info(f" {'Avg Stream Duration:':<30} {stats.stream_duration_avg:>10.1f} ms")
logger.info(f" {'P50 Stream Duration:':<30} {stats.stream_duration_p50:>10.1f} ms")
logger.info(f" {'P95 Stream Duration:':<30} {stats.stream_duration_p95:>10.1f} ms")
logger.info(f" {'Avg Events per Stream:':<30} {stats.events_per_stream_avg:>10.1f}")
logger.info("")
logger.info("INTER-EVENT LATENCY")
logger.info(f" {'Average:':<30} {stats.inter_event_latency_avg:>10.1f} ms")
logger.info(f" {'Median (P50):':<30} {stats.inter_event_latency_p50:>10.1f} ms")
logger.info(f" {'95th Percentile:':<30} {stats.inter_event_latency_p95:>10.1f} ms")
logger.info("")
logger.info("TIME TO FIRST EVENT (ms)")
logger.info(f" {'Average:':<30} {stats.ttfe_avg:>10.1f} ms")
logger.info(f" {'Median (P50):':<30} {stats.ttfe_p50:>10.1f} ms")
logger.info(f" {'95th Percentile:':<30} {stats.ttfe_p95:>10.1f} ms")
logger.info(f" {'Minimum:':<30} {stats.ttfe_min:>10.1f} ms")
logger.info(f" {'Maximum:':<30} {stats.ttfe_max:>10.1f} ms")
logger.info(
f" {'Window Samples:':<30} {stats.ttfe_samples:>10,d} (last {min(10000, stats.ttfe_total_samples):,d})"
)
logger.info(f" {'Total Samples:':<30} {stats.ttfe_total_samples:>10,d}")
# Error summary
if any(stats.error_counts.values()):
logger.info("")
logger.info("ERRORS")
for error_type, count in stats.error_counts.items():
if isinstance(count, int) and count > 0:
logger.info(f" {error_type:<30} {count:>10,d}")
logger.info("=" * 80 + "\n")
# Export machine-readable report (only on master node)
is_master = not getattr(environment.runner, "worker_id", None) if hasattr(environment, "runner") else True
if is_master:
export_json_report(stats, test_duration, environment)
def export_json_report(stats: MetricsSnapshot, duration: float, environment: object) -> None:
"""Export metrics to JSON file for CI/CD analysis"""
reports_dir = Path(__file__).parent / "reports"
reports_dir.mkdir(exist_ok=True)
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
report_file = reports_dir / f"sse_metrics_{timestamp}.json"
# Access environment.stats.total attributes safely
locust_stats: LocustStats | None = None
if hasattr(environment, "stats") and hasattr(environment.stats, "total"):
total = environment.stats.total
if hasattr(total, "num_requests") and total.num_requests > 0:
locust_stats = LocustStats(
total_requests=total.num_requests,
total_failures=total.num_failures,
avg_response_time=total.avg_response_time,
min_response_time=total.min_response_time,
max_response_time=total.max_response_time,
)
report_data = ReportData(
timestamp=datetime.now().isoformat(),
duration_seconds=duration,
metrics=asdict(stats), # type: ignore[arg-type]
locust_stats=locust_stats,
)
with open(report_file, "w") as f:
json.dump(report_data, f, indent=2)
logger.info(f"Exported metrics to {report_file}")
@@ -0,0 +1,118 @@
#!/usr/bin/env python3
import base64
import importlib.util
import sys
from pathlib import Path
from common.config_helper import ConfigHelper
def _load_setup_module(name: str):
module_path = Path(__file__).parent / "setup" / f"{name}.py"
spec = importlib.util.spec_from_file_location(f"stress_test_{name}", module_path)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def _load_stress_test_module(name: str):
module_path = Path(__file__).parent / f"{name}.py"
spec = importlib.util.spec_from_file_location(f"stress_test_{name}", module_path)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def test_config_helper_getters_read_state_sections(tmp_path):
helper = ConfigHelper(base_dir=tmp_path)
helper.write_state(
{
"auth": {"access_token": "console-token", "csrf_token": "csrf-token"},
"app": {"app_id": "app-id"},
"api_key": {"token": "app-token"},
}
)
assert helper.get_token() == "console-token"
assert helper.get_csrf_token() == "csrf-token"
assert helper.get_app_id() == "app-id"
assert helper.get_api_key() == "app-token"
assert helper.console_auth_headers() == {
"authorization": "Bearer console-token",
"X-CSRF-Token": "csrf-token",
}
assert helper.console_auth_cookies() == {
"locale": "en-US",
"access_token": "console-token",
"csrf_token": "csrf-token",
}
def test_login_admin_encodes_password_like_web_client():
login_admin = _load_setup_module("login_admin")
encoded = login_admin.encode_sensitive_field("password123")
assert encoded == base64.b64encode(b"password123").decode()
def test_setup_admin_reads_credentials_from_environment(monkeypatch):
setup_admin = _load_setup_module("setup_admin")
monkeypatch.setenv("STRESS_TEST_ADMIN_EMAIL", "real-admin@example.com")
monkeypatch.setenv("STRESS_TEST_ADMIN_USERNAME", "real-admin")
monkeypatch.setenv("STRESS_TEST_ADMIN_PASSWORD", "secret")
assert setup_admin.build_admin_config() == {
"email": "real-admin@example.com",
"username": "real-admin",
"password": "secret",
}
def test_setup_all_reads_credentials_from_environment(monkeypatch):
setup_all = _load_stress_test_module("setup_all")
monkeypatch.setenv("STRESS_TEST_ADMIN_EMAIL", "real-admin@example.com")
monkeypatch.setenv("STRESS_TEST_ADMIN_USERNAME", "real-admin")
monkeypatch.setenv("STRESS_TEST_ADMIN_PASSWORD", "secret")
assert setup_all.build_admin_config() == {
"email": "real-admin@example.com",
"username": "real-admin",
"password": "secret",
}
def test_setup_all_confirm_defaults_to_yes(monkeypatch):
setup_all = _load_stress_test_module("setup_all")
monkeypatch.setattr("builtins.input", lambda prompt: "")
assert setup_all.confirm("Continue?") is True
def test_create_api_key_fails_without_app_id(tmp_path):
create_api_key = _load_setup_module("create_api_key")
create_api_key.config_helper.base_dir = tmp_path
create_api_key.config_helper.write_state({"auth": {"access_token": "token"}})
assert create_api_key.create_api_key() is False
def test_plugin_install_response_without_task_is_non_blocking():
install_openai_plugin = _load_setup_module("install_openai_plugin")
assert install_openai_plugin.is_non_blocking_install_response({"ok": True}) is True
assert install_openai_plugin.is_non_blocking_install_response({}) is True
assert install_openai_plugin.is_non_blocking_install_response({"code": "plugin_error"}) is False
def test_import_response_with_warnings_and_app_id_is_success():
import_workflow_app = _load_setup_module("import_workflow_app")
assert import_workflow_app.is_successful_import_response({"status": "completed-with-warnings", "app_id": "app-id"})
assert not import_workflow_app.is_successful_import_response({"status": "failed", "app_id": "app-id"})
assert not import_workflow_app.is_successful_import_response({"status": "completed"})