chore: 纳入本地 Dify 1.19.0 源码及品牌与免登改动
演示目录只保留可重建的源码。运行密钥、数据卷、依赖和带口令的本地配置不进入仓库。 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
60158935f4
commit
6519682885
15831 files changed
+3063428
No files matched your search
@@ -0,0 +1,85 @@
|
||||
from collections.abc import Callable
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from enums import DeploymentEdition
|
||||
from extensions.application_services.tracing import build_tracing_services
|
||||
from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository
|
||||
from repositories.application_trace_repository import ApplicationTraceRepository
|
||||
from repositories.trace_record_repository import TraceRecordRepository
|
||||
from services.tracing.adapters import (
|
||||
CeleryExportRetryQueue,
|
||||
CeleryTraceEmitter,
|
||||
WorkspaceTracingSecretCipher,
|
||||
schedule_agent_trace_export,
|
||||
send_to_enterprise_telemetry,
|
||||
)
|
||||
from services.tracing.egress import SSRFEgress
|
||||
from services.tracing.registry import EntryPointTraceProviderCatalog
|
||||
from services.tracing.runtime_bridge import RuntimeApplicationTracingHooks
|
||||
|
||||
|
||||
def test_delivery_reads_through_the_deployment_adapters(sqlite_session_factory: sessionmaker[Session]) -> None:
|
||||
tracing = build_tracing_services(database_client=sqlite_session_factory)
|
||||
delivery = tracing.delivery
|
||||
|
||||
assert isinstance(delivery._records, TraceRecordRepository)
|
||||
assert delivery._records._sessions is sqlite_session_factory
|
||||
assert isinstance(delivery._destinations, SQLAlchemyAppTracingConfigRepository)
|
||||
assert isinstance(delivery._providers, EntryPointTraceProviderCatalog)
|
||||
assert isinstance(delivery._codec._cipher, WorkspaceTracingSecretCipher)
|
||||
assert isinstance(delivery._egress, SSRFEgress)
|
||||
assert isinstance(delivery._retries, CeleryExportRetryQueue)
|
||||
assert isinstance(tracing.scopes._emitter, CeleryTraceEmitter)
|
||||
assert isinstance(tracing.scopes._policies, SQLAlchemyAppTracingConfigRepository)
|
||||
assert tracing.provider_configs._providers is delivery._providers
|
||||
assert tracing.provider_configs._codec is delivery._codec
|
||||
|
||||
|
||||
def test_agent_collection_shares_the_providers_and_schedules_exports_on_the_trace_queue(
|
||||
sqlite_session_factory: sessionmaker[Session],
|
||||
) -> None:
|
||||
tracing = build_tracing_services(database_client=sqlite_session_factory)
|
||||
agent_traces = tracing.agent_traces
|
||||
|
||||
# One repository persists the run aggregates and answers the run and source queries.
|
||||
assert isinstance(agent_traces._runs, ApplicationTraceRepository)
|
||||
assert agent_traces._run_query is agent_traces._runs
|
||||
assert agent_traces._sources is agent_traces._runs
|
||||
assert agent_traces._runs._sessions is sqlite_session_factory
|
||||
assert isinstance(agent_traces._records, TraceRecordRepository)
|
||||
# Agent traces never carry the enterprise telemetry dimensions.
|
||||
assert agent_traces._records._enterprise_details is False
|
||||
assert agent_traces._providers is tracing.delivery._providers
|
||||
assert agent_traces._schedule_export is schedule_agent_trace_export
|
||||
|
||||
|
||||
@pytest.mark.parametrize("enabled", [True, False])
|
||||
def test_agent_collection_joins_delivery_only_when_enabled(
|
||||
sqlite_session_factory: sessionmaker[Session], config_overrides: Callable[..., None], enabled: bool
|
||||
) -> None:
|
||||
config_overrides(OPS_AGENT_TRACE_ENABLED=enabled, OPS_AGENT_TRACE_MAX_ATTEMPTS=3)
|
||||
|
||||
tracing = build_tracing_services(database_client=sqlite_session_factory)
|
||||
|
||||
assert (tracing.delivery._collector is tracing.agent_traces) is enabled
|
||||
assert isinstance(tracing.hooks, RuntimeApplicationTracingHooks) is enabled
|
||||
assert tracing.agent_traces._settings.enabled is enabled
|
||||
assert tracing.agent_traces._settings.max_export_attempts == 3
|
||||
assert tracing.provider_capabilities._api_collection_enabled is enabled
|
||||
|
||||
|
||||
@pytest.mark.parametrize("edition", [DeploymentEdition.ENTERPRISE, DeploymentEdition.COMMUNITY])
|
||||
def test_enterprise_telemetry_receives_records_with_its_dimensions_only_when_enabled(
|
||||
sqlite_session_factory: sessionmaker[Session], config_overrides: Callable[..., None], edition: DeploymentEdition
|
||||
) -> None:
|
||||
config_overrides(DEPLOYMENT_EDITION=edition, ENTERPRISE_TELEMETRY_ENABLED=True, OPS_TRACE_EXPORT_MAX_ATTEMPTS=4)
|
||||
enterprise = edition == DeploymentEdition.ENTERPRISE
|
||||
|
||||
delivery = build_tracing_services(database_client=sqlite_session_factory).delivery
|
||||
|
||||
assert (delivery._enterprise_sink is send_to_enterprise_telemetry) is enterprise
|
||||
assert isinstance(delivery._records, TraceRecordRepository)
|
||||
assert delivery._records._enterprise_details is enterprise
|
||||
assert delivery._max_export_attempts == 4
|
||||
@@ -0,0 +1 @@
|
||||
"""LogStore extension unit tests."""
|
||||
Whitespace-only changes.
+410
@@ -0,0 +1,410 @@
|
||||
import datetime
|
||||
import sqlite3
|
||||
import time
|
||||
from collections.abc import Generator
|
||||
from contextlib import closing
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from aliyun.log import GetLogsRequest, GetLogsResponse
|
||||
from pytest_mock import MockerFixture
|
||||
|
||||
from extensions.logstore.aliyun_logstore import AliyunLogStore
|
||||
from extensions.logstore.aliyun_logstore_pg import AliyunLogStorePG
|
||||
from extensions.logstore.repositories.logstore_api_workflow_node_execution_repository import (
|
||||
LogstoreAPIWorkflowNodeExecutionRepository,
|
||||
_dict_to_workflow_node_execution_model,
|
||||
)
|
||||
from models.workflow import WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def logstore_database() -> Generator[sqlite3.Connection]:
|
||||
with closing(sqlite3.connect(":memory:")) as database:
|
||||
database.row_factory = sqlite3.Row
|
||||
database.execute(
|
||||
"CREATE TABLE workflow_node_execution ("
|
||||
"id TEXT, node_execution_id TEXT, tenant_id TEXT, app_id TEXT, workflow_id TEXT, "
|
||||
"workflow_run_id TEXT, triggered_from_workflow_id TEXT, triggered_from_node_execution_id TEXT, "
|
||||
'triggered_from TEXT, node_id TEXT, node_type TEXT, title TEXT, "index" INTEGER, '
|
||||
"status TEXT, elapsed_time REAL, created_at INTEGER, finished_at INTEGER, execution_metadata TEXT, "
|
||||
"log_version INTEGER, __time__ INTEGER, inputs TEXT, process_data TEXT, outputs TEXT)"
|
||||
)
|
||||
yield database
|
||||
|
||||
|
||||
@pytest.fixture(params=["pg", "sdk"])
|
||||
def sql_repository(
|
||||
request: pytest.FixtureRequest,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
mocker: MockerFixture,
|
||||
logstore_database: sqlite3.Connection,
|
||||
):
|
||||
"""Exercise the real protocol adapter while executing its SQL against local records."""
|
||||
monkeypatch.setattr(AliyunLogStore, "_instance", None)
|
||||
monkeypatch.setattr(AliyunLogStore, "_initialized", False)
|
||||
monkeypatch.setenv("ALIYUN_SLS_ENDPOINT", "localhost")
|
||||
mocker.patch.object(AliyunLogStore, "_check_endpoint_connectivity")
|
||||
store = AliyunLogStore()
|
||||
store._use_pg_protocol = request.param == "pg"
|
||||
store._pg_client = AliyunLogStorePG("key", "secret", "localhost", "test")
|
||||
store.project_name = "test"
|
||||
store.log_enabled = False
|
||||
|
||||
def execute(sql: str, *_args: object) -> list[dict[str, object]]:
|
||||
return [dict(row) for row in logstore_database.execute(sql)]
|
||||
|
||||
def get_logs(log_request: GetLogsRequest):
|
||||
search, separator, sql = log_request.get_query().partition(" | ")
|
||||
assert separator
|
||||
assert search
|
||||
assert log_request.get_from() == 0
|
||||
assert log_request.get_to() <= int(time.time())
|
||||
return GetLogsResponse({"meta": {"progress": "Complete"}, "data": execute(sql)}, {})
|
||||
|
||||
mocker.patch.object(store._pg_client, "execute_sql", side_effect=execute)
|
||||
mocker.patch.object(store.client, "get_logs", side_effect=get_logs)
|
||||
repository = LogstoreAPIWorkflowNodeExecutionRepository(session_maker=None)
|
||||
return repository, logstore_database
|
||||
|
||||
|
||||
def test_sql_history_excludes_recursive_tool_executions_and_retains_legacy_rows(sql_repository) -> None:
|
||||
repository, database = sql_repository
|
||||
rows = [
|
||||
{
|
||||
"id": execution_id,
|
||||
"tenant_id": "tenant",
|
||||
"app_id": "app",
|
||||
"workflow_id": "workflow",
|
||||
"workflow_run_id": "run",
|
||||
"node_id": "same-node",
|
||||
"triggered_from": origin.value if origin is not None else None,
|
||||
"status": "succeeded",
|
||||
"index": index,
|
||||
"created_at": index,
|
||||
"log_version": 1,
|
||||
"__time__": 1,
|
||||
}
|
||||
for execution_id, index, origin in (
|
||||
("root", 1, WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN),
|
||||
("legacy", 2, None),
|
||||
("child", 3, WorkflowNodeExecutionTriggeredFrom.WORKFLOW_TOOL),
|
||||
)
|
||||
]
|
||||
insert = (
|
||||
"INSERT INTO workflow_node_execution (id, tenant_id, app_id, workflow_id, workflow_run_id, "
|
||||
'node_id, triggered_from, status, "index", created_at, log_version, __time__) VALUES '
|
||||
"(:id, :tenant_id, :app_id, :workflow_id, :workflow_run_id, :node_id, :triggered_from, :status, "
|
||||
":index, :created_at, :log_version, :__time__)"
|
||||
)
|
||||
database.executemany(insert, rows)
|
||||
history = repository.get_executions_by_workflow_run(tenant_id="tenant", app_id="app", workflow_run_id="run")
|
||||
# A newer paused/hidden version must not expose the earlier visible version.
|
||||
rows = [
|
||||
{**rows[0], "id": "paused", "created_at": 10, "status": "running"},
|
||||
{**rows[0], "id": "paused", "created_at": 10, "status": "paused", "log_version": 2},
|
||||
{**rows[0], "id": "hidden", "created_at": 11},
|
||||
{**rows[2], "id": "hidden", "created_at": 11, "log_version": 2},
|
||||
{**rows[1], "id": "null-status", "created_at": 0, "status": None},
|
||||
*[
|
||||
{**rows[0], "id": f"other-{field}", field: "other", "created_at": 99}
|
||||
for field in ("tenant_id", "app_id", "workflow_id", "node_id")
|
||||
],
|
||||
]
|
||||
database.executemany(insert, rows)
|
||||
latest = repository.get_node_last_execution(
|
||||
tenant_id="tenant", app_id="app", workflow_id="workflow", node_id="same-node"
|
||||
)
|
||||
|
||||
assert [execution.id for execution in history] == ["legacy", "root"]
|
||||
assert latest is not None
|
||||
assert latest.id == "legacy"
|
||||
assert repository.get_node_last_execution("tenant", "app", "workflow", "unknown") is None
|
||||
database.execute("DELETE FROM workflow_node_execution WHERE id IN ('legacy', 'root')")
|
||||
assert repository.get_node_last_execution("tenant", "app", "workflow", "same-node").id == "null-status"
|
||||
|
||||
|
||||
def test_sdk_history_searches_use_owner_scope() -> None:
|
||||
with patch("extensions.logstore.repositories.logstore_api_workflow_node_execution_repository.AliyunLogStore"):
|
||||
repository = LogstoreAPIWorkflowNodeExecutionRepository(session_maker=None)
|
||||
repository.logstore_client = MagicMock(supports_pg_protocol=False)
|
||||
repository.logstore_client.execute_sql.return_value = []
|
||||
|
||||
repository.get_executions_by_workflow_run(tenant_id="tenant", app_id="app", workflow_run_id="run")
|
||||
repository.get_node_last_execution(tenant_id="tenant", app_id="app", workflow_id="workflow", node_id="same-node")
|
||||
|
||||
history_query, latest_query = [call.kwargs for call in repository.logstore_client.execute_sql.call_args_list]
|
||||
assert latest_query["query"] == (
|
||||
'tenant_id: "tenant" and app_id: "app" and workflow_id: "workflow" and node_id: "same-node"'
|
||||
)
|
||||
assert history_query["query"] == 'tenant_id: "tenant" and app_id: "app" and workflow_run_id: "run"'
|
||||
|
||||
|
||||
def test_resumption_snapshots_read_latest_nodes_with_full_owner_scope(logstore_database: sqlite3.Connection) -> None:
|
||||
with patch("extensions.logstore.repositories.logstore_api_workflow_node_execution_repository.AliyunLogStore"):
|
||||
repository = LogstoreAPIWorkflowNodeExecutionRepository(session_maker=None)
|
||||
database = logstore_database
|
||||
database.row_factory = sqlite3.Row
|
||||
base = {
|
||||
"id": "row-id",
|
||||
"node_execution_id": "engine-id",
|
||||
"tenant_id": "tenant",
|
||||
"app_id": "app",
|
||||
"workflow_id": "workflow",
|
||||
"workflow_run_id": "run",
|
||||
"triggered_from": "workflow-run",
|
||||
"node_id": "tool",
|
||||
"node_type": "tool",
|
||||
"title": "Approval",
|
||||
"index": 2,
|
||||
"status": "paused",
|
||||
"elapsed_time": 3.5,
|
||||
"created_at": 100,
|
||||
"finished_at": 104,
|
||||
"execution_metadata": '{"iteration_id":"iteration"}',
|
||||
"log_version": 2,
|
||||
"__time__": 1,
|
||||
"inputs": '{"private":"input"}',
|
||||
"process_data": '{"private":"trace"}',
|
||||
"outputs": '{"private":"output"}',
|
||||
}
|
||||
rows = [base, {**base, "log_version": 1, "status": "running"}]
|
||||
rows.extend(
|
||||
{**base, "id": f"page-{index}", "index": index, "execution_metadata": "[]", "elapsed_time": None}
|
||||
for index in range(3, 1003)
|
||||
)
|
||||
for field in ("tenant_id", "app_id", "workflow_id", "workflow_run_id", "triggered_from"):
|
||||
rows.append({**base, "id": f"other-{field}", field: "other", "index": 99})
|
||||
database.executemany(
|
||||
'INSERT INTO workflow_node_execution ("'
|
||||
+ '", "'.join(base)
|
||||
+ '") VALUES ('
|
||||
+ ", ".join(f":{key}" for key in base)
|
||||
+ ")",
|
||||
rows,
|
||||
)
|
||||
|
||||
def execute_query(*, sql: str, **_kwargs: object) -> list[dict[str, object]]:
|
||||
result = [dict(row) for row in database.execute(sql)]
|
||||
assert all(row.keys().isdisjoint({"inputs", "process_data", "outputs"}) for row in result)
|
||||
return result
|
||||
|
||||
repository.logstore_client.execute_sql.side_effect = execute_query
|
||||
snapshots = repository.get_execution_snapshots_by_workflow_run(
|
||||
tenant_id="tenant",
|
||||
app_id="app",
|
||||
workflow_id="workflow",
|
||||
triggered_from="workflow-run",
|
||||
workflow_run_id="run",
|
||||
)
|
||||
assert len(snapshots) == 1001
|
||||
snapshot = snapshots[0]
|
||||
assert (snapshot.execution_id, snapshot.title, snapshot.index, snapshot.status) == (
|
||||
"engine-id",
|
||||
"Approval",
|
||||
2,
|
||||
"paused",
|
||||
)
|
||||
assert snapshot.iteration_id == "iteration"
|
||||
assert snapshot.elapsed_time == 3.5
|
||||
assert snapshot.created_at == datetime.datetime(1970, 1, 1, 0, 1, 40)
|
||||
assert snapshots[-1].index == 1002
|
||||
assert snapshots[-1].iteration_id is None
|
||||
assert snapshots[-1].elapsed_time == 4
|
||||
assert repository.logstore_client.execute_sql.call_count == 2
|
||||
query = repository.logstore_client.execute_sql.call_args.kwargs
|
||||
assert (
|
||||
query["query"] == 'tenant_id: "tenant" and app_id: "app" and workflow_run_id: "run" and workflow_id: "workflow"'
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("requested_id", ["parent-row", "parent-engine"])
|
||||
def test_workflow_tool_children_only_fetch_matching_latest_payloads(
|
||||
requested_id: str, sql_repository, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
repository, database = sql_repository
|
||||
rows = [
|
||||
{
|
||||
"id": "parent-row",
|
||||
"node_execution_id": "parent-engine",
|
||||
"workflow_id": "caller-workflow",
|
||||
"node_type": "tool",
|
||||
"log_version": 1,
|
||||
"created_at": 0,
|
||||
},
|
||||
*[
|
||||
{
|
||||
"id": "child",
|
||||
"node_type": "human-input",
|
||||
"triggered_from": "workflow-tool",
|
||||
"triggered_from_node_execution_id": "parent-engine",
|
||||
"log_version": version,
|
||||
"status": status,
|
||||
"created_at": 1,
|
||||
"index": 2,
|
||||
"outputs": '{"answer":"approved"}',
|
||||
}
|
||||
for version, status in ((1, "running"), (2, "paused"))
|
||||
],
|
||||
{
|
||||
"id": "other-call",
|
||||
"triggered_from": "workflow-tool",
|
||||
"triggered_from_node_execution_id": "other-engine",
|
||||
"created_at": 2,
|
||||
},
|
||||
{
|
||||
"id": "earlier-child",
|
||||
"triggered_from": "workflow-tool",
|
||||
"triggered_from_node_execution_id": "parent-engine",
|
||||
"created_at": 1,
|
||||
"index": 1,
|
||||
},
|
||||
{
|
||||
"id": "wrong-origin",
|
||||
"triggered_from": "workflow-run",
|
||||
"triggered_from_node_execution_id": "parent-engine",
|
||||
},
|
||||
{
|
||||
"id": "other-tenant",
|
||||
"tenant_id": "other-tenant",
|
||||
"triggered_from": "workflow-tool",
|
||||
"triggered_from_node_execution_id": "parent-engine",
|
||||
},
|
||||
{
|
||||
"id": "other-run",
|
||||
"workflow_run_id": "other-run",
|
||||
"triggered_from": "workflow-tool",
|
||||
"triggered_from_node_execution_id": "parent-engine",
|
||||
},
|
||||
{
|
||||
"id": "other-caller",
|
||||
"triggered_from": "workflow-tool",
|
||||
"triggered_from_workflow_id": "other-workflow",
|
||||
"triggered_from_node_execution_id": "parent-engine",
|
||||
},
|
||||
*[{"id": f"unrelated-{index}"} for index in range(1001)],
|
||||
]
|
||||
defaults = {
|
||||
"id": "",
|
||||
"tenant_id": "tenant",
|
||||
"app_id": "source-app",
|
||||
"workflow_run_id": "run",
|
||||
"node_execution_id": None,
|
||||
"workflow_id": "source-workflow",
|
||||
"triggered_from_workflow_id": "caller-workflow",
|
||||
"triggered_from_node_execution_id": None,
|
||||
"node_type": "start",
|
||||
"triggered_from": "workflow-run",
|
||||
"process_data": None,
|
||||
"outputs": None,
|
||||
"status": "succeeded",
|
||||
"index": 0,
|
||||
"created_at": 0,
|
||||
"log_version": 1,
|
||||
"__time__": 1,
|
||||
}
|
||||
rows = [{**defaults, **row} for row in rows]
|
||||
fetched_payload_ids: list[str] = []
|
||||
database.executemany(
|
||||
"INSERT INTO workflow_node_execution (id, tenant_id, app_id, workflow_run_id, node_execution_id, "
|
||||
"workflow_id, triggered_from_workflow_id, triggered_from_node_execution_id, node_type, triggered_from, "
|
||||
'process_data, outputs, status, "index", created_at, log_version, __time__) VALUES '
|
||||
"(:id, :tenant_id, :app_id, :workflow_run_id, :node_execution_id, :workflow_id, "
|
||||
":triggered_from_workflow_id, :triggered_from_node_execution_id, :node_type, :triggered_from, "
|
||||
":process_data, :outputs, :status, :index, :created_at, :log_version, :__time__)",
|
||||
rows,
|
||||
)
|
||||
|
||||
execute_sql = repository.logstore_client.execute_sql
|
||||
|
||||
def execute_query(*, sql: str, **kwargs: object) -> list[dict[str, object]]:
|
||||
result = execute_sql(sql=sql, **kwargs)
|
||||
fetched_payload_ids.extend(str(row["id"]) for row in result if "outputs" in row)
|
||||
return result
|
||||
|
||||
monkeypatch.setattr(repository.logstore_client, "execute_sql", execute_query)
|
||||
children = repository.get_workflow_tool_executions("tenant", "run", requested_id)
|
||||
assert repository.get_workflow_tool_executions("tenant", "run", "unknown") == []
|
||||
assert repository.get_workflow_tool_executions("tenant", "run", "child") == []
|
||||
|
||||
assert [child.id for child in children] == ["earlier-child", "child"]
|
||||
assert children[1].status.value == "paused"
|
||||
assert children[1].outputs_dict == {"answer": "approved"}
|
||||
assert fetched_payload_ids == ["earlier-child", "child"]
|
||||
|
||||
|
||||
def test_load_full_process_data_returns_logstore_mapping() -> None:
|
||||
with patch("extensions.logstore.repositories.logstore_api_workflow_node_execution_repository.AliyunLogStore"):
|
||||
repository = LogstoreAPIWorkflowNodeExecutionRepository(session_maker=None)
|
||||
execution = WorkflowNodeExecutionModel()
|
||||
execution.process_data = '{"__dify_retry_history": [{"retry_index": 1}]}'
|
||||
|
||||
assert repository.load_full_process_data(execution) == {"__dify_retry_history": [{"retry_index": 1}]}
|
||||
|
||||
|
||||
def test_get_execution_by_id_keeps_process_data_from_highest_failed_log_version(sql_repository) -> None:
|
||||
repository, database = sql_repository
|
||||
execution_id, tenant_id = "execution'1", 'tenant"1'
|
||||
database.executemany(
|
||||
"INSERT INTO workflow_node_execution (id, tenant_id, log_version, status, process_data, __time__) "
|
||||
"VALUES (?, ?, ?, ?, ?, 1)",
|
||||
[
|
||||
(execution_id, tenant_id, 1, "running", "{}"),
|
||||
(execution_id, tenant_id, 2, "failed", '{"workflow_agent_binding_id":"binding-1"}'),
|
||||
("other-execution", "other-tenant", 3, "succeeded", "{}"),
|
||||
],
|
||||
)
|
||||
execution = repository.get_execution_by_id(execution_id, tenant_id)
|
||||
|
||||
assert execution is not None
|
||||
assert execution.status.value == "failed"
|
||||
assert execution.process_data_dict == {"workflow_agent_binding_id": "binding-1"}
|
||||
if not repository.logstore_client.supports_pg_protocol:
|
||||
sdk_query = repository.logstore_client.client.get_logs.call_args.args[0].get_query()
|
||||
assert sdk_query.startswith('id: "execution\'1" and tenant_id: "tenant\\"1" | ')
|
||||
assert repository.get_execution_by_id(execution_id).id == execution_id
|
||||
assert repository.get_execution_by_id(execution_id, "other-tenant") is None
|
||||
assert repository.get_execution_by_id("unknown") is None
|
||||
assert repository.get_execution_by_id("execution' OR '1'='1") is None
|
||||
|
||||
|
||||
_CREATED_AT = datetime.datetime(2026, 8, 18, 2, 0, 0, tzinfo=datetime.UTC)
|
||||
_FINISHED_AT = _CREATED_AT + datetime.timedelta(seconds=30)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def non_utc_host_timezone(monkeypatch: pytest.MonkeyPatch) -> Generator[None]:
|
||||
"""Run the host clock in UTC+05:30 so local-time conversions become observable."""
|
||||
monkeypatch.setenv("TZ", "Asia/Kolkata")
|
||||
time.tzset()
|
||||
yield
|
||||
monkeypatch.undo()
|
||||
time.tzset()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("case", "payload"),
|
||||
[
|
||||
("both epoch", {"created_at": _CREATED_AT.timestamp(), "finished_at": _FINISHED_AT.timestamp()}),
|
||||
("aware iso and epoch", {"created_at": _CREATED_AT.isoformat(), "finished_at": _FINISHED_AT.timestamp()}),
|
||||
(
|
||||
"naive iso and epoch",
|
||||
{"created_at": _CREATED_AT.replace(tzinfo=None).isoformat(), "finished_at": _FINISHED_AT.timestamp()},
|
||||
),
|
||||
("both datetime", {"created_at": _CREATED_AT, "finished_at": _FINISHED_AT}),
|
||||
],
|
||||
)
|
||||
@pytest.mark.usefixtures("non_utc_host_timezone")
|
||||
def test_dict_to_node_execution_normalizes_timestamps_to_naive_utc(case: str, payload: dict[str, object]) -> None:
|
||||
model = _dict_to_workflow_node_execution_model({"id": "execution-1", **payload})
|
||||
|
||||
assert model.created_at == _CREATED_AT.replace(tzinfo=None), case
|
||||
assert model.finished_at == _FINISHED_AT.replace(tzinfo=None), case
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("non_utc_host_timezone")
|
||||
def test_dict_to_node_execution_defaults_missing_created_at_to_naive_utc_now() -> None:
|
||||
model = _dict_to_workflow_node_execution_model({"id": "execution-1"})
|
||||
|
||||
assert model.created_at.tzinfo is None
|
||||
assert abs((model.created_at - datetime.datetime.now(tz=datetime.UTC).replace(tzinfo=None)).total_seconds()) < 60
|
||||
+155
@@ -0,0 +1,155 @@
|
||||
import datetime
|
||||
import time
|
||||
from collections.abc import Generator
|
||||
from typing import cast, override
|
||||
|
||||
import pytest
|
||||
|
||||
from extensions.logstore.aliyun_logstore import AliyunLogStore
|
||||
from extensions.logstore.repositories.logstore_api_workflow_run_repository import (
|
||||
LogstoreAPIWorkflowRunRepository,
|
||||
_dict_to_workflow_run,
|
||||
)
|
||||
from graphon.enums import WorkflowExecutionStatus
|
||||
from models.enums import WorkflowRunTriggeredFrom
|
||||
from repositories.api_workflow_run_repository import AppWorkflowRunStats
|
||||
|
||||
_START = datetime.datetime(2026, 8, 18, 2, 0, 0, tzinfo=datetime.UTC)
|
||||
_FINISH = _START + datetime.timedelta(seconds=30)
|
||||
_EXPECTED_CREATED_AT = _START.replace(tzinfo=None)
|
||||
_EXPECTED_FINISHED_AT = _FINISH.replace(tzinfo=None)
|
||||
|
||||
_BASE: dict[str, object] = {"id": "run-1", "tenant_id": "tenant-1", "app_id": "app-1", "workflow_id": "workflow-1"}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def non_utc_host_timezone(monkeypatch: pytest.MonkeyPatch) -> Generator[None]:
|
||||
"""Run the host clock in UTC+05:30 so local-time conversions become observable."""
|
||||
monkeypatch.setenv("TZ", "Asia/Kolkata")
|
||||
time.tzset()
|
||||
yield
|
||||
monkeypatch.undo()
|
||||
time.tzset()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("case", "payload"),
|
||||
[
|
||||
("both epoch", {"started_at": _START.timestamp(), "finished_at": _FINISH.timestamp()}),
|
||||
("aware iso and epoch", {"started_at": _START.isoformat(), "finished_at": _FINISH.timestamp()}),
|
||||
(
|
||||
"naive iso and epoch",
|
||||
{"started_at": _START.replace(tzinfo=None).isoformat(), "finished_at": _FINISH.timestamp()},
|
||||
),
|
||||
("both datetime", {"started_at": _START, "finished_at": _FINISH}),
|
||||
],
|
||||
)
|
||||
@pytest.mark.usefixtures("non_utc_host_timezone")
|
||||
def test_dict_to_workflow_run_normalizes_timestamps_to_naive_utc(case: str, payload: dict[str, object]) -> None:
|
||||
model = _dict_to_workflow_run({**_BASE, **payload})
|
||||
|
||||
assert model.created_at == _EXPECTED_CREATED_AT, case
|
||||
assert model.finished_at == _EXPECTED_FINISHED_AT, case
|
||||
assert model.elapsed_time == 30.0, case
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("non_utc_host_timezone")
|
||||
def test_dict_to_workflow_run_defaults_missing_started_at_to_naive_utc_now() -> None:
|
||||
model = _dict_to_workflow_run({**_BASE, "finished_at": _FINISH.timestamp()})
|
||||
|
||||
assert model.created_at.tzinfo is None
|
||||
# A naive local-time default would sit 5h30m ahead of UTC and drive elapsed_time negative.
|
||||
assert abs((model.created_at - datetime.datetime.now(tz=datetime.UTC).replace(tzinfo=None)).total_seconds()) < 60
|
||||
|
||||
|
||||
class _FakeLogStore:
|
||||
def __init__(self, rows: list[dict[str, object]]) -> None:
|
||||
self.rows = rows
|
||||
self.sql: list[str] = []
|
||||
|
||||
def execute_sql(self, sql: str, logstore: str | None = None, **_options: object) -> list[dict[str, object]]:
|
||||
assert logstore == AliyunLogStore.workflow_execution_logstore
|
||||
self.sql.append(sql)
|
||||
return self.rows
|
||||
|
||||
|
||||
def _repository(rows: list[dict[str, object]]) -> tuple[LogstoreAPIWorkflowRunRepository, _FakeLogStore]:
|
||||
repository = LogstoreAPIWorkflowRunRepository.__new__(LogstoreAPIWorkflowRunRepository)
|
||||
client = _FakeLogStore(rows)
|
||||
repository.logstore_client = cast(AliyunLogStore, client)
|
||||
return repository, client
|
||||
|
||||
|
||||
_APPS = {"app-1": "tenant-1", "app-'2": "tenant-1"}
|
||||
_SOURCES = [WorkflowRunTriggeredFrom.APP_RUN, WorkflowRunTriggeredFrom.WEBHOOK]
|
||||
|
||||
|
||||
class _RoutingLogStore(_FakeLogStore):
|
||||
"""Answer the windowed count query and the newest-start query with separate rows."""
|
||||
|
||||
def __init__(self, counts: list[dict[str, object]], last: list[dict[str, object]]) -> None:
|
||||
super().__init__([])
|
||||
self.counts = counts
|
||||
self.last = last
|
||||
|
||||
@override
|
||||
def execute_sql(self, sql: str, logstore: str | None = None, **_options: object) -> list[dict[str, object]]:
|
||||
super().execute_sql(sql, logstore)
|
||||
return self.last if "MAX(started_at)" in sql else self.counts
|
||||
|
||||
|
||||
def test_stats_by_app_ids_dedupes_versions_and_keeps_each_apps_tenant() -> None:
|
||||
repository = LogstoreAPIWorkflowRunRepository.__new__(LogstoreAPIWorkflowRunRepository)
|
||||
client = _RoutingLogStore(
|
||||
counts=[
|
||||
{"tenant_id": "tenant-1", "app_id": "app-1", "status": "succeeded", "count": "3"},
|
||||
{"tenant_id": "tenant-1", "app_id": "app-1", "status": "running", "count": 1},
|
||||
{"tenant_id": "tenant-2", "app_id": "app-1", "status": "failed", "count": 9},
|
||||
{"tenant_id": "tenant-1", "app_id": "app-1", "status": "bogus", "count": 9},
|
||||
],
|
||||
last=[
|
||||
{"tenant_id": "tenant-1", "app_id": "app-1", "last_started_at": "2026-10-14T12:30:00"},
|
||||
{"tenant_id": "tenant-1", "app_id": "app-'2", "last_started_at": "2026-01-01T00:00:00"},
|
||||
{"tenant_id": "tenant-2", "app_id": "app-'2", "last_started_at": "2026-10-14T13:00:00"},
|
||||
],
|
||||
)
|
||||
repository.logstore_client = cast(AliyunLogStore, client)
|
||||
since = datetime.datetime(2026, 10, 14, 12)
|
||||
|
||||
stats = repository.get_stats_by_app_ids(_APPS, _SOURCES, since)
|
||||
|
||||
assert stats == {
|
||||
"app-1": AppWorkflowRunStats(
|
||||
status_counts={WorkflowExecutionStatus.SUCCEEDED: 3, WorkflowExecutionStatus.RUNNING: 1},
|
||||
last_created_at=datetime.datetime(2026, 10, 14, 12, 30),
|
||||
),
|
||||
"app-'2": AppWorkflowRunStats(status_counts={}, last_created_at=datetime.datetime(2026, 1, 1)),
|
||||
}
|
||||
counts_sql, last_sql = client.sql
|
||||
assert "ROW_NUMBER() OVER (PARTITION BY id ORDER BY log_version DESC)" in counts_sql
|
||||
assert "app_id IN ('app-1', 'app-''2')" in counts_sql
|
||||
assert "triggered_from IN ('app-run', 'webhook')" in counts_sql
|
||||
assert f"__time__ >= {int(since.replace(tzinfo=datetime.UTC).timestamp())}" in counts_sql
|
||||
assert "started_at >= '2026-10-14T12:00:00'" in counts_sql
|
||||
assert "app_id IN ('app-1', 'app-''2')" in last_sql
|
||||
|
||||
|
||||
def test_stats_by_app_ids_skips_the_queries_without_apps() -> None:
|
||||
repository, client = _repository([])
|
||||
|
||||
assert repository.get_stats_by_app_ids({}, _SOURCES, datetime.datetime(2026, 10, 14)) == {}
|
||||
assert client.sql == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("elapsed_time", ["0.0", "5.5"])
|
||||
def test_dict_to_workflow_run_prefers_stored_active_duration(elapsed_time: str) -> None:
|
||||
model = _dict_to_workflow_run({**_BASE, "started_at": _START, "finished_at": _FINISH, "elapsed_time": elapsed_time})
|
||||
|
||||
assert model.elapsed_time == float(elapsed_time)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("elapsed_time", [None, "null"])
|
||||
def test_dict_to_workflow_run_defaults_legacy_unfinished_duration_to_zero(elapsed_time: str | None) -> None:
|
||||
model = _dict_to_workflow_run({**_BASE, "elapsed_time": elapsed_time})
|
||||
|
||||
assert model.elapsed_time == 0.0
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
from collections.abc import Callable
|
||||
from types import SimpleNamespace
|
||||
from typing import cast
|
||||
from unittest.mock import patch
|
||||
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from extensions.logstore.repositories.logstore_workflow_execution_repository import (
|
||||
LogstoreWorkflowExecutionRepository,
|
||||
)
|
||||
from models.account import Account
|
||||
from models.enums import WorkflowRunTriggeredFrom
|
||||
|
||||
|
||||
def test_repository_uses_typed_logstore_migration_flags(
|
||||
config_overrides: Callable[..., None],
|
||||
sqlite_session_factory: sessionmaker[Session],
|
||||
) -> None:
|
||||
config_overrides(
|
||||
LOGSTORE_DUAL_WRITE_ENABLED=True,
|
||||
LOGSTORE_ENABLE_PUT_GRAPH_FIELD=False,
|
||||
)
|
||||
with (
|
||||
patch("extensions.logstore.repositories.logstore_workflow_execution_repository.AliyunLogStore"),
|
||||
patch(
|
||||
"extensions.logstore.repositories.logstore_workflow_execution_repository."
|
||||
"SQLAlchemyWorkflowExecutionRepository"
|
||||
),
|
||||
):
|
||||
repository = LogstoreWorkflowExecutionRepository(
|
||||
session_factory=sqlite_session_factory,
|
||||
tenant_id="tenant-1",
|
||||
user=cast(Account, SimpleNamespace(id="account-1")),
|
||||
app_id="app-1",
|
||||
triggered_from=WorkflowRunTriggeredFrom.APP_RUN,
|
||||
)
|
||||
|
||||
assert repository._enable_dual_write is True
|
||||
assert repository._enable_put_graph_field is False
|
||||
+152
@@ -0,0 +1,152 @@
|
||||
from collections.abc import Callable
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import Engine
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from core.workflow.node_execution_process_data import WORKFLOW_TOOL_ROOT_APP_ID_KEY
|
||||
from extensions.logstore.repositories.logstore_workflow_node_execution_repository import (
|
||||
LogstoreWorkflowNodeExecutionRepository,
|
||||
)
|
||||
from graphon.entities import WorkflowNodeExecution
|
||||
from graphon.enums import BuiltinNodeTypes, WorkflowNodeExecutionStatus
|
||||
from libs.datetime_utils import naive_utc_now
|
||||
from models.account import Account
|
||||
from models.workflow import WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom
|
||||
|
||||
|
||||
def _make_account() -> Account:
|
||||
account = Account(name="Logstore User", email="logstore@example.com")
|
||||
account.id = "account-1"
|
||||
return account
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def logstore_schema(sqlite_engine: Engine) -> None:
|
||||
with sqlite_engine.begin() as connection:
|
||||
connection.exec_driver_sql(
|
||||
"CREATE TABLE workflow_node_execution AS SELECT *, 1 AS log_version FROM workflow_node_executions WHERE 0"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("logstore_schema")
|
||||
@pytest.mark.parametrize("source_app", ["caller-app", "source-app"])
|
||||
def test_workflow_tool_scope_preserves_logstore_and_source_scoped_synchronous_agent_rows(
|
||||
config_overrides: Callable[..., None],
|
||||
sqlite_session_factory: sessionmaker[Session],
|
||||
sqlite_engine: Engine,
|
||||
source_app: str,
|
||||
) -> None:
|
||||
config_overrides(LOGSTORE_DUAL_WRITE_ENABLED=False)
|
||||
with patch(
|
||||
"extensions.logstore.repositories.logstore_workflow_node_execution_repository.AliyunLogStore"
|
||||
) as logstore:
|
||||
caller = LogstoreWorkflowNodeExecutionRepository(
|
||||
session_factory=sqlite_session_factory,
|
||||
tenant_id="tenant-1",
|
||||
user=_make_account(),
|
||||
app_id="caller-app",
|
||||
triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,
|
||||
)
|
||||
source = caller.for_workflow_tool(source_app)
|
||||
|
||||
node = WorkflowNodeExecution(
|
||||
id="source-exec",
|
||||
node_execution_id="source-exec",
|
||||
workflow_id="source-workflow",
|
||||
workflow_execution_id="caller-run",
|
||||
index=1,
|
||||
node_id="agent",
|
||||
node_type=BuiltinNodeTypes.AGENT,
|
||||
title="Agent",
|
||||
created_at=naive_utc_now(),
|
||||
process_data={WORKFLOW_TOOL_ROOT_APP_ID_KEY: "caller-app"},
|
||||
)
|
||||
source.save_synchronously(node)
|
||||
with sqlite_session_factory() as session:
|
||||
row = session.get(WorkflowNodeExecutionModel, "source-exec")
|
||||
assert row is not None
|
||||
assert (row.app_id, row.tenant_id, row.created_by, row.workflow_run_id) == (
|
||||
source_app,
|
||||
"tenant-1",
|
||||
"account-1",
|
||||
"caller-run",
|
||||
)
|
||||
assert row.triggered_from == WorkflowNodeExecutionTriggeredFrom.WORKFLOW_TOOL
|
||||
|
||||
node.status = WorkflowNodeExecutionStatus.SUCCEEDED
|
||||
source.save(node)
|
||||
logstore.return_value.put_log.assert_called_once()
|
||||
payload = dict(logstore.return_value.put_log.call_args.args[1])
|
||||
assert (payload["app_id"], payload["tenant_id"], payload["created_by"], payload["status"]) == (
|
||||
source_app,
|
||||
"tenant-1",
|
||||
"account-1",
|
||||
"succeeded",
|
||||
)
|
||||
assert payload["triggered_from"] == WorkflowNodeExecutionTriggeredFrom.WORKFLOW_TOOL
|
||||
|
||||
caller.save_synchronously(node.model_copy(update={"id": "caller-exec", "node_execution_id": "caller-exec"}))
|
||||
caller.save_synchronously(node.model_copy(update={"id": "legacy-exec", "node_execution_id": "legacy-exec"}))
|
||||
for execution_id in ("foreign-root", "foreign-run", "foreign-tenant"):
|
||||
source.save_synchronously(node.model_copy(update={"id": execution_id, "node_execution_id": execution_id}))
|
||||
with sqlite_session_factory() as session:
|
||||
foreign_root = session.get(WorkflowNodeExecutionModel, "foreign-root")
|
||||
foreign_run = session.get(WorkflowNodeExecutionModel, "foreign-run")
|
||||
foreign_tenant = session.get(WorkflowNodeExecutionModel, "foreign-tenant")
|
||||
assert foreign_root is not None
|
||||
assert foreign_run is not None
|
||||
assert foreign_tenant is not None
|
||||
foreign_root.process_data = '{"workflow_tool_root_app_id": "other-app"}'
|
||||
foreign_run.workflow_run_id = "other-run"
|
||||
foreign_tenant.tenant_id = "other-tenant"
|
||||
session.commit()
|
||||
with sqlite_engine.begin() as connection:
|
||||
connection.exec_driver_sql(
|
||||
"INSERT INTO workflow_node_execution SELECT *, 1 AS log_version FROM workflow_node_executions"
|
||||
)
|
||||
connection.exec_driver_sql("UPDATE workflow_node_execution SET triggered_from = NULL WHERE id = 'legacy-exec'")
|
||||
|
||||
def execute_query(*, sql: str, **_kwargs: object) -> list[dict[str, object]]:
|
||||
with sqlite_engine.connect() as connection:
|
||||
return [
|
||||
dict(row)
|
||||
for row in connection.exec_driver_sql(sql.replace("json_extract_scalar", "json_extract")).mappings()
|
||||
]
|
||||
|
||||
logstore.return_value.execute_sql.side_effect = execute_query
|
||||
assert {execution.id for execution in caller.get_by_workflow_execution("caller-run")} == {
|
||||
"caller-exec",
|
||||
"legacy-exec",
|
||||
}
|
||||
assert {execution.id for execution in source.get_by_workflow_execution("caller-run")} == {
|
||||
"source-exec",
|
||||
"foreign-root",
|
||||
}
|
||||
|
||||
|
||||
def test_save_synchronously_writes_sql_when_dual_write_is_disabled(
|
||||
config_overrides: Callable[..., None], sqlite_session_factory: sessionmaker[Session]
|
||||
) -> None:
|
||||
config_overrides(LOGSTORE_DUAL_WRITE_ENABLED=False)
|
||||
with (
|
||||
patch("extensions.logstore.repositories.logstore_workflow_node_execution_repository.AliyunLogStore"),
|
||||
patch(
|
||||
"extensions.logstore.repositories.logstore_workflow_node_execution_repository."
|
||||
"SQLAlchemyWorkflowNodeExecutionRepository"
|
||||
) as sql_repository_type,
|
||||
):
|
||||
repository = LogstoreWorkflowNodeExecutionRepository(
|
||||
session_factory=sqlite_session_factory,
|
||||
tenant_id="tenant-1",
|
||||
user=_make_account(),
|
||||
app_id="app-1",
|
||||
triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,
|
||||
)
|
||||
|
||||
execution = MagicMock()
|
||||
repository.save_synchronously(execution)
|
||||
|
||||
assert repository._enable_dual_write is False
|
||||
sql_repository_type.return_value.save_synchronously.assert_called_once_with(execution)
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Test that the AliyunLogStorePG port-connectivity logger captures the traceback.
|
||||
|
||||
Cycle 21 (sibling of #41066 / merged in #41068): `logger.debug("...: %s", str(e))`
|
||||
sites in `extensions/logstore/aliyun_logstore_pg.py` were converted to
|
||||
`logger.debug("...", exc_info=True)` so the traceback is captured at the
|
||||
same log level rather than being silently dropped. This test exercises one
|
||||
of the modified sites — `_check_port_connectivity` — to confirm the fix
|
||||
shape is correct: the captured log record has a non-empty traceback.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from extensions.logstore.aliyun_logstore_pg import AliyunLogStorePG
|
||||
|
||||
|
||||
def test_check_port_connectivity_captures_traceback_on_exception(caplog: pytest.LogCaptureFixture) -> None:
|
||||
"""A socket failure during the port check must produce a log record with exc_info set.
|
||||
|
||||
Before cycle 21: the record was logged as
|
||||
`Port connectivity check failed for host:port: <str(exception)>` with no
|
||||
traceback. After cycle 21: the record is logged as
|
||||
`Port connectivity check failed for host:port` with `exc_info=True`, so
|
||||
the traceback is part of the log record (and the host/port are still
|
||||
included via the format args).
|
||||
"""
|
||||
store = AliyunLogStorePG(
|
||||
access_key_id="ak",
|
||||
access_key_secret="sk",
|
||||
endpoint="https://example.com",
|
||||
project_name="p",
|
||||
)
|
||||
|
||||
with (
|
||||
patch("extensions.logstore.aliyun_logstore_pg.socket.socket", side_effect=OSError("boom")),
|
||||
caplog.at_level("DEBUG", logger="extensions.logstore.aliyun_logstore_pg"),
|
||||
):
|
||||
result = store._check_port_connectivity("example.invalid", 9999)
|
||||
|
||||
assert result is False
|
||||
|
||||
matching_records = [r for r in caplog.records if r.name == "extensions.logstore.aliyun_logstore_pg"]
|
||||
assert len(matching_records) == 1
|
||||
record = matching_records[0]
|
||||
assert record.levelname == "DEBUG"
|
||||
assert record.exc_info is not None
|
||||
formatted = caplog.text
|
||||
assert "Port connectivity check failed for example.invalid:9999" in formatted
|
||||
assert "Traceback (most recent call last)" in formatted
|
||||
assert "OSError: boom" in formatted
|
||||
# The exception is captured via the traceback, not interpolated into the
|
||||
# format string — so the message text is the short form, not the long one.
|
||||
assert "boom" not in record.getMessage()
|
||||
@@ -0,0 +1,469 @@
|
||||
"""
|
||||
Unit tests for SQL escape utility functions.
|
||||
|
||||
These tests ensure that SQL injection attacks are properly prevented
|
||||
in LogStore queries, particularly for cross-tenant access scenarios.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from extensions.logstore.sql_escape import escape_identifier, escape_logstore_query_value, escape_sql_string
|
||||
|
||||
|
||||
class TestEscapeSQLString:
|
||||
"""Test escape_sql_string function."""
|
||||
|
||||
def test_escape_empty_string(self):
|
||||
"""Test escaping empty string."""
|
||||
assert escape_sql_string("") == ""
|
||||
|
||||
def test_escape_normal_string(self):
|
||||
"""Test escaping string without special characters."""
|
||||
assert escape_sql_string("tenant_abc123") == "tenant_abc123"
|
||||
assert escape_sql_string("app-uuid-1234") == "app-uuid-1234"
|
||||
|
||||
def test_escape_single_quote(self):
|
||||
"""Test escaping single quote."""
|
||||
# Single quote should be doubled
|
||||
assert escape_sql_string("tenant'id") == "tenant''id"
|
||||
assert escape_sql_string("O'Reilly") == "O''Reilly"
|
||||
|
||||
def test_escape_multiple_quotes(self):
|
||||
"""Test escaping multiple single quotes."""
|
||||
assert escape_sql_string("a'b'c") == "a''b''c"
|
||||
assert escape_sql_string("'''") == "''''''"
|
||||
|
||||
# === SQL Injection Attack Scenarios ===
|
||||
|
||||
def test_prevent_boolean_injection(self):
|
||||
"""Test prevention of boolean injection attacks."""
|
||||
# Classic OR 1=1 attack
|
||||
malicious_input = "tenant' OR '1'='1"
|
||||
escaped = escape_sql_string(malicious_input)
|
||||
assert escaped == "tenant'' OR ''1''=''1"
|
||||
|
||||
# When used in SQL, this becomes a safe string literal
|
||||
sql = f"WHERE tenant_id='{escaped}'"
|
||||
assert sql == "WHERE tenant_id='tenant'' OR ''1''=''1'"
|
||||
# The entire input is now a string literal that won't match any tenant
|
||||
|
||||
def test_prevent_or_injection(self):
|
||||
"""Test prevention of OR-based injection."""
|
||||
malicious_input = "tenant_a' OR tenant_id='tenant_b"
|
||||
escaped = escape_sql_string(malicious_input)
|
||||
assert escaped == "tenant_a'' OR tenant_id=''tenant_b"
|
||||
|
||||
sql = f"WHERE tenant_id='{escaped}'"
|
||||
# The OR is now part of the string literal, not SQL logic
|
||||
assert "OR tenant_id=" in sql
|
||||
# The SQL has: opening ', doubled internal quotes '', and closing '
|
||||
assert sql == "WHERE tenant_id='tenant_a'' OR tenant_id=''tenant_b'"
|
||||
|
||||
def test_prevent_union_injection(self):
|
||||
"""Test prevention of UNION-based injection."""
|
||||
malicious_input = "xxx' UNION SELECT password FROM users WHERE '1'='1"
|
||||
escaped = escape_sql_string(malicious_input)
|
||||
assert escaped == "xxx'' UNION SELECT password FROM users WHERE ''1''=''1"
|
||||
|
||||
# UNION becomes part of the string literal
|
||||
assert "UNION" in escaped
|
||||
assert escaped.count("''") == 4 # All internal quotes are doubled
|
||||
|
||||
def test_prevent_comment_injection(self):
|
||||
"""Test prevention of comment-based injection."""
|
||||
# SQL comment to bypass remaining conditions
|
||||
malicious_input = "tenant' --"
|
||||
escaped = escape_sql_string(malicious_input)
|
||||
assert escaped == "tenant'' --"
|
||||
|
||||
sql = f"WHERE tenant_id='{escaped}' AND deleted=false"
|
||||
# The -- is now inside the string, not a SQL comment
|
||||
assert "--" in sql
|
||||
assert "AND deleted=false" in sql # This part is NOT commented out
|
||||
|
||||
def test_prevent_semicolon_injection(self):
|
||||
"""Test prevention of semicolon-based multi-statement injection."""
|
||||
malicious_input = "tenant'; DROP TABLE users; --"
|
||||
escaped = escape_sql_string(malicious_input)
|
||||
assert escaped == "tenant''; DROP TABLE users; --"
|
||||
|
||||
# Semicolons and DROP are now part of the string
|
||||
assert "DROP TABLE" in escaped
|
||||
|
||||
def test_prevent_time_based_blind_injection(self):
|
||||
"""Test prevention of time-based blind SQL injection."""
|
||||
malicious_input = "tenant' AND SLEEP(5) --"
|
||||
escaped = escape_sql_string(malicious_input)
|
||||
assert escaped == "tenant'' AND SLEEP(5) --"
|
||||
|
||||
# SLEEP becomes part of the string
|
||||
assert "SLEEP" in escaped
|
||||
|
||||
def test_prevent_wildcard_injection(self):
|
||||
"""Test prevention of wildcard-based injection."""
|
||||
malicious_input = "tenant' OR tenant_id LIKE '%"
|
||||
escaped = escape_sql_string(malicious_input)
|
||||
assert escaped == "tenant'' OR tenant_id LIKE ''%"
|
||||
|
||||
# The LIKE and wildcard are now part of the string
|
||||
assert "LIKE" in escaped
|
||||
|
||||
def test_prevent_null_byte_injection(self):
|
||||
"""Test handling of null bytes."""
|
||||
# Null bytes can sometimes bypass filters
|
||||
malicious_input = "tenant\x00' OR '1'='1"
|
||||
escaped = escape_sql_string(malicious_input)
|
||||
# Null byte is preserved, but quote is escaped
|
||||
assert "''1''=''1" in escaped
|
||||
|
||||
# === Real-world SAAS Scenarios ===
|
||||
|
||||
def test_cross_tenant_access_attempt(self):
|
||||
"""Test prevention of cross-tenant data access."""
|
||||
# Attacker tries to access another tenant's data
|
||||
attacker_input = "tenant_b' OR tenant_id='tenant_a"
|
||||
escaped = escape_sql_string(attacker_input)
|
||||
|
||||
sql = f"SELECT * FROM workflow_runs WHERE tenant_id='{escaped}'"
|
||||
# The query will look for a tenant literally named "tenant_b' OR tenant_id='tenant_a"
|
||||
# which doesn't exist - preventing access to either tenant's data
|
||||
assert "tenant_b'' OR tenant_id=''tenant_a" in sql
|
||||
|
||||
def test_cross_app_access_attempt(self):
|
||||
"""Test prevention of cross-application data access."""
|
||||
attacker_input = "app1' OR app_id='app2"
|
||||
escaped = escape_sql_string(attacker_input)
|
||||
|
||||
sql = f"WHERE app_id='{escaped}'"
|
||||
# Cannot access app2's data
|
||||
assert "app1'' OR app_id=''app2" in sql
|
||||
|
||||
def test_bypass_status_filter(self):
|
||||
"""Test prevention of bypassing status filters."""
|
||||
# Try to see all statuses instead of just 'running'
|
||||
attacker_input = "running' OR status LIKE '%"
|
||||
escaped = escape_sql_string(attacker_input)
|
||||
|
||||
sql = f"WHERE status='{escaped}'"
|
||||
# Status condition is not bypassed
|
||||
assert "running'' OR status LIKE ''%" in sql
|
||||
|
||||
# === Edge Cases ===
|
||||
|
||||
def test_escape_only_quotes(self):
|
||||
"""Test string with only quotes."""
|
||||
assert escape_sql_string("'") == "''"
|
||||
assert escape_sql_string("''") == "''''"
|
||||
|
||||
def test_escape_mixed_content(self):
|
||||
"""Test string with mixed quotes and other chars."""
|
||||
input_str = "It's a 'test' of O'Reilly's code"
|
||||
escaped = escape_sql_string(input_str)
|
||||
assert escaped == "It''s a ''test'' of O''Reilly''s code"
|
||||
|
||||
def test_escape_unicode_with_quotes(self):
|
||||
"""Test Unicode strings with quotes."""
|
||||
input_str = "租户' OR '1'='1"
|
||||
escaped = escape_sql_string(input_str)
|
||||
assert escaped == "租户'' OR ''1''=''1"
|
||||
|
||||
|
||||
class TestEscapeIdentifier:
|
||||
"""Test escape_identifier function."""
|
||||
|
||||
def test_escape_uuid(self):
|
||||
"""Test escaping UUID identifiers."""
|
||||
uuid = "550e8400-e29b-41d4-a716-446655440000"
|
||||
assert escape_identifier(uuid) == uuid
|
||||
|
||||
def test_escape_alphanumeric_id(self):
|
||||
"""Test escaping alphanumeric identifiers."""
|
||||
assert escape_identifier("tenant_123") == "tenant_123"
|
||||
assert escape_identifier("app-abc-123") == "app-abc-123"
|
||||
|
||||
def test_escape_identifier_with_quote(self):
|
||||
"""Test escaping identifier with single quote."""
|
||||
malicious = "tenant' OR '1'='1"
|
||||
escaped = escape_identifier(malicious)
|
||||
assert escaped == "tenant'' OR ''1''=''1"
|
||||
|
||||
def test_identifier_injection_attempt(self):
|
||||
"""Test prevention of injection through identifiers."""
|
||||
# Common identifier injection patterns
|
||||
test_cases = [
|
||||
("id' OR '1'='1", "id'' OR ''1''=''1"),
|
||||
("id'; DROP TABLE", "id''; DROP TABLE"),
|
||||
("id' UNION SELECT", "id'' UNION SELECT"),
|
||||
]
|
||||
|
||||
for malicious, expected in test_cases:
|
||||
assert escape_identifier(malicious) == expected
|
||||
|
||||
|
||||
class TestSQLInjectionIntegration:
|
||||
"""Integration tests simulating real SQL construction scenarios."""
|
||||
|
||||
def test_complete_where_clause_safety(self):
|
||||
"""Test that a complete WHERE clause is safe from injection."""
|
||||
# Simulating typical query construction
|
||||
tenant_id = "tenant' OR '1'='1"
|
||||
app_id = "app' UNION SELECT"
|
||||
run_id = "run' --"
|
||||
|
||||
escaped_tenant = escape_identifier(tenant_id)
|
||||
escaped_app = escape_identifier(app_id)
|
||||
escaped_run = escape_identifier(run_id)
|
||||
|
||||
sql = f"""
|
||||
SELECT * FROM workflow_runs
|
||||
WHERE tenant_id='{escaped_tenant}'
|
||||
AND app_id='{escaped_app}'
|
||||
AND id='{escaped_run}'
|
||||
"""
|
||||
|
||||
# Verify all special characters are escaped
|
||||
assert "tenant'' OR ''1''=''1" in sql
|
||||
assert "app'' UNION SELECT" in sql
|
||||
assert "run'' --" in sql
|
||||
|
||||
# Verify SQL structure is preserved (3 conditions with AND)
|
||||
assert sql.count("AND") == 2
|
||||
|
||||
def test_multiple_conditions_with_injection_attempts(self):
|
||||
"""Test multiple conditions all attempting injection."""
|
||||
conditions = {
|
||||
"tenant_id": "t1' OR tenant_id='t2",
|
||||
"app_id": "a1' OR app_id='a2",
|
||||
"status": "running' OR '1'='1",
|
||||
}
|
||||
|
||||
where_parts = []
|
||||
for field, value in conditions.items():
|
||||
escaped = escape_sql_string(value)
|
||||
where_parts.append(f"{field}='{escaped}'")
|
||||
|
||||
where_clause = " AND ".join(where_parts)
|
||||
|
||||
# All injection attempts are neutralized
|
||||
assert "t1'' OR tenant_id=''t2" in where_clause
|
||||
assert "a1'' OR app_id=''a2" in where_clause
|
||||
assert "running'' OR ''1''=''1" in where_clause
|
||||
|
||||
# AND structure is preserved
|
||||
assert where_clause.count(" AND ") == 2
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("attack_vector", "description"),
|
||||
[
|
||||
("' OR '1'='1", "Boolean injection"),
|
||||
("' OR '1'='1' --", "Boolean with comment"),
|
||||
("' UNION SELECT * FROM users --", "Union injection"),
|
||||
("'; DROP TABLE workflow_runs; --", "Destructive command"),
|
||||
("' AND SLEEP(10) --", "Time-based blind"),
|
||||
("' OR tenant_id LIKE '%", "Wildcard injection"),
|
||||
("admin' --", "Comment bypass"),
|
||||
("' OR 1=1 LIMIT 1 --", "Limit bypass"),
|
||||
],
|
||||
)
|
||||
def test_common_injection_vectors(self, attack_vector, description):
|
||||
"""Test protection against common injection attack vectors."""
|
||||
escaped = escape_sql_string(attack_vector)
|
||||
|
||||
# Build SQL
|
||||
sql = f"WHERE tenant_id='{escaped}'"
|
||||
|
||||
# Verify the attack string is now a safe literal
|
||||
# The key indicator: all internal single quotes are doubled
|
||||
internal_quotes = escaped.count("''")
|
||||
original_quotes = attack_vector.count("'")
|
||||
|
||||
# Each original quote should be doubled
|
||||
assert internal_quotes == original_quotes
|
||||
|
||||
# Verify SQL has exactly 2 quotes (opening and closing)
|
||||
assert sql.count("'") >= 2 # At least opening and closing
|
||||
|
||||
def test_logstore_specific_scenario(self):
|
||||
"""Test SQL injection prevention in LogStore-specific scenarios."""
|
||||
# Simulate LogStore query with window function
|
||||
tenant_id = "tenant' OR '1'='1"
|
||||
app_id = "app' UNION SELECT"
|
||||
|
||||
escaped_tenant = escape_identifier(tenant_id)
|
||||
escaped_app = escape_identifier(app_id)
|
||||
|
||||
sql = f"""
|
||||
SELECT * FROM (
|
||||
SELECT *, ROW_NUMBER() OVER (PARTITION BY id ORDER BY log_version DESC) as rn
|
||||
FROM workflow_execution_logstore
|
||||
WHERE tenant_id='{escaped_tenant}'
|
||||
AND app_id='{escaped_app}'
|
||||
AND __time__ > 0
|
||||
) AS subquery WHERE rn = 1
|
||||
"""
|
||||
|
||||
# Complex query structure is maintained
|
||||
assert "ROW_NUMBER()" in sql
|
||||
assert "PARTITION BY id" in sql
|
||||
|
||||
# Injection attempts are escaped
|
||||
assert "tenant'' OR ''1''=''1" in sql
|
||||
assert "app'' UNION SELECT" in sql
|
||||
|
||||
|
||||
# ====================================================================================
|
||||
# Tests for LogStore Query Syntax (SDK Mode)
|
||||
# ====================================================================================
|
||||
|
||||
|
||||
class TestLogStoreQueryEscape:
|
||||
"""Test escape_logstore_query_value for SDK mode query syntax."""
|
||||
|
||||
def test_normal_value(self):
|
||||
"""Test escaping normal alphanumeric value."""
|
||||
value = "550e8400-e29b-41d4-a716-446655440000"
|
||||
escaped = escape_logstore_query_value(value)
|
||||
|
||||
# Should be wrapped in double quotes
|
||||
assert escaped == '"550e8400-e29b-41d4-a716-446655440000"'
|
||||
|
||||
def test_empty_value(self):
|
||||
"""Test escaping empty string."""
|
||||
assert escape_logstore_query_value("") == '""'
|
||||
|
||||
def test_value_with_and_keyword(self):
|
||||
"""Test that 'and' keyword is neutralized when quoted."""
|
||||
malicious = "value and field:evil"
|
||||
escaped = escape_logstore_query_value(malicious)
|
||||
|
||||
# Should be wrapped in quotes, making 'and' a literal
|
||||
assert escaped == '"value and field:evil"'
|
||||
|
||||
# Simulate using in query
|
||||
query = f"tenant_id:{escaped}"
|
||||
assert query == 'tenant_id:"value and field:evil"'
|
||||
|
||||
def test_value_with_or_keyword(self):
|
||||
"""Test that 'or' keyword is neutralized when quoted."""
|
||||
malicious = "tenant_a or tenant_id:tenant_b"
|
||||
escaped = escape_logstore_query_value(malicious)
|
||||
|
||||
assert escaped == '"tenant_a or tenant_id:tenant_b"'
|
||||
|
||||
query = f"tenant_id:{escaped}"
|
||||
assert "or" in query # Present but as literal string
|
||||
|
||||
def test_value_with_not_keyword(self):
|
||||
"""Test that 'not' keyword is neutralized when quoted."""
|
||||
malicious = "not field:value"
|
||||
escaped = escape_logstore_query_value(malicious)
|
||||
|
||||
assert escaped == '"not field:value"'
|
||||
|
||||
def test_value_with_parentheses(self):
|
||||
"""Test that parentheses are neutralized when quoted."""
|
||||
malicious = "(tenant_a or tenant_b)"
|
||||
escaped = escape_logstore_query_value(malicious)
|
||||
|
||||
assert escaped == '"(tenant_a or tenant_b)"'
|
||||
assert "(" in escaped # Present as literal
|
||||
assert ")" in escaped # Present as literal
|
||||
|
||||
def test_value_with_colon(self):
|
||||
"""Test that colons are neutralized when quoted."""
|
||||
malicious = "field:value"
|
||||
escaped = escape_logstore_query_value(malicious)
|
||||
|
||||
assert escaped == '"field:value"'
|
||||
assert ":" in escaped # Present as literal
|
||||
|
||||
def test_value_with_double_quotes(self):
|
||||
"""Test that internal double quotes are escaped."""
|
||||
value_with_quotes = 'tenant"test"value'
|
||||
escaped = escape_logstore_query_value(value_with_quotes)
|
||||
|
||||
# Double quotes should be escaped with backslash
|
||||
assert escaped == '"tenant\\"test\\"value"'
|
||||
# Should have outer quotes plus escaped inner quotes
|
||||
assert '\\"' in escaped
|
||||
|
||||
def test_value_with_backslash(self):
|
||||
"""Test that backslashes are escaped."""
|
||||
value_with_backslash = "tenant\\test"
|
||||
escaped = escape_logstore_query_value(value_with_backslash)
|
||||
|
||||
# Backslash should be escaped
|
||||
assert escaped == '"tenant\\\\test"'
|
||||
assert "\\\\" in escaped
|
||||
|
||||
def test_value_with_backslash_and_quote(self):
|
||||
"""Test escaping both backslash and double quote."""
|
||||
value = 'path\\to\\"file"'
|
||||
escaped = escape_logstore_query_value(value)
|
||||
|
||||
# Both should be escaped
|
||||
assert escaped == '"path\\\\to\\\\\\"file\\""'
|
||||
# Verify escape order is correct
|
||||
assert "\\\\" in escaped # Escaped backslash
|
||||
assert '\\"' in escaped # Escaped double quote
|
||||
|
||||
def test_complex_injection_attempt(self):
|
||||
"""Test complex injection combining multiple operators."""
|
||||
malicious = 'tenant_a" or (tenant_id:"tenant_b" and app_id:"evil")'
|
||||
escaped = escape_logstore_query_value(malicious)
|
||||
|
||||
# All special chars should be literals or escaped
|
||||
assert escaped.startswith('"')
|
||||
assert escaped.endswith('"')
|
||||
# Inner double quotes escaped, operators become literals
|
||||
assert "or" in escaped
|
||||
assert "and" in escaped
|
||||
assert '\\"' in escaped # Escaped quotes
|
||||
|
||||
def test_only_backslash(self):
|
||||
"""Test escaping a single backslash."""
|
||||
assert escape_logstore_query_value("\\") == '"\\\\"'
|
||||
|
||||
def test_only_double_quote(self):
|
||||
"""Test escaping a single double quote."""
|
||||
assert escape_logstore_query_value('"') == '"\\""'
|
||||
|
||||
def test_multiple_backslashes(self):
|
||||
"""Test escaping multiple consecutive backslashes."""
|
||||
assert escape_logstore_query_value("\\\\\\") == '"\\\\\\\\\\\\"' # 3 backslashes -> 6
|
||||
|
||||
def test_escape_sequence_like_input(self):
|
||||
"""Test that existing escape sequences are properly escaped."""
|
||||
# Input looks like already escaped, but we still escape it
|
||||
value = 'value\\"test'
|
||||
escaped = escape_logstore_query_value(value)
|
||||
# \\ -> \\\\, " -> \"
|
||||
assert escaped == '"value\\\\\\"test"'
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("attack_scenario", "field", "malicious_value"),
|
||||
[
|
||||
("Cross-tenant via OR", "tenant_id", "tenant_a or tenant_id:tenant_b"),
|
||||
("Cross-app via AND", "app_id", "app_a and (app_id:app_b or app_id:app_c)"),
|
||||
("Boolean logic", "status", "succeeded or status:failed"),
|
||||
("Negation", "tenant_id", "not tenant_a"),
|
||||
("Field injection", "run_id", "run123 and tenant_id:evil_tenant"),
|
||||
("Parentheses grouping", "app_id", "app1 or (app_id:app2 and tenant_id:tenant2)"),
|
||||
("Quote breaking attempt", "tenant_id", 'tenant" or "1"="1'),
|
||||
("Backslash escape bypass", "app_id", "app\\ and app_id:evil"),
|
||||
],
|
||||
)
|
||||
def test_logstore_query_injection_scenarios(attack_scenario: str, field: str, malicious_value: str):
|
||||
"""Test that various LogStore query injection attempts are neutralized."""
|
||||
escaped = escape_logstore_query_value(malicious_value)
|
||||
|
||||
# Build query
|
||||
query = f"{field}:{escaped}"
|
||||
|
||||
# All operators should be within quoted string (literals)
|
||||
assert escaped.startswith('"')
|
||||
assert escaped.endswith('"')
|
||||
|
||||
# Verify the full query structure is safe
|
||||
assert query.count(":") >= 1 # At least the main field:value separator
|
||||
Whitespace-only changes.
@@ -0,0 +1,90 @@
|
||||
"""
|
||||
Shared fixtures for OTel tests.
|
||||
|
||||
Provides:
|
||||
- Mock TracerProvider with MemorySpanExporter
|
||||
- Mock configurations
|
||||
- Test data factories
|
||||
"""
|
||||
|
||||
from collections.abc import Iterator
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
|
||||
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
|
||||
from opentelemetry.trace import set_tracer_provider
|
||||
|
||||
from models import Account, App, EndUser
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def memory_span_exporter() -> InMemorySpanExporter:
|
||||
"""Provide an in-memory span exporter for testing."""
|
||||
return InMemorySpanExporter()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tracer_provider_with_memory_exporter(memory_span_exporter: InMemorySpanExporter) -> Iterator[TracerProvider]:
|
||||
"""Provide a TracerProvider configured with memory exporter."""
|
||||
import opentelemetry.trace as trace_api
|
||||
|
||||
trace_api._TRACER_PROVIDER = None
|
||||
trace_api._TRACER_PROVIDER_SET_ONCE._done = False
|
||||
|
||||
provider = TracerProvider()
|
||||
processor = SimpleSpanProcessor(memory_span_exporter)
|
||||
provider.add_span_processor(processor)
|
||||
set_tracer_provider(provider)
|
||||
|
||||
yield provider
|
||||
|
||||
provider.force_flush()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_app_model() -> App:
|
||||
"""Create a real transient App model."""
|
||||
return App(id="test-app-id", tenant_id="test-tenant-id")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_account_user() -> Account:
|
||||
"""Create a real transient Account user."""
|
||||
user = Account(name="Test User", email="otel@example.com")
|
||||
user.id = "test-user-id"
|
||||
return user
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_end_user() -> EndUser:
|
||||
"""Create a real transient EndUser."""
|
||||
return EndUser(id="test-end-user-id", tenant_id="test-tenant-id")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_workflow_runner() -> MagicMock:
|
||||
"""Create a mock WorkflowAppRunner."""
|
||||
runner = MagicMock()
|
||||
runner.application_generate_entity = MagicMock()
|
||||
runner.application_generate_entity.user_id = "test-user-id"
|
||||
runner.application_generate_entity.stream = True
|
||||
runner.application_generate_entity.app_config = MagicMock()
|
||||
runner.application_generate_entity.app_config.app_id = "test-app-id"
|
||||
runner.application_generate_entity.app_config.tenant_id = "test-tenant-id"
|
||||
runner.application_generate_entity.app_config.workflow_id = "test-workflow-id"
|
||||
return runner
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_handler_instances() -> Iterator[None]:
|
||||
"""Reset handler singleton instances before each test."""
|
||||
from extensions.otel.decorators.base import _HANDLER_INSTANCES
|
||||
|
||||
_HANDLER_INSTANCES.clear()
|
||||
from extensions.otel.decorators.handler import SpanHandler
|
||||
|
||||
_HANDLER_INSTANCES[SpanHandler] = SpanHandler()
|
||||
yield
|
||||
_HANDLER_INSTANCES.clear()
|
||||
Whitespace-only changes.
Whitespace-only changes.
+91
@@ -0,0 +1,91 @@
|
||||
"""
|
||||
Tests for AppGenerateHandler.
|
||||
|
||||
Test objectives:
|
||||
1. Verify handler compatibility with real function signature (fails when parameters change)
|
||||
2. Verify span attribute mapping correctness
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from core.app.entities.app_invoke_entities import InvokeFrom
|
||||
from extensions.otel.decorators.handlers.generate_handler import AppGenerateHandler
|
||||
from extensions.otel.semconv import DifySpanAttributes, GenAIAttributes
|
||||
from tests.helpers.unit.config_override import config_overrides_context
|
||||
|
||||
|
||||
class TestAppGenerateHandler:
|
||||
"""Core tests for AppGenerateHandler"""
|
||||
|
||||
@config_overrides_context(ENABLE_OTEL=True)
|
||||
def test_compatible_with_real_function_signature(
|
||||
self, tracer_provider_with_memory_exporter, mock_app_model, mock_account_user
|
||||
):
|
||||
"""
|
||||
Verify handler compatibility with real AppGenerateService.generate signature.
|
||||
|
||||
If AppGenerateService.generate parameters change, this test will fail,
|
||||
prompting developers to update the handler's parameter extraction logic.
|
||||
"""
|
||||
from services.app_generate_service import AppGenerateService
|
||||
|
||||
handler = AppGenerateHandler()
|
||||
|
||||
kwargs = {
|
||||
"session": MagicMock(),
|
||||
"app_model": mock_app_model,
|
||||
"user": mock_account_user,
|
||||
"args": {"workflow_id": "test-wf-123"},
|
||||
"invoke_from": InvokeFrom.DEBUGGER,
|
||||
"streaming": True,
|
||||
"root_node_id": None,
|
||||
}
|
||||
|
||||
arguments = handler._extract_arguments(AppGenerateService.generate, **kwargs)
|
||||
|
||||
assert arguments is not None, "Failed to extract arguments from AppGenerateService.generate"
|
||||
assert "app_model" in arguments, "Handler uses app_model but parameter is missing"
|
||||
assert "user" in arguments, "Handler uses user but parameter is missing"
|
||||
assert "args" in arguments, "Handler uses args but parameter is missing"
|
||||
assert "streaming" in arguments, "Handler uses streaming but parameter is missing"
|
||||
|
||||
@config_overrides_context(ENABLE_OTEL=True)
|
||||
def test_all_span_attributes_set_correctly(
|
||||
self, tracer_provider_with_memory_exporter, memory_span_exporter, mock_app_model, mock_account_user
|
||||
):
|
||||
"""Verify all span attributes are mapped correctly"""
|
||||
handler = AppGenerateHandler()
|
||||
tracer = tracer_provider_with_memory_exporter.get_tracer(__name__)
|
||||
|
||||
test_app_id = "app-456"
|
||||
test_tenant_id = "tenant-789"
|
||||
test_user_id = "user-111"
|
||||
test_workflow_id = "wf-222"
|
||||
|
||||
mock_app_model.id = test_app_id
|
||||
mock_app_model.tenant_id = test_tenant_id
|
||||
mock_account_user.id = test_user_id
|
||||
|
||||
def dummy_func(app_model, user, args, invoke_from, streaming=True):
|
||||
return "result"
|
||||
|
||||
handler.wrapper(
|
||||
tracer,
|
||||
dummy_func,
|
||||
app_model=mock_app_model,
|
||||
user=mock_account_user,
|
||||
args={"workflow_id": test_workflow_id},
|
||||
invoke_from=InvokeFrom.DEBUGGER,
|
||||
streaming=False,
|
||||
)
|
||||
|
||||
spans = memory_span_exporter.get_finished_spans()
|
||||
assert len(spans) == 1
|
||||
attrs = spans[0].attributes
|
||||
|
||||
assert attrs[DifySpanAttributes.APP_ID] == test_app_id
|
||||
assert attrs[DifySpanAttributes.TENANT_ID] == test_tenant_id
|
||||
assert attrs[GenAIAttributes.USER_ID] == test_user_id
|
||||
assert attrs[DifySpanAttributes.WORKFLOW_ID] == test_workflow_id
|
||||
assert attrs[DifySpanAttributes.USER_TYPE] == "Account"
|
||||
assert attrs[DifySpanAttributes.STREAMING] is False
|
||||
+102
@@ -0,0 +1,102 @@
|
||||
"""
|
||||
Tests for WorkflowAppRunnerHandler.
|
||||
|
||||
Test objectives:
|
||||
1. Verify handler compatibility with real WorkflowAppRunner structure (fails when structure changes)
|
||||
2. Verify span attribute mapping correctness
|
||||
"""
|
||||
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
|
||||
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
|
||||
from opentelemetry.trace import StatusCode
|
||||
from sqlalchemy import Engine
|
||||
|
||||
from extensions.otel.decorators.handlers.workflow_app_runner_handler import WorkflowAppRunnerHandler
|
||||
from extensions.otel.semconv import DifySpanAttributes, GenAIAttributes
|
||||
from services.workflow_run_agg import WorkflowRunAgg
|
||||
from tests.helpers.unit.config_override import config_overrides_context
|
||||
from tests.unit_tests.services.test_workflow_run_agg import make_workflow_runner
|
||||
|
||||
|
||||
def test_service_driver_emits_workflow_span(sqlite_engine: Engine) -> None:
|
||||
runner, runs, nodes = make_workflow_runner(sqlite_engine)
|
||||
exporter = InMemorySpanExporter()
|
||||
provider = TracerProvider()
|
||||
provider.add_span_processor(SimpleSpanProcessor(exporter))
|
||||
try:
|
||||
WorkflowAppRunnerHandler().wrapper(provider.get_tracer(__name__), WorkflowRunAgg.run, runner, None, runs, nodes)
|
||||
spans = exporter.get_finished_spans()
|
||||
assert len(spans) == 1
|
||||
assert spans[0].status.status_code == StatusCode.OK
|
||||
attributes = spans[0].attributes
|
||||
assert attributes is not None
|
||||
assert attributes[DifySpanAttributes.WORKFLOW_ID] == runner.application_generate_entity.app_config.workflow_id
|
||||
assert attributes[DifySpanAttributes.APP_ID] == runner.application_generate_entity.app_config.app_id
|
||||
assert attributes[GenAIAttributes.USER_ID] == runner.application_generate_entity.user_id
|
||||
finally:
|
||||
provider.shutdown()
|
||||
|
||||
|
||||
class TestWorkflowAppRunnerHandler:
|
||||
"""Core tests for WorkflowAppRunnerHandler"""
|
||||
|
||||
def test_handler_structure_dependencies(self):
|
||||
"""
|
||||
Verify handler dependencies on WorkflowAppRunner structure.
|
||||
|
||||
Handler depends on:
|
||||
- runner.application_generate_entity (WorkflowAppGenerateEntity)
|
||||
- entity.app_config (WorkflowAppConfig)
|
||||
- entity.user_id, entity.stream
|
||||
- app_config.app_id, app_config.tenant_id, app_config.workflow_id
|
||||
|
||||
If these attribute paths change in real types, this test will fail,
|
||||
prompting developers to update the handler's attribute access logic.
|
||||
"""
|
||||
from core.app.app_config.entities import WorkflowUIBasedAppConfig
|
||||
from core.app.entities.app_invoke_entities import WorkflowAppGenerateEntity
|
||||
|
||||
required_entity_fields = ["user_id", "stream", "app_config"]
|
||||
entity_fields = WorkflowAppGenerateEntity.model_fields
|
||||
for field in required_entity_fields:
|
||||
assert field in entity_fields, f"Handler expects WorkflowAppGenerateEntity.{field} but field is missing"
|
||||
|
||||
required_config_fields = ["app_id", "tenant_id", "workflow_id"]
|
||||
config_fields = WorkflowUIBasedAppConfig.model_fields
|
||||
for field in required_config_fields:
|
||||
assert field in config_fields, f"Handler expects app_config.{field} but field is missing"
|
||||
|
||||
@config_overrides_context(ENABLE_OTEL=True)
|
||||
def test_all_span_attributes_set_correctly(
|
||||
self, tracer_provider_with_memory_exporter, memory_span_exporter, mock_workflow_runner
|
||||
):
|
||||
"""Verify all span attributes are mapped correctly"""
|
||||
handler = WorkflowAppRunnerHandler()
|
||||
tracer = tracer_provider_with_memory_exporter.get_tracer(__name__)
|
||||
|
||||
test_app_id = "app-999"
|
||||
test_tenant_id = "tenant-888"
|
||||
test_user_id = "user-777"
|
||||
test_workflow_id = "wf-666"
|
||||
|
||||
mock_workflow_runner.application_generate_entity.user_id = test_user_id
|
||||
mock_workflow_runner.application_generate_entity.stream = False
|
||||
mock_workflow_runner.application_generate_entity.app_config.app_id = test_app_id
|
||||
mock_workflow_runner.application_generate_entity.app_config.tenant_id = test_tenant_id
|
||||
mock_workflow_runner.application_generate_entity.app_config.workflow_id = test_workflow_id
|
||||
|
||||
def runner_run(runner):
|
||||
return "result"
|
||||
|
||||
handler.wrapper(tracer, runner_run, mock_workflow_runner)
|
||||
|
||||
spans = memory_span_exporter.get_finished_spans()
|
||||
assert len(spans) == 1
|
||||
attrs = spans[0].attributes
|
||||
|
||||
assert attrs[DifySpanAttributes.APP_ID] == test_app_id
|
||||
assert attrs[DifySpanAttributes.TENANT_ID] == test_tenant_id
|
||||
assert attrs[GenAIAttributes.USER_ID] == test_user_id
|
||||
assert attrs[DifySpanAttributes.WORKFLOW_ID] == test_workflow_id
|
||||
assert attrs[DifySpanAttributes.STREAMING] is False
|
||||
@@ -0,0 +1,125 @@
|
||||
"""
|
||||
Tests for trace_span decorator.
|
||||
|
||||
Test coverage:
|
||||
- Decorator basic functionality
|
||||
- Enable/disable logic
|
||||
- Handler singleton management
|
||||
- Integration with OpenTelemetry SDK
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
import pytest
|
||||
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
|
||||
from opentelemetry.trace import StatusCode
|
||||
|
||||
from extensions.otel.decorators.base import trace_span
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _otel_enabled(config_overrides: Callable[..., None]) -> None:
|
||||
config_overrides(ENABLE_OTEL=True)
|
||||
|
||||
|
||||
class TestTraceSpanDecorator:
|
||||
"""Test trace_span decorator basic functionality."""
|
||||
|
||||
@pytest.mark.usefixtures("tracer_provider_with_memory_exporter")
|
||||
def test_decorated_function_executes_normally(self) -> None:
|
||||
"""Test that decorated function executes and returns correct value."""
|
||||
|
||||
@trace_span()
|
||||
def test_func(x: int, y: int) -> int:
|
||||
return x + y
|
||||
|
||||
result = test_func(2, 3)
|
||||
assert result == 5
|
||||
|
||||
@pytest.mark.usefixtures("tracer_provider_with_memory_exporter")
|
||||
def test_decorator_with_args_and_kwargs(self) -> None:
|
||||
"""Test that decorator correctly handles args and kwargs."""
|
||||
|
||||
@trace_span()
|
||||
def test_func(a: int, b: int, c: int = 10) -> int:
|
||||
return a + b + c
|
||||
|
||||
result = test_func(1, 2, c=3)
|
||||
assert result == 6
|
||||
|
||||
|
||||
class TestTraceSpanWithMemoryExporter:
|
||||
"""Test trace_span with MemorySpanExporter to verify span creation."""
|
||||
|
||||
@pytest.mark.usefixtures("tracer_provider_with_memory_exporter")
|
||||
def test_span_is_created_and_exported(self, memory_span_exporter: InMemorySpanExporter) -> None:
|
||||
"""Test that span is created and exported to memory exporter."""
|
||||
|
||||
@trace_span()
|
||||
def test_func() -> str:
|
||||
return "result"
|
||||
|
||||
test_func()
|
||||
|
||||
spans = memory_span_exporter.get_finished_spans()
|
||||
assert len(spans) == 1
|
||||
|
||||
@pytest.mark.usefixtures("tracer_provider_with_memory_exporter")
|
||||
def test_span_name_matches_function(self, memory_span_exporter: InMemorySpanExporter) -> None:
|
||||
"""Test that span name matches the decorated function."""
|
||||
|
||||
@trace_span()
|
||||
def my_test_function() -> str:
|
||||
return "result"
|
||||
|
||||
my_test_function()
|
||||
|
||||
spans = memory_span_exporter.get_finished_spans()
|
||||
assert len(spans) == 1
|
||||
assert "my_test_function" in spans[0].name
|
||||
|
||||
@pytest.mark.usefixtures("tracer_provider_with_memory_exporter")
|
||||
def test_span_status_is_ok_on_success(self, memory_span_exporter: InMemorySpanExporter) -> None:
|
||||
"""Test that span status is OK when function succeeds."""
|
||||
|
||||
@trace_span()
|
||||
def test_func() -> str:
|
||||
return "result"
|
||||
|
||||
test_func()
|
||||
|
||||
spans = memory_span_exporter.get_finished_spans()
|
||||
assert len(spans) == 1
|
||||
assert spans[0].status.status_code == StatusCode.OK
|
||||
|
||||
@pytest.mark.usefixtures("tracer_provider_with_memory_exporter")
|
||||
def test_span_status_is_error_on_exception(self, memory_span_exporter: InMemorySpanExporter) -> None:
|
||||
"""Test that span status is ERROR when function raises exception."""
|
||||
|
||||
@trace_span()
|
||||
def test_func() -> None:
|
||||
raise ValueError("test error")
|
||||
|
||||
with pytest.raises(ValueError, match="test error"):
|
||||
test_func()
|
||||
|
||||
spans = memory_span_exporter.get_finished_spans()
|
||||
assert len(spans) == 1
|
||||
assert spans[0].status.status_code == StatusCode.ERROR
|
||||
|
||||
@pytest.mark.usefixtures("tracer_provider_with_memory_exporter")
|
||||
def test_exception_is_recorded_in_span(self, memory_span_exporter: InMemorySpanExporter) -> None:
|
||||
"""Test that exception details are recorded in span events."""
|
||||
|
||||
@trace_span()
|
||||
def test_func() -> None:
|
||||
raise ValueError("test error")
|
||||
|
||||
with pytest.raises(ValueError, match="test error"):
|
||||
test_func()
|
||||
|
||||
spans = memory_span_exporter.get_finished_spans()
|
||||
assert len(spans) == 1
|
||||
events = spans[0].events
|
||||
assert len(events) > 0
|
||||
assert any("exception" in event.name.lower() for event in events)
|
||||
@@ -0,0 +1,258 @@
|
||||
"""
|
||||
Tests for SpanHandler base class.
|
||||
|
||||
Test coverage:
|
||||
- _build_span_name method
|
||||
- _extract_arguments method
|
||||
- wrapper method default implementation
|
||||
- Signature caching
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
import pytest
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
|
||||
from opentelemetry.trace import StatusCode
|
||||
|
||||
from extensions.otel.decorators.handler import SpanHandler
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _otel_enabled(config_overrides: Callable[..., None]) -> None:
|
||||
config_overrides(ENABLE_OTEL=True)
|
||||
|
||||
|
||||
class TestSpanHandlerExtractArguments:
|
||||
"""Test SpanHandler._extract_arguments method."""
|
||||
|
||||
def test_extract_positional_arguments(self) -> None:
|
||||
"""Test extracting positional arguments."""
|
||||
handler = SpanHandler()
|
||||
|
||||
def func(a: int, b: int, c: int) -> None:
|
||||
pass
|
||||
|
||||
result = handler._extract_arguments(func, 1, 2, 3)
|
||||
|
||||
assert result is not None
|
||||
assert result["a"] == 1
|
||||
assert result["b"] == 2
|
||||
assert result["c"] == 3
|
||||
|
||||
def test_extract_keyword_arguments(self) -> None:
|
||||
"""Test extracting keyword arguments."""
|
||||
handler = SpanHandler()
|
||||
|
||||
def func(a: int, b: int, c: int) -> None:
|
||||
pass
|
||||
|
||||
result = handler._extract_arguments(func, a=1, b=2, c=3)
|
||||
|
||||
assert result is not None
|
||||
assert result["a"] == 1
|
||||
assert result["b"] == 2
|
||||
assert result["c"] == 3
|
||||
|
||||
def test_extract_mixed_arguments(self) -> None:
|
||||
"""Test extracting mixed positional and keyword arguments."""
|
||||
handler = SpanHandler()
|
||||
|
||||
def func(a: int, b: int, c: int) -> None:
|
||||
pass
|
||||
|
||||
result = handler._extract_arguments(func, 1, b=2, c=3)
|
||||
|
||||
assert result is not None
|
||||
assert result["a"] == 1
|
||||
assert result["b"] == 2
|
||||
assert result["c"] == 3
|
||||
|
||||
def test_extract_arguments_with_defaults(self) -> None:
|
||||
"""Test extracting arguments with default values."""
|
||||
handler = SpanHandler()
|
||||
|
||||
def func(a: int, b: int = 10, c: int = 20) -> None:
|
||||
pass
|
||||
|
||||
result = handler._extract_arguments(func, 1)
|
||||
|
||||
assert result is not None
|
||||
assert result["a"] == 1
|
||||
assert result["b"] == 10
|
||||
assert result["c"] == 20
|
||||
|
||||
def test_extract_arguments_handles_self(self) -> None:
|
||||
"""Test extracting arguments from instance method (with self)."""
|
||||
handler = SpanHandler()
|
||||
|
||||
class MyClass:
|
||||
def method(self, a: int, b: int) -> None:
|
||||
pass
|
||||
|
||||
instance = MyClass()
|
||||
result = handler._extract_arguments(instance.method, 1, 2)
|
||||
|
||||
assert result is not None
|
||||
assert result["a"] == 1
|
||||
assert result["b"] == 2
|
||||
|
||||
def test_extract_arguments_returns_none_on_error(self) -> None:
|
||||
"""Test that _extract_arguments returns None when extraction fails."""
|
||||
handler = SpanHandler()
|
||||
|
||||
def func(a: int, b: int) -> None:
|
||||
pass
|
||||
|
||||
result = handler._extract_arguments(func, 1) # pyrefly: ignore[missing-argument]
|
||||
|
||||
assert result is None
|
||||
|
||||
def test_signature_caching(self) -> None:
|
||||
"""Test that function signatures are cached."""
|
||||
handler = SpanHandler()
|
||||
|
||||
def func(a: int, b: int) -> None:
|
||||
pass
|
||||
|
||||
assert func not in handler._signature_cache
|
||||
|
||||
handler._extract_arguments(func, 1, 2)
|
||||
assert func in handler._signature_cache
|
||||
|
||||
cached_sig = handler._signature_cache[func]
|
||||
handler._extract_arguments(func, 3, 4)
|
||||
assert handler._signature_cache[func] is cached_sig
|
||||
|
||||
|
||||
class TestSpanHandlerWrapper:
|
||||
"""Test SpanHandler.wrapper default implementation."""
|
||||
|
||||
def test_wrapper_creates_span(
|
||||
self, tracer_provider_with_memory_exporter: TracerProvider, memory_span_exporter: InMemorySpanExporter
|
||||
) -> None:
|
||||
"""Test that wrapper creates a span."""
|
||||
handler = SpanHandler()
|
||||
tracer = tracer_provider_with_memory_exporter.get_tracer(__name__)
|
||||
|
||||
def test_func() -> str:
|
||||
return "result"
|
||||
|
||||
result = handler.wrapper(tracer, test_func)
|
||||
|
||||
assert result == "result"
|
||||
spans = memory_span_exporter.get_finished_spans()
|
||||
assert len(spans) == 1
|
||||
|
||||
def test_wrapper_sets_span_kind_internal(
|
||||
self, tracer_provider_with_memory_exporter: TracerProvider, memory_span_exporter: InMemorySpanExporter
|
||||
) -> None:
|
||||
"""Test that wrapper sets SpanKind to INTERNAL."""
|
||||
from opentelemetry.trace import SpanKind
|
||||
|
||||
handler = SpanHandler()
|
||||
tracer = tracer_provider_with_memory_exporter.get_tracer(__name__)
|
||||
|
||||
def test_func() -> str:
|
||||
return "result"
|
||||
|
||||
handler.wrapper(tracer, test_func)
|
||||
|
||||
spans = memory_span_exporter.get_finished_spans()
|
||||
assert len(spans) == 1
|
||||
assert spans[0].kind == SpanKind.INTERNAL
|
||||
|
||||
def test_wrapper_sets_status_ok_on_success(
|
||||
self, tracer_provider_with_memory_exporter: TracerProvider, memory_span_exporter: InMemorySpanExporter
|
||||
) -> None:
|
||||
"""Test that wrapper sets status to OK when function succeeds."""
|
||||
handler = SpanHandler()
|
||||
tracer = tracer_provider_with_memory_exporter.get_tracer(__name__)
|
||||
|
||||
def test_func() -> str:
|
||||
return "result"
|
||||
|
||||
handler.wrapper(tracer, test_func)
|
||||
|
||||
spans = memory_span_exporter.get_finished_spans()
|
||||
assert len(spans) == 1
|
||||
assert spans[0].status.status_code == StatusCode.OK
|
||||
|
||||
def test_wrapper_records_exception_on_error(
|
||||
self, tracer_provider_with_memory_exporter: TracerProvider, memory_span_exporter: InMemorySpanExporter
|
||||
) -> None:
|
||||
"""Test that wrapper records exception when function raises."""
|
||||
handler = SpanHandler()
|
||||
tracer = tracer_provider_with_memory_exporter.get_tracer(__name__)
|
||||
|
||||
def test_func() -> None:
|
||||
raise ValueError("test error")
|
||||
|
||||
with pytest.raises(ValueError, match="test error"):
|
||||
handler.wrapper(tracer, test_func)
|
||||
|
||||
spans = memory_span_exporter.get_finished_spans()
|
||||
assert len(spans) == 1
|
||||
events = spans[0].events
|
||||
assert len(events) > 0
|
||||
assert any("exception" in event.name.lower() for event in events)
|
||||
|
||||
def test_wrapper_sets_status_error_on_exception(
|
||||
self, tracer_provider_with_memory_exporter: TracerProvider, memory_span_exporter: InMemorySpanExporter
|
||||
) -> None:
|
||||
"""Test that wrapper sets status to ERROR when function raises exception."""
|
||||
handler = SpanHandler()
|
||||
tracer = tracer_provider_with_memory_exporter.get_tracer(__name__)
|
||||
|
||||
def test_func() -> None:
|
||||
raise ValueError("test error")
|
||||
|
||||
with pytest.raises(ValueError, match="test error"):
|
||||
handler.wrapper(tracer, test_func)
|
||||
|
||||
spans = memory_span_exporter.get_finished_spans()
|
||||
assert len(spans) == 1
|
||||
assert spans[0].status.status_code == StatusCode.ERROR
|
||||
assert spans[0].status.description is not None
|
||||
assert "test error" in spans[0].status.description
|
||||
|
||||
def test_wrapper_re_raises_exception(self, tracer_provider_with_memory_exporter: TracerProvider) -> None:
|
||||
"""Test that wrapper re-raises exception after recording it."""
|
||||
handler = SpanHandler()
|
||||
tracer = tracer_provider_with_memory_exporter.get_tracer(__name__)
|
||||
|
||||
def test_func() -> None:
|
||||
raise ValueError("test error")
|
||||
|
||||
with pytest.raises(ValueError, match="test error"):
|
||||
handler.wrapper(tracer, test_func)
|
||||
|
||||
def test_wrapper_passes_arguments_correctly(self, tracer_provider_with_memory_exporter: TracerProvider) -> None:
|
||||
"""Test that wrapper correctly passes arguments to wrapped function."""
|
||||
handler = SpanHandler()
|
||||
tracer = tracer_provider_with_memory_exporter.get_tracer(__name__)
|
||||
|
||||
def test_func(a: int, b: int, c: int = 10) -> int:
|
||||
return a + b + c
|
||||
|
||||
result = handler.wrapper(tracer, test_func, 1, 2, c=3)
|
||||
|
||||
assert result == 6
|
||||
|
||||
def test_wrapper_with_memory_exporter(
|
||||
self, tracer_provider_with_memory_exporter: TracerProvider, memory_span_exporter: InMemorySpanExporter
|
||||
) -> None:
|
||||
"""Test wrapper end-to-end with memory exporter."""
|
||||
handler = SpanHandler()
|
||||
tracer = tracer_provider_with_memory_exporter.get_tracer(__name__)
|
||||
|
||||
def my_function(x: int) -> int:
|
||||
return x * 2
|
||||
|
||||
result = handler.wrapper(tracer, my_function, 5)
|
||||
|
||||
assert result == 10
|
||||
spans = memory_span_exporter.get_finished_spans()
|
||||
assert len(spans) == 1
|
||||
assert "my_function" in spans[0].name
|
||||
assert spans[0].status.status_code == StatusCode.OK
|
||||
@@ -0,0 +1,307 @@
|
||||
"""SQL comment scope through real Celery requests and OTel task instrumentation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Iterator
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from threading import Event
|
||||
from typing import Protocol, cast
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from celery import Celery, Task
|
||||
from celery.app.task import Context
|
||||
from celery.result import EagerResult
|
||||
from celery.signals import task_postrun, task_prerun
|
||||
from opentelemetry import context, trace
|
||||
from opentelemetry.instrumentation.celery import CeleryInstrumentor
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
|
||||
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
|
||||
from opentelemetry.sdk.trace.sampling import ALWAYS_OFF, ALWAYS_ON
|
||||
|
||||
from core.app.apps.exc import GenerateTaskStoppedError
|
||||
from extensions.otel.celery_sqlcommenter import (
|
||||
_SQLCOMMENTER_CONTEXT_KEY,
|
||||
_TOKEN_ATTR,
|
||||
CelerySqlcommenterTagsDict,
|
||||
_build_celery_sqlcommenter_tags,
|
||||
_on_task_postrun,
|
||||
_on_task_prerun,
|
||||
setup_celery_sqlcommenter,
|
||||
)
|
||||
|
||||
|
||||
class _DisconnectableSignal(Protocol):
|
||||
"""Celery's signal stubs omit the public disconnect method."""
|
||||
|
||||
def disconnect(self, receiver: Callable[..., None]) -> bool: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Invocation:
|
||||
task_object: int
|
||||
request: Context
|
||||
span_context: trace.SpanContext
|
||||
tags: CelerySqlcommenterTagsDict
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def celery_task() -> Iterator[Task[..., object]]:
|
||||
app = Celery("sqlcommenter-unit-test", broker="memory://", set_as_current=False)
|
||||
|
||||
@app.task(name=f"sqlcommenter.{uuid4()}")
|
||||
def unused() -> None:
|
||||
pass
|
||||
|
||||
task = app.tasks[unused.name]
|
||||
task.push_request(id=str(uuid4()), retries=0, delivery_info={})
|
||||
try:
|
||||
yield task
|
||||
finally:
|
||||
task.pop_request()
|
||||
app.close()
|
||||
|
||||
|
||||
def test_build_tags_includes_celery_framework_and_task_name(celery_task: Task[..., object]) -> None:
|
||||
tags = _build_celery_sqlcommenter_tags(celery_task)
|
||||
|
||||
assert tags["framework"].startswith("celery:")
|
||||
assert tags["task_name"] == celery_task.name
|
||||
|
||||
|
||||
@pytest.mark.parametrize("retries", [0, 3])
|
||||
def test_build_tags_includes_only_nonzero_retries(celery_task: Task[..., object], retries: int) -> None:
|
||||
celery_task.request.retries = retries
|
||||
|
||||
tags = _build_celery_sqlcommenter_tags(celery_task)
|
||||
|
||||
if retries:
|
||||
assert tags["celery_retries"] == retries
|
||||
else:
|
||||
assert "celery_retries" not in tags
|
||||
|
||||
|
||||
def test_build_tags_includes_request_routing_key(celery_task: Task[..., object]) -> None:
|
||||
celery_task.request.delivery_info = {"routing_key": "workflow_based_app_execution"}
|
||||
|
||||
assert _build_celery_sqlcommenter_tags(celery_task)["routing_key"] == "workflow_based_app_execution"
|
||||
|
||||
|
||||
def test_build_tags_includes_current_traceparent(celery_task: Task[..., object]) -> None:
|
||||
span_context = trace.SpanContext(trace_id=0x123, span_id=0x456, is_remote=True, trace_flags=trace.TraceFlags(1))
|
||||
span = trace.NonRecordingSpan(span_context)
|
||||
|
||||
with trace.use_span(span):
|
||||
tags = _build_celery_sqlcommenter_tags(celery_task)
|
||||
|
||||
assert tags["traceparent"] == f"00-{span_context.trace_id:032x}-{span_context.span_id:016x}-01"
|
||||
|
||||
|
||||
def test_handlers_restore_context_and_remove_request_token(celery_task: Task[..., object]) -> None:
|
||||
previous_context = context.get_current()
|
||||
|
||||
_on_task_prerun(task=celery_task)
|
||||
assert hasattr(celery_task.request, _TOKEN_ATTR)
|
||||
assert not hasattr(celery_task, _TOKEN_ATTR)
|
||||
tags = context.get_value(_SQLCOMMENTER_CONTEXT_KEY)
|
||||
assert isinstance(tags, dict)
|
||||
assert tags["task_name"] == celery_task.name
|
||||
|
||||
_on_task_postrun(task=celery_task)
|
||||
_on_task_postrun(task=celery_task)
|
||||
|
||||
assert context.get_current() is previous_context
|
||||
assert not hasattr(celery_task.request, _TOKEN_ATTR)
|
||||
|
||||
|
||||
def test_handlers_skip_missing_task_or_request_token(celery_task: Task[..., object]) -> None:
|
||||
previous_context = context.get_current()
|
||||
|
||||
_on_task_prerun()
|
||||
_on_task_postrun()
|
||||
_on_task_postrun(task=celery_task)
|
||||
|
||||
assert context.get_current() is previous_context
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sampled", [False, True])
|
||||
@pytest.mark.parametrize("outcome", ["success", "failure", "retry", "cancel"])
|
||||
@pytest.mark.parametrize("finish_order", [(0, 1), (1, 0)])
|
||||
def test_overlapping_task_invocations_restore_otel_parent(
|
||||
sampled: bool,
|
||||
outcome: str,
|
||||
finish_order: tuple[int, int],
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
"""Real eager task tracing includes signals, exception and retry handling.
|
||||
|
||||
Two threads execute the same registered Task singleton. Each Celery request
|
||||
has its own token and incoming parent. Cancellation models the Workflow's
|
||||
cooperative stop returning through Celery postrun, not killing a process.
|
||||
"""
|
||||
provider = TracerProvider(sampler=ALWAYS_ON if sampled else ALWAYS_OFF)
|
||||
exporter = InMemorySpanExporter()
|
||||
provider.add_span_processor(SimpleSpanProcessor(exporter))
|
||||
meter_provider = MeterProvider()
|
||||
setup_celery_sqlcommenter(tracer_provider=provider, meter_provider=meter_provider)
|
||||
app = Celery("sqlcommenter-concurrency-test", broker="memory://", set_as_current=False)
|
||||
started = [Event(), Event()]
|
||||
release = [Event(), Event()]
|
||||
attempts: dict[int, list[_Invocation]] = {0: [], 1: []}
|
||||
restored: dict[int, bool] = {}
|
||||
parents: dict[int, trace.SpanContext] = {}
|
||||
name = f"sqlcommenter.concurrent.{uuid4()}"
|
||||
|
||||
@app.task(bind=True, name=name, max_retries=1)
|
||||
def execute_task(task: Task[..., object], marker: int) -> dict[str, str]:
|
||||
span = trace.get_current_span()
|
||||
attempts[marker].append(
|
||||
_Invocation(
|
||||
task_object=id(task),
|
||||
request=task.request,
|
||||
span_context=span.get_span_context(),
|
||||
tags=cast(CelerySqlcommenterTagsDict, context.get_value(_SQLCOMMENTER_CONTEXT_KEY)),
|
||||
)
|
||||
)
|
||||
if task.request.retries == 0:
|
||||
started[marker].set()
|
||||
assert release[marker].wait(timeout=10), "concurrent invocation was never released"
|
||||
if outcome == "failure":
|
||||
raise ValueError(f"synthetic failure {marker}")
|
||||
if outcome == "retry" and task.request.retries == 0:
|
||||
raise task.retry(countdown=0)
|
||||
if outcome == "cancel":
|
||||
try:
|
||||
raise GenerateTaskStoppedError("synthetic cooperative stop")
|
||||
except GenerateTaskStoppedError:
|
||||
return {"status": "stopped"}
|
||||
return {"status": "succeeded"}
|
||||
|
||||
task = app.tasks[name]
|
||||
|
||||
def invoke(marker: int) -> EagerResult[object]:
|
||||
parent = provider.get_tracer(__name__).start_span(f"caller-{marker}")
|
||||
parents[marker] = parent.get_span_context()
|
||||
baseline = context.set_value(_SQLCOMMENTER_CONTEXT_KEY, {"caller": marker}, trace.set_span_in_context(parent))
|
||||
token = context.attach(baseline)
|
||||
try:
|
||||
result = task.apply(
|
||||
kwargs={"marker": marker}, task_id=str(uuid4()), throw=False, routing_key=f"route-{marker}"
|
||||
)
|
||||
restored[marker] = context.get_current() is baseline and trace.get_current_span() is parent
|
||||
assert context.get_value(_SQLCOMMENTER_CONTEXT_KEY) == {"caller": marker}
|
||||
return result
|
||||
finally:
|
||||
context.detach(token)
|
||||
parent.end()
|
||||
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
futures = [executor.submit(invoke, 0)]
|
||||
try:
|
||||
assert started[0].wait(timeout=10)
|
||||
futures.append(executor.submit(invoke, 1))
|
||||
assert started[1].wait(timeout=10)
|
||||
for marker in finish_order:
|
||||
release[marker].set()
|
||||
result = futures[marker].result(timeout=10)
|
||||
if outcome == "failure":
|
||||
assert result.state == "FAILURE"
|
||||
assert isinstance(result.result, ValueError)
|
||||
assert str(result.result) == f"synthetic failure {marker}"
|
||||
else:
|
||||
assert result.state == "SUCCESS"
|
||||
assert result.result == {"status": "stopped" if outcome == "cancel" else "succeeded"}
|
||||
finally:
|
||||
for event in release:
|
||||
event.set()
|
||||
|
||||
assert restored == {0: True, 1: True}
|
||||
assert not hasattr(task, _TOKEN_ATTR)
|
||||
assert parents[0].trace_id != parents[1].trace_id
|
||||
expected_attempts = 2 if outcome == "retry" else 1
|
||||
for marker, invocations in attempts.items():
|
||||
assert len(invocations) == expected_attempts
|
||||
for index, invocation in enumerate(invocations):
|
||||
span_context = invocation.span_context
|
||||
tags = invocation.tags
|
||||
assert invocation.task_object == id(task)
|
||||
assert not hasattr(invocation.request, _TOKEN_ATTR)
|
||||
assert span_context.trace_id == parents[marker].trace_id
|
||||
assert span_context.trace_flags.sampled is sampled
|
||||
assert tags["routing_key"] == f"route-{marker}"
|
||||
assert tags["traceparent"] == (
|
||||
f"00-{span_context.trace_id:032x}-{span_context.span_id:016x}-{int(span_context.trace_flags):02x}"
|
||||
)
|
||||
if index:
|
||||
assert tags["celery_retries"] == index
|
||||
else:
|
||||
assert "celery_retries" not in tags
|
||||
task_spans = [
|
||||
span
|
||||
for span in exporter.get_finished_spans()
|
||||
if span.attributes is not None and span.attributes.get("celery.task_name") == name
|
||||
]
|
||||
assert len(task_spans) == (2 * expected_attempts if sampled else 0)
|
||||
for span in task_spans:
|
||||
assert span.parent is not None
|
||||
assert span.context is not None
|
||||
parent = next(parent for parent in parents.values() if parent.trace_id == span.context.trace_id)
|
||||
assert span.parent.span_id == parent.span_id
|
||||
assert "Failed to detach context" not in caplog.text
|
||||
finally:
|
||||
cast(_DisconnectableSignal, task_prerun).disconnect(_on_task_prerun)
|
||||
cast(_DisconnectableSignal, task_postrun).disconnect(_on_task_postrun)
|
||||
CeleryInstrumentor().uninstrument()
|
||||
app.close()
|
||||
provider.shutdown()
|
||||
meter_provider.shutdown()
|
||||
|
||||
|
||||
def test_external_instrumentation_keeps_its_provider_and_context(caplog: pytest.LogCaptureFixture) -> None:
|
||||
"""An existing instrumentor has no public signal-ordering API to nest under."""
|
||||
external_provider = TracerProvider()
|
||||
external_exporter = InMemorySpanExporter()
|
||||
external_provider.add_span_processor(SimpleSpanProcessor(external_exporter))
|
||||
unused_provider = TracerProvider()
|
||||
unused_exporter = InMemorySpanExporter()
|
||||
unused_provider.add_span_processor(SimpleSpanProcessor(unused_exporter))
|
||||
meter_provider = MeterProvider()
|
||||
instrumentor = CeleryInstrumentor()
|
||||
instrumentor.instrument(tracer_provider=external_provider, meter_provider=meter_provider)
|
||||
setup_celery_sqlcommenter(tracer_provider=unused_provider, meter_provider=meter_provider)
|
||||
app = Celery("sqlcommenter-external-instrumentation", broker="memory://", set_as_current=False)
|
||||
baseline_tags = {"outer": "external-owner"}
|
||||
|
||||
@app.task(name=f"sqlcommenter.external.{uuid4()}")
|
||||
def execute_task() -> None:
|
||||
assert context.get_value(_SQLCOMMENTER_CONTEXT_KEY) == baseline_tags
|
||||
assert not hasattr(app.tasks[execute_task.name].request, _TOKEN_ATTR)
|
||||
|
||||
parent = external_provider.get_tracer(__name__).start_span("external-parent")
|
||||
baseline = context.set_value(_SQLCOMMENTER_CONTEXT_KEY, baseline_tags, trace.set_span_in_context(parent))
|
||||
token = context.attach(baseline)
|
||||
try:
|
||||
result = execute_task.apply(throw=True)
|
||||
|
||||
assert result.state == "SUCCESS"
|
||||
assert context.get_current() is baseline
|
||||
assert trace.get_current_span() is parent
|
||||
spans = external_exporter.get_finished_spans()
|
||||
assert len(spans) == 1
|
||||
assert spans[0].parent is not None
|
||||
assert spans[0].parent.span_id == parent.get_span_context().span_id
|
||||
assert not unused_exporter.get_finished_spans()
|
||||
assert "skipping SQL commenter signal registration" in caplog.text
|
||||
assert "Failed to detach context" not in caplog.text
|
||||
finally:
|
||||
context.detach(token)
|
||||
parent.end()
|
||||
instrumentor.uninstrument()
|
||||
app.close()
|
||||
external_provider.shutdown()
|
||||
unused_provider.shutdown()
|
||||
meter_provider.shutdown()
|
||||
@@ -0,0 +1,51 @@
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import pytest
|
||||
from opentelemetry import context as otel_context
|
||||
|
||||
from extensions.otel.context import propagate_context
|
||||
|
||||
|
||||
def test_propagate_context_captures_context_when_wrapped() -> None:
|
||||
context_key = otel_context.create_key("test-context")
|
||||
captured_context = otel_context.set_value(context_key, "captured")
|
||||
|
||||
token = otel_context.attach(captured_context)
|
||||
try:
|
||||
wrapped = propagate_context(lambda: otel_context.get_value(context_key))
|
||||
finally:
|
||||
otel_context.detach(token)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
assert executor.submit(wrapped).result() == "captured"
|
||||
|
||||
|
||||
def test_propagate_context_detaches_context_after_exception() -> None:
|
||||
context_key = otel_context.create_key("test-context")
|
||||
captured_context = otel_context.set_value(context_key, "captured")
|
||||
|
||||
def raise_error() -> None:
|
||||
raise RuntimeError("retrieval failed")
|
||||
|
||||
token = otel_context.attach(captured_context)
|
||||
try:
|
||||
wrapped = propagate_context(raise_error)
|
||||
finally:
|
||||
otel_context.detach(token)
|
||||
|
||||
def invoke_and_read_context() -> object:
|
||||
with pytest.raises(RuntimeError, match="retrieval failed"):
|
||||
wrapped()
|
||||
return otel_context.get_value(context_key)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
assert executor.submit(invoke_and_read_context).result() is None
|
||||
|
||||
|
||||
def test_propagate_context_preserves_function_metadata() -> None:
|
||||
def retrieve() -> None:
|
||||
pass
|
||||
|
||||
wrapped = propagate_context(retrieve)
|
||||
|
||||
assert wrapped.__name__ == "retrieve"
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Exercise collaboration metric startup through the existing OTel composition root."""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from textwrap import dedent
|
||||
|
||||
|
||||
def test_init_instruments_registers_app_owned_collaboration_metrics_before_events() -> None:
|
||||
"""Real instrumentors bind one app-owned instance and collect zero gauges before traffic.
|
||||
|
||||
A child interpreter isolates gevent patching, SDK providers, and global instrumentors
|
||||
from other tests. SQLite supplies a real SQLAlchemy engine without external services.
|
||||
"""
|
||||
script = dedent(
|
||||
"""
|
||||
from gevent import monkey
|
||||
monkey.patch_all()
|
||||
|
||||
from flask_sqlalchemy import SQLAlchemy
|
||||
from opentelemetry.metrics import set_meter_provider
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import Gauge, InMemoryMetricReader
|
||||
|
||||
from dify_app import DifyApp
|
||||
from extensions.otel.instrumentation import init_instruments
|
||||
from extensions.otel.workflow_collaboration import WorkflowCollaborationMetrics
|
||||
|
||||
app = DifyApp(__name__)
|
||||
app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///:memory:"
|
||||
database = SQLAlchemy(app)
|
||||
reader = InMemoryMetricReader()
|
||||
provider = MeterProvider(metric_readers=[reader])
|
||||
set_meter_provider(provider)
|
||||
try:
|
||||
init_instruments(app)
|
||||
metrics = app.extensions["workflow_collaboration_metrics"]
|
||||
assert isinstance(metrics, WorkflowCollaborationMetrics)
|
||||
assert [
|
||||
extension for extension in app.extensions.values()
|
||||
if isinstance(extension, WorkflowCollaborationMetrics)
|
||||
] == [metrics]
|
||||
|
||||
data = reader.get_metrics_data()
|
||||
assert data is not None
|
||||
exported = {
|
||||
metric.name: metric
|
||||
for resource in data.resource_metrics
|
||||
for scope in resource.scope_metrics
|
||||
for metric in scope.metrics
|
||||
if metric.name.startswith("dify.collaboration.")
|
||||
}
|
||||
assert set(exported) == {
|
||||
"dify.collaboration.connections",
|
||||
"dify.collaboration.rooms",
|
||||
"dify.collaboration.room_sessions",
|
||||
}
|
||||
for name in ("connections", "rooms"):
|
||||
gauge = exported[f"dify.collaboration.{name}"].data
|
||||
assert isinstance(gauge, Gauge)
|
||||
assert len(gauge.data_points) == 1
|
||||
assert gauge.data_points[0].value == 0
|
||||
assert gauge.data_points[0].attributes == {"aggregation_scope": "worker"}
|
||||
distribution = exported["dify.collaboration.room_sessions"].data
|
||||
assert isinstance(distribution, Gauge)
|
||||
assert len(distribution.data_points) == 5
|
||||
assert {point.attributes["size_range"]: point.value for point in distribution.data_points} == {
|
||||
"1": 0, "2-5": 0, "6-10": 0, "11-20": 0, "21+": 0
|
||||
}
|
||||
finally:
|
||||
provider.shutdown()
|
||||
with app.app_context():
|
||||
database.engine.dispose()
|
||||
"""
|
||||
)
|
||||
completed = subprocess.run(
|
||||
[sys.executable, "-B", "-c", script],
|
||||
cwd=Path(__file__).resolve().parents[4],
|
||||
env={**os.environ, "DEBUG": "false", "ENABLE_OTEL": "true"},
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
check=False,
|
||||
)
|
||||
assert completed.returncode == 0, completed.stdout + completed.stderr
|
||||
@@ -0,0 +1,84 @@
|
||||
from collections.abc import Iterator
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
from opentelemetry import trace
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
|
||||
from core.logging.context import clear_request_context
|
||||
from models import Account
|
||||
from tests.helpers.unit.config_override import config_overrides_context
|
||||
|
||||
|
||||
def _user() -> Account:
|
||||
user = Account(name="Test User", email="otel@example.com")
|
||||
user.id = "user-id"
|
||||
return user
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_logging_context() -> Iterator[None]:
|
||||
clear_request_context()
|
||||
yield
|
||||
clear_request_context()
|
||||
|
||||
|
||||
def test_on_user_loaded_does_not_write_to_non_recording_span() -> None:
|
||||
from extensions.otel import runtime
|
||||
|
||||
span = mock.MagicMock()
|
||||
span.is_recording.return_value = False
|
||||
user = _user()
|
||||
|
||||
with (
|
||||
config_overrides_context(ENABLE_OTEL=True),
|
||||
mock.patch("opentelemetry.trace.get_current_span", return_value=span),
|
||||
mock.patch.object(runtime, "extract_tenant_id", return_value="tenant-id"),
|
||||
):
|
||||
runtime.on_user_loaded(None, user)
|
||||
|
||||
span.is_recording.assert_called_once_with()
|
||||
span.set_attribute.assert_not_called()
|
||||
span.set_attributes.assert_not_called()
|
||||
|
||||
|
||||
def test_on_user_loaded_sets_attributes_on_recording_span() -> None:
|
||||
from extensions.otel import runtime
|
||||
from extensions.otel.semconv import DifySpanAttributes, GenAIAttributes
|
||||
|
||||
span = mock.MagicMock()
|
||||
span.is_recording.return_value = True
|
||||
user = _user()
|
||||
|
||||
with (
|
||||
config_overrides_context(ENABLE_OTEL=True),
|
||||
mock.patch("opentelemetry.trace.get_current_span", return_value=span),
|
||||
mock.patch.object(runtime, "extract_tenant_id", return_value="tenant-id"),
|
||||
):
|
||||
runtime.on_user_loaded(None, user)
|
||||
|
||||
span.set_attributes.assert_called_once_with(
|
||||
{
|
||||
DifySpanAttributes.TENANT_ID: "tenant-id",
|
||||
GenAIAttributes.USER_ID: "user-id",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_on_user_loaded_ignores_ended_sdk_span(caplog: pytest.LogCaptureFixture) -> None:
|
||||
from extensions.otel import runtime
|
||||
|
||||
tracer_provider = TracerProvider()
|
||||
span = tracer_provider.get_tracer(__name__).start_span("ended")
|
||||
span.end()
|
||||
user = _user()
|
||||
|
||||
with (
|
||||
trace.use_span(span, end_on_exit=False),
|
||||
config_overrides_context(ENABLE_OTEL=True),
|
||||
mock.patch.object(runtime, "extract_tenant_id", return_value="tenant-id"),
|
||||
caplog.at_level("WARNING", logger="opentelemetry.sdk.trace"),
|
||||
):
|
||||
runtime.on_user_loaded(None, user)
|
||||
|
||||
assert "Setting attribute on ended span" not in caplog.text
|
||||
@@ -0,0 +1,605 @@
|
||||
"""Exercise worker-local collaboration metrics with real SDK collection and room membership."""
|
||||
|
||||
from collections.abc import Callable, Iterator
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Barrier, get_ident
|
||||
from typing import Never
|
||||
|
||||
import pytest
|
||||
import socketio
|
||||
from flask import Flask
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import Gauge, Histogram, InMemoryMetricReader, Metric, Sum
|
||||
|
||||
from extensions.otel import workflow_collaboration as instrumentation
|
||||
|
||||
SIZE_RANGES = ("1", "2-5", "6-10", "11-20", "21+")
|
||||
type ObservedMetrics = tuple[instrumentation.WorkflowCollaborationMetrics, InMemoryMetricReader]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def metrics_app() -> Flask:
|
||||
"""Create a real app with isolated extension ownership and no external services."""
|
||||
return Flask(__name__)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def observed_metrics(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
config_overrides: Callable[..., None],
|
||||
metrics_app: Flask,
|
||||
) -> Iterator[ObservedMetrics]:
|
||||
"""Initialize app-owned instruments with a real SDK meter and isolate global configuration."""
|
||||
config_overrides(ENABLE_OTEL=True)
|
||||
reader = InMemoryMetricReader()
|
||||
provider = MeterProvider(metric_readers=[reader])
|
||||
monkeypatch.setattr(instrumentation, "get_meter", provider.get_meter)
|
||||
instrumentation.init_workflow_collaboration_metrics(metrics_app)
|
||||
metrics = metrics_app.extensions["workflow_collaboration_metrics"]
|
||||
assert isinstance(metrics, instrumentation.WorkflowCollaborationMetrics)
|
||||
yield metrics, reader
|
||||
provider.shutdown()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def socket_server(metrics_app: Flask, monkeypatch: pytest.MonkeyPatch) -> socketio.Server:
|
||||
"""Attach the real app to a network-free server as the application factory does."""
|
||||
server = socketio.Server(async_mode="threading")
|
||||
monkeypatch.setattr(server, "app", metrics_app, raising=False)
|
||||
return server
|
||||
|
||||
|
||||
def _metrics(reader: InMemoryMetricReader) -> dict[str, Metric]:
|
||||
"""Collect actual SDK data, requiring a successful observable callback collection."""
|
||||
data = reader.get_metrics_data()
|
||||
assert data is not None
|
||||
return {
|
||||
metric.name: metric
|
||||
for resource in data.resource_metrics
|
||||
for scope in resource.scope_metrics
|
||||
for metric in scope.metrics
|
||||
}
|
||||
|
||||
|
||||
def _assert_population(reader: InMemoryMetricReader, connections: int, rooms: int, sizes: dict[str, int]) -> None:
|
||||
"""Require exact worker-scope labels and all fixed buckets, including zeroes."""
|
||||
collected = _metrics(reader)
|
||||
for name, expected in (("connections", connections), ("rooms", rooms)):
|
||||
data = collected[f"dify.collaboration.{name}"].data
|
||||
assert isinstance(data, Gauge)
|
||||
assert len(data.data_points) == 1
|
||||
point = data.data_points[0]
|
||||
assert point.attributes == {"aggregation_scope": "worker"}
|
||||
assert point.value == expected
|
||||
|
||||
distribution = collected["dify.collaboration.room_sessions"].data
|
||||
assert isinstance(distribution, Gauge)
|
||||
assert len(distribution.data_points) == len(SIZE_RANGES)
|
||||
actual: dict[str, int | float] = {}
|
||||
for point in distribution.data_points:
|
||||
assert point.attributes is not None
|
||||
assert set(point.attributes) == {"aggregation_scope", "size_range"}
|
||||
assert point.attributes["aggregation_scope"] == "worker"
|
||||
size_range = point.attributes["size_range"]
|
||||
assert isinstance(size_range, str)
|
||||
actual[size_range] = point.value
|
||||
assert actual == {size_range: sizes.get(size_range, 0) for size_range in SIZE_RANGES}
|
||||
|
||||
|
||||
def _assert_event(reader: InMemoryMetricReader, event: str, event_type: str, outcome: str) -> None:
|
||||
"""Require one matching counter and duration sample without payload-derived labels."""
|
||||
collected = _metrics(reader)
|
||||
expected = {"event": event, "event_type": event_type, "outcome": outcome}
|
||||
events = collected["dify.collaboration.events"].data
|
||||
durations = collected["dify.collaboration.event_duration"].data
|
||||
assert isinstance(events, Sum)
|
||||
assert isinstance(durations, Histogram)
|
||||
assert len(events.data_points) == len(durations.data_points) == 1
|
||||
assert events.data_points[0].attributes == expected
|
||||
assert events.data_points[0].value == 1
|
||||
assert durations.data_points[0].attributes == expected
|
||||
assert durations.data_points[0].count == 1
|
||||
assert durations.data_points[0].sum >= 0
|
||||
|
||||
|
||||
def test_empty_worker_emits_all_zero_gauges(observed_metrics: ObservedMetrics) -> None:
|
||||
"""An unused worker still exports zero-valued populations and every size bucket."""
|
||||
_, reader = observed_metrics
|
||||
_assert_population(reader, connections=0, rooms=0, sizes={})
|
||||
|
||||
|
||||
def test_duplicate_connections_and_joins_do_not_inflate_population(observed_metrics: ObservedMetrics) -> None:
|
||||
"""Repeated lifecycle notifications preserve unique local SID and room membership."""
|
||||
metrics, reader = observed_metrics
|
||||
metrics.connected("sid-a")
|
||||
metrics.connected("sid-a")
|
||||
metrics.update_rooms("sid-a", ["sid-a", "workflow-a", "workflow-a"])
|
||||
metrics.update_rooms("sid-a", ["sid-a", "workflow-a"])
|
||||
_assert_population(reader, connections=1, rooms=1, sizes={"1": 1})
|
||||
|
||||
metrics.connected("sid-b")
|
||||
metrics.update_rooms("sid-b", ["sid-b", "workflow-a"])
|
||||
_assert_population(reader, connections=2, rooms=1, sizes={"2-5": 1})
|
||||
|
||||
|
||||
def test_room_snapshot_replaces_membership_and_disconnect_clears_it(observed_metrics: ObservedMetrics) -> None:
|
||||
"""Multiple actual rooms are counted until removed; late joins cannot resurrect a SID."""
|
||||
metrics, reader = observed_metrics
|
||||
metrics.connected("sid-a")
|
||||
metrics.connected("sid-b")
|
||||
metrics.update_rooms("sid-a", ["sid-a", "workflow-a", "workflow-b"])
|
||||
metrics.update_rooms("sid-b", ["sid-b", "workflow-a"])
|
||||
_assert_population(reader, connections=2, rooms=2, sizes={"1": 1, "2-5": 1})
|
||||
|
||||
metrics.update_rooms("sid-a", ["sid-a", "workflow-b"])
|
||||
_assert_population(reader, connections=2, rooms=2, sizes={"1": 2})
|
||||
metrics.disconnected("sid-a")
|
||||
metrics.disconnected("sid-a")
|
||||
metrics.update_rooms("sid-a", ["workflow-b"])
|
||||
_assert_population(reader, connections=1, rooms=1, sizes={"1": 1})
|
||||
|
||||
metrics.disconnected("sid-b")
|
||||
metrics.update_rooms("unknown-sid", ["new-workflow"])
|
||||
_assert_population(reader, connections=0, rooms=0, sizes={})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("members", "before", "after"),
|
||||
[
|
||||
(1, "1", None),
|
||||
(2, "2-5", "1"),
|
||||
(5, "2-5", "2-5"),
|
||||
(6, "6-10", "2-5"),
|
||||
(10, "6-10", "6-10"),
|
||||
(11, "11-20", "6-10"),
|
||||
(20, "11-20", "11-20"),
|
||||
(21, "21+", "11-20"),
|
||||
],
|
||||
)
|
||||
def test_room_distribution_crosses_fixed_bucket_boundaries(
|
||||
observed_metrics: ObservedMetrics, members: int, before: str, after: str | None
|
||||
) -> None:
|
||||
"""Room-size buckets describe current rooms and move immediately on member departure."""
|
||||
metrics, reader = observed_metrics
|
||||
for index in range(members):
|
||||
sid = f"sid-{index}"
|
||||
metrics.connected(sid)
|
||||
metrics.update_rooms(sid, [sid, "workflow-a"])
|
||||
_assert_population(reader, connections=members, rooms=1, sizes={before: 1})
|
||||
|
||||
metrics.disconnected("sid-0")
|
||||
_assert_population(reader, connections=members - 1, rooms=int(members > 1), sizes={after: 1} if after else {})
|
||||
|
||||
|
||||
def test_record_event_exports_seconds_and_matching_counter(observed_metrics: ObservedMetrics) -> None:
|
||||
"""Counter and histogram retain the same bounded dimensions and seconds unit."""
|
||||
metrics, reader = observed_metrics
|
||||
metrics.record_event("graph_event", "none", "success", 0.025)
|
||||
_assert_event(reader, "graph_event", "none", "success")
|
||||
duration = _metrics(reader)["dify.collaboration.event_duration"]
|
||||
assert duration.unit == "s"
|
||||
assert isinstance(duration.data, Histogram)
|
||||
assert duration.data.data_points[0].sum == pytest.approx(0.025)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("result", "outcome"),
|
||||
[
|
||||
(None, "success"),
|
||||
(True, "success"),
|
||||
(False, "rejected"),
|
||||
(0, "success"),
|
||||
({"status": 500}, "success"),
|
||||
(({"msg": "ok"}, 200), "success"),
|
||||
(({"msg": "denied"}, 403), "rejected"),
|
||||
(({"msg": "failed"}, 503), "error"),
|
||||
(("ack", "opaque"), "success"),
|
||||
],
|
||||
)
|
||||
def test_event_outcome_preserves_exact_handler_return(
|
||||
observed_metrics: ObservedMetrics, socket_server: socketio.Server, result: object, outcome: str
|
||||
) -> None:
|
||||
"""Only explicit rejection or HTTP-like status tuples classify a normal return as failure."""
|
||||
assert isinstance(socket_server, instrumentation._SocketIO)
|
||||
_, reader = observed_metrics
|
||||
|
||||
@instrumentation.observe_workflow_event("graph_event", socket_server)
|
||||
def handler(_sid: str, _data: object) -> object:
|
||||
return result
|
||||
|
||||
assert handler("sid-a", b"graph-data") is result
|
||||
assert handler.__name__ == "handler"
|
||||
_assert_event(reader, "graph_event", "none", outcome)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("payload", "expected_type"),
|
||||
[
|
||||
({"type": "mouse_move"}, "mouse_move"),
|
||||
({"type": "workflow_update"}, "workflow_update"),
|
||||
({"type": "untrusted-user-specific-event"}, "other"),
|
||||
({}, "other"),
|
||||
({"type": ["mouse_move"]}, "other"),
|
||||
({"type": None}, "other"),
|
||||
(None, "other"),
|
||||
(b"data", "other"),
|
||||
],
|
||||
)
|
||||
def test_event_types_are_bounded_without_changing_payload(
|
||||
observed_metrics: ObservedMetrics, socket_server: socketio.Server, payload: object, expected_type: str
|
||||
) -> None:
|
||||
"""Unknown and malformed payloads stay handler-owned and cannot add metric series."""
|
||||
assert isinstance(socket_server, instrumentation._SocketIO)
|
||||
_, reader = observed_metrics
|
||||
|
||||
@instrumentation.observe_workflow_event("collaboration_event", socket_server)
|
||||
def handler(_sid: str, data: object) -> object:
|
||||
return data
|
||||
|
||||
assert handler("sid-a", payload) is payload
|
||||
_assert_event(reader, "collaboration_event", expected_type, "success")
|
||||
|
||||
|
||||
def test_event_exception_is_recorded_and_reraised_unchanged(
|
||||
observed_metrics: ObservedMetrics, socket_server: socketio.Server
|
||||
) -> None:
|
||||
"""Instrumentation preserves the original exception instance while reporting failure."""
|
||||
assert isinstance(socket_server, instrumentation._SocketIO)
|
||||
_, reader = observed_metrics
|
||||
failure = ValueError("handler failed")
|
||||
|
||||
@instrumentation.observe_workflow_event("collaboration_event", socket_server)
|
||||
def handler(_sid: str, _data: object) -> Never:
|
||||
raise failure
|
||||
|
||||
with pytest.raises(ValueError, match="handler failed") as caught:
|
||||
handler("sid-a", {"type": "mouse_move"})
|
||||
assert caught.value is failure
|
||||
_assert_event(reader, "collaboration_event", "mouse_move", "error")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("accepted", [True, False])
|
||||
def test_connect_counts_only_authenticated_sockets(
|
||||
observed_metrics: ObservedMetrics, socket_server: socketio.Server, accepted: bool
|
||||
) -> None:
|
||||
"""A provisional Socket.IO room exists before authentication, but rejected SIDs do not count."""
|
||||
assert isinstance(socket_server, instrumentation._SocketIO)
|
||||
assert isinstance(socket_server.manager, socketio.Manager)
|
||||
_, reader = observed_metrics
|
||||
sid = socket_server.manager.connect("engine-sid", "/")
|
||||
assert isinstance(sid, str)
|
||||
|
||||
@instrumentation.observe_workflow_event("connect", socket_server)
|
||||
def handler(_sid: str, _environ: object, _auth: object) -> bool:
|
||||
return accepted
|
||||
|
||||
assert handler(sid, {}, None) is accepted
|
||||
_assert_population(reader, connections=int(accepted), rooms=0, sizes={})
|
||||
_assert_event(reader, "connect", "none", "success" if accepted else "rejected")
|
||||
|
||||
|
||||
def test_connect_completed_after_disconnect_does_not_restore_socket(
|
||||
observed_metrics: ObservedMetrics, socket_server: socketio.Server
|
||||
) -> None:
|
||||
"""A successful handler cannot reintroduce a SID that disconnected during authentication."""
|
||||
assert isinstance(socket_server, instrumentation._SocketIO)
|
||||
assert isinstance(socket_server.manager, socketio.Manager)
|
||||
_, reader = observed_metrics
|
||||
sid = socket_server.manager.connect("engine-sid", "/")
|
||||
assert isinstance(sid, str)
|
||||
|
||||
@instrumentation.observe_workflow_event("connect", socket_server)
|
||||
def handler(sid: str, _environ: object, _auth: object) -> bool:
|
||||
socket_server.manager.disconnect(sid, "/")
|
||||
return True
|
||||
|
||||
assert handler(sid, {}, None) is True
|
||||
_assert_population(reader, connections=0, rooms=0, sizes={})
|
||||
_assert_event(reader, "connect", "none", "success")
|
||||
|
||||
|
||||
def test_partial_join_failure_reflects_real_membership(
|
||||
observed_metrics: ObservedMetrics, socket_server: socketio.Server
|
||||
) -> None:
|
||||
"""An exception after enter_room retains actual local membership and the original failure."""
|
||||
assert isinstance(socket_server, instrumentation._SocketIO)
|
||||
assert isinstance(socket_server.manager, socketio.Manager)
|
||||
metrics, reader = observed_metrics
|
||||
sid = socket_server.manager.connect("engine-sid", "/")
|
||||
assert isinstance(sid, str)
|
||||
metrics.connected(sid)
|
||||
failure = RuntimeError("broadcast failed after join")
|
||||
|
||||
@instrumentation.observe_workflow_event("user_connect", socket_server)
|
||||
def handler(sid: str, data: dict[str, str]) -> Never:
|
||||
socket_server.enter_room(sid, data["workflow_id"])
|
||||
raise failure
|
||||
|
||||
with pytest.raises(RuntimeError) as caught:
|
||||
handler(sid, {"workflow_id": "workflow-a"})
|
||||
assert caught.value is failure
|
||||
_assert_population(reader, connections=1, rooms=1, sizes={"1": 1})
|
||||
_assert_event(reader, "user_connect", "none", "error")
|
||||
|
||||
|
||||
def test_disconnect_failure_still_removes_all_local_membership(
|
||||
observed_metrics: ObservedMetrics, socket_server: socketio.Server
|
||||
) -> None:
|
||||
"""Population cleanup does not depend on the service's Redis cleanup succeeding."""
|
||||
assert isinstance(socket_server, instrumentation._SocketIO)
|
||||
assert isinstance(socket_server.manager, socketio.Manager)
|
||||
metrics, reader = observed_metrics
|
||||
sid = socket_server.manager.connect("engine-sid", "/")
|
||||
assert isinstance(sid, str)
|
||||
metrics.connected(sid)
|
||||
socket_server.enter_room(sid, "workflow-a")
|
||||
metrics.update_rooms(sid, socket_server.rooms(sid))
|
||||
socket_server.manager.pre_disconnect(sid, "/")
|
||||
failure = RuntimeError("service cleanup failed")
|
||||
|
||||
@instrumentation.observe_workflow_event("disconnect", socket_server)
|
||||
def handler(_sid: str) -> Never:
|
||||
raise failure
|
||||
|
||||
with pytest.raises(RuntimeError) as caught:
|
||||
handler(sid)
|
||||
assert caught.value is failure
|
||||
_assert_population(reader, connections=0, rooms=0, sizes={})
|
||||
_assert_event(reader, "disconnect", "none", "error")
|
||||
|
||||
|
||||
def test_room_lookup_failure_does_not_hide_return_or_event(
|
||||
observed_metrics: ObservedMetrics, socket_server: socketio.Server, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""A metrics-only membership lookup failure cannot change an ACK or suppress event accounting."""
|
||||
assert isinstance(socket_server, instrumentation._SocketIO)
|
||||
assert isinstance(socket_server.manager, socketio.Manager)
|
||||
metrics, reader = observed_metrics
|
||||
sid = socket_server.manager.connect("engine-sid", "/")
|
||||
assert isinstance(sid, str)
|
||||
metrics.connected(sid)
|
||||
result = {"msg": "connected"}
|
||||
|
||||
def unavailable_rooms(*_args: object, **_kwargs: object) -> Never:
|
||||
raise RuntimeError("room lookup failed")
|
||||
|
||||
monkeypatch.setattr(socket_server, "rooms", unavailable_rooms)
|
||||
|
||||
@instrumentation.observe_workflow_event("user_connect", socket_server)
|
||||
def handler(_sid: str, _data: object) -> dict[str, str]:
|
||||
return result
|
||||
|
||||
assert handler(sid, {}) is result
|
||||
_assert_event(reader, "user_connect", "none", "success")
|
||||
|
||||
|
||||
def test_collecting_gauges_never_reads_live_socket_manager(
|
||||
observed_metrics: ObservedMetrics, socket_server: socketio.Server, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""SDK collection consumes the published snapshot even when every room accessor is unavailable."""
|
||||
assert isinstance(socket_server, instrumentation._SocketIO)
|
||||
assert isinstance(socket_server.manager, socketio.Manager)
|
||||
metrics, reader = observed_metrics
|
||||
sid = socket_server.manager.connect("engine-sid", "/")
|
||||
assert isinstance(sid, str)
|
||||
metrics.connected(sid)
|
||||
socket_server.enter_room(sid, "workflow-a")
|
||||
|
||||
@instrumentation.observe_workflow_event("user_connect", socket_server)
|
||||
def handler(_sid: str, _data: object) -> dict[str, str]:
|
||||
return {"msg": "connected"}
|
||||
|
||||
handler(sid, {})
|
||||
|
||||
def forbidden_access(*_args: object, **_kwargs: object) -> Never:
|
||||
raise AssertionError("observable callbacks must not touch live manager state")
|
||||
|
||||
monkeypatch.setattr(socket_server, "rooms", forbidden_access)
|
||||
monkeypatch.setattr(socket_server.manager, "get_rooms", forbidden_access)
|
||||
monkeypatch.setattr(socket_server.manager, "get_participants", forbidden_access)
|
||||
_assert_population(reader, connections=1, rooms=1, sizes={"1": 1})
|
||||
|
||||
|
||||
def test_native_thread_collection_uses_complete_numeric_snapshots(observed_metrics: ObservedMetrics) -> None:
|
||||
"""Native SDK collection tolerates handler-side churn without traversing mutable membership."""
|
||||
metrics, reader = observed_metrics
|
||||
ready = Barrier(2)
|
||||
handler_thread = get_ident()
|
||||
|
||||
def collect() -> int:
|
||||
ready.wait(timeout=5)
|
||||
for _ in range(50):
|
||||
distribution = _metrics(reader)["dify.collaboration.room_sessions"].data
|
||||
assert isinstance(distribution, Gauge)
|
||||
assert len(distribution.data_points) == len(SIZE_RANGES)
|
||||
counts = [point.value for point in distribution.data_points]
|
||||
assert all(count >= 0 for count in counts)
|
||||
assert sum(counts) in (0, 1)
|
||||
return get_ident()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
collection = executor.submit(collect)
|
||||
ready.wait(timeout=5)
|
||||
for index in range(1000):
|
||||
sid = f"sid-{index}"
|
||||
metrics.connected(sid)
|
||||
metrics.update_rooms(sid, [sid, "workflow-a"])
|
||||
metrics.disconnected(sid)
|
||||
assert collection.result(timeout=10) != handler_thread
|
||||
_assert_population(reader, connections=0, rooms=0, sizes={})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("handler_fails", [False, True])
|
||||
def test_sdk_failure_never_changes_handler_result_or_exception(
|
||||
observed_metrics: ObservedMetrics,
|
||||
socket_server: socketio.Server,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
handler_fails: bool,
|
||||
) -> None:
|
||||
"""An instrument failure stays isolated from the successful ACK or original business exception."""
|
||||
assert isinstance(socket_server, instrumentation._SocketIO)
|
||||
metrics, _ = observed_metrics
|
||||
result = {"msg": "received"}
|
||||
handler_failure = ValueError("original handler failure")
|
||||
|
||||
def unavailable_sdk(*_args: object, **_kwargs: object) -> Never:
|
||||
raise RuntimeError("metric SDK failure")
|
||||
|
||||
monkeypatch.setattr(metrics._events, "add", unavailable_sdk)
|
||||
|
||||
@instrumentation.observe_workflow_event("graph_event", socket_server)
|
||||
def handler(_sid: str, _data: object) -> dict[str, str]:
|
||||
if handler_fails:
|
||||
raise handler_failure
|
||||
return result
|
||||
|
||||
if handler_fails:
|
||||
with pytest.raises(ValueError, match="original handler failure") as caught:
|
||||
handler("sid-a", b"graph-data")
|
||||
assert caught.value is handler_failure
|
||||
else:
|
||||
assert handler("sid-a", b"graph-data") is result
|
||||
|
||||
|
||||
def test_startup_instance_is_shared_and_repeated_initialization_preserves_state(
|
||||
observed_metrics: ObservedMetrics,
|
||||
metrics_app: Flask,
|
||||
socket_server: socketio.Server,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Handlers share startup-owned state; repeated initialization neither registers nor resets instruments."""
|
||||
assert isinstance(socket_server, instrumentation._SocketIO)
|
||||
assert isinstance(socket_server.manager, socketio.Manager)
|
||||
metrics, reader = observed_metrics
|
||||
registration_attempts: list[object] = []
|
||||
|
||||
def forbidden_registration(*args: object, **_kwargs: object) -> Never:
|
||||
registration_attempts.extend(args)
|
||||
raise AssertionError("event handling must use the initialized app instance")
|
||||
|
||||
monkeypatch.setattr(instrumentation, "get_meter", forbidden_registration)
|
||||
|
||||
@instrumentation.observe_workflow_event("connect", socket_server)
|
||||
def connected(_sid: str, _environ: object, _auth: object) -> bool:
|
||||
return True
|
||||
|
||||
@instrumentation.observe_workflow_event("user_connect", socket_server)
|
||||
def joined(sid: str, _data: object) -> dict[str, str]:
|
||||
socket_server.enter_room(sid, "workflow-a")
|
||||
return {"msg": "connected"}
|
||||
|
||||
for index in range(2):
|
||||
sid = socket_server.manager.connect(f"engine-{index}", "/")
|
||||
assert isinstance(sid, str)
|
||||
assert connected(sid, {}, None) is True
|
||||
assert joined(sid, {}) == {"msg": "connected"}
|
||||
instrumentation.init_workflow_collaboration_metrics(metrics_app)
|
||||
assert metrics_app.extensions["workflow_collaboration_metrics"] is metrics
|
||||
|
||||
assert registration_attempts == []
|
||||
_assert_population(reader, connections=2, rooms=1, sizes={"2-5": 1})
|
||||
events = _metrics(reader)["dify.collaboration.events"].data
|
||||
assert isinstance(events, Sum)
|
||||
assert len(events.data_points) == 2
|
||||
assert all(point.value == 2 for point in events.data_points)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("initialized", [False, True])
|
||||
@pytest.mark.parametrize("handler_fails", [False, True])
|
||||
def test_disabled_metrics_bypass_all_instrumentation_with_or_without_app_instance(
|
||||
observed_metrics: ObservedMetrics,
|
||||
metrics_app: Flask,
|
||||
socket_server: socketio.Server,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
config_overrides: Callable[..., None],
|
||||
initialized: bool,
|
||||
handler_fails: bool,
|
||||
) -> None:
|
||||
"""Disabled instrumentation preserves business behavior without timing, app lookup, membership, or SDK access."""
|
||||
assert isinstance(socket_server, instrumentation._SocketIO)
|
||||
metrics, reader = observed_metrics
|
||||
if not initialized:
|
||||
metrics_app.extensions.pop("workflow_collaboration_metrics")
|
||||
accesses: list[str] = []
|
||||
result = {"msg": "connected"}
|
||||
failure = ValueError("business failure while metrics disabled")
|
||||
|
||||
def forbidden_access(*_args: object, **_kwargs: object) -> Never:
|
||||
accesses.append("instrumentation")
|
||||
raise AssertionError("disabled metrics must not access instrumentation")
|
||||
|
||||
monkeypatch.setattr(instrumentation, "perf_counter", forbidden_access)
|
||||
monkeypatch.setattr(instrumentation, "get_meter", forbidden_access)
|
||||
monkeypatch.setattr(type(socket_server), "app", property(forbidden_access), raising=False)
|
||||
monkeypatch.setattr(socket_server, "rooms", forbidden_access)
|
||||
monkeypatch.setattr(metrics._events, "add", forbidden_access)
|
||||
|
||||
@instrumentation.observe_workflow_event("user_connect", socket_server)
|
||||
def handler(_sid: str, _data: object) -> dict[str, str]:
|
||||
if handler_fails:
|
||||
raise failure
|
||||
return result
|
||||
|
||||
config_overrides(ENABLE_OTEL=False)
|
||||
if handler_fails:
|
||||
with pytest.raises(ValueError, match="business failure while metrics disabled") as caught:
|
||||
handler("sid-a", {})
|
||||
assert caught.value is failure
|
||||
else:
|
||||
assert handler("sid-a", {}) is result
|
||||
|
||||
assert accesses == []
|
||||
_assert_population(reader, connections=0, rooms=0, sizes={})
|
||||
assert "dify.collaboration.events" not in _metrics(reader)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("initialization_fails", [False, True])
|
||||
@pytest.mark.parametrize("handler_fails", [False, True])
|
||||
def test_enabled_metrics_without_initialized_extension_leave_business_behavior_unchanged(
|
||||
metrics_app: Flask,
|
||||
socket_server: socketio.Server,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
config_overrides: Callable[..., None],
|
||||
initialization_fails: bool,
|
||||
handler_fails: bool,
|
||||
) -> None:
|
||||
"""Missing or failed startup instrumentation never registers lazily, times, or replaces handler results."""
|
||||
assert isinstance(socket_server, instrumentation._SocketIO)
|
||||
config_overrides(ENABLE_OTEL=True)
|
||||
registration_attempts: list[str] = []
|
||||
accesses: list[str] = []
|
||||
result = {"msg": "connected"}
|
||||
failure = ValueError("business failure without metrics instance")
|
||||
|
||||
def unavailable_meter(*_args: object, **_kwargs: object) -> Never:
|
||||
registration_attempts.append("get_meter")
|
||||
raise RuntimeError("SDK initialization failed")
|
||||
|
||||
def forbidden_access(*_args: object, **_kwargs: object) -> Never:
|
||||
accesses.append("handler instrumentation")
|
||||
raise AssertionError("missing instance must not trigger event instrumentation")
|
||||
|
||||
monkeypatch.setattr(instrumentation, "get_meter", unavailable_meter)
|
||||
if initialization_fails:
|
||||
instrumentation.init_workflow_collaboration_metrics(metrics_app)
|
||||
assert "workflow_collaboration_metrics" not in metrics_app.extensions
|
||||
|
||||
monkeypatch.setattr(instrumentation, "perf_counter", forbidden_access)
|
||||
monkeypatch.setattr(socket_server, "rooms", forbidden_access)
|
||||
|
||||
@instrumentation.observe_workflow_event("user_connect", socket_server)
|
||||
def handler(_sid: str, _data: object) -> dict[str, str]:
|
||||
if handler_fails:
|
||||
raise failure
|
||||
return result
|
||||
|
||||
if handler_fails:
|
||||
with pytest.raises(ValueError, match="business failure without metrics instance") as caught:
|
||||
handler("sid-a", {})
|
||||
assert caught.value is failure
|
||||
else:
|
||||
assert handler("sid-a", {}) is result
|
||||
|
||||
assert accesses == []
|
||||
assert registration_attempts == (["get_meter"] if initialization_fails else [])
|
||||
assert "workflow_collaboration_metrics" not in metrics_app.extensions
|
||||
Whitespace-only changes.
+132
@@ -0,0 +1,132 @@
|
||||
"""Unit coverage for ClickZetta configuration and path generation."""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from extensions.storage.clickzetta_volume.clickzetta_volume_storage import (
|
||||
ClickZettaVolumeConfig,
|
||||
ClickZettaVolumeStorage,
|
||||
)
|
||||
|
||||
|
||||
def _table_storage() -> ClickZettaVolumeStorage:
|
||||
config = ClickZettaVolumeConfig(
|
||||
username="test_user",
|
||||
password="test_pass",
|
||||
instance="test_instance",
|
||||
service="uat-api.clickzetta.com",
|
||||
workspace="quick_start",
|
||||
vcluster="default_ap",
|
||||
schema_name="dify",
|
||||
volume_type="table",
|
||||
table_prefix="test_dataset_",
|
||||
)
|
||||
storage = ClickZettaVolumeStorage.__new__(ClickZettaVolumeStorage)
|
||||
storage._config = config
|
||||
return storage
|
||||
|
||||
|
||||
def test_config_validation() -> None:
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match=(
|
||||
"1 validation error for ClickZettaVolumeConfig\\\n Value error, "
|
||||
"CLICKZETTA_VOLUME_USERNAME or CLICKZETTA_USERNAME is required"
|
||||
),
|
||||
):
|
||||
ClickZettaVolumeConfig(username="", password="pass", instance="instance")
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match=(
|
||||
"1 validation error for ClickZettaVolumeConfig\\\n Value error, "
|
||||
"CLICKZETTA_VOLUME_TYPE must be one of: table, user, external"
|
||||
),
|
||||
):
|
||||
ClickZettaVolumeConfig(username="user", password="pass", instance="instance", volume_type="invalid_type")
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match=(
|
||||
"1 validation error for ClickZettaVolumeConfig\\\n Value error, "
|
||||
"CLICKZETTA_VOLUME_NAME is required for external volume type"
|
||||
),
|
||||
):
|
||||
ClickZettaVolumeConfig(
|
||||
username="user",
|
||||
password="pass",
|
||||
instance="instance",
|
||||
volume_type="external",
|
||||
)
|
||||
|
||||
|
||||
def test_volume_path_generation() -> None:
|
||||
storage = _table_storage()
|
||||
|
||||
assert storage._get_volume_path("test.txt", "12345") == "test_dataset_12345/test.txt"
|
||||
assert storage._get_volume_path("12345/test.txt") == "12345/test.txt"
|
||||
|
||||
storage._config.volume_type = "user"
|
||||
assert storage._get_volume_path("test.txt") == "dify_km/test.txt"
|
||||
|
||||
|
||||
def test_sql_prefix_generation() -> None:
|
||||
storage = _table_storage()
|
||||
|
||||
assert storage._get_volume_sql_prefix("12345") == "TABLE VOLUME test_dataset_12345"
|
||||
|
||||
storage._config.volume_type = "user"
|
||||
assert storage._get_volume_sql_prefix() == "USER VOLUME"
|
||||
|
||||
storage._config.volume_type = "external"
|
||||
storage._config.volume_name = "my_external_volume"
|
||||
assert storage._get_volume_sql_prefix() == "VOLUME my_external_volume"
|
||||
|
||||
|
||||
def test_scan_table_prefix_matches_file_storage_path(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
storage = _table_storage()
|
||||
execute = MagicMock(return_value=[("app/trace.json",), ("another-app/trace.json",)])
|
||||
monkeypatch.setattr(storage, "_execute_sql", execute)
|
||||
assert storage.scan("ops_trace/app/") == ["ops_trace/app/trace.json"]
|
||||
execute.assert_called_once_with("LIST TABLE VOLUME test_dataset_ops_trace SUBDIRECTORY 'app/'", fetch=True)
|
||||
|
||||
|
||||
def test_scan_special_table_prefix_restores_logical_root(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
storage = _table_storage()
|
||||
execute = MagicMock(return_value=[("dify_km/tenant/file.txt",)])
|
||||
monkeypatch.setattr(storage, "_execute_sql", execute)
|
||||
assert storage.scan("upload_files/tenant/") == ["upload_files/tenant/file.txt"]
|
||||
execute.assert_called_once_with("LIST USER VOLUME SUBDIRECTORY 'dify_km/tenant/'", fetch=True)
|
||||
|
||||
|
||||
def test_scan_preserves_bare_dataset_root_behavior(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
storage = _table_storage()
|
||||
execute = MagicMock(return_value=[("file.txt",)])
|
||||
monkeypatch.setattr(storage, "_execute_sql", execute)
|
||||
assert storage.scan("12345") == ["file.txt"]
|
||||
execute.assert_called_once_with("LIST TABLE VOLUME test_dataset_12345", fetch=True)
|
||||
|
||||
|
||||
def test_scan_propagates_backend_failure(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
storage = _table_storage()
|
||||
monkeypatch.setattr(storage, "_execute_sql", MagicMock(side_effect=RuntimeError("backend unavailable")))
|
||||
with pytest.raises(RuntimeError, match="backend unavailable"):
|
||||
storage.scan("ops_trace/app/")
|
||||
|
||||
|
||||
def test_delete_propagates_failed_existence_check(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
storage = _table_storage()
|
||||
execute = MagicMock(side_effect=RuntimeError("backend unavailable"))
|
||||
monkeypatch.setattr(storage, "_execute_sql", execute)
|
||||
with pytest.raises(RuntimeError, match="backend unavailable"):
|
||||
storage.delete("ops_trace/app/trace.json")
|
||||
assert execute.call_count == 1
|
||||
|
||||
|
||||
def test_delete_absent_file_remains_idempotent(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
storage = _table_storage()
|
||||
execute = MagicMock(return_value=[])
|
||||
monkeypatch.setattr(storage, "_execute_sql", execute)
|
||||
storage.delete("ops_trace/app/trace.json")
|
||||
assert execute.call_count == 1
|
||||
@@ -0,0 +1,25 @@
|
||||
from typing import override
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from oss2 import Auth
|
||||
|
||||
from extensions.storage.aliyun_oss_storage import AliyunOssStorage
|
||||
from tests.helpers.unit.oss.__mock.base import (
|
||||
BaseStorageTest,
|
||||
get_example_bucket,
|
||||
get_example_folder,
|
||||
)
|
||||
|
||||
pytest_plugins = ("tests.helpers.unit.oss.__mock.aliyun_oss",)
|
||||
|
||||
|
||||
class TestAliyunOss(BaseStorageTest):
|
||||
@pytest.fixture(autouse=True)
|
||||
@override
|
||||
def setup_method(self, setup_aliyun_oss_mock: None) -> None:
|
||||
"""Executed before each test method."""
|
||||
with patch.object(Auth, "__init__", return_value=None):
|
||||
self.storage = AliyunOssStorage()
|
||||
self.storage.bucket_name = get_example_bucket()
|
||||
self.storage.folder = get_example_folder()
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Unit tests for the common AWS S3 storage adapter."""
|
||||
|
||||
from collections.abc import Callable
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from extensions.storage.aws_s3_storage import AwsS3Storage
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def s3_storage(config_overrides: Callable[..., None]) -> tuple[AwsS3Storage, MagicMock]:
|
||||
client = MagicMock()
|
||||
client.head_bucket.return_value = dict[str, object]()
|
||||
config_overrides(S3_USE_AWS_MANAGED_IAM=False, S3_BUCKET_NAME="dify-files")
|
||||
with patch("extensions.storage.aws_s3_storage.boto3.client", return_value=client):
|
||||
storage = AwsS3Storage()
|
||||
return storage, client
|
||||
|
||||
|
||||
def test_generate_presigned_url(s3_storage: tuple[AwsS3Storage, MagicMock]) -> None:
|
||||
storage, client = s3_storage
|
||||
client.generate_presigned_url.return_value = "https://s3.example.com/icon.png?signature=test"
|
||||
|
||||
result = storage.generate_presigned_url(
|
||||
"upload_files/tenant/icon.png",
|
||||
expires_in=300,
|
||||
content_type="image/png",
|
||||
)
|
||||
|
||||
assert result == "https://s3.example.com/icon.png?signature=test"
|
||||
client.generate_presigned_url.assert_called_once_with(
|
||||
"get_object",
|
||||
Params={
|
||||
"Bucket": "dify-files",
|
||||
"Key": "upload_files/tenant/icon.png",
|
||||
"ResponseContentType": "image/png",
|
||||
},
|
||||
ExpiresIn=300,
|
||||
)
|
||||
|
||||
|
||||
def test_scan_lists_recursive_files_and_derived_directories(
|
||||
s3_storage: tuple[AwsS3Storage, MagicMock],
|
||||
) -> None:
|
||||
storage, client = s3_storage
|
||||
paginator = client.get_paginator.return_value
|
||||
paginator.paginate.return_value = [
|
||||
{
|
||||
"Contents": [
|
||||
{"Key": "knowledge-fs/objects/tenant-1/a.txt"},
|
||||
{"Key": "knowledge-fs/objects/tenant-1/nested/b.txt"},
|
||||
{"Key": "knowledge-fs/objects/tenant-1/empty/"},
|
||||
]
|
||||
},
|
||||
{"Contents": [{"Key": "knowledge-fs/objects/tenant-1/nested/c.txt"}]},
|
||||
]
|
||||
|
||||
result = storage.scan(
|
||||
"knowledge-fs/objects/tenant-1",
|
||||
files=True,
|
||||
directories=True,
|
||||
)
|
||||
|
||||
assert result == [
|
||||
"knowledge-fs/objects/tenant-1/a.txt",
|
||||
"knowledge-fs/objects/tenant-1/empty/",
|
||||
"knowledge-fs/objects/tenant-1/nested/",
|
||||
"knowledge-fs/objects/tenant-1/nested/b.txt",
|
||||
"knowledge-fs/objects/tenant-1/nested/c.txt",
|
||||
]
|
||||
client.get_paginator.assert_called_once_with("list_objects_v2")
|
||||
paginator.paginate.assert_called_once_with(
|
||||
Bucket="dify-files",
|
||||
Prefix="knowledge-fs/objects/tenant-1/",
|
||||
)
|
||||
|
||||
|
||||
def test_scan_rejects_request_without_files_or_directories(
|
||||
s3_storage: tuple[AwsS3Storage, MagicMock],
|
||||
) -> None:
|
||||
storage, _ = s3_storage
|
||||
|
||||
with pytest.raises(ValueError, match="At least one"):
|
||||
storage.scan("knowledge-fs", files=False, directories=False)
|
||||
@@ -0,0 +1,62 @@
|
||||
from typing import override
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from baidubce.auth.bce_credentials import BceCredentials
|
||||
from baidubce.bce_client_configuration import BceClientConfiguration
|
||||
|
||||
from extensions.storage.baidu_obs_storage import BaiduObsStorage
|
||||
from tests.helpers.unit.oss.__mock.base import (
|
||||
BaseStorageTest,
|
||||
get_example_bucket,
|
||||
)
|
||||
|
||||
pytest_plugins = ("tests.helpers.unit.oss.__mock.baidu_obs",)
|
||||
|
||||
|
||||
class TestBaiduObs(BaseStorageTest):
|
||||
@pytest.fixture(autouse=True)
|
||||
@override
|
||||
def setup_method(self, setup_baidu_obs_mock: None) -> None:
|
||||
"""Executed before each test method."""
|
||||
with (
|
||||
patch.object(BceCredentials, "__init__", return_value=None),
|
||||
patch.object(BceClientConfiguration, "__init__", return_value=None),
|
||||
):
|
||||
self.storage = BaiduObsStorage()
|
||||
self.storage.bucket_name = get_example_bucket()
|
||||
|
||||
|
||||
class TestBaiduObsConfiguration:
|
||||
def test_init_with_config(self) -> None:
|
||||
mock_dify_config = MagicMock()
|
||||
mock_dify_config.BAIDU_OBS_BUCKET_NAME = "test-bucket"
|
||||
mock_dify_config.BAIDU_OBS_ACCESS_KEY = "test-access-key"
|
||||
mock_dify_config.BAIDU_OBS_SECRET_KEY = "test-secret-key"
|
||||
mock_dify_config.BAIDU_OBS_ENDPOINT = "https://bj.bcebos.com"
|
||||
|
||||
mock_credentials = MagicMock(name="credentials")
|
||||
mock_config = MagicMock(name="config")
|
||||
mock_client = MagicMock(name="client")
|
||||
|
||||
with (
|
||||
patch("extensions.storage.baidu_obs_storage.dify_config", mock_dify_config),
|
||||
patch("extensions.storage.baidu_obs_storage.BceCredentials", return_value=mock_credentials) as credentials,
|
||||
patch(
|
||||
"extensions.storage.baidu_obs_storage.BceClientConfiguration", return_value=mock_config
|
||||
) as configuration,
|
||||
patch("extensions.storage.baidu_obs_storage.BosClient", return_value=mock_client) as client_cls,
|
||||
):
|
||||
storage = BaiduObsStorage()
|
||||
|
||||
assert storage.bucket_name == "test-bucket"
|
||||
assert storage.client == mock_client
|
||||
credentials.assert_called_once_with(
|
||||
access_key_id="test-access-key",
|
||||
secret_access_key="test-secret-key",
|
||||
)
|
||||
configuration.assert_called_once_with(
|
||||
credentials=mock_credentials,
|
||||
endpoint="https://bj.bcebos.com",
|
||||
)
|
||||
client_cls.assert_called_once_with(config=mock_config)
|
||||
@@ -0,0 +1,143 @@
|
||||
from collections.abc import Generator
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from extensions.storage.opendal_storage import OpenDALStorage
|
||||
from tests.helpers.unit.oss.__mock.base import (
|
||||
get_example_data,
|
||||
get_example_filename,
|
||||
get_opendal_bucket,
|
||||
)
|
||||
|
||||
"""Unit coverage for OpenDAL filesystem root selection."""
|
||||
|
||||
|
||||
class TestOpenDALFsDefaultRoot:
|
||||
"""Test that OpenDALStorage with scheme='fs' works correctly when no root is provided."""
|
||||
|
||||
def test_fs_without_root_uses_default(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""When no root is specified, the default 'storage' should be used and passed to the Operator."""
|
||||
# Change to tmp_path so the default "storage" dir is created there
|
||||
monkeypatch.chdir(tmp_path)
|
||||
# Ensure no OPENDAL_FS_ROOT env var is set
|
||||
monkeypatch.delenv("OPENDAL_FS_ROOT", raising=False)
|
||||
|
||||
storage = OpenDALStorage(scheme="fs")
|
||||
|
||||
# The default directory should have been created
|
||||
assert (tmp_path / "storage").is_dir()
|
||||
# The storage should be functional
|
||||
storage.save("test_default_root.txt", b"hello")
|
||||
assert storage.exists("test_default_root.txt")
|
||||
assert storage.load_once("test_default_root.txt") == b"hello"
|
||||
|
||||
# Cleanup
|
||||
storage.delete("test_default_root.txt")
|
||||
|
||||
def test_fs_with_explicit_root(self, tmp_path: Path) -> None:
|
||||
"""When root is explicitly provided, it should be used."""
|
||||
custom_root = str(tmp_path / "custom_storage")
|
||||
storage = OpenDALStorage(scheme="fs", root=custom_root)
|
||||
|
||||
assert Path(custom_root).is_dir()
|
||||
storage.save("test_explicit_root.txt", b"world")
|
||||
assert storage.exists("test_explicit_root.txt")
|
||||
assert storage.load_once("test_explicit_root.txt") == b"world"
|
||||
|
||||
# Cleanup
|
||||
storage.delete("test_explicit_root.txt")
|
||||
|
||||
def test_fs_with_env_var_root(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""When OPENDAL_FS_ROOT env var is set, it should be picked up via _get_opendal_kwargs."""
|
||||
env_root = str(tmp_path / "env_storage")
|
||||
monkeypatch.setenv("OPENDAL_FS_ROOT", env_root)
|
||||
# Ensure .env file doesn't interfere
|
||||
monkeypatch.chdir(tmp_path)
|
||||
|
||||
storage = OpenDALStorage(scheme="fs")
|
||||
|
||||
assert Path(env_root).is_dir()
|
||||
storage.save("test_env_root.txt", b"env_data")
|
||||
assert storage.exists("test_env_root.txt")
|
||||
assert storage.load_once("test_env_root.txt") == b"env_data"
|
||||
|
||||
# Cleanup
|
||||
storage.delete("test_env_root.txt")
|
||||
|
||||
|
||||
class TestOpenDAL:
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_method(self) -> None:
|
||||
"""Executed before each test method."""
|
||||
self.storage = OpenDALStorage(
|
||||
scheme="fs",
|
||||
root=get_opendal_bucket(),
|
||||
)
|
||||
|
||||
@pytest.fixture(scope="class", autouse=True)
|
||||
def teardown_class(self) -> Generator[None]:
|
||||
"""Clean up after all tests in the class."""
|
||||
|
||||
yield
|
||||
|
||||
folder = Path(get_opendal_bucket())
|
||||
if folder.exists() and folder.is_dir():
|
||||
import shutil
|
||||
|
||||
shutil.rmtree(folder, ignore_errors=True)
|
||||
|
||||
def test_save_and_exists(self) -> None:
|
||||
"""Test saving data and checking existence."""
|
||||
filename = get_example_filename()
|
||||
data = get_example_data()
|
||||
|
||||
assert not self.storage.exists(filename)
|
||||
self.storage.save(filename, data)
|
||||
assert self.storage.exists(filename)
|
||||
|
||||
def test_load_once(self) -> None:
|
||||
"""Test loading data once."""
|
||||
filename = get_example_filename()
|
||||
data = get_example_data()
|
||||
|
||||
self.storage.save(filename, data)
|
||||
loaded_data = self.storage.load_once(filename)
|
||||
assert loaded_data == data
|
||||
|
||||
def test_load_stream(self) -> None:
|
||||
"""Test loading data as a stream."""
|
||||
filename = get_example_filename()
|
||||
chunks = 5
|
||||
chunk_size = 4096
|
||||
data = get_example_data(length=chunk_size * chunks)
|
||||
|
||||
self.storage.save(filename, data)
|
||||
generator = self.storage.load_stream(filename)
|
||||
assert isinstance(generator, Generator)
|
||||
for i in range(chunks):
|
||||
fetched = next(generator)
|
||||
assert len(fetched) == chunk_size
|
||||
assert fetched == data[i * chunk_size : (i + 1) * chunk_size]
|
||||
with pytest.raises(StopIteration):
|
||||
next(generator)
|
||||
|
||||
def test_download(self) -> None:
|
||||
"""Test downloading data to a file."""
|
||||
filename = get_example_filename()
|
||||
filepath = str(Path(get_opendal_bucket()) / filename)
|
||||
data = get_example_data()
|
||||
|
||||
self.storage.save(filename, data)
|
||||
self.storage.download(filename, filepath)
|
||||
|
||||
def test_delete(self) -> None:
|
||||
"""Test deleting a file."""
|
||||
filename = get_example_filename()
|
||||
data = get_example_data()
|
||||
|
||||
self.storage.save(filename, data)
|
||||
assert self.storage.exists(filename)
|
||||
|
||||
self.storage.delete(filename)
|
||||
assert not self.storage.exists(filename)
|
||||
@@ -0,0 +1,239 @@
|
||||
from collections.abc import Callable, Generator
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from extensions.storage.supabase_storage import SupabaseStorage
|
||||
|
||||
|
||||
class TestSupabaseStorage:
|
||||
"""Test suite for SupabaseStorage class."""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _supabase_config(self, config_overrides: Callable[..., None]) -> None:
|
||||
config_overrides(
|
||||
SUPABASE_URL="https://test.supabase.co",
|
||||
SUPABASE_API_KEY="test-api-key",
|
||||
SUPABASE_BUCKET_NAME="test-bucket",
|
||||
)
|
||||
|
||||
def test_init_success_with_all_config(self) -> None:
|
||||
"""Test successful initialization when all required config is provided."""
|
||||
with patch("extensions.storage.supabase_storage.Client", autospec=True) as mock_client_class:
|
||||
mock_client = Mock()
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
# Mock bucket_exists to return True so create_bucket is not called
|
||||
with patch.object(SupabaseStorage, "bucket_exists", return_value=True):
|
||||
storage = SupabaseStorage()
|
||||
|
||||
assert storage.bucket_name == "test-bucket"
|
||||
mock_client_class.assert_called_once_with(
|
||||
supabase_url="https://test.supabase.co", supabase_key="test-api-key"
|
||||
)
|
||||
|
||||
def test_init_raises_error_when_url_missing(self, config_overrides: Callable[..., None]) -> None:
|
||||
"""Test initialization raises ValueError when SUPABASE_URL is None."""
|
||||
config_overrides(SUPABASE_URL=None)
|
||||
with pytest.raises(ValueError, match="SUPABASE_URL is not set"):
|
||||
SupabaseStorage()
|
||||
|
||||
def test_init_raises_error_when_api_key_missing(self, config_overrides: Callable[..., None]) -> None:
|
||||
"""Test initialization raises ValueError when SUPABASE_API_KEY is None."""
|
||||
config_overrides(SUPABASE_API_KEY=None)
|
||||
with pytest.raises(ValueError, match="SUPABASE_API_KEY is not set"):
|
||||
SupabaseStorage()
|
||||
|
||||
def test_init_raises_error_when_bucket_name_missing(self, config_overrides: Callable[..., None]) -> None:
|
||||
"""Test initialization raises ValueError when SUPABASE_BUCKET_NAME is None."""
|
||||
config_overrides(SUPABASE_BUCKET_NAME=None)
|
||||
with pytest.raises(ValueError, match="SUPABASE_BUCKET_NAME is not set"):
|
||||
SupabaseStorage()
|
||||
|
||||
def test_create_bucket_when_not_exists(self) -> None:
|
||||
"""Test create_bucket creates bucket when it doesn't exist."""
|
||||
with patch("extensions.storage.supabase_storage.Client", autospec=True) as mock_client_class:
|
||||
mock_client = Mock()
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
with patch.object(SupabaseStorage, "bucket_exists", return_value=False):
|
||||
SupabaseStorage()
|
||||
|
||||
mock_client.storage.create_bucket.assert_called_once_with(id="test-bucket", name="test-bucket")
|
||||
|
||||
def test_create_bucket_when_exists(self) -> None:
|
||||
"""Test create_bucket does not create bucket when it already exists."""
|
||||
with patch("extensions.storage.supabase_storage.Client", autospec=True) as mock_client_class:
|
||||
mock_client = Mock()
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
with patch.object(SupabaseStorage, "bucket_exists", return_value=True):
|
||||
SupabaseStorage()
|
||||
|
||||
mock_client.storage.create_bucket.assert_not_called()
|
||||
|
||||
@pytest.fixture
|
||||
def storage_with_mock_client(self) -> Generator[tuple[SupabaseStorage, Mock]]:
|
||||
"""Fixture providing SupabaseStorage with mocked client."""
|
||||
with patch("extensions.storage.supabase_storage.Client", autospec=True) as mock_client_class:
|
||||
mock_client = Mock()
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
with patch.object(SupabaseStorage, "bucket_exists", return_value=True):
|
||||
storage = SupabaseStorage()
|
||||
# Create fresh mock for each test
|
||||
mock_client.reset_mock()
|
||||
yield storage, mock_client
|
||||
|
||||
def test_save(self, storage_with_mock_client: tuple[SupabaseStorage, Mock]) -> None:
|
||||
"""Test save calls client.storage.from_(bucket).upload(path, data)."""
|
||||
storage, mock_client = storage_with_mock_client
|
||||
|
||||
filename = "test.txt"
|
||||
data = b"test data"
|
||||
|
||||
storage.save(filename, data)
|
||||
|
||||
mock_client.storage.from_.assert_called_once_with("test-bucket")
|
||||
mock_client.storage.from_().upload.assert_called_once_with(filename, data)
|
||||
|
||||
def test_load_once_returns_bytes(self, storage_with_mock_client: tuple[SupabaseStorage, Mock]) -> None:
|
||||
"""Test load_once returns bytes."""
|
||||
storage, mock_client = storage_with_mock_client
|
||||
|
||||
expected_data = b"test content"
|
||||
mock_client.storage.from_().download.return_value = expected_data
|
||||
|
||||
result = storage.load_once("test.txt")
|
||||
|
||||
assert result == expected_data
|
||||
# Verify the correct calls were made
|
||||
assert "test-bucket" in [call[0][0] for call in mock_client.storage.from_.call_args_list if call[0]]
|
||||
mock_client.storage.from_().download.assert_called_with("test.txt")
|
||||
|
||||
def test_load_stream_yields_chunks(self, storage_with_mock_client: tuple[SupabaseStorage, Mock]) -> None:
|
||||
"""Test load_stream yields chunks."""
|
||||
storage, mock_client = storage_with_mock_client
|
||||
|
||||
test_data = b"test content for streaming"
|
||||
mock_client.storage.from_().download.return_value = test_data
|
||||
|
||||
result = storage.load_stream("test.txt")
|
||||
|
||||
assert isinstance(result, Generator)
|
||||
|
||||
# Collect all chunks
|
||||
chunks = list(result)
|
||||
|
||||
# Verify chunks contain the expected data
|
||||
assert b"".join(chunks) == test_data
|
||||
# Verify the correct calls were made
|
||||
assert "test-bucket" in [call[0][0] for call in mock_client.storage.from_.call_args_list if call[0]]
|
||||
mock_client.storage.from_().download.assert_called_with("test.txt")
|
||||
|
||||
def test_download_writes_bytes_to_disk(
|
||||
self, storage_with_mock_client: tuple[SupabaseStorage, Mock], tmp_path: Path
|
||||
) -> None:
|
||||
"""Test download writes expected bytes to disk."""
|
||||
storage, mock_client = storage_with_mock_client
|
||||
|
||||
test_data = b"test file content"
|
||||
mock_client.storage.from_().download.return_value = test_data
|
||||
|
||||
target_file = tmp_path / "downloaded_file.txt"
|
||||
|
||||
storage.download("test.txt", str(target_file))
|
||||
|
||||
# Verify file was written with correct content
|
||||
assert target_file.read_bytes() == test_data
|
||||
# Verify the correct calls were made
|
||||
assert "test-bucket" in [call[0][0] for call in mock_client.storage.from_.call_args_list if call[0]]
|
||||
mock_client.storage.from_().download.assert_called_with("test.txt")
|
||||
|
||||
def test_exists_returns_true_when_file_found(self, storage_with_mock_client: tuple[SupabaseStorage, Mock]) -> None:
|
||||
"""Test exists returns True when list() returns items."""
|
||||
storage, mock_client = storage_with_mock_client
|
||||
|
||||
mock_client.storage.from_().list.return_value = [{"name": "test.txt"}]
|
||||
|
||||
result = storage.exists("test.txt")
|
||||
|
||||
assert result is True
|
||||
assert "test-bucket" in [call[0][0] for call in mock_client.storage.from_.call_args_list if call[0]]
|
||||
mock_client.storage.from_().list.assert_called_with(path="test.txt")
|
||||
|
||||
def test_exists_returns_false_when_file_not_found(
|
||||
self, storage_with_mock_client: tuple[SupabaseStorage, Mock]
|
||||
) -> None:
|
||||
"""Test exists returns False when list() returns an empty list."""
|
||||
storage, mock_client = storage_with_mock_client
|
||||
|
||||
mock_client.storage.from_().list.return_value = []
|
||||
|
||||
result = storage.exists("test.txt")
|
||||
|
||||
assert result is False
|
||||
assert "test-bucket" in [call[0][0] for call in mock_client.storage.from_.call_args_list if call[0]]
|
||||
mock_client.storage.from_().list.assert_called_with(path="test.txt")
|
||||
|
||||
def test_delete_calls_remove_with_filename_in_list(
|
||||
self, storage_with_mock_client: tuple[SupabaseStorage, Mock]
|
||||
) -> None:
|
||||
"""Test delete calls remove([...]) (some client versions require a list)."""
|
||||
storage, mock_client = storage_with_mock_client
|
||||
|
||||
filename = "test.txt"
|
||||
|
||||
storage.delete(filename)
|
||||
|
||||
mock_client.storage.from_.assert_called_once_with("test-bucket")
|
||||
mock_client.storage.from_().remove.assert_called_once_with([filename])
|
||||
|
||||
def test_bucket_exists_returns_true_when_bucket_found(self) -> None:
|
||||
"""Test bucket_exists returns True when bucket is found in list."""
|
||||
with patch("extensions.storage.supabase_storage.Client", autospec=True) as mock_client_class:
|
||||
mock_client = Mock()
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
mock_bucket = Mock()
|
||||
mock_bucket.name = "test-bucket"
|
||||
mock_client.storage.list_buckets.return_value = [mock_bucket]
|
||||
storage = SupabaseStorage()
|
||||
result = storage.bucket_exists()
|
||||
|
||||
assert result is True
|
||||
assert mock_client.storage.list_buckets.call_count >= 1
|
||||
|
||||
def test_bucket_exists_returns_false_when_bucket_not_found(self) -> None:
|
||||
"""Test bucket_exists returns False when bucket is not found in list."""
|
||||
with patch("extensions.storage.supabase_storage.Client", autospec=True) as mock_client_class:
|
||||
mock_client = Mock()
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
# Mock different bucket
|
||||
mock_bucket = Mock()
|
||||
mock_bucket.name = "different-bucket"
|
||||
mock_client.storage.list_buckets.return_value = [mock_bucket]
|
||||
mock_client.storage.create_bucket = Mock()
|
||||
|
||||
storage = SupabaseStorage()
|
||||
result = storage.bucket_exists()
|
||||
|
||||
assert result is False
|
||||
assert mock_client.storage.list_buckets.call_count >= 1
|
||||
|
||||
def test_bucket_exists_returns_false_when_no_buckets(self) -> None:
|
||||
"""Test bucket_exists returns False when no buckets exist."""
|
||||
with patch("extensions.storage.supabase_storage.Client", autospec=True) as mock_client_class:
|
||||
mock_client = Mock()
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
mock_client.storage.list_buckets.return_value = list[object]()
|
||||
mock_client.storage.create_bucket = Mock()
|
||||
|
||||
storage = SupabaseStorage()
|
||||
result = storage.bucket_exists()
|
||||
|
||||
assert result is False
|
||||
assert mock_client.storage.list_buckets.call_count >= 1
|
||||
@@ -0,0 +1,102 @@
|
||||
from pathlib import Path
|
||||
from typing import override
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from qcloud_cos import CosConfig
|
||||
|
||||
from extensions.storage.tencent_cos_storage import TencentCosStorage
|
||||
from tests.helpers.unit.oss.__mock.base import (
|
||||
BaseStorageTest,
|
||||
get_example_bucket,
|
||||
get_example_data,
|
||||
get_example_filename,
|
||||
)
|
||||
|
||||
pytest_plugins = ("tests.helpers.unit.oss.__mock.tencent_cos",)
|
||||
|
||||
|
||||
class TestTencentCos(BaseStorageTest):
|
||||
@pytest.fixture(autouse=True)
|
||||
@override
|
||||
def setup_method(self, setup_tencent_cos_mock: None) -> None:
|
||||
"""Executed before each test method."""
|
||||
with patch.object(CosConfig, "__init__", return_value=None, autospec=True):
|
||||
self.storage = TencentCosStorage()
|
||||
self.storage.bucket_name = get_example_bucket()
|
||||
|
||||
# Replaces the base test so the download lands in tmp_path; the extra fixture argument narrows the signature.
|
||||
@override
|
||||
def test_download(self, tmp_path: Path) -> None: # pyrefly: ignore[bad-override]
|
||||
destination = tmp_path / "download.txt"
|
||||
self.storage.download(get_example_filename(), str(destination))
|
||||
assert destination.read_bytes() == get_example_data()
|
||||
|
||||
|
||||
class TestTencentCosConfiguration:
|
||||
"""Tests for TencentCosStorage initialization with different configurations."""
|
||||
|
||||
def test_init_with_custom_domain(self) -> None:
|
||||
"""Test initialization with custom domain configured."""
|
||||
# Mock dify_config to return custom domain configuration
|
||||
mock_dify_config = MagicMock()
|
||||
mock_dify_config.TENCENT_COS_CUSTOM_DOMAIN = "cos.example.com"
|
||||
mock_dify_config.TENCENT_COS_SECRET_ID = "test-secret-id"
|
||||
mock_dify_config.TENCENT_COS_SECRET_KEY = "test-secret-key"
|
||||
mock_dify_config.TENCENT_COS_SCHEME = "https"
|
||||
|
||||
# Mock CosConfig and CosS3Client
|
||||
mock_config_instance = MagicMock()
|
||||
mock_client = MagicMock()
|
||||
|
||||
with (
|
||||
patch("extensions.storage.tencent_cos_storage.dify_config", mock_dify_config),
|
||||
patch(
|
||||
"extensions.storage.tencent_cos_storage.CosConfig", return_value=mock_config_instance, autospec=True
|
||||
) as mock_cos_config,
|
||||
patch("extensions.storage.tencent_cos_storage.CosS3Client", return_value=mock_client, autospec=True),
|
||||
):
|
||||
TencentCosStorage()
|
||||
|
||||
# Verify CosConfig was called with Domain parameter (not Region)
|
||||
mock_cos_config.assert_called_once()
|
||||
call_kwargs = mock_cos_config.call_args[1]
|
||||
assert "Domain" in call_kwargs
|
||||
assert call_kwargs["Domain"] == "cos.example.com"
|
||||
assert "Region" not in call_kwargs
|
||||
assert call_kwargs["SecretId"] == "test-secret-id"
|
||||
assert call_kwargs["SecretKey"] == "test-secret-key"
|
||||
assert call_kwargs["Scheme"] == "https"
|
||||
|
||||
def test_init_with_region(self) -> None:
|
||||
"""Test initialization with region configured (no custom domain)."""
|
||||
# Mock dify_config to return region configuration
|
||||
mock_dify_config = MagicMock()
|
||||
mock_dify_config.TENCENT_COS_CUSTOM_DOMAIN = None
|
||||
mock_dify_config.TENCENT_COS_REGION = "ap-guangzhou"
|
||||
mock_dify_config.TENCENT_COS_SECRET_ID = "test-secret-id"
|
||||
mock_dify_config.TENCENT_COS_SECRET_KEY = "test-secret-key"
|
||||
mock_dify_config.TENCENT_COS_SCHEME = "https"
|
||||
|
||||
# Mock CosConfig and CosS3Client
|
||||
mock_config_instance = MagicMock()
|
||||
mock_client = MagicMock()
|
||||
|
||||
with (
|
||||
patch("extensions.storage.tencent_cos_storage.dify_config", mock_dify_config),
|
||||
patch(
|
||||
"extensions.storage.tencent_cos_storage.CosConfig", return_value=mock_config_instance, autospec=True
|
||||
) as mock_cos_config,
|
||||
patch("extensions.storage.tencent_cos_storage.CosS3Client", return_value=mock_client, autospec=True),
|
||||
):
|
||||
TencentCosStorage()
|
||||
|
||||
# Verify CosConfig was called with Region parameter (not Domain)
|
||||
mock_cos_config.assert_called_once()
|
||||
call_kwargs = mock_cos_config.call_args[1]
|
||||
assert "Region" in call_kwargs
|
||||
assert call_kwargs["Region"] == "ap-guangzhou"
|
||||
assert "Domain" not in call_kwargs
|
||||
assert call_kwargs["SecretId"] == "test-secret-id"
|
||||
assert call_kwargs["SecretKey"] == "test-secret-key"
|
||||
assert call_kwargs["Scheme"] == "https"
|
||||
@@ -0,0 +1,35 @@
|
||||
from collections.abc import Callable
|
||||
from typing import override
|
||||
|
||||
import pytest
|
||||
from tos import TosClientV2
|
||||
|
||||
from extensions.storage.volcengine_tos_storage import VolcengineTosStorage
|
||||
from tests.helpers.unit.oss.__mock.base import (
|
||||
BaseStorageTest,
|
||||
get_example_bucket,
|
||||
)
|
||||
|
||||
pytest_plugins = ("tests.helpers.unit.oss.__mock.volcengine_tos",)
|
||||
|
||||
|
||||
class TestVolcengineTos(BaseStorageTest):
|
||||
@pytest.fixture(autouse=True)
|
||||
@override
|
||||
def setup_method(self, setup_volcengine_tos_mock: None, config_overrides: Callable[..., None]) -> None:
|
||||
"""Executed before each test method."""
|
||||
config_overrides(
|
||||
VOLCENGINE_TOS_ACCESS_KEY="test_access_key",
|
||||
VOLCENGINE_TOS_SECRET_KEY="test_secret_key",
|
||||
VOLCENGINE_TOS_ENDPOINT="test_endpoint",
|
||||
VOLCENGINE_TOS_REGION="test_region",
|
||||
)
|
||||
self.storage = VolcengineTosStorage()
|
||||
|
||||
self.storage.bucket_name = get_example_bucket()
|
||||
self.storage.client = TosClientV2(
|
||||
ak="dify",
|
||||
sk="dify",
|
||||
endpoint="https://xxx.volces.com",
|
||||
region="cn-beijing",
|
||||
)
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,391 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import builtins
|
||||
import importlib
|
||||
|
||||
import pytest
|
||||
from flask import Blueprint, Flask, Response
|
||||
from flask.views import MethodView
|
||||
from flask_cors import CORS
|
||||
from flask_restx import Resource
|
||||
|
||||
from configs import dify_config
|
||||
from constants import HEADER_NAME_CATALOG
|
||||
from dify_app import DifyApp
|
||||
from extensions import ext_blueprints
|
||||
from extensions.ext_blueprints import (
|
||||
AUTHENTICATED_HEADERS,
|
||||
EXPOSED_HEADERS,
|
||||
OPENAPI_HEADERS,
|
||||
OPENAPI_MAX_AGE_SECONDS,
|
||||
_apply_cors_once,
|
||||
)
|
||||
from libs.external_api import ExternalApi
|
||||
from tests.helpers.unit.config_override import apply_config_overrides
|
||||
|
||||
"""CORS posture for /openapi/v1/* — default empty allowlist (same-origin),
|
||||
expandable via OPENAPI_CORS_ALLOW_ORIGINS. Cross-origin requests from
|
||||
disallowed origins do not receive the Access-Control-Allow-Origin
|
||||
header, which the browser then blocks.
|
||||
|
||||
Tests use a fresh Blueprint + Flask-CORS per case because the production
|
||||
blueprint is a module-level singleton and can't be reconfigured once
|
||||
registered.
|
||||
"""
|
||||
|
||||
|
||||
if not hasattr(builtins, "MethodView"):
|
||||
builtins.MethodView = MethodView # type: ignore[attr-defined]
|
||||
|
||||
|
||||
def _make_app(allowed_origins: list[str], blueprint_name: str) -> Flask:
|
||||
"""Build a Flask app with a fresh openapi-style blueprint mirroring
|
||||
production CORS settings, parameterised on the origin allowlist.
|
||||
"""
|
||||
bp = Blueprint(blueprint_name, __name__, url_prefix="/openapi/v1")
|
||||
api = ExternalApi(bp, version="1.0", title="OpenAPI Test", description="")
|
||||
|
||||
@api.route("/_health")
|
||||
class _Health(Resource):
|
||||
def get(self) -> dict[str, bool]:
|
||||
return {"ok": True}
|
||||
|
||||
CORS(
|
||||
bp,
|
||||
resources={r"/*": {"origins": allowed_origins}},
|
||||
supports_credentials=True,
|
||||
allow_headers=list(OPENAPI_HEADERS),
|
||||
methods=["GET", "POST", "PATCH", "DELETE", "OPTIONS"],
|
||||
expose_headers=list(EXPOSED_HEADERS),
|
||||
max_age=OPENAPI_MAX_AGE_SECONDS,
|
||||
)
|
||||
|
||||
app = Flask(__name__)
|
||||
app.config["TESTING"] = True
|
||||
app.register_blueprint(bp)
|
||||
return app
|
||||
|
||||
|
||||
class TestCorsCases:
|
||||
from tests.unit_tests.controllers.conftest import error_response_contract
|
||||
|
||||
error_response_contract = staticmethod(error_response_contract)
|
||||
|
||||
from tests.unit_tests.controllers.conftest import datasource_application_dependencies
|
||||
|
||||
datasource_application_dependencies = staticmethod(datasource_application_dependencies)
|
||||
|
||||
from tests.unit_tests.controllers.conftest import pipeline_application
|
||||
|
||||
pipeline_application = staticmethod(pipeline_application)
|
||||
|
||||
from tests.unit_tests.controllers.conftest import app_query_services
|
||||
|
||||
app_query_services = staticmethod(app_query_services)
|
||||
|
||||
from tests.unit_tests.controllers.openapi.conftest import openapi_app
|
||||
|
||||
openapi_app = staticmethod(openapi_app)
|
||||
|
||||
from tests.unit_tests.controllers.openapi.conftest import app
|
||||
|
||||
app = staticmethod(app)
|
||||
|
||||
from tests.unit_tests.controllers.openapi.conftest import admitted_bearer
|
||||
|
||||
admitted_bearer = staticmethod(admitted_bearer)
|
||||
|
||||
from tests.unit_tests.controllers.openapi.conftest import _account_services
|
||||
|
||||
_account_services = staticmethod(_account_services)
|
||||
|
||||
@staticmethod
|
||||
def test_default_openapi_cors_allowlist_is_empty() -> None:
|
||||
"""Default config admits no cross-origin until operator opts in."""
|
||||
assert dify_config.OPENAPI_CORS_ALLOW_ORIGINS == []
|
||||
|
||||
@staticmethod
|
||||
def test_preflight_allowed_origin_returns_cors_headers() -> None:
|
||||
app = _make_app(["https://app.example.com"], "openapi_t1")
|
||||
client = app.test_client()
|
||||
response = client.options(
|
||||
"/openapi/v1/_health",
|
||||
headers={
|
||||
"Origin": "https://app.example.com",
|
||||
"Access-Control-Request-Method": "GET",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.headers.get("Access-Control-Allow-Origin") == "https://app.example.com"
|
||||
assert response.headers.get("Access-Control-Max-Age") == str(OPENAPI_MAX_AGE_SECONDS)
|
||||
|
||||
@staticmethod
|
||||
def test_preflight_disallowed_origin_omits_cors_headers() -> None:
|
||||
app = _make_app(["https://app.example.com"], "openapi_t2")
|
||||
client = app.test_client()
|
||||
response = client.options(
|
||||
"/openapi/v1/_health",
|
||||
headers={
|
||||
"Origin": "https://attacker.example",
|
||||
"Access-Control-Request-Method": "GET",
|
||||
},
|
||||
)
|
||||
|
||||
# flask-cors omits Allow-Origin for disallowed origins; browser blocks.
|
||||
assert "Access-Control-Allow-Origin" not in response.headers
|
||||
|
||||
@staticmethod
|
||||
def test_preflight_with_default_empty_allowlist_omits_cors_headers() -> None:
|
||||
app = _make_app([], "openapi_t3")
|
||||
client = app.test_client()
|
||||
response = client.options(
|
||||
"/openapi/v1/_health",
|
||||
headers={
|
||||
"Origin": "https://app.example.com",
|
||||
"Access-Control-Request-Method": "GET",
|
||||
},
|
||||
)
|
||||
|
||||
assert "Access-Control-Allow-Origin" not in response.headers
|
||||
|
||||
@staticmethod
|
||||
def test_same_origin_request_succeeds_without_origin_header() -> None:
|
||||
app = _make_app(["https://app.example.com"], "openapi_t4")
|
||||
client = app.test_client()
|
||||
# Browsers don't send Origin on same-origin GETs.
|
||||
response = client.get("/openapi/v1/_health")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.get_json() == {"ok": True}
|
||||
|
||||
@staticmethod
|
||||
def test_authorization_header_is_in_allow_headers() -> None:
|
||||
"""Bearer-authed routes need Authorization in the preflight response."""
|
||||
app = _make_app(["https://app.example.com"], "openapi_t5")
|
||||
client = app.test_client()
|
||||
response = client.options(
|
||||
"/openapi/v1/_health",
|
||||
headers={
|
||||
"Origin": "https://app.example.com",
|
||||
"Access-Control-Request-Method": "GET",
|
||||
"Access-Control-Request-Headers": "Authorization",
|
||||
},
|
||||
)
|
||||
|
||||
allow_headers = response.headers.get("Access-Control-Allow-Headers", "").lower()
|
||||
assert "authorization" in allow_headers
|
||||
|
||||
@staticmethod
|
||||
def test_catalog_fingerprint_header_is_exposed() -> None:
|
||||
app = _make_app(["https://app.example.com"], "openapi_t6")
|
||||
client = app.test_client()
|
||||
response = client.get("/openapi/v1/_health", headers={"Origin": "https://app.example.com"})
|
||||
|
||||
expose_headers = response.headers.get("Access-Control-Expose-Headers", "")
|
||||
assert HEADER_NAME_CATALOG in expose_headers
|
||||
|
||||
@staticmethod
|
||||
def test_catalog_fingerprint_header_is_allowed_on_requests() -> None:
|
||||
"""Every guarded route needs the fingerprint on the request, so preflight must admit it."""
|
||||
app = _make_app(["https://app.example.com"], "openapi_t7")
|
||||
client = app.test_client()
|
||||
response = client.options(
|
||||
"/openapi/v1/_health",
|
||||
headers={
|
||||
"Origin": "https://app.example.com",
|
||||
"Access-Control-Request-Method": "GET",
|
||||
"Access-Control-Request-Headers": HEADER_NAME_CATALOG,
|
||||
},
|
||||
)
|
||||
|
||||
allow_headers = response.headers.get("Access-Control-Allow-Headers", "").lower()
|
||||
assert HEADER_NAME_CATALOG.lower() in allow_headers
|
||||
|
||||
|
||||
"""Regression coverage for authenticated browser CORS headers."""
|
||||
|
||||
|
||||
class TestExtBlueprintsCorsCases:
|
||||
@staticmethod
|
||||
def test_authenticated_cors_allows_request_metadata_headers() -> None:
|
||||
app = Flask(__name__)
|
||||
blueprint = Blueprint("cors_probe", __name__, url_prefix="/console/api")
|
||||
|
||||
@blueprint.post("/probe")
|
||||
def probe() -> tuple[str, int]:
|
||||
return "", 204
|
||||
|
||||
_apply_cors_once(
|
||||
blueprint,
|
||||
resources={r"/*": {"origins": ["http://localhost:3000"]}},
|
||||
supports_credentials=True,
|
||||
allow_headers=list(AUTHENTICATED_HEADERS),
|
||||
methods=["POST", "OPTIONS"],
|
||||
)
|
||||
app.register_blueprint(blueprint)
|
||||
|
||||
response = app.test_client().options(
|
||||
"/console/api/probe",
|
||||
headers={
|
||||
"Access-Control-Request-Headers": "Idempotency-Key, X-Request-ID, X-Trace-ID",
|
||||
"Access-Control-Request-Method": "POST",
|
||||
"Origin": "http://localhost:3000",
|
||||
},
|
||||
)
|
||||
|
||||
allowed_headers = response.headers.get("Access-Control-Allow-Headers", "").lower()
|
||||
assert "idempotency-key" in allowed_headers
|
||||
assert "x-request-id" in allowed_headers
|
||||
assert "x-trace-id" in allowed_headers
|
||||
|
||||
|
||||
"""Verifies the OPENAPI_ENABLED gate in `extensions.ext_blueprints.init_app`.
|
||||
|
||||
Contract:
|
||||
- When `dify_config.OPENAPI_ENABLED` is True, the `/openapi/v1/*` blueprint
|
||||
must be registered on the Flask app AND have CORS configured (signalled
|
||||
by the idempotent `_dify_cors_applied` marker that `_apply_cors_once`
|
||||
sets after wiring `flask_cors.CORS`).
|
||||
- When False, the openapi blueprint must NOT be registered and CORS must
|
||||
NOT be wired for it. The cross-origin posture is otherwise undefined for
|
||||
the disabled feature.
|
||||
|
||||
Why the blueprints are swapped for fresh ones:
|
||||
`init_app` operates on module-level blueprint singletons (e.g.
|
||||
`controllers.service_api.bp`). Other unit tests register those same
|
||||
singletons onto throwaway Flask apps without going through `init_app`
|
||||
(see `tests/unit_tests/controllers/service_api/test___init__.py` and the
|
||||
`tests/unit_tests/controllers/openapi/*` suite). Once a blueprint has
|
||||
been registered to any app, Flask flips `_got_registered_once = True`
|
||||
and rejects further `after_request` hookup -- which is what
|
||||
`flask_cors.CORS(bp, ...)` does internally. To make this gate test
|
||||
order-independent we monkeypatch each consumed `controllers.<ns>.bp` to
|
||||
a pristine, never-registered `Blueprint` for the duration of the test.
|
||||
`init_app` resolves these via `from controllers.<ns> import bp as ...`
|
||||
inside its function body, so the patched value is what it sees.
|
||||
"""
|
||||
|
||||
|
||||
# Modules whose `bp` attribute is consumed by `ext_blueprints.init_app`.
|
||||
# Keep in sync with the imports inside `init_app`.
|
||||
_BLUEPRINT_MODULES: tuple[str, ...] = (
|
||||
"controllers.console",
|
||||
"controllers.files",
|
||||
"controllers.inner_api",
|
||||
"controllers.mcp",
|
||||
"controllers.openapi",
|
||||
"controllers.service_api",
|
||||
"controllers.trigger",
|
||||
"controllers.web",
|
||||
)
|
||||
|
||||
|
||||
def _probe_view() -> tuple[str, int]:
|
||||
return "ok", 200
|
||||
|
||||
|
||||
def _download_probe_view() -> Response:
|
||||
response = Response("document", content_type="text/plain")
|
||||
response.headers["Content-Disposition"] = "attachment; filename=document.txt"
|
||||
return response
|
||||
|
||||
|
||||
def _build_app() -> DifyApp:
|
||||
app = DifyApp(__name__)
|
||||
app.config["TESTING"] = True
|
||||
return app
|
||||
|
||||
|
||||
class TestExtBlueprintsOpenapiCases:
|
||||
@staticmethod
|
||||
def test_openapi_blueprint_registered_with_cors_when_enabled(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
fresh_blueprints: dict[str, Blueprint],
|
||||
) -> None:
|
||||
"""Enabled gate: blueprint mounted, CORS wired, `/openapi/v1/*` rules live."""
|
||||
apply_config_overrides(monkeypatch, OPENAPI_ENABLED=True)
|
||||
|
||||
app = _build_app()
|
||||
ext_blueprints.init_app(app)
|
||||
|
||||
openapi_bp = fresh_blueprints["controllers.openapi"]
|
||||
assert "openapi" in app.blueprints
|
||||
assert app.blueprints["openapi"] is openapi_bp
|
||||
# `_apply_cors_once` only sets this after wiring flask_cors.CORS, so
|
||||
# it is a faithful proxy for "CORS was applied to this blueprint".
|
||||
assert getattr(openapi_bp, "_dify_cors_applied", False) is True
|
||||
|
||||
openapi_rules = [r for r in app.url_map.iter_rules() if r.rule.startswith("/openapi/v1")]
|
||||
assert openapi_rules, "expected at least one /openapi/v1/* route once enabled"
|
||||
|
||||
@staticmethod
|
||||
def test_openapi_blueprint_absent_when_disabled(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
fresh_blueprints: dict[str, Blueprint],
|
||||
) -> None:
|
||||
"""Disabled gate: no blueprint, no CORS, no `/openapi/v1/*` URL rules."""
|
||||
apply_config_overrides(monkeypatch, OPENAPI_ENABLED=False)
|
||||
|
||||
app = _build_app()
|
||||
ext_blueprints.init_app(app)
|
||||
|
||||
openapi_bp = fresh_blueprints["controllers.openapi"]
|
||||
assert "openapi" not in app.blueprints
|
||||
# No CORS wiring should have run for the openapi blueprint.
|
||||
assert not hasattr(openapi_bp, "_dify_cors_applied")
|
||||
openapi_rules = [r for r in app.url_map.iter_rules() if r.rule.startswith("/openapi/v1")]
|
||||
assert openapi_rules == []
|
||||
|
||||
@staticmethod
|
||||
def test_console_cors_exposes_download_filename(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
fresh_blueprints: dict[str, Blueprint],
|
||||
) -> None:
|
||||
origin = "http://localhost:3000"
|
||||
apply_config_overrides(
|
||||
monkeypatch,
|
||||
OPENAPI_ENABLED=False,
|
||||
inner_CONSOLE_CORS_ALLOW_ORIGINS=origin,
|
||||
)
|
||||
|
||||
app = _build_app()
|
||||
ext_blueprints.init_app(app)
|
||||
console_bp = fresh_blueprints["controllers.console"]
|
||||
assert app.blueprints[console_bp.name] is console_bp
|
||||
|
||||
response = app.test_client().get(
|
||||
"/console/api/_download",
|
||||
headers={"Origin": origin},
|
||||
)
|
||||
|
||||
exposed_headers = {
|
||||
header.strip().lower() for header in response.headers["Access-Control-Expose-Headers"].split(",")
|
||||
}
|
||||
assert "content-disposition" in exposed_headers
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fresh_blueprints(monkeypatch: pytest.MonkeyPatch) -> dict[str, Blueprint]:
|
||||
"""Replace each production blueprint singleton with a fresh, unregistered copy.
|
||||
|
||||
Mirrors the production `name` and `url_prefix` so the gate can still
|
||||
be asserted via `app.blueprints[...]` and url_map prefix checks. The
|
||||
openapi replacement gets a dummy `/_probe` rule so the test can
|
||||
observe at least one `/openapi/v1/*` rule after registration.
|
||||
"""
|
||||
fresh: dict[str, Blueprint] = {}
|
||||
for module_name in _BLUEPRINT_MODULES:
|
||||
module = importlib.import_module(module_name)
|
||||
original = module.bp
|
||||
replacement = Blueprint(
|
||||
original.name,
|
||||
original.import_name,
|
||||
url_prefix=original.url_prefix,
|
||||
)
|
||||
if module_name == "controllers.openapi":
|
||||
replacement.add_url_rule("/_probe", endpoint="_probe", view_func=_probe_view)
|
||||
if module_name == "controllers.console":
|
||||
replacement.add_url_rule("/_download", endpoint="_download", view_func=_download_probe_view)
|
||||
monkeypatch.setattr(module, "bp", replacement)
|
||||
fresh[module_name] = replacement
|
||||
return fresh
|
||||
@@ -0,0 +1,556 @@
|
||||
import ssl
|
||||
from collections.abc import Iterator
|
||||
from datetime import timedelta
|
||||
from importlib import import_module
|
||||
from queue import Empty
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
from celery import Celery
|
||||
from celery.contrib.testing.app import TestApp, setup_default_app
|
||||
from celery.exceptions import Retry
|
||||
|
||||
from dify_app import DifyApp
|
||||
from enums import DeploymentEdition
|
||||
from extensions.ext_celery import (
|
||||
_enqueue_initial_community_telemetry_heartbeat,
|
||||
get_celery_broker_transport_options,
|
||||
init_app,
|
||||
)
|
||||
from schedule.collect_agent_sandbox_usage import collect_agent_sandbox_usage
|
||||
from services.knowledge_fs.background_contract import OPERATION_QUEUES, OPERATION_TASK
|
||||
from services.knowledge_fs.lifecycle_readiness import KnowledgeFSLifecycleWorkerReadiness
|
||||
from tasks.application_trace_task import export_application_trace
|
||||
from tasks.trace_delivery_task import deliver_trace_event, retry_trace_export
|
||||
from tests.helpers.unit.config_override import apply_config_overrides, config_overrides_context
|
||||
|
||||
"""Tests for Celery SSL configuration."""
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tracing_celery(request: pytest.FixtureRequest) -> Iterator[Celery]:
|
||||
with (
|
||||
setup_default_app(TestApp()),
|
||||
config_overrides_context(
|
||||
CELERY_BROKER_URL="memory://",
|
||||
CELERY_BACKEND="disabled",
|
||||
OPS_TRACE_QUEUE=request.param,
|
||||
OPS_AGENT_TRACE_ENABLED=True,
|
||||
ENABLE_APPLICATION_TRACE_RECONCILE_TASK=True,
|
||||
ENABLE_TRACE_PAYLOAD_SWEEP_TASK=True,
|
||||
DISABLE_TELEMETRY=True,
|
||||
),
|
||||
):
|
||||
app = init_app(DifyApp(__name__))
|
||||
try:
|
||||
yield app
|
||||
finally:
|
||||
app.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tracing_celery", ["ops_trace_v2", "ops_trace", "tracing-upgrade"], indirect=True)
|
||||
def test_tracing_dispatch_and_beat_use_configured_queue(tracing_celery: Celery) -> None:
|
||||
queue = tracing_celery.conf.task_routes["tasks.trace_delivery_task.*"]["queue"]
|
||||
calls = {
|
||||
deliver_trace_event.name: ({"event": "{}"},),
|
||||
retry_trace_export.name: ("trace_payloads/job.json",),
|
||||
export_application_trace.name: ("run-1",),
|
||||
tracing_celery.conf.beat_schedule["sweep_trace_payloads"]["task"]: (),
|
||||
tracing_celery.conf.beat_schedule["reconcile_application_traces"]["task"]: (),
|
||||
collect_agent_sandbox_usage.name: (),
|
||||
}
|
||||
with tracing_celery.connection_for_write() as connection:
|
||||
if queue != "ops_trace":
|
||||
tracing_celery.send_task("tasks.ops_trace_task.process_trace_tasks", args=({},), queue="ops_trace")
|
||||
for name, args in calls.items():
|
||||
tracing_celery.tasks[name].apply_async(args=args)
|
||||
with connection.SimpleQueue(queue) as inbox:
|
||||
for name, args in calls.items():
|
||||
message = inbox.get(block=False)
|
||||
assert message.headers["task"] == name
|
||||
assert message.payload[0] == list(args)
|
||||
message.ack()
|
||||
with pytest.raises(Empty):
|
||||
inbox.get(block=False)
|
||||
if queue != "ops_trace":
|
||||
with connection.SimpleQueue("ops_trace") as legacy:
|
||||
message = legacy.get(block=False)
|
||||
assert message.headers["task"] == "tasks.ops_trace_task.process_trace_tasks"
|
||||
message.ack()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tracing_celery", ["ops_trace"], indirect=True)
|
||||
def test_redelivery_stays_on_original_queue_during_switchback(tracing_celery: Celery) -> None:
|
||||
task = tracing_celery.tasks["tasks.trace_delivery_task.deliver_trace_event"]
|
||||
task.push_request(
|
||||
id="trace-redelivery",
|
||||
args=({"event": "{}"},),
|
||||
kwargs={},
|
||||
retries=0,
|
||||
called_directly=False,
|
||||
delivery_info={"exchange": "", "routing_key": "ops_trace_v2"},
|
||||
)
|
||||
try:
|
||||
with pytest.raises(Retry):
|
||||
task.retry(countdown=5)
|
||||
with tracing_celery.connection_for_read() as connection:
|
||||
with connection.SimpleQueue("ops_trace_v2") as original:
|
||||
message = original.get(block=False)
|
||||
assert message.headers["task"] == task.name
|
||||
assert message.headers["retries"] == 1
|
||||
message.ack()
|
||||
with connection.SimpleQueue("ops_trace") as restored:
|
||||
with pytest.raises(Empty):
|
||||
restored.get(block=False)
|
||||
finally:
|
||||
task.pop_request()
|
||||
|
||||
|
||||
class TestCelerySslCases:
|
||||
@staticmethod
|
||||
@pytest.fixture(autouse=True)
|
||||
def restore_celery_app_state() -> Iterator[None]:
|
||||
# init_app replaces Celery's process-wide current/default app. Restore both
|
||||
# so later shared tasks do not enter this module's Flask app context.
|
||||
with setup_default_app(TestApp()):
|
||||
yield
|
||||
|
||||
class TestCelerySSLConfiguration:
|
||||
"""Test suite for Celery SSL configuration."""
|
||||
|
||||
def test_get_celery_broker_transport_options_includes_global_keyprefix_for_redis(self):
|
||||
mock_config = MagicMock()
|
||||
mock_config.CELERY_USE_SENTINEL = False
|
||||
mock_config.REDIS_KEY_PREFIX = "enterprise-a"
|
||||
|
||||
with patch("extensions.ext_celery.dify_config", mock_config):
|
||||
from extensions.ext_celery import get_celery_broker_transport_options
|
||||
|
||||
result = get_celery_broker_transport_options()
|
||||
|
||||
assert result["global_keyprefix"] == "enterprise-a:"
|
||||
|
||||
def test_get_celery_broker_transport_options_omits_global_keyprefix_when_prefix_empty(self):
|
||||
mock_config = MagicMock()
|
||||
mock_config.CELERY_USE_SENTINEL = False
|
||||
mock_config.REDIS_KEY_PREFIX = " "
|
||||
|
||||
with patch("extensions.ext_celery.dify_config", mock_config):
|
||||
from extensions.ext_celery import get_celery_broker_transport_options
|
||||
|
||||
result = get_celery_broker_transport_options()
|
||||
|
||||
assert "global_keyprefix" not in result
|
||||
|
||||
def test_get_celery_broker_transport_options_keeps_sentinel_and_adds_global_keyprefix(self):
|
||||
mock_config = MagicMock()
|
||||
mock_config.CELERY_USE_SENTINEL = True
|
||||
mock_config.CELERY_SENTINEL_MASTER_NAME = "mymaster"
|
||||
mock_config.CELERY_SENTINEL_SOCKET_TIMEOUT = 0.1
|
||||
mock_config.CELERY_SENTINEL_PASSWORD = "secret"
|
||||
mock_config.REDIS_KEY_PREFIX = "enterprise-a"
|
||||
|
||||
with patch("extensions.ext_celery.dify_config", mock_config):
|
||||
from extensions.ext_celery import get_celery_broker_transport_options
|
||||
|
||||
result = get_celery_broker_transport_options()
|
||||
|
||||
assert result["master_name"] == "mymaster"
|
||||
assert result["sentinel_kwargs"]["password"] == "secret"
|
||||
assert result["global_keyprefix"] == "enterprise-a:"
|
||||
|
||||
def test_get_celery_ssl_options_when_ssl_disabled(self):
|
||||
"""Test SSL options when BROKER_USE_SSL is False."""
|
||||
from configs import DifyConfig
|
||||
|
||||
dify_config = DifyConfig(CELERY_BROKER_URL="redis://localhost:6379/0")
|
||||
|
||||
with patch("extensions.ext_celery.dify_config", dify_config):
|
||||
from extensions.ext_celery import get_celery_ssl_options
|
||||
|
||||
result = get_celery_ssl_options()
|
||||
assert result is None
|
||||
|
||||
def test_get_celery_ssl_options_when_broker_not_redis(self):
|
||||
"""Test SSL options when broker is not Redis."""
|
||||
mock_config = MagicMock()
|
||||
mock_config.CELERY_BROKER_URL = "amqp://localhost:5672"
|
||||
|
||||
with patch("extensions.ext_celery.dify_config", mock_config):
|
||||
from extensions.ext_celery import get_celery_ssl_options
|
||||
|
||||
result = get_celery_ssl_options()
|
||||
assert result is None
|
||||
|
||||
def test_get_celery_ssl_options_with_cert_none(self):
|
||||
"""Test SSL options with CERT_NONE requirement."""
|
||||
mock_config = MagicMock()
|
||||
mock_config.CELERY_BROKER_URL = "redis://localhost:6379/0"
|
||||
mock_config.REDIS_SSL_CERT_REQS = "CERT_NONE"
|
||||
mock_config.REDIS_SSL_CA_CERTS = None
|
||||
mock_config.REDIS_SSL_CERTFILE = None
|
||||
mock_config.REDIS_SSL_KEYFILE = None
|
||||
|
||||
with patch("extensions.ext_celery.dify_config", mock_config):
|
||||
from extensions.ext_celery import get_celery_ssl_options
|
||||
|
||||
result = get_celery_ssl_options()
|
||||
assert result is not None
|
||||
assert result["ssl_cert_reqs"] == ssl.CERT_NONE
|
||||
assert result["ssl_ca_certs"] is None
|
||||
assert result["ssl_certfile"] is None
|
||||
assert result["ssl_keyfile"] is None
|
||||
|
||||
def test_get_celery_ssl_options_with_cert_required(self):
|
||||
"""Test SSL options with CERT_REQUIRED and certificates."""
|
||||
mock_config = MagicMock()
|
||||
mock_config.CELERY_BROKER_URL = "rediss://localhost:6380/0"
|
||||
mock_config.REDIS_SSL_CERT_REQS = "CERT_REQUIRED"
|
||||
mock_config.REDIS_SSL_CA_CERTS = "/path/to/ca.crt"
|
||||
mock_config.REDIS_SSL_CERTFILE = "/path/to/client.crt"
|
||||
mock_config.REDIS_SSL_KEYFILE = "/path/to/client.key"
|
||||
|
||||
with patch("extensions.ext_celery.dify_config", mock_config):
|
||||
from extensions.ext_celery import get_celery_ssl_options
|
||||
|
||||
result = get_celery_ssl_options()
|
||||
assert result is not None
|
||||
assert result["ssl_cert_reqs"] == ssl.CERT_REQUIRED
|
||||
assert result["ssl_ca_certs"] == "/path/to/ca.crt"
|
||||
assert result["ssl_certfile"] == "/path/to/client.crt"
|
||||
assert result["ssl_keyfile"] == "/path/to/client.key"
|
||||
|
||||
def test_get_celery_ssl_options_with_cert_optional(self):
|
||||
"""Test SSL options with CERT_OPTIONAL requirement."""
|
||||
mock_config = MagicMock()
|
||||
mock_config.CELERY_BROKER_URL = "redis://localhost:6379/0"
|
||||
mock_config.REDIS_SSL_CERT_REQS = "CERT_OPTIONAL"
|
||||
mock_config.REDIS_SSL_CA_CERTS = "/path/to/ca.crt"
|
||||
mock_config.REDIS_SSL_CERTFILE = None
|
||||
mock_config.REDIS_SSL_KEYFILE = None
|
||||
|
||||
with patch("extensions.ext_celery.dify_config", mock_config):
|
||||
from extensions.ext_celery import get_celery_ssl_options
|
||||
|
||||
result = get_celery_ssl_options()
|
||||
assert result is not None
|
||||
assert result["ssl_cert_reqs"] == ssl.CERT_OPTIONAL
|
||||
assert result["ssl_ca_certs"] == "/path/to/ca.crt"
|
||||
|
||||
def test_get_celery_ssl_options_with_invalid_cert_reqs(self):
|
||||
"""Test SSL options with invalid cert requirement defaults to CERT_NONE."""
|
||||
mock_config = MagicMock()
|
||||
mock_config.CELERY_BROKER_URL = "redis://localhost:6379/0"
|
||||
mock_config.REDIS_SSL_CERT_REQS = "INVALID_VALUE"
|
||||
mock_config.REDIS_SSL_CA_CERTS = None
|
||||
mock_config.REDIS_SSL_CERTFILE = None
|
||||
mock_config.REDIS_SSL_KEYFILE = None
|
||||
|
||||
with patch("extensions.ext_celery.dify_config", mock_config):
|
||||
from extensions.ext_celery import get_celery_ssl_options
|
||||
|
||||
result = get_celery_ssl_options()
|
||||
assert result is not None
|
||||
assert result["ssl_cert_reqs"] == ssl.CERT_NONE # Should default to CERT_NONE
|
||||
|
||||
def test_celery_init_applies_ssl_to_broker_and_backend(self):
|
||||
"""Test that SSL options are applied to both broker and backend when using Redis."""
|
||||
mock_config = MagicMock()
|
||||
mock_config.HUMAN_INPUT_TIMEOUT_TASK_INTERVAL = 1
|
||||
mock_config.CELERY_BROKER_URL = "redis://localhost:6379/0"
|
||||
mock_config.CELERY_BACKEND = "redis"
|
||||
mock_config.CELERY_RESULT_BACKEND = "redis://localhost:6379/0"
|
||||
mock_config.REDIS_SSL_CERT_REQS = "CERT_NONE"
|
||||
mock_config.REDIS_SSL_CA_CERTS = None
|
||||
mock_config.REDIS_SSL_CERTFILE = None
|
||||
mock_config.REDIS_SSL_KEYFILE = None
|
||||
mock_config.CELERY_USE_SENTINEL = False
|
||||
mock_config.LOG_FORMAT = "%(message)s"
|
||||
mock_config.LOG_TZ = "UTC"
|
||||
mock_config.LOG_FILE = None
|
||||
|
||||
# Mock all the scheduler configs
|
||||
mock_config.CELERY_BEAT_SCHEDULER_TIME = 1
|
||||
mock_config.AGENT_SANDBOX_METERING_ENABLED = False
|
||||
mock_config.ENABLE_CONVERSATION_CLEANUP_TASK = False
|
||||
mock_config.CONVERSATION_CLEANUP_TASK_INTERVAL = 5
|
||||
mock_config.ENABLE_CLEAN_EMBEDDING_CACHE_TASK = False
|
||||
mock_config.ENABLE_CLEAN_UNUSED_DATASETS_TASK = False
|
||||
mock_config.ENABLE_CREATE_TIDB_SERVERLESS_TASK = False
|
||||
mock_config.ENABLE_UPDATE_TIDB_SERVERLESS_STATUS_TASK = False
|
||||
mock_config.ENABLE_CLEAN_MESSAGES = False
|
||||
mock_config.ENABLE_MAIL_CLEAN_DOCUMENT_NOTIFY_TASK = False
|
||||
mock_config.ENABLE_DATASETS_QUEUE_MONITOR = False
|
||||
mock_config.ENABLE_CHECK_UPGRADABLE_PLUGIN_TASK = False
|
||||
mock_config.ENABLE_WORKFLOW_SCHEDULE_POLLER_TASK = False
|
||||
mock_config.WORKFLOW_SCHEDULE_POLLER_INTERVAL = 1
|
||||
mock_config.WORKFLOW_SCHEDULE_POLLER_BATCH_SIZE = 100
|
||||
mock_config.WORKFLOW_SCHEDULE_MAX_DISPATCH_PER_TICK = 0
|
||||
mock_config.ENABLE_TRIGGER_PROVIDER_REFRESH_TASK = False
|
||||
mock_config.TRIGGER_PROVIDER_REFRESH_INTERVAL = 15
|
||||
mock_config.ENABLE_API_TOKEN_LAST_USED_UPDATE_TASK = False
|
||||
mock_config.API_TOKEN_LAST_USED_UPDATE_INTERVAL = 30
|
||||
|
||||
with (
|
||||
patch("extensions.ext_celery.dify_config", mock_config),
|
||||
patch(
|
||||
"services.knowledge_fs.lifecycle_readiness.get_configured_knowledge_fs_lifecycle_worker_readiness",
|
||||
return_value=KnowledgeFSLifecycleWorkerReadiness(ready=False, blockers=()),
|
||||
),
|
||||
):
|
||||
from dify_app import DifyApp
|
||||
from extensions.ext_celery import init_app
|
||||
|
||||
app = DifyApp(__name__)
|
||||
celery_app = init_app(app)
|
||||
|
||||
# Check that SSL options were applied
|
||||
assert "broker_use_ssl" in celery_app.conf
|
||||
assert celery_app.conf["broker_use_ssl"] is not None
|
||||
assert celery_app.conf["broker_use_ssl"]["ssl_cert_reqs"] == ssl.CERT_NONE
|
||||
|
||||
# Check that SSL is also applied to Redis backend
|
||||
assert "redis_backend_use_ssl" in celery_app.conf
|
||||
assert celery_app.conf["redis_backend_use_ssl"] is not None
|
||||
|
||||
def test_celery_init_registers_required_agent_and_conversation_tasks(self):
|
||||
mock_config = MagicMock()
|
||||
mock_config.BROKER_USE_SSL = False
|
||||
mock_config.REDIS_KEY_PREFIX = "enterprise-a"
|
||||
mock_config.HUMAN_INPUT_TIMEOUT_TASK_INTERVAL = 1
|
||||
mock_config.CELERY_BROKER_URL = "redis://localhost:6379/0"
|
||||
mock_config.CELERY_BACKEND = "redis"
|
||||
mock_config.CELERY_RESULT_BACKEND = "redis://localhost:6379/0"
|
||||
mock_config.CELERY_USE_SENTINEL = False
|
||||
mock_config.LOG_FORMAT = "%(message)s"
|
||||
mock_config.LOG_TZ = "UTC"
|
||||
mock_config.LOG_FILE = None
|
||||
mock_config.CELERY_TASK_ANNOTATIONS = {}
|
||||
|
||||
mock_config.CELERY_BEAT_SCHEDULER_TIME = 1
|
||||
mock_config.AGENT_SANDBOX_METERING_ENABLED = False
|
||||
mock_config.ENABLE_CONVERSATION_CLEANUP_TASK = True
|
||||
mock_config.CONVERSATION_CLEANUP_TASK_INTERVAL = 5
|
||||
mock_config.ENABLE_CLEAN_EMBEDDING_CACHE_TASK = False
|
||||
mock_config.ENABLE_CLEAN_UNUSED_DATASETS_TASK = False
|
||||
mock_config.ENABLE_CREATE_TIDB_SERVERLESS_TASK = False
|
||||
mock_config.ENABLE_UPDATE_TIDB_SERVERLESS_STATUS_TASK = False
|
||||
mock_config.ENABLE_CLEAN_MESSAGES = False
|
||||
mock_config.ENABLE_MAIL_CLEAN_DOCUMENT_NOTIFY_TASK = False
|
||||
mock_config.ENABLE_DATASETS_QUEUE_MONITOR = False
|
||||
mock_config.ENABLE_HUMAN_INPUT_TIMEOUT_TASK = False
|
||||
mock_config.ENABLE_CHECK_UPGRADABLE_PLUGIN_TASK = False
|
||||
mock_config.MARKETPLACE_ENABLED = False
|
||||
mock_config.WORKFLOW_LOG_CLEANUP_ENABLED = False
|
||||
mock_config.ENABLE_WORKFLOW_RUN_CLEANUP_TASK = False
|
||||
mock_config.ENABLE_WORKFLOW_SCHEDULE_POLLER_TASK = False
|
||||
mock_config.WORKFLOW_SCHEDULE_POLLER_INTERVAL = 1
|
||||
mock_config.ENABLE_TRIGGER_PROVIDER_REFRESH_TASK = False
|
||||
mock_config.TRIGGER_PROVIDER_REFRESH_INTERVAL = 15
|
||||
mock_config.ENABLE_API_TOKEN_LAST_USED_UPDATE_TASK = False
|
||||
mock_config.API_TOKEN_LAST_USED_UPDATE_INTERVAL = 30
|
||||
mock_config.DEPLOYMENT_EDITION = DeploymentEdition.COMMUNITY
|
||||
mock_config.ENTERPRISE_TELEMETRY_ENABLED = False
|
||||
|
||||
with (
|
||||
patch("extensions.ext_celery.dify_config", mock_config),
|
||||
patch(
|
||||
"services.knowledge_fs.lifecycle_readiness.get_configured_knowledge_fs_lifecycle_worker_readiness",
|
||||
return_value=KnowledgeFSLifecycleWorkerReadiness(ready=False, blockers=()),
|
||||
),
|
||||
):
|
||||
from dify_app import DifyApp
|
||||
from extensions.ext_celery import init_app
|
||||
|
||||
app = DifyApp(__name__)
|
||||
celery_app = init_app(app)
|
||||
|
||||
assert celery_app.conf["broker_transport_options"]["global_keyprefix"] == "enterprise-a:"
|
||||
assert celery_app.conf["result_backend_transport_options"]["global_keyprefix"] == "enterprise-a:"
|
||||
assert "tasks.collect_agent_resources_task" in celery_app.conf["imports"]
|
||||
assert "tasks.delete_conversation_task" in celery_app.conf["imports"]
|
||||
assert "tasks.workflow_run_archive_deletion_tasks" in celery_app.conf["imports"]
|
||||
import_module("tasks.workflow_run_archive_deletion_tasks")
|
||||
task_name = "tasks.workflow_run_archive_deletion_tasks.delete_workflow_run_archive_task"
|
||||
archive_deletion = celery_app.tasks[task_name]
|
||||
assert archive_deletion.queue == "workflow_archive"
|
||||
assert archive_deletion.acks_late is True
|
||||
assert archive_deletion.reject_on_worker_lost is True
|
||||
assert archive_deletion.autoretry_for == (Exception,)
|
||||
assert archive_deletion.retry_kwargs["max_retries"] == 5
|
||||
assert celery_app.conf["beat_schedule"]["conversation_cleanup_sweeper"]["task"] == (
|
||||
"tasks.delete_conversation_task.sweep_deleted_conversations"
|
||||
)
|
||||
|
||||
|
||||
class TestCommunityTelemetryCeleryCases:
|
||||
@staticmethod
|
||||
def test_beat_start_enqueues_community_telemetry_heartbeat() -> None:
|
||||
task = Mock()
|
||||
sender = SimpleNamespace(
|
||||
app=SimpleNamespace(
|
||||
conf=SimpleNamespace(beat_schedule={"community_telemetry_heartbeat": {}}),
|
||||
tasks={"community_telemetry.send_heartbeat": task},
|
||||
)
|
||||
)
|
||||
|
||||
_enqueue_initial_community_telemetry_heartbeat(sender)
|
||||
|
||||
task.apply_async.assert_called_once_with()
|
||||
|
||||
@staticmethod
|
||||
def test_beat_start_skips_community_telemetry_when_not_scheduled() -> None:
|
||||
task = Mock()
|
||||
sender = SimpleNamespace(
|
||||
app=SimpleNamespace(
|
||||
conf=SimpleNamespace(beat_schedule={}),
|
||||
tasks={"community_telemetry.send_heartbeat": task},
|
||||
)
|
||||
)
|
||||
|
||||
_enqueue_initial_community_telemetry_heartbeat(sender)
|
||||
|
||||
task.apply_async.assert_not_called()
|
||||
|
||||
@staticmethod
|
||||
def test_beat_start_skips_community_telemetry_when_task_is_unavailable() -> None:
|
||||
sender = SimpleNamespace(
|
||||
app=SimpleNamespace(
|
||||
conf=SimpleNamespace(beat_schedule={"community_telemetry_heartbeat": {}}),
|
||||
tasks={},
|
||||
)
|
||||
)
|
||||
|
||||
_enqueue_initial_community_telemetry_heartbeat(sender)
|
||||
|
||||
|
||||
class TestKnowledgeFsCeleryCases:
|
||||
@staticmethod
|
||||
@pytest.fixture(autouse=True)
|
||||
def restore_celery_app_state() -> Iterator[None]:
|
||||
# init_app replaces Celery's process-wide current/default app; restore both so
|
||||
# later shared tasks still resolve against the app that registered them.
|
||||
with setup_default_app(TestApp()):
|
||||
yield
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.parametrize("background_enabled", [False, True])
|
||||
@pytest.mark.parametrize("vector_store", ["qdrant", "tidb_on_qdrant", "elasticsearch", "weaviate"])
|
||||
def test_celery_registers_initial_source_task_when_knowledge_fs_lifecycle_is_ready(
|
||||
background_enabled: bool, vector_store: str, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
# A stale deployment variable must not suppress vector cleanup for the selected provider.
|
||||
monkeypatch.setenv("KNOWLEDGE_VECTOR_STORAGE", "postgres")
|
||||
apply_config_overrides(
|
||||
monkeypatch,
|
||||
REDIS_KEY_PREFIX="test",
|
||||
HUMAN_INPUT_TIMEOUT_TASK_INTERVAL=1,
|
||||
CELERY_BROKER_URL="redis://localhost:6379/0",
|
||||
CELERY_BACKEND="redis",
|
||||
CELERY_USE_SENTINEL=False,
|
||||
CELERY_BROKER_VISIBILITY_TIMEOUT=10800,
|
||||
KNOWLEDGE_FS_BACKGROUND_WORKER_ENABLED=background_enabled,
|
||||
VECTOR_STORE=vector_store,
|
||||
KNOWLEDGE_FS_BACKGROUND_POLL_INTERVAL_SECONDS=5,
|
||||
KNOWLEDGE_FS_BACKGROUND_TASK_TIMEOUT_SECONDS=7200,
|
||||
LOG_FORMAT="%(message)s",
|
||||
LOG_TZ="UTC",
|
||||
LOG_FILE=None,
|
||||
CELERY_TASK_ANNOTATIONS=dict[str, object](),
|
||||
CELERY_BEAT_SCHEDULER_TIME=1,
|
||||
KNOWLEDGE_FS_LIFECYCLE_POLL_INTERVAL_SECONDS=2,
|
||||
ENABLE_CONVERSATION_CLEANUP_TASK=True,
|
||||
CONVERSATION_CLEANUP_TASK_INTERVAL=5,
|
||||
ENABLE_CLEAN_EMBEDDING_CACHE_TASK=False,
|
||||
ENABLE_CLEAN_UNUSED_DATASETS_TASK=False,
|
||||
ENABLE_CREATE_TIDB_SERVERLESS_TASK=False,
|
||||
ENABLE_UPDATE_TIDB_SERVERLESS_STATUS_TASK=False,
|
||||
ENABLE_CLEAN_MESSAGES=False,
|
||||
ENABLE_MAIL_CLEAN_DOCUMENT_NOTIFY_TASK=False,
|
||||
ENABLE_DATASETS_QUEUE_MONITOR=False,
|
||||
ENABLE_HUMAN_INPUT_TIMEOUT_TASK=False,
|
||||
ENABLE_CHECK_UPGRADABLE_PLUGIN_TASK=False,
|
||||
MARKETPLACE_ENABLED=False,
|
||||
WORKFLOW_LOG_CLEANUP_ENABLED=False,
|
||||
ENABLE_WORKFLOW_RUN_CLEANUP_TASK=False,
|
||||
ENABLE_WORKFLOW_SCHEDULE_POLLER_TASK=False,
|
||||
WORKFLOW_SCHEDULE_POLLER_INTERVAL=1,
|
||||
ENABLE_TRIGGER_PROVIDER_REFRESH_TASK=False,
|
||||
TRIGGER_PROVIDER_REFRESH_INTERVAL=15,
|
||||
ENABLE_API_TOKEN_LAST_USED_UPDATE_TASK=False,
|
||||
API_TOKEN_LAST_USED_UPDATE_INTERVAL=30,
|
||||
ENTERPRISE_TELEMETRY_ENABLED=False,
|
||||
)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"services.knowledge_fs.lifecycle_readiness.get_configured_knowledge_fs_lifecycle_worker_readiness",
|
||||
return_value=KnowledgeFSLifecycleWorkerReadiness(ready=True, blockers=()),
|
||||
),
|
||||
):
|
||||
celery_app = init_app(DifyApp(__name__))
|
||||
|
||||
assert "tasks.knowledge_fs_initial_source_tasks" in celery_app.conf["imports"]
|
||||
assert "tasks.knowledge_fs_initial_source_preview_tasks" in celery_app.conf["imports"]
|
||||
assert "tasks.knowledge_fs_failed_retrieval_tasks" in celery_app.conf["imports"]
|
||||
assert "tasks.knowledge_fs_lifecycle_tasks" in celery_app.conf["imports"]
|
||||
assert "tasks.knowledge_fs_source_import_tasks" in celery_app.conf["imports"]
|
||||
assert "tasks.delete_conversation_task" in celery_app.conf["imports"]
|
||||
assert celery_app.conf["beat_schedule"]["conversation_cleanup_sweeper"] == {
|
||||
"task": "tasks.delete_conversation_task.sweep_deleted_conversations",
|
||||
"schedule": timedelta(minutes=5),
|
||||
}
|
||||
assert celery_app.conf["beat_schedule"]["knowledge_fs_staged_upload_cleanup"] == {
|
||||
"task": "tasks.knowledge_fs_lifecycle_tasks.cleanup_knowledge_fs_staged_uploads",
|
||||
"schedule": timedelta(seconds=2),
|
||||
}
|
||||
assert celery_app.conf["beat_schedule"]["knowledge_fs_upgrade_file_cleanup"] == {
|
||||
"task": "tasks.knowledge_fs_upgrade_tasks.cleanup_deferred_knowledge_fs_upgrade_files",
|
||||
"schedule": timedelta(seconds=2),
|
||||
}
|
||||
assert celery_app.conf["beat_schedule"]["knowledge_fs_upgrade_recovery"] == {
|
||||
"task": "tasks.knowledge_fs_upgrade_tasks.recover_stalled_knowledge_fs_upgrades",
|
||||
"schedule": timedelta(seconds=60),
|
||||
"options": {"expires": 60},
|
||||
}
|
||||
schedules = celery_app.conf["beat_schedule"]
|
||||
if background_enabled:
|
||||
assert celery_app.conf["broker_transport_options"]["visibility_timeout"] == 10800
|
||||
for operation, queue in OPERATION_QUEUES.items():
|
||||
assert schedules[f"knowledge_fs_background_{operation}"] == {
|
||||
"task": OPERATION_TASK,
|
||||
"schedule": timedelta(seconds=5),
|
||||
"kwargs": {"operation": operation},
|
||||
"options": {"queue": queue, "expires": 30},
|
||||
}
|
||||
else:
|
||||
assert not any(name.startswith("knowledge_fs_background_") for name in schedules)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"services.knowledge_fs.lifecycle_readiness.get_configured_knowledge_fs_lifecycle_worker_readiness",
|
||||
return_value=KnowledgeFSLifecycleWorkerReadiness(ready=False, blockers=("test",)),
|
||||
),
|
||||
):
|
||||
preview_only_app = init_app(DifyApp(f"{__name__}.preview_only"))
|
||||
|
||||
assert "tasks.knowledge_fs_initial_source_preview_tasks" in preview_only_app.conf["imports"]
|
||||
assert "tasks.knowledge_fs_failed_retrieval_tasks" in preview_only_app.conf["imports"]
|
||||
assert "tasks.knowledge_fs_initial_source_tasks" not in preview_only_app.conf["imports"]
|
||||
assert "tasks.knowledge_fs_lifecycle_tasks" not in preview_only_app.conf["imports"]
|
||||
assert "tasks.knowledge_fs_source_import_tasks" not in preview_only_app.conf["imports"]
|
||||
assert "knowledge_fs_upgrade_recovery" not in preview_only_app.conf["beat_schedule"]
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.parametrize("visibility", [None, 3600, 7200, 7230])
|
||||
def test_long_running_delivery_rejects_an_unsafe_redis_visibility(visibility: int | None) -> None:
|
||||
with patch("extensions.ext_celery.dify_config") as config:
|
||||
config.CELERY_USE_SENTINEL = False
|
||||
config.REDIS_KEY_PREFIX = ""
|
||||
config.CELERY_BROKER_URL = "redis://localhost:6379/0"
|
||||
config.CELERY_BROKER_VISIBILITY_TIMEOUT = visibility
|
||||
config.KNOWLEDGE_FS_BACKGROUND_WORKER_ENABLED = True
|
||||
config.KNOWLEDGE_FS_BACKGROUND_TASK_TIMEOUT_SECONDS = 7200
|
||||
with pytest.raises(ValueError, match="VISIBILITY_TIMEOUT"):
|
||||
get_celery_broker_transport_options()
|
||||
@@ -0,0 +1,53 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
from flask import Flask
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from extensions import ext_knowledge_fs_observability
|
||||
from repositories.sqlalchemy_knowledge_fs_control_space_state_metrics import (
|
||||
SQLAlchemyKnowledgeFSControlSpaceStateCountReader,
|
||||
)
|
||||
from tests.helpers.unit.tasks.knowledge_fs_task_fakes import RecordingKnowledgeFSMetrics
|
||||
|
||||
|
||||
def test_extension_registers_the_database_backed_control_space_gauge() -> None:
|
||||
app = Flask(__name__)
|
||||
session_maker = sessionmaker()
|
||||
metrics = RecordingKnowledgeFSMetrics()
|
||||
reader = SQLAlchemyKnowledgeFSControlSpaceStateCountReader(session_maker)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"extensions.ext_knowledge_fs_observability.session_factory.get_session_maker",
|
||||
return_value=session_maker,
|
||||
),
|
||||
patch(
|
||||
"extensions.ext_knowledge_fs_observability.SQLAlchemyKnowledgeFSControlSpaceStateCountReader",
|
||||
return_value=reader,
|
||||
) as reader_type,
|
||||
patch(
|
||||
"extensions.ext_knowledge_fs_observability.get_knowledge_fs_operational_metrics",
|
||||
return_value=metrics,
|
||||
),
|
||||
):
|
||||
ext_knowledge_fs_observability.init_app(app)
|
||||
|
||||
reader_type.assert_called_once_with(session_maker)
|
||||
assert metrics.gauge_readers == [reader]
|
||||
|
||||
|
||||
def test_extension_does_not_block_startup_when_metric_registration_fails() -> None:
|
||||
app = Flask(__name__)
|
||||
metrics = RecordingKnowledgeFSMetrics(gauge_registration_error=RuntimeError("metrics unavailable"))
|
||||
|
||||
with (
|
||||
patch("extensions.ext_knowledge_fs_observability.session_factory.get_session_maker"),
|
||||
patch("extensions.ext_knowledge_fs_observability.SQLAlchemyKnowledgeFSControlSpaceStateCountReader"),
|
||||
patch(
|
||||
"extensions.ext_knowledge_fs_observability.get_knowledge_fs_operational_metrics",
|
||||
return_value=metrics,
|
||||
),
|
||||
):
|
||||
ext_knowledge_fs_observability.init_app(app)
|
||||
@@ -0,0 +1,128 @@
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from typing import cast
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
from flask import Flask, Response, request
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from constants import COOKIE_NAME_ACCESS_TOKEN
|
||||
from core.logging.context import clear_request_context, get_identity_context
|
||||
from extensions import ext_login
|
||||
from extensions.ext_login import unauthorized_handler
|
||||
from models.account import TenantAccountRole
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_logging_context():
|
||||
clear_request_context()
|
||||
yield
|
||||
clear_request_context()
|
||||
|
||||
|
||||
def test_unauthorized_handler_returns_json_response() -> None:
|
||||
response = unauthorized_handler()
|
||||
|
||||
assert isinstance(response, Response)
|
||||
assert response.status_code == 401
|
||||
assert response.content_type == "application/json"
|
||||
assert json.loads(response.get_data(as_text=True)) == {
|
||||
"code": "unauthorized",
|
||||
"message": "Unauthorized.",
|
||||
"status": 401,
|
||||
}
|
||||
|
||||
|
||||
def test_on_user_logged_in_sets_account_logging_identity() -> None:
|
||||
account = ext_login.Account(name="Test Account", email="test@example.com")
|
||||
account.id = "account-id"
|
||||
account._current_tenant = ext_login.Tenant(name="Test Tenant")
|
||||
account._current_tenant.id = "tenant-id"
|
||||
clear_request_context()
|
||||
|
||||
ext_login.on_user_logged_in(None, account)
|
||||
|
||||
assert get_identity_context() == ("tenant-id", "account-id", "account")
|
||||
|
||||
|
||||
def test_on_user_logged_in_sets_end_user_logging_identity() -> None:
|
||||
end_user = ext_login.EndUser(
|
||||
id="end-user-id",
|
||||
tenant_id="tenant-id",
|
||||
type="browser",
|
||||
)
|
||||
clear_request_context()
|
||||
|
||||
ext_login.on_user_logged_in(None, end_user)
|
||||
|
||||
assert get_identity_context() == ("tenant-id", "end-user-id", "browser")
|
||||
|
||||
|
||||
def test_on_user_logged_in_does_not_break_auth_when_identity_is_unavailable(caplog: pytest.LogCaptureFixture) -> None:
|
||||
account = ext_login.Account(name="Test Account", email="test@example.com")
|
||||
account.id = "account-id"
|
||||
clear_request_context()
|
||||
|
||||
with (
|
||||
mock.patch.object(
|
||||
ext_login.Account,
|
||||
"current_tenant_id",
|
||||
new_callable=mock.PropertyMock,
|
||||
side_effect=RuntimeError("unavailable"),
|
||||
),
|
||||
caplog.at_level("ERROR", logger=ext_login.logger.name),
|
||||
):
|
||||
ext_login.on_user_logged_in(None, account)
|
||||
|
||||
assert get_identity_context() == ("", "", "")
|
||||
assert "Failed to set logging identity context" in caplog.text
|
||||
|
||||
|
||||
def test_on_user_logged_in_logs_unsupported_user_type(caplog: pytest.LogCaptureFixture) -> None:
|
||||
unsupported_user = cast(ext_login.LoginUser, object())
|
||||
clear_request_context()
|
||||
|
||||
with caplog.at_level("ERROR", logger=ext_login.logger.name):
|
||||
ext_login.on_user_logged_in(None, unsupported_user)
|
||||
|
||||
assert get_identity_context() == ("", "", "")
|
||||
assert "Failed to set logging identity context" in caplog.text
|
||||
|
||||
|
||||
def test_admin_api_key_header_takes_precedence_over_console_cookie(
|
||||
sqlite_session: Session, config_overrides: Callable[..., None]
|
||||
) -> None:
|
||||
app = Flask(__name__)
|
||||
tenant = ext_login.Tenant(name="Test Tenant")
|
||||
tenant.id = "workspace-id"
|
||||
account = ext_login.Account(name="Test Account", email="test@example.com")
|
||||
sqlite_session.add_all([tenant, account])
|
||||
sqlite_session.flush()
|
||||
tenant_account_join = ext_login.TenantAccountJoin(
|
||||
tenant_id=tenant.id,
|
||||
account_id=account.id,
|
||||
role=TenantAccountRole.OWNER,
|
||||
)
|
||||
sqlite_session.add(tenant_account_join)
|
||||
sqlite_session.commit()
|
||||
config_overrides(
|
||||
ADMIN_API_KEY_ENABLE=True,
|
||||
ADMIN_API_KEY="admin-key",
|
||||
CONSOLE_WEB_URL="http://console.example.com",
|
||||
CONSOLE_API_URL="http://api.example.com",
|
||||
COOKIE_DOMAIN="",
|
||||
)
|
||||
|
||||
with app.test_request_context(
|
||||
"/console/api/test",
|
||||
headers={
|
||||
"Authorization": "Bearer admin-key",
|
||||
"Cookie": f"{COOKIE_NAME_ACCESS_TOKEN}=console-session",
|
||||
"X-WORKSPACE-ID": "workspace-id",
|
||||
},
|
||||
):
|
||||
result = ext_login._load_user_from_request(request, sqlite_session)
|
||||
|
||||
assert result is account
|
||||
assert account.current_tenant is tenant
|
||||
@@ -0,0 +1,67 @@
|
||||
"""
|
||||
Guards the OpenTelemetry Flask instrumentation contract Dify relies on.
|
||||
|
||||
Flask can run teardown handlers more than once for a request. Instrumentation
|
||||
versions before 0.63b0 left the span activation and context token in
|
||||
``request.environ`` after the first teardown, so the second one detached an
|
||||
already-used token and the OTel API logged "Failed to detach context" for every
|
||||
affected request.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from collections.abc import Iterator
|
||||
|
||||
import flask
|
||||
import pytest
|
||||
from opentelemetry.instrumentation.flask import FlaskInstrumentor
|
||||
|
||||
from tests.unit_tests.extensions.otel.conftest import memory_span_exporter as memory_span_exporter # noqa: PLC0414
|
||||
from tests.unit_tests.extensions.otel.conftest import mock_account_user as mock_account_user # noqa: PLC0414
|
||||
from tests.unit_tests.extensions.otel.conftest import mock_app_model as mock_app_model # noqa: PLC0414
|
||||
from tests.unit_tests.extensions.otel.conftest import mock_end_user as mock_end_user # noqa: PLC0414
|
||||
from tests.unit_tests.extensions.otel.conftest import mock_workflow_runner as mock_workflow_runner # noqa: PLC0414
|
||||
from tests.unit_tests.extensions.otel.conftest import reset_handler_instances as reset_handler_instances # noqa: PLC0414
|
||||
from tests.unit_tests.extensions.otel.conftest import (
|
||||
tracer_provider_with_memory_exporter as tracer_provider_with_memory_exporter, # noqa: PLC0414
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def instrumented_app() -> Iterator[flask.Flask]:
|
||||
app = flask.Flask(__name__)
|
||||
|
||||
@app.route("/ping")
|
||||
def ping() -> str:
|
||||
return "pong"
|
||||
|
||||
instrumentor = FlaskInstrumentor()
|
||||
instrumentor.instrument_app(app)
|
||||
yield app
|
||||
instrumentor.uninstrument_app(app)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("tracer_provider_with_memory_exporter")
|
||||
def test_duplicate_teardown_does_not_log_detach_failure(
|
||||
instrumented_app: flask.Flask, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
handlers = instrumented_app.teardown_request_funcs[None]
|
||||
# instrument_app registers its teardown handler last.
|
||||
index = len(handlers) - 1
|
||||
instrumentation_teardown = handlers[index]
|
||||
calls = 0
|
||||
|
||||
def teardown_twice(exc: BaseException | None) -> None:
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
instrumentation_teardown(exc)
|
||||
instrumentation_teardown(exc)
|
||||
|
||||
handlers[index] = teardown_twice
|
||||
try:
|
||||
with caplog.at_level(logging.ERROR, logger="opentelemetry.context"):
|
||||
assert instrumented_app.test_client().get("/ping").status_code == 200
|
||||
finally:
|
||||
handlers[index] = instrumentation_teardown
|
||||
|
||||
assert calls == 1, "the instrumentation teardown never ran, so nothing was exercised"
|
||||
assert "Failed to detach context" not in caplog.text
|
||||
@@ -0,0 +1,322 @@
|
||||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import pytest
|
||||
from redis import RedisError
|
||||
from redis.retry import Retry
|
||||
|
||||
from extensions import ext_redis
|
||||
from extensions.ext_redis import (
|
||||
RedisClientWrapper,
|
||||
_get_base_redis_params,
|
||||
_get_cluster_connection_health_params,
|
||||
_get_connection_health_params,
|
||||
_normalize_redis_key_prefix,
|
||||
_parse_redis_nodes,
|
||||
_serialize_redis_name,
|
||||
redis_fallback,
|
||||
)
|
||||
from libs.broadcast_channel.redis.pubsub_channel import BroadcastChannel as RedisBroadcastChannel
|
||||
from libs.broadcast_channel.redis.sharded_channel import ShardedRedisBroadcastChannel
|
||||
from tests.helpers.unit.config_override import apply_config_overrides
|
||||
|
||||
|
||||
class TestPubsubChannelCases:
|
||||
@staticmethod
|
||||
def test_get_pubsub_broadcast_channel_defaults_to_pubsub(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
apply_config_overrides(monkeypatch, PUBSUB_REDIS_CHANNEL_TYPE="pubsub")
|
||||
monkeypatch.setattr(ext_redis, "_pubsub_redis_client", object())
|
||||
|
||||
channel = ext_redis.get_pubsub_broadcast_channel()
|
||||
|
||||
assert isinstance(channel, RedisBroadcastChannel)
|
||||
|
||||
@staticmethod
|
||||
def test_get_pubsub_broadcast_channel_sharded(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
apply_config_overrides(monkeypatch, PUBSUB_REDIS_CHANNEL_TYPE="sharded")
|
||||
monkeypatch.setattr(ext_redis, "_pubsub_redis_client", object())
|
||||
|
||||
channel = ext_redis.get_pubsub_broadcast_channel()
|
||||
|
||||
assert isinstance(channel, ShardedRedisBroadcastChannel)
|
||||
|
||||
|
||||
type _RecordedCall = tuple[str, tuple[object, ...], dict[str, object]]
|
||||
|
||||
|
||||
@dataclass
|
||||
class _RecordingRedisClient:
|
||||
calls: list[_RecordedCall] = field(default_factory=list)
|
||||
|
||||
def _record(self, method: str, *args: object, **kwargs: object) -> None:
|
||||
self.calls.append((method, args, kwargs))
|
||||
|
||||
def register_script(self, script: str) -> Callable[..., str]:
|
||||
self._record("register_script", script)
|
||||
|
||||
def execute(*, keys: object, args: object, client: object) -> str:
|
||||
self._record("script", keys, args, client)
|
||||
return "result"
|
||||
|
||||
return execute
|
||||
|
||||
def get(self, name: str | bytes) -> None:
|
||||
self._record("get", name)
|
||||
|
||||
def delete(self, *names: str | bytes) -> None:
|
||||
self._record("delete", *names)
|
||||
|
||||
def lock(self, name: str, **kwargs: object) -> None:
|
||||
self._record("lock", name, **kwargs)
|
||||
|
||||
def hset(self, name: str | bytes, *args: object, **kwargs: object) -> None:
|
||||
self._record("hset", name, *args, **kwargs)
|
||||
|
||||
def hgetall(self, name: str | bytes) -> None:
|
||||
self._record("hgetall", name)
|
||||
|
||||
def hkeys(self, name: str | bytes) -> None:
|
||||
self._record("hkeys", name)
|
||||
|
||||
def hexists(self, name: str | bytes, key: str | bytes) -> None:
|
||||
self._record("hexists", name, key)
|
||||
|
||||
def zadd(self, name: str | bytes, mapping: Mapping[object, object], **kwargs: object) -> None:
|
||||
self._record("zadd", name, mapping, **kwargs)
|
||||
|
||||
def zrem(self, name: str | bytes, *values: str | bytes) -> None:
|
||||
self._record("zrem", name, *values)
|
||||
|
||||
|
||||
class TestRedisCases:
|
||||
@staticmethod
|
||||
@pytest.fixture(autouse=True)
|
||||
def _redis_config(config_overrides: Callable[..., None]) -> None:
|
||||
config_overrides(
|
||||
REDIS_USERNAME=None,
|
||||
REDIS_PASSWORD=None,
|
||||
REDIS_DB=0,
|
||||
REDIS_SERIALIZATION_PROTOCOL=3,
|
||||
REDIS_ENABLE_CLIENT_SIDE_CACHE=False,
|
||||
REDIS_RETRY_RETRIES=3,
|
||||
REDIS_RETRY_BACKOFF_BASE=1.0,
|
||||
REDIS_RETRY_BACKOFF_CAP=10.0,
|
||||
REDIS_SOCKET_TIMEOUT=5.0,
|
||||
REDIS_SOCKET_CONNECT_TIMEOUT=5.0,
|
||||
REDIS_HEALTH_CHECK_INTERVAL=30,
|
||||
REDIS_KEEPALIVE=True,
|
||||
REDIS_KEEPALIVE_IDLE=60,
|
||||
REDIS_KEEPALIVE_INTERVAL=10,
|
||||
REDIS_KEEPALIVE_COUNT=3,
|
||||
REDIS_KEY_PREFIX="",
|
||||
)
|
||||
|
||||
class TestGetConnectionHealthParams:
|
||||
def test_includes_all_health_params(self) -> None:
|
||||
params = _get_connection_health_params()
|
||||
|
||||
assert "retry" in params
|
||||
assert "socket_timeout" in params
|
||||
assert "socket_connect_timeout" in params
|
||||
assert "health_check_interval" in params
|
||||
assert isinstance(params["retry"], Retry)
|
||||
assert params["retry"]._retries == 3 # pyrefly: ignore[missing-attribute]
|
||||
assert params["socket_timeout"] == 5.0
|
||||
assert params["socket_connect_timeout"] == 5.0
|
||||
assert params["health_check_interval"] == 30
|
||||
|
||||
class TestGetClusterConnectionHealthParams:
|
||||
def test_excludes_health_check_interval(self) -> None:
|
||||
params = _get_cluster_connection_health_params()
|
||||
|
||||
assert "retry" in params
|
||||
assert "socket_timeout" in params
|
||||
assert "socket_connect_timeout" in params
|
||||
assert "health_check_interval" not in params
|
||||
|
||||
class TestGetBaseRedisParams:
|
||||
def test_includes_retry_and_health_params(self) -> None:
|
||||
params = _get_base_redis_params()
|
||||
|
||||
assert "retry" in params
|
||||
assert isinstance(params["retry"], Retry)
|
||||
assert params["socket_timeout"] == 5.0
|
||||
assert params["socket_connect_timeout"] == 5.0
|
||||
assert params["health_check_interval"] == 30
|
||||
assert params["socket_keepalive"] is True
|
||||
assert isinstance(params["socket_keepalive_options"], dict)
|
||||
# Existing params still present
|
||||
assert params["db"] == 0
|
||||
assert params["encoding"] == "utf-8"
|
||||
|
||||
class TestParseRedisNodes:
|
||||
def test_trims_nodes(self) -> None:
|
||||
assert _parse_redis_nodes("redis-a:6379, redis-b:6380") == [("redis-a", 6379), ("redis-b", 6380)]
|
||||
|
||||
def test_supports_bracketed_ipv6(self) -> None:
|
||||
assert _parse_redis_nodes("[2001:db8::10]:6379") == [("2001:db8::10", 6379)]
|
||||
|
||||
class TestRedisFallback:
|
||||
def test_redis_fallback_success(self) -> None:
|
||||
@redis_fallback(default_return=None)
|
||||
def test_func() -> str:
|
||||
return "success"
|
||||
|
||||
assert test_func() == "success"
|
||||
|
||||
def test_redis_fallback_error(self) -> None:
|
||||
@redis_fallback(default_return="fallback")
|
||||
def test_func() -> None:
|
||||
raise RedisError("Redis error")
|
||||
|
||||
assert test_func() == "fallback"
|
||||
|
||||
def test_redis_fallback_none_default(self) -> None:
|
||||
@redis_fallback()
|
||||
def test_func() -> None:
|
||||
raise RedisError("Redis error")
|
||||
|
||||
assert test_func() is None
|
||||
|
||||
def test_redis_fallback_with_args(self) -> None:
|
||||
received: list[tuple[int, int]] = []
|
||||
|
||||
@redis_fallback(default_return=0)
|
||||
def test_func(x: int, y: int) -> None:
|
||||
received.append((x, y))
|
||||
raise RedisError("Redis error")
|
||||
|
||||
assert test_func(1, 2) == 0
|
||||
assert received == [(1, 2)]
|
||||
|
||||
def test_redis_fallback_with_kwargs(self) -> None:
|
||||
received: list[tuple[int | None, int | None]] = []
|
||||
|
||||
@redis_fallback(default_return=dict[str, object]())
|
||||
def test_func(x: int | None = None, y: int | None = None) -> None:
|
||||
received.append((x, y))
|
||||
raise RedisError("Redis error")
|
||||
|
||||
assert test_func(x=1, y=2) == {}
|
||||
assert received == [(1, 2)]
|
||||
|
||||
def test_redis_fallback_preserves_function_metadata(self) -> None:
|
||||
@redis_fallback(default_return=None)
|
||||
def test_func() -> None:
|
||||
"""Test function docstring"""
|
||||
pass
|
||||
|
||||
assert test_func.__name__ == "test_func"
|
||||
assert test_func.__doc__ == "Test function docstring"
|
||||
|
||||
class TestRedisKeyPrefixHelpers:
|
||||
def test_normalize_redis_key_prefix_trims_whitespace(self) -> None:
|
||||
assert _normalize_redis_key_prefix(" enterprise-a ") == "enterprise-a"
|
||||
|
||||
def test_normalize_redis_key_prefix_treats_whitespace_only_as_empty(self) -> None:
|
||||
assert _normalize_redis_key_prefix(" ") == ""
|
||||
|
||||
def test_serialize_redis_name_returns_original_when_prefix_empty(self) -> None:
|
||||
assert _serialize_redis_name("model_lb_index:test", "") == "model_lb_index:test"
|
||||
|
||||
def test_serialize_redis_name_adds_single_colon_separator(self) -> None:
|
||||
assert _serialize_redis_name("model_lb_index:test", "enterprise-a") == "enterprise-a:model_lb_index:test"
|
||||
|
||||
class TestRedisClientWrapperKeyPrefix:
|
||||
def test_wrapper_zrem_prefixes_key_but_not_members(self, config_overrides: Callable[..., None]) -> None:
|
||||
client = _RecordingRedisClient()
|
||||
wrapper = RedisClientWrapper()
|
||||
wrapper.initialize(client) # type: ignore[arg-type]
|
||||
config_overrides(REDIS_KEY_PREFIX="enterprise-a")
|
||||
|
||||
wrapper.zrem("reset:user@example.com", "reservation")
|
||||
|
||||
assert client.calls == [("zrem", ("enterprise-a:reset:user@example.com", "reservation"), {})]
|
||||
|
||||
def test_wrapper_registered_script_prefixes_key_arguments(self, config_overrides: Callable[..., None]) -> None:
|
||||
raw_client = _RecordingRedisClient()
|
||||
wrapper = RedisClientWrapper()
|
||||
wrapper.initialize(raw_client) # type: ignore[arg-type]
|
||||
|
||||
config_overrides(REDIS_KEY_PREFIX="enterprise-a")
|
||||
script = wrapper.register_script("return redis.call('GET', KEYS[1])")
|
||||
|
||||
assert script(keys=["device_code:abc"], args=["argument"]) == "result"
|
||||
assert raw_client.calls == [
|
||||
("register_script", ("return redis.call('GET', KEYS[1])",), {}),
|
||||
("script", (("enterprise-a:device_code:abc",), ["argument"], raw_client), {}),
|
||||
]
|
||||
|
||||
def test_wrapper_get_prefixes_string_keys(self, config_overrides: Callable[..., None]) -> None:
|
||||
client = _RecordingRedisClient()
|
||||
wrapper = RedisClientWrapper()
|
||||
wrapper.initialize(client) # type: ignore[arg-type]
|
||||
|
||||
config_overrides(REDIS_KEY_PREFIX="enterprise-a")
|
||||
wrapper.get("oauth_state:abc")
|
||||
|
||||
assert client.calls == [("get", ("enterprise-a:oauth_state:abc",), {})]
|
||||
|
||||
def test_wrapper_delete_prefixes_multiple_keys(self, config_overrides: Callable[..., None]) -> None:
|
||||
client = _RecordingRedisClient()
|
||||
wrapper = RedisClientWrapper()
|
||||
wrapper.initialize(client) # type: ignore[arg-type]
|
||||
|
||||
config_overrides(REDIS_KEY_PREFIX="enterprise-a")
|
||||
wrapper.delete("key:a", "key:b")
|
||||
|
||||
assert client.calls == [("delete", ("enterprise-a:key:a", "enterprise-a:key:b"), {})]
|
||||
|
||||
def test_wrapper_lock_prefixes_lock_name(self, config_overrides: Callable[..., None]) -> None:
|
||||
client = _RecordingRedisClient()
|
||||
wrapper = RedisClientWrapper()
|
||||
wrapper.initialize(client) # type: ignore[arg-type]
|
||||
|
||||
config_overrides(REDIS_KEY_PREFIX="enterprise-a")
|
||||
wrapper.lock("resource-lock", timeout=10)
|
||||
|
||||
method, args, kwargs = client.calls[0]
|
||||
assert method == "lock"
|
||||
assert args == ("enterprise-a:resource-lock",)
|
||||
assert kwargs["timeout"] == 10
|
||||
|
||||
def test_wrapper_hash_operations_prefix_key_name(self, config_overrides: Callable[..., None]) -> None:
|
||||
client = _RecordingRedisClient()
|
||||
wrapper = RedisClientWrapper()
|
||||
wrapper.initialize(client) # type: ignore[arg-type]
|
||||
|
||||
config_overrides(REDIS_KEY_PREFIX="enterprise-a")
|
||||
wrapper.hset("hash:key", "field", "value")
|
||||
wrapper.hgetall("hash:key")
|
||||
wrapper.hkeys("hash:key")
|
||||
wrapper.hexists("hash:key", "field")
|
||||
|
||||
assert client.calls == [
|
||||
("hset", ("enterprise-a:hash:key", "field", "value"), {}),
|
||||
("hgetall", ("enterprise-a:hash:key",), {}),
|
||||
("hkeys", ("enterprise-a:hash:key",), {}),
|
||||
("hexists", ("enterprise-a:hash:key", "field"), {}),
|
||||
]
|
||||
|
||||
def test_wrapper_zadd_prefixes_sorted_set_name(self, config_overrides: Callable[..., None]) -> None:
|
||||
client = _RecordingRedisClient()
|
||||
wrapper = RedisClientWrapper()
|
||||
wrapper.initialize(client) # type: ignore[arg-type]
|
||||
|
||||
config_overrides(REDIS_KEY_PREFIX="enterprise-a")
|
||||
wrapper.zadd("zset:key", {"member": 1})
|
||||
|
||||
method, args, kwargs = client.calls[0]
|
||||
assert method == "zadd"
|
||||
assert args == ("enterprise-a:zset:key", {"member": 1})
|
||||
assert kwargs["nx"] is False
|
||||
|
||||
def test_wrapper_preserves_keys_when_prefix_is_empty(self, config_overrides: Callable[..., None]) -> None:
|
||||
client = _RecordingRedisClient()
|
||||
wrapper = RedisClientWrapper()
|
||||
wrapper.initialize(client) # type: ignore[arg-type]
|
||||
|
||||
config_overrides(REDIS_KEY_PREFIX=" ")
|
||||
wrapper.get("plain:key")
|
||||
|
||||
assert client.calls == [("get", ("plain:key",), {})]
|
||||
@@ -0,0 +1,341 @@
|
||||
import json
|
||||
import logging
|
||||
from unittest import mock
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from flask import Flask, Response
|
||||
|
||||
from extensions import ext_request_logging
|
||||
from extensions.ext_request_logging import _is_content_type_json, _log_request_finished, init_app
|
||||
from tests.helpers.unit.config_override import apply_config_overrides
|
||||
|
||||
|
||||
def test_is_content_type_json() -> None:
|
||||
"""
|
||||
Test the _is_content_type_json function.
|
||||
"""
|
||||
|
||||
assert _is_content_type_json("application/json") is True
|
||||
# content type header with charset option.
|
||||
assert _is_content_type_json("application/json; charset=utf-8") is True
|
||||
# content type header with charset option, in uppercase.
|
||||
assert _is_content_type_json("APPLICATION/JSON; CHARSET=UTF-8") is True
|
||||
assert _is_content_type_json("text/html") is False
|
||||
assert _is_content_type_json("") is False
|
||||
|
||||
|
||||
_KEY_NEEDLE = "needle"
|
||||
_VALUE_NEEDLE = _KEY_NEEDLE[::-1]
|
||||
_RESPONSE_NEEDLE = "response"
|
||||
|
||||
|
||||
def _get_test_app() -> Flask:
|
||||
app = Flask(__name__)
|
||||
|
||||
@app.route("/", methods=["GET", "POST"])
|
||||
def handler() -> str:
|
||||
return _RESPONSE_NEEDLE
|
||||
|
||||
return app
|
||||
|
||||
|
||||
# NOTE(QuantumGhost): Due to the design of Flask, we need to use monkey patch to write tests.
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_request_receiver(monkeypatch: pytest.MonkeyPatch) -> mock.Mock:
|
||||
mock_log_request_started = mock.Mock()
|
||||
monkeypatch.setattr(ext_request_logging, "_log_request_started", mock_log_request_started)
|
||||
return mock_log_request_started
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_response_receiver(monkeypatch: pytest.MonkeyPatch) -> mock.Mock:
|
||||
mock_log_request_finished = mock.Mock()
|
||||
monkeypatch.setattr(ext_request_logging, "_log_request_finished", mock_log_request_finished)
|
||||
return mock_log_request_finished
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def enable_request_logging(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
apply_config_overrides(monkeypatch, ENABLE_REQUEST_LOGGING=True)
|
||||
|
||||
|
||||
def _captured_records(caplog: pytest.LogCaptureFixture, level: int) -> list[logging.LogRecord]:
|
||||
return [
|
||||
record
|
||||
for record in caplog.records
|
||||
if record.name == ext_request_logging.logger.name and record.levelno == level
|
||||
]
|
||||
|
||||
|
||||
def _log_args(record: logging.LogRecord) -> list[str]:
|
||||
assert isinstance(record.args, tuple)
|
||||
args = [arg for arg in record.args if isinstance(arg, str)]
|
||||
assert len(args) == len(record.args)
|
||||
return args
|
||||
|
||||
|
||||
class TestRequestLoggingExtension:
|
||||
def test_receiver_should_not_be_invoked_if_configuration_is_disabled(
|
||||
self,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
mock_request_receiver: MagicMock,
|
||||
mock_response_receiver: MagicMock,
|
||||
) -> None:
|
||||
apply_config_overrides(monkeypatch, ENABLE_REQUEST_LOGGING=False)
|
||||
|
||||
app = _get_test_app()
|
||||
init_app(app)
|
||||
|
||||
with app.test_client() as client:
|
||||
client.get("/")
|
||||
|
||||
mock_request_receiver.assert_not_called()
|
||||
mock_response_receiver.assert_not_called()
|
||||
|
||||
@pytest.mark.usefixtures("enable_request_logging")
|
||||
def test_receiver_should_be_called_if_enabled(
|
||||
self,
|
||||
mock_request_receiver: MagicMock,
|
||||
mock_response_receiver: MagicMock,
|
||||
) -> None:
|
||||
"""
|
||||
Test the request logging extension with JSON data.
|
||||
"""
|
||||
|
||||
app = _get_test_app()
|
||||
init_app(app)
|
||||
|
||||
with app.test_client() as client:
|
||||
client.post("/", json={_KEY_NEEDLE: _VALUE_NEEDLE})
|
||||
|
||||
mock_request_receiver.assert_called_once()
|
||||
mock_response_receiver.assert_called_once()
|
||||
|
||||
|
||||
class TestLoggingLevel:
|
||||
@pytest.mark.usefixtures("enable_request_logging")
|
||||
def test_logging_should_be_skipped_if_level_is_above_debug(self, caplog: pytest.LogCaptureFixture) -> None:
|
||||
caplog.set_level(logging.INFO, logger=ext_request_logging.logger.name)
|
||||
app = _get_test_app()
|
||||
init_app(app)
|
||||
|
||||
with app.test_client() as client:
|
||||
client.post("/", json={_KEY_NEEDLE: _VALUE_NEEDLE})
|
||||
assert not _captured_records(caplog, logging.DEBUG)
|
||||
|
||||
|
||||
class TestRequestReceiverLogging:
|
||||
@pytest.mark.usefixtures("enable_request_logging", "mock_response_receiver")
|
||||
def test_non_json_request(self, caplog: pytest.LogCaptureFixture) -> None:
|
||||
caplog.set_level(logging.DEBUG, logger=ext_request_logging.logger.name)
|
||||
app = _get_test_app()
|
||||
init_app(app)
|
||||
|
||||
with app.test_client() as client:
|
||||
client.post("/", data="plain text")
|
||||
debug_records = _captured_records(caplog, logging.DEBUG)
|
||||
assert len(debug_records) == 1
|
||||
record = debug_records[0]
|
||||
assert "Received Request" in record.msg
|
||||
assert record.args == ("POST", "/")
|
||||
assert "Request Body" not in record.msg
|
||||
|
||||
@pytest.mark.usefixtures("enable_request_logging", "mock_response_receiver")
|
||||
def test_json_request(self, caplog: pytest.LogCaptureFixture) -> None:
|
||||
caplog.set_level(logging.DEBUG, logger=ext_request_logging.logger.name)
|
||||
app = _get_test_app()
|
||||
init_app(app)
|
||||
|
||||
with app.test_client() as client:
|
||||
client.post("/", json={_KEY_NEEDLE: _VALUE_NEEDLE})
|
||||
debug_records = _captured_records(caplog, logging.DEBUG)
|
||||
assert len(debug_records) == 1
|
||||
record = debug_records[0]
|
||||
args = _log_args(record)
|
||||
assert "Received Request" in record.msg
|
||||
assert "Request Body" in record.msg
|
||||
assert args[0] == "POST"
|
||||
assert args[1] == "/"
|
||||
assert _KEY_NEEDLE in args[2]
|
||||
|
||||
@pytest.mark.usefixtures("enable_request_logging", "mock_response_receiver")
|
||||
def test_json_request_with_empty_body(self, caplog: pytest.LogCaptureFixture) -> None:
|
||||
caplog.set_level(logging.DEBUG, logger=ext_request_logging.logger.name)
|
||||
app = _get_test_app()
|
||||
init_app(app)
|
||||
|
||||
with app.test_client() as client:
|
||||
client.post("/", headers={"Content-Type": "application/json"})
|
||||
|
||||
debug_records = _captured_records(caplog, logging.DEBUG)
|
||||
assert len(debug_records) == 1
|
||||
record = debug_records[0]
|
||||
assert "Received Request" in record.msg
|
||||
assert "Request Body" not in record.msg
|
||||
assert record.args == ("POST", "/")
|
||||
|
||||
@pytest.mark.usefixtures("enable_request_logging", "mock_response_receiver")
|
||||
def test_json_request_with_invalid_json_as_body(self, caplog: pytest.LogCaptureFixture) -> None:
|
||||
caplog.set_level(logging.DEBUG, logger=ext_request_logging.logger.name)
|
||||
app = _get_test_app()
|
||||
init_app(app)
|
||||
|
||||
with app.test_client() as client:
|
||||
client.post(
|
||||
"/",
|
||||
headers={"Content-Type": "application/json"},
|
||||
data="{",
|
||||
)
|
||||
assert not _captured_records(caplog, logging.DEBUG)
|
||||
error_records = _captured_records(caplog, logging.ERROR)
|
||||
assert len(error_records) == 1
|
||||
assert error_records[0].message == "Failed to parse JSON request"
|
||||
|
||||
|
||||
class TestResponseReceiverLogging:
|
||||
@pytest.mark.usefixtures("enable_request_logging")
|
||||
def test_non_json_response(self, caplog: pytest.LogCaptureFixture) -> None:
|
||||
caplog.set_level(logging.DEBUG, logger=ext_request_logging.logger.name)
|
||||
app = _get_test_app()
|
||||
response = Response(
|
||||
"OK",
|
||||
headers={"Content-Type": "text/plain"},
|
||||
)
|
||||
_log_request_finished(app, response)
|
||||
debug_records = _captured_records(caplog, logging.DEBUG)
|
||||
assert len(debug_records) == 1
|
||||
record = debug_records[0]
|
||||
args = _log_args(record)
|
||||
assert "Response" in record.msg
|
||||
assert "200" in args[0]
|
||||
assert args[1] == "text/plain"
|
||||
assert "Response Body" not in record.msg
|
||||
|
||||
@pytest.mark.usefixtures("enable_request_logging", "mock_response_receiver")
|
||||
def test_json_response(self, caplog: pytest.LogCaptureFixture) -> None:
|
||||
caplog.set_level(logging.DEBUG, logger=ext_request_logging.logger.name)
|
||||
app = _get_test_app()
|
||||
response = Response(
|
||||
json.dumps({_KEY_NEEDLE: _VALUE_NEEDLE}),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
_log_request_finished(app, response)
|
||||
debug_records = _captured_records(caplog, logging.DEBUG)
|
||||
assert len(debug_records) == 1
|
||||
record = debug_records[0]
|
||||
args = _log_args(record)
|
||||
assert "Response" in record.msg
|
||||
assert "Response Body" in record.msg
|
||||
assert "200" in args[0]
|
||||
assert args[1] == "application/json"
|
||||
assert _KEY_NEEDLE in args[2]
|
||||
|
||||
@pytest.mark.usefixtures("enable_request_logging", "mock_response_receiver")
|
||||
def test_json_request_with_invalid_json_as_body(self, caplog: pytest.LogCaptureFixture) -> None:
|
||||
caplog.set_level(logging.DEBUG, logger=ext_request_logging.logger.name)
|
||||
app = _get_test_app()
|
||||
|
||||
response = Response(
|
||||
"{",
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
_log_request_finished(app, response)
|
||||
assert not _captured_records(caplog, logging.DEBUG)
|
||||
error_records = _captured_records(caplog, logging.ERROR)
|
||||
assert len(error_records) == 1
|
||||
assert error_records[0].message == "Failed to parse JSON response"
|
||||
|
||||
|
||||
class TestResponseUnmodified:
|
||||
def test_when_request_logging_disabled(self) -> None:
|
||||
app = _get_test_app()
|
||||
init_app(app)
|
||||
|
||||
with app.test_client() as client:
|
||||
response = client.post(
|
||||
"/",
|
||||
headers={"Content-Type": "application/json"},
|
||||
data="{",
|
||||
)
|
||||
assert response.text == _RESPONSE_NEEDLE
|
||||
assert response.status_code == 200
|
||||
|
||||
@pytest.mark.usefixtures("enable_request_logging")
|
||||
def test_when_request_logging_enabled(self) -> None:
|
||||
app = _get_test_app()
|
||||
init_app(app)
|
||||
|
||||
with app.test_client() as client:
|
||||
response = client.post(
|
||||
"/",
|
||||
headers={"Content-Type": "application/json"},
|
||||
data="{",
|
||||
)
|
||||
assert response.text == _RESPONSE_NEEDLE
|
||||
assert response.status_code == 200
|
||||
|
||||
|
||||
class TestRequestFinishedInfoAccessLine:
|
||||
def test_info_access_log_includes_method_path_status_duration_trace_id(
|
||||
self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
"""Ensure INFO access line contains expected fields with computed duration and trace id."""
|
||||
app = _get_test_app()
|
||||
# Push a real request context so flask.request and g are available
|
||||
with app.test_request_context("/foo", method="GET"):
|
||||
# Seed start timestamp via the extension's own start hook and control perf_counter deterministically
|
||||
seq = iter([100.0, 100.123456])
|
||||
monkeypatch.setattr(ext_request_logging.time, "perf_counter", lambda: next(seq))
|
||||
# Provide a deterministic trace id
|
||||
monkeypatch.setattr(
|
||||
ext_request_logging,
|
||||
"get_trace_id_from_otel_context",
|
||||
lambda: "trace-xyz",
|
||||
)
|
||||
# Simulate request_started to record start timestamp on g
|
||||
ext_request_logging._log_request_started(app)
|
||||
|
||||
# Capture logs from the real logger at INFO level only (skip DEBUG branch)
|
||||
caplog.set_level(logging.INFO, logger=ext_request_logging.__name__)
|
||||
response = Response(json.dumps({"ok": True}), mimetype="application/json", status=200)
|
||||
_log_request_finished(app, response)
|
||||
|
||||
# Verify a single INFO record with the five fields in order
|
||||
info_records = [rec for rec in caplog.records if rec.levelno == logging.INFO]
|
||||
assert len(info_records) == 1
|
||||
msg = info_records[0].getMessage()
|
||||
# Expected format: METHOD PATH STATUS DURATION_MS TRACE_ID
|
||||
assert "GET" in msg
|
||||
assert "/foo" in msg
|
||||
assert "200" in msg
|
||||
assert "123.456" in msg # rounded to 3 decimals
|
||||
assert "trace-xyz" in msg
|
||||
|
||||
def test_info_access_log_uses_dash_without_start_timestamp(
|
||||
self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
app = _get_test_app()
|
||||
with app.test_request_context("/bar", method="POST"):
|
||||
# No g.__request_started_ts set -> duration should be '-'
|
||||
monkeypatch.setattr(
|
||||
ext_request_logging,
|
||||
"get_trace_id_from_otel_context",
|
||||
lambda: "tid-no-start",
|
||||
)
|
||||
caplog.set_level(logging.INFO, logger=ext_request_logging.__name__)
|
||||
response = Response("OK", mimetype="text/plain", status=204)
|
||||
_log_request_finished(app, response)
|
||||
|
||||
info_records = [rec for rec in caplog.records if rec.levelno == logging.INFO]
|
||||
assert len(info_records) == 1
|
||||
msg = info_records[0].getMessage()
|
||||
assert "POST" in msg
|
||||
assert "/bar" in msg
|
||||
assert "204" in msg
|
||||
# Duration placeholder
|
||||
# The fields are space separated; ensure a standalone '-' appears
|
||||
assert " - " in msg or msg.endswith(" -")
|
||||
assert "tid-no-start" in msg
|
||||
@@ -0,0 +1,75 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from dify_app import DifyApp
|
||||
from extensions import ext_set_secretkey
|
||||
from tests.helpers.unit.config_override import apply_config_overrides
|
||||
|
||||
|
||||
class InMemoryStorage:
|
||||
def __init__(self, files: dict[str, bytes] | None = None) -> None:
|
||||
self.files = files or {}
|
||||
self.saved_files: list[tuple[str, bytes]] = []
|
||||
|
||||
def load_once(self, filename: str) -> bytes:
|
||||
try:
|
||||
return self.files[filename]
|
||||
except KeyError as caught_error:
|
||||
raise FileNotFoundError(filename) from caught_error
|
||||
|
||||
def save(self, filename: str, data: bytes) -> None:
|
||||
self.files[filename] = data
|
||||
self.saved_files.append((filename, data))
|
||||
|
||||
|
||||
def test_init_app_uses_configured_secret_key(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
secret_key = "configured-secret-key"
|
||||
storage = InMemoryStorage()
|
||||
apply_config_overrides(monkeypatch, SECRET_KEY=secret_key)
|
||||
monkeypatch.setattr("configs.secret_key.storage", storage)
|
||||
app = DifyApp(__name__)
|
||||
app.config["SECRET_KEY"] = secret_key
|
||||
|
||||
ext_set_secretkey.init_app(app)
|
||||
|
||||
assert app.secret_key == secret_key
|
||||
assert app.config["SECRET_KEY"] == secret_key
|
||||
assert storage.saved_files == []
|
||||
|
||||
|
||||
def test_init_app_generates_and_persists_secret_key_when_missing(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
storage = InMemoryStorage()
|
||||
apply_config_overrides(monkeypatch, SECRET_KEY="")
|
||||
monkeypatch.setattr("configs.secret_key.storage", storage)
|
||||
app = DifyApp(__name__)
|
||||
app.config["SECRET_KEY"] = ""
|
||||
|
||||
ext_set_secretkey.init_app(app)
|
||||
|
||||
persisted_key = storage.files[".dify_secret_key"].decode("utf-8").strip()
|
||||
assert persisted_key
|
||||
assert storage.saved_files == [(".dify_secret_key", f"{persisted_key}\n".encode())]
|
||||
assert persisted_key == ext_set_secretkey.dify_config.SECRET_KEY
|
||||
assert persisted_key == app.config["SECRET_KEY"]
|
||||
assert persisted_key == app.secret_key
|
||||
|
||||
|
||||
def test_init_app_reuses_persisted_secret_key_when_missing(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
persisted_key = "persisted-secret-key"
|
||||
storage = InMemoryStorage({".dify_secret_key": f"{persisted_key}\n".encode()})
|
||||
apply_config_overrides(monkeypatch, SECRET_KEY="")
|
||||
monkeypatch.setattr("configs.secret_key.storage", storage)
|
||||
app = DifyApp(__name__)
|
||||
app.config["SECRET_KEY"] = ""
|
||||
|
||||
ext_set_secretkey.init_app(app)
|
||||
|
||||
assert persisted_key == ext_set_secretkey.dify_config.SECRET_KEY
|
||||
assert persisted_key == app.config["SECRET_KEY"]
|
||||
assert persisted_key == app.secret_key
|
||||
assert storage.saved_files == []
|
||||
@@ -0,0 +1,54 @@
|
||||
import ssl
|
||||
from collections.abc import Callable
|
||||
|
||||
import socketio
|
||||
|
||||
from configs import dify_config
|
||||
from extensions import ext_socketio
|
||||
|
||||
|
||||
def test_socketio_server_uses_redis_manager() -> None:
|
||||
assert isinstance(ext_socketio.sio.manager, socketio.RedisManager)
|
||||
|
||||
|
||||
def test_create_socketio_client_manager_uses_pubsub_url_and_prefixed_channel(
|
||||
config_overrides: Callable[..., None],
|
||||
) -> None:
|
||||
config_overrides(PUBSUB_REDIS_URL="redis://redis.example.com:6380/3", REDIS_KEY_PREFIX="tenant-a")
|
||||
|
||||
manager = ext_socketio.create_socketio_client_manager()
|
||||
|
||||
assert manager.redis_url == "redis://redis.example.com:6380/3"
|
||||
assert manager.channel == "tenant-a:socketio"
|
||||
|
||||
|
||||
def test_build_redis_options_includes_tls_options_for_rediss(config_overrides: Callable[..., None]) -> None:
|
||||
config_overrides(
|
||||
REDIS_SSL_CERT_REQS="CERT_REQUIRED",
|
||||
REDIS_SSL_CA_CERTS="/ca.pem",
|
||||
REDIS_SSL_CERTFILE="/cert.pem",
|
||||
REDIS_SSL_KEYFILE="/key.pem",
|
||||
)
|
||||
|
||||
options = ext_socketio._build_redis_options("rediss://redis.example.com:6380/3")
|
||||
|
||||
assert options["ssl_cert_reqs"] == ssl.CERT_REQUIRED
|
||||
assert options["ssl_ca_certs"] == "/ca.pem"
|
||||
assert options["ssl_certfile"] == "/cert.pem"
|
||||
assert options["ssl_keyfile"] == "/key.pem"
|
||||
|
||||
|
||||
def test_build_redis_options_omits_socket_timeout(config_overrides: Callable[..., None]) -> None:
|
||||
# socket_timeout must not be passed to RedisManager because the pub/sub
|
||||
# listen loop blocks indefinitely between messages; a read timeout there
|
||||
# triggers an infinite reconnect storm (issue #39423).
|
||||
config_overrides(REDIS_SOCKET_TIMEOUT=5.0)
|
||||
|
||||
options = ext_socketio._build_redis_options("redis://redis.example.com:6380/3")
|
||||
|
||||
assert "socket_timeout" not in options
|
||||
assert "socket_connect_timeout" in options
|
||||
|
||||
|
||||
def test_socketio_server_uses_configured_max_http_buffer_size() -> None:
|
||||
assert ext_socketio.sio.eio.max_http_buffer_size == dify_config.WEBSOCKET_MAX_HTTP_BUFFER_SIZE
|
||||
@@ -0,0 +1,186 @@
|
||||
from typing import TYPE_CHECKING, Literal, override
|
||||
|
||||
from dify_agent.layers.execution_context import ApplicationTracingContext
|
||||
from dify_trace_sdk.emitter import TraceScope
|
||||
from dify_trace_sdk.events import TraceCorrelation, TraceEvent
|
||||
from flask import Flask
|
||||
|
||||
from core.helper.trace_id_helper import ParentTraceContext
|
||||
from extensions.tracing import (
|
||||
ApplicationTracingHooks,
|
||||
ExecutionTraceScopes,
|
||||
application_tracing_hooks,
|
||||
bind_tracing,
|
||||
correlation_from_extras,
|
||||
create_trace_scope,
|
||||
create_trace_scope_for_app,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from core.app.apps.agent_app.runtime_request_builder import AgentAppRuntimeBuildContext
|
||||
from core.app.entities.app_invoke_entities import AdvancedChatAppGenerateEntity, WorkflowAppGenerateEntity
|
||||
from core.workflow.nodes.agent_v2.runtime_request_builder import WorkflowAgentRuntimeBuildContext
|
||||
|
||||
|
||||
class _Recorder:
|
||||
def __init__(self) -> None:
|
||||
self.events: list[TraceEvent] = []
|
||||
|
||||
def emit(self, event: TraceEvent) -> None:
|
||||
self.events.append(event)
|
||||
|
||||
|
||||
class _Scopes(ExecutionTraceScopes):
|
||||
def __init__(self, *, fails: bool) -> None:
|
||||
self.fails = fails
|
||||
self.policies: list[str | None] = []
|
||||
self.correlations: list[TraceCorrelation] = []
|
||||
|
||||
def _scope(
|
||||
self, *, tenant_id: str, app_id: str | None, user_id: str | None, correlation: TraceCorrelation
|
||||
) -> TraceScope:
|
||||
if self.fails:
|
||||
raise RuntimeError("policy lookup failed")
|
||||
self.correlations.append(correlation)
|
||||
return TraceScope(
|
||||
emitter=_Recorder(), tenant_id=tenant_id, app_id=app_id, user_id=user_id, correlation=correlation
|
||||
)
|
||||
|
||||
@override
|
||||
def for_execution(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
app_id: str | None,
|
||||
app_tracing: str | None,
|
||||
user_id: str | None,
|
||||
correlation: TraceCorrelation,
|
||||
) -> TraceScope | None:
|
||||
self.policies.append(app_tracing)
|
||||
return self._scope(tenant_id=tenant_id, app_id=app_id, user_id=user_id, correlation=correlation)
|
||||
|
||||
@override
|
||||
def for_app(
|
||||
self, *, tenant_id: str, app_id: str | None, user_id: str | None, correlation: TraceCorrelation
|
||||
) -> TraceScope | None:
|
||||
return self._scope(tenant_id=tenant_id, app_id=app_id, user_id=user_id, correlation=correlation)
|
||||
|
||||
|
||||
class _Hooks(ApplicationTracingHooks):
|
||||
@override
|
||||
def prepare_agent_app(self, context: "AgentAppRuntimeBuildContext") -> ApplicationTracingContext | None:
|
||||
return None
|
||||
|
||||
@override
|
||||
def prepare_workflow_agent(
|
||||
self, context: "WorkflowAgentRuntimeBuildContext", conversation_id: str | None
|
||||
) -> ApplicationTracingContext | None:
|
||||
return None
|
||||
|
||||
@override
|
||||
def freeze_workflow_run(
|
||||
self,
|
||||
entity: "AdvancedChatAppGenerateEntity | WorkflowAppGenerateEntity",
|
||||
*,
|
||||
workflow_id: str,
|
||||
workflow_run_id: str,
|
||||
conversation_id: object,
|
||||
graph_data: object,
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
@override
|
||||
def capture_agent_app_terminal(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
app_id: str,
|
||||
message_id: str,
|
||||
outcome: Literal["succeeded", "failed", "cancelled"],
|
||||
output: str | None,
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _app(scopes: _Scopes, hooks: ApplicationTracingHooks | None) -> Flask:
|
||||
app = Flask(__name__)
|
||||
bind_tracing(app, scopes=scopes, hooks=hooks)
|
||||
return app
|
||||
|
||||
|
||||
def test_correlation_reads_the_caller_extras() -> None:
|
||||
nested = correlation_from_extras(
|
||||
{
|
||||
"external_trace_id": "caller-trace",
|
||||
"trace_session_id": "session-1",
|
||||
"parent_trace_context": {"parent_workflow_run_id": "outer-run", "parent_node_execution_id": "outer-node"},
|
||||
}
|
||||
)
|
||||
typed = correlation_from_extras(
|
||||
{"parent_trace_context": ParentTraceContext(parent_workflow_run_id="outer-run", parent_node_execution_id=None)}
|
||||
)
|
||||
|
||||
assert nested == TraceCorrelation(
|
||||
external_trace_id="caller-trace",
|
||||
session_id="session-1",
|
||||
parent_workflow_run_id="outer-run",
|
||||
parent_node_execution_id="outer-node",
|
||||
)
|
||||
assert typed.parent_workflow_run_id == "outer-run"
|
||||
assert correlation_from_extras({"external_trace_id": "", "parent_trace_context": {"x": 1}}) == TraceCorrelation(
|
||||
external_trace_id=None, session_id=None, parent_workflow_run_id=None, parent_node_execution_id=None
|
||||
)
|
||||
|
||||
|
||||
def test_scopes_come_from_the_bound_scopes_with_the_raw_policy_and_correlation() -> None:
|
||||
scopes = _Scopes(fails=False)
|
||||
with _app(scopes, None).app_context():
|
||||
scope = create_trace_scope(
|
||||
tenant_id="tenant-1",
|
||||
app_id="app-1",
|
||||
app_tracing='{"enabled": true}',
|
||||
user_id="u",
|
||||
extras={"external_trace_id": "caller-trace"},
|
||||
)
|
||||
app_scope = create_trace_scope_for_app(tenant_id="tenant-1", app_id=None, user_id=None, extras={})
|
||||
|
||||
assert scope is not None
|
||||
assert (scope.tenant_id, scope.app_id, scope.user_id) == ("tenant-1", "app-1", "u")
|
||||
assert scopes.policies == ['{"enabled": true}']
|
||||
assert scope.correlation.external_trace_id == "caller-trace"
|
||||
assert app_scope is not None
|
||||
assert app_scope.app_id is None
|
||||
|
||||
|
||||
def test_scopes_fail_open() -> None:
|
||||
assert create_trace_scope(tenant_id="tenant-1", app_id="app-1", app_tracing=None, user_id=None, extras={}) is None
|
||||
with _app(_Scopes(fails=True), None).app_context():
|
||||
assert (
|
||||
create_trace_scope(tenant_id="tenant-1", app_id="app-1", app_tracing=None, user_id=None, extras={}) is None
|
||||
)
|
||||
assert create_trace_scope_for_app(tenant_id="tenant-1", app_id="app-1", user_id=None, extras={}) is None
|
||||
with Flask(__name__).app_context():
|
||||
assert create_trace_scope_for_app(tenant_id="tenant-1", app_id="app-1", user_id=None, extras={}) is None
|
||||
|
||||
|
||||
def test_agent_hooks_are_the_ones_the_composition_root_bound() -> None:
|
||||
hooks = _Hooks()
|
||||
|
||||
with _app(_Scopes(fails=False), hooks).app_context():
|
||||
assert application_tracing_hooks() is hooks
|
||||
with _app(_Scopes(fails=False), None).app_context():
|
||||
assert application_tracing_hooks() is None
|
||||
assert application_tracing_hooks() is None
|
||||
|
||||
|
||||
def test_invalid_extension_bindings_fail_open() -> None:
|
||||
app = Flask(__name__)
|
||||
app.extensions["trace_scopes"] = object()
|
||||
app.extensions["application_tracing_hooks"] = object()
|
||||
|
||||
with app.app_context():
|
||||
assert (
|
||||
create_trace_scope(tenant_id="tenant-1", app_id="app-1", app_tracing=None, user_id=None, extras={}) is None
|
||||
)
|
||||
assert create_trace_scope_for_app(tenant_id="tenant-1", app_id="app-1", user_id=None, extras={}) is None
|
||||
assert application_tracing_hooks() is None
|
||||
@@ -0,0 +1,130 @@
|
||||
import logging
|
||||
from collections.abc import Generator
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from core.app.apps.workflow.active_workflow_tasks import reset_active_workflow_tasks
|
||||
from core.app.apps.workflow.command_channels import CelerySignalCommandChannel
|
||||
from extensions import workflow_warm_shutdown
|
||||
from graphon.engine.command import AbortCommand
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_warm_shutdown_state() -> Generator[None]:
|
||||
reset_active_workflow_tasks()
|
||||
workflow_warm_shutdown._celery_warm_shutdown_started.clear()
|
||||
yield
|
||||
reset_active_workflow_tasks()
|
||||
workflow_warm_shutdown._celery_warm_shutdown_started.clear()
|
||||
|
||||
|
||||
def _create_warm_shutdown_command_channel() -> CelerySignalCommandChannel:
|
||||
return CelerySignalCommandChannel(
|
||||
shutdown_state_getter=workflow_warm_shutdown.celery_warm_shutdown_started,
|
||||
abort_reason=workflow_warm_shutdown.WORKFLOW_WARM_SHUTDOWN_ABORT_REASON,
|
||||
)
|
||||
|
||||
|
||||
def test_worker_shutting_down_skips_non_warm_shutdown(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
mark_shutdown = MagicMock()
|
||||
monkeypatch.setattr(workflow_warm_shutdown, "mark_celery_warm_shutdown_started", mark_shutdown)
|
||||
|
||||
workflow_warm_shutdown._on_worker_shutting_down(how="cold")
|
||||
|
||||
mark_shutdown.assert_not_called()
|
||||
|
||||
|
||||
def test_worker_shutting_down_marks_warm_shutdown(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
mark_shutdown = MagicMock()
|
||||
monkeypatch.setattr(workflow_warm_shutdown, "mark_celery_warm_shutdown_started", mark_shutdown)
|
||||
monkeypatch.setattr(workflow_warm_shutdown, "get_active_workflow_task_count", lambda: 2)
|
||||
|
||||
workflow_warm_shutdown._on_worker_shutting_down(how="warm")
|
||||
|
||||
mark_shutdown.assert_called_once_with()
|
||||
|
||||
|
||||
def test_warm_shutdown_state_tracks_started_flag() -> None:
|
||||
assert workflow_warm_shutdown.celery_warm_shutdown_started() is False
|
||||
|
||||
workflow_warm_shutdown.mark_celery_warm_shutdown_started()
|
||||
|
||||
assert workflow_warm_shutdown.celery_warm_shutdown_started() is True
|
||||
|
||||
|
||||
def test_setup_configures_warm_shutdown_command_channel(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(workflow_warm_shutdown.worker_shutting_down, "connect", MagicMock())
|
||||
monkeypatch.setattr(workflow_warm_shutdown.worker_shutdown, "connect", MagicMock())
|
||||
|
||||
workflow_warm_shutdown.setup_workflow_warm_shutdown_handler()
|
||||
workflow_warm_shutdown.mark_celery_warm_shutdown_started()
|
||||
|
||||
commands = _create_warm_shutdown_command_channel().fetch_commands()
|
||||
|
||||
assert len(commands) == 1
|
||||
assert isinstance(commands[0], AbortCommand)
|
||||
assert commands[0].reason == workflow_warm_shutdown.WORKFLOW_WARM_SHUTDOWN_ABORT_REASON
|
||||
|
||||
|
||||
def test_warm_shutdown_command_stays_available_for_late_channels(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(workflow_warm_shutdown.worker_shutting_down, "connect", MagicMock())
|
||||
monkeypatch.setattr(workflow_warm_shutdown.worker_shutdown, "connect", MagicMock())
|
||||
|
||||
workflow_warm_shutdown.setup_workflow_warm_shutdown_handler()
|
||||
workflow_warm_shutdown.mark_celery_warm_shutdown_started()
|
||||
|
||||
first_channel = _create_warm_shutdown_command_channel()
|
||||
late_channel = _create_warm_shutdown_command_channel()
|
||||
|
||||
assert len(first_channel.fetch_commands()) == 1
|
||||
assert len(late_channel.fetch_commands()) == 1
|
||||
|
||||
|
||||
def test_worker_shutdown_logs_when_all_workflow_runs_ended(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
caplog.set_level(logging.INFO, logger=workflow_warm_shutdown.logger.name)
|
||||
monkeypatch.setattr(workflow_warm_shutdown, "get_active_workflow_task_count", lambda: 0)
|
||||
|
||||
workflow_warm_shutdown._on_worker_shutdown()
|
||||
|
||||
assert "after all tracked workflow runs ended" in caplog.text
|
||||
|
||||
|
||||
def test_worker_shutdown_logs_remaining_workflow_runs(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
caplog.set_level(logging.INFO, logger=workflow_warm_shutdown.logger.name)
|
||||
monkeypatch.setattr(workflow_warm_shutdown, "get_active_workflow_task_count", lambda: 2)
|
||||
|
||||
workflow_warm_shutdown._on_worker_shutdown()
|
||||
|
||||
assert "with 2 workflow run(s) still active after warm shutdown wait" in caplog.text
|
||||
|
||||
|
||||
def test_setup_connects_shutdown_handlers(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
connect_shutting_down = MagicMock()
|
||||
connect_shutdown = MagicMock()
|
||||
monkeypatch.setattr(workflow_warm_shutdown.worker_shutting_down, "connect", connect_shutting_down)
|
||||
monkeypatch.setattr(workflow_warm_shutdown.worker_shutdown, "connect", connect_shutdown)
|
||||
|
||||
workflow_warm_shutdown.setup_workflow_warm_shutdown_handler()
|
||||
|
||||
connect_shutting_down.assert_called_once()
|
||||
connect_shutdown.assert_called_once()
|
||||
|
||||
|
||||
def test_setup_preserves_warm_shutdown_state(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(workflow_warm_shutdown.worker_shutting_down, "connect", MagicMock())
|
||||
monkeypatch.setattr(workflow_warm_shutdown.worker_shutdown, "connect", MagicMock())
|
||||
|
||||
workflow_warm_shutdown.mark_celery_warm_shutdown_started()
|
||||
workflow_warm_shutdown.setup_workflow_warm_shutdown_handler()
|
||||
|
||||
commands = _create_warm_shutdown_command_channel().fetch_commands()
|
||||
|
||||
assert workflow_warm_shutdown.celery_warm_shutdown_started() is True
|
||||
assert len(commands) == 1
|
||||
Reference in new issue
Block a user