chore: 纳入本地 Dify 1.19.0 源码及品牌与免登改动

演示目录只保留可重建的源码。运行密钥、数据卷、依赖和带口令的本地配置不进入仓库。

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
wuzilongandCursor committed 2026-11-18 13:59:25 +08:00
1 parent 60158935f4
commit 6519682885
15831 files changed
+3063428

No files matched your search

Whitespace-only changes.
@@ -0,0 +1,179 @@
"""Exercise Celery entrypoints against durable application trace state."""
from dataclasses import replace
from datetime import UTC, datetime, timedelta
from uuid import UUID, uuid4
import httpx
import pytest
from dify_trace_sdk.events import MessageCompleted, TraceCorrelation, TraceEvent
from flask import Flask
from sqlalchemy.orm import Session, sessionmaker
from core.helper import ssrf_proxy
from domain.tracing.trace_run_agg import TraceRunState
from extensions.application_services.tracing import build_tracing_services
from extensions.ext_application_services import ApplicationServices
from libs.datetime_utils import naive_utc_now
from models.application_trace import ApplicationTraceRun
from models.enums import ConversationFromSource
from models.model import AppMode, Message
from repositories.factory import DifyAPIRepositoryFactory
from repositories.trace_record_repository import TraceRecordRepository
from services.tracing.canonical import CanonicalTraceBuilder
from services.tracing.collection import AgentTraceSettings
from services.tracing.egress import SSRFEgress
from services.tracing.providers import ProviderConfigCodec
from services.tracing.records import MessageTraceInfo
from services.tracing.registry import EntryPointTraceProviderCatalog
from services.tracing.service import ApplicationTracingService
from tasks.application_trace_task import export_application_trace, reconcile_application_traces
from tests.helpers.unit.model_factories import make_conversation, make_tenant
from tests.helpers.unit.services.tracing_ports import AGENT_TRACE_SETTINGS, WorkspaceBoundCipher
from tests.unit_tests.repositories.test_application_trace_repository import (
Collection,
batch_for,
ingest,
seed,
terminal,
)
def bind_worker(
app: Flask,
services: ApplicationServices,
collection: Collection,
*,
settings: AgentTraceSettings,
scheduled: list[str],
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Hand the tasks a worker over the seeded ledger with the installed providers and the SSRF egress."""
worker = ApplicationTracingService(
runs=collection.repository,
run_query=collection.repository,
sources=collection.repository,
records=TraceRecordRepository(
collection.sessions,
workflow_runs=DifyAPIRepositoryFactory.create_api_workflow_run_repository(collection.sessions),
node_executions=DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository(collection.sessions),
files_url="https://files.test",
enterprise_details=False,
),
builder=CanonicalTraceBuilder(),
providers=EntryPointTraceProviderCatalog(),
codec=ProviderConfigCodec(WorkspaceBoundCipher()),
egress=SSRFEgress(),
schedule_export=scheduled.append,
settings=settings,
)
monkeypatch.setitem(
app.extensions,
"application_services",
replace(services, tracing=replace(services.tracing, agent_traces=worker)),
)
def test_worker_task_exports_once_through_the_ssrf_proxy_and_replay_observes_checkpoint(
app: Flask,
account_application_services: ApplicationServices,
sqlite_session_factory: sessionmaker[Session],
monkeypatch: pytest.MonkeyPatch,
) -> None:
case = seed(sqlite_session_factory, "worker-task")
ingest(case, batch_for(case.run, case.invocation()))
terminal(case)
bind_worker(
app,
account_application_services,
case,
settings=AGENT_TRACE_SETTINGS,
scheduled=[],
monkeypatch=monkeypatch,
)
proxied: list[tuple[str, str]] = []
def send(method: str, url: str, **_kwargs: object) -> httpx.Response:
proxied.append((method, url))
return httpx.Response(200, json={}, request=httpx.Request(method, url))
monkeypatch.setattr(ssrf_proxy, "make_request", send)
export_application_trace.run(case.run.id)
export_application_trace.run(case.run.id)
assert proxied == [("POST", "https://langfuse.test/api/public/ingestion")]
result = case.repository.get(case.run.id)
assert result is not None
assert result.state is TraceRunState.ACCEPTED
assert result.attempts == 1
provider_trace_id = str(UUID(hex=case.run.trace_id))
assert result.provider_trace_id == provider_trace_id
assert result.trace_url == f"https://langfuse.test/project/original/traces/{provider_trace_id}"
def test_worker_records_carry_no_enterprise_names(sqlite_session_factory: sessionmaker[Session]) -> None:
case = seed(sqlite_session_factory, "worker-names")
with case.sessions.begin() as session:
message = session.get(Message, case.run.source_id)
assert message is not None
conversation_id = message.conversation_id
session.add(
make_conversation(
conversation_id=conversation_id,
app_id=case.run.app_id,
mode=AppMode.AGENT,
inputs={},
from_source=ConversationFromSource.CONSOLE,
)
)
session.add(make_tenant(tenant_id=case.run.tenant_id))
# The tasks reach this Agent trace service through the composition root; its records feed recovery.
worker = build_tracing_services(database_client=sqlite_session_factory).agent_traces
record = worker._records.assemble(
TraceEvent(
event_id=uuid4().hex,
tenant_id=case.run.tenant_id,
app_id=case.run.app_id,
user_id=None,
occurred_at=datetime.now(UTC),
correlation=TraceCorrelation(
external_trace_id=None, session_id=None, parent_workflow_run_id=None, parent_node_execution_id=None
),
payload=MessageCompleted(message_id=case.run.source_id, conversation_id=conversation_id),
)
)
assert isinstance(record, MessageTraceInfo)
assert (record.metadata["app_name"], record.metadata["workspace_name"]) == ("", "")
def test_reconcile_only_schedules_due_pending_rows_and_applies_retention(
app: Flask,
account_application_services: ApplicationServices,
sqlite_session_factory: sessionmaker[Session],
monkeypatch: pytest.MonkeyPatch,
) -> None:
due = seed(sqlite_session_factory, "due")
deferred = seed(sqlite_session_factory, "future")
accepted = seed(sqlite_session_factory, "accepted-expired")
with sqlite_session_factory.begin() as session:
future = session.get(ApplicationTraceRun, deferred.run.id)
expired = session.get(ApplicationTraceRun, accepted.run.id)
assert future is not None
assert expired is not None
future.state = TraceRunState.RETRYING
future.next_attempt_at = naive_utc_now() + timedelta(hours=1)
expired.state = TraceRunState.ACCEPTED
expired.created_at = naive_utc_now() - timedelta(days=4)
scheduled: list[str] = []
bind_worker(
app,
account_application_services,
due,
settings=replace(AGENT_TRACE_SETTINGS, retention_days=3),
scheduled=scheduled,
monkeypatch=monkeypatch,
)
reconcile_application_traces.run()
assert scheduled == [due.run.id]
assert due.repository.get(accepted.run.id) is None
assert due.repository.get(deferred.run.id) is not None
assert due.repository.get(due.run.id) is not None
@@ -0,0 +1,88 @@
import ast
import inspect
from services.errors.app import WorkflowNotFoundError
from tasks import async_workflow_tasks
def test_workflow_not_found_error_message_is_a_single_formatted_string() -> None:
# Regression: previously raised with printf-style positional args
# ("...%s...", id1, id2), which made Exception store them as a tuple
# and str(exc) rendered the literal tuple instead of a readable message.
workflow_run_id = "wr-123"
workflow_id = "wf-456"
exc = WorkflowNotFoundError(f"Workflow not found: workflow_run_id={workflow_run_id}, workflow_id={workflow_id}")
# The exception must stringify to the fully-interpolated message and must
# not contain unresolved %s placeholders nor tuple punctuation.
rendered = str(exc)
assert rendered == "Workflow not found: workflow_run_id=wr-123, workflow_id=wf-456"
assert "%s" not in rendered
assert not rendered.startswith("(")
assert exc.args == ("Workflow not found: workflow_run_id=wr-123, workflow_id=wf-456",)
def test_app_not_found_error_message_is_a_single_formatted_string() -> None:
# Same regression check for the _AppNotFoundError raise site in the same
# function (resume_workflow_execution).
app_id = "app-789"
workflow_run_id = "wr-123"
exc = async_workflow_tasks._AppNotFoundError(f"App not found: app_id={app_id}, workflow_run_id={workflow_run_id}")
rendered = str(exc)
assert rendered == "App not found: app_id=app-789, workflow_run_id=wr-123"
assert "%s" not in rendered
assert not rendered.startswith("(")
assert exc.args == ("App not found: app_id=app-789, workflow_run_id=wr-123",)
def _collect_raise_calls_in(func_name: str) -> list[ast.Call]:
"""Walk the AST of async_workflow_tasks and return every Call inside a Raise
inside the function named ``func_name``.
This lets us assert at unit-test time that no raise site in a critical
function passes printf-style positional args to an Exception constructor
(which would otherwise be stored verbatim in ``exc.args`` and stringify
as a tuple-repr instead of a human-readable message).
"""
source = inspect.getsource(async_workflow_tasks)
tree = ast.parse(source)
raises: list[ast.Call] = []
for node in ast.walk(tree):
if isinstance(node, ast.FunctionDef) and node.name == func_name:
for sub in ast.walk(node):
if isinstance(sub, ast.Raise) and isinstance(sub.exc, ast.Call):
raises.append(sub.exc)
return raises
def test_resume_workflow_execution_raises_do_not_use_printf_positional_args() -> None:
"""Regression: ``resume_workflow_execution`` previously raised
``WorkflowNotFoundError("Workflow not found: workflow_run_id=%s, ...", run_id, wf_id)``
and ``_AppNotFoundError("App not found: app_id=%s, ...", app_id, run_id)``.
Python's ``Exception.__init__`` then stored all three as ``exc.args`` and
``str(exc)`` produced ``("Workflow not found: ...%s...", 'id1', 'id2')``
rather than a usable message.
Lock this in by walking the function's AST: every ``raise SomeError(...)``
inside ``resume_workflow_execution`` must pass exactly one positional
argument (the rendered message), or be a no-arg raise. If anyone reverts
to the printf pattern, this test fails fast at unit-test time.
"""
raise_calls = _collect_raise_calls_in("resume_workflow_execution")
assert raise_calls, "Expected raise sites inside resume_workflow_execution to inspect"
for call in raise_calls:
assert len(call.args) <= 1, (
f"raise {ast.unparse(call)} passes {len(call.args)} positional args; "
"Exception classes accept a single rendered message. Use an f-string "
"instead of printf-style ('...%s...', val1, val2)."
)
# Also reject %s in a literal first arg as belt-and-braces.
if call.args and isinstance(call.args[0], ast.Constant) and isinstance(call.args[0].value, str):
assert "%s" not in call.args[0].value, (
f"raise {ast.unparse(call)} still contains '%s' in its message; use an f-string."
)
@@ -0,0 +1,214 @@
import uuid
from datetime import UTC, datetime, timedelta
from unittest.mock import patch
import pytest
from sqlalchemy.orm import Session
import tasks.batch_clean_document_task as task_module
from extensions.storage.storage_type import StorageType
from models.dataset import Dataset, DocumentSegment, SegmentAttachmentBinding
from models.enums import CreatorUserRole, DataSourceType
from models.knowledge_fs import KnowledgeFSUpgradeFileLease, KnowledgeFSUpgradeJob
from models.model import UploadFile
from tasks.batch_clean_document_task import batch_clean_document_task
@pytest.fixture
def cleanup_rows(sqlite_session: Session, monkeypatch: pytest.MonkeyPatch) -> tuple[str, str, str]:
tenant_id = str(uuid.uuid4())
dataset_id = str(uuid.uuid4())
document_id = str(uuid.uuid4())
created_by = str(uuid.uuid4())
dataset = Dataset(
id=dataset_id,
tenant_id=tenant_id,
name="Batch cleanup dataset",
data_source_type=DataSourceType.UPLOAD_FILE,
created_by=created_by,
)
segment = DocumentSegment(
tenant_id=tenant_id,
dataset_id=dataset_id,
document_id=document_id,
position=1,
content="content",
word_count=1,
tokens=1,
created_by=created_by,
index_node_id="node-1",
)
sqlite_session.add_all([dataset, segment])
sqlite_session.commit()
engine = sqlite_session.get_bind()
monkeypatch.setattr(
task_module.session_factory,
"create_session",
lambda: Session(engine, expire_on_commit=False),
)
return dataset_id, document_id, tenant_id
def test_successful_vector_cleanup_schedules_billing_refresh(cleanup_rows: tuple[str, str, str]) -> None:
dataset_id, document_id, tenant_id = cleanup_rows
with (
patch("tasks.batch_clean_document_task.get_image_upload_file_ids", return_value=[]),
patch("tasks.batch_clean_document_task.clean_document_indexes", return_value=tenant_id) as index_cleanup,
patch("tasks.batch_clean_document_task.schedule_billing_vector_space_refresh") as schedule_refresh,
):
batch_clean_document_task(
document_ids=[document_id],
dataset_id=dataset_id,
doc_form="paragraph",
file_ids=[],
)
index_cleanup.assert_called_once()
schedule_refresh.assert_called_once_with(tenant_id)
def test_failed_vector_cleanup_does_not_schedule_billing_refresh(cleanup_rows: tuple[str, str, str]) -> None:
dataset_id, document_id, _tenant_id = cleanup_rows
with (
patch("tasks.batch_clean_document_task.get_image_upload_file_ids", return_value=[]),
patch("tasks.batch_clean_document_task.clean_document_indexes") as index_cleanup,
patch("tasks.batch_clean_document_task.schedule_billing_vector_space_refresh") as schedule_refresh,
):
index_cleanup.side_effect = RuntimeError("vector cleanup failed")
batch_clean_document_task(
document_ids=[document_id],
dataset_id=dataset_id,
doc_form="paragraph",
file_ids=[],
)
schedule_refresh.assert_not_called()
def test_batch_cleanup_keeps_only_the_leased_legacy_source_file(
cleanup_rows: tuple[str, str, str], sqlite_session: Session
) -> None:
dataset_id, document_id, tenant_id = cleanup_rows
account_id = str(uuid.uuid4())
now = datetime.now(UTC).replace(tzinfo=None)
leased_file = UploadFile(
tenant_id=tenant_id,
storage_type=StorageType.LOCAL,
key=f"upload_files/{tenant_id}/leased.txt",
name="leased.txt",
size=10,
extension="txt",
mime_type="text/plain",
created_by_role=CreatorUserRole.ACCOUNT,
created_by=account_id,
created_at=now,
used=False,
)
deletable_file = UploadFile(
tenant_id=tenant_id,
storage_type=StorageType.LOCAL,
key=f"upload_files/{tenant_id}/deletable.txt",
name="deletable.txt",
size=10,
extension="txt",
mime_type="text/plain",
created_by_role=CreatorUserRole.ACCOUNT,
created_by=account_id,
created_at=now,
used=False,
)
job = KnowledgeFSUpgradeJob(
tenant_id=tenant_id,
old_dataset_id=dataset_id,
requested_by_account_id=account_id,
owner_account_id=account_id,
idempotency_key="batch-cleanup-lease-test",
snapshot_at=now,
config_snapshot={},
permission_snapshot={},
app_binding_snapshot=[],
tag_ids_snapshot=[],
)
sqlite_session.add_all([leased_file, deletable_file, job])
sqlite_session.flush()
sqlite_session.add(
KnowledgeFSUpgradeFileLease(
job_id=job.id,
old_upload_file_id=leased_file.id,
expires_at=now + timedelta(hours=1),
)
)
sqlite_session.commit()
leased_file_id = leased_file.id
deletable_file_id = deletable_file.id
deletable_file_key = deletable_file.key
with (
patch("tasks.batch_clean_document_task.get_image_upload_file_ids", return_value=[]),
patch("tasks.batch_clean_document_task.clean_document_indexes", return_value=tenant_id),
patch("tasks.batch_clean_document_task.schedule_billing_vector_space_refresh"),
patch("tasks.batch_clean_document_task.storage") as storage,
):
batch_clean_document_task(
document_ids=[document_id],
dataset_id=dataset_id,
doc_form="paragraph",
file_ids=[leased_file_id, deletable_file_id],
)
sqlite_session.expire_all()
assert sqlite_session.get(UploadFile, leased_file_id) is not None
assert sqlite_session.get(UploadFile, deletable_file_id) is None
storage.delete.assert_called_once_with(deletable_file_key)
def test_cleans_segment_attachment_bindings_and_files(
cleanup_rows: tuple[str, str, str], sqlite_session: Session
) -> None:
dataset_id, document_id, tenant_id = cleanup_rows
segment = sqlite_session.query(DocumentSegment).filter_by(document_id=document_id).one()
attachment = UploadFile(
tenant_id=tenant_id,
storage_type=StorageType.LOCAL,
key="attachments/image.png",
name="image.png",
size=10,
extension="png",
mime_type="image/png",
created_by_role=CreatorUserRole.ACCOUNT,
created_by=segment.created_by,
created_at=datetime.now(UTC),
used=True,
)
binding = SegmentAttachmentBinding(
tenant_id=tenant_id,
dataset_id=dataset_id,
document_id=document_id,
segment_id=segment.id,
attachment_id=attachment.id,
)
sqlite_session.add_all([attachment, binding])
sqlite_session.commit()
attachment_id = attachment.id
attachment_key = attachment.key
binding_id = binding.id
with (
patch("tasks.batch_clean_document_task.get_image_upload_file_ids", return_value=[]),
patch("tasks.batch_clean_document_task.clean_document_indexes"),
patch("tasks.batch_clean_document_task.schedule_billing_vector_space_refresh"),
patch("tasks.batch_clean_document_task.storage.delete") as storage_delete,
):
batch_clean_document_task(
document_ids=[document_id],
dataset_id=dataset_id,
doc_form="paragraph",
file_ids=[],
)
sqlite_session.expire_all()
assert sqlite_session.get(SegmentAttachmentBinding, binding_id) is None
assert sqlite_session.get(UploadFile, attachment_id) is None
storage_delete.assert_called_once_with(attachment_key)
@@ -0,0 +1,140 @@
from datetime import UTC, datetime
from pathlib import Path
from unittest.mock import patch
from uuid import uuid4
import pytest
from sqlalchemy import event, select
from sqlalchemy.engine import Engine
from sqlalchemy.orm import Session
from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType
from extensions.storage.storage_type import StorageType
from models.dataset import Dataset, Document, DocumentSegment
from models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, IndexingStatus, SegmentStatus
from models.model import UploadFile
from services.knowledge.resource_scope import DatasetRef, DocumentRef
from tasks.batch_create_segment_to_index_task import batch_create_segment_to_index_task
@pytest.mark.parametrize("foreign_owner", [None, "dataset", "document_tenant", "document_dataset", "upload"])
def test_batch_create_segment_to_index_task_validates_scope_and_indexes_committed_batch(
sqlite_session: Session,
sqlite_engine: Engine,
foreign_owner: str | None,
) -> None:
tenant_id = str(uuid4())
user_id = str(uuid4())
dataset = Dataset(
id=str(uuid4()),
tenant_id=tenant_id,
name="Batch import dataset",
data_source_type=DataSourceType.UPLOAD_FILE,
created_by=user_id,
indexing_technique=IndexTechniqueType.ECONOMY,
)
document = Document(
id=str(uuid4()),
tenant_id=tenant_id,
dataset_id=dataset.id,
position=1,
data_source_type=DataSourceType.UPLOAD_FILE,
batch="batch-1",
name="document.txt",
created_from=DocumentCreatedFrom.WEB,
created_by=user_id,
indexing_status=IndexingStatus.COMPLETED,
enabled=True,
archived=False,
doc_form=IndexStructureType.PARAGRAPH_INDEX,
word_count=0,
)
upload_file = UploadFile(
tenant_id=tenant_id,
storage_type=StorageType.LOCAL,
key="batch-import.csv",
name="batch-import.csv",
size=32,
extension=".csv",
mime_type="text/csv",
created_by_role=CreatorUserRole.ACCOUNT,
created_by=user_id,
created_at=datetime.now(UTC),
used=False,
)
existing_segment = DocumentSegment(
tenant_id=tenant_id,
dataset_id=dataset.id,
document_id=document.id,
position=7,
content="existing",
word_count=1,
tokens=1,
created_by=user_id,
index_node_id="existing-node",
index_node_hash="existing-hash",
status=SegmentStatus.COMPLETED,
)
if foreign_owner == "dataset":
dataset.tenant_id = str(uuid4())
elif foreign_owner == "document_tenant":
document.tenant_id = str(uuid4())
elif foreign_owner == "document_dataset":
document.dataset_id = str(uuid4())
elif foreign_owner == "upload":
upload_file.tenant_id = str(uuid4())
sqlite_session.add_all([dataset, document, upload_file, existing_segment])
sqlite_session.commit()
def mock_download(_key: str, file_path: str) -> None:
Path(file_path).write_text("content\nfirst\nsecond\nthird\n", encoding="utf-8")
max_position_queries: list[str] = []
def count_max_position_query(
_connection: object,
_cursor: object,
statement: str,
_parameters: object,
_context: object,
_executemany: bool,
) -> None:
normalized_statement = statement.lower()
if "max(" in normalized_statement and "document_segments.position" in normalized_statement:
max_position_queries.append(statement)
event.listen(sqlite_engine, "before_cursor_execute", count_max_position_query)
try:
with (
patch("tasks.batch_create_segment_to_index_task.storage.download", side_effect=mock_download) as download,
patch("extensions.ext_application_services.application_services") as services,
):
batch_create_segment_to_index_task(
job_id=str(uuid4()),
upload_file_id=upload_file.id,
dataset_id=dataset.id,
document_id=document.id,
tenant_id=tenant_id,
user_id=user_id,
)
finally:
event.remove(sqlite_engine, "before_cursor_execute", count_max_position_query)
if foreign_owner is not None:
download.assert_not_called()
services.assert_not_called()
assert not max_position_queries
assert sqlite_session.scalars(select(DocumentSegment.id)).all() == [existing_segment.id]
return
positions = sqlite_session.scalars(
select(DocumentSegment.position)
.where(DocumentSegment.document_id == document.id)
.order_by(DocumentSegment.position)
).all()
assert positions == [7, 8, 9, 10]
assert len(max_position_queries) == 1
indexing = services.return_value.knowledge.segments.mutations.index_segments
indexing.assert_called_once()
assert indexing.call_args.args == (DocumentRef(DatasetRef(tenant_id, dataset.id), document.id),)
assert len(indexing.call_args.kwargs["segment_ids"]) == 3
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,388 @@
"""SQLite-backed resilience tests for ``clean_document_task``.
The task must continue PostgreSQL cleanup when vector cleanup fails. Each test
starts from the production incident shape: the caller has already deleted the
``Document`` row while its segments and metadata bindings remain.
"""
import uuid
from datetime import UTC, datetime, timedelta
from unittest.mock import patch
import pytest
from sqlalchemy import select
from sqlalchemy.orm import Session
import tasks.clean_document_task as clean_document_task_module
from extensions.storage.storage_type import StorageType
from models.dataset import (
Dataset,
DatasetMetadataBinding,
Document,
DocumentSegment,
SegmentAttachmentBinding,
)
from models.enums import CreatorUserRole, DataSourceType
from models.knowledge_fs import KnowledgeFSUpgradeFileLease, KnowledgeFSUpgradeJob
from models.model import UploadFile
from tasks.clean_document_task import clean_document_task
from tests.helpers.unit.model_factories import make_document
SQLITE_MODELS = (
Dataset,
Document,
DocumentSegment,
SegmentAttachmentBinding,
UploadFile,
DatasetMetadataBinding,
)
pytestmark = pytest.mark.parametrize("sqlite_session", [SQLITE_MODELS], indirect=True)
@pytest.fixture
def document_id() -> str:
return str(uuid.uuid4())
@pytest.fixture
def dataset_id() -> str:
return str(uuid.uuid4())
@pytest.fixture
def tenant_id() -> str:
return str(uuid.uuid4())
@pytest.fixture
def bind_task_sessions(monkeypatch: pytest.MonkeyPatch, sqlite_session: Session) -> None:
"""Bind every short-lived task transaction to the isolated SQLite engine."""
engine = sqlite_session.get_bind()
monkeypatch.setattr(
clean_document_task_module.session_factory,
"create_session",
lambda: Session(engine, expire_on_commit=False),
)
@pytest.fixture
def mock_storage():
with patch("tasks.clean_document_task.storage", autospec=True) as mock:
mock.delete.return_value = None
yield mock
@pytest.fixture
def mock_index_cleanup(tenant_id: str):
"""Keep task resilience tests independent of the external index backend."""
with patch("tasks.clean_document_task.clean_document_indexes", return_value=tenant_id) as cleanup:
yield cleanup
def _document(*, document_id: str, dataset_id: str, tenant_id: str, created_by: str) -> Document:
return make_document(
document_id=document_id,
tenant_id=tenant_id,
dataset_id=dataset_id,
name=f"{document_id}.txt",
created_by=created_by,
)
def _segment(*, segment_id: str, document_id: str, dataset_id: str, tenant_id: str, created_by: str) -> DocumentSegment:
segment = DocumentSegment(
tenant_id=tenant_id,
dataset_id=dataset_id,
document_id=document_id,
position=1,
content="segment content",
word_count=2,
tokens=2,
created_by=created_by,
index_node_id=f"node-{segment_id}",
)
segment.id = segment_id
return segment
def _persist_deleted_document_state(
session: Session,
*,
document_id: str,
dataset_id: str,
tenant_id: str,
target_segment_ids: list[str],
) -> tuple[str, str]:
"""Persist target children after deleting their document, plus scoped control rows."""
created_by = str(uuid.uuid4())
other_document_id = str(uuid.uuid4())
survivor_segment_id = str(uuid.uuid4())
dataset = Dataset(
id=dataset_id,
tenant_id=tenant_id,
name="Cleanup dataset",
data_source_type=DataSourceType.UPLOAD_FILE,
created_by=created_by,
)
target_document = _document(
document_id=document_id,
dataset_id=dataset_id,
tenant_id=tenant_id,
created_by=created_by,
)
other_document = _document(
document_id=other_document_id,
dataset_id=dataset_id,
tenant_id=tenant_id,
created_by=created_by,
)
segments = [
_segment(
segment_id=segment_id,
document_id=document_id,
dataset_id=dataset_id,
tenant_id=tenant_id,
created_by=created_by,
)
for segment_id in target_segment_ids
]
segments.append(
_segment(
segment_id=survivor_segment_id,
document_id=other_document_id,
dataset_id=dataset_id,
tenant_id=tenant_id,
created_by=created_by,
)
)
metadata_bindings = [
DatasetMetadataBinding(
tenant_id=tenant_id,
dataset_id=dataset_id,
metadata_id=str(uuid.uuid4()),
document_id=current_document_id,
created_by=created_by,
)
for current_document_id in (document_id, other_document_id)
]
session.add_all([dataset, target_document, other_document, *segments, *metadata_bindings])
session.commit()
session.delete(target_document)
session.commit()
return other_document_id, survivor_segment_id
def _assert_relational_cleanup(
session: Session,
*,
document_id: str,
other_document_id: str,
survivor_segment_id: str,
) -> None:
session.expire_all()
assert session.get(Document, document_id) is None
remaining_segments = session.scalars(select(DocumentSegment)).all()
assert [(segment.id, segment.document_id) for segment in remaining_segments] == [
(survivor_segment_id, other_document_id)
]
remaining_binding_document_ids = set(session.scalars(select(DatasetMetadataBinding.document_id)).all())
assert remaining_binding_document_ids == {other_document_id}
class TestVectorCleanupResilience:
"""Vector/index failures must not abort relational cleanup."""
def test_billing_failure_during_vector_cleanup_does_not_skip_pg_cleanup(
self,
document_id: str,
dataset_id: str,
tenant_id: str,
sqlite_session: Session,
bind_task_sessions: None,
mock_storage,
mock_index_cleanup,
) -> None:
"""A transient billing failure leaves only the unrelated document's rows."""
other_document_id, survivor_segment_id = _persist_deleted_document_state(
sqlite_session,
document_id=document_id,
dataset_id=dataset_id,
tenant_id=tenant_id,
target_segment_ids=["seg-1", "seg-2"],
)
mock_index_cleanup.side_effect = ValueError(
"Unable to retrieve billing information. Please try again later or contact support."
)
# Act — must not raise out of the task even though clean() raises.
with patch("tasks.clean_document_task.schedule_billing_vector_space_refresh") as schedule_refresh:
clean_document_task(
document_id=document_id,
dataset_id=dataset_id,
doc_form="paragraph",
file_id=None,
)
mock_index_cleanup.assert_called_once()
_assert_relational_cleanup(
sqlite_session,
document_id=document_id,
other_document_id=other_document_id,
survivor_segment_id=survivor_segment_id,
)
schedule_refresh.assert_not_called()
def test_active_upgrade_lease_keeps_the_legacy_source_file(
document_id: str,
dataset_id: str,
tenant_id: str,
sqlite_session: Session,
bind_task_sessions: None,
mock_storage,
mock_index_cleanup,
) -> None:
del bind_task_sessions, mock_index_cleanup
_persist_deleted_document_state(
sqlite_session,
document_id=document_id,
dataset_id=dataset_id,
tenant_id=tenant_id,
target_segment_ids=[],
)
account_id = str(uuid.uuid4())
upload_file = UploadFile(
tenant_id=tenant_id,
storage_type=StorageType.LOCAL,
key=f"upload_files/{tenant_id}/source.txt",
name="source.txt",
size=12,
extension="txt",
mime_type="text/plain",
created_by_role=CreatorUserRole.ACCOUNT,
created_by=account_id,
created_at=datetime(2026, 8, 17, tzinfo=UTC),
used=False,
)
now = datetime.now(UTC).replace(tzinfo=None)
job = KnowledgeFSUpgradeJob(
tenant_id=tenant_id,
old_dataset_id=dataset_id,
requested_by_account_id=account_id,
owner_account_id=account_id,
idempotency_key="cleanup-lease-test",
snapshot_at=now,
config_snapshot={},
permission_snapshot={},
app_binding_snapshot=[],
tag_ids_snapshot=[],
)
sqlite_session.add_all([upload_file, job])
sqlite_session.flush()
lease = KnowledgeFSUpgradeFileLease(
job_id=job.id,
old_upload_file_id=upload_file.id,
expires_at=now + timedelta(hours=1),
)
sqlite_session.add(lease)
sqlite_session.commit()
clean_document_task(
document_id=document_id,
dataset_id=dataset_id,
doc_form="paragraph",
file_id=upload_file.id,
)
sqlite_session.expire_all()
assert sqlite_session.get(UploadFile, upload_file.id) is not None
persisted_lease = sqlite_session.get(KnowledgeFSUpgradeFileLease, lease.id)
assert persisted_lease is not None
assert persisted_lease.cleanup_requested_at is not None
mock_storage.delete.assert_not_called()
class TestVectorCleanupSuccessPaths:
def test_vector_cleanup_success_path_remains_unaffected(
self,
document_id: str,
dataset_id: str,
tenant_id: str,
sqlite_session: Session,
bind_task_sessions: None,
mock_storage,
mock_index_cleanup,
) -> None:
"""The happy path calls the index boundary and completes scoped cleanup."""
other_document_id, survivor_segment_id = _persist_deleted_document_state(
sqlite_session,
document_id=document_id,
dataset_id=dataset_id,
tenant_id=tenant_id,
target_segment_ids=["seg-1"],
)
with patch("tasks.clean_document_task.schedule_billing_vector_space_refresh") as schedule_refresh:
clean_document_task(
document_id=document_id,
dataset_id=dataset_id,
doc_form="paragraph",
file_id=None,
)
mock_index_cleanup.assert_called_once()
_, kwargs = mock_index_cleanup.call_args
new_session = kwargs.pop("new_session")
with new_session() as cleanup_session:
assert cleanup_session.get_bind() is sqlite_session.get_bind()
assert kwargs == {
"dataset_id": dataset_id,
"document_ids": [document_id],
"doc_form": "paragraph",
}
_assert_relational_cleanup(
sqlite_session,
document_id=document_id,
other_document_id=other_document_id,
survivor_segment_id=survivor_segment_id,
)
schedule_refresh.assert_called_once_with(tenant_id)
def test_no_index_state_does_not_refresh_billing(
self,
document_id: str,
dataset_id: str,
tenant_id: str,
sqlite_session: Session,
bind_task_sessions: None,
mock_storage,
mock_index_cleanup,
) -> None:
"""Cleanup checks for orphan summaries even when no segments remain."""
mock_index_cleanup.return_value = None
other_document_id, survivor_segment_id = _persist_deleted_document_state(
sqlite_session,
document_id=document_id,
dataset_id=dataset_id,
tenant_id=tenant_id,
target_segment_ids=[],
)
with patch("tasks.clean_document_task.schedule_billing_vector_space_refresh") as schedule_refresh:
clean_document_task(
document_id=document_id,
dataset_id=dataset_id,
doc_form="paragraph",
file_id=None,
)
mock_index_cleanup.assert_called_once()
_assert_relational_cleanup(
sqlite_session,
document_id=document_id,
other_document_id=other_document_id,
survivor_segment_id=survivor_segment_id,
)
schedule_refresh.assert_not_called()
@@ -0,0 +1,207 @@
from typing import Protocol, cast
import pytest
from services.agent.deletion_service import AgentDeletionService
from services.agent.home_snapshot_service import AgentHomeSnapshotService
from services.agent.workspace_service import AgentWorkspaceService
from tasks.collect_agent_resources_task import (
collect_agent_resources,
enqueue_agent_resource_collection,
)
class _TaskWithQueue(Protocol):
queue: str
def test_collection_task_uses_retention_queue() -> None:
task = cast(_TaskWithQueue, collect_agent_resources)
assert task.queue == "retention"
def test_enqueue_deduplicates_ids_and_skips_empty_input(monkeypatch: pytest.MonkeyPatch) -> None:
delay_calls: list[dict[str, object]] = []
def delay(**kwargs: object) -> None:
delay_calls.append(kwargs)
monkeypatch.setattr(collect_agent_resources, "delay", delay)
enqueue_agent_resource_collection(tenant_id="tenant-1")
enqueue_agent_resource_collection(
tenant_id="tenant-1",
binding_ids=["binding-2", "binding-1", "binding-2"],
workspace_ids=["workspace-1"],
purge_agent_ids=["agent-2", "", "agent-1", "agent-2"],
)
assert delay_calls == [
{
"tenant_id": "tenant-1",
"binding_ids": ["binding-1", "binding-2"],
"workspace_ids": ["workspace-1"],
"home_snapshot_ids": [],
"purge_agent_ids": ["agent-1", "agent-2"],
}
]
def test_collection_runs_in_workspace_binding_snapshot_order(monkeypatch: pytest.MonkeyPatch) -> None:
calls: list[str] = []
monkeypatch.setattr(
AgentWorkspaceService,
"collect_retired_workspace",
lambda **_kwargs: calls.append("workspace"),
)
monkeypatch.setattr(
AgentWorkspaceService,
"collect_retired_binding",
lambda **_kwargs: calls.append("binding"),
)
monkeypatch.setattr(
AgentHomeSnapshotService,
"collect_retired_home_snapshot",
lambda **_kwargs: calls.append("home"),
)
purge_calls: list[tuple[str, list[str]]] = []
def purge(*, tenant_id: str, agent_ids: list[str]) -> None:
purge_calls.append((tenant_id, agent_ids))
calls.append("purge")
monkeypatch.setattr(AgentDeletionService, "purge_archived_agents", purge)
collect_agent_resources.run(
tenant_id="tenant-1",
workspace_ids=["workspace-1"],
binding_ids=["binding-1"],
home_snapshot_ids=["home-1"],
purge_agent_ids=["agent-1"],
)
assert calls == ["workspace", "binding", "home", "purge"]
assert purge_calls == [("tenant-1", ["agent-1"])]
def test_collection_failure_propagates_after_attempting_remaining_resources(monkeypatch: pytest.MonkeyPatch) -> None:
calls: list[str] = []
first_error = RuntimeError("workspace-1 failed")
errors = {
"workspace-1": first_error,
"workspace-2": RuntimeError("workspace-2 failed"),
"binding-1": RuntimeError("binding-1 failed"),
"home-2": RuntimeError("home-2 failed"),
}
logged_exceptions: list[tuple[str, dict[str, object]]] = []
def log_exception(message: str, *, extra: dict[str, object]) -> None:
logged_exceptions.append((message, extra))
def collect_workspace(*, workspace_id: str, **_kwargs: object) -> None:
calls.append(f"workspace:{workspace_id}")
if error := errors.get(workspace_id):
raise error
def collect_binding(*, binding_id: str, **_kwargs: object) -> None:
calls.append(f"binding:{binding_id}")
if error := errors.get(binding_id):
raise error
def collect_home(*, home_snapshot_id: str, **_kwargs: object) -> None:
calls.append(f"home:{home_snapshot_id}")
if error := errors.get(home_snapshot_id):
raise error
monkeypatch.setattr(AgentWorkspaceService, "collect_retired_workspace", collect_workspace)
monkeypatch.setattr(AgentWorkspaceService, "collect_retired_binding", collect_binding)
monkeypatch.setattr(AgentHomeSnapshotService, "collect_retired_home_snapshot", collect_home)
monkeypatch.setattr("tasks.collect_agent_resources_task.logger.exception", log_exception)
purge_calls: list[tuple[str, list[str]]] = []
def purge(*, tenant_id: str, agent_ids: list[str]) -> None:
purge_calls.append((tenant_id, agent_ids))
monkeypatch.setattr(AgentDeletionService, "purge_archived_agents", purge)
with pytest.raises(RuntimeError) as exc_info:
collect_agent_resources.run(
tenant_id="tenant-1",
workspace_ids=["workspace-1", "workspace-2", "workspace-3"],
binding_ids=["binding-1", "binding-2"],
home_snapshot_ids=["home-1", "home-2", "home-3"],
purge_agent_ids=["agent-1"],
)
assert exc_info.value.__cause__ is first_error
assert str(exc_info.value) == (
"Failed to collect 4 retired Agent resource(s): "
"workspace:workspace-1, workspace:workspace-2, binding:binding-1, home_snapshot:home-2"
)
assert calls == [
"workspace:workspace-1",
"workspace:workspace-2",
"workspace:workspace-3",
"binding:binding-1",
"binding:binding-2",
"home:home-1",
"home:home-2",
"home:home-3",
]
assert purge_calls == []
assert logged_exceptions == [
(
"Failed to collect retired Agent resource",
{
"tenant_id": "tenant-1",
"resource_type": resource_type,
"resource_id": resource_id,
},
)
for resource_type, resource_id in (
("workspace", "workspace-1"),
("workspace", "workspace-2"),
("binding", "binding-1"),
("home_snapshot", "home-2"),
)
]
def test_enqueue_failure_propagates(monkeypatch: pytest.MonkeyPatch) -> None:
error = RuntimeError("queue unavailable")
delay_calls: list[dict[str, object]] = []
logged_exceptions: list[tuple[str, dict[str, object]]] = []
def delay(**kwargs: object) -> None:
delay_calls.append(kwargs)
raise error
def log_exception(message: str, *, extra: dict[str, object]) -> None:
logged_exceptions.append((message, extra))
monkeypatch.setattr(collect_agent_resources, "delay", delay)
monkeypatch.setattr("tasks.collect_agent_resources_task.logger.exception", log_exception)
with pytest.raises(RuntimeError) as exc_info:
enqueue_agent_resource_collection(
tenant_id="tenant-1",
binding_ids=["binding-1"],
workspace_ids=["workspace-1"],
home_snapshot_ids=["home-1"],
purge_agent_ids=["agent-2", "agent-1", "agent-2"],
)
assert exc_info.value is error
payload = {
"binding_ids": ["binding-1"],
"workspace_ids": ["workspace-1"],
"home_snapshot_ids": ["home-1"],
"purge_agent_ids": ["agent-1", "agent-2"],
}
assert delay_calls == [{"tenant_id": "tenant-1", **payload}]
assert logged_exceptions == [
(
"Failed to enqueue retired Agent resource collection",
{"tenant_id": "tenant-1", **payload},
)
]
@@ -0,0 +1,50 @@
import logging
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
from sqlalchemy.engine import Engine
from sqlalchemy.orm import Session
from tasks import community_telemetry_task
def _bind_task_to_sqlite(monkeypatch: pytest.MonkeyPatch, sqlite_engine: Engine) -> None:
"""Bind the task-local sessionmaker to the isolated SQLite database."""
monkeypatch.setattr(community_telemetry_task, "db", SimpleNamespace(engine=sqlite_engine))
def test_send_community_telemetry_heartbeat_reports_with_a_database_session(
monkeypatch: pytest.MonkeyPatch, sqlite_engine: Engine
) -> None:
_bind_task_to_sqlite(monkeypatch, sqlite_engine)
received_sessions: list[Session] = []
def report_heartbeat(*, session: Session) -> None:
received_sessions.append(session)
assert session.get_bind() is sqlite_engine
monkeypatch.setattr(community_telemetry_task.CommunityTelemetryService, "report_heartbeat", report_heartbeat)
community_telemetry_task.send_community_telemetry_heartbeat.run()
assert len(received_sessions) == 1
assert isinstance(received_sessions[0], Session)
def test_send_community_telemetry_heartbeat_swallows_report_errors(
monkeypatch: pytest.MonkeyPatch,
sqlite_engine: Engine,
caplog: pytest.LogCaptureFixture,
) -> None:
_bind_task_to_sqlite(monkeypatch, sqlite_engine)
monkeypatch.setattr(
community_telemetry_task.CommunityTelemetryService,
"report_heartbeat",
Mock(side_effect=RuntimeError("telemetry unavailable")),
)
caplog.set_level(logging.DEBUG, logger=community_telemetry_task.logger.name)
community_telemetry_task.send_community_telemetry_heartbeat.run()
assert "Failed to process community telemetry heartbeat" in caplog.text
@@ -0,0 +1,285 @@
from datetime import UTC, datetime, timedelta
from decimal import Decimal
from unittest.mock import MagicMock, patch
import pytest
from sqlalchemy import select
from sqlalchemy.orm import Session
from core.workflow.human_input_adapter import DeliveryMethodType
from core.workflow.nodes.human_input.enums import HumanInputFormKind, HumanInputFormStatus
from graphon.file import FileTransferMethod, FileType
from models import (
AgentDebugConversation,
Conversation,
ConversationVariable,
HumanInputForm,
HumanInputFormUploadFile,
HumanInputFormUploadToken,
Message,
MessageAgentThought,
MessageAnnotation,
MessageChain,
MessageFeedback,
MessageFile,
PinnedConversation,
SavedMessage,
)
from models.agent import AgentConfigDraftType
from models.enums import (
ConversationFromSource,
ConversationStatus,
CreatorUserRole,
FeedbackFromSource,
FeedbackRating,
InvokeFrom,
MessageChainType,
)
from models.human_input import HumanInputDelivery, HumanInputFormRecipient, RecipientType
from models.tools import ToolConversationVariables, ToolFile
from tasks.delete_conversation_task import _cleanup_conversation_related_data, sweep_deleted_conversations
from tests.helpers.unit.model_factories import make_conversation, make_message
TENANT_ID = "11111111-1111-1111-1111-111111111111"
APP_ID = "22222222-2222-2222-2222-222222222222"
ACCOUNT_ID = "33333333-3333-3333-3333-333333333333"
CONVERSATION_ID = "44444444-4444-4444-4444-444444444444"
OTHER_CONVERSATION_ID = "55555555-5555-5555-5555-555555555555"
MESSAGE_ID = "66666666-6666-6666-6666-666666666666"
AGENT_ID = "77777777-7777-7777-7777-777777777777"
def _conversation(conversation_id: str, *, deleted: bool) -> Conversation:
return make_conversation(
conversation_id=conversation_id,
app_id=APP_ID,
name="Test conversation",
inputs={},
status=ConversationStatus.NORMAL,
from_source=ConversationFromSource.CONSOLE,
from_account_id=ACCOUNT_ID,
is_deleted=deleted,
)
def _message() -> Message:
return make_message(
message_id=MESSAGE_ID,
app_id=APP_ID,
conversation_id=CONVERSATION_ID,
inputs={},
query="hello",
message={"role": "user", "content": "hello"},
answer="world",
message_unit_price=Decimal(0),
answer_unit_price=Decimal(0),
currency="USD",
invoke_from=InvokeFrom.WEB_APP,
from_source=ConversationFromSource.CONSOLE,
from_account_id=ACCOUNT_ID,
)
def _tool_file(*, name: str, conversation_id: str | None = CONVERSATION_ID) -> ToolFile:
return ToolFile(
user_id=ACCOUNT_ID,
tenant_id=TENANT_ID,
conversation_id=conversation_id,
file_key=f"tools/{TENANT_ID}/{name}",
mimetype="text/plain",
name=name,
size=5,
)
def test_cleanup_removes_owned_resources(sqlite_session: Session) -> None:
conversation = _conversation(CONVERSATION_ID, deleted=True)
other_conversation = _conversation(OTHER_CONVERSATION_ID, deleted=False)
message = _message()
owned_file = _tool_file(name="owned.txt")
other_file = _tool_file(name="other.txt", conversation_id=OTHER_CONVERSATION_ID)
sqlite_session.add_all([conversation, other_conversation, message, owned_file, other_file])
sqlite_session.flush()
message_chain = MessageChain(message_id=MESSAGE_ID, type=MessageChainType.SYSTEM, input=None, output=None)
form = HumanInputForm(
tenant_id=TENANT_ID,
app_id=APP_ID,
workflow_run_id=None,
conversation_id=CONVERSATION_ID,
form_kind=HumanInputFormKind.RUNTIME,
node_id="ask-human",
form_definition="{}",
rendered_content="form",
status=HumanInputFormStatus.WAITING,
expiration_time=datetime.now(UTC) + timedelta(hours=1),
)
sqlite_session.add_all([message_chain, form])
sqlite_session.flush()
delivery = HumanInputDelivery(
form_id=form.id,
delivery_method_type=DeliveryMethodType.WEBAPP,
delivery_config_id=None,
channel_payload="{}",
)
sqlite_session.add(delivery)
sqlite_session.flush()
upload_token = HumanInputFormUploadToken(
tenant_id=TENANT_ID,
app_id=APP_ID,
form_id=form.id,
recipient_id="88888888-8888-8888-8888-888888888888",
token="upload-token",
)
sqlite_session.add(upload_token)
sqlite_session.flush()
related_rows = [
MessageAgentThought(
message_id=MESSAGE_ID,
position=1,
created_by_role=CreatorUserRole.ACCOUNT,
created_by=ACCOUNT_ID,
message_chain_id=message_chain.id,
),
MessageFile(
message_id=MESSAGE_ID,
type=FileType.DOCUMENT,
transfer_method=FileTransferMethod.REMOTE_URL,
created_by_role=CreatorUserRole.ACCOUNT,
created_by=ACCOUNT_ID,
url="https://example.com/file.txt",
),
SavedMessage(
app_id=APP_ID,
message_id=MESSAGE_ID,
created_by_role=CreatorUserRole.ACCOUNT,
created_by=ACCOUNT_ID,
),
MessageAnnotation(
app_id=APP_ID,
question="hello",
content="world",
account_id=ACCOUNT_ID,
conversation_id=CONVERSATION_ID,
message_id=MESSAGE_ID,
),
MessageFeedback(
app_id=APP_ID,
conversation_id=CONVERSATION_ID,
message_id=MESSAGE_ID,
rating=FeedbackRating.LIKE,
from_source=FeedbackFromSource.ADMIN,
from_account_id=ACCOUNT_ID,
),
ToolConversationVariables(
user_id=ACCOUNT_ID,
tenant_id=TENANT_ID,
conversation_id=CONVERSATION_ID,
variables_str="{}",
),
ConversationVariable(
id="99999999-9999-9999-9999-999999999999",
conversation_id=CONVERSATION_ID,
app_id=APP_ID,
data="{}",
),
PinnedConversation(
app_id=APP_ID,
conversation_id=CONVERSATION_ID,
created_by_role=CreatorUserRole.ACCOUNT,
created_by=ACCOUNT_ID,
),
AgentDebugConversation(
tenant_id=TENANT_ID,
agent_id=AGENT_ID,
app_id=APP_ID,
account_id=ACCOUNT_ID,
draft_type=AgentConfigDraftType.DEBUG_BUILD,
conversation_id=CONVERSATION_ID,
),
HumanInputFormRecipient(
form_id=form.id,
delivery_id=delivery.id,
recipient_type=RecipientType.CONSOLE,
recipient_payload="{}",
),
HumanInputFormUploadFile(
tenant_id=TENANT_ID,
app_id=APP_ID,
form_id=form.id,
upload_file_id="aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
upload_token_id=upload_token.id,
),
]
sqlite_session.add_all(related_rows)
sqlite_session.commit()
form_id = form.id
owned_file_id = owned_file.id
owned_file_key = owned_file.file_key
other_file_id = other_file.id
with patch("tasks.delete_conversation_task.storage") as storage_mock:
assert _cleanup_conversation_related_data(CONVERSATION_ID) is True
storage_mock.delete.assert_called_once_with(owned_file_key)
sqlite_session.expire_all()
assert sqlite_session.get(Conversation, CONVERSATION_ID) is None
assert sqlite_session.get(Message, MESSAGE_ID) is None
assert (
sqlite_session.scalar(select(MessageAgentThought).where(MessageAgentThought.message_id == MESSAGE_ID)) is None
)
assert sqlite_session.scalar(select(HumanInputForm).where(HumanInputForm.id == form_id)) is None
assert sqlite_session.get(ToolFile, owned_file_id) is None
assert sqlite_session.get(ToolFile, other_file_id) is not None
assert sqlite_session.get(Conversation, OTHER_CONVERSATION_ID) is not None
def test_cleanup_storage_failure_retains_marker_and_file_key(sqlite_session: Session) -> None:
conversation = _conversation(CONVERSATION_ID, deleted=True)
tool_file = _tool_file(name="retry.txt")
sqlite_session.add_all([conversation, tool_file])
sqlite_session.commit()
with patch("tasks.delete_conversation_task.storage") as storage_mock:
storage_mock.delete.side_effect = RuntimeError("storage unavailable")
storage_mock.exists.return_value = True
with pytest.raises(RuntimeError, match="storage unavailable"):
_cleanup_conversation_related_data(CONVERSATION_ID)
sqlite_session.expire_all()
persisted_conversation = sqlite_session.get(Conversation, CONVERSATION_ID)
assert persisted_conversation is not None
assert persisted_conversation.is_deleted is True
assert sqlite_session.get(ToolFile, tool_file.id) is not None
def test_cleanup_skips_active_conversation(sqlite_session: Session) -> None:
conversation = _conversation(CONVERSATION_ID, deleted=False)
tool_file = _tool_file(name="active.txt")
sqlite_session.add_all([conversation, tool_file])
sqlite_session.commit()
with patch("tasks.delete_conversation_task.storage") as storage_mock:
assert _cleanup_conversation_related_data(CONVERSATION_ID) is False
storage_mock.delete.assert_not_called()
assert sqlite_session.get(Conversation, CONVERSATION_ID) is not None
assert sqlite_session.get(ToolFile, tool_file.id) is not None
def test_sweeper_dispatches_only_soft_deleted_conversations(sqlite_session: Session) -> None:
sqlite_session.add_all(
[
_conversation(CONVERSATION_ID, deleted=True),
_conversation(OTHER_CONVERSATION_ID, deleted=False),
]
)
sqlite_session.commit()
with patch("tasks.delete_conversation_task.delete_conversation_related_data.delay", MagicMock()) as delay:
assert sweep_deleted_conversations.run() == 1
delay.assert_called_once_with(CONVERSATION_ID)
@@ -0,0 +1,321 @@
import uuid
from collections.abc import Generator
from contextlib import contextmanager
from datetime import UTC, datetime
from unittest.mock import MagicMock, patch
import pytest
from sqlalchemy import event
from sqlalchemy.orm import Session, sessionmaker
from core.rag.index_processor.constant.index_type import IndexStructureType
from extensions.storage.storage_type import StorageType
from models.dataset import Dataset, Document, DocumentSegment, SegmentAttachmentBinding
from models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, IndexingStatus, SegmentStatus
from models.model import UploadFile
from tasks.delete_segment_from_index_task import delete_segment_from_index_task
from tasks.disable_segment_from_index_task import disable_segment_from_index_task
from tasks.disable_segments_from_index_task import disable_segments_from_index_task
@pytest.fixture
def indexed_segment(sqlite_session: Session) -> tuple[Dataset, Document, DocumentSegment]:
"""Persist the complete owner chain consumed by segment cleanup tasks."""
tenant_id = str(uuid.uuid4())
created_by = str(uuid.uuid4())
dataset = Dataset(
id=str(uuid.uuid4()),
tenant_id=tenant_id,
name="Cleanup dataset",
data_source_type=DataSourceType.UPLOAD_FILE,
created_by=created_by,
is_multimodal=False,
)
document = Document(
id=str(uuid.uuid4()),
tenant_id=tenant_id,
dataset_id=dataset.id,
position=1,
data_source_type=DataSourceType.UPLOAD_FILE,
batch="batch-1",
name="document.txt",
created_from=DocumentCreatedFrom.WEB,
created_by=created_by,
indexing_status=IndexingStatus.COMPLETED,
doc_form=IndexStructureType.PARAGRAPH_INDEX,
)
segment = DocumentSegment(
tenant_id=tenant_id,
dataset_id=dataset.id,
document_id=document.id,
position=1,
content="content",
word_count=1,
tokens=1,
created_by=created_by,
index_node_id="node-1",
index_node_hash="hash-1",
disabled_by=created_by,
status=SegmentStatus.COMPLETED,
)
sqlite_session.add_all([dataset, document, segment])
sqlite_session.commit()
return dataset, document, segment
@contextmanager
def _record_transaction_events(
sqlite_session_factory: sessionmaker[Session], phase_events: list[str]
) -> Generator[None]:
"""Record real commits made by the task-owned SQLite session."""
session_type = sqlite_session_factory.class_
def after_commit(_session: Session) -> None:
phase_events.append("commit")
event.listen(session_type, "after_commit", after_commit)
try:
yield
finally:
event.remove(session_type, "after_commit", after_commit)
def test_disable_segment_commits_index_cleanup(
indexed_segment: tuple[Dataset, Document, DocumentSegment],
sqlite_session_factory: sessionmaker[Session],
) -> None:
dataset, _document, segment = indexed_segment
phase_events: list[str] = []
processor = MagicMock()
processor.clean.side_effect = lambda *_args, **_kwargs: phase_events.append("clean")
disable_summaries = MagicMock(side_effect=lambda *_args, **_kwargs: phase_events.append("summary"))
with (
_record_transaction_events(sqlite_session_factory, phase_events),
patch("tasks.disable_segment_from_index_task.IndexProcessorFactory") as processor_factory,
patch(
"services.knowledge.summaries.adapters.SummaryIndexAdapter.disable_summaries_for_segments",
disable_summaries,
),
patch("tasks.disable_segment_from_index_task.redis_client.delete"),
):
processor_factory.return_value.init_index_processor.return_value = processor
disable_segment_from_index_task.run(segment.id)
assert phase_events == ["clean", "commit", "summary"]
disable_summaries.assert_called_once()
assert disable_summaries.call_args.kwargs["dataset"].id == dataset.id
assert disable_summaries.call_args.kwargs["segment_ids"] == [segment.id]
assert disable_summaries.call_args.kwargs["disabled_by"] == segment.disabled_by
def test_disable_segments_commits_index_cleanup(
indexed_segment: tuple[Dataset, Document, DocumentSegment],
sqlite_session_factory: sessionmaker[Session],
) -> None:
dataset, document, segment = indexed_segment
phase_events: list[str] = []
processor = MagicMock()
processor.clean.side_effect = lambda *_args, **_kwargs: phase_events.append("clean")
disable_summaries = MagicMock(side_effect=lambda *_args, **_kwargs: phase_events.append("summary"))
with (
_record_transaction_events(sqlite_session_factory, phase_events),
patch("tasks.disable_segments_from_index_task.IndexProcessorFactory") as processor_factory,
patch(
"services.knowledge.summaries.adapters.SummaryIndexAdapter.disable_summaries_for_segments",
disable_summaries,
),
patch("tasks.disable_segments_from_index_task.redis_client.delete"),
):
processor_factory.return_value.init_index_processor.return_value = processor
disable_segments_from_index_task.run([segment.id], dataset.id, document.id)
assert phase_events == ["clean", "commit", "summary"]
disable_summaries.assert_called_once()
assert disable_summaries.call_args.kwargs["dataset"].id == dataset.id
assert disable_summaries.call_args.kwargs["segment_ids"] == [segment.id]
assert disable_summaries.call_args.kwargs["disabled_by"] == segment.disabled_by
def test_delete_segment_commits_index_cleanup_without_attachments(
indexed_segment: tuple[Dataset, Document, DocumentSegment],
sqlite_session_factory: sessionmaker[Session],
) -> None:
dataset, document, segment = indexed_segment
phase_events: list[str] = []
processor = MagicMock()
processor.clean.side_effect = lambda *_args, **_kwargs: phase_events.append("clean")
with (
_record_transaction_events(sqlite_session_factory, phase_events),
patch("tasks.delete_segment_from_index_task.IndexProcessorFactory") as processor_factory,
):
processor_factory.return_value.init_index_processor.return_value = processor
delete_segment_from_index_task.run(["node-1"], dataset.id, document.id, [segment.id])
assert phase_events == ["clean", "commit"]
def test_delete_segment_removes_attachment_blobs_from_storage(
indexed_segment: tuple[Dataset, Document, DocumentSegment],
sqlite_session: Session,
) -> None:
dataset, document, segment = indexed_segment
dataset.is_multimodal = True
attachment = UploadFile(
tenant_id=dataset.tenant_id,
storage_type=StorageType.LOCAL,
key="attachments/segment-image.png",
name="segment-image.png",
size=10,
extension="png",
mime_type="image/png",
created_by_role=CreatorUserRole.ACCOUNT,
created_by=segment.created_by,
created_at=datetime.now(UTC),
used=True,
)
binding = SegmentAttachmentBinding(
tenant_id=dataset.tenant_id,
dataset_id=dataset.id,
document_id=document.id,
segment_id=segment.id,
attachment_id=attachment.id,
)
sqlite_session.add_all([dataset, attachment, binding])
sqlite_session.commit()
attachment_id = attachment.id
attachment_key = attachment.key
binding_id = binding.id
with (
patch("tasks.delete_segment_from_index_task.IndexProcessorFactory"),
patch("tasks.delete_segment_from_index_task.storage.delete") as storage_delete,
):
delete_segment_from_index_task.run(["node-1"], dataset.id, document.id, [segment.id])
storage_delete.assert_called_once_with(attachment_key)
sqlite_session.expire_all()
assert sqlite_session.get(SegmentAttachmentBinding, binding_id) is None
assert sqlite_session.get(UploadFile, attachment_id) is None
def test_delete_segment_preserves_attachment_shared_by_another_segment(
indexed_segment: tuple[Dataset, Document, DocumentSegment],
sqlite_session: Session,
) -> None:
dataset, document, segment = indexed_segment
dataset.is_multimodal = True
other_segment = DocumentSegment(
tenant_id=dataset.tenant_id,
dataset_id=dataset.id,
document_id=document.id,
position=2,
content="other content",
word_count=1,
tokens=1,
created_by=segment.created_by,
index_node_id="node-2",
index_node_hash="hash-2",
disabled_by=segment.disabled_by,
status=SegmentStatus.COMPLETED,
)
attachment = UploadFile(
tenant_id=dataset.tenant_id,
storage_type=StorageType.LOCAL,
key="attachments/shared-segment-image.png",
name="shared-segment-image.png",
size=10,
extension="png",
mime_type="image/png",
created_by_role=CreatorUserRole.ACCOUNT,
created_by=segment.created_by,
created_at=datetime.now(UTC),
used=True,
)
binding = SegmentAttachmentBinding(
tenant_id=dataset.tenant_id,
dataset_id=dataset.id,
document_id=document.id,
segment_id=segment.id,
attachment_id=attachment.id,
)
shared_binding = SegmentAttachmentBinding(
tenant_id=dataset.tenant_id,
dataset_id=dataset.id,
document_id=document.id,
segment_id=other_segment.id,
attachment_id=attachment.id,
)
sqlite_session.add_all([dataset, other_segment, attachment, binding, shared_binding])
sqlite_session.commit()
attachment_id = attachment.id
binding_id = binding.id
shared_binding_id = shared_binding.id
with (
patch("tasks.delete_segment_from_index_task.IndexProcessorFactory") as processor_factory,
patch("tasks.delete_segment_from_index_task.storage.delete") as storage_delete,
):
delete_segment_from_index_task.run(["node-1"], dataset.id, document.id, [segment.id])
processor = processor_factory.return_value.init_index_processor.return_value
assert processor.clean.call_count == 1
assert processor.clean.call_args.args[1] == ["node-1"]
assert processor.clean.call_args.kwargs["with_keywords"] is True
storage_delete.assert_not_called()
sqlite_session.expire_all()
assert sqlite_session.get(SegmentAttachmentBinding, binding_id) is None
assert sqlite_session.get(SegmentAttachmentBinding, shared_binding_id) is not None
assert sqlite_session.get(UploadFile, attachment_id) is not None
def test_delete_segment_keeps_database_cleanup_when_attachment_storage_delete_fails(
indexed_segment: tuple[Dataset, Document, DocumentSegment],
sqlite_session: Session,
caplog: pytest.LogCaptureFixture,
) -> None:
dataset, document, segment = indexed_segment
dataset.is_multimodal = True
attachment = UploadFile(
tenant_id=dataset.tenant_id,
storage_type=StorageType.LOCAL,
key="attachments/failing-segment-image.png",
name="failing-segment-image.png",
size=10,
extension="png",
mime_type="image/png",
created_by_role=CreatorUserRole.ACCOUNT,
created_by=segment.created_by,
created_at=datetime.now(UTC),
used=True,
)
binding = SegmentAttachmentBinding(
tenant_id=dataset.tenant_id,
dataset_id=dataset.id,
document_id=document.id,
segment_id=segment.id,
attachment_id=attachment.id,
)
sqlite_session.add_all([dataset, attachment, binding])
sqlite_session.commit()
attachment_id = attachment.id
attachment_key = attachment.key
binding_id = binding.id
with (
patch("tasks.delete_segment_from_index_task.IndexProcessorFactory"),
patch(
"tasks.delete_segment_from_index_task.storage.delete",
side_effect=RuntimeError("storage unavailable"),
) as storage_delete,
caplog.at_level("ERROR", logger="tasks.delete_segment_from_index_task"),
):
delete_segment_from_index_task.run(["node-1"], dataset.id, document.id, [segment.id])
storage_delete.assert_called_once_with(attachment_key)
assert "Failed to delete segment attachment from storage" in caplog.text
sqlite_session.expire_all()
assert sqlite_session.get(SegmentAttachmentBinding, binding_id) is None
assert sqlite_session.get(UploadFile, attachment_id) is None
@@ -0,0 +1,284 @@
"""Unit tests for document sync persistence and external collaborator wiring.
The task's Dataset and Document lookups use real SQLite transactions. Datasource,
Notion extraction, index cleanup, and indexing remain mocked I/O boundaries.
"""
import json
import uuid
from unittest.mock import ANY, MagicMock, patch
import pytest
from sqlalchemy.orm import Session, sessionmaker
from core.db import session_factory as session_factory_module
from core.rag.index_processor.constant.index_type import IndexStructureType
from models.dataset import Dataset, Document, DocumentSegment
from models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus
from tasks.document_indexing_sync_task import document_indexing_sync_task
pytestmark = pytest.mark.parametrize(
"sqlite_session",
[(Dataset, Document, DocumentSegment)],
indirect=True,
)
@pytest.fixture
def dataset_id() -> str:
"""Generate a dataset id."""
return str(uuid.uuid4())
@pytest.fixture
def document_id() -> str:
"""Generate a document id."""
return str(uuid.uuid4())
@pytest.fixture
def notion_workspace_id() -> str:
"""Generate a notion workspace id."""
return str(uuid.uuid4())
@pytest.fixture
def notion_page_id() -> str:
"""Generate a notion page id."""
return str(uuid.uuid4())
@pytest.fixture
def credential_id() -> str:
"""Generate a credential id."""
return str(uuid.uuid4())
@pytest.fixture
def tenant_id() -> str:
"""Generate the tenant that owns the persisted dataset and document."""
return str(uuid.uuid4())
@pytest.fixture
def dataset(sqlite_session: Session, dataset_id: str, tenant_id: str) -> Dataset:
"""Persist the dataset resolved by the task's initial and cleanup transactions."""
dataset = Dataset(
id=dataset_id,
tenant_id=tenant_id,
name="Notion dataset",
data_source_type=DataSourceType.NOTION_IMPORT,
created_by=str(uuid.uuid4()),
)
sqlite_session.add(dataset)
sqlite_session.commit()
return dataset
@pytest.fixture
def document(
sqlite_session: Session,
dataset: Dataset,
document_id: str,
tenant_id: str,
notion_workspace_id: str,
notion_page_id: str,
credential_id: str,
) -> Document:
"""Persist a completed Notion document for collaborator and update assertions."""
document = Document(
id=document_id,
tenant_id=tenant_id,
dataset_id=dataset.id,
position=1,
data_source_type=DataSourceType.NOTION_IMPORT,
data_source_info=json.dumps(
{
"notion_workspace_id": notion_workspace_id,
"notion_page_id": notion_page_id,
"type": "page",
"last_edited_time": "2024-01-01T00:00:00Z",
"credential_id": credential_id,
}
),
batch="batch-1",
name="Notion page",
created_from=DocumentCreatedFrom.API,
created_by=str(uuid.uuid4()),
indexing_status=IndexingStatus.COMPLETED,
doc_form=IndexStructureType.PARAGRAPH_INDEX,
)
sqlite_session.add(document)
sqlite_session.commit()
return document
@pytest.fixture(autouse=True)
def _bind_sqlite_session_factory(monkeypatch: pytest.MonkeyPatch, sqlite_session: Session) -> None:
"""Bind each task-owned session to the test's isolated SQLite database."""
monkeypatch.setattr(
session_factory_module,
"_session_maker",
sessionmaker(bind=sqlite_session.get_bind(), expire_on_commit=False),
)
@pytest.fixture
def mock_datasource_provider_service():
"""Mock datasource credential provider."""
with patch("tasks.document_indexing_sync_task.build_data_source_credentials", autospec=True) as mock_service_class:
mock_service = MagicMock()
mock_service.get_datasource_credentials.return_value = {"integration_secret": "test_token"}
mock_service_class.return_value.providers = mock_service
yield mock_service
@pytest.fixture
def mock_notion_extractor():
"""Mock notion extractor class and instance."""
with patch("tasks.document_indexing_sync_task.NotionExtractor", autospec=True) as mock_extractor_class:
mock_extractor = MagicMock()
mock_extractor.get_notion_last_edited_time.return_value = "2024-01-01T00:00:00Z"
mock_extractor_class.return_value = mock_extractor
yield {"class": mock_extractor_class, "instance": mock_extractor}
class TestDocumentIndexingSyncTaskCollaboratorParams:
"""Unit tests for collaborator parameter passing in document_indexing_sync_task."""
def test_notion_extractor_initialized_with_correct_params(
self,
mock_datasource_provider_service,
mock_notion_extractor,
document: Document,
dataset_id: str,
document_id: str,
notion_workspace_id: str,
notion_page_id: str,
):
"""Test that NotionExtractor is initialized with expected arguments."""
# Arrange
expected_token = "test_token"
# Act
document_indexing_sync_task(dataset_id, document_id)
# Assert
mock_notion_extractor["class"].assert_called_once_with(
notion_token_loader=ANY,
notion_workspace_id=notion_workspace_id,
notion_obj_id=notion_page_id,
notion_page_type="page",
notion_access_token=expected_token,
tenant_id=document.tenant_id,
)
def test_datasource_credentials_requested_correctly(
self,
mock_datasource_provider_service,
mock_notion_extractor,
document: Document,
dataset_id: str,
document_id: str,
credential_id: str,
):
"""Test that datasource credentials are requested with expected identifiers."""
# Arrange
expected_tenant_id = document.tenant_id
# Act
document_indexing_sync_task(dataset_id, document_id)
# Assert
mock_datasource_provider_service.get_datasource_credentials.assert_called_once_with(
tenant_id=expected_tenant_id,
credential_id=credential_id,
provider="notion_datasource",
plugin_id="langgenius/notion_datasource",
)
def test_credential_id_missing_uses_none(
self,
mock_datasource_provider_service,
mock_notion_extractor,
document: Document,
sqlite_session: Session,
dataset_id: str,
document_id: str,
):
"""Test that missing credential_id is forwarded as None."""
# Arrange
document.data_source_info = json.dumps(
{
"notion_workspace_id": "workspace-id",
"notion_page_id": "page-id",
"type": "page",
"last_edited_time": "2024-01-01T00:00:00Z",
}
)
sqlite_session.commit()
# Act
document_indexing_sync_task(dataset_id, document_id)
# Assert
mock_datasource_provider_service.get_datasource_credentials.assert_called_once_with(
tenant_id=document.tenant_id,
credential_id=None,
provider="notion_datasource",
plugin_id="langgenius/notion_datasource",
)
class TestDataSourceInfoSerialization:
"""Regression test: data_source_info must be written as a JSON string, not a raw dict.
See https://github.com/langgenius/dify/issues/32705
psycopg2 raises ``ProgrammingError: can't adapt type 'dict'`` when a Python
dict is passed directly to a text/LongText column.
"""
def test_data_source_info_serialized_as_json_string(
self,
document: Document,
sqlite_session: Session,
dataset_id: str,
document_id: str,
):
"""data_source_info must be serialized with json.dumps before DB write."""
with (
patch("tasks.document_indexing_sync_task.build_data_source_credentials") as mock_service_class,
patch("tasks.document_indexing_sync_task.NotionExtractor") as mock_extractor_class,
patch("tasks.document_indexing_sync_task.IndexProcessorFactory") as mock_ipf,
patch("tasks.document_indexing_sync_task.build_document_indexing_service") as mock_runner_class,
):
# External collaborators
mock_service = MagicMock()
mock_service.get_datasource_credentials.return_value = {"integration_secret": "token"}
mock_service_class.return_value.providers = mock_service
mock_extractor = MagicMock()
# Return a *different* timestamp so the task enters the sync/update branch
mock_extractor.get_notion_last_edited_time.return_value = "2024-02-01T00:00:00Z"
mock_extractor_class.return_value = mock_extractor
mock_ip = MagicMock()
mock_ipf.return_value.init_index_processor.return_value = mock_ip
mock_runner = MagicMock()
mock_runner_class.return_value = mock_runner
# Act
document_indexing_sync_task(dataset_id, document_id)
# Assert: data_source_info must be a JSON *string*, not a dict
sqlite_session.expire_all()
stored_document = sqlite_session.get(Document, document.id)
assert stored_document is not None
assert isinstance(stored_document.data_source_info, str), (
f"data_source_info should be a JSON string, got {type(stored_document.data_source_info).__name__}"
)
parsed = json.loads(stored_document.data_source_info)
assert parsed["last_edited_time"] == "2024-02-01T00:00:00Z"
assert stored_document.indexing_status == IndexingStatus.PARSING
assert stored_document.processing_started_at is not None
@@ -0,0 +1,605 @@
"""SQLite-backed tests for document indexing tasks.
The indexing task deliberately uses separate transactions for validation,
status persistence, indexing, and summary dispatch. These tests persist real
ORM rows so each phase observes only committed database state.
"""
import uuid
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import ANY, MagicMock, Mock, patch
import pytest
from sqlalchemy.orm import Session
from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType
from enums import CloudPlan, DeploymentEdition
from extensions.ext_redis import redis_client
from models.dataset import Dataset, Document
from models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus
from services.document_indexing_proxy.document_indexing_task_proxy import DocumentIndexingTaskProxy
from services.knowledge.indexing.errors import DocumentIsPausedError
from services.knowledge.resource_scope import DocumentRef
from tasks.document_indexing_task import (
_document_indexing,
_document_indexing_with_tenant_queue,
document_indexing_task,
normal_document_indexing_task,
priority_document_indexing_task,
)
from tasks.recover_document_indexing_task import recover_document_indexing_task
from tests.helpers.unit.config_override import apply_config_overrides, config_overrides_context
@pytest.mark.parametrize(
("status", "entry"),
[
("waiting", "run"),
("parsing", "run"),
("cleaning", "run"),
("splitting", "run_in_splitting_status"),
("indexing", "run_in_indexing_status"),
("completed", None),
],
)
def test_recovery_dispatches_detached_owner_reference(
sqlite_session, tenant_id, dataset_id, document_ids, status, entry
):
_persist_indexing_rows(sqlite_session, tenant_id=tenant_id, dataset_id=dataset_id, document_ids=document_ids[:1])
row = sqlite_session.get(Document, document_ids[0])
row.indexing_status = status
sqlite_session.commit()
with patch("tasks.recover_document_indexing_task.build_document_indexing_service") as factory:
recover_document_indexing_task.run(dataset_id, document_ids[0])
service = factory.return_value
if entry is None:
assert service.mock_calls == []
else:
method = {
"run": service.run,
"run_in_splitting_status": service.run_in_splitting_status,
"run_in_indexing_status": service.run_in_indexing_status,
}[entry]
method.assert_called_once()
(argument,) = method.call_args.args
ref = argument[0] if entry == "run" else argument
assert isinstance(ref, DocumentRef)
assert (ref.dataset.tenant_id, ref.dataset.dataset_id, ref.document_id) == (
tenant_id,
dataset_id,
document_ids[0],
)
@pytest.fixture
def tenant_id() -> str:
return str(uuid.uuid4())
@pytest.fixture
def dataset_id() -> str:
return str(uuid.uuid4())
@pytest.fixture
def document_ids() -> list[str]:
return [str(uuid.uuid4()) for _ in range(3)]
@pytest.fixture
def mock_redis() -> MagicMock:
"""Reset the external Redis boundary used by tenant-isolated queues."""
redis_client.reset_mock()
redis_client.get.return_value = None
redis_client.setex.return_value = True
redis_client.delete.return_value = True
redis_client.lpush.return_value = 1
redis_client.rpop.return_value = None
return redis_client
@pytest.fixture
def indexing_runner(monkeypatch: pytest.MonkeyPatch) -> MagicMock:
runner = MagicMock()
runner_class = MagicMock(return_value=runner)
monkeypatch.setattr("tasks.document_indexing_task.build_document_indexing_service", runner_class)
runner._constructor_mock = runner_class
return runner
def _features(
*,
plan: CloudPlan = CloudPlan.PROFESSIONAL,
vector_limit: int = 1000,
vector_size: int = 0,
) -> SimpleNamespace:
return SimpleNamespace(
billing=SimpleNamespace(subscription=SimpleNamespace(plan=plan)),
vector_space=SimpleNamespace(limit=vector_limit, size=vector_size),
)
def _patch_features(monkeypatch: pytest.MonkeyPatch, features: SimpleNamespace) -> MagicMock:
get_features = MagicMock(return_value=features)
monkeypatch.setattr("tasks.document_indexing_task.FeatureService.get_features", get_features)
return get_features
def _persist_indexing_rows(
session: Session,
*,
tenant_id: str,
dataset_id: str,
document_ids: list[str],
indexing_technique: IndexTechniqueType = IndexTechniqueType.HIGH_QUALITY,
summary_index_setting: dict[str, bool] | None = None,
document_forms: list[IndexStructureType] | None = None,
need_summary: list[bool] | None = None,
) -> tuple[Dataset, list[Document]]:
"""Persist one tenant-owned dataset and the requested document rows."""
created_by = str(uuid.uuid4())
dataset = Dataset(
id=dataset_id,
tenant_id=tenant_id,
name="Indexing dataset",
data_source_type=DataSourceType.UPLOAD_FILE,
indexing_technique=indexing_technique,
embedding_model_provider="openai",
embedding_model="text-embedding-3-small",
summary_index_setting=summary_index_setting,
created_by=created_by,
)
documents = [
Document(
id=document_id,
tenant_id=tenant_id,
dataset_id=dataset_id,
position=position,
data_source_type=DataSourceType.UPLOAD_FILE,
batch="batch-1",
name=f"document-{position}.txt",
created_from=DocumentCreatedFrom.WEB,
created_by=created_by,
indexing_status=IndexingStatus.WAITING,
doc_form=(document_forms or [IndexStructureType.PARAGRAPH_INDEX] * len(document_ids))[position - 1],
need_summary=(need_summary or [False] * len(document_ids))[position - 1],
)
for position, document_id in enumerate(document_ids, start=1)
]
session.add_all([dataset, *documents])
session.commit()
return dataset, documents
def _persisted_documents(session: Session, document_ids: list[str]) -> list[Document]:
session.expire_all()
return [document for document_id in document_ids if (document := session.get(Document, document_id)) is not None]
class TestTaskEnqueuing:
@config_overrides_context(DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY)
def test_self_hosted_dispatches_directly_to_priority_task(
self, tenant_id: str, dataset_id: str, document_ids: list[str], mock_redis: MagicMock
) -> None:
with (
patch.object(DocumentIndexingTaskProxy, "features"),
patch.object(DocumentIndexingTaskProxy, "PRIORITY_TASK_FUNC", Mock()) as task,
):
DocumentIndexingTaskProxy(tenant_id, dataset_id, document_ids).delay()
task.delay.assert_called_once_with(
tenant_id=tenant_id,
dataset_id=dataset_id,
document_ids=document_ids,
)
@config_overrides_context(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD)
@pytest.mark.parametrize(
("plan", "task_attribute"),
[
(CloudPlan.SANDBOX, "NORMAL_TASK_FUNC"),
(CloudPlan.PROFESSIONAL, "PRIORITY_TASK_FUNC"),
],
)
def test_cloud_dispatches_first_task_through_tenant_queue(
self,
tenant_id: str,
dataset_id: str,
document_ids: list[str],
mock_redis: MagicMock,
plan: CloudPlan,
task_attribute: str,
) -> None:
with (
patch.object(DocumentIndexingTaskProxy, "features") as features,
patch.object(DocumentIndexingTaskProxy, task_attribute, Mock()) as task,
):
features.billing.subscription.plan = plan
DocumentIndexingTaskProxy(tenant_id, dataset_id, document_ids).delay()
mock_redis.setex.assert_called()
task.delay.assert_called_once()
@config_overrides_context(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD)
def test_running_tenant_task_queues_followup_work(
self, tenant_id: str, dataset_id: str, document_ids: list[str], mock_redis: MagicMock
) -> None:
mock_redis.get.return_value = b"1"
with (
patch.object(DocumentIndexingTaskProxy, "features") as features,
patch.object(DocumentIndexingTaskProxy, "PRIORITY_TASK_FUNC", Mock()) as task,
):
features.billing.subscription.plan = CloudPlan.PROFESSIONAL
DocumentIndexingTaskProxy(tenant_id, dataset_id, document_ids).delay()
mock_redis.lpush.assert_called_once()
task.delay.assert_not_called()
class TestDocumentIndexing:
def test_legacy_task_persists_parsing_before_running(
self,
sqlite_session: Session,
tenant_id: str,
dataset_id: str,
document_ids: list[str],
indexing_runner: MagicMock,
monkeypatch: pytest.MonkeyPatch,
) -> None:
_persist_indexing_rows(
sqlite_session,
tenant_id=tenant_id,
dataset_id=dataset_id,
document_ids=document_ids,
)
_patch_features(monkeypatch, _features())
def assert_committed_parsing(refs: list[DocumentRef]) -> None:
with Session(sqlite_session.get_bind()) as observer:
documents = [observer.get(Document, ref.document_id) for ref in refs]
assert all(document.indexing_status == IndexingStatus.PARSING for document in documents)
assert all(document.processing_started_at is not None for document in documents)
indexing_runner.run.side_effect = assert_committed_parsing
document_indexing_task.run(dataset_id, document_ids)
persisted = _persisted_documents(sqlite_session, document_ids)
assert [document.indexing_status for document in persisted] == [IndexingStatus.PARSING] * 3
indexing_runner._constructor_mock.assert_called_once_with(
session_factory=ANY, enforce_vector_space_admission=True
)
indexing_runner.run.assert_called_once()
assert len(indexing_runner.run.call_args.args) == 1
def test_only_existing_documents_are_processed(
self,
sqlite_session: Session,
tenant_id: str,
dataset_id: str,
document_ids: list[str],
indexing_runner: MagicMock,
monkeypatch: pytest.MonkeyPatch,
) -> None:
existing_ids = [document_ids[0], document_ids[2]]
_persist_indexing_rows(
sqlite_session,
tenant_id=tenant_id,
dataset_id=dataset_id,
document_ids=existing_ids,
)
_patch_features(monkeypatch, _features())
_document_indexing(dataset_id, document_ids)
processed = indexing_runner.run.call_args.args[0]
assert {ref.document_id for ref in processed} == set(existing_ids)
assert sqlite_session.get(Document, document_ids[1]) is None
def test_empty_batch_still_reaches_runner(
self,
sqlite_session: Session,
tenant_id: str,
dataset_id: str,
indexing_runner: MagicMock,
monkeypatch: pytest.MonkeyPatch,
) -> None:
_persist_indexing_rows(
sqlite_session,
tenant_id=tenant_id,
dataset_id=dataset_id,
document_ids=[],
)
_patch_features(monkeypatch, _features())
_document_indexing(dataset_id, [])
assert indexing_runner.run.call_args.args[0] == []
assert len(indexing_runner.run.call_args.args) == 1
def test_missing_dataset_returns_before_feature_lookup(
self, dataset_id: str, document_ids: list[str], monkeypatch: pytest.MonkeyPatch
) -> None:
get_features = _patch_features(monkeypatch, _features())
runner_class = MagicMock()
monkeypatch.setattr("tasks.document_indexing_task.build_document_indexing_service", runner_class)
_document_indexing(dataset_id, document_ids)
get_features.assert_not_called()
runner_class.assert_not_called()
@config_overrides_context(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD)
@pytest.mark.parametrize(
("features", "batch_limit", "message"),
[
(_features(), 1, "batch upload limit"),
(_features(plan=CloudPlan.SANDBOX), 100, "does not support batch upload"),
(_features(vector_limit=100, vector_size=100), 100, "over the limit"),
],
)
def test_validation_failure_marks_every_scoped_document_error(
self,
sqlite_session: Session,
tenant_id: str,
dataset_id: str,
document_ids: list[str],
monkeypatch: pytest.MonkeyPatch,
features: SimpleNamespace,
batch_limit: int,
message: str,
) -> None:
_persist_indexing_rows(
sqlite_session,
tenant_id=tenant_id,
dataset_id=dataset_id,
document_ids=document_ids,
)
control_dataset_id = str(uuid.uuid4())
control_document_id = str(uuid.uuid4())
_persist_indexing_rows(
sqlite_session,
tenant_id=str(uuid.uuid4()),
dataset_id=control_dataset_id,
document_ids=[control_document_id],
)
_patch_features(monkeypatch, features)
apply_config_overrides(monkeypatch, BATCH_UPLOAD_LIMIT=str(batch_limit))
_document_indexing(dataset_id, document_ids)
persisted = _persisted_documents(sqlite_session, document_ids)
assert all(document.indexing_status == IndexingStatus.ERROR for document in persisted)
assert all(document.error and message in document.error for document in persisted)
assert all(document.stopped_at is not None for document in persisted)
control = sqlite_session.get(Document, control_document_id)
assert control is not None
assert control.indexing_status == IndexingStatus.WAITING
@pytest.mark.parametrize("error", [DocumentIsPausedError("paused"), RuntimeError("boom")])
def test_runner_failure_stops_before_summary_dispatch(
self,
sqlite_session: Session,
tenant_id: str,
dataset_id: str,
document_ids: list[str],
indexing_runner: MagicMock,
monkeypatch: pytest.MonkeyPatch,
error: Exception,
) -> None:
_persist_indexing_rows(
sqlite_session,
tenant_id=tenant_id,
dataset_id=dataset_id,
document_ids=document_ids,
summary_index_setting={"enable": True},
need_summary=[True] * len(document_ids),
)
_patch_features(monkeypatch, _features())
indexing_runner.run.side_effect = error
summary_delay = MagicMock()
monkeypatch.setattr("tasks.document_indexing_task.generate_summary_index_task.delay", summary_delay)
_document_indexing(dataset_id, document_ids)
summary_delay.assert_not_called()
persisted = _persisted_documents(sqlite_session, document_ids)
assert all(document.indexing_status == IndexingStatus.PARSING for document in persisted)
class TestSummaryDispatch:
def test_only_eligible_completed_documents_queue_summaries(
self,
sqlite_session: Session,
tenant_id: str,
dataset_id: str,
document_ids: list[str],
indexing_runner: MagicMock,
monkeypatch: pytest.MonkeyPatch,
) -> None:
_persist_indexing_rows(
sqlite_session,
tenant_id=tenant_id,
dataset_id=dataset_id,
document_ids=document_ids,
summary_index_setting={"enable": True},
document_forms=[
IndexStructureType.PARAGRAPH_INDEX,
IndexStructureType.QA_INDEX,
IndexStructureType.PARAGRAPH_INDEX,
],
need_summary=[True, True, True],
)
_patch_features(monkeypatch, _features())
def finish_documents(refs: list[DocumentRef]) -> None:
with Session(sqlite_session.get_bind()) as writer, writer.begin():
for ref, status in zip(
refs, [IndexingStatus.COMPLETED, IndexingStatus.COMPLETED, IndexingStatus.INDEXING], strict=True
):
writer.get(Document, ref.document_id).indexing_status = status
indexing_runner.run.side_effect = finish_documents
summary_delay = MagicMock()
monkeypatch.setattr("tasks.document_indexing_task.generate_summary_index_task.delay", summary_delay)
_document_indexing(dataset_id, document_ids)
summary_delay.assert_called_once_with(dataset_id, document_ids[0], None)
def test_summary_queue_failure_does_not_fail_indexing(
self,
sqlite_session: Session,
tenant_id: str,
dataset_id: str,
indexing_runner: MagicMock,
monkeypatch: pytest.MonkeyPatch,
) -> None:
document_id = str(uuid.uuid4())
_persist_indexing_rows(
sqlite_session,
tenant_id=tenant_id,
dataset_id=dataset_id,
document_ids=[document_id],
summary_index_setting={"enable": True},
need_summary=[True],
)
_patch_features(monkeypatch, _features())
def complete_documents(refs: list[DocumentRef]) -> None:
with Session(sqlite_session.get_bind()) as writer, writer.begin():
writer.get(Document, refs[0].document_id).indexing_status = IndexingStatus.COMPLETED
indexing_runner.run.side_effect = complete_documents
summary_delay = MagicMock(side_effect=RuntimeError("queue unavailable"))
monkeypatch.setattr("tasks.document_indexing_task.generate_summary_index_task.delay", summary_delay)
_document_indexing(dataset_id, [document_id])
summary_delay.assert_called_once_with(dataset_id, document_id, None)
persisted = _persisted_documents(sqlite_session, [document_id])[0]
assert persisted.indexing_status == IndexingStatus.COMPLETED
def test_economy_indexing_skips_summary_generation(
self,
sqlite_session: Session,
tenant_id: str,
dataset_id: str,
indexing_runner: MagicMock,
monkeypatch: pytest.MonkeyPatch,
) -> None:
document_id = str(uuid.uuid4())
_persist_indexing_rows(
sqlite_session,
tenant_id=tenant_id,
dataset_id=dataset_id,
document_ids=[document_id],
indexing_technique=IndexTechniqueType.ECONOMY,
summary_index_setting={"enable": True},
need_summary=[True],
)
_patch_features(monkeypatch, _features())
def complete_documents(refs: list[DocumentRef]) -> None:
with Session(sqlite_session.get_bind()) as writer, writer.begin():
writer.get(Document, refs[0].document_id).indexing_status = IndexingStatus.COMPLETED
indexing_runner.run.side_effect = complete_documents
summary_delay = MagicMock()
monkeypatch.setattr("tasks.document_indexing_task.generate_summary_index_task.delay", summary_delay)
_document_indexing(dataset_id, [document_id])
summary_delay.assert_not_called()
def test_dataset_removed_by_runner_is_absent_from_summary_phase(
self,
sqlite_session: Session,
tenant_id: str,
dataset_id: str,
indexing_runner: MagicMock,
monkeypatch: pytest.MonkeyPatch,
) -> None:
document_id = str(uuid.uuid4())
_persist_indexing_rows(
sqlite_session,
tenant_id=tenant_id,
dataset_id=dataset_id,
document_ids=[document_id],
summary_index_setting={"enable": True},
need_summary=[True],
)
_patch_features(monkeypatch, _features())
def remove_dataset(_refs: list[DocumentRef]) -> None:
with Session(sqlite_session.get_bind()) as writer, writer.begin():
dataset = writer.get(Dataset, dataset_id)
assert dataset is not None
writer.delete(dataset)
indexing_runner.run.side_effect = remove_dataset
summary_delay = MagicMock()
monkeypatch.setattr("tasks.document_indexing_task.generate_summary_index_task.delay", summary_delay)
_document_indexing(dataset_id, [document_id])
sqlite_session.expire_all()
assert sqlite_session.get(Dataset, dataset_id) is None
summary_delay.assert_not_called()
class TestTenantQueue:
def test_followup_tasks_are_dispatched_with_one_shared_producer(
self, tenant_id: str, dataset_id: str, document_ids: list[str], monkeypatch: pytest.MonkeyPatch
) -> None:
next_documents = [str(uuid.uuid4())]
queue = MagicMock()
queue.pull_tasks.return_value = [
{"tenant_id": tenant_id, "dataset_id": dataset_id, "document_ids": next_documents}
]
monkeypatch.setattr("tasks.document_indexing_task.TenantIsolatedTaskQueue", MagicMock(return_value=queue))
monkeypatch.setattr("tasks.document_indexing_task._document_indexing", MagicMock())
producer = object()
monkeypatch.setattr(
"tasks.document_indexing_task.current_app.producer_or_acquire",
MagicMock(return_value=nullcontext(producer)),
)
task = MagicMock()
_document_indexing_with_tenant_queue(tenant_id, dataset_id, document_ids, task)
task.apply_async.assert_called_once_with(
kwargs={"tenant_id": tenant_id, "dataset_id": dataset_id, "document_ids": next_documents},
producer=producer,
)
queue.set_task_waiting_time.assert_called_once()
queue.delete_task_key.assert_not_called()
def test_queue_cleanup_runs_when_indexing_fails(
self, tenant_id: str, dataset_id: str, document_ids: list[str], monkeypatch: pytest.MonkeyPatch
) -> None:
queue = MagicMock()
queue.pull_tasks.return_value = []
monkeypatch.setattr("tasks.document_indexing_task.TenantIsolatedTaskQueue", MagicMock(return_value=queue))
indexing = MagicMock(side_effect=RuntimeError("indexing failed"))
monkeypatch.setattr("tasks.document_indexing_task._document_indexing", indexing)
_document_indexing_with_tenant_queue(tenant_id, dataset_id, document_ids, MagicMock())
queue.delete_task_key.assert_called_once()
@pytest.mark.parametrize("task", [normal_document_indexing_task, priority_document_indexing_task])
def test_celery_entrypoints_delegate_to_tenant_queue(
self,
task: object,
tenant_id: str,
dataset_id: str,
document_ids: list[str],
monkeypatch: pytest.MonkeyPatch,
) -> None:
delegate = MagicMock()
monkeypatch.setattr("tasks.document_indexing_task._document_indexing_with_tenant_queue", delegate)
task.run(tenant_id, dataset_id, document_ids) # type: ignore[attr-defined]
delegate.assert_called_once()
@@ -0,0 +1,458 @@
"""SQLite-backed tests for document update indexing and summary generation."""
from __future__ import annotations
import uuid
from collections.abc import Callable
from datetime import UTC, datetime
from functools import partial
from unittest.mock import MagicMock
import pytest
from sqlalchemy import select
from sqlalchemy.orm import Session
import tasks.document_indexing_update_task as task_module
from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType
from extensions.storage.storage_type import StorageType
from models.dataset import Dataset, Document, DocumentSegment, SegmentAttachmentBinding
from models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, IndexingStatus
from models.model import UploadFile
from services.knowledge.indexing.errors import DocumentIsPausedError
from tasks.document_indexing_update_task import document_indexing_update_task
@pytest.fixture
def task_harness(
sqlite_session: Session,
monkeypatch: pytest.MonkeyPatch,
) -> tuple[MagicMock, MagicMock]:
"""Bind task-owned sessions to SQLite and keep only external boundaries mocked."""
engine = sqlite_session.get_bind()
monkeypatch.setattr(
task_module.session_factory,
"create_session",
lambda: Session(engine, expire_on_commit=False),
)
runner = MagicMock()
processor = MagicMock()
monkeypatch.setattr(task_module, "build_document_indexing_service", MagicMock(return_value=runner))
monkeypatch.setattr(
task_module,
"IndexProcessorFactory",
MagicMock(return_value=MagicMock(init_index_processor=MagicMock(return_value=processor))),
)
return runner, processor
def _persist_rows(
session: Session,
*,
indexing_technique: IndexTechniqueType = IndexTechniqueType.HIGH_QUALITY,
summary_index_setting: dict | None = None,
doc_form: IndexStructureType = IndexStructureType.PARAGRAPH_INDEX,
need_summary: bool = True,
with_segment: bool = False,
) -> tuple[Dataset, Document]:
tenant_id = str(uuid.uuid4())
dataset_id = str(uuid.uuid4())
document_id = str(uuid.uuid4())
created_by = str(uuid.uuid4())
dataset = Dataset(
id=dataset_id,
tenant_id=tenant_id,
name="Update dataset",
data_source_type=DataSourceType.UPLOAD_FILE,
created_by=created_by,
indexing_technique=indexing_technique,
summary_index_setting=summary_index_setting,
)
document = Document(
id=document_id,
tenant_id=tenant_id,
dataset_id=dataset_id,
position=1,
data_source_type=DataSourceType.UPLOAD_FILE,
batch="batch-1",
name="document.txt",
created_from=DocumentCreatedFrom.WEB,
created_by=created_by,
indexing_status=IndexingStatus.WAITING,
doc_form=doc_form,
need_summary=need_summary,
)
rows: list[object] = [dataset, document]
if with_segment:
rows.append(
DocumentSegment(
tenant_id=tenant_id,
dataset_id=dataset_id,
document_id=document_id,
position=1,
content="segment",
word_count=1,
tokens=1,
created_by=created_by,
index_node_id="node-1",
)
)
session.add_all(rows)
session.commit()
return dataset, document
def _complete_indexing(refs, *, bind) -> None:
with Session(bind) as writer, writer.begin():
for ref in refs:
writer.get(Document, ref.document_id).indexing_status = IndexingStatus.COMPLETED
def _persist_attachment(
session: Session,
*,
dataset: Dataset,
document: Document,
segment: DocumentSegment,
key: str,
) -> tuple[UploadFile, SegmentAttachmentBinding]:
attachment = UploadFile(
tenant_id=dataset.tenant_id,
storage_type=StorageType.LOCAL,
key=key,
name="image.png",
size=10,
extension="png",
mime_type="image/png",
created_by_role=CreatorUserRole.ACCOUNT,
created_by=document.created_by,
created_at=datetime.now(UTC),
used=True,
)
binding = SegmentAttachmentBinding(
tenant_id=dataset.tenant_id,
dataset_id=dataset.id,
document_id=document.id,
segment_id=segment.id,
attachment_id=attachment.id,
)
session.add_all([attachment, binding])
session.commit()
return attachment, binding
def test_queues_summary_when_all_persisted_conditions_match(
sqlite_session: Session,
task_harness: tuple[MagicMock, MagicMock],
monkeypatch: pytest.MonkeyPatch,
) -> None:
runner, _processor = task_harness
dataset, document = _persist_rows(sqlite_session, summary_index_setting={"enable": True})
runner.run.side_effect = partial(_complete_indexing, bind=sqlite_session.get_bind())
delay = MagicMock()
monkeypatch.setattr(task_module.generate_summary_index_task, "delay", delay)
document_indexing_update_task(dataset.id, document.id)
delay.assert_called_once_with(dataset.id, document.id, None)
sqlite_session.expire_all()
assert sqlite_session.get(Document, document.id).indexing_status == IndexingStatus.COMPLETED # type: ignore[union-attr]
@pytest.mark.parametrize(
("dataset_changes", "document_changes"),
[
({"indexing_technique": IndexTechniqueType.ECONOMY}, {}),
({"summary_index_setting": None}, {}),
({"summary_index_setting": {"enable": False}}, {}),
({"summary_index_setting": {"enable": True}}, {"need_summary": False}),
(
{"summary_index_setting": {"enable": True}},
{"doc_form": IndexStructureType.QA_INDEX},
),
],
)
def test_skips_summary_when_persisted_eligibility_does_not_match(
sqlite_session: Session,
task_harness: tuple[MagicMock, MagicMock],
monkeypatch: pytest.MonkeyPatch,
dataset_changes: dict,
document_changes: dict,
) -> None:
runner, _processor = task_harness
dataset, document = _persist_rows(
sqlite_session,
summary_index_setting={"enable": True},
)
for key, value in dataset_changes.items():
setattr(dataset, key, value)
for key, value in document_changes.items():
setattr(document, key, value)
sqlite_session.commit()
runner.run.side_effect = partial(_complete_indexing, bind=sqlite_session.get_bind())
delay = MagicMock()
monkeypatch.setattr(task_module.generate_summary_index_task, "delay", delay)
document_indexing_update_task(dataset.id, document.id)
delay.assert_not_called()
@pytest.mark.parametrize(
"error_factory",
[
lambda _document_id: RuntimeError("indexing failed"),
lambda document_id: DocumentIsPausedError(f"{document_id} is paused"),
],
)
def test_skips_summary_when_indexing_fails_or_is_paused(
sqlite_session: Session,
task_harness: tuple[MagicMock, MagicMock],
monkeypatch: pytest.MonkeyPatch,
error_factory: Callable[[str], Exception],
) -> None:
runner, _processor = task_harness
dataset, document = _persist_rows(sqlite_session, summary_index_setting={"enable": True})
runner.run.side_effect = error_factory(document.id)
delay = MagicMock()
monkeypatch.setattr(task_module.generate_summary_index_task, "delay", delay)
document_indexing_update_task(dataset.id, document.id)
delay.assert_not_called()
def test_returns_without_opening_external_boundaries_when_document_is_missing(
task_harness: tuple[MagicMock, MagicMock],
) -> None:
runner, processor = task_harness
document_indexing_update_task(str(uuid.uuid4()), str(uuid.uuid4()))
runner.run.assert_not_called()
processor.clean.assert_not_called()
def test_skips_summary_when_dataset_is_removed_after_indexing(
sqlite_session: Session,
task_harness: tuple[MagicMock, MagicMock],
monkeypatch: pytest.MonkeyPatch,
) -> None:
runner, _processor = task_harness
dataset, document = _persist_rows(sqlite_session, summary_index_setting={"enable": True})
def complete_and_remove(documents: list[Document], session: Session) -> None:
_complete_indexing(documents, session)
persisted_dataset = session.get(Dataset, dataset.id)
assert persisted_dataset is not None
session.delete(persisted_dataset)
runner.run.side_effect = complete_and_remove
delay = MagicMock()
monkeypatch.setattr(task_module.generate_summary_index_task, "delay", delay)
document_indexing_update_task(dataset.id, document.id)
delay.assert_not_called()
def test_skips_summary_when_runner_leaves_document_incomplete(
sqlite_session: Session,
task_harness: tuple[MagicMock, MagicMock],
monkeypatch: pytest.MonkeyPatch,
) -> None:
runner, _processor = task_harness
dataset, document = _persist_rows(sqlite_session, summary_index_setting={"enable": True})
runner.run.return_value = None
delay = MagicMock()
monkeypatch.setattr(task_module.generate_summary_index_task, "delay", delay)
document_indexing_update_task(dataset.id, document.id)
delay.assert_not_called()
def test_queue_failure_is_swallowed_after_successful_indexing(
sqlite_session: Session,
task_harness: tuple[MagicMock, MagicMock],
monkeypatch: pytest.MonkeyPatch,
) -> None:
runner, _processor = task_harness
dataset, document = _persist_rows(sqlite_session, summary_index_setting={"enable": True})
runner.run.side_effect = partial(_complete_indexing, bind=sqlite_session.get_bind())
monkeypatch.setattr(
task_module.generate_summary_index_task,
"delay",
MagicMock(side_effect=RuntimeError("queue unavailable")),
)
document_indexing_update_task(dataset.id, document.id)
sqlite_session.expire_all()
assert sqlite_session.get(Document, document.id).indexing_status == IndexingStatus.COMPLETED # type: ignore[union-attr]
def test_cleans_and_deletes_persisted_segments_with_real_session(
sqlite_session: Session,
task_harness: tuple[MagicMock, MagicMock],
monkeypatch: pytest.MonkeyPatch,
) -> None:
runner, processor = task_harness
dataset, document = _persist_rows(
sqlite_session,
summary_index_setting={"enable": True},
with_segment=True,
)
runner.run.side_effect = partial(_complete_indexing, bind=sqlite_session.get_bind())
delay = MagicMock()
monkeypatch.setattr(task_module.generate_summary_index_task, "delay", delay)
document_indexing_update_task(dataset.id, document.id)
processor.clean.assert_called_once()
assert isinstance(processor.clean.call_args.kwargs["session"], Session)
assert sqlite_session.scalars(select(DocumentSegment).where(DocumentSegment.document_id == document.id)).all() == []
delay.assert_called_once_with(dataset.id, document.id, None)
def test_removes_orphaned_multimodal_attachments_during_reindex(
sqlite_session: Session,
task_harness: tuple[MagicMock, MagicMock],
monkeypatch: pytest.MonkeyPatch,
) -> None:
runner, processor = task_harness
dataset, document = _persist_rows(sqlite_session, with_segment=True)
dataset.is_multimodal = True
segment = sqlite_session.scalar(select(DocumentSegment).where(DocumentSegment.document_id == document.id))
assert segment is not None
attachment, binding = _persist_attachment(
sqlite_session,
dataset=dataset,
document=document,
segment=segment,
key="attachments/orphaned-image.png",
)
attachment_id = attachment.id
binding_id = binding.id
storage_delete = MagicMock()
monkeypatch.setattr(task_module.storage, "delete", storage_delete)
document_indexing_update_task(dataset.id, document.id)
assert processor.clean.call_count == 2
assert processor.clean.call_args_list[0].args[1] == ["node-1"]
assert processor.clean.call_args_list[1].kwargs["node_ids"] == [attachment_id]
assert processor.clean.call_args_list[1].kwargs["with_keywords"] is False
storage_delete.assert_called_once_with("attachments/orphaned-image.png")
sqlite_session.expire_all()
assert sqlite_session.get(SegmentAttachmentBinding, binding_id) is None
assert sqlite_session.get(UploadFile, attachment_id) is None
assert sqlite_session.scalars(select(DocumentSegment).where(DocumentSegment.document_id == document.id)).all() == []
runner.run.assert_called_once()
def test_preserves_multimodal_attachment_referenced_by_another_document_during_reindex(
sqlite_session: Session,
task_harness: tuple[MagicMock, MagicMock],
monkeypatch: pytest.MonkeyPatch,
) -> None:
runner, processor = task_harness
dataset, document = _persist_rows(sqlite_session, with_segment=True)
dataset.is_multimodal = True
segment = sqlite_session.scalar(select(DocumentSegment).where(DocumentSegment.document_id == document.id))
assert segment is not None
attachment, binding = _persist_attachment(
sqlite_session,
dataset=dataset,
document=document,
segment=segment,
key="attachments/shared-image.png",
)
other_document = Document(
id=str(uuid.uuid4()),
tenant_id=dataset.tenant_id,
dataset_id=dataset.id,
position=2,
data_source_type=DataSourceType.UPLOAD_FILE,
batch="batch-2",
name="other-document.txt",
created_from=DocumentCreatedFrom.WEB,
created_by=document.created_by,
indexing_status=IndexingStatus.COMPLETED,
doc_form=IndexStructureType.PARAGRAPH_INDEX,
)
sqlite_session.add(other_document)
sqlite_session.flush()
other_segment = DocumentSegment(
tenant_id=dataset.tenant_id,
dataset_id=dataset.id,
document_id=other_document.id,
position=1,
content="other segment",
word_count=2,
tokens=2,
created_by=document.created_by,
index_node_id="node-2",
)
sqlite_session.add(other_segment)
sqlite_session.flush()
shared_binding = SegmentAttachmentBinding(
tenant_id=dataset.tenant_id,
dataset_id=dataset.id,
document_id=other_document.id,
segment_id=other_segment.id,
attachment_id=attachment.id,
)
sqlite_session.add(shared_binding)
sqlite_session.commit()
attachment_id = attachment.id
binding_id = binding.id
shared_binding_id = shared_binding.id
storage_delete = MagicMock()
monkeypatch.setattr(task_module.storage, "delete", storage_delete)
document_indexing_update_task(dataset.id, document.id)
processor.clean.assert_called_once()
assert processor.clean.call_args.args[1] == ["node-1"]
storage_delete.assert_not_called()
sqlite_session.expire_all()
assert sqlite_session.get(SegmentAttachmentBinding, binding_id) is None
assert sqlite_session.get(SegmentAttachmentBinding, shared_binding_id) is not None
assert sqlite_session.get(UploadFile, attachment_id) is not None
runner.run.assert_called_once()
def test_keeps_database_cleanup_when_reindex_attachment_storage_delete_fails(
sqlite_session: Session,
task_harness: tuple[MagicMock, MagicMock],
monkeypatch: pytest.MonkeyPatch,
caplog: pytest.LogCaptureFixture,
) -> None:
runner, _processor = task_harness
dataset, document = _persist_rows(sqlite_session, with_segment=True)
dataset.is_multimodal = True
segment = sqlite_session.scalar(select(DocumentSegment).where(DocumentSegment.document_id == document.id))
assert segment is not None
attachment, binding = _persist_attachment(
sqlite_session,
dataset=dataset,
document=document,
segment=segment,
key="attachments/failing-image.png",
)
attachment_id = attachment.id
binding_id = binding.id
storage_delete = MagicMock(side_effect=RuntimeError("storage unavailable"))
monkeypatch.setattr(task_module.storage, "delete", storage_delete)
with caplog.at_level("ERROR", logger="tasks.document_indexing_update_task"):
document_indexing_update_task(dataset.id, document.id)
storage_delete.assert_called_once_with("attachments/failing-image.png")
assert "Failed to delete document attachment from storage during re-indexing" in caplog.text
sqlite_session.expire_all()
assert sqlite_session.get(SegmentAttachmentBinding, binding_id) is None
assert sqlite_session.get(UploadFile, attachment_id) is None
assert sqlite_session.scalars(select(DocumentSegment).where(DocumentSegment.document_id == document.id)).all() == []
runner.run.assert_called_once()
@@ -0,0 +1,256 @@
"""Unit tests for queue/wrapper behaviors in duplicate document indexing tasks (non-database logic)."""
import json
import uuid
from unittest.mock import MagicMock, Mock, create_autospec, patch
import pytest
from tasks.duplicate_document_indexing_task import (
_duplicate_document_indexing_task,
_duplicate_document_indexing_task_with_tenant_queue,
duplicate_document_indexing_task,
normal_duplicate_document_indexing_task,
priority_duplicate_document_indexing_task,
)
@pytest.fixture
def tenant_id() -> str:
return str(uuid.uuid4())
@pytest.fixture
def dataset_id() -> str:
return str(uuid.uuid4())
@pytest.fixture
def document_ids() -> list[str]:
return [str(uuid.uuid4()) for _ in range(3)]
class TestDuplicateDocumentIndexingTask:
"""Tests for the deprecated duplicate_document_indexing_task function."""
@patch("tasks.duplicate_document_indexing_task._duplicate_document_indexing_task", autospec=True)
def test_duplicate_document_indexing_task_calls_core_function(
self, mock_core_func: MagicMock, dataset_id: str, document_ids: list[str]
) -> None:
"""Test that duplicate_document_indexing_task calls the core _duplicate_document_indexing_task function."""
# Act
duplicate_document_indexing_task(dataset_id, document_ids)
# Assert
mock_core_func.assert_called_once_with(dataset_id, document_ids)
@patch("tasks.duplicate_document_indexing_task._duplicate_document_indexing_task", autospec=True)
def test_duplicate_document_indexing_task_with_empty_document_ids(
self, mock_core_func: MagicMock, dataset_id: str
) -> None:
"""Test duplicate_document_indexing_task with empty document_ids list."""
# Arrange
document_ids = []
# Act
duplicate_document_indexing_task(dataset_id, document_ids)
# Assert
mock_core_func.assert_called_once_with(dataset_id, document_ids)
class TestDuplicateDocumentIndexingTaskWithTenantQueue:
"""Tests for _duplicate_document_indexing_task_with_tenant_queue function."""
@patch("tasks.duplicate_document_indexing_task._duplicate_document_indexing_task", autospec=True)
@pytest.mark.usefixtures("tenant_queue_commands")
def test_tenant_queue_wrapper_calls_core_function(
self,
mock_core_func: MagicMock,
tenant_id: str,
dataset_id: str,
document_ids: list[str],
) -> None:
"""Test that tenant queue wrapper calls the core function."""
# Arrange
mock_task_func = Mock()
# Act
_duplicate_document_indexing_task_with_tenant_queue(tenant_id, dataset_id, document_ids, mock_task_func)
# Assert
mock_core_func.assert_called_once_with(dataset_id, document_ids)
@patch(
"tasks.duplicate_document_indexing_task._duplicate_document_indexing_task",
new=create_autospec(_duplicate_document_indexing_task),
)
def test_tenant_queue_wrapper_deletes_key_when_no_tasks(
self,
tenant_queue_commands: MagicMock,
tenant_id: str,
dataset_id: str,
document_ids: list[str],
) -> None:
"""Test that tenant queue wrapper deletes task key when no more tasks."""
# Arrange
mock_task_func = Mock()
tenant_queue_commands.return_value = None
# Act
_duplicate_document_indexing_task_with_tenant_queue(tenant_id, dataset_id, document_ids, mock_task_func)
# Assert
tenant_queue_commands.assert_any_call("DEL", f"tenant_duplicate_document_indexing_task:{tenant_id}")
@patch(
"tasks.duplicate_document_indexing_task._duplicate_document_indexing_task",
new=create_autospec(_duplicate_document_indexing_task),
)
def test_tenant_queue_wrapper_processes_next_tasks(
self,
tenant_queue_commands: MagicMock,
tenant_id: str,
dataset_id: str,
document_ids: list[str],
) -> None:
"""Test that tenant queue wrapper processes next tasks from queue."""
# Arrange
mock_task_func = Mock()
next_task = {
"tenant_id": tenant_id,
"dataset_id": dataset_id,
"document_ids": document_ids,
}
tenant_queue_commands.side_effect = [json.dumps({"data": next_task}), None, None]
# Act
_duplicate_document_indexing_task_with_tenant_queue(tenant_id, dataset_id, document_ids, mock_task_func)
# Assert
tenant_queue_commands.assert_any_call("SETEX", f"tenant_duplicate_document_indexing_task:{tenant_id}", 3600, 1)
mock_task_func.delay.assert_called_once_with(
tenant_id=tenant_id,
dataset_id=dataset_id,
document_ids=document_ids,
)
@patch("tasks.duplicate_document_indexing_task._duplicate_document_indexing_task", autospec=True)
def test_tenant_queue_wrapper_handles_core_function_error(
self,
mock_core_func: MagicMock,
tenant_queue_commands: MagicMock,
tenant_id: str,
dataset_id: str,
document_ids: list[str],
) -> None:
"""Test that tenant queue wrapper handles errors from core function."""
# Arrange
mock_task_func = Mock()
mock_core_func.side_effect = Exception("Core function error")
# Act
_duplicate_document_indexing_task_with_tenant_queue(tenant_id, dataset_id, document_ids, mock_task_func)
# Assert
# Should still check for next tasks even after error
tenant_queue_commands.assert_any_call("RPOP", f"tenant_self_duplicate_document_indexing_task_queue:{tenant_id}")
class TestNormalDuplicateDocumentIndexingTask:
"""Tests for normal_duplicate_document_indexing_task function."""
@patch("tasks.duplicate_document_indexing_task._duplicate_document_indexing_task_with_tenant_queue", autospec=True)
def test_normal_task_calls_tenant_queue_wrapper(
self,
mock_wrapper_func: MagicMock,
tenant_id: str,
dataset_id: str,
document_ids: list[str],
) -> None:
"""Test that normal task calls tenant queue wrapper."""
# Act
normal_duplicate_document_indexing_task(tenant_id, dataset_id, document_ids)
# Assert
mock_wrapper_func.assert_called_once_with(
tenant_id, dataset_id, document_ids, normal_duplicate_document_indexing_task
)
@patch("tasks.duplicate_document_indexing_task._duplicate_document_indexing_task_with_tenant_queue", autospec=True)
def test_normal_task_with_empty_document_ids(
self,
mock_wrapper_func: MagicMock,
tenant_id: str,
dataset_id: str,
) -> None:
"""Test normal task with empty document_ids list."""
# Arrange
document_ids = []
# Act
normal_duplicate_document_indexing_task(tenant_id, dataset_id, document_ids)
# Assert
mock_wrapper_func.assert_called_once_with(
tenant_id, dataset_id, document_ids, normal_duplicate_document_indexing_task
)
class TestPriorityDuplicateDocumentIndexingTask:
"""Tests for priority_duplicate_document_indexing_task function."""
@patch("tasks.duplicate_document_indexing_task._duplicate_document_indexing_task_with_tenant_queue", autospec=True)
def test_priority_task_calls_tenant_queue_wrapper(
self,
mock_wrapper_func: MagicMock,
tenant_id: str,
dataset_id: str,
document_ids: list[str],
) -> None:
"""Test that priority task calls tenant queue wrapper."""
# Act
priority_duplicate_document_indexing_task(tenant_id, dataset_id, document_ids)
# Assert
mock_wrapper_func.assert_called_once_with(
tenant_id, dataset_id, document_ids, priority_duplicate_document_indexing_task
)
@patch("tasks.duplicate_document_indexing_task._duplicate_document_indexing_task_with_tenant_queue", autospec=True)
def test_priority_task_with_single_document(
self,
mock_wrapper_func: MagicMock,
tenant_id: str,
dataset_id: str,
) -> None:
"""Test priority task with single document."""
# Arrange
document_ids = ["doc-1"]
# Act
priority_duplicate_document_indexing_task(tenant_id, dataset_id, document_ids)
# Assert
mock_wrapper_func.assert_called_once_with(
tenant_id, dataset_id, document_ids, priority_duplicate_document_indexing_task
)
@patch("tasks.duplicate_document_indexing_task._duplicate_document_indexing_task_with_tenant_queue", autospec=True)
def test_priority_task_with_large_batch(
self,
mock_wrapper_func: MagicMock,
tenant_id: str,
dataset_id: str,
) -> None:
"""Test priority task with large batch of documents."""
# Arrange
document_ids = [f"doc-{i}" for i in range(100)]
# Act
priority_duplicate_document_indexing_task(tenant_id, dataset_id, document_ids)
# Assert
mock_wrapper_func.assert_called_once_with(
tenant_id, dataset_id, document_ids, priority_duplicate_document_indexing_task
)
@@ -0,0 +1,236 @@
import uuid
from collections.abc import Generator
from contextlib import contextmanager
from unittest.mock import MagicMock, patch
import pytest
from sqlalchemy import event
from sqlalchemy.orm import Session, sessionmaker
from core.rag.index_processor.constant.index_type import IndexStructureType
from models.dataset import Dataset, Document, DocumentSegment
from models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus, SegmentStatus
from tasks.enable_segment_to_index_task import enable_segment_to_index_task
from tasks.enable_segments_to_index_task import enable_segments_to_index_task
@pytest.fixture
def indexed_segment(sqlite_session: Session) -> tuple[Dataset, Document, DocumentSegment]:
"""Persist the complete owner chain consumed by segment indexing tasks."""
tenant_id = str(uuid.uuid4())
created_by = str(uuid.uuid4())
dataset = Dataset(
id=str(uuid.uuid4()),
tenant_id=tenant_id,
name="Indexing dataset",
data_source_type=DataSourceType.UPLOAD_FILE,
created_by=created_by,
is_multimodal=False,
)
document = Document(
id=str(uuid.uuid4()),
tenant_id=tenant_id,
dataset_id=dataset.id,
position=1,
data_source_type=DataSourceType.UPLOAD_FILE,
batch="batch-1",
name="document.txt",
created_from=DocumentCreatedFrom.WEB,
created_by=created_by,
indexing_status=IndexingStatus.COMPLETED,
doc_form=IndexStructureType.PARAGRAPH_INDEX,
)
segment = DocumentSegment(
tenant_id=tenant_id,
dataset_id=dataset.id,
document_id=document.id,
position=1,
content="content",
word_count=1,
tokens=1,
created_by=created_by,
index_node_id="node-1",
index_node_hash="hash-1",
status=SegmentStatus.COMPLETED,
)
sqlite_session.add_all([dataset, document, segment])
sqlite_session.commit()
return dataset, document, segment
@contextmanager
def _record_transaction_events(
sqlite_session_factory: sessionmaker[Session], phase_events: list[str]
) -> Generator[None]:
"""Record real transaction boundaries from task-owned SQLite sessions."""
session_type = sqlite_session_factory.class_
def after_commit(_session: Session) -> None:
phase_events.append("commit")
def after_rollback(_session: Session) -> None:
phase_events.append("rollback")
event.listen(session_type, "after_commit", after_commit)
event.listen(session_type, "after_rollback", after_rollback)
try:
yield
finally:
event.remove(session_type, "after_commit", after_commit)
event.remove(session_type, "after_rollback", after_rollback)
def test_enable_segment_commits_index_rows_after_loading(
indexed_segment: tuple[Dataset, Document, DocumentSegment],
sqlite_session_factory: sessionmaker[Session],
) -> None:
dataset, _document, segment = indexed_segment
phase_events: list[str] = []
index_processor = MagicMock()
index_processor.load.side_effect = lambda *_args, **_kwargs: phase_events.append("load")
enable_summaries = MagicMock(side_effect=lambda *_args, **_kwargs: phase_events.append("summary"))
with (
_record_transaction_events(sqlite_session_factory, phase_events),
patch("tasks.enable_segment_to_index_task.IndexProcessorFactory") as processor_factory,
patch(
"services.knowledge.summaries.adapters.SummaryIndexAdapter.enable_summaries_for_segments",
enable_summaries,
),
patch("tasks.enable_segment_to_index_task.redis_client.delete"),
):
processor_factory.return_value.init_index_processor.return_value = index_processor
enable_segment_to_index_task.run(segment.id)
assert phase_events == ["load", "commit", "summary"]
enable_summaries.assert_called_once()
assert enable_summaries.call_args.kwargs["dataset"].id == dataset.id
assert enable_summaries.call_args.kwargs["segment_ids"] == [segment.id]
def test_enable_segment_rolls_back_before_error_compensation(
sqlite_session: Session,
indexed_segment: tuple[Dataset, Document, DocumentSegment],
sqlite_session_factory: sessionmaker[Session],
) -> None:
_dataset, _document, segment = indexed_segment
phase_events: list[str] = []
index_processor = MagicMock()
def fail_load(*_args: object, **_kwargs: object) -> None:
phase_events.append("load")
raise RuntimeError("load failed")
index_processor.load.side_effect = fail_load
with (
_record_transaction_events(sqlite_session_factory, phase_events),
patch("tasks.enable_segment_to_index_task.IndexProcessorFactory") as processor_factory,
patch(
"services.knowledge.summaries.adapters.SummaryIndexAdapter.enable_summaries_for_segments"
) as enable_summaries,
patch("tasks.enable_segment_to_index_task.redis_client.delete"),
):
processor_factory.return_value.init_index_processor.return_value = index_processor
enable_segment_to_index_task.run(segment.id)
sqlite_session.expire_all()
persisted_segment = sqlite_session.get(DocumentSegment, segment.id)
assert persisted_segment is not None
assert persisted_segment.enabled is False
assert persisted_segment.status == SegmentStatus.ERROR
assert persisted_segment.error == "load failed"
assert persisted_segment.disabled_at is not None
assert phase_events == ["load", "rollback", "commit"]
enable_summaries.assert_not_called()
def test_enable_segments_commits_index_rows_after_loading(
indexed_segment: tuple[Dataset, Document, DocumentSegment],
sqlite_session_factory: sessionmaker[Session],
) -> None:
dataset, document, segment = indexed_segment
phase_events: list[str] = []
index_processor = MagicMock()
index_processor.load.side_effect = lambda *_args, **_kwargs: phase_events.append("load")
enable_summaries = MagicMock(side_effect=lambda *_args, **_kwargs: phase_events.append("summary"))
with (
_record_transaction_events(sqlite_session_factory, phase_events),
patch("tasks.enable_segments_to_index_task.IndexProcessorFactory") as processor_factory,
patch(
"services.knowledge.summaries.adapters.SummaryIndexAdapter.enable_summaries_for_segments",
enable_summaries,
),
patch("tasks.enable_segments_to_index_task.redis_client.delete"),
):
processor_factory.return_value.init_index_processor.return_value = index_processor
enable_segments_to_index_task.run([segment.id], dataset.id, document.id)
assert phase_events == ["load", "commit", "summary"]
enable_summaries.assert_called_once()
assert enable_summaries.call_args.kwargs["dataset"].id == dataset.id
assert enable_summaries.call_args.kwargs["segment_ids"] == [segment.id]
def test_enable_segments_skips_segments_that_are_not_completed(
sqlite_session: Session,
indexed_segment: tuple[Dataset, Document, DocumentSegment],
) -> None:
"""A batch enable must not push a segment that never finished indexing."""
dataset, document, segment = indexed_segment
segment.status = SegmentStatus.ERROR
segment.enabled = False
sqlite_session.commit()
index_processor = MagicMock()
with (
patch("tasks.enable_segments_to_index_task.IndexProcessorFactory") as processor_factory,
patch(
"services.knowledge.summaries.adapters.SummaryIndexAdapter.enable_summaries_for_segments"
) as enable_summaries,
patch("tasks.enable_segments_to_index_task.redis_client.delete"),
):
processor_factory.return_value.init_index_processor.return_value = index_processor
enable_segments_to_index_task.run([segment.id], dataset.id, document.id)
index_processor.load.assert_not_called()
enable_summaries.assert_not_called()
def test_enable_segments_rolls_back_before_error_compensation(
sqlite_session: Session,
indexed_segment: tuple[Dataset, Document, DocumentSegment],
sqlite_session_factory: sessionmaker[Session],
) -> None:
dataset, document, segment = indexed_segment
phase_events: list[str] = []
index_processor = MagicMock()
def fail_load(*_args: object, **_kwargs: object) -> None:
phase_events.append("load")
raise RuntimeError("load failed")
index_processor.load.side_effect = fail_load
with (
_record_transaction_events(sqlite_session_factory, phase_events),
patch("tasks.enable_segments_to_index_task.IndexProcessorFactory") as processor_factory,
patch(
"services.knowledge.summaries.adapters.SummaryIndexAdapter.enable_summaries_for_segments"
) as enable_summaries,
patch("tasks.enable_segments_to_index_task.redis_client.delete"),
):
processor_factory.return_value.init_index_processor.return_value = index_processor
enable_segments_to_index_task.run([segment.id], dataset.id, document.id)
sqlite_session.expire_all()
persisted_segment = sqlite_session.get(DocumentSegment, segment.id)
assert persisted_segment is not None
assert persisted_segment.enabled is False
assert persisted_segment.status == SegmentStatus.ERROR
assert persisted_segment.error == "load failed"
assert persisted_segment.disabled_at is not None
assert phase_events == ["load", "rollback", "commit"]
enable_summaries.assert_not_called()
@@ -0,0 +1,71 @@
"""Unit tests for enterprise telemetry Celery task."""
import json
from unittest.mock import MagicMock, patch
import pytest
from enterprise.telemetry.contracts import TelemetryCase, TelemetryEnvelope
from tasks.enterprise_telemetry_task import process_enterprise_telemetry
@pytest.fixture
def sample_envelope_json() -> str:
envelope = TelemetryEnvelope(
case=TelemetryCase.APP_CREATED,
tenant_id="test-tenant",
event_id="test-event-123",
payload={"app_id": "app-123"},
)
return envelope.model_dump_json()
def test_process_enterprise_telemetry_success(sample_envelope_json: str) -> None:
with patch("tasks.enterprise_telemetry_task.EnterpriseMetricHandler") as mock_handler_class:
mock_handler = MagicMock()
mock_handler_class.return_value = mock_handler
process_enterprise_telemetry(sample_envelope_json)
mock_handler.handle.assert_called_once()
call_args = mock_handler.handle.call_args[0][0]
assert isinstance(call_args, TelemetryEnvelope)
assert call_args.case == TelemetryCase.APP_CREATED
assert call_args.tenant_id == "test-tenant"
assert call_args.event_id == "test-event-123"
def test_process_enterprise_telemetry_invalid_json(caplog: pytest.LogCaptureFixture) -> None:
invalid_json = "not valid json"
process_enterprise_telemetry(invalid_json)
assert "Failed to process enterprise telemetry envelope" in caplog.text
def test_process_enterprise_telemetry_handler_exception(
sample_envelope_json: str, caplog: pytest.LogCaptureFixture
) -> None:
with patch("tasks.enterprise_telemetry_task.EnterpriseMetricHandler") as mock_handler_class:
mock_handler = MagicMock()
mock_handler.handle.side_effect = Exception("Handler error")
mock_handler_class.return_value = mock_handler
process_enterprise_telemetry(sample_envelope_json)
assert "Failed to process enterprise telemetry envelope" in caplog.text
def test_process_enterprise_telemetry_validation_error(caplog: pytest.LogCaptureFixture) -> None:
invalid_envelope = json.dumps(
{
"case": "INVALID_CASE",
"tenant_id": "test-tenant",
"event_id": "test-event",
"payload": {},
}
)
process_enterprise_telemetry(invalid_envelope)
assert "Failed to process enterprise telemetry envelope" in caplog.text
@@ -0,0 +1,78 @@
"""
Unit tests for summary index task queue isolation.
These tasks must NOT run on the shared 'dataset' queue because they invoke LLMs
for each document segment and can occupy all worker slots for hours, blocking
document indexing tasks.
"""
from typing import Protocol, runtime_checkable
import pytest
from tasks.generate_summary_index_task import generate_summary_index_task
from tasks.regenerate_summary_index_task import regenerate_summary_index_task
SUMMARY_QUEUE = "dataset_summary"
INDEXING_QUEUE = "dataset"
@runtime_checkable
class _Queue(Protocol):
name: str
class _Router(Protocol):
def route(
self,
options: dict[str, object],
name: str,
args: tuple[object, ...],
kwargs: dict[str, object],
) -> dict[str, object]: ...
class _Amqp(Protocol):
router: _Router
class _CeleryApp(Protocol):
amqp: _Amqp
@runtime_checkable
class _RoutableTask(Protocol):
name: str
app: _CeleryApp
def _get_exec_options(self) -> dict[str, object]: ...
def _routed_queue(task: object) -> str | None:
assert isinstance(task, _RoutableTask), "Expected a Celery task with routing metadata"
options = task._get_exec_options()
routed = task.app.amqp.router.route(options, task.name, args=(), kwargs={})
queue = routed.get("queue")
if queue is None:
return None
assert isinstance(queue, _Queue), "Expected a routed queue with a name"
return queue.name
@pytest.mark.parametrize(
("task", "task_name"),
[
(generate_summary_index_task, "generate_summary_index_task"),
(regenerate_summary_index_task, "regenerate_summary_index_task"),
],
)
def test_summary_task_routes_to_dedicated_queue(task: object, task_name: str) -> None:
"""Summary tasks must route to the dataset_summary queue, not the shared dataset queue.
Summary generation is LLM-heavy and will block document indexing if placed
on the shared queue.
"""
assert _routed_queue(task) == SUMMARY_QUEUE, (
f"{task_name} must route to '{SUMMARY_QUEUE}' queue (not '{INDEXING_QUEUE}'). "
"Summary generation is LLM-heavy and will block document indexing if placed on the shared queue."
)
@@ -0,0 +1,252 @@
from __future__ import annotations
from datetime import datetime, timedelta
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
from sqlalchemy.engine import Engine
from sqlalchemy.orm import Session, sessionmaker
import core.db.session_factory as session_factory_module
from core.repositories.human_input_repository import HumanInputFormSubmissionRepository
from core.workflow.nodes.human_input.entities import FormDefinition
from core.workflow.nodes.human_input.enums import HumanInputFormKind, HumanInputFormStatus
from models.human_input import HumanInputForm
from tasks import human_input_timeout_tasks as task_module
from tests.helpers.unit.config_override import apply_config_overrides
class _FakeService:
def __init__(self):
self.enqueued: list[str] = []
self.agent_app_resumed: list[tuple[str, str]] = []
def enqueue_resume(self, workflow_run_id: str | None) -> None:
if workflow_run_id is not None:
self.enqueued.append(workflow_run_id)
def enqueue_agent_app_resume(self, *, conversation_id: str, form_id: str) -> None:
self.agent_app_resumed.append((conversation_id, form_id))
def _build_form(
*,
form_id: str,
form_kind: HumanInputFormKind,
created_at: datetime,
expiration_time: datetime,
workflow_run_id: str | None,
node_id: str,
conversation_id: str | None = None,
) -> HumanInputForm:
form_definition = FormDefinition(
form_content="",
rendered_content="",
expiration_time=expiration_time,
)
return HumanInputForm(
id=form_id,
tenant_id="tenant-1",
app_id="app-1",
form_kind=form_kind,
created_at=created_at,
expiration_time=expiration_time,
workflow_run_id=workflow_run_id,
conversation_id=conversation_id,
node_id=node_id,
form_definition=form_definition.model_dump_json(),
rendered_content="",
status=HumanInputFormStatus.WAITING,
)
@pytest.fixture
def sqlite_task_database(
sqlite_engine: Engine,
sqlite_session: Session,
monkeypatch: pytest.MonkeyPatch,
) -> None:
repository_session_maker = sessionmaker(bind=sqlite_engine, expire_on_commit=False)
monkeypatch.setattr(session_factory_module, "_session_maker", repository_session_maker)
monkeypatch.setattr(task_module, "db", SimpleNamespace(engine=sqlite_engine))
def test_is_global_timeout_uses_created_at():
now = datetime(2025, 1, 1, 12, 0, 0)
form = _build_form(
form_id="form-1",
form_kind=HumanInputFormKind.RUNTIME,
created_at=now - timedelta(seconds=61),
expiration_time=now + timedelta(hours=1),
workflow_run_id="run-1",
node_id="node-1",
)
assert task_module._is_global_timeout(form, 60, now=now) is True
form.workflow_run_id = None
assert task_module._is_global_timeout(form, 60, now=now) is False
form.workflow_run_id = "run-1"
form.created_at = now - timedelta(seconds=59)
assert task_module._is_global_timeout(form, 60, now=now) is False
assert task_module._is_global_timeout(form, 0, now=now) is False
@pytest.mark.parametrize("sqlite_session", [(HumanInputForm,)], indirect=True)
def test_check_and_handle_human_input_timeouts_marks_and_routes(
monkeypatch: pytest.MonkeyPatch,
sqlite_task_database: None,
sqlite_engine: Engine,
sqlite_session: Session,
):
now = datetime(2025, 1, 1, 12, 0, 0)
monkeypatch.setattr(task_module, "naive_utc_now", lambda: now)
apply_config_overrides(monkeypatch, HUMAN_INPUT_GLOBAL_TIMEOUT_SECONDS=3600)
forms = [
_build_form(
form_id="form-global",
form_kind=HumanInputFormKind.RUNTIME,
created_at=now - timedelta(hours=2),
expiration_time=now + timedelta(hours=1),
workflow_run_id="run-global",
node_id="node-global",
),
_build_form(
form_id="form-node",
form_kind=HumanInputFormKind.RUNTIME,
created_at=now - timedelta(minutes=5),
expiration_time=now - timedelta(seconds=1),
workflow_run_id="run-node",
node_id="node-node",
),
_build_form(
form_id="form-delivery",
form_kind=HumanInputFormKind.DELIVERY_TEST,
created_at=now - timedelta(minutes=1),
expiration_time=now - timedelta(seconds=1),
workflow_run_id=None,
node_id="node-delivery",
),
]
sqlite_session.add_all(forms)
sqlite_session.commit()
repo = HumanInputFormSubmissionRepository()
mark_timeout_spy = MagicMock(wraps=repo.mark_timeout)
monkeypatch.setattr(repo, "mark_timeout", mark_timeout_spy)
service = _FakeService()
service_factory = MagicMock(return_value=service)
global_timeout_handler = MagicMock()
monkeypatch.setattr(task_module, "HumanInputFormSubmissionRepository", lambda: repo)
monkeypatch.setattr(task_module, "HumanInputService", service_factory)
monkeypatch.setattr(task_module, "_handle_global_timeout", global_timeout_handler)
task_module.check_and_handle_human_input_timeouts(limit=100)
assert {
(call.kwargs["form_id"], call.kwargs["timeout_status"], call.kwargs["reason"])
for call in mark_timeout_spy.call_args_list
} == {
("form-global", HumanInputFormStatus.EXPIRED, "global_timeout"),
("form-node", HumanInputFormStatus.TIMEOUT, "node_timeout"),
("form-delivery", HumanInputFormStatus.TIMEOUT, "delivery_test_timeout"),
}
assert service.enqueued == ["run-node"]
global_timeout_handler.assert_called_once()
global_timeout_call = global_timeout_handler.call_args.kwargs
assert global_timeout_call["form_id"] == "form-global"
assert global_timeout_call["workflow_run_id"] == "run-global"
assert global_timeout_call["node_id"] == "node-global"
task_session_maker = global_timeout_call["session_factory"]
assert isinstance(task_session_maker, sessionmaker)
assert task_session_maker.kw["bind"] is sqlite_engine
service_factory.assert_called_once_with(task_session_maker, form_repository=repo)
sqlite_session.expire_all()
assert sqlite_session.get(HumanInputForm, "form-global").status == HumanInputFormStatus.EXPIRED
assert sqlite_session.get(HumanInputForm, "form-node").status == HumanInputFormStatus.TIMEOUT
assert sqlite_session.get(HumanInputForm, "form-delivery").status == HumanInputFormStatus.TIMEOUT
@pytest.mark.parametrize("sqlite_session", [(HumanInputForm,)], indirect=True)
def test_check_and_handle_human_input_timeouts_orders_by_id_before_limit(
monkeypatch: pytest.MonkeyPatch,
sqlite_task_database: None,
sqlite_session: Session,
):
now = datetime(2025, 1, 1, 12, 0, 0)
monkeypatch.setattr(task_module, "naive_utc_now", lambda: now)
apply_config_overrides(monkeypatch, HUMAN_INPUT_GLOBAL_TIMEOUT_SECONDS=0)
forms = [
_build_form(
form_id=form_id,
form_kind=HumanInputFormKind.DELIVERY_TEST,
created_at=now - timedelta(minutes=1),
expiration_time=now - timedelta(seconds=1),
workflow_run_id=None,
node_id=f"node-{form_id}",
)
for form_id in ("form-b", "form-a")
]
sqlite_session.add_all(forms)
sqlite_session.commit()
repo = HumanInputFormSubmissionRepository()
mark_timeout_spy = MagicMock(wraps=repo.mark_timeout)
monkeypatch.setattr(repo, "mark_timeout", mark_timeout_spy)
monkeypatch.setattr(task_module, "HumanInputFormSubmissionRepository", lambda: repo)
monkeypatch.setattr(task_module, "HumanInputService", MagicMock(return_value=_FakeService()))
task_module.check_and_handle_human_input_timeouts(limit=1)
mark_timeout_spy.assert_called_once_with(
form_id="form-a",
timeout_status=HumanInputFormStatus.TIMEOUT,
reason="delivery_test_timeout",
)
sqlite_session.expire_all()
assert sqlite_session.get(HumanInputForm, "form-a").status == HumanInputFormStatus.TIMEOUT
assert sqlite_session.get(HumanInputForm, "form-b").status == HumanInputFormStatus.WAITING
@pytest.mark.parametrize("sqlite_session", [(HumanInputForm,)], indirect=True)
def test_check_and_handle_human_input_timeouts_omits_global_filter_when_disabled(
monkeypatch: pytest.MonkeyPatch,
sqlite_task_database: None,
sqlite_session: Session,
):
now = datetime(2025, 1, 1, 12, 0, 0)
monkeypatch.setattr(task_module, "naive_utc_now", lambda: now)
apply_config_overrides(monkeypatch, HUMAN_INPUT_GLOBAL_TIMEOUT_SECONDS=0)
old_unexpired_form = _build_form(
form_id="form-old",
form_kind=HumanInputFormKind.RUNTIME,
created_at=now - timedelta(hours=2),
expiration_time=now + timedelta(hours=1),
workflow_run_id="run-old",
node_id="node-old",
)
sqlite_session.add(old_unexpired_form)
sqlite_session.commit()
repo = HumanInputFormSubmissionRepository()
mark_timeout_spy = MagicMock(wraps=repo.mark_timeout)
monkeypatch.setattr(repo, "mark_timeout", mark_timeout_spy)
monkeypatch.setattr(task_module, "HumanInputFormSubmissionRepository", lambda: repo)
monkeypatch.setattr(task_module, "HumanInputService", MagicMock(return_value=_FakeService()))
global_timeout_handler = MagicMock()
monkeypatch.setattr(task_module, "_handle_global_timeout", global_timeout_handler)
task_module.check_and_handle_human_input_timeouts(limit=1)
mark_timeout_spy.assert_not_called()
global_timeout_handler.assert_not_called()
sqlite_session.refresh(old_unexpired_form)
assert old_unexpired_form.status == HumanInputFormStatus.WAITING
@@ -0,0 +1,137 @@
"""Tests for Celery-owned IM callback execution and retry."""
from datetime import UTC, datetime
import pytest
from celery.exceptions import Retry
from sqlalchemy.engine import Engine
from sqlalchemy.orm import sessionmaker
from core.human_input_v2.entities import IMProvider
from core.human_input_v2.im_integration.adapters import AuthenticatedIMEvent, IMEventIngressKind
from core.human_input_v2.im_message_inbox import IMInboxRecord, IMInboxRecordId
from dify_app import DifyApp
from repositories.human_input_v2.im_channel_repository import IMChannelId
from services.human_input_v2.im_message_inbox.processor import IMInboxProcessor
from services.human_input_v2.im_message_inbox.storage import IMInboxStorage
from tasks.im_message_inbox_tasks import (
IMInboxRuntimeNotConfiguredError,
IMInboxTaskRetryError,
configure_im_inbox_task_runtime,
process_im_message_inbox_record,
)
_NOW = datetime(2026, 8, 2, 8, tzinfo=UTC)
class _RecordProcessor:
record_ids: list[IMInboxRecordId]
def __init__(self) -> None:
self.record_ids = []
def process(self, record_id: IMInboxRecordId) -> None:
self.record_ids.append(record_id)
class _FailingRecordProcessor:
def process(self, record_id: IMInboxRecordId) -> None:
raise RuntimeError(f"payload must-not-log for {record_id}")
class _Consumer:
records: list[IMInboxRecord]
def __init__(self) -> None:
self.records = []
def consume(self, record: IMInboxRecord) -> None:
self.records.append(record)
def _event() -> AuthenticatedIMEvent:
return AuthenticatedIMEvent(
provider=IMProvider.FEISHU,
provider_tenant_id="tenant-1",
event_id="event-1",
event_type="card.action",
occurred_at=None,
received_at=datetime(2026, 8, 2, 8),
ingress_kind=IMEventIngressKind.WEBHOOK,
payload='{"secret":"must-not-log"}',
)
def test_processing_task_uses_human_input_delivery_queue(monkeypatch: pytest.MonkeyPatch) -> None:
dispatches: list[dict[str, object]] = []
def capture_dispatch(*_args: object, **options: object) -> None:
dispatches.append(options)
monkeypatch.setitem(process_im_message_inbox_record.app.conf, "task_always_eager", False)
monkeypatch.setattr(process_im_message_inbox_record.app, "send_task", capture_dispatch)
process_im_message_inbox_record.apply_async(args=("record-1",))
assert len(dispatches) == 1
assert dispatches[0]["queue"] == "human_input_delivery"
def test_processing_task_fails_closed_when_runtime_is_not_configured() -> None:
app = DifyApp(__name__)
with app.app_context(), pytest.raises(IMInboxRuntimeNotConfiguredError):
process_im_message_inbox_record.run("record-1")
def test_processing_task_passes_typed_record_id_to_processor() -> None:
app = DifyApp(__name__)
processor = _RecordProcessor()
configure_im_inbox_task_runtime(app, processor_factory=lambda: processor)
with app.app_context():
result = process_im_message_inbox_record.run("record-1")
assert result is None
assert processor.record_ids == [IMInboxRecordId("record-1")]
def test_processing_task_retries_sanitized_processor_failure(
monkeypatch: pytest.MonkeyPatch,
caplog: pytest.LogCaptureFixture,
) -> None:
app = DifyApp(__name__)
configure_im_inbox_task_runtime(app, processor_factory=_FailingRecordProcessor)
retry_errors: list[Exception] = []
def retry(*, exc: Exception, **_: object) -> None:
retry_errors.append(exc)
raise Retry()
monkeypatch.setattr(process_im_message_inbox_record, "retry", retry)
with app.app_context(), pytest.raises(Retry):
process_im_message_inbox_record.run("record-1")
assert len(retry_errors) == 1
assert isinstance(retry_errors[0], IMInboxTaskRetryError)
assert "must-not-log" not in caplog.text
assert "must-not-log" not in str(retry_errors[0])
def test_processing_task_loads_callback_and_records_processed_fact(sqlite_engine: Engine) -> None:
repository = IMInboxStorage(sessionmaker(bind=sqlite_engine, expire_on_commit=False))
accepted = repository.insert_or_resolve(IMChannelId("channel-1"), _event(), now=_NOW)
consumer = _Consumer()
processor = IMInboxProcessor(repository=repository, consumer=consumer, clock=lambda: _NOW)
app = DifyApp(__name__)
configure_im_inbox_task_runtime(app, processor_factory=lambda: processor)
with app.app_context():
process_im_message_inbox_record.run(str(accepted.record_id))
assert len(consumer.records) == 1
assert consumer.records[0].event == _event()
stored = repository.get(accepted.record_id)
assert stored is not None
assert stored.processed_at == datetime(2026, 8, 2, 8)
@@ -0,0 +1,161 @@
"""Task behavior with real workspace queries and an explicit RBAC transport."""
from collections.abc import Callable, Iterator
from typing import Protocol, cast
import pytest
from flask import Flask
import tasks.initialize_created_app_rbac_access_task as task_module
from enums import DeploymentEdition
from enums.account import TenantAccountRole
from extensions.ext_application_services import build_application_services
from extensions.ext_redis import redis_client
from models import TenantAccountJoin
from services.rbac import contracts as dto
from tasks.initialize_created_app_rbac_access_task import (
initialize_created_app_rbac_access_task,
sync_joined_workspace_member_rbac_access_task,
)
from tests.helpers.unit.rbac_fakes import RBACDomain
class _TaskWithQueue(Protocol):
queue: str
@pytest.fixture(autouse=True)
def workspace_services(
monkeypatch: pytest.MonkeyPatch,
app: Flask,
rbac_domain: RBACDomain,
config_overrides: Callable[..., None],
) -> None:
config_overrides(RBAC_ENABLED=True)
services = build_application_services(
database_client=rbac_domain.sessions,
deployment_edition=DeploymentEdition.COMMUNITY,
initialization_password="",
redis=redis_client,
)
monkeypatch.setitem(app.extensions, "application_services", services)
def _seed_members(domain: RBACDomain, members: tuple[str, ...]) -> None:
with domain.sessions.begin() as session:
for account_id in members:
membership = TenantAccountJoin(tenant_id="tenant-1", account_id=account_id, role=TenantAccountRole.NORMAL)
membership.id = account_id
session.add(membership)
session.add(TenantAccountJoin(tenant_id="other", account_id="foreign", role=TenantAccountRole.NORMAL))
def test_initialize_created_app_rbac_access_task_uses_rbac_queue() -> None:
task = cast(_TaskWithQueue, initialize_created_app_rbac_access_task)
assert task.queue == "app_rbac"
def test_sync_joined_workspace_member_rbac_access_task_uses_rbac_queue() -> None:
task = cast(_TaskWithQueue, sync_joined_workspace_member_rbac_access_task)
assert task.queue == "app_rbac"
def test_initialize_created_app_rbac_access_task_batches_workspace_members(
monkeypatch: pytest.MonkeyPatch, rbac_domain: RBACDomain
) -> None:
_seed_members(rbac_domain, ("acct-1", "acct-2", "acct-3"))
monkeypatch.setattr(task_module, "APP_RBAC_ACCOUNT_POLICY_BATCH_SIZE", 2)
initialize_created_app_rbac_access_task.run("tenant-1", "actor-1", "app-1")
first, second = rbac_domain.transport.requests
for request, account_ids in ((first, ["acct-1", "acct-2"]), (second, ["acct-3"])):
assert request.method == "POST"
assert request.endpoint == "/rbac/apps/whitelist/members/batch"
assert (request.tenant_id, request.account_id) == ("tenant-1", "actor-1")
assert request.params is None
assert request.json == {"data": [{"app_id": "app-1", "account_ids": account_ids, "policy_id": "default"}]}
@pytest.mark.parametrize("kind", ["app", "dataset", "agent"])
def test_initialize_created_app_rbac_access_task_targets_the_resource_that_was_passed(
rbac_domain: RBACDomain, kind: str
) -> None:
_seed_members(rbac_domain, ("acct-1",))
resource_id = f"{kind}-1"
initialize_created_app_rbac_access_task.run("tenant-1", "actor-1", **{f"{kind}_id": resource_id})
request = rbac_domain.transport.only_request
assert (request.method, request.endpoint) == ("POST", f"/rbac/{kind}s/whitelist/members/batch")
assert (request.tenant_id, request.account_id) == ("tenant-1", "actor-1")
assert request.params is None
assert request.json == {"data": [{f"{kind}_id": resource_id, "account_ids": ["acct-1"], "policy_id": "default"}]}
def test_initialize_created_app_rbac_access_task_retries_on_failure(
monkeypatch: pytest.MonkeyPatch, rbac_domain: RBACDomain
) -> None:
_seed_members(rbac_domain, ("acct-1",))
failure = ConnectionError("RBAC unavailable")
rbac_domain.transport.failure = failure
retries: list[Exception] = []
def retry(*, exc: Exception) -> RuntimeError:
retries.append(exc)
return RuntimeError("retry requested")
monkeypatch.setattr(initialize_created_app_rbac_access_task, "retry", retry)
with pytest.raises(RuntimeError, match="retry requested"):
initialize_created_app_rbac_access_task.run("tenant-1", "actor-1", "app-1")
assert retries == [failure]
assert rbac_domain.transport.only_request.endpoint == "/rbac/apps/whitelist/members/batch"
def test_sync_joined_workspace_member_rbac_access_task_appends_auto_included_resources(
monkeypatch: pytest.MonkeyPatch, rbac_domain: RBACDomain
) -> None:
resources = [
dto.ResourceWhitelistConfigResource(resource_type=kind, resource_id=resource_id)
for kind, resource_id in (
(dto.RBACResourceType.APP, "app-1"),
(dto.RBACResourceType.DATASET, "dataset-1"),
(dto.RBACResourceType.APP, "app-2"),
(dto.RBACResourceType.AGENT, "agent-1"),
(dto.RBACResourceType.AGENT, "agent-2"),
)
]
def resource_batches(tenant_id: str, batch_size: int) -> Iterator[list[dto.ResourceWhitelistConfigResource]]:
assert tenant_id == "tenant-1"
assert batch_size == task_module.APP_RBAC_RESOURCE_CONFIG_BATCH_SIZE
yield resources
configs = dto.ResourceWhitelistConfigsResponse(
data=[
dto.ResourceWhitelistConfigItem(
resource_type=resource.resource_type,
resource_id=resource.resource_id,
automatic_include_workspace_members=resource.resource_id.endswith("-1"),
)
for resource in resources
]
)
rbac_domain.transport.responses["/rbac/whitelist/configs"] = configs.model_dump(mode="json")
monkeypatch.setattr(task_module, "_iter_resource_config_batches", resource_batches)
sync_joined_workspace_member_rbac_access_task.run("tenant-1", "member-1", "actor-1")
lookup, *appends = rbac_domain.transport.requests
assert (lookup.method, lookup.endpoint) == ("POST", "/rbac/whitelist/configs")
assert lookup.json == {"resources": [resource.model_dump(mode="json") for resource in resources]}
assert len(appends) == 3
for kind, request in zip(("app", "dataset", "agent"), appends, strict=True):
assert (request.method, request.endpoint) == ("POST", f"/rbac/{kind}s/whitelist/members/batch")
assert request.json == {
"data": [{f"{kind}_id": f"{kind}-1", "account_ids": ["member-1"], "policy_id": "default"}]
}
for request in (lookup, *appends):
assert (request.tenant_id, request.account_id) == ("tenant-1", "actor-1")
@@ -0,0 +1,203 @@
import logging
from types import SimpleNamespace
from typing import Protocol, cast
from unittest.mock import MagicMock, call, patch
import pytest
from celery.exceptions import Retry
from core.plugin.entities.plugin_daemon import PluginInstallTaskStartResponse
from tests.helpers.unit.config_override import apply_config_overrides
class _TaskWithQueue(Protocol):
queue: str
def test_install_default_plugins_task_uses_plugin_queue() -> None:
from tasks.install_default_plugins_task import install_default_plugins_task
assert cast(_TaskWithQueue, install_default_plugins_task).queue == "plugin"
def test_configure_default_models_task_uses_plugin_queue() -> None:
from tasks.install_default_plugins_task import configure_default_models_task
assert cast(_TaskWithQueue, configure_default_models_task).queue == "plugin"
def test_install_default_plugins_task_installs_latest_identifiers(monkeypatch: pytest.MonkeyPatch) -> None:
import tasks.install_default_plugins_task as task_module
from tasks.install_default_plugins_task import install_default_plugins_task
plugin_ids = ["langgenius/openai", "langgenius/gemini"]
plugin_identifiers = ["langgenius/openai:1.0.0@aaa", "langgenius/gemini:0.9.1@bbb"]
fetch = MagicMock(
return_value=[
SimpleNamespace(plugin_id=plugin_ids[1], latest_package_identifier=plugin_identifiers[1]),
SimpleNamespace(plugin_id=plugin_ids[0], latest_package_identifier=plugin_identifiers[0]),
]
)
install = MagicMock()
monkeypatch.setattr(task_module.marketplace, "batch_fetch_plugin_manifests", fetch)
monkeypatch.setattr(task_module.PluginService, "install_from_marketplace_pkg", install)
with patch.object(task_module.configure_default_models_task, "delay") as configure:
install_default_plugins_task.run("tenant-1", plugin_ids)
fetch.assert_called_once_with(plugin_ids)
install.assert_called_once_with("tenant-1", plugin_identifiers)
configure.assert_called_once_with("tenant-1", None)
def test_install_default_plugins_task_skips_missing_plugins(
monkeypatch: pytest.MonkeyPatch,
caplog: pytest.LogCaptureFixture,
) -> None:
import tasks.install_default_plugins_task as task_module
from tasks.install_default_plugins_task import install_default_plugins_task
fetch = MagicMock(
return_value=[
SimpleNamespace(plugin_id="langgenius/openai", latest_package_identifier="langgenius/openai:1.0.0@aaa")
]
)
install = MagicMock()
monkeypatch.setattr(task_module.marketplace, "batch_fetch_plugin_manifests", fetch)
monkeypatch.setattr(task_module.PluginService, "install_from_marketplace_pkg", install)
with (
caplog.at_level(logging.WARNING, logger=task_module.logger.name),
patch.object(task_module.configure_default_models_task, "delay"),
):
install_default_plugins_task.run("tenant-1", ["langgenius/openai", "missing/plugin"])
install.assert_called_once_with("tenant-1", ["langgenius/openai:1.0.0@aaa"])
assert "Default plugins not found in marketplace: missing/plugin" in caplog.text
@pytest.mark.parametrize("configured_models", ["", "llm:provider:model"])
@pytest.mark.parametrize("all_installed", [False, True])
def test_install_default_plugins_task_queues_model_configuration_after_daemon_install(
monkeypatch: pytest.MonkeyPatch,
configured_models: str,
all_installed: bool,
) -> None:
import tasks.install_default_plugins_task as task_module
from tasks.install_default_plugins_task import install_default_plugins_task
plugin_id = "langgenius/openai"
plugin_identifier = "langgenius/openai:1.0.0@aaa"
apply_config_overrides(monkeypatch, NEW_USER_DEFAULT_MODELS=configured_models)
monkeypatch.setattr(
task_module.marketplace,
"batch_fetch_plugin_manifests",
MagicMock(return_value=[SimpleNamespace(plugin_id=plugin_id, latest_package_identifier=plugin_identifier)]),
)
with (
patch.object(
task_module.PluginService,
"install_from_marketplace_pkg",
return_value=PluginInstallTaskStartResponse(all_installed=all_installed, task_id="install-task-1"),
),
patch.object(task_module.configure_default_models_task, "delay") as delay,
):
install_default_plugins_task.run("tenant-1", [plugin_id])
delay.assert_called_once_with("tenant-1", None if all_installed else "install-task-1")
@pytest.mark.parametrize("configured_models", ["", "llm:provider:model"])
def test_configure_default_models_task_retries_while_plugins_are_installing(
monkeypatch: pytest.MonkeyPatch,
configured_models: str,
) -> None:
import tasks.install_default_plugins_task as task_module
from tasks.install_default_plugins_task import configure_default_models_task
apply_config_overrides(monkeypatch, NEW_USER_DEFAULT_MODELS=configured_models)
monkeypatch.setattr(
task_module.PluginService,
"fetch_install_task",
MagicMock(return_value=SimpleNamespace(status=task_module.PluginInstallTaskStatus.Running)),
)
retry = MagicMock(side_effect=Retry())
monkeypatch.setattr(configure_default_models_task, "retry", retry)
with pytest.raises(Retry):
configure_default_models_task.run("tenant-1", "install-task-1")
retry.assert_called_once_with()
def test_configure_default_models_task_sets_each_explicit_model(monkeypatch: pytest.MonkeyPatch) -> None:
import tasks.install_default_plugins_task as task_module
from tasks.install_default_plugins_task import configure_default_models_task
apply_config_overrides(
monkeypatch,
NEW_USER_DEFAULT_MODELS=(
"llm:langgenius/openai/openai:gpt-4o-mini,text-embedding:langgenius/openai/openai:text-embedding-3-small"
),
)
fetch_install_task = MagicMock(
return_value=SimpleNamespace(
status=task_module.PluginInstallTaskStatus.Success,
plugins=[],
)
)
monkeypatch.setattr(task_module.PluginService, "fetch_install_task", fetch_install_task)
model_provider_service = MagicMock()
monkeypatch.setattr(task_module, "ModelProviderService", MagicMock(return_value=model_provider_service))
configure_default_models_task.run("tenant-1", "install-task-1")
fetch_install_task.assert_called_once_with("tenant-1", "install-task-1")
assert model_provider_service.update_default_model_of_model_type.call_args_list == [
call(
tenant_id="tenant-1",
model_type="llm",
provider="langgenius/openai/openai",
model="gpt-4o-mini",
),
call(
tenant_id="tenant-1",
model_type="text-embedding",
provider="langgenius/openai/openai",
model="text-embedding-3-small",
),
]
model_provider_service.initialize_default_llm.assert_not_called()
@pytest.mark.parametrize("configured_models", ["", "text-embedding:provider:embedding-model"])
def test_configure_default_models_task_initializes_llm_without_an_explicit_llm(
monkeypatch: pytest.MonkeyPatch, configured_models: str
) -> None:
import tasks.install_default_plugins_task as task_module
apply_config_overrides(monkeypatch, NEW_USER_DEFAULT_MODELS=configured_models)
with (
patch.object(task_module.ModelProviderService, "initialize_default_llm") as initialize,
patch.object(task_module.ModelProviderService, "update_default_model_of_model_type") as update,
):
task_module.configure_default_models_task.run("tenant-1", None)
initialize.assert_called_once_with("tenant-1")
if configured_models:
update.assert_called_once_with(
tenant_id="tenant-1", model_type="text-embedding", provider="provider", model="embedding-model"
)
else:
update.assert_not_called()
def test_configure_default_models_task_reports_initialization_failure(monkeypatch: pytest.MonkeyPatch) -> None:
import tasks.install_default_plugins_task as task_module
apply_config_overrides(monkeypatch, NEW_USER_DEFAULT_MODELS="")
with patch.object(
task_module.ModelProviderService, "initialize_default_llm", side_effect=RuntimeError("unavailable")
):
with pytest.raises(RuntimeError, match="Failed to initialize defaults for tenant tenant-1; model types: llm"):
task_module.configure_default_models_task.run("tenant-1", None)
@@ -0,0 +1,117 @@
from dataclasses import dataclass, field
from typing import Literal
from uuid import uuid4
import pytest
from celery.exceptions import Retry
from dify_agent.protocol.knowledge_fs import KnowledgeFsError
from dify_agent.protocol.knowledge_investigation import KnowledgeInvestigationPayload
from services.knowledge_fs.product_remote import KnowledgeFSProductRequestRejectedError
from tasks import knowledge_fs_agent_investigation_tasks as module
from tests.helpers.unit.tasks.knowledge_fs_task_fakes import RecordingRetry
from tests.helpers.unit.tasks.task_options import task_options
@dataclass
class _Gateway:
"""Records investigation captures; raises ``error`` when one is configured."""
error: Exception | None = None
captures: list[tuple[KnowledgeInvestigationPayload, str]] = field(default_factory=list)
def capture_investigation(self, report: KnowledgeInvestigationPayload, control_space_id: str) -> None:
self.captures.append((report, control_space_id))
if self.error is not None:
raise self.error
def report() -> tuple[dict[str, object], str]:
space = str(uuid4())
attempts: list[dict[str, object]] = []
return {
"investigation_id": str(uuid4()),
"execution_context": {"tenant_id": "tenant", "agent_mode": "agent_app", "invoke_from": "published"},
"bindings": [{"id": "docs", "name": "Docs", "control_space_id": space}],
"query": "refund",
"status": "completed",
"attempts": attempts,
}, space
def test_dataset_worker_reconstructs_report_for_each_delivery(monkeypatch: pytest.MonkeyPatch) -> None:
gateway = _Gateway()
monkeypatch.setattr(module, "AgentKnowledgeGateway", lambda: gateway)
payload, space = report()
task = module.capture_agent_knowledge_investigation_task
task.run(report=payload, control_space_id=space)
task.run(report=payload, control_space_id=space)
assert task_options(task)["queue"] == "dataset"
first, second = gateway.captures
assert first == second
assert first[1] == space
assert str(first[0].investigation_id) == payload["investigation_id"]
def test_revoked_authority_stops_publication_and_transient_failures_retry(monkeypatch: pytest.MonkeyPatch) -> None:
gateway = _Gateway(error=KnowledgeFsError("KNOWLEDGE_SCOPE_CHANGED", "revoked", 403))
monkeypatch.setattr(module, "AgentKnowledgeGateway", lambda: gateway)
payload, space = report()
task = module.capture_agent_knowledge_investigation_task
retry = RecordingRetry()
monkeypatch.setattr(task, "retry", retry)
task.run(report=payload, control_space_id=space)
assert retry.calls == []
gateway.error = OSError("offline")
with pytest.raises(Retry):
task.run(report=payload, control_space_id=space)
assert len(retry.calls) == 1
@pytest.mark.parametrize("status_code", [400, 403, 409, 413, 422])
def test_permanent_remote_rejections_do_not_retry(
monkeypatch: pytest.MonkeyPatch, status_code: Literal[400, 403, 409, 413, 422]
) -> None:
gateway = _Gateway(error=KnowledgeFSProductRequestRejectedError(status_code=status_code))
monkeypatch.setattr(module, "AgentKnowledgeGateway", lambda: gateway)
task = module.capture_agent_knowledge_investigation_task
retry = RecordingRetry()
monkeypatch.setattr(task, "retry", retry)
payload, space = report()
task.run(report=payload, control_space_id=space)
assert len(gateway.captures) == 1
assert retry.calls == []
@pytest.mark.parametrize(("retries", "countdown"), [(0, 30), (1, 60), (2, 120)])
def test_rate_limited_report_retries_with_backoff(
monkeypatch: pytest.MonkeyPatch, retries: int, countdown: int
) -> None:
error = KnowledgeFSProductRequestRejectedError(status_code=429)
gateway = _Gateway(error=error)
monkeypatch.setattr(module, "AgentKnowledgeGateway", lambda: gateway)
task = module.capture_agent_knowledge_investigation_task
retry = RecordingRetry()
monkeypatch.setattr(task, "retry", retry)
monkeypatch.setattr(task.request, "retries", retries)
payload, space = report()
with pytest.raises(Retry):
task.run(report=payload, control_space_id=space)
assert retry.calls == [{"exc": error, "countdown": countdown}]
def test_malformed_queued_report_is_rejected_before_gateway_publication(monkeypatch: pytest.MonkeyPatch) -> None:
gateway = _Gateway()
monkeypatch.setattr(module, "AgentKnowledgeGateway", lambda: gateway)
task = module.capture_agent_knowledge_investigation_task
retry = RecordingRetry()
monkeypatch.setattr(task, "retry", retry)
task.run(report={"investigation_id": "invalid"}, control_space_id=str(uuid4()))
assert gateway.captures == []
assert retry.calls == []
@@ -0,0 +1,66 @@
from unittest.mock import patch
import pytest
from celery.app.task import Context
from celery.exceptions import Reject, Retry
from tasks.knowledge_fs_background_tasks import execute_delivery, execute_operation
JOB: dict[str, object] = {
"id": "018f0d60-7a49-7cc2-9c1b-5b36f18f8a01",
"type": "document.compile",
"payload": {"attemptId": "018f0d60-7a49-7cc2-9c1b-5b36f18f8a02"},
"attempts": 1,
}
def test_delivery_uses_late_ack_and_passes_broker_retry_count_to_engine() -> None:
assert execute_delivery.acks_late is True
assert execute_delivery.reject_on_worker_lost is True
execute_delivery.request_stack.push(Context(id=JOB["id"], retries=2))
try:
with patch(
"tasks.knowledge_fs_background_tasks.background_engine.execute", return_value={"outcome": "completed"}
) as engine:
assert execute_delivery.run(delivery=JOB) == {"outcome": "completed"}
engine.assert_called_once_with("delivery", {**JOB, "attempts": 3})
finally:
execute_delivery.request_stack.pop()
def test_bad_delivery_identity_never_enters_engine() -> None:
execute_delivery.request_stack.push(Context(id="different"))
try:
with patch("tasks.knowledge_fs_background_tasks.background_engine.execute") as engine:
with pytest.raises(Reject):
execute_delivery.run(delivery=JOB)
engine.assert_not_called()
finally:
execute_delivery.request_stack.pop()
@pytest.mark.parametrize("response", [{"outcome": "retry", "runAfter": 0}, OSError("engine died")])
def test_early_delivery_and_engine_loss_retry_without_acknowledging(response: object) -> None:
execute_delivery.request_stack.push(Context(id=JOB["id"], retries=0))
try:
with (
patch("tasks.knowledge_fs_background_tasks.background_engine.execute") as engine,
patch.object(execute_delivery, "retry", side_effect=Retry()) as retry,
):
if isinstance(response, Exception):
engine.side_effect = response
else:
engine.return_value = response
with pytest.raises(Retry):
execute_delivery.run(delivery=JOB)
retry.assert_called_once()
assert 1 <= retry.call_args.kwargs["countdown"] <= 300
finally:
execute_delivery.request_stack.pop()
def test_unknown_operation_is_not_executable() -> None:
with patch("tasks.knowledge_fs_background_tasks.background_engine.execute") as engine:
with pytest.raises(Reject):
execute_operation.run(operation="shell")
engine.assert_not_called()
@@ -0,0 +1,186 @@
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Literal
import pytest
from celery.app.task import Context
from celery.exceptions import Retry
from typing_extensions import TypedDict
from services.knowledge_fs.app_admission_service import KnowledgeFSAppAdmissionError
from services.knowledge_fs.product_dto import (
KnowledgeFSWorkflowFailedRetrievalCapturePayload,
KnowledgeFSWorkflowFailedRetrievalCaptureResponse,
)
from services.knowledge_fs.product_remote import KnowledgeFSProductRequestRejectedError
from services.knowledge_fs.resource import KnowledgeResourceRef
from tasks import knowledge_fs_failed_retrieval_tasks as task_module
from tests.helpers.unit.tasks.knowledge_fs_task_fakes import RecordingRetry
from tests.helpers.unit.tasks.task_options import task_options
EVENT_ID = "019fac9f-bfb0-75ee-9af5-252ebafbac1e"
class FailedRetrievalTaskArguments(TypedDict, closed=True):
event_id: str
tenant_id: str
app_id: str
control_space_id: str
query: str
mode: Literal["fast", "deep", "research"]
retrieval_trace_id: str
def _task_kwargs() -> FailedRetrievalTaskArguments:
return {
"event_id": EVENT_ID,
"tenant_id": "tenant-1",
"app_id": "app-1",
"control_space_id": "control-1",
"query": "missing answer",
"mode": "fast",
"retrieval_trace_id": "retrieval-trace-1",
}
@dataclass(frozen=True)
class _CaptureCall:
tenant_id: str
app_id: str
resource: KnowledgeResourceRef
payload: KnowledgeFSWorkflowFailedRetrievalCapturePayload
@dataclass
class _AppCapabilities:
outcome: KnowledgeFSWorkflowFailedRetrievalCaptureResponse | Exception
calls: list[_CaptureCall] = field(default_factory=list)
def capture_workflow_failed_retrieval(
self,
*,
tenant_id: str,
app_id: str,
resource: KnowledgeResourceRef,
payload: KnowledgeFSWorkflowFailedRetrievalCapturePayload,
) -> KnowledgeFSWorkflowFailedRetrievalCaptureResponse:
self.calls.append(_CaptureCall(tenant_id=tenant_id, app_id=app_id, resource=resource, payload=payload))
if isinstance(self.outcome, Exception):
raise self.outcome
return self.outcome
@dataclass(frozen=True)
class _Runtime:
app_capabilities: _AppCapabilities
def _install_capability(
monkeypatch: pytest.MonkeyPatch, outcome: KnowledgeFSWorkflowFailedRetrievalCaptureResponse | Exception
) -> _AppCapabilities:
capability = _AppCapabilities(outcome)
monkeypatch.setattr(task_module, "get_knowledge_fs_runtime", lambda _session_maker: _Runtime(capability))
monkeypatch.setattr(task_module.session_factory, "get_session_maker", lambda: object())
return capability
def _run_task_with_retries(retries: int) -> None:
task = task_module.capture_workflow_failed_retrieval_task
task.request_stack.push(Context(retries=retries))
try:
task.run(**_task_kwargs())
finally:
task.pop_request()
def test_failed_retrieval_task_runs_on_dataset_queue_and_reauthorizes_app(
monkeypatch: pytest.MonkeyPatch,
) -> None:
capability = _install_capability(
monkeypatch,
KnowledgeFSWorkflowFailedRetrievalCaptureResponse.model_construct(
failed_query_id="failed-query-1",
verdict="retrieval-miss",
bad_case_id="bad-case-1",
),
)
task_module.capture_workflow_failed_retrieval_task.run(**_task_kwargs())
assert task_options(task_module.capture_workflow_failed_retrieval_task)["queue"] == "dataset"
assert len(capability.calls) == 1
call = capability.calls[0]
assert call.tenant_id == "tenant-1"
assert call.app_id == "app-1"
assert call.resource.control_space_id == "control-1"
assert str(call.payload.event_id) == EVENT_ID
assert call.payload.query == "missing answer"
assert call.payload.retrieval_trace_id == "retrieval-trace-1"
def test_enqueue_is_idempotency_aware_and_never_propagates_broker_failure(monkeypatch: pytest.MonkeyPatch) -> None:
delayed: list[dict[str, object]] = []
monkeypatch.setattr(
task_module.capture_workflow_failed_retrieval_task, "delay", lambda **kwargs: delayed.append(kwargs)
)
task_module.enqueue_workflow_failed_retrieval_capture(**_task_kwargs())
assert delayed == [_task_kwargs()]
def unavailable_broker(**kwargs: object) -> None:
delayed.append(kwargs)
raise RuntimeError("broker unavailable")
monkeypatch.setattr(task_module.capture_workflow_failed_retrieval_task, "delay", unavailable_broker)
task_module.enqueue_workflow_failed_retrieval_capture(**_task_kwargs())
assert len(delayed) == 2
def test_terminal_admission_rejection_is_swallowed(monkeypatch: pytest.MonkeyPatch) -> None:
capability = _install_capability(monkeypatch, KnowledgeFSAppAdmissionError("not admitted"))
task_module.capture_workflow_failed_retrieval_task.run(**_task_kwargs())
assert len(capability.calls) == 1
def test_rate_limit_rejection_retries_with_initial_backoff(monkeypatch: pytest.MonkeyPatch) -> None:
error = KnowledgeFSProductRequestRejectedError(status_code=429)
capability = _install_capability(monkeypatch, error)
retry = RecordingRetry()
monkeypatch.setattr(task_module.capture_workflow_failed_retrieval_task, "retry", retry)
with pytest.raises(Retry):
_run_task_with_retries(0)
assert len(capability.calls) == 1
assert retry.calls == [{"exc": error, "countdown": 30}]
def test_transient_failure_retries_with_exponential_backoff(monkeypatch: pytest.MonkeyPatch) -> None:
error = RuntimeError("KnowledgeFS temporarily unavailable")
capability = _install_capability(monkeypatch, error)
retry = RecordingRetry()
monkeypatch.setattr(task_module.capture_workflow_failed_retrieval_task, "retry", retry)
with pytest.raises(Retry):
_run_task_with_retries(1)
assert len(capability.calls) == 1
assert retry.calls == [{"exc": error, "countdown": 60}]
def test_retry_budget_exhaustion_raises_original_failure(monkeypatch: pytest.MonkeyPatch) -> None:
error = RuntimeError("KnowledgeFS remained unavailable")
capability = _install_capability(monkeypatch, error)
retry = RecordingRetry()
monkeypatch.setattr(task_module.capture_workflow_failed_retrieval_task, "retry", retry)
with pytest.raises(RuntimeError, match="KnowledgeFS remained unavailable") as exc_info:
_run_task_with_retries(3)
assert exc_info.value is error
assert len(capability.calls) == 1
assert retry.calls == []
@@ -0,0 +1,183 @@
from collections.abc import Callable
from dataclasses import dataclass, field
from typing import Never
from unittest.mock import patch
from sqlalchemy import inspect
from core.db.session_factory import session_factory
from models.account import Account, AccountStatus, Tenant, TenantAccountJoin, TenantAccountRole
from services.knowledge_fs.initial_source_preview import KnowledgeFSInitialSourcePreviewCanceledError
from services.knowledge_fs.product_dto import (
KnowledgeFSInitialSourcePreviewJobResponse,
KnowledgeFSInitialSourcePreviewResponse,
)
from tasks.knowledge_fs_initial_source_preview_tasks import run_knowledge_fs_initial_source_preview
from tests.helpers.unit.tasks.task_options import task_options
_JOB_KEY = {"tenant_id": "tenant-1", "account_id": "account-1", "job_id": "job-1"}
@dataclass
class _PreviewJobService:
"""Records preview job transitions; ``transition_results`` are consumed in order."""
transition_results: list[bool]
job_status: str = "running"
transitions: list[dict[str, object]] = field(default_factory=list)
stored: list[dict[str, object]] = field(default_factory=list)
released: list[dict[str, str]] = field(default_factory=list)
lookups: list[dict[str, str]] = field(default_factory=list)
def transition_status(self, **kwargs: object) -> bool:
self.transitions.append(kwargs)
return self.transition_results.pop(0)
def store_content(self, **kwargs: object) -> None:
self.stored.append(kwargs)
def release_active_job(self, *, tenant_id: str, account_id: str, job_id: str) -> None:
self.released.append({"tenant_id": tenant_id, "account_id": account_id, "job_id": job_id})
def get(self, *, tenant_id: str, account_id: str, job_id: str) -> KnowledgeFSInitialSourcePreviewJobResponse:
self.lookups.append({"tenant_id": tenant_id, "account_id": account_id, "job_id": job_id})
return KnowledgeFSInitialSourcePreviewJobResponse.model_construct(status=self.job_status)
@dataclass
class _PreviewService:
run: Callable[..., KnowledgeFSInitialSourcePreviewResponse]
calls: list[dict[str, object]] = field(default_factory=list)
def preview(self, **kwargs: object) -> KnowledgeFSInitialSourcePreviewResponse:
self.calls.append(kwargs)
return self.run(**kwargs)
def _payload() -> dict[str, object]:
return {
"credentialId": "credential-1",
"datasource": "crawl",
"kind": "website_crawl",
"parameters": {"url": "https://docs.dify.ai"},
"pluginId": "langgenius/firecrawl_datasource",
"provider": "firecrawl",
}
def _seed_account(*, member: bool) -> None:
tenant = Tenant(name="Preview tenant")
tenant.id = "tenant-1"
account = Account(name="Preview user", email="preview-user@example.com", status=AccountStatus.ACTIVE)
account.id = "account-1"
with session_factory.create_session() as session:
session.add_all([tenant, account])
if member:
session.add(
TenantAccountJoin(
tenant_id=tenant.id,
account_id=account.id,
current=True,
role=TenantAccountRole.OWNER,
)
)
session.commit()
def _run_preview(job_service: _PreviewJobService, preview_service: _PreviewService | None) -> list[dict[str, object]]:
"""Run the task and return the cleanup schedules it requested."""
scheduled_cleanups: list[dict[str, object]] = []
with (
patch(
"tasks.knowledge_fs_initial_source_preview_tasks.KnowledgeFSInitialSourcePreviewJobService",
return_value=job_service,
),
patch(
"tasks.knowledge_fs_initial_source_preview_tasks.KnowledgeFSInitialSourcePreviewService",
return_value=preview_service,
) as preview_service_type,
patch(
"tasks.knowledge_fs_initial_source_preview_tasks.cleanup_knowledge_fs_initial_source_preview.apply_async",
side_effect=lambda **kwargs: scheduled_cleanups.append(kwargs),
),
):
run_knowledge_fs_initial_source_preview.run(
tenant_id="tenant-1",
account_id="account-1",
job_id="job-1",
payload=_payload(),
)
if preview_service is None:
preview_service_type.assert_not_called()
return scheduled_cleanups
def test_preview_task_uses_the_dataset_queue() -> None:
assert task_options(run_knowledge_fs_initial_source_preview)["queue"] == "dataset"
def test_preview_task_persists_running_and_completed_states() -> None:
_seed_account(member=True)
job_service = _PreviewJobService(transition_results=[True, True])
result = KnowledgeFSInitialSourcePreviewResponse(kind="website_crawl")
preview_service = _PreviewService(run=lambda **_: result)
scheduled_cleanups = _run_preview(job_service, preview_service)
assert job_service.transitions == [
{**_JOB_KEY, "status": "running", "allowed_from": ("pending",)},
{**_JOB_KEY, "status": "completed", "allowed_from": ("running",), "result": result},
]
assert job_service.stored == [{**_JOB_KEY, "result": result}]
assert scheduled_cleanups == [
{"kwargs": {"account_id": "account-1", "job_id": "job-1", "tenant_id": "tenant-1"}, "countdown": 3600}
]
assert job_service.released == [_JOB_KEY]
assert len(preview_service.calls) == 1
account = preview_service.calls[0]["account"]
assert isinstance(account, Account)
assert account.id == "account-1"
# The tenant was selected through the task's session before the account was detached from it.
assert account.current_tenant_id == "tenant-1"
assert inspect(account).detached
def test_preview_task_does_not_start_a_canceled_job() -> None:
job_service = _PreviewJobService(transition_results=[False])
_run_preview(job_service, preview_service=None)
assert job_service.transitions == [{**_JOB_KEY, "status": "running", "allowed_from": ("pending",)}]
assert job_service.released == [_JOB_KEY]
def test_preview_task_keeps_a_cooperatively_canceled_job_terminal() -> None:
_seed_account(member=True)
job_service = _PreviewJobService(transition_results=[True], job_status="canceled")
def preview(**kwargs: object) -> Never:
is_canceled = kwargs["is_canceled"]
assert callable(is_canceled)
assert is_canceled() is True
raise KnowledgeFSInitialSourcePreviewCanceledError("canceled")
preview_service = _PreviewService(run=preview)
_run_preview(job_service, preview_service)
assert job_service.transitions == [{**_JOB_KEY, "status": "running", "allowed_from": ("pending",)}]
assert len(preview_service.calls) == 1
assert job_service.lookups == [_JOB_KEY]
def test_preview_task_rejects_an_account_removed_from_the_tenant() -> None:
_seed_account(member=False)
job_service = _PreviewJobService(transition_results=[True, True])
_run_preview(job_service, preview_service=None)
assert job_service.transitions[-1] == {
**_JOB_KEY,
"status": "failed",
"allowed_from": ("pending", "running"),
}
@@ -0,0 +1,262 @@
from unittest.mock import patch
import pytest
from services.knowledge_fs.initial_source_import import KnowledgeFSInitialSourceNotReadyError
from services.knowledge_fs.product_dto import (
KnowledgeFSInitialOnlineDocumentSourcePayload,
KnowledgeFSInitialOnlineDriveSourcePayload,
KnowledgeFSInitialWebsiteSourcePayload,
)
from services.knowledge_fs.product_remote import (
KnowledgeFSProductRemoteError,
KnowledgeFSProductRequestRejectedError,
KnowledgeFSProductResourceNotFoundError,
)
from tasks.knowledge_fs_initial_source_tasks import import_initial_source, import_initial_website_source
def _payload(sync_policy: str = "daily") -> KnowledgeFSInitialWebsiteSourcePayload:
return KnowledgeFSInitialWebsiteSourcePayload.model_validate(
{
"kind": "website_crawl",
"name": "Dify docs",
"provider": "firecrawl",
"providerDisplayName": "Firecrawl",
"parameters": {"url": "https://docs.dify.ai", "limit": 25},
"root_url": "https://docs.dify.ai",
"crawl_options": {"include_subpages": True, "limit": 25},
"selection": [
{"source_url": "https://docs.dify.ai/a", "title": "A"},
{"source_url": "https://docs.dify.ai/b", "title": "B"},
],
**({"custom_interval_seconds": 129_600} if sync_policy == "custom" else {}),
"sync_policy": sync_policy,
}
)
def _legacy_payload() -> KnowledgeFSInitialWebsiteSourcePayload:
return KnowledgeFSInitialWebsiteSourcePayload.model_validate(
{
"kind": "website_crawl",
"name": "Dify docs",
"provider": "firecrawl",
"providerDisplayName": "Firecrawl",
"root_url": "https://docs.dify.ai",
"crawl_options": {"include_subpages": False, "limit": 25},
"selection": [{"source_url": "https://docs.dify.ai/a", "title": "A"}],
"sync_policy": "manual",
}
)
def _document_payload() -> KnowledgeFSInitialOnlineDocumentSourcePayload:
return KnowledgeFSInitialOnlineDocumentSourcePayload.model_validate(
{
"kind": "online_document",
"name": "Product wiki",
"pluginId": "langgenius/notion_datasource",
"provider": "notion",
"providerDisplayName": "Notion",
"datasource": "pages",
"credentialId": "notion-credential-1",
"selection": [
{
"lastEditedTime": "2026-08-10T00:00:00Z",
"name": "Roadmap",
"pageId": "page-1",
"providerItemId": "notion:page-1",
"type": "page",
"workspaceId": "workspace-1",
}
],
}
)
def _drive_payload() -> KnowledgeFSInitialOnlineDriveSourcePayload:
return KnowledgeFSInitialOnlineDriveSourcePayload.model_validate(
{
"kind": "online_drive",
"name": "Team drive",
"pluginId": "langgenius/google_drive",
"provider": "google_drive",
"providerDisplayName": "Google Drive",
"datasource": "google_drive",
"credentialId": "drive-credential-1",
"selection": [
{
"id": "file-1",
"mimeType": "application/pdf",
"name": "Plan.pdf",
"providerItemId": "google-drive:file-1",
}
],
"sync_policy": "manual",
}
)
def test_initial_website_source_task_returns_result_and_retries_not_ready_error() -> None:
serialized_payload = _payload().model_dump(mode="json")
with patch(
"tasks.knowledge_fs_initial_source_tasks.start_initial_website_source_import",
return_value="workflow-1",
):
assert (
import_initial_website_source.run(
tenant_id="tenant-1",
account_id="account-1",
control_space_id="control-1",
operation_id="operation-1",
payload=serialized_payload,
)
== "workflow-1"
)
retry_error = RuntimeError("retry requested")
with (
patch(
"tasks.knowledge_fs_initial_source_tasks.start_initial_website_source_import",
side_effect=KnowledgeFSInitialSourceNotReadyError("not ready"),
),
patch.object(import_initial_website_source, "retry", side_effect=retry_error) as retry,
pytest.raises(RuntimeError, match="retry requested"),
):
import_initial_website_source.run(
tenant_id="tenant-1",
account_id="account-1",
control_space_id="control-1",
operation_id="operation-1",
payload=serialized_payload,
)
retry.assert_called_once()
def test_initial_website_source_task_retries_with_workflow_id() -> None:
serialized_payload = _payload().model_dump(mode="json")
retry_error = RuntimeError("retry requested")
with (
patch(
"tasks.knowledge_fs_initial_source_tasks.start_initial_website_source_import",
side_effect=KnowledgeFSInitialSourceNotReadyError(
"not ready",
workflow_id="workflow-1",
),
),
patch.object(import_initial_website_source, "retry", side_effect=retry_error) as retry,
pytest.raises(RuntimeError, match="retry requested"),
):
import_initial_website_source.run(
tenant_id="tenant-1",
account_id="account-1",
control_space_id="control-1",
operation_id="operation-1",
payload=serialized_payload,
)
retry.assert_called_once_with(
exc=retry.call_args.kwargs["exc"],
kwargs={
"tenant_id": "tenant-1",
"account_id": "account-1",
"control_space_id": "control-1",
"operation_id": "operation-1",
"payload": serialized_payload,
"workflow_id": "workflow-1",
},
)
def test_initial_source_task_validates_discriminated_connector_payload() -> None:
serialized_payload = _document_payload().model_dump(mode="json", by_alias=True)
with patch(
"tasks.knowledge_fs_initial_source_tasks.start_initial_source_import",
return_value="workflow-1",
) as start_import:
assert (
import_initial_source.run(
tenant_id="tenant-1",
account_id="account-1",
control_space_id="control-1",
operation_id="operation-1",
payload=serialized_payload,
)
== "workflow-1"
)
parsed_payload = start_import.call_args.kwargs["payload"]
assert isinstance(parsed_payload, KnowledgeFSInitialOnlineDocumentSourcePayload)
assert parsed_payload.credential_id == "notion-credential-1"
def test_initial_source_task_retries_transient_remote_error() -> None:
serialized_payload = _drive_payload().model_dump(mode="json", by_alias=True)
remote_error = KnowledgeFSProductRemoteError("temporary outage")
retry_error = RuntimeError("retry requested")
with (
patch(
"tasks.knowledge_fs_initial_source_tasks.start_initial_source_import",
side_effect=remote_error,
),
patch.object(import_initial_source, "retry", side_effect=retry_error) as retry,
pytest.raises(RuntimeError, match="retry requested"),
):
import_initial_source.run(
tenant_id="tenant-1",
account_id="account-1",
control_space_id="control-1",
operation_id="operation-1",
payload=serialized_payload,
workflow_id="workflow-1",
)
retry.assert_called_once_with(exc=remote_error)
def test_initial_source_task_retries_rate_limited_request() -> None:
serialized_payload = _payload().model_dump(mode="json", by_alias=True)
rejected = KnowledgeFSProductRequestRejectedError(status_code=429)
retry_error = RuntimeError("retry requested")
with (
patch(
"tasks.knowledge_fs_initial_source_tasks.start_initial_source_import",
side_effect=rejected,
),
patch.object(import_initial_source, "retry", side_effect=retry_error) as retry,
pytest.raises(RuntimeError, match="retry requested"),
):
import_initial_source.run(
tenant_id="tenant-1",
account_id="account-1",
control_space_id="control-1",
operation_id="operation-1",
payload=serialized_payload,
workflow_id=None,
)
retry.assert_called_once_with(exc=rejected)
def test_initial_source_task_does_not_retry_authoritative_missing_resource() -> None:
serialized_payload = _drive_payload().model_dump(mode="json", by_alias=True)
missing_resource = KnowledgeFSProductResourceNotFoundError("workflow was not found")
with (
patch(
"tasks.knowledge_fs_initial_source_tasks.start_initial_source_import",
side_effect=missing_resource,
),
patch.object(import_initial_source, "retry") as retry,
pytest.raises(KnowledgeFSProductResourceNotFoundError, match="workflow was not found"),
):
import_initial_source.run(
tenant_id="tenant-1",
account_id="account-1",
control_space_id="control-1",
operation_id="operation-1",
payload=serialized_payload,
workflow_id="workflow-1",
)
retry.assert_not_called()
@@ -0,0 +1,157 @@
from __future__ import annotations
from dataclasses import dataclass, field
from datetime import timedelta
from unittest.mock import patch
import pytest
from services.knowledge_fs.lifecycle_readiness import KnowledgeFSLifecycleWorkerReadiness
from services.knowledge_fs.lifecycle_saga import KnowledgeFSLifecycleDispatchResult
from services.knowledge_fs.orphan_reconciler import KnowledgeFSOrphanReport
from tasks.knowledge_fs_lifecycle_tasks import (
cleanup_knowledge_fs_staged_uploads,
run_knowledge_fs_lifecycle_worker,
)
from tests.helpers.unit.config_override import apply_config_overrides
@dataclass
class _SagaRunner:
results: list[KnowledgeFSLifecycleDispatchResult]
dispatches: list[dict[str, object]] = field(default_factory=list)
def dispatch_one(self, **kwargs: object) -> KnowledgeFSLifecycleDispatchResult:
self.dispatches.append(kwargs)
return self.results.pop(0)
@dataclass
class _OrphanReconciler:
report: KnowledgeFSOrphanReport
calls: list[dict[str, object]] = field(default_factory=list)
def reconcile(self, *, limit: int, apply_repairs: bool) -> KnowledgeFSOrphanReport:
self.calls.append({"limit": limit, "apply_repairs": apply_repairs})
return self.report
@dataclass
class _StagedUploads:
expired: int
limits: list[int] = field(default_factory=list)
def cleanup_expired(self, *, limit: int) -> int:
self.limits.append(limit)
return self.expired
@dataclass(frozen=True)
class _Runtime:
facade: str
def test_worker_self_check_returns_before_assembling_remote_when_rollout_is_not_ready() -> None:
with (
patch(
"tasks.knowledge_fs_lifecycle_tasks.get_configured_knowledge_fs_lifecycle_worker_readiness",
return_value=KnowledgeFSLifecycleWorkerReadiness(False, ("legacy_acl_freeze",)),
),
patch("tasks.knowledge_fs_lifecycle_tasks.get_knowledge_fs_lifecycle_remote") as remote,
):
result = run_knowledge_fs_lifecycle_worker.run()
assert result == {"status": "disabled", "dispatched": 0, "completed": 0, "reconciled": 0}
remote.assert_not_called()
def test_ready_worker_dispatches_a_bounded_batch_then_repairs_orphans(
monkeypatch: pytest.MonkeyPatch,
) -> None:
runner = _SagaRunner(
results=[
KnowledgeFSLifecycleDispatchResult(claimed=True, completed=True, outbox_id="outbox-1"),
KnowledgeFSLifecycleDispatchResult(claimed=True, completed=False, outbox_id="outbox-2"),
KnowledgeFSLifecycleDispatchResult(claimed=False, completed=False, outbox_id=None),
]
)
reconciler = _OrphanReconciler(
report=KnowledgeFSOrphanReport(
repaired_control_space_ids=("control-1",),
remote_orphan_space_ids=(),
local_missing_remote_ids=(),
cleanup_control_space_ids=("control-2", "control-3"),
)
)
apply_config_overrides(
monkeypatch,
KNOWLEDGE_FS_LIFECYCLE_BATCH_SIZE=5,
KNOWLEDGE_FS_LIFECYCLE_LEASE_SECONDS=30,
KNOWLEDGE_FS_ENABLED=True,
)
with (
patch(
"tasks.knowledge_fs_lifecycle_tasks.get_configured_knowledge_fs_lifecycle_worker_readiness",
return_value=KnowledgeFSLifecycleWorkerReadiness(True, ()),
),
patch("tasks.knowledge_fs_lifecycle_tasks.get_knowledge_fs_lifecycle_remote", return_value="remote-1"),
patch("tasks.knowledge_fs_lifecycle_tasks.session_factory.get_session_maker", return_value="maker-1"),
patch("tasks.knowledge_fs_lifecycle_tasks.KnowledgeFSLifecycleSagaRunner", return_value=runner),
patch("tasks.knowledge_fs_lifecycle_tasks.KnowledgeFSOrphanReconciler", return_value=reconciler),
patch("tasks.knowledge_fs_lifecycle_tasks.socket.gethostname", return_value="worker-host"),
patch("tasks.knowledge_fs_lifecycle_tasks.naive_utc_now", return_value="now-1"),
):
result = run_knowledge_fs_lifecycle_worker.run()
assert result == {"status": "ok", "dispatched": 2, "completed": 1, "reconciled": 3}
assert len(runner.dispatches) == 3
assert runner.dispatches[-1] == {
"worker_id": "worker-host:manual",
"now": "now-1",
"lease_duration": timedelta(seconds=30),
"product_enabled": True,
}
assert reconciler.calls == [{"limit": 5, "apply_repairs": True}]
def test_staged_upload_cleanup_returns_before_runtime_assembly_when_not_ready() -> None:
with (
patch(
"tasks.knowledge_fs_lifecycle_tasks.get_configured_knowledge_fs_lifecycle_worker_readiness",
return_value=KnowledgeFSLifecycleWorkerReadiness(False, ("worker_disabled",)),
),
patch("tasks.knowledge_fs_lifecycle_tasks.get_knowledge_fs_runtime") as runtime,
):
result = cleanup_knowledge_fs_staged_uploads.run()
assert result == 0
runtime.assert_not_called()
def test_staged_upload_cleanup_uses_the_configured_batch_limit(monkeypatch: pytest.MonkeyPatch) -> None:
staged_uploads = _StagedUploads(expired=4)
apply_config_overrides(monkeypatch, KNOWLEDGE_FS_LIFECYCLE_BATCH_SIZE=25)
with (
patch(
"tasks.knowledge_fs_lifecycle_tasks.get_configured_knowledge_fs_lifecycle_worker_readiness",
return_value=KnowledgeFSLifecycleWorkerReadiness(True, ()),
),
patch(
"tasks.knowledge_fs_lifecycle_tasks.session_factory.get_session_maker",
return_value="maker-1",
),
patch(
"tasks.knowledge_fs_lifecycle_tasks.get_knowledge_fs_runtime",
return_value=_Runtime(facade="facade-1"),
) as runtime,
patch(
"tasks.knowledge_fs_lifecycle_tasks.KnowledgeFSStagedUploadService",
return_value=staged_uploads,
) as service,
):
result = cleanup_knowledge_fs_staged_uploads.run()
assert result == 4
runtime.assert_called_once_with("maker-1")
service.assert_called_once_with("maker-1", facade="facade-1")
assert staged_uploads.limits == [25]
@@ -0,0 +1,327 @@
from typing import Literal, Never
from unittest.mock import patch
import pytest
from celery.exceptions import Retry
from services.knowledge_fs.product_dto import (
KnowledgeFSSourceResponse,
KnowledgeFSSourceSyncPolicyPayload,
KnowledgeFSSourceUpdatePayload,
KnowledgeFSSourceWorkflowResponse,
)
from services.knowledge_fs.product_remote import (
KnowledgeFSProductRequestRejectedError,
KnowledgeFSProductResourceNotFoundError,
)
from tasks import knowledge_fs_source_import_tasks as task_module
from tasks.knowledge_fs_source_import_tasks import (
KnowledgeFSSourceImportNotReadyError,
finalize_source_import_once,
)
from tests.helpers.unit.tasks.knowledge_fs_task_fakes import (
FacadeRuntime,
RecordingKnowledgeFSFacade,
RecordingRetry,
json_object,
source_response,
source_workflow_response,
)
def _source(*, status: str = "syncing") -> KnowledgeFSSourceResponse:
return source_response(
metadata={
"parameters": {"limit": 99},
"preview": False,
"pendingImport": {
"kind": "crawl-preview-selection",
"previewWorkflowId": "preview-1",
"workflowId": "import-1",
"syncPolicy": {"enabled": False, "mode": "manual"},
},
},
status=status,
version=4,
)
def _completed_workflow() -> KnowledgeFSSourceWorkflowResponse:
return source_workflow_response(workflow_id="import-1", state="completed")
def _facade(workflow: KnowledgeFSSourceWorkflowResponse) -> RecordingKnowledgeFSFacade:
facade = RecordingKnowledgeFSFacade()
facade.script("get_source_workflow", workflow)
facade.script("get_source_sync_policy", KnowledgeFSProductResourceNotFoundError("missing"))
facade.script("update_source_sync_policy", None)
return facade
def _run(facade: RecordingKnowledgeFSFacade) -> str:
with patch(
"tasks.knowledge_fs_source_import_tasks.get_knowledge_fs_runtime",
return_value=FacadeRuntime(facade),
):
return finalize_source_import_once(
tenant_id="tenant-1",
account_id="account-1",
control_space_id="control-1",
source_id="source-1",
workflow_id="import-1",
)
def _raise(error: Exception) -> Never:
raise error
def test_finalize_source_import_waits_for_terminal_workflow() -> None:
facade = _facade(source_workflow_response(workflow_id="import-1", state="importing"))
with pytest.raises(KnowledgeFSSourceImportNotReadyError):
_run(facade)
assert facade.call_count("get_source") == 0
def test_finalize_source_import_activates_source_then_applies_policy_for_new_version() -> None:
facade = _facade(_completed_workflow())
facade.script("get_source", _source())
facade.script(
"update_source",
source_response(
metadata={
"lastImport": {
"kind": "crawl-preview-selection",
"previewWorkflowId": "preview-1",
"state": "completed",
"syncPolicy": {"enabled": False, "mode": "manual"},
"workflowId": "import-1",
},
"preview": False,
},
status="active",
version=5,
),
)
assert _run(facade) == "import-1"
policy = facade.payload("update_source_sync_policy", KnowledgeFSSourceSyncPolicyPayload)
assert policy.enabled is False
assert policy.mode == "manual"
assert policy.expected_revision == 0
assert policy.expected_source_version == 5
update = facade.payload("update_source", KnowledgeFSSourceUpdatePayload)
update_metadata = json_object(update.metadata)
assert update.status == "active"
assert "parameters" not in update_metadata
assert update_metadata["pendingImport"] is None
assert json_object(update_metadata["lastImport"])["state"] == "completed"
def test_finalize_source_import_retries_policy_after_source_activation() -> None:
facade = _facade(_completed_workflow())
facade.script(
"get_source",
source_response(
metadata={
"lastImport": {
"kind": "crawl-preview-selection",
"previewWorkflowId": "preview-1",
"state": "completed",
"syncPolicy": {"enabled": True, "mode": "interval"},
"workflowId": "import-1",
},
"preview": False,
},
status="active",
version=5,
),
)
assert _run(facade) == "import-1"
assert facade.call_count("update_source") == 0
policy = facade.payload("update_source_sync_policy", KnowledgeFSSourceSyncPolicyPayload)
assert policy.expected_source_version == 5
def test_finalize_source_import_recovers_a_stale_failed_marker_after_workflow_completion() -> None:
facade = _facade(_completed_workflow())
facade.script(
"get_source",
source_response(
metadata={
"lastImport": {
"errorCode": "SOURCE_OPERATION_FAILED",
"errorMessage": "provider failed",
"kind": "crawl-preview-selection",
"previewWorkflowId": "preview-1",
"state": "failed",
"syncPolicy": {"enabled": False, "mode": "manual"},
"workflowId": "import-1",
},
"pendingImport": None,
"preview": False,
},
status="error",
version=5,
),
)
facade.script(
"update_source",
source_response(
metadata={
"lastImport": {
"kind": "crawl-preview-selection",
"previewWorkflowId": "preview-1",
"state": "completed",
"syncPolicy": {"enabled": False, "mode": "manual"},
"workflowId": "import-1",
},
"pendingImport": None,
"preview": False,
},
status="active",
version=6,
),
)
assert _run(facade) == "import-1"
update = facade.payload("update_source", KnowledgeFSSourceUpdatePayload)
update_metadata = json_object(update.metadata)
assert update.status == "active"
assert json_object(update_metadata["lastImport"])["state"] == "completed"
assert update_metadata["pendingImport"] is None
policy = facade.payload("update_source_sync_policy", KnowledgeFSSourceSyncPolicyPayload)
assert policy.expected_source_version == 6
def test_finalize_source_import_persists_failure_on_visible_source() -> None:
facade = _facade(
source_workflow_response(
workflow_id="import-1",
state="failed",
failure_message="provider failed",
last_error_code="PROVIDER_FAILED",
)
)
facade.script("get_source", _source())
facade.script("update_source", _source(status="error"))
assert _run(facade) == "import-1"
update = facade.payload("update_source", KnowledgeFSSourceUpdatePayload)
update_metadata = json_object(update.metadata)
assert update.status == "error"
assert "parameters" not in update_metadata
assert update_metadata["lastImport"] == {
"errorCode": "PROVIDER_FAILED",
"errorMessage": "provider failed",
"kind": "crawl-preview-selection",
"previewWorkflowId": "preview-1",
"state": "failed",
"syncPolicy": {"enabled": False, "mode": "manual"},
"workflowId": "import-1",
}
assert update_metadata["pendingImport"] is None
assert facade.call_count("update_source_sync_policy") == 0
def test_finalize_website_source_import_waits_for_policy_write_fence() -> None:
facade = _facade(_completed_workflow())
facade.script(
"get_source",
source_response(
metadata={
"lastImport": {
"kind": "website-crawl-import",
"state": "completed",
"syncPolicy": {"enabled": True, "mode": "interval"},
"workflowId": "import-1",
},
"preview": False,
},
status="active",
version=5,
),
)
facade.script("update_source_sync_policy", KnowledgeFSProductRequestRejectedError(status_code=400))
with pytest.raises(KnowledgeFSSourceImportNotReadyError):
_run(facade)
@pytest.mark.parametrize("import_kind", ["online-document-import", "online-drive-import"])
def test_finalize_non_website_source_import_does_not_reclassify_a_bad_request(
import_kind: str,
) -> None:
facade = _facade(_completed_workflow())
facade.script(
"get_source",
source_response(
metadata={
"lastImport": {
"kind": import_kind,
"state": "completed",
"syncPolicy": {"enabled": True, "mode": "interval"},
"workflowId": "import-1",
},
"preview": False,
},
status="active",
version=5,
),
)
rejection = KnowledgeFSProductRequestRejectedError(status_code=400)
facade.script("update_source_sync_policy", rejection)
with pytest.raises(KnowledgeFSProductRequestRejectedError) as raised:
_run(facade)
assert raised.value is rejection
def test_finalize_source_import_retries_a_source_revision_conflict(monkeypatch: pytest.MonkeyPatch) -> None:
error = KnowledgeFSProductRequestRejectedError(status_code=409)
monkeypatch.setattr(task_module, "finalize_source_import_once", lambda **_: _raise(error))
retry = RecordingRetry()
monkeypatch.setattr(task_module.finalize_source_import, "retry", retry)
with pytest.raises(Retry):
task_module.finalize_source_import.run(
tenant_id="tenant-1",
account_id="account-1",
control_space_id="control-1",
source_id="source-1",
workflow_id="import-1",
)
assert retry.calls == [{"exc": error}]
@pytest.mark.parametrize("status_code", [400, 422])
def test_finalize_source_import_does_not_retry_a_permanent_rejection(
monkeypatch: pytest.MonkeyPatch, status_code: Literal[400, 422]
) -> None:
error = KnowledgeFSProductRequestRejectedError(status_code=status_code)
monkeypatch.setattr(task_module, "finalize_source_import_once", lambda **_: _raise(error))
retry = RecordingRetry()
monkeypatch.setattr(task_module.finalize_source_import, "retry", retry)
with pytest.raises(KnowledgeFSProductRequestRejectedError) as raised:
task_module.finalize_source_import.run(
tenant_id="tenant-1",
account_id="account-1",
control_space_id="control-1",
source_id="source-1",
workflow_id="import-1",
)
assert raised.value is error
assert retry.calls == []
@@ -0,0 +1,185 @@
from __future__ import annotations
from dataclasses import dataclass, field
from unittest.mock import patch
import pytest
from celery.app.task import Context, Task
from services.dataset_knowledge_fs_upgrade_service import (
UPGRADE_CHECKPOINT_TIME_LIMIT_SECONDS,
UPGRADE_DELIVERY_TIMEOUT,
KnowledgeFSUpgradeBusyError,
KnowledgeFSUpgradeNotReadyError,
)
from tasks.knowledge_fs_upgrade_tasks import (
KNOWLEDGE_FS_UPGRADE_QUEUE,
cleanup_deferred_knowledge_fs_upgrade_files,
reconcile_knowledge_fs_upgrade_documents,
recover_stalled_knowledge_fs_upgrades,
run_knowledge_fs_upgrade,
)
from tests.helpers.unit.tasks.task_options import task_options
@dataclass
class _UpgradeRunner:
"""Returns ``outcome`` from ``run_next`` or raises it, and records failure reports."""
outcome: bool | None | Exception
runs: list[dict[str, object]] = field(default_factory=list)
failures: list[dict[str, object]] = field(default_factory=list)
releases: list[tuple[str, str | None]] = field(default_factory=list)
released_tenant_id: str | None = "tenant-1"
def run_next(self, **kwargs: object) -> bool | None:
self.runs.append(kwargs)
if isinstance(self.outcome, Exception):
raise self.outcome
return self.outcome
def fail(self, *, job_id: str, celery_task_id: str, error: Exception) -> None:
self.failures.append({"job_id": job_id, "celery_task_id": celery_task_id, "error": error})
def release_for_retry(self, *, job_id: str, celery_task_id: str) -> None:
self.releases.append((job_id, celery_task_id))
def release_for_dispatch(self, *, job_id: str, celery_task_id: str | None) -> str | None:
self.releases.append((job_id, celery_task_id))
return self.released_tenant_id
def test_upgrade_tasks_are_pinned_to_the_dedicated_queue() -> None:
assert task_options(run_knowledge_fs_upgrade)["queue"] == KNOWLEDGE_FS_UPGRADE_QUEUE
assert task_options(reconcile_knowledge_fs_upgrade_documents)["queue"] == KNOWLEDGE_FS_UPGRADE_QUEUE
assert task_options(cleanup_deferred_knowledge_fs_upgrade_files)["queue"] == KNOWLEDGE_FS_UPGRADE_QUEUE
def test_deferred_file_cleanup_uses_the_upgrade_session_factory() -> None:
with (
patch("tasks.knowledge_fs_upgrade_tasks.session_factory.get_session_maker", return_value="maker"),
patch("tasks.knowledge_fs_upgrade_tasks.cleanup_deferred_upgrade_files", return_value=3) as cleanup,
):
assert cleanup_deferred_knowledge_fs_upgrade_files.run() == 3
cleanup.assert_called_once_with("maker")
@pytest.mark.parametrize("has_more", [True, False])
def test_worker_enqueues_only_the_expected_follow_up(has_more: bool) -> None:
runner = _UpgradeRunner(outcome=has_more)
with (
patch("tasks.knowledge_fs_upgrade_tasks.session_factory.get_session_maker", return_value="maker"),
patch("tasks.knowledge_fs_upgrade_tasks.KnowledgeFSUpgradeRunner", return_value=runner),
patch("services.knowledge.datasets.upgrades.enqueue_upgrade_job") as continue_upgrade,
patch("tasks.knowledge_fs_upgrade_tasks._dispatch_reconciliations") as reconcile,
):
run_knowledge_fs_upgrade.run(job_id="job-1")
assert len(runner.runs) == 1
assert runner.runs[0]["job_id"] == "job-1"
if has_more:
assert runner.releases == [("job-1", None)]
continue_upgrade.assert_called_once()
assert continue_upgrade.call_args.args[1:] == ("tenant-1", "job-1")
reconcile.assert_not_called()
else:
assert runner.releases == []
continue_upgrade.assert_not_called()
reconcile.assert_called_once_with(job_id="job-1")
def test_worker_skips_dispatch_when_the_job_moved_on() -> None:
runner = _UpgradeRunner(outcome=True, released_tenant_id=None)
with (
patch("tasks.knowledge_fs_upgrade_tasks.session_factory.get_session_maker", return_value="maker"),
patch("tasks.knowledge_fs_upgrade_tasks.KnowledgeFSUpgradeRunner", return_value=runner),
patch("services.knowledge.datasets.upgrades.enqueue_upgrade_job") as continue_upgrade,
):
run_knowledge_fs_upgrade.run(job_id="job-1")
continue_upgrade.assert_not_called()
def test_worker_marks_parent_failed_when_a_checkpoint_raises() -> None:
error = RuntimeError("ordinary document upload failed")
runner = _UpgradeRunner(outcome=error)
with (
patch("tasks.knowledge_fs_upgrade_tasks.session_factory.get_session_maker", return_value="maker"),
patch("tasks.knowledge_fs_upgrade_tasks.KnowledgeFSUpgradeRunner", return_value=runner),
pytest.raises(RuntimeError, match="ordinary document upload failed"),
):
run_knowledge_fs_upgrade.run(job_id="job-1")
assert runner.failures == [{"job_id": "job-1", "celery_task_id": None, "error": error}]
def test_worker_marks_parent_failed_when_provisioning_retries_are_exhausted() -> None:
error = KnowledgeFSUpgradeNotReadyError("Space is still provisioning")
runner = _UpgradeRunner(outcome=error)
run_knowledge_fs_upgrade.request_stack.push(Context(retries=run_knowledge_fs_upgrade.max_retries))
try:
with (
patch("tasks.knowledge_fs_upgrade_tasks.session_factory.get_session_maker", return_value="maker"),
patch("tasks.knowledge_fs_upgrade_tasks.KnowledgeFSUpgradeRunner", return_value=runner),
pytest.raises(KnowledgeFSUpgradeNotReadyError, match="still provisioning"),
):
run_knowledge_fs_upgrade.run(job_id="job-1")
finally:
run_knowledge_fs_upgrade.pop_request()
assert runner.failures == [{"job_id": "job-1", "celery_task_id": None, "error": error}]
@pytest.mark.parametrize("task", [run_knowledge_fs_upgrade, reconcile_knowledge_fs_upgrade_documents])
def test_upgrade_checkpoint_is_redelivered_on_worker_loss_and_bounded_before_recovery(task: Task[..., None]) -> None:
assert task.acks_late is True
assert task.reject_on_worker_lost is True
options = task_options(task)
assert options["time_limit"] == UPGRADE_CHECKPOINT_TIME_LIMIT_SECONDS
assert options["soft_time_limit"] == UPGRADE_CHECKPOINT_TIME_LIMIT_SECONDS - 30
assert UPGRADE_DELIVERY_TIMEOUT.total_seconds() > UPGRADE_CHECKPOINT_TIME_LIMIT_SECONDS
def test_superseded_delivery_does_not_schedule_any_follow_up() -> None:
runner = _UpgradeRunner(outcome=None)
with (
patch("tasks.knowledge_fs_upgrade_tasks.KnowledgeFSUpgradeRunner", return_value=runner),
patch("services.knowledge.datasets.upgrades.enqueue_upgrade_job") as dispatch,
patch.object(reconcile_knowledge_fs_upgrade_documents, "apply_async") as reconcile,
):
run_knowledge_fs_upgrade.run(job_id="job-1")
dispatch.assert_not_called()
reconcile.assert_not_called()
@pytest.mark.parametrize("busy", [False, True])
def test_waiting_delivery_preserves_live_execution_and_releases_provisioning_retry(busy: bool) -> None:
error = KnowledgeFSUpgradeBusyError("busy") if busy else KnowledgeFSUpgradeNotReadyError("provisioning")
runner = _UpgradeRunner(outcome=error)
run_knowledge_fs_upgrade.request_stack.push(Context(id="task-1", retries=0))
try:
with (
patch("tasks.knowledge_fs_upgrade_tasks.KnowledgeFSUpgradeRunner", return_value=runner),
patch.object(run_knowledge_fs_upgrade, "retry", side_effect=RuntimeError("retry sent")) as retry,
pytest.raises(RuntimeError, match="retry sent"),
):
run_knowledge_fs_upgrade.run(job_id="job-1")
assert runner.releases == ([] if busy else [("job-1", "task-1")])
assert runner.failures == []
retry.assert_called_once()
finally:
run_knowledge_fs_upgrade.pop_request()
def test_recovery_sweep_dispatches_each_abandoned_job_even_if_one_publish_fails() -> None:
with (
patch("tasks.knowledge_fs_upgrade_tasks._dispatch_reconciliations", return_value=0),
patch("tasks.knowledge_fs_upgrade_tasks.KnowledgeFSUpgradeSnapshotService") as snapshots,
patch(
"services.knowledge.datasets.upgrades.enqueue_upgrade_job", side_effect=[OSError("broker down"), None]
) as dispatch,
):
snapshots.return_value.recover_stalled_jobs.return_value = [("tenant-1", "job-1"), ("tenant-2", "job-2")]
assert recover_stalled_knowledge_fs_upgrades.run() == 2
assert [call.args[1:] for call in dispatch.call_args_list] == [("tenant-1", "job-1"), ("tenant-2", "job-2")]
@@ -0,0 +1,143 @@
from collections.abc import Sequence
from datetime import UTC, datetime, timedelta
from uuid import uuid4
import pytest
from sqlalchemy.engine import Engine
from sqlalchemy.orm import Session, sessionmaker
from graphon.runtime import VariablePool
from models.human_input import HumanInputDelivery, HumanInputForm, HumanInputFormRecipient
from services.human_input_form_delivery_provider import (
HumanInputFormDeliveryContext,
HumanInputFormDeliveryDispatcher,
HumanInputFormDeliveryProviderRegistry,
)
from tasks import mail_human_input_delivery_task as task_module
@pytest.mark.parametrize(
"sqlite_session", [(HumanInputForm, HumanInputDelivery, HumanInputFormRecipient)], indirect=True
)
@pytest.mark.parametrize(
("email_only", "mail_inited", "feature_enabled", "feature_error", "expected_methods"),
[
(True, True, True, False, (task_module.DeliveryMethodType.EMAIL,)),
(True, True, False, False, ()),
(True, False, True, False, ()),
(False, True, True, False, task_module.FORM_DELIVERY_METHOD_TYPES),
(False, False, True, False, (task_module.DeliveryMethodType.IM,)),
(False, True, False, False, (task_module.DeliveryMethodType.IM,)),
(False, True, True, True, (task_module.DeliveryMethodType.IM,)),
],
ids=[
"legacy-email",
"legacy-disabled",
"legacy-unconfigured",
"all-channels",
"im-without-mail",
"im-without-feature",
"im-after-feature-error",
],
)
def test_delivery_task_dispatches_available_persisted_channels(
monkeypatch: pytest.MonkeyPatch,
sqlite_engine: Engine,
sqlite_session: Session,
email_only: bool,
mail_inited: bool,
feature_enabled: bool,
feature_error: bool,
expected_methods: tuple[task_module.DeliveryMethodType, ...],
) -> None:
form = HumanInputForm(
tenant_id=str(uuid4()),
app_id=str(uuid4()),
workflow_run_id=None,
conversation_id=None,
node_id="human-input",
form_definition="{}",
rendered_content="content",
expiration_time=datetime.now(UTC) + timedelta(hours=1),
)
sqlite_session.add(form)
sqlite_session.flush()
deliveries = [
HumanInputDelivery(form_id=form.id, delivery_method_type=method, channel_payload="{}")
for method in task_module.FORM_DELIVERY_METHOD_TYPES
]
sqlite_session.add_all(deliveries)
sqlite_session.commit()
pool = VariablePool()
sent: list[HumanInputFormDeliveryContext] = []
loading_sessions: list[Session] = []
feature_tenants: list[str] = []
load_contexts = HumanInputFormDeliveryDispatcher.load_form_contexts
def load(
dispatcher: HumanInputFormDeliveryDispatcher,
*,
session: Session,
form: HumanInputForm,
variable_pool: VariablePool | None = None,
delivery_method_types: Sequence[task_module.DeliveryMethodType] | None = None,
) -> tuple[HumanInputFormDeliveryContext, ...]:
loading_sessions.append(session)
assert session.bind is sqlite_engine
assert variable_pool is pool
assert delivery_method_types == (
(task_module.DeliveryMethodType.EMAIL,) if email_only else task_module.FORM_DELIVERY_METHOD_TYPES
)
return load_contexts(
dispatcher,
session=session,
form=form,
variable_pool=variable_pool,
delivery_method_types=delivery_method_types,
)
def dispatch(
_registry: HumanInputFormDeliveryProviderRegistry,
*,
context: HumanInputFormDeliveryContext,
continue_on_error: bool = True,
) -> bool:
assert len(loading_sessions) == 1
assert not loading_sessions[0].in_transaction()
assert not loading_sessions[0].identity_map
assert continue_on_error
assert context.variable_pool is pool
sent.append(context)
return True
from services.entities.feature_entities import FeatureModel
from services.feature_service import FeatureService
def features(tenant_id: str, **_kwargs: object) -> FeatureModel:
feature_tenants.append(tenant_id)
if feature_error:
raise RuntimeError("feature service unavailable")
return FeatureModel(human_input_email_delivery_enabled=feature_enabled)
monkeypatch.setattr(task_module.mail, "is_inited", lambda: mail_inited)
monkeypatch.setattr(FeatureService, "get_features", features)
monkeypatch.setattr(task_module, "_load_variable_pool", lambda _run_id: pool)
monkeypatch.setattr(HumanInputFormDeliveryProviderRegistry, "dispatch", dispatch)
monkeypatch.setattr(HumanInputFormDeliveryDispatcher, "load_form_contexts", load)
task = (
task_module.dispatch_human_input_email_task
if email_only
else task_module.dispatch_human_input_form_delivery_task
)
task(
form_id=form.id, node_title="Approve", session_factory=sessionmaker(bind=sqlite_engine, expire_on_commit=False)
)
assert tuple(context.delivery_method_type for context in sent) == expected_methods
assert len(loading_sessions) == (0 if email_only and (not mail_inited or not feature_enabled) else 1)
assert feature_tenants == ([form.tenant_id] if mail_inited else [])
for context in sent:
assert context.form_id == form.id
assert context.tenant_id == form.tenant_id
assert context.app_id == form.app_id
assert context.rendered_content == "content"
assert context.delivery_id in {delivery.id for delivery in deliveries}
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,37 @@
import pytest
from tasks import ops_trace_task
class _Storage:
def __init__(self) -> None:
self.files: dict[str, bytes] = {}
def save(self, key: str, data: bytes) -> None:
self.files[key] = data
def delete(self, key: str) -> None:
self.files.pop(key, None)
@pytest.fixture
def storage(monkeypatch: pytest.MonkeyPatch) -> _Storage:
fake = _Storage()
monkeypatch.setattr(ops_trace_task, "storage", fake)
return fake
def test_legacy_trace_tasks_are_drained(storage: _Storage) -> None:
storage.save("ops_trace/app-1/file-1.json", b"{}")
ops_trace_task.process_trace_tasks.run({"app_id": "app-1", "file_id": "file-1"})
assert storage.files == {}
def test_legacy_trace_tasks_never_delete_outside_their_folder(storage: _Storage) -> None:
storage.save("ops_trace/app-1/file-1.json", b"{}")
ops_trace_task.process_trace_tasks.run({"app_id": "..", "file_id": "../secrets"})
assert list(storage.files) == ["ops_trace/app-1/file-1.json"]
@@ -0,0 +1,431 @@
from __future__ import annotations
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
from core.plugin.entities.marketplace import MarketplacePluginSnapshot
from core.plugin.entities.plugin import PluginCategory, PluginInstallationSource
from models.account import (
TenantPluginAutoUpgradeCategory,
TenantPluginAutoUpgradeMode,
TenantPluginAutoUpgradeStrategySetting,
)
MODULE = "tasks.process_tenant_plugin_autoupgrade_check_task"
def _make_plugin(
plugin_id: str,
version: str,
source: PluginInstallationSource = PluginInstallationSource.Marketplace,
category: PluginCategory = PluginCategory.Tool,
) -> SimpleNamespace:
"""Build a minimal stand-in for a PluginInstallation entry returned by manager.list_plugins."""
return SimpleNamespace(
plugin_id=plugin_id,
version=version,
plugin_unique_identifier=f"{plugin_id}:{version}@deadbeef",
source=source,
declaration=SimpleNamespace(category=category),
)
def _make_manifest(plugin_id: str, latest_version: str) -> MarketplacePluginSnapshot:
org, name = plugin_id.split("/", 1)
return MarketplacePluginSnapshot(
org=org,
name=name,
latest_version=latest_version,
latest_package_identifier=f"{plugin_id}:{latest_version}@cafe1234",
latest_package_url=f"https://marketplace.example/{plugin_id}/{latest_version}.difypkg",
)
def _run_task(
*,
plugins: list[SimpleNamespace],
manifests: list[MarketplacePluginSnapshot],
strategy_setting: TenantPluginAutoUpgradeStrategySetting = TenantPluginAutoUpgradeStrategySetting.LATEST,
upgrade_mode: TenantPluginAutoUpgradeMode = TenantPluginAutoUpgradeMode.ALL,
exclude_plugins: list[str] | None = None,
include_plugins: list[str] | None = None,
category: PluginCategory | str | None = None,
) -> tuple[MagicMock, list[tuple[str, str, str]]]:
"""
Execute the celery task synchronously with mocks for the plugin manager,
the marketplace cache and PluginService.upgrade_plugin_with_marketplace.
Returns the upgrade-call recorder so each test can assert on it.
"""
fake_manager = MagicMock()
fake_manager.list_plugins.return_value = plugins
upgrade_calls: list[tuple[str, str, str]] = []
def _record_upgrade(tenant_id: str, original: str, new: str) -> None:
upgrade_calls.append((tenant_id, original, new))
with (
patch(f"{MODULE}.PluginInstaller", return_value=fake_manager),
patch(f"{MODULE}.marketplace_batch_fetch_plugin_manifests", return_value=manifests),
patch(
f"{MODULE}.PluginService.upgrade_plugin_with_marketplace",
side_effect=_record_upgrade,
) as upgrade_mock,
):
from tasks.process_tenant_plugin_autoupgrade_check_task import (
process_tenant_plugin_autoupgrade_check_task,
)
process_tenant_plugin_autoupgrade_check_task(
"tenant-1",
strategy_setting,
0,
upgrade_mode,
exclude_plugins or [],
include_plugins or [],
category,
)
return upgrade_mock, upgrade_calls
class TestUpgradeCallsMarketplaceService:
"""
Regression test for the bug where the auto-upgrade task called
manager.upgrade_plugin directly, which skipped downloading the new package
from marketplace and uploading it to the daemon. The daemon then failed with
"package file not found" and the upgrade silently never completed.
"""
def test_upgrade_routes_through_plugin_service(self) -> None:
plugin = _make_plugin("acme/foo", "1.0.0")
manifest = _make_manifest("acme/foo", "1.0.1")
upgrade_mock, calls = _run_task(plugins=[plugin], manifests=[manifest])
upgrade_mock.assert_called_once()
assert calls == [("tenant-1", plugin.plugin_unique_identifier, manifest.latest_package_identifier)]
def test_does_not_call_manager_upgrade_plugin_directly(self) -> None:
"""Locks in that we never go back to the broken path that bypassed download/upload."""
plugin = _make_plugin("acme/foo", "1.0.0")
manifest = _make_manifest("acme/foo", "1.0.1")
fake_manager = MagicMock()
fake_manager.list_plugins.return_value = [plugin]
with (
patch(f"{MODULE}.PluginInstaller", return_value=fake_manager),
patch(f"{MODULE}.marketplace_batch_fetch_plugin_manifests", return_value=[manifest]),
patch(f"{MODULE}.PluginService.upgrade_plugin_with_marketplace"),
):
from tasks.process_tenant_plugin_autoupgrade_check_task import (
process_tenant_plugin_autoupgrade_check_task,
)
process_tenant_plugin_autoupgrade_check_task(
"tenant-1",
TenantPluginAutoUpgradeStrategySetting.LATEST,
0,
TenantPluginAutoUpgradeMode.ALL,
[],
[],
)
fake_manager.upgrade_plugin.assert_not_called()
class TestStrategySetting:
def test_disabled_strategy_skips_everything(self) -> None:
upgrade_mock, _ = _run_task(
plugins=[_make_plugin("acme/foo", "1.0.0")],
manifests=[_make_manifest("acme/foo", "1.0.1")],
strategy_setting=TenantPluginAutoUpgradeStrategySetting.DISABLED,
)
upgrade_mock.assert_not_called()
def test_fix_only_upgrades_patch_version(self) -> None:
upgrade_mock, calls = _run_task(
plugins=[_make_plugin("acme/foo", "1.0.0")],
manifests=[_make_manifest("acme/foo", "1.0.5")],
strategy_setting=TenantPluginAutoUpgradeStrategySetting.FIX_ONLY,
)
upgrade_mock.assert_called_once()
assert calls[0][2].endswith(":1.0.5@cafe1234")
def test_fix_only_skips_minor_bump(self) -> None:
upgrade_mock, _ = _run_task(
plugins=[_make_plugin("acme/foo", "1.0.0")],
manifests=[_make_manifest("acme/foo", "1.1.0")],
strategy_setting=TenantPluginAutoUpgradeStrategySetting.FIX_ONLY,
)
upgrade_mock.assert_not_called()
def test_fix_only_skips_major_bump(self) -> None:
upgrade_mock, _ = _run_task(
plugins=[_make_plugin("acme/foo", "1.0.0")],
manifests=[_make_manifest("acme/foo", "2.0.0")],
strategy_setting=TenantPluginAutoUpgradeStrategySetting.FIX_ONLY,
)
upgrade_mock.assert_not_called()
def test_latest_strategy_skips_when_versions_equal(self) -> None:
upgrade_mock, _ = _run_task(
plugins=[_make_plugin("acme/foo", "1.0.0")],
manifests=[_make_manifest("acme/foo", "1.0.0")],
strategy_setting=TenantPluginAutoUpgradeStrategySetting.LATEST,
)
upgrade_mock.assert_not_called()
class TestUpgradeMode:
def test_mode_all_upgrades_every_marketplace_plugin(self) -> None:
plugins = [
_make_plugin("acme/foo", "1.0.0"),
_make_plugin("acme/bar", "2.0.0"),
]
manifests = [
_make_manifest("acme/foo", "1.0.1"),
_make_manifest("acme/bar", "2.0.1"),
]
upgrade_mock, calls = _run_task(
plugins=plugins,
manifests=manifests,
upgrade_mode=TenantPluginAutoUpgradeMode.ALL,
)
assert upgrade_mock.call_count == 2
upgraded_ids = sorted(c[1] for c in calls)
assert upgraded_ids == sorted(p.plugin_unique_identifier for p in plugins)
def test_mode_all_skips_non_marketplace_sources(self) -> None:
plugins = [
_make_plugin("acme/foo", "1.0.0"),
_make_plugin("acme/bar", "2.0.0", source=PluginInstallationSource.Github),
]
manifests = [
_make_manifest("acme/foo", "1.0.1"),
_make_manifest("acme/bar", "2.0.1"),
]
upgrade_mock, calls = _run_task(
plugins=plugins,
manifests=manifests,
upgrade_mode=TenantPluginAutoUpgradeMode.ALL,
)
assert upgrade_mock.call_count == 1
assert calls[0][1] == plugins[0].plugin_unique_identifier
def test_mode_partial_only_upgrades_included_plugins(self) -> None:
plugins = [
_make_plugin("acme/foo", "1.0.0"),
_make_plugin("acme/bar", "2.0.0"),
]
manifests = [
_make_manifest("acme/foo", "1.0.1"),
_make_manifest("acme/bar", "2.0.1"),
]
upgrade_mock, calls = _run_task(
plugins=plugins,
manifests=manifests,
upgrade_mode=TenantPluginAutoUpgradeMode.PARTIAL,
include_plugins=["acme/foo"],
)
assert upgrade_mock.call_count == 1
assert calls[0][1] == plugins[0].plugin_unique_identifier
def test_mode_exclude_skips_excluded_plugins(self) -> None:
plugins = [
_make_plugin("acme/foo", "1.0.0"),
_make_plugin("acme/bar", "2.0.0"),
]
manifests = [
_make_manifest("acme/foo", "1.0.1"),
_make_manifest("acme/bar", "2.0.1"),
]
upgrade_mock, calls = _run_task(
plugins=plugins,
manifests=manifests,
upgrade_mode=TenantPluginAutoUpgradeMode.EXCLUDE,
exclude_plugins=["acme/bar"],
)
assert upgrade_mock.call_count == 1
assert calls[0][1] == plugins[0].plugin_unique_identifier
def test_category_strategy_only_upgrades_matching_category(self) -> None:
plugins = [
_make_plugin("acme/model-provider", "1.0.0", category=PluginCategory.Model),
_make_plugin("acme/tool-provider", "1.0.0", category=PluginCategory.Tool),
]
manifests = [
_make_manifest("acme/model-provider", "1.0.1"),
_make_manifest("acme/tool-provider", "1.0.1"),
]
upgrade_mock, calls = _run_task(
plugins=plugins,
manifests=manifests,
upgrade_mode=TenantPluginAutoUpgradeMode.ALL,
category=TenantPluginAutoUpgradeCategory.MODEL,
)
upgrade_mock.assert_called_once()
assert calls[0][1] == plugins[0].plugin_unique_identifier
class TestErrorIsolation:
def test_one_plugin_failure_does_not_block_others(self) -> None:
plugins = [
_make_plugin("acme/foo", "1.0.0"),
_make_plugin("acme/bar", "2.0.0"),
]
manifests = [
_make_manifest("acme/foo", "1.0.1"),
_make_manifest("acme/bar", "2.0.1"),
]
fake_manager = MagicMock()
fake_manager.list_plugins.return_value = plugins
seen: list[str] = []
def _upgrade(tenant_id: str, original: str, new: str) -> None:
seen.append(original)
if "foo" in original:
raise RuntimeError("boom")
with (
patch(f"{MODULE}.PluginInstaller", return_value=fake_manager),
patch(f"{MODULE}.marketplace_batch_fetch_plugin_manifests", return_value=manifests),
patch(f"{MODULE}.PluginService.upgrade_plugin_with_marketplace", side_effect=_upgrade),
):
from tasks.process_tenant_plugin_autoupgrade_check_task import (
process_tenant_plugin_autoupgrade_check_task,
)
process_tenant_plugin_autoupgrade_check_task(
"tenant-1",
TenantPluginAutoUpgradeStrategySetting.LATEST,
0,
TenantPluginAutoUpgradeMode.ALL,
[],
[],
)
assert any("foo" in s for s in seen)
assert any("bar" in s for s in seen)
def _make_declaration(plugin_id: str, latest_version: str) -> SimpleNamespace:
"""Minimal stand-in for a MarketplacePluginDeclaration returned by the batch API."""
org, name = plugin_id.split("/", 1)
return SimpleNamespace(
plugin_id=plugin_id,
org=org,
name=name,
latest_version=latest_version,
latest_package_identifier=f"{plugin_id}:{latest_version}@cafe1234",
)
class TestManifestCacheFallback:
"""
The global snapshot is a best-effort optimisation. When it does not cover a plugin,
the check must fall back to the batch API instead of silently skipping the upgrade.
"""
def _fetch(
self,
plugin_ids: list[str],
*,
cached: dict[str, str],
declarations: list[SimpleNamespace] | None = None,
batch_error: Exception | None = None,
) -> tuple[list[MarketplacePluginSnapshot], MagicMock, MagicMock]:
from tasks.process_tenant_plugin_autoupgrade_check_task import (
marketplace_batch_fetch_plugin_manifests,
)
redis_mock = MagicMock()
redis_mock.get.side_effect = lambda key: cached.get(key)
batch_mock = MagicMock()
if batch_error is not None:
batch_mock.side_effect = batch_error
else:
batch_mock.return_value = declarations or []
with (
patch(f"{MODULE}.redis_client", redis_mock),
patch(f"{MODULE}.batch_fetch_plugin_manifests", batch_mock),
):
result = marketplace_batch_fetch_plugin_manifests(plugin_ids)
return result, batch_mock, redis_mock
def test_cache_hits_do_not_touch_the_marketplace(self) -> None:
"""The whole point of the snapshot: a warm cache must not generate any request."""
key = "plugin_autoupgrade_check_task:cached_plugin_snapshot:langgenius/foo"
cached = {key: _make_manifest("langgenius/foo", "1.1.0").model_dump_json()}
result, batch_mock, _ = self._fetch(["langgenius/foo"], cached=cached)
assert [r.latest_version for r in result] == ["1.1.0"]
batch_mock.assert_not_called()
def test_empty_snapshot_falls_back_to_batch_api(self) -> None:
"""Regression: an empty global snapshot used to skip every plugin silently."""
result, batch_mock, _ = self._fetch(
["langgenius/foo"],
cached={},
declarations=[_make_declaration("langgenius/foo", "1.2.0")],
)
batch_mock.assert_called_once_with(["langgenius/foo"])
assert [r.plugin_id for r in result] == ["langgenius/foo"]
assert [r.latest_version for r in result] == ["1.2.0"]
def test_fallback_results_are_cached(self) -> None:
_, _, redis_mock = self._fetch(
["langgenius/foo"],
cached={},
declarations=[_make_declaration("langgenius/foo", "1.2.0")],
)
cached_keys = [call.args[0] for call in redis_mock.setex.call_args_list]
assert "plugin_autoupgrade_check_task:cached_plugin_snapshot:langgenius/foo" in cached_keys
def test_plugins_unknown_to_the_marketplace_are_negatively_cached(self) -> None:
"""Avoid re-asking about a plugin the marketplace does not serve on every cycle."""
_, _, redis_mock = self._fetch(["langgenius/ghost"], cached={}, declarations=[])
redis_mock.setex.assert_called_once()
key, _ttl, value = redis_mock.setex.call_args.args
assert key == "plugin_autoupgrade_check_task:cached_plugin_snapshot:langgenius/ghost"
assert value == "null"
def test_negatively_cached_plugins_are_not_refetched(self) -> None:
key = "plugin_autoupgrade_check_task:cached_plugin_snapshot:langgenius/ghost"
result, batch_mock, _ = self._fetch(["langgenius/ghost"], cached={key: "null"})
assert result == []
batch_mock.assert_not_called()
def test_marketplace_failure_keeps_cached_results(self) -> None:
"""A failing batch call must not lose the plugins the snapshot did cover."""
key = "plugin_autoupgrade_check_task:cached_plugin_snapshot:langgenius/foo"
cached = {key: _make_manifest("langgenius/foo", "1.1.0").model_dump_json()}
result, _, _ = self._fetch(
["langgenius/foo", "langgenius/bar"],
cached=cached,
batch_error=RuntimeError("marketplace down"),
)
assert [r.plugin_id for r in result] == ["langgenius/foo"]
@@ -0,0 +1,49 @@
from collections.abc import Callable
from unittest.mock import patch
import pytest
from enums import DeploymentEdition
from tasks.refresh_billing_vector_space_task import (
refresh_billing_vector_space_task,
schedule_billing_vector_space_refresh,
)
@pytest.fixture(autouse=True)
def _cloud_edition(config_overrides: Callable[..., None]) -> None:
config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD)
def test_refresh_invalidates_vector_space_cache() -> None:
with (
patch(
"tasks.refresh_billing_vector_space_task.BillingService.invalidate_vector_space_cache"
) as invalidate_cache,
):
refresh_billing_vector_space_task.run("tenant-1")
invalidate_cache.assert_called_once_with("tenant-1")
def test_refresh_failure_schedules_retry() -> None:
error = RuntimeError("billing unavailable")
with (
patch(
"tasks.refresh_billing_vector_space_task.BillingService.invalidate_vector_space_cache",
side_effect=error,
),
patch.object(refresh_billing_vector_space_task, "retry", side_effect=RuntimeError("retry scheduled")) as retry,
pytest.raises(RuntimeError, match="retry scheduled"),
):
refresh_billing_vector_space_task.run("tenant-1")
retry.assert_called_once_with(exc=error, countdown=30)
def test_dispatch_failure_does_not_propagate() -> None:
with (
patch.object(refresh_billing_vector_space_task, "delay", side_effect=RuntimeError("broker unavailable")),
):
schedule_billing_vector_space_refresh("tenant-1")
@@ -0,0 +1,627 @@
import logging
from collections.abc import Generator, Iterator, Sequence
from contextlib import AbstractContextManager, contextmanager
from datetime import UTC, datetime
from unittest.mock import MagicMock, call, patch
from uuid import uuid4
import pytest
from pytest_mock import MockerFixture
from sqlalchemy.orm import Session, sessionmaker
import tasks.remove_app_and_related_data_task as remove_app_task_module
from enums import DeploymentEdition
from extensions.ext_application_services import ApplicationServices
from graphon.enums import WorkflowExecutionStatus
from graphon.variables.segments import StringSegment
from libs.archive_storage import ArchiveStorageNotConfiguredError
from models import AppStar
from models.agent import WorkflowAgentBindingType, WorkflowAgentNodeBinding
from models.enums import CreatorUserRole, WorkflowRunTriggeredFrom
from models.workflow import WorkflowArchiveLog, WorkflowDraftVariable
from services.network_access_group_service import NetworkAccessGroupUpstreamError
from tasks.remove_app_and_related_data_task import (
_delete_app_stars,
_delete_app_workflow_archive_logs,
_delete_archived_workflow_run_files,
_delete_draft_variable_offload_data,
_delete_draft_variables,
_delete_workflow_agent_node_bindings,
delete_draft_variables_batch,
)
from tests.helpers.unit.config_override import apply_config_overrides
def test_delete_workflow_agent_node_bindings_is_scoped_to_tenant_and_app(sqlite_session: Session) -> None:
target = WorkflowAgentNodeBinding(
tenant_id="tenant-1",
app_id="app-1",
workflow_id="workflow-1",
workflow_version="draft",
node_id="node-1",
binding_type=WorkflowAgentBindingType.INLINE_AGENT,
agent_id="agent-1",
current_snapshot_id="snapshot-1",
node_job_config={},
)
kept = WorkflowAgentNodeBinding(
tenant_id="tenant-1",
app_id="app-2",
workflow_id="workflow-2",
workflow_version="draft",
node_id="node-2",
binding_type=WorkflowAgentBindingType.INLINE_AGENT,
agent_id="agent-2",
current_snapshot_id="snapshot-2",
node_job_config={},
)
other_tenant = WorkflowAgentNodeBinding(
tenant_id="tenant-2",
app_id="app-1",
workflow_id="workflow-3",
workflow_version="draft",
node_id="node-3",
binding_type=WorkflowAgentBindingType.INLINE_AGENT,
agent_id="agent-3",
current_snapshot_id="snapshot-3",
node_job_config={},
)
sqlite_session.add_all([target, kept, other_tenant])
sqlite_session.commit()
target_id = target.id
kept_id = kept.id
other_tenant_id = other_tenant.id
_delete_workflow_agent_node_bindings("tenant-1", "app-1")
sqlite_session.expire_all()
assert sqlite_session.get(WorkflowAgentNodeBinding, target_id) is None
assert sqlite_session.get(WorkflowAgentNodeBinding, kept_id) is not None
assert sqlite_session.get(WorkflowAgentNodeBinding, other_tenant_id) is not None
def test_app_cleanup_removes_agent_bindings_before_workflows(monkeypatch: pytest.MonkeyPatch) -> None:
events: list[str] = []
apply_config_overrides(monkeypatch, DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY)
other_cleanup_names = (
"cleanup_app_storage",
"_delete_app_model_configs",
"_delete_app_site",
"_delete_app_mcp_servers",
"_delete_app_api_tokens",
"_delete_installed_apps",
"_delete_app_stars",
"_delete_recommended_apps",
"_delete_app_annotation_data",
"_delete_app_dataset_joins",
"_delete_app_workflow_runs",
"_delete_app_workflow_node_executions",
"_delete_app_workflow_app_logs",
"_delete_app_conversations",
"_delete_app_messages",
"_delete_workflow_tool_providers",
"_delete_app_tag_bindings",
"_delete_end_users",
"_delete_trace_app_configs",
"_delete_conversation_variables",
"_delete_draft_variables",
"_delete_app_triggers",
"_delete_workflow_plugin_triggers",
"_delete_workflow_webhook_triggers",
"_delete_workflow_schedule_plans",
"_delete_workflow_trigger_logs",
)
for name in other_cleanup_names:
monkeypatch.setattr(remove_app_task_module, name, MagicMock())
delete_bindings = MagicMock(side_effect=lambda *_args: events.append("bindings"))
delete_workflows = MagicMock(side_effect=lambda *_args: events.append("workflows"))
monkeypatch.setattr(remove_app_task_module, "_delete_workflow_agent_node_bindings", delete_bindings)
monkeypatch.setattr(remove_app_task_module, "_delete_app_workflows", delete_workflows)
monkeypatch.setattr(
remove_app_task_module, "_delete_app_workflow_node_executions", lambda *_args: events.append("executions")
)
monkeypatch.setattr(remove_app_task_module, "_delete_app_workflow_runs", lambda *_args: events.append("runs"))
remove_app_task_module.remove_app_and_related_data_task.run(tenant_id="tenant-1", app_id="app-1")
assert events == ["bindings", "workflows", "executions", "runs"]
delete_bindings.assert_called_once_with("tenant-1", "app-1")
delete_workflows.assert_called_once_with("tenant-1", "app-1")
def test_cloud_app_cleanup_schedules_independent_binding_cleanup(
monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture
) -> None:
apply_config_overrides(monkeypatch, DEPLOYMENT_EDITION=DeploymentEdition.CLOUD)
apply_async = mocker.patch.object(
remove_app_task_module.cleanup_app_network_access_group_binding_task,
"apply_async",
)
remove_app_task_module._schedule_app_network_access_group_binding_cleanup("tenant-1", "app-1")
apply_async.assert_called_once_with(
kwargs={"tenant_id": "tenant-1", "app_id": "app-1"},
countdown=10,
)
def test_community_app_cleanup_does_not_schedule_binding_cleanup(
monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture
) -> None:
apply_config_overrides(monkeypatch, DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY)
apply_async = mocker.patch.object(
remove_app_task_module.cleanup_app_network_access_group_binding_task,
"apply_async",
)
remove_app_task_module._schedule_app_network_access_group_binding_cleanup("tenant-1", "app-1")
apply_async.assert_not_called()
def test_independent_binding_cleanup_task_succeeds(
monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture, account_application_services: ApplicationServices
) -> None:
cleanup_binding = mocker.patch.object(
account_application_services.network_access_groups, "cleanup_app_binding", return_value=True
)
apply_config_overrides(monkeypatch, DEPLOYMENT_EDITION=DeploymentEdition.CLOUD)
with patch(
"extensions.ext_application_services.application_services",
return_value=account_application_services,
):
remove_app_task_module.cleanup_app_network_access_group_binding_task.run(
tenant_id="tenant-1",
app_id="app-1",
)
cleanup_binding.assert_called_once_with(workspace_id="tenant-1", app_id="app-1")
assert remove_app_task_module.cleanup_app_network_access_group_binding_task.max_retries == 24
def test_independent_binding_cleanup_task_retries_with_exponential_backoff(
monkeypatch: pytest.MonkeyPatch,
mocker: MockerFixture,
account_application_services: ApplicationServices,
) -> None:
upstream_error = NetworkAccessGroupUpstreamError(unavailable=False)
retry_error = RuntimeError("retry scheduled")
mocker.patch.object(
account_application_services.network_access_groups, "cleanup_app_binding", side_effect=upstream_error
)
apply_config_overrides(monkeypatch, DEPLOYMENT_EDITION=DeploymentEdition.CLOUD)
retry = mocker.patch.object(
remove_app_task_module.cleanup_app_network_access_group_binding_task, "retry", side_effect=retry_error
)
with (
patch(
"extensions.ext_application_services.application_services",
return_value=account_application_services,
),
pytest.raises(RuntimeError, match="retry scheduled"),
):
remove_app_task_module.cleanup_app_network_access_group_binding_task.run(
tenant_id="tenant-1",
app_id="app-1",
)
retry.assert_called_once_with(exc=upstream_error, countdown=15)
assert remove_app_task_module._network_access_binding_cleanup_retry_delay(5) == 480
assert remove_app_task_module._network_access_binding_cleanup_retry_delay(6) == 600
assert remove_app_task_module._network_access_binding_cleanup_retry_delay(24) == 600
def test_parent_local_cleanup_failure_happens_after_binding_cleanup_is_scheduled(
monkeypatch: pytest.MonkeyPatch,
mocker: MockerFixture,
) -> None:
local_error = RuntimeError("local cleanup failed")
retry_error = RuntimeError("parent retry scheduled")
apply_config_overrides(monkeypatch, DEPLOYMENT_EDITION=DeploymentEdition.CLOUD)
apply_async = mocker.patch.object(
remove_app_task_module.cleanup_app_network_access_group_binding_task,
"apply_async",
)
mocker.patch.object(remove_app_task_module, "_delete_app_model_configs", side_effect=local_error)
retry = mocker.patch.object(
remove_app_task_module.remove_app_and_related_data_task, "retry", side_effect=retry_error
)
with pytest.raises(RuntimeError, match="parent retry scheduled"):
remove_app_task_module.remove_app_and_related_data_task.run(tenant_id="tenant-1", app_id="app-1")
apply_async.assert_called_once_with(
kwargs={"tenant_id": "tenant-1", "app_id": "app-1"},
countdown=10,
)
retry.assert_called_once_with(exc=local_error, countdown=60)
def test_parent_retries_before_local_cleanup_when_binding_schedule_fails(
monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture
) -> None:
broker_error = RuntimeError("broker unavailable")
retry_error = RuntimeError("parent retry scheduled")
apply_config_overrides(monkeypatch, DEPLOYMENT_EDITION=DeploymentEdition.CLOUD)
mocker.patch.object(
remove_app_task_module.cleanup_app_network_access_group_binding_task,
"apply_async",
side_effect=broker_error,
)
first_local_cleanup = mocker.patch.object(remove_app_task_module, "_delete_app_model_configs")
retry = mocker.patch.object(
remove_app_task_module.remove_app_and_related_data_task, "retry", side_effect=retry_error
)
with pytest.raises(RuntimeError, match="parent retry scheduled"):
remove_app_task_module.remove_app_and_related_data_task.run(tenant_id="tenant-1", app_id="app-1")
retry.assert_called_once_with(exc=broker_error, countdown=60)
first_local_cleanup.assert_not_called()
class TestDeleteDraftVariablesBatch:
def test_delete_draft_variables_batch_invalid_batch_size(self) -> None:
"""Test that invalid batch size raises ValueError."""
app_id = "test-app-id"
with pytest.raises(ValueError, match="batch_size must be positive"):
delete_draft_variables_batch(app_id, -1)
with pytest.raises(ValueError, match="batch_size must be positive"):
delete_draft_variables_batch(app_id, 0)
@patch("tasks.remove_app_and_related_data_task.delete_draft_variables_batch")
def test_delete_draft_variables_calls_batch_function(self, mock_batch_delete: MagicMock) -> None:
"""Test that _delete_draft_variables calls the batch function correctly."""
app_id = "test-app-id"
expected_return = 42
mock_batch_delete.return_value = expected_return
result = _delete_draft_variables(app_id)
assert result == expected_return
mock_batch_delete.assert_called_once_with(app_id, batch_size=1000)
class _FakeResult:
"""Minimal stand-in for a SQLAlchemy ``Result``."""
def __init__(self, rows: Sequence[Sequence[object]] = (), rowcount: int = 0) -> None:
self._rows = [tuple(row) for row in rows]
self.rowcount = rowcount
def __iter__(self) -> Iterator[Sequence[object]]:
return iter(self._rows)
class _FakeSessionFactory:
"""Session factory stub that counts open sessions and hands out queued results.
Lets the tests assert that object storage cleanup runs with no session open.
"""
def __init__(self, results: list[_FakeResult] | None = None) -> None:
self._results = list(results or [])
self.open_count = 0
self.open_count_at_storage_delete: list[int] = []
def create_session(self) -> AbstractContextManager[MagicMock]:
factory = self
class _TransactionContext:
def __init__(self, session: MagicMock) -> None:
self._session = session
def __enter__(self) -> MagicMock:
factory.open_count += 1
return self._session
def __exit__(self, *_exc_info: object) -> bool:
factory.open_count -= 1
return False
class _SessionContext:
def __enter__(self) -> MagicMock:
factory.open_count += 1
session = MagicMock()
session.execute.side_effect = factory.next_result
session.begin.return_value = _TransactionContext(session)
return session
def __exit__(self, *_exc_info: object) -> bool:
factory.open_count -= 1
return False
return _SessionContext()
def next_result(self, *_args: object, **_kwargs: object) -> _FakeResult:
if self._results:
return self._results.pop(0)
return _FakeResult([])
class TestDeleteDraftVariableOffloadData:
"""Test the Offload data cleanup functionality."""
def test_delete_draft_variable_offload_data_empty_file_ids(self) -> None:
"""An empty file_ids list must not open a session or touch object storage."""
with (
patch("tasks.remove_app_and_related_data_task.session_factory") as mock_session_factory,
patch("extensions.ext_storage.storage") as mock_storage,
):
result = _delete_draft_variable_offload_data([])
assert result == 0
mock_session_factory.create_session.assert_not_called()
mock_storage.delete.assert_not_called()
def test_delete_draft_variable_offload_data_database_failure(self, caplog: pytest.LogCaptureFixture) -> None:
"""Test handling of database operation failures."""
mock_session_factory = MagicMock()
mock_session_factory.create_session.return_value.__enter__.return_value.execute.side_effect = Exception(
"Database error"
)
with patch("tasks.remove_app_and_related_data_task.session_factory", mock_session_factory):
with caplog.at_level(logging.ERROR):
result = _delete_draft_variable_offload_data(["file-1"])
assert result == 0
assert "Error deleting draft variable offload data:" in caplog.text
def test_storage_delete_runs_with_no_open_transaction(self, monkeypatch: pytest.MonkeyPatch) -> None:
"""Object storage cleanup must not run while a database transaction is open."""
factory = _FakeSessionFactory([_FakeResult([("vf-1", "key-1", "uf-1"), ("vf-2", "key-2", "uf-2")])])
mock_storage = MagicMock()
mock_storage.delete.side_effect = lambda _key: factory.open_count_at_storage_delete.append(factory.open_count)
monkeypatch.setattr(remove_app_task_module, "session_factory", factory)
monkeypatch.setattr("extensions.ext_storage.storage", mock_storage)
result = _delete_draft_variable_offload_data(["vf-1", "vf-2"])
assert result == 2
assert mock_storage.delete.call_count == 2
assert factory.open_count_at_storage_delete == [0, 0]
assert factory.open_count == 0
def test_storage_failure_is_logged_and_not_counted(
self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
"""A failing object is logged and not counted, and the DB cleanup still runs."""
factory = _FakeSessionFactory([_FakeResult([("vf-1", "key-1", "uf-1"), ("vf-2", "key-2", "uf-2")])])
mock_storage = MagicMock()
mock_storage.delete.side_effect = [Exception("Storage error"), None]
monkeypatch.setattr(remove_app_task_module, "session_factory", factory)
monkeypatch.setattr("extensions.ext_storage.storage", mock_storage)
with caplog.at_level(logging.ERROR):
result = _delete_draft_variable_offload_data(["vf-1", "vf-2"])
assert result == 1
assert "Failed to delete storage object key-1" in caplog.text
assert factory.open_count == 0
def test_file_ids_with_no_matching_rows_delete_nothing(self, monkeypatch: pytest.MonkeyPatch) -> None:
"""file_ids that no longer resolve to rows must not raise or touch storage."""
factory = _FakeSessionFactory([_FakeResult([])])
mock_storage = MagicMock()
monkeypatch.setattr(remove_app_task_module, "session_factory", factory)
monkeypatch.setattr("extensions.ext_storage.storage", mock_storage)
result = _delete_draft_variable_offload_data(["vf-gone"])
assert result == 0
mock_storage.delete.assert_not_called()
assert factory.open_count == 0
class TestDeleteDraftVariablesBatchTransactionScope:
"""The batch loop must not hold a transaction while object storage is cleaned up."""
def test_offload_cleanup_is_called_with_no_session_open(
self, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, sqlite_session_factory: sessionmaker[Session]
) -> None:
variable = WorkflowDraftVariable.new_node_variable(
app_id="app-1",
user_id="user-1",
node_id="node-1",
name="value",
value=StringSegment(value="truncated"),
node_execution_id="exec-1",
file_id="vf-1",
)
other = WorkflowDraftVariable.new_node_variable(
app_id="app-2",
user_id="user-1",
node_id="node-1",
name="value",
value=StringSegment(value="untouched"),
node_execution_id="exec-2",
)
sqlite_session.add_all([variable, other])
sqlite_session.commit()
variable_id, other_id = variable.id, other.id
open_sessions: list[Session] = []
@contextmanager
def tracked_session() -> Generator[Session]:
with sqlite_session_factory() as session:
open_sessions.append(session)
try:
yield session
finally:
open_sessions.remove(session)
cleaned: list[list[str]] = []
def _record_cleanup(file_ids: list[str]) -> int:
assert not open_sessions
cleaned.append(file_ids)
return len(file_ids)
monkeypatch.setattr(remove_app_task_module.session_factory, "create_session", tracked_session)
monkeypatch.setattr(remove_app_task_module, "_delete_draft_variable_offload_data", _record_cleanup)
total_deleted = delete_draft_variables_batch("app-1", batch_size=10)
assert total_deleted == 1
assert cleaned == [["vf-1"]]
assert not open_sessions
sqlite_session.expire_all()
assert sqlite_session.get(WorkflowDraftVariable, variable_id) is None
assert sqlite_session.get(WorkflowDraftVariable, other_id) is not None
def test_rows_without_offloaded_files_skip_cleanup(
self, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session
) -> None:
"""A batch whose rows carry no file_id must not invoke the offload cleanup."""
variable = WorkflowDraftVariable.new_node_variable(
app_id="app-1",
user_id="user-1",
node_id="node-1",
name="value",
value=StringSegment(value="plain"),
node_execution_id="exec-1",
)
sqlite_session.add(variable)
sqlite_session.commit()
variable_id = variable.id
cleanup = MagicMock(return_value=0)
monkeypatch.setattr(remove_app_task_module, "_delete_draft_variable_offload_data", cleanup)
total_deleted = delete_draft_variables_batch("app-1", batch_size=10)
assert total_deleted == 1
cleanup.assert_not_called()
sqlite_session.expire_all()
assert sqlite_session.get(WorkflowDraftVariable, variable_id) is None
class TestDeleteWorkflowArchiveLogs:
@pytest.mark.parametrize("sqlite_session", [(WorkflowArchiveLog,)], indirect=True)
@patch("tasks.remove_app_and_related_data_task._delete_records")
@patch("tasks.remove_app_and_related_data_task.db")
def test_delete_app_workflow_archive_logs_calls_delete_records(
self, mock_db: MagicMock, mock_delete_records: MagicMock, sqlite_session: Session
) -> None:
tenant_id = "tenant-1"
app_id = "app-1"
_delete_app_workflow_archive_logs(tenant_id, app_id)
mock_delete_records.assert_called_once()
query_sql, params, delete_func, name = mock_delete_records.call_args[0]
assert "workflow_archive_logs" in query_sql
assert params == {"tenant_id": tenant_id, "app_id": app_id}
assert name == "workflow archive log"
archive_log = WorkflowArchiveLog(
tenant_id=str(uuid4()),
app_id=str(uuid4()),
workflow_id=str(uuid4()),
workflow_run_id=str(uuid4()),
created_by_role=CreatorUserRole.ACCOUNT,
created_by=str(uuid4()),
log_id=None,
log_created_at=None,
log_created_from=None,
run_version="1",
run_status=WorkflowExecutionStatus.SUCCEEDED,
run_triggered_from=WorkflowRunTriggeredFrom.APP_RUN,
run_error=None,
run_elapsed_time=0,
run_total_tokens=0,
run_total_steps=1,
run_created_at=datetime.now(UTC),
run_finished_at=datetime.now(UTC),
run_exceptions_count=0,
trigger_metadata=None,
)
sqlite_session.add(archive_log)
sqlite_session.commit()
delete_func(sqlite_session, archive_log.id)
sqlite_session.commit()
sqlite_session.expunge_all()
assert sqlite_session.get(WorkflowArchiveLog, archive_log.id) is None
class TestDeleteAppStars:
@pytest.mark.parametrize("sqlite_session", [(AppStar,)], indirect=True)
@patch("tasks.remove_app_and_related_data_task._delete_records")
def test_delete_app_stars_calls_delete_records(
self, mock_delete_records: MagicMock, sqlite_session: Session
) -> None:
tenant_id = "tenant-1"
app_id = "app-1"
_delete_app_stars(tenant_id, app_id)
mock_delete_records.assert_called_once()
query_sql, params, delete_func, name = mock_delete_records.call_args[0]
assert "app_stars" in query_sql
assert params == {"tenant_id": tenant_id, "app_id": app_id}
assert name == "app star"
app_star = AppStar(tenant_id=str(uuid4()), app_id=str(uuid4()), account_id=str(uuid4()))
sqlite_session.add(app_star)
sqlite_session.commit()
delete_func(sqlite_session, app_star.id)
sqlite_session.commit()
sqlite_session.expunge_all()
assert sqlite_session.get(AppStar, app_star.id) is None
class TestDeleteArchivedWorkflowRunFiles:
@patch("tasks.remove_app_and_related_data_task.get_archive_storage")
def test_delete_archived_workflow_run_files_not_configured(
self, mock_get_storage: MagicMock, caplog: pytest.LogCaptureFixture
) -> None:
mock_get_storage.side_effect = ArchiveStorageNotConfiguredError("missing config")
with caplog.at_level(logging.INFO, logger="tasks.remove_app_and_related_data_task"):
_delete_archived_workflow_run_files("tenant-1", "app-1")
assert caplog.text.count("Archive storage not configured") == 1
@patch("tasks.remove_app_and_related_data_task.get_archive_storage")
def test_delete_archived_workflow_run_files_list_failure(
self, mock_get_storage: MagicMock, caplog: pytest.LogCaptureFixture
) -> None:
storage = MagicMock()
storage.list_objects.side_effect = Exception("list failed")
mock_get_storage.return_value = storage
with caplog.at_level(logging.ERROR, logger="tasks.remove_app_and_related_data_task"):
_delete_archived_workflow_run_files("tenant-1", "app-1")
storage.list_objects.assert_called_once_with("tenant-1/app_id=app-1/")
storage.delete_object.assert_not_called()
assert "Failed to list archive files for app app-1" in caplog.text
@patch("tasks.remove_app_and_related_data_task.get_archive_storage")
def test_delete_archived_workflow_run_files_success(
self, mock_get_storage: MagicMock, caplog: pytest.LogCaptureFixture
) -> None:
storage = MagicMock()
storage.list_objects.return_value = ["key-1", "key-2"]
mock_get_storage.return_value = storage
with caplog.at_level(logging.INFO, logger="tasks.remove_app_and_related_data_task"):
_delete_archived_workflow_run_files("tenant-1", "app-1")
storage.list_objects.assert_called_once_with("tenant-1/app_id=app-1/")
storage.delete_object.assert_has_calls([call("key-1"), call("key-2")], any_order=False)
assert "Deleted 2 archive objects for app app-1" in caplog.text
@@ -0,0 +1,80 @@
from unittest.mock import ANY, MagicMock, patch
from uuid import uuid4
import pytest
from sqlalchemy.orm import Session
from core.app.apps.pipeline.pipeline_generator import PipelineGenerator
from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType
from enums import DeploymentEdition
from models import Account, Tenant, TenantAccountJoin
from models.account import TenantAccountRole
from models.dataset import Dataset, Document
from models.enums import DatasetRuntimeMode, DataSourceType, DocumentCreatedFrom, IndexingStatus
from tasks.retry_document_indexing_task import retry_document_indexing_task
from tests.helpers.unit.config_override import config_overrides_context
@config_overrides_context(DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY)
@pytest.mark.parametrize("runtime_mode", [DatasetRuntimeMode.GENERAL, DatasetRuntimeMode.RAG_PIPELINE])
def test_retry_uses_dataset_runtime(sqlite_session: Session, runtime_mode: DatasetRuntimeMode) -> None:
tenant = Tenant(name="Retry tenant")
user = Account(name="Retry user", email=f"retry-{uuid4()}@example.com")
membership = TenantAccountJoin(
tenant_id=tenant.id,
account_id=user.id,
current=True,
role=TenantAccountRole.OWNER,
)
dataset = Dataset(
id=str(uuid4()),
tenant_id=tenant.id,
name="Retry dataset",
created_by=user.id,
data_source_type=DataSourceType.UPLOAD_FILE,
indexing_technique=IndexTechniqueType.ECONOMY,
chunk_structure=IndexStructureType.PARAGRAPH_INDEX,
runtime_mode=runtime_mode,
)
document = Document(
id=str(uuid4()),
tenant_id=tenant.id,
dataset_id=dataset.id,
position=1,
data_source_type=DataSourceType.UPLOAD_FILE,
data_source_info="{}",
batch="retry-batch",
name="Retry document",
created_from=DocumentCreatedFrom.WEB,
created_by=user.id,
indexing_status=IndexingStatus.COMPLETED,
enabled=True,
archived=False,
doc_form=IndexStructureType.PARAGRAPH_INDEX,
)
sqlite_session.add_all([tenant, user, membership, dataset, document])
sqlite_session.commit()
features = MagicMock()
with (
patch("tasks.retry_document_indexing_task.FeatureService.get_features", return_value=features),
patch("tasks.retry_document_indexing_task.IndexProcessorFactory"),
patch("tasks.retry_document_indexing_task.build_document_indexing_service") as indexing_runner,
patch("tasks.retry_document_indexing_task.RagPipelineService") as pipeline_service,
patch("tasks.retry_document_indexing_task.redis_client"),
):
retry_document_indexing_task.run(dataset.id, [document.id], user.id)
if runtime_mode == DatasetRuntimeMode.GENERAL:
pipeline_service.assert_not_called()
indexing_runner.assert_called_once_with(session_factory=ANY, enforce_vector_space_admission=True)
(run_documents,) = indexing_runner.return_value.run.call_args.args
assert [item.document_id for item in run_documents] == [document.id]
assert run_documents[0].dataset.tenant_id == dataset.tenant_id
else:
indexing_runner.assert_not_called()
retry = pipeline_service.return_value.retry_error_document
retry.assert_called_once()
retry_dataset, retry_document, retry_user = retry.call_args.args
assert (retry_dataset.id, retry_document.id, retry_user.id) == (dataset.id, document.id, user.id)
assert isinstance(retry.call_args.kwargs["generator"], PipelineGenerator)
@@ -0,0 +1,98 @@
import uuid
from unittest.mock import MagicMock, patch
from sqlalchemy import select
from sqlalchemy.orm import Session
from core.rag.index_processor.constant.index_type import IndexStructureType
from enums import DeploymentEdition
from models.dataset import Dataset, Document, DocumentSegment
from models.enums import DataSourceType
from tasks.sync_website_document_indexing_task import sync_website_document_indexing_task
from tests.helpers.unit.config_override import config_overrides_context
from tests.helpers.unit.model_factories import make_dataset, make_document
def _dataset(tenant_id: str) -> Dataset:
return make_dataset(
dataset_id=str(uuid.uuid4()),
tenant_id=tenant_id,
name="Website dataset",
data_source_type=DataSourceType.WEBSITE_CRAWL,
created_by=str(uuid.uuid4()),
)
def _document(dataset: Dataset) -> Document:
return make_document(
document_id=str(uuid.uuid4()),
tenant_id=dataset.tenant_id,
dataset_id=dataset.id,
data_source_type=DataSourceType.WEBSITE_CRAWL,
name="Website document",
created_by=str(uuid.uuid4()),
doc_form=IndexStructureType.PARAGRAPH_INDEX,
)
def _segment(*, tenant_id: str, dataset_id: str, document_id: str) -> DocumentSegment:
return DocumentSegment(
tenant_id=tenant_id,
dataset_id=dataset_id,
document_id=document_id,
position=1,
content="content",
word_count=1,
tokens=1,
created_by=str(uuid.uuid4()),
)
def test_rejects_document_outside_dataset_before_side_effects(sqlite_session: Session) -> None:
tenant_id = str(uuid.uuid4())
requested_dataset = _dataset(tenant_id)
foreign_dataset = _dataset(tenant_id)
foreign_document = _document(foreign_dataset)
sqlite_session.add_all([requested_dataset, foreign_dataset, foreign_document])
sqlite_session.commit()
with (
patch("tasks.sync_website_document_indexing_task.FeatureService") as feature_service,
patch("tasks.sync_website_document_indexing_task.IndexProcessorFactory") as processor_factory,
):
sync_website_document_indexing_task(requested_dataset.id, foreign_document.id)
feature_service.get_features.assert_not_called()
processor_factory.assert_not_called()
@config_overrides_context(DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY)
def test_cleanup_is_owner_scoped_and_skips_empty_vector_ids(sqlite_session: Session) -> None:
tenant_id = str(uuid.uuid4())
dataset = _dataset(tenant_id)
document = _document(dataset)
owned_segment = _segment(tenant_id=tenant_id, dataset_id=dataset.id, document_id=document.id)
other_dataset = _dataset(tenant_id)
decoy_segments = [
_segment(tenant_id=tenant_id, dataset_id=dataset.id, document_id=str(uuid.uuid4())),
_segment(tenant_id=tenant_id, dataset_id=other_dataset.id, document_id=document.id),
_segment(tenant_id=str(uuid.uuid4()), dataset_id=dataset.id, document_id=document.id),
]
for index, segment in enumerate(decoy_segments):
segment.index_node_id = f"decoy-node-{index}"
sqlite_session.add_all([dataset, document, owned_segment, other_dataset, *decoy_segments])
sqlite_session.commit()
features = MagicMock()
with (
patch("tasks.sync_website_document_indexing_task.FeatureService.get_features", return_value=features),
patch("tasks.sync_website_document_indexing_task.IndexProcessorFactory") as processor_factory,
patch("tasks.sync_website_document_indexing_task.build_document_indexing_service") as indexing_runner,
patch("tasks.sync_website_document_indexing_task.redis_client"),
):
sync_website_document_indexing_task(dataset.id, document.id)
processor_factory.return_value.init_index_processor.return_value.clean.assert_not_called()
indexing_runner.return_value.run.assert_called_once()
sqlite_session.expire_all()
assert set(sqlite_session.scalars(select(DocumentSegment.id))) == {segment.id for segment in decoy_segments}
@@ -0,0 +1,276 @@
from dataclasses import replace
from datetime import UTC, datetime, timedelta
from typing import override
import pytest
from celery.exceptions import Retry
from dify_trace_sdk.events import MessageCompleted, TraceCorrelation, TraceEvent
from dify_trace_sdk.testing import RecordingEgress, message_trace
from flask import Flask
from sqlalchemy.orm import Session, sessionmaker
from enums import DeploymentEdition
from extensions.ext_application_services import build_application_services
from extensions.ext_redis import redis_client
from services.tracing.adapters import payload_key
from services.tracing.canonical import CanonicalTraceBuilder
from services.tracing.delivery import (
ExportJob,
ExportRetryQueue,
StoredTraceDestination,
TraceCollector,
TraceDeliveryService,
TraceDestinationQuery,
TraceRecordSource,
)
from services.tracing.errors import SourceNotFoundError, SourceNotReadyError
from services.tracing.providers import ProviderConfigCodec
from services.tracing.records import BaseTraceInfo, GenerateNameTraceInfo, WorkflowTraceInfo
from tasks import trace_delivery_task
from tasks.trace_delivery_task import deliver_trace_event, retry_trace_export, sweep_trace_payloads
from tests.helpers.unit.services.tracing_ports import InstalledTraceProviders, WorkspaceBoundCipher
class _Storage:
def __init__(self, files: dict[str, bytes]) -> None:
self.files = dict(files)
def load(self, key: str) -> bytes:
return self.files[key]
def save(self, key: str, data: bytes) -> None:
self.files[key] = data
def delete(self, key: str) -> None:
self.files.pop(key, None)
def scan(self, path: str, files: bool = True) -> list[str]:
keys: list[str] = [key for key in self.files if key.startswith(path)] if files else []
if not keys:
raise FileNotFoundError(path)
return keys
class _Records(TraceRecordSource):
"""Answers every assembly with the scripted record or error after recording the event."""
def __init__(self) -> None:
self.result: BaseTraceInfo | Exception = SourceNotFoundError()
self.events: list[TraceEvent] = []
@override
def assemble(self, event: TraceEvent) -> BaseTraceInfo:
self.events.append(event)
if isinstance(self.result, Exception):
raise self.result
return self.result
class _NoDestinations(TraceDestinationQuery):
def __init__(self) -> None:
self.lookups: list[tuple[str, str]] = []
@override
def active_destination(self, *, tenant_id: str, app_id: str) -> StoredTraceDestination | None:
self.lookups.append((tenant_id, app_id))
return None
class _NoRetries(ExportRetryQueue):
@override
def schedule(self, job: ExportJob, *, delay_seconds: int) -> None:
raise AssertionError("no send is attempted")
class _UnavailableCollector(TraceCollector):
@override
def capture_terminal(self, *, tenant_id: str, info: BaseTraceInfo) -> bool:
raise RuntimeError("ledger unavailable")
@override
def link_managed_parent(self, *, tenant_id: str, info: WorkflowTraceInfo) -> WorkflowTraceInfo:
return info
class _Delivery:
def __init__(self) -> None:
self.records = _Records()
self.destinations = _NoDestinations()
def service(self, collector: TraceCollector | None) -> TraceDeliveryService:
return TraceDeliveryService(
records=self.records,
destinations=self.destinations,
providers=InstalledTraceProviders(),
codec=ProviderConfigCodec(WorkspaceBoundCipher()),
builder=CanonicalTraceBuilder(),
egress=RecordingEgress([], grpc_target=None),
retries=_NoRetries(),
enterprise_sink=None,
collector=collector,
max_export_attempts=3,
)
class _RetryRequests:
def __init__(self) -> None:
self.calls: list[dict[str, object]] = []
def __call__(self, **kwargs: object) -> Retry:
self.calls.append(kwargs)
raise Retry("redelivery scheduled", None)
def _event() -> TraceEvent:
return TraceEvent(
event_id="event-1",
tenant_id="tenant-1",
app_id="app-1",
user_id=None,
occurred_at=datetime.now(UTC),
correlation=TraceCorrelation(
external_trace_id=None, session_id=None, parent_workflow_run_id=None, parent_node_execution_id=None
),
payload=MessageCompleted(message_id="message-1", conversation_id=None),
)
@pytest.fixture
def storage(monkeypatch: pytest.MonkeyPatch) -> _Storage:
fake = _Storage({})
monkeypatch.setattr(trace_delivery_task, "storage", fake)
return fake
def _install(
monkeypatch: pytest.MonkeyPatch,
app: Flask,
sessions: sessionmaker[Session],
delivery: _Delivery,
collector: TraceCollector | None,
) -> None:
services = build_application_services(
database_client=sessions,
deployment_edition=DeploymentEdition.COMMUNITY,
initialization_password="",
redis=redis_client,
)
tracing = replace(services.tracing, delivery=delivery.service(collector))
monkeypatch.setitem(app.extensions, "application_services", replace(services, tracing=tracing))
@pytest.fixture
def delivery(monkeypatch: pytest.MonkeyPatch, app: Flask, sqlite_session_factory: sessionmaker[Session]) -> _Delivery:
delivery = _Delivery()
_install(monkeypatch, app, sqlite_session_factory, delivery, None)
return delivery
@pytest.mark.usefixtures("storage")
def test_inline_events_are_delivered(delivery: _Delivery) -> None:
deliver_trace_event.run({"event": _event().model_dump_json()})
assert [event.event_id for event in delivery.records.events] == ["event-1"]
def test_stored_events_are_loaded_and_their_payload_removed(delivery: _Delivery, storage: _Storage) -> None:
key = payload_key(datetime.now(UTC), "events", "event-1")
storage.save(key, _event().model_dump_json().encode())
deliver_trace_event.run({"payload_key": key})
assert len(delivery.records.events) == 1
assert key not in storage.files
def test_unreadable_events_are_dropped(delivery: _Delivery, storage: _Storage) -> None:
storage.save("trace_payloads/2026-01-01/events/bad.json", b"not json")
deliver_trace_event.run({"payload_key": "trace_payloads/2026-01-01/events/bad.json"})
assert delivery.records.events == []
assert storage.files == {}
@pytest.mark.usefixtures("storage")
def test_missing_sources_are_dropped(delivery: _Delivery) -> None:
deliver_trace_event.run({"event": _event().model_dump_json()})
assert len(delivery.records.events) == 1
def test_sources_not_ready_are_redelivered_and_keep_their_payload(
monkeypatch: pytest.MonkeyPatch, delivery: _Delivery, storage: _Storage
) -> None:
delivery.records.result = SourceNotReadyError()
retries = _RetryRequests()
monkeypatch.setattr(deliver_trace_event, "retry", retries)
key = payload_key(datetime.now(UTC), "events", "event-1")
storage.save(key, _event().model_dump_json().encode())
with pytest.raises(Retry):
deliver_trace_event.run({"payload_key": key})
assert retries.calls == [{"args": ({"payload_key": key},), "countdown": 5}]
assert key in storage.files
@pytest.mark.usefixtures("storage")
def test_collector_outages_redeliver_without_repeating_enterprise_telemetry(
monkeypatch: pytest.MonkeyPatch, app: Flask, sqlite_session_factory: sessionmaker[Session]
) -> None:
delivery = _Delivery()
delivery.records.result = GenerateNameTraceInfo(
message_id=None,
message_data=None,
inputs="hello",
outputs="Greeting",
start_time=datetime(2026, 1, 1),
end_time=datetime(2026, 1, 1),
metadata={"trace_event_id": "event-1"},
conversation_id=None,
tenant_id="tenant-1",
)
_install(monkeypatch, app, sqlite_session_factory, delivery, _UnavailableCollector())
retries = _RetryRequests()
monkeypatch.setattr(deliver_trace_event, "retry", retries)
envelope = {"event": _event().model_dump_json()}
with pytest.raises(Retry):
deliver_trace_event.run(envelope)
assert retries.calls[0]["args"] == ({**envelope, "enterprise_delivered": True},)
def test_export_retries_run_the_stored_job_once(delivery: _Delivery, storage: _Storage) -> None:
job = ExportJob(tenant_id="tenant-1", app_id="app-1", provider_id="langfuse", attempt=2, trace=message_trace())
key = payload_key(datetime.now(UTC), "exports", "job-1")
storage.save(key, job.model_dump_json().encode())
retry_trace_export.run(key)
assert delivery.destinations.lookups == [("tenant-1", "app-1")]
assert key not in storage.files
def test_unreadable_export_jobs_are_dropped(delivery: _Delivery, storage: _Storage) -> None:
storage.save("trace_payloads/2026-01-01/exports/bad.json", b"{}")
retry_trace_export.run("trace_payloads/2026-01-01/exports/bad.json")
assert delivery.destinations.lookups == []
assert storage.files == {}
def test_sweep_preserves_legacy_payloads_while_old_workers_drain(storage: _Storage) -> None:
today = datetime.now(UTC)
fresh = payload_key(today, "events", "fresh")
expired = payload_key(today - timedelta(days=3), "events", "expired")
for key in (fresh, expired, "trace_payloads/not-a-date/x.json", "ops_trace/app-1/legacy.json", "other/keep.json"):
storage.save(key, b"{}")
sweep_trace_payloads.run()
assert sorted(storage.files) == sorted(
[fresh, "trace_payloads/not-a-date/x.json", "ops_trace/app-1/legacy.json", "other/keep.json"]
)
@@ -0,0 +1,267 @@
from collections.abc import Generator, Mapping
from unittest.mock import MagicMock, patch
import pytest
from typing_extensions import TypedDict
import tasks.trigger_processing_tasks as trigger_processing_tasks_module
from core.plugin.entities.plugin_daemon import CredentialType
from models.enums import EndUserType
from models.model import EndUser
from models.trigger import TriggerSubscription, WorkflowPluginTrigger
from models.workflow import Workflow
from services.errors.app import QuotaExceededError
from tasks.trigger_processing_tasks import dispatch_triggered_workflow
from tests.helpers.unit.model_factories import make_end_user, make_workflow
def _workflow(*, app_id: str = "app-123") -> Workflow:
return make_workflow(
workflow_id="workflow-123",
tenant_id="tenant-123",
app_id=app_id,
version="published",
graph={
"nodes": [
{
"id": "node-123",
"data": {"type": trigger_processing_tasks_module.TRIGGER_PLUGIN_NODE_TYPE},
}
],
"edges": [],
},
created_by="user-123",
)
def _end_user() -> EndUser:
return make_end_user(
end_user_id="end-user-123",
tenant_id="tenant-123",
app_id="app-123",
end_user_type=EndUserType.TRIGGER,
session_id="trigger-session",
)
class _EndUserServiceStub:
def __init__(self) -> None:
self.result: Mapping[str, EndUser] = {}
self.calls: list[tuple[EndUserType, str, list[str], str]] = []
def create_end_user_batch(
self,
type: EndUserType,
tenant_id: str,
app_ids: list[str],
user_id: str,
) -> Mapping[str, EndUser]:
self.calls.append((type, tenant_id, app_ids, user_id))
return self.result
class _DispatchMocks(TypedDict, closed=True):
get_workflows: MagicMock
reserve: MagicMock
quota_charge: MagicMock
mark_rate_limited: MagicMock
invoke_trigger_event: MagicMock
invoke_response: MagicMock
end_users: _EndUserServiceStub
trigger_workflow_async: MagicMock
class TestDispatchTriggeredWorkflow:
"""Unit tests covering branch behaviours of ``dispatch_triggered_workflow``.
The covered branches are:
- workflow missing for ``plugin_trigger.app_id`` → log + ``continue``
- ``QuotaService.reserve`` raising ``QuotaExceededError`` →
``mark_tenant_triggers_rate_limited`` + early ``return``
- ``trigger_workflow_async`` succeeds →
``quota_charge.commit()`` + ``dispatched_count`` increments
"""
@pytest.fixture
def subscription(self) -> TriggerSubscription:
subscription = TriggerSubscription(
tenant_id="tenant-123",
user_id="user-123",
name="Test Subscription",
endpoint_id="endpoint-123",
provider_id="langgenius/test_plugin/test_plugin",
parameters={},
properties={},
credentials={},
credential_type=CredentialType.API_KEY,
)
subscription.id = "subscription-123"
return subscription
@pytest.fixture
def plugin_trigger(self) -> WorkflowPluginTrigger:
trigger = WorkflowPluginTrigger(
app_id="app-123",
node_id="node-123",
tenant_id="tenant-123",
provider_id="langgenius/test_plugin/test_plugin",
event_name="test_event",
subscription_id="subscription-123",
)
trigger.id = "plugin-trigger-123"
return trigger
@pytest.fixture
def provider_controller(self) -> MagicMock:
controller = MagicMock()
controller.plugin_unique_identifier = "langgenius/test_plugin:0.0.1"
controller.entity.identity.name = "Test Plugin"
controller.entity.identity.icon = "icon.svg"
controller.entity.identity.icon_dark = "icon_dark.svg"
return controller
@pytest.fixture
def dispatch_mocks(
self, plugin_trigger: WorkflowPluginTrigger, provider_controller: MagicMock
) -> Generator[_DispatchMocks]:
"""Patch all external dependencies reached by ``dispatch_triggered_workflow``.
Defaults are configured so the code flow can reach the final async
trigger block (line ~385); each test overrides specific handles
(``get_workflows``, ``reserve``, ``end_users``, ...) to
drive the path it targets.
"""
invoke_response = MagicMock()
invoke_response.cancelled = False
invoke_response.variables = dict[str, object]()
quota_charge = MagicMock()
end_users = _EndUserServiceStub()
with (
patch.object(
trigger_processing_tasks_module.TriggerHttpRequestCachingService,
"get_request",
return_value=MagicMock(),
),
patch.object(
trigger_processing_tasks_module.TriggerHttpRequestCachingService,
"get_payload",
return_value=MagicMock(),
),
patch.object(
trigger_processing_tasks_module.TriggerSubscriptionOperatorService,
"get_subscriber_triggers",
return_value=[plugin_trigger],
),
patch.object(
trigger_processing_tasks_module.TriggerManager,
"get_trigger_provider",
return_value=provider_controller,
),
patch.object(
trigger_processing_tasks_module.TriggerManager,
"invoke_trigger_event",
return_value=invoke_response,
) as invoke_trigger_event,
patch.object(
trigger_processing_tasks_module.TriggerEventNodeData,
"model_validate",
return_value=MagicMock(),
),
patch.object(
trigger_processing_tasks_module,
"_get_published_workflows_by_app_ids",
) as get_workflows,
patch.object(
trigger_processing_tasks_module.QuotaService,
"reserve",
return_value=quota_charge,
) as reserve,
patch.object(
trigger_processing_tasks_module.AppTriggerService,
"mark_tenant_triggers_rate_limited",
) as mark_rate_limited,
patch.object(
trigger_processing_tasks_module.AsyncWorkflowService,
"trigger_workflow_async",
) as trigger_workflow_async,
):
yield {
"get_workflows": get_workflows,
"reserve": reserve,
"quota_charge": quota_charge,
"mark_rate_limited": mark_rate_limited,
"invoke_trigger_event": invoke_trigger_event,
"invoke_response": invoke_response,
"end_users": end_users,
"trigger_workflow_async": trigger_workflow_async,
}
def test_dispatch_skips_when_workflow_missing(
self, subscription: TriggerSubscription, dispatch_mocks: _DispatchMocks
) -> None:
"""Covers missing workflow → log + ``continue``."""
dispatch_mocks["get_workflows"].return_value = dict[str, object]()
dispatched = dispatch_triggered_workflow(
user_id="user-123",
subscription=subscription,
event_name="test_event",
request_id="request-123",
end_users=dispatch_mocks["end_users"],
)
assert dispatched == 0
dispatch_mocks["reserve"].assert_not_called()
dispatch_mocks["invoke_trigger_event"].assert_not_called()
dispatch_mocks["mark_rate_limited"].assert_not_called()
def test_dispatch_marks_rate_limited_when_quota_exceeded(
self, subscription: TriggerSubscription, plugin_trigger: WorkflowPluginTrigger, dispatch_mocks: _DispatchMocks
) -> None:
"""Covers QuotaExceededError → mark rate-limited + early return."""
workflow = _workflow()
dispatch_mocks["get_workflows"].return_value = {plugin_trigger.app_id: workflow}
dispatch_mocks["reserve"].side_effect = QuotaExceededError(
feature="trigger", tenant_id=subscription.tenant_id, required=1
)
dispatched = dispatch_triggered_workflow(
user_id="user-123",
subscription=subscription,
event_name="test_event",
request_id="request-123",
end_users=dispatch_mocks["end_users"],
)
assert dispatched == 0
dispatch_mocks["reserve"].assert_called_once()
dispatch_mocks["mark_rate_limited"].assert_called_once_with(subscription.tenant_id)
dispatch_mocks["invoke_trigger_event"].assert_not_called()
def test_dispatch_commits_quota_and_counts_when_workflow_triggered(
self, subscription: TriggerSubscription, plugin_trigger: WorkflowPluginTrigger, dispatch_mocks: _DispatchMocks
) -> None:
"""Happy path: end user exists and async trigger succeeds."""
workflow = _workflow()
dispatch_mocks["get_workflows"].return_value = {plugin_trigger.app_id: workflow}
end_user = _end_user()
dispatch_mocks["end_users"].result = {plugin_trigger.app_id: end_user}
dispatched = dispatch_triggered_workflow(
user_id="user-123",
subscription=subscription,
event_name="test_event",
request_id="request-123",
end_users=dispatch_mocks["end_users"],
)
assert dispatched == 1
dispatch_mocks["trigger_workflow_async"].assert_called_once()
_, kwargs = dispatch_mocks["trigger_workflow_async"].call_args
assert kwargs["user"] is end_user
dispatch_mocks["quota_charge"].commit.assert_called_once()
dispatch_mocks["quota_charge"].refund.assert_not_called()
dispatch_mocks["mark_rate_limited"].assert_not_called()
@@ -0,0 +1,96 @@
from datetime import datetime
from unittest.mock import patch
from uuid import uuid4
from sqlalchemy.orm import Session, sessionmaker
from core.repositories.celery_workflow_execution_repository import CeleryWorkflowExecutionRepository
from graphon.enums import WorkflowExecutionStatus, WorkflowType
from models import Account, CreatorUserRole, WorkflowRun
from models.enums import WorkflowRunTriggeredFrom
from repositories.entities.workflow_execution import WorkflowExecution
from tasks.workflow_execution_tasks import save_workflow_execution_task
TABLES = (WorkflowRun,)
def test_legacy_celery_payload_preserves_timestamp_based_duration(
sqlite_session_factory: sessionmaker[Session],
) -> None:
execution_id = str(uuid4())
graph: dict[str, object] = {}
inputs: dict[str, object] = {}
payload = {
"id_": execution_id,
"workflow_id": str(uuid4()),
"workflow_version": "1",
"workflow_type": WorkflowType.WORKFLOW.value,
"graph": graph,
"inputs": inputs,
"status": WorkflowExecutionStatus.SUCCEEDED.value,
"started_at": "2026-01-01T00:00:00",
"finished_at": "2026-01-01T00:00:15",
}
assert (
save_workflow_execution_task.run(
execution_data=payload,
tenant_id=str(uuid4()),
app_id=str(uuid4()),
triggered_from=WorkflowRunTriggeredFrom.APP_RUN.value,
creator_user_id=str(uuid4()),
creator_user_role=CreatorUserRole.ACCOUNT.value,
)
is True
)
with sqlite_session_factory() as session:
stored = session.get(WorkflowRun, execution_id)
assert stored is not None
assert stored.elapsed_time == 15.0
def test_celery_roundtrip_preserves_accumulated_duration_on_create_update_and_redelivery(
sqlite_session_factory: sessionmaker[Session],
) -> None:
account = Account(name="Test", email="test@example.com")
account.id = str(uuid4())
repository = CeleryWorkflowExecutionRepository(
session_factory=sqlite_session_factory,
tenant_id=str(uuid4()),
user=account,
app_id=str(uuid4()),
triggered_from=WorkflowRunTriggeredFrom.APP_RUN,
)
execution = WorkflowExecution(
id_=str(uuid4()),
workflow_id=str(uuid4()),
workflow_version="1",
workflow_type=WorkflowType.WORKFLOW,
graph={},
inputs={},
started_at=datetime(2026, 1, 1),
accumulated_elapsed_time=5.0,
)
for status, duration in [
(WorkflowExecutionStatus.RUNNING, 5.0),
(WorkflowExecutionStatus.PAUSED, 7.0),
(WorkflowExecutionStatus.SUCCEEDED, 9.0),
]:
execution.status = status
execution.accumulated_elapsed_time = duration
if status == WorkflowExecutionStatus.SUCCEEDED:
execution.finished_at = datetime(2026, 1, 2)
with patch("core.repositories.celery_workflow_execution_repository.save_workflow_execution_task") as task:
repository.save(execution)
payload = task.delay.call_args.kwargs
assert payload["execution_data"]["accumulated_elapsed_time"] == duration
assert payload["creator_user_role"] == CreatorUserRole.ACCOUNT
for _ in range(2):
assert save_workflow_execution_task.run(**payload) is True
with sqlite_session_factory() as session:
stored = session.get(WorkflowRun, execution.id_)
assert stored is not None
assert stored.elapsed_time == duration
assert stored.created_at == execution.started_at
@@ -0,0 +1,490 @@
import json
from datetime import UTC, datetime, timedelta
from decimal import Decimal
from typing import Protocol, cast
from unittest.mock import MagicMock
import pytest
from sqlalchemy import func, select
from sqlalchemy.orm import Session, sessionmaker
from typing_extensions import TypedDict
from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepository
from graphon.entities import WorkflowNodeExecution
from graphon.entities.workflow_node_execution import WorkflowNodeExecutionMetadataKey
from graphon.enums import BuiltinNodeTypes, WorkflowExecutionStatus, WorkflowNodeExecutionStatus, WorkflowType
from models import Account
from models.enums import CreatorUserRole, WorkflowRunTriggeredFrom
from models.workflow import WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom, WorkflowRun
from models.workflow import WorkflowType as ModelWorkflowType
from tasks.workflow_node_execution_tasks import (
_create_node_execution_from_domain,
_update_node_execution_from_domain,
save_workflow_node_execution_task,
save_workflow_node_executions_task,
)
def test_celery_update_preserves_workflow_agent_binding_identity() -> None:
stored = WorkflowNodeExecutionModel(
process_data='{"workflow_agent_binding_id": "workflow-binding-1"}',
)
incoming = WorkflowNodeExecution(
id="execution-1",
workflow_id="workflow-1",
node_id="node-1",
node_type="agent",
title="Agent",
index=1,
created_at=datetime.now(UTC),
process_data={"retry": True},
)
_update_node_execution_from_domain(stored, incoming)
assert stored.process_data_dict == {
"retry": True,
"workflow_agent_binding_id": "workflow-binding-1",
}
"""SQLite-backed tests for asynchronous workflow node-execution persistence."""
TENANT_ID = "00000000-0000-0000-0000-000000000010"
APP_ID = "00000000-0000-0000-0000-000000000020"
WORKFLOW_ID = "00000000-0000-0000-0000-000000000030"
WORKFLOW_RUN_ID = "00000000-0000-0000-0000-000000000040"
ACCOUNT_ID = "00000000-0000-0000-0000-000000000050"
EXECUTION_ID = "00000000-0000-0000-0000-000000000060"
class _ExecutionTaskKwargs(TypedDict, closed=True):
tenant_id: str
app_id: str
triggered_from: str
creator_user_id: str
creator_user_role: str
class _TaskWithRequestContext(Protocol):
def push_request(self, *, retries: int) -> None: ...
def pop_request(self) -> None: ...
def _execution(
*,
execution_id: str = EXECUTION_ID,
status: WorkflowNodeExecutionStatus = WorkflowNodeExecutionStatus.RUNNING,
inputs: dict[str, object] | None = None,
process_data: dict[str, object] | None = None,
outputs: dict[str, object] | None = None,
error: str | None = None,
elapsed_time: float = 0,
created_at: datetime | None = None,
finished_at: datetime | None = None,
) -> WorkflowNodeExecution:
return WorkflowNodeExecution(
id=execution_id,
node_execution_id="runtime-node-execution-1",
workflow_id=WORKFLOW_ID,
workflow_execution_id=WORKFLOW_RUN_ID,
index=3,
predecessor_node_id="previous-node",
node_id="llm-node",
node_type=BuiltinNodeTypes.LLM,
title="Generate answer",
inputs=inputs if inputs is not None else {"question": "hello"},
process_data=process_data if process_data is not None else {"attempt": 1},
outputs=outputs,
status=status,
error=error,
elapsed_time=elapsed_time,
metadata={
WorkflowNodeExecutionMetadataKey.TOTAL_TOKENS: 12,
WorkflowNodeExecutionMetadataKey.TOTAL_PRICE: Decimal("0.01"),
},
created_at=created_at or datetime(2026, 8, 12, 1, tzinfo=UTC),
finished_at=finished_at,
)
def _stored_execution(
*,
execution_id: str = EXECUTION_ID,
tenant_id: str = TENANT_ID,
app_id: str = APP_ID,
process_data: dict[str, object] | None = None,
) -> WorkflowNodeExecutionModel:
run = WorkflowNodeExecutionModel(
id=execution_id,
tenant_id=tenant_id,
app_id=app_id,
workflow_id=WORKFLOW_ID,
triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,
workflow_run_id=WORKFLOW_RUN_ID,
index=1,
predecessor_node_id=None,
node_execution_id="old-runtime-id",
node_id="llm-node",
node_type=BuiltinNodeTypes.LLM,
title="Old title",
agent_workspace_binding_id="00000000-0000-0000-0000-000000000070",
inputs=json.dumps({"old": "input"}),
process_data=json.dumps(process_data or {"workflow_agent_binding_id": "workflow-binding-1"}),
outputs=json.dumps({"old": "output"}),
status=WorkflowNodeExecutionStatus.RUNNING,
error=None,
elapsed_time=0,
execution_metadata="{}",
created_by_role=CreatorUserRole.ACCOUNT,
created_by=ACCOUNT_ID,
finished_at=None,
)
run.created_at = datetime(2026, 8, 12, 1)
return run
def test_create_helper_builds_real_mapped_model_with_serialized_runtime_values() -> None:
execution = _execution(outputs={"answer": "world"})
model = _create_node_execution_from_domain(
execution,
tenant_id=TENANT_ID,
app_id=APP_ID,
triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,
creator_user_id=ACCOUNT_ID,
creator_user_role=CreatorUserRole.ACCOUNT,
)
assert isinstance(model, WorkflowNodeExecutionModel)
assert model.id == EXECUTION_ID
assert model.tenant_id == TENANT_ID
assert model.workflow_run_id == WORKFLOW_RUN_ID
assert model.node_type == BuiltinNodeTypes.LLM
assert model.inputs_dict == {"question": "hello"}
assert model.process_data_dict == {"attempt": 1}
assert model.outputs_dict == {"answer": "world"}
assert json.loads(model.execution_metadata or "{}") == {"total_tokens": 12, "total_price": 0.01}
def test_update_helper_preserves_binding_identity_and_immutable_ownership() -> None:
stored = _stored_execution()
incoming = _execution(
status=WorkflowNodeExecutionStatus.SUCCEEDED,
process_data={"attempt": 2},
outputs={"answer": "updated"},
elapsed_time=1.25,
finished_at=datetime(2026, 8, 12, 1, 0, 2, tzinfo=UTC),
)
_update_node_execution_from_domain(stored, incoming)
assert stored.tenant_id == TENANT_ID
assert stored.app_id == APP_ID
assert stored.workflow_id == WORKFLOW_ID
assert stored.created_by == ACCOUNT_ID
assert stored.node_execution_id == "old-runtime-id"
assert stored.process_data_dict == {
"attempt": 2,
"workflow_agent_binding_id": "workflow-binding-1",
}
assert stored.outputs_dict == {"answer": "updated"}
assert stored.status is WorkflowNodeExecutionStatus.SUCCEEDED
assert stored.elapsed_time == 1.25
def test_task_creates_and_commits_node_execution(
sqlite_session_factory: sessionmaker[Session],
) -> None:
execution = _execution(outputs={"answer": "created"})
result = save_workflow_node_execution_task.run(
execution_data=execution.model_dump(),
tenant_id=TENANT_ID,
app_id=APP_ID,
triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN.value,
creator_user_id=ACCOUNT_ID,
creator_user_role=CreatorUserRole.ACCOUNT.value,
)
assert result is True
with sqlite_session_factory() as observer:
persisted = observer.get(WorkflowNodeExecutionModel, EXECUTION_ID)
assert persisted is not None
assert persisted.tenant_id == TENANT_ID
assert persisted.app_id == APP_ID
assert persisted.outputs_dict == {"answer": "created"}
assert persisted.created_by_role is CreatorUserRole.ACCOUNT
assert persisted.created_by == ACCOUNT_ID
def test_task_updates_only_mutable_execution_state(
sqlite_session_factory: sessionmaker[Session],
) -> None:
with sqlite_session_factory.begin() as seed_session:
seed_session.add(_stored_execution())
finished_at = datetime(2026, 8, 12, 1, 0, 3, tzinfo=UTC)
execution = _execution(
status=WorkflowNodeExecutionStatus.FAILED,
inputs={"question": "updated"},
process_data={"attempt": 3},
outputs={"partial": True},
error="provider failed",
elapsed_time=2.5,
created_at=datetime(2026, 8, 12, 2, tzinfo=UTC),
finished_at=finished_at,
)
result = save_workflow_node_execution_task.run(
execution_data=execution.model_dump(),
tenant_id=TENANT_ID,
app_id=APP_ID,
triggered_from=WorkflowNodeExecutionTriggeredFrom.SINGLE_STEP.value,
creator_user_id="different-account",
creator_user_role=CreatorUserRole.END_USER.value,
)
assert result is True
with sqlite_session_factory() as observer:
persisted = observer.get(WorkflowNodeExecutionModel, EXECUTION_ID)
assert persisted is not None
assert persisted.triggered_from is WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN
assert persisted.created_by_role is CreatorUserRole.ACCOUNT
assert persisted.created_by == ACCOUNT_ID
assert persisted.created_at == datetime(2026, 8, 12, 1)
assert persisted.inputs_dict == {"question": "updated"}
assert persisted.process_data_dict == {
"attempt": 3,
"workflow_agent_binding_id": "workflow-binding-1",
}
assert persisted.outputs_dict == {"partial": True}
assert persisted.status is WorkflowNodeExecutionStatus.FAILED
assert persisted.error == "provider failed"
assert persisted.elapsed_time == 2.5
assert persisted.finished_at == finished_at.replace(tzinfo=None)
def test_task_does_not_overwrite_same_id_from_another_tenant(
monkeypatch: pytest.MonkeyPatch,
sqlite_session_factory: sessionmaker[Session],
) -> None:
other_tenant_id = "00000000-0000-0000-0000-000000000099"
with sqlite_session_factory.begin() as seed_session:
seed_session.add(_stored_execution(tenant_id=other_tenant_id))
retry = MagicMock(side_effect=RuntimeError("retry requested"))
monkeypatch.setattr(save_workflow_node_execution_task, "retry", retry)
with pytest.raises(RuntimeError, match="retry requested"):
save_workflow_node_execution_task.run(
execution_data=_execution(outputs={"unsafe": True}).model_dump(),
tenant_id=TENANT_ID,
app_id=APP_ID,
triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN.value,
creator_user_id=ACCOUNT_ID,
creator_user_role=CreatorUserRole.ACCOUNT.value,
)
retry.assert_called_once()
assert retry.call_args.kwargs["countdown"] == 60
with sqlite_session_factory() as observer:
persisted = observer.get(WorkflowNodeExecutionModel, EXECUTION_ID)
assert persisted is not None
assert persisted.tenant_id == other_tenant_id
assert persisted.outputs_dict == {"old": "output"}
def test_task_retries_invalid_payload_without_committing(
monkeypatch: pytest.MonkeyPatch,
sqlite_session_factory: sessionmaker[Session],
) -> None:
retry = MagicMock(side_effect=RuntimeError("retry requested"))
monkeypatch.setattr(save_workflow_node_execution_task, "retry", retry)
invalid_payload = _execution().model_dump()
invalid_payload["node_id"] = None
with pytest.raises(RuntimeError, match="retry requested"):
save_workflow_node_execution_task.run(
execution_data=invalid_payload,
tenant_id=TENANT_ID,
app_id=APP_ID,
triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN.value,
creator_user_id=ACCOUNT_ID,
creator_user_role=CreatorUserRole.ACCOUNT.value,
)
retry.assert_called_once()
with sqlite_session_factory() as observer:
assert observer.get(WorkflowNodeExecutionModel, EXECUTION_ID) is None
def test_task_uses_exponential_retry_delay_for_redelivery(monkeypatch: pytest.MonkeyPatch) -> None:
retry = MagicMock(side_effect=RuntimeError("retry requested"))
monkeypatch.setattr(save_workflow_node_execution_task, "retry", retry)
task = cast(_TaskWithRequestContext, save_workflow_node_execution_task)
task.push_request(retries=2)
try:
with pytest.raises(RuntimeError, match="retry requested"):
save_workflow_node_execution_task.run(
execution_data={"id": EXECUTION_ID},
tenant_id=TENANT_ID,
app_id=APP_ID,
triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN.value,
creator_user_id=ACCOUNT_ID,
creator_user_role=CreatorUserRole.ACCOUNT.value,
)
finally:
task.pop_request()
assert retry.call_args.kwargs["countdown"] == 240
def test_task_repeated_delivery_updates_one_row(
sqlite_session_factory: sessionmaker[Session],
) -> None:
created_at = datetime(2026, 8, 12, 1, tzinfo=UTC)
first = _execution(created_at=created_at, outputs={"delivery": 1})
second = _execution(
created_at=created_at + timedelta(minutes=1),
status=WorkflowNodeExecutionStatus.SUCCEEDED,
outputs={"delivery": 2},
finished_at=created_at + timedelta(seconds=5),
)
task_kwargs: _ExecutionTaskKwargs = {
"tenant_id": TENANT_ID,
"app_id": APP_ID,
"triggered_from": WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN.value,
"creator_user_id": ACCOUNT_ID,
"creator_user_role": CreatorUserRole.ACCOUNT.value,
}
assert save_workflow_node_execution_task.run(execution_data=first.model_dump(), **task_kwargs)
assert save_workflow_node_execution_task.run(execution_data=second.model_dump(), **task_kwargs)
with sqlite_session_factory() as observer:
assert observer.scalar(select(func.count()).select_from(WorkflowNodeExecutionModel)) == 1
persisted = observer.get(WorkflowNodeExecutionModel, EXECUTION_ID)
assert persisted is not None
assert persisted.outputs_dict == {"delivery": 2}
assert persisted.created_at == created_at.replace(tzinfo=None)
@pytest.mark.parametrize("writer", ["celery", "celery-batch", "sql", "sql-batch", "sql-data"])
@pytest.mark.parametrize("existing", [False, True])
@pytest.mark.parametrize(
"status",
[WorkflowNodeExecutionStatus.RUNNING, WorkflowNodeExecutionStatus.PAUSED, WorkflowNodeExecutionStatus.RETRY],
)
def test_delayed_nonterminal_node_write_cannot_revive_or_insert_a_live_node_after_stop(
sqlite_session_factory: sessionmaker[Session], writer: str, existing: bool, status: WorkflowNodeExecutionStatus
) -> None:
finished_at = datetime(2026, 8, 12, 1, 1)
with sqlite_session_factory.begin() as session:
session.add(
WorkflowRun(
id=WORKFLOW_RUN_ID,
tenant_id=TENANT_ID,
app_id=APP_ID,
workflow_id=WORKFLOW_ID,
type=ModelWorkflowType(WorkflowType.WORKFLOW),
triggered_from=WorkflowRunTriggeredFrom.APP_RUN,
version="1",
status=WorkflowExecutionStatus.STOPPED,
error="User requested stop",
stop_requested_at=finished_at,
finished_at=finished_at,
created_by_role=CreatorUserRole.ACCOUNT,
created_by=ACCOUNT_ID,
)
)
if existing:
node = _stored_execution()
node.status = WorkflowNodeExecutionStatus.FAILED
node.finished_at = finished_at
node.error = "User requested stop"
session.add(node)
execution = _execution(status=status)
if writer.startswith("celery"):
kwargs: _ExecutionTaskKwargs = {
"tenant_id": TENANT_ID,
"app_id": APP_ID,
"triggered_from": WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN.value,
"creator_user_id": ACCOUNT_ID,
"creator_user_role": CreatorUserRole.ACCOUNT.value,
}
if writer == "celery":
assert save_workflow_node_execution_task.run(execution_data=execution.model_dump(), **kwargs)
else:
assert save_workflow_node_executions_task.run(executions_data=[execution.model_dump()], **kwargs)
else:
user = Account(name="Test", email="test@example.com")
user.id = ACCOUNT_ID
repository = SQLAlchemyWorkflowNodeExecutionRepository(
session_factory=sqlite_session_factory,
tenant_id=TENANT_ID,
app_id=APP_ID,
user=user,
triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,
)
if writer == "sql":
repository.save(execution)
elif writer == "sql-batch":
repository.save_many([execution])
else:
repository.save_execution_data(execution)
with sqlite_session_factory() as session:
stored = session.get(WorkflowNodeExecutionModel, EXECUTION_ID)
assert stored is not None
assert stored.status == WorkflowNodeExecutionStatus.FAILED
assert stored.finished_at == finished_at
assert stored.error == "User requested stop"
@pytest.mark.parametrize(
"status",
[WorkflowNodeExecutionStatus.RUNNING, WorkflowNodeExecutionStatus.PAUSED, WorkflowNodeExecutionStatus.RETRY],
)
def test_delayed_write_preserves_node_cleanup_before_workflow_summary_finishes(
sqlite_session_factory: sessionmaker[Session], status: WorkflowNodeExecutionStatus
) -> None:
with sqlite_session_factory.begin() as session:
session.add(
WorkflowRun(
id=WORKFLOW_RUN_ID,
tenant_id=TENANT_ID,
app_id=APP_ID,
workflow_id=WORKFLOW_ID,
type=ModelWorkflowType(WorkflowType.WORKFLOW),
triggered_from=WorkflowRunTriggeredFrom.APP_RUN,
version="1",
status=WorkflowExecutionStatus.RUNNING,
created_by_role=CreatorUserRole.ACCOUNT,
created_by=ACCOUNT_ID,
)
)
node = _stored_execution()
node.status = WorkflowNodeExecutionStatus.FAILED
node.finished_at = datetime(2026, 8, 12, 1, 1)
node.elapsed_time = 60
node.error = "Execution aborted"
session.add(node)
session.flush()
original = dict(session.execute(select(WorkflowNodeExecutionModel.__table__)).mappings().one())
assert save_workflow_node_execution_task.run(
execution_data=_execution(status=status).model_dump(),
tenant_id=TENANT_ID,
app_id=APP_ID,
triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN.value,
creator_user_id=ACCOUNT_ID,
creator_user_role=CreatorUserRole.ACCOUNT.value,
)
with sqlite_session_factory() as session:
assert dict(session.execute(select(WorkflowNodeExecutionModel.__table__)).mappings().one()) == original
run = session.get(WorkflowRun, WORKFLOW_RUN_ID)
assert run is not None
assert run.status == WorkflowExecutionStatus.RUNNING
assert run.stop_requested_at is None