chore: 纳入本地 Dify 1.19.0 源码及品牌与免登改动
演示目录只保留可重建的源码。运行密钥、数据卷、依赖和带口令的本地配置不进入仓库。 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
60158935f4
commit
6519682885
15831 files changed
+3063428
No files matched your search
Whitespace-only changes.
+1380
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,179 @@
|
||||
"""Exercise Celery entrypoints against durable application trace state."""
|
||||
|
||||
from dataclasses import replace
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from dify_trace_sdk.events import MessageCompleted, TraceCorrelation, TraceEvent
|
||||
from flask import Flask
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from core.helper import ssrf_proxy
|
||||
from domain.tracing.trace_run_agg import TraceRunState
|
||||
from extensions.application_services.tracing import build_tracing_services
|
||||
from extensions.ext_application_services import ApplicationServices
|
||||
from libs.datetime_utils import naive_utc_now
|
||||
from models.application_trace import ApplicationTraceRun
|
||||
from models.enums import ConversationFromSource
|
||||
from models.model import AppMode, Message
|
||||
from repositories.factory import DifyAPIRepositoryFactory
|
||||
from repositories.trace_record_repository import TraceRecordRepository
|
||||
from services.tracing.canonical import CanonicalTraceBuilder
|
||||
from services.tracing.collection import AgentTraceSettings
|
||||
from services.tracing.egress import SSRFEgress
|
||||
from services.tracing.providers import ProviderConfigCodec
|
||||
from services.tracing.records import MessageTraceInfo
|
||||
from services.tracing.registry import EntryPointTraceProviderCatalog
|
||||
from services.tracing.service import ApplicationTracingService
|
||||
from tasks.application_trace_task import export_application_trace, reconcile_application_traces
|
||||
from tests.helpers.unit.model_factories import make_conversation, make_tenant
|
||||
from tests.helpers.unit.services.tracing_ports import AGENT_TRACE_SETTINGS, WorkspaceBoundCipher
|
||||
from tests.unit_tests.repositories.test_application_trace_repository import (
|
||||
Collection,
|
||||
batch_for,
|
||||
ingest,
|
||||
seed,
|
||||
terminal,
|
||||
)
|
||||
|
||||
|
||||
def bind_worker(
|
||||
app: Flask,
|
||||
services: ApplicationServices,
|
||||
collection: Collection,
|
||||
*,
|
||||
settings: AgentTraceSettings,
|
||||
scheduled: list[str],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Hand the tasks a worker over the seeded ledger with the installed providers and the SSRF egress."""
|
||||
worker = ApplicationTracingService(
|
||||
runs=collection.repository,
|
||||
run_query=collection.repository,
|
||||
sources=collection.repository,
|
||||
records=TraceRecordRepository(
|
||||
collection.sessions,
|
||||
workflow_runs=DifyAPIRepositoryFactory.create_api_workflow_run_repository(collection.sessions),
|
||||
node_executions=DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository(collection.sessions),
|
||||
files_url="https://files.test",
|
||||
enterprise_details=False,
|
||||
),
|
||||
builder=CanonicalTraceBuilder(),
|
||||
providers=EntryPointTraceProviderCatalog(),
|
||||
codec=ProviderConfigCodec(WorkspaceBoundCipher()),
|
||||
egress=SSRFEgress(),
|
||||
schedule_export=scheduled.append,
|
||||
settings=settings,
|
||||
)
|
||||
monkeypatch.setitem(
|
||||
app.extensions,
|
||||
"application_services",
|
||||
replace(services, tracing=replace(services.tracing, agent_traces=worker)),
|
||||
)
|
||||
|
||||
|
||||
def test_worker_task_exports_once_through_the_ssrf_proxy_and_replay_observes_checkpoint(
|
||||
app: Flask,
|
||||
account_application_services: ApplicationServices,
|
||||
sqlite_session_factory: sessionmaker[Session],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
case = seed(sqlite_session_factory, "worker-task")
|
||||
ingest(case, batch_for(case.run, case.invocation()))
|
||||
terminal(case)
|
||||
bind_worker(
|
||||
app,
|
||||
account_application_services,
|
||||
case,
|
||||
settings=AGENT_TRACE_SETTINGS,
|
||||
scheduled=[],
|
||||
monkeypatch=monkeypatch,
|
||||
)
|
||||
proxied: list[tuple[str, str]] = []
|
||||
|
||||
def send(method: str, url: str, **_kwargs: object) -> httpx.Response:
|
||||
proxied.append((method, url))
|
||||
return httpx.Response(200, json={}, request=httpx.Request(method, url))
|
||||
|
||||
monkeypatch.setattr(ssrf_proxy, "make_request", send)
|
||||
export_application_trace.run(case.run.id)
|
||||
export_application_trace.run(case.run.id)
|
||||
assert proxied == [("POST", "https://langfuse.test/api/public/ingestion")]
|
||||
result = case.repository.get(case.run.id)
|
||||
assert result is not None
|
||||
assert result.state is TraceRunState.ACCEPTED
|
||||
assert result.attempts == 1
|
||||
provider_trace_id = str(UUID(hex=case.run.trace_id))
|
||||
assert result.provider_trace_id == provider_trace_id
|
||||
assert result.trace_url == f"https://langfuse.test/project/original/traces/{provider_trace_id}"
|
||||
|
||||
|
||||
def test_worker_records_carry_no_enterprise_names(sqlite_session_factory: sessionmaker[Session]) -> None:
|
||||
case = seed(sqlite_session_factory, "worker-names")
|
||||
with case.sessions.begin() as session:
|
||||
message = session.get(Message, case.run.source_id)
|
||||
assert message is not None
|
||||
conversation_id = message.conversation_id
|
||||
session.add(
|
||||
make_conversation(
|
||||
conversation_id=conversation_id,
|
||||
app_id=case.run.app_id,
|
||||
mode=AppMode.AGENT,
|
||||
inputs={},
|
||||
from_source=ConversationFromSource.CONSOLE,
|
||||
)
|
||||
)
|
||||
session.add(make_tenant(tenant_id=case.run.tenant_id))
|
||||
# The tasks reach this Agent trace service through the composition root; its records feed recovery.
|
||||
worker = build_tracing_services(database_client=sqlite_session_factory).agent_traces
|
||||
record = worker._records.assemble(
|
||||
TraceEvent(
|
||||
event_id=uuid4().hex,
|
||||
tenant_id=case.run.tenant_id,
|
||||
app_id=case.run.app_id,
|
||||
user_id=None,
|
||||
occurred_at=datetime.now(UTC),
|
||||
correlation=TraceCorrelation(
|
||||
external_trace_id=None, session_id=None, parent_workflow_run_id=None, parent_node_execution_id=None
|
||||
),
|
||||
payload=MessageCompleted(message_id=case.run.source_id, conversation_id=conversation_id),
|
||||
)
|
||||
)
|
||||
assert isinstance(record, MessageTraceInfo)
|
||||
assert (record.metadata["app_name"], record.metadata["workspace_name"]) == ("", "")
|
||||
|
||||
|
||||
def test_reconcile_only_schedules_due_pending_rows_and_applies_retention(
|
||||
app: Flask,
|
||||
account_application_services: ApplicationServices,
|
||||
sqlite_session_factory: sessionmaker[Session],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
due = seed(sqlite_session_factory, "due")
|
||||
deferred = seed(sqlite_session_factory, "future")
|
||||
accepted = seed(sqlite_session_factory, "accepted-expired")
|
||||
with sqlite_session_factory.begin() as session:
|
||||
future = session.get(ApplicationTraceRun, deferred.run.id)
|
||||
expired = session.get(ApplicationTraceRun, accepted.run.id)
|
||||
assert future is not None
|
||||
assert expired is not None
|
||||
future.state = TraceRunState.RETRYING
|
||||
future.next_attempt_at = naive_utc_now() + timedelta(hours=1)
|
||||
expired.state = TraceRunState.ACCEPTED
|
||||
expired.created_at = naive_utc_now() - timedelta(days=4)
|
||||
scheduled: list[str] = []
|
||||
bind_worker(
|
||||
app,
|
||||
account_application_services,
|
||||
due,
|
||||
settings=replace(AGENT_TRACE_SETTINGS, retention_days=3),
|
||||
scheduled=scheduled,
|
||||
monkeypatch=monkeypatch,
|
||||
)
|
||||
reconcile_application_traces.run()
|
||||
assert scheduled == [due.run.id]
|
||||
assert due.repository.get(accepted.run.id) is None
|
||||
assert due.repository.get(deferred.run.id) is not None
|
||||
assert due.repository.get(due.run.id) is not None
|
||||
@@ -0,0 +1,88 @@
|
||||
import ast
|
||||
import inspect
|
||||
|
||||
from services.errors.app import WorkflowNotFoundError
|
||||
from tasks import async_workflow_tasks
|
||||
|
||||
|
||||
def test_workflow_not_found_error_message_is_a_single_formatted_string() -> None:
|
||||
# Regression: previously raised with printf-style positional args
|
||||
# ("...%s...", id1, id2), which made Exception store them as a tuple
|
||||
# and str(exc) rendered the literal tuple instead of a readable message.
|
||||
workflow_run_id = "wr-123"
|
||||
workflow_id = "wf-456"
|
||||
|
||||
exc = WorkflowNotFoundError(f"Workflow not found: workflow_run_id={workflow_run_id}, workflow_id={workflow_id}")
|
||||
|
||||
# The exception must stringify to the fully-interpolated message and must
|
||||
# not contain unresolved %s placeholders nor tuple punctuation.
|
||||
rendered = str(exc)
|
||||
assert rendered == "Workflow not found: workflow_run_id=wr-123, workflow_id=wf-456"
|
||||
assert "%s" not in rendered
|
||||
assert not rendered.startswith("(")
|
||||
assert exc.args == ("Workflow not found: workflow_run_id=wr-123, workflow_id=wf-456",)
|
||||
|
||||
|
||||
def test_app_not_found_error_message_is_a_single_formatted_string() -> None:
|
||||
# Same regression check for the _AppNotFoundError raise site in the same
|
||||
# function (resume_workflow_execution).
|
||||
app_id = "app-789"
|
||||
workflow_run_id = "wr-123"
|
||||
|
||||
exc = async_workflow_tasks._AppNotFoundError(f"App not found: app_id={app_id}, workflow_run_id={workflow_run_id}")
|
||||
|
||||
rendered = str(exc)
|
||||
assert rendered == "App not found: app_id=app-789, workflow_run_id=wr-123"
|
||||
assert "%s" not in rendered
|
||||
assert not rendered.startswith("(")
|
||||
assert exc.args == ("App not found: app_id=app-789, workflow_run_id=wr-123",)
|
||||
|
||||
|
||||
def _collect_raise_calls_in(func_name: str) -> list[ast.Call]:
|
||||
"""Walk the AST of async_workflow_tasks and return every Call inside a Raise
|
||||
inside the function named ``func_name``.
|
||||
|
||||
This lets us assert at unit-test time that no raise site in a critical
|
||||
function passes printf-style positional args to an Exception constructor
|
||||
(which would otherwise be stored verbatim in ``exc.args`` and stringify
|
||||
as a tuple-repr instead of a human-readable message).
|
||||
"""
|
||||
source = inspect.getsource(async_workflow_tasks)
|
||||
tree = ast.parse(source)
|
||||
raises: list[ast.Call] = []
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.FunctionDef) and node.name == func_name:
|
||||
for sub in ast.walk(node):
|
||||
if isinstance(sub, ast.Raise) and isinstance(sub.exc, ast.Call):
|
||||
raises.append(sub.exc)
|
||||
return raises
|
||||
|
||||
|
||||
def test_resume_workflow_execution_raises_do_not_use_printf_positional_args() -> None:
|
||||
"""Regression: ``resume_workflow_execution`` previously raised
|
||||
``WorkflowNotFoundError("Workflow not found: workflow_run_id=%s, ...", run_id, wf_id)``
|
||||
and ``_AppNotFoundError("App not found: app_id=%s, ...", app_id, run_id)``.
|
||||
|
||||
Python's ``Exception.__init__`` then stored all three as ``exc.args`` and
|
||||
``str(exc)`` produced ``("Workflow not found: ...%s...", 'id1', 'id2')``
|
||||
rather than a usable message.
|
||||
|
||||
Lock this in by walking the function's AST: every ``raise SomeError(...)``
|
||||
inside ``resume_workflow_execution`` must pass exactly one positional
|
||||
argument (the rendered message), or be a no-arg raise. If anyone reverts
|
||||
to the printf pattern, this test fails fast at unit-test time.
|
||||
"""
|
||||
raise_calls = _collect_raise_calls_in("resume_workflow_execution")
|
||||
|
||||
assert raise_calls, "Expected raise sites inside resume_workflow_execution to inspect"
|
||||
for call in raise_calls:
|
||||
assert len(call.args) <= 1, (
|
||||
f"raise {ast.unparse(call)} passes {len(call.args)} positional args; "
|
||||
"Exception classes accept a single rendered message. Use an f-string "
|
||||
"instead of printf-style ('...%s...', val1, val2)."
|
||||
)
|
||||
# Also reject %s in a literal first arg as belt-and-braces.
|
||||
if call.args and isinstance(call.args[0], ast.Constant) and isinstance(call.args[0].value, str):
|
||||
assert "%s" not in call.args[0].value, (
|
||||
f"raise {ast.unparse(call)} still contains '%s' in its message; use an f-string."
|
||||
)
|
||||
@@ -0,0 +1,214 @@
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
import tasks.batch_clean_document_task as task_module
|
||||
from extensions.storage.storage_type import StorageType
|
||||
from models.dataset import Dataset, DocumentSegment, SegmentAttachmentBinding
|
||||
from models.enums import CreatorUserRole, DataSourceType
|
||||
from models.knowledge_fs import KnowledgeFSUpgradeFileLease, KnowledgeFSUpgradeJob
|
||||
from models.model import UploadFile
|
||||
from tasks.batch_clean_document_task import batch_clean_document_task
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def cleanup_rows(sqlite_session: Session, monkeypatch: pytest.MonkeyPatch) -> tuple[str, str, str]:
|
||||
tenant_id = str(uuid.uuid4())
|
||||
dataset_id = str(uuid.uuid4())
|
||||
document_id = str(uuid.uuid4())
|
||||
created_by = str(uuid.uuid4())
|
||||
dataset = Dataset(
|
||||
id=dataset_id,
|
||||
tenant_id=tenant_id,
|
||||
name="Batch cleanup dataset",
|
||||
data_source_type=DataSourceType.UPLOAD_FILE,
|
||||
created_by=created_by,
|
||||
)
|
||||
segment = DocumentSegment(
|
||||
tenant_id=tenant_id,
|
||||
dataset_id=dataset_id,
|
||||
document_id=document_id,
|
||||
position=1,
|
||||
content="content",
|
||||
word_count=1,
|
||||
tokens=1,
|
||||
created_by=created_by,
|
||||
index_node_id="node-1",
|
||||
)
|
||||
sqlite_session.add_all([dataset, segment])
|
||||
sqlite_session.commit()
|
||||
engine = sqlite_session.get_bind()
|
||||
monkeypatch.setattr(
|
||||
task_module.session_factory,
|
||||
"create_session",
|
||||
lambda: Session(engine, expire_on_commit=False),
|
||||
)
|
||||
return dataset_id, document_id, tenant_id
|
||||
|
||||
|
||||
def test_successful_vector_cleanup_schedules_billing_refresh(cleanup_rows: tuple[str, str, str]) -> None:
|
||||
dataset_id, document_id, tenant_id = cleanup_rows
|
||||
|
||||
with (
|
||||
patch("tasks.batch_clean_document_task.get_image_upload_file_ids", return_value=[]),
|
||||
patch("tasks.batch_clean_document_task.clean_document_indexes", return_value=tenant_id) as index_cleanup,
|
||||
patch("tasks.batch_clean_document_task.schedule_billing_vector_space_refresh") as schedule_refresh,
|
||||
):
|
||||
batch_clean_document_task(
|
||||
document_ids=[document_id],
|
||||
dataset_id=dataset_id,
|
||||
doc_form="paragraph",
|
||||
file_ids=[],
|
||||
)
|
||||
|
||||
index_cleanup.assert_called_once()
|
||||
schedule_refresh.assert_called_once_with(tenant_id)
|
||||
|
||||
|
||||
def test_failed_vector_cleanup_does_not_schedule_billing_refresh(cleanup_rows: tuple[str, str, str]) -> None:
|
||||
dataset_id, document_id, _tenant_id = cleanup_rows
|
||||
|
||||
with (
|
||||
patch("tasks.batch_clean_document_task.get_image_upload_file_ids", return_value=[]),
|
||||
patch("tasks.batch_clean_document_task.clean_document_indexes") as index_cleanup,
|
||||
patch("tasks.batch_clean_document_task.schedule_billing_vector_space_refresh") as schedule_refresh,
|
||||
):
|
||||
index_cleanup.side_effect = RuntimeError("vector cleanup failed")
|
||||
batch_clean_document_task(
|
||||
document_ids=[document_id],
|
||||
dataset_id=dataset_id,
|
||||
doc_form="paragraph",
|
||||
file_ids=[],
|
||||
)
|
||||
|
||||
schedule_refresh.assert_not_called()
|
||||
|
||||
|
||||
def test_batch_cleanup_keeps_only_the_leased_legacy_source_file(
|
||||
cleanup_rows: tuple[str, str, str], sqlite_session: Session
|
||||
) -> None:
|
||||
dataset_id, document_id, tenant_id = cleanup_rows
|
||||
account_id = str(uuid.uuid4())
|
||||
now = datetime.now(UTC).replace(tzinfo=None)
|
||||
leased_file = UploadFile(
|
||||
tenant_id=tenant_id,
|
||||
storage_type=StorageType.LOCAL,
|
||||
key=f"upload_files/{tenant_id}/leased.txt",
|
||||
name="leased.txt",
|
||||
size=10,
|
||||
extension="txt",
|
||||
mime_type="text/plain",
|
||||
created_by_role=CreatorUserRole.ACCOUNT,
|
||||
created_by=account_id,
|
||||
created_at=now,
|
||||
used=False,
|
||||
)
|
||||
deletable_file = UploadFile(
|
||||
tenant_id=tenant_id,
|
||||
storage_type=StorageType.LOCAL,
|
||||
key=f"upload_files/{tenant_id}/deletable.txt",
|
||||
name="deletable.txt",
|
||||
size=10,
|
||||
extension="txt",
|
||||
mime_type="text/plain",
|
||||
created_by_role=CreatorUserRole.ACCOUNT,
|
||||
created_by=account_id,
|
||||
created_at=now,
|
||||
used=False,
|
||||
)
|
||||
job = KnowledgeFSUpgradeJob(
|
||||
tenant_id=tenant_id,
|
||||
old_dataset_id=dataset_id,
|
||||
requested_by_account_id=account_id,
|
||||
owner_account_id=account_id,
|
||||
idempotency_key="batch-cleanup-lease-test",
|
||||
snapshot_at=now,
|
||||
config_snapshot={},
|
||||
permission_snapshot={},
|
||||
app_binding_snapshot=[],
|
||||
tag_ids_snapshot=[],
|
||||
)
|
||||
sqlite_session.add_all([leased_file, deletable_file, job])
|
||||
sqlite_session.flush()
|
||||
sqlite_session.add(
|
||||
KnowledgeFSUpgradeFileLease(
|
||||
job_id=job.id,
|
||||
old_upload_file_id=leased_file.id,
|
||||
expires_at=now + timedelta(hours=1),
|
||||
)
|
||||
)
|
||||
sqlite_session.commit()
|
||||
leased_file_id = leased_file.id
|
||||
deletable_file_id = deletable_file.id
|
||||
deletable_file_key = deletable_file.key
|
||||
|
||||
with (
|
||||
patch("tasks.batch_clean_document_task.get_image_upload_file_ids", return_value=[]),
|
||||
patch("tasks.batch_clean_document_task.clean_document_indexes", return_value=tenant_id),
|
||||
patch("tasks.batch_clean_document_task.schedule_billing_vector_space_refresh"),
|
||||
patch("tasks.batch_clean_document_task.storage") as storage,
|
||||
):
|
||||
batch_clean_document_task(
|
||||
document_ids=[document_id],
|
||||
dataset_id=dataset_id,
|
||||
doc_form="paragraph",
|
||||
file_ids=[leased_file_id, deletable_file_id],
|
||||
)
|
||||
|
||||
sqlite_session.expire_all()
|
||||
assert sqlite_session.get(UploadFile, leased_file_id) is not None
|
||||
assert sqlite_session.get(UploadFile, deletable_file_id) is None
|
||||
storage.delete.assert_called_once_with(deletable_file_key)
|
||||
|
||||
|
||||
def test_cleans_segment_attachment_bindings_and_files(
|
||||
cleanup_rows: tuple[str, str, str], sqlite_session: Session
|
||||
) -> None:
|
||||
dataset_id, document_id, tenant_id = cleanup_rows
|
||||
segment = sqlite_session.query(DocumentSegment).filter_by(document_id=document_id).one()
|
||||
attachment = UploadFile(
|
||||
tenant_id=tenant_id,
|
||||
storage_type=StorageType.LOCAL,
|
||||
key="attachments/image.png",
|
||||
name="image.png",
|
||||
size=10,
|
||||
extension="png",
|
||||
mime_type="image/png",
|
||||
created_by_role=CreatorUserRole.ACCOUNT,
|
||||
created_by=segment.created_by,
|
||||
created_at=datetime.now(UTC),
|
||||
used=True,
|
||||
)
|
||||
binding = SegmentAttachmentBinding(
|
||||
tenant_id=tenant_id,
|
||||
dataset_id=dataset_id,
|
||||
document_id=document_id,
|
||||
segment_id=segment.id,
|
||||
attachment_id=attachment.id,
|
||||
)
|
||||
sqlite_session.add_all([attachment, binding])
|
||||
sqlite_session.commit()
|
||||
attachment_id = attachment.id
|
||||
attachment_key = attachment.key
|
||||
binding_id = binding.id
|
||||
|
||||
with (
|
||||
patch("tasks.batch_clean_document_task.get_image_upload_file_ids", return_value=[]),
|
||||
patch("tasks.batch_clean_document_task.clean_document_indexes"),
|
||||
patch("tasks.batch_clean_document_task.schedule_billing_vector_space_refresh"),
|
||||
patch("tasks.batch_clean_document_task.storage.delete") as storage_delete,
|
||||
):
|
||||
batch_clean_document_task(
|
||||
document_ids=[document_id],
|
||||
dataset_id=dataset_id,
|
||||
doc_form="paragraph",
|
||||
file_ids=[],
|
||||
)
|
||||
|
||||
sqlite_session.expire_all()
|
||||
assert sqlite_session.get(SegmentAttachmentBinding, binding_id) is None
|
||||
assert sqlite_session.get(UploadFile, attachment_id) is None
|
||||
storage_delete.assert_called_once_with(attachment_key)
|
||||
@@ -0,0 +1,140 @@
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import event, select
|
||||
from sqlalchemy.engine import Engine
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType
|
||||
from extensions.storage.storage_type import StorageType
|
||||
from models.dataset import Dataset, Document, DocumentSegment
|
||||
from models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, IndexingStatus, SegmentStatus
|
||||
from models.model import UploadFile
|
||||
from services.knowledge.resource_scope import DatasetRef, DocumentRef
|
||||
from tasks.batch_create_segment_to_index_task import batch_create_segment_to_index_task
|
||||
|
||||
|
||||
@pytest.mark.parametrize("foreign_owner", [None, "dataset", "document_tenant", "document_dataset", "upload"])
|
||||
def test_batch_create_segment_to_index_task_validates_scope_and_indexes_committed_batch(
|
||||
sqlite_session: Session,
|
||||
sqlite_engine: Engine,
|
||||
foreign_owner: str | None,
|
||||
) -> None:
|
||||
tenant_id = str(uuid4())
|
||||
user_id = str(uuid4())
|
||||
dataset = Dataset(
|
||||
id=str(uuid4()),
|
||||
tenant_id=tenant_id,
|
||||
name="Batch import dataset",
|
||||
data_source_type=DataSourceType.UPLOAD_FILE,
|
||||
created_by=user_id,
|
||||
indexing_technique=IndexTechniqueType.ECONOMY,
|
||||
)
|
||||
document = Document(
|
||||
id=str(uuid4()),
|
||||
tenant_id=tenant_id,
|
||||
dataset_id=dataset.id,
|
||||
position=1,
|
||||
data_source_type=DataSourceType.UPLOAD_FILE,
|
||||
batch="batch-1",
|
||||
name="document.txt",
|
||||
created_from=DocumentCreatedFrom.WEB,
|
||||
created_by=user_id,
|
||||
indexing_status=IndexingStatus.COMPLETED,
|
||||
enabled=True,
|
||||
archived=False,
|
||||
doc_form=IndexStructureType.PARAGRAPH_INDEX,
|
||||
word_count=0,
|
||||
)
|
||||
upload_file = UploadFile(
|
||||
tenant_id=tenant_id,
|
||||
storage_type=StorageType.LOCAL,
|
||||
key="batch-import.csv",
|
||||
name="batch-import.csv",
|
||||
size=32,
|
||||
extension=".csv",
|
||||
mime_type="text/csv",
|
||||
created_by_role=CreatorUserRole.ACCOUNT,
|
||||
created_by=user_id,
|
||||
created_at=datetime.now(UTC),
|
||||
used=False,
|
||||
)
|
||||
existing_segment = DocumentSegment(
|
||||
tenant_id=tenant_id,
|
||||
dataset_id=dataset.id,
|
||||
document_id=document.id,
|
||||
position=7,
|
||||
content="existing",
|
||||
word_count=1,
|
||||
tokens=1,
|
||||
created_by=user_id,
|
||||
index_node_id="existing-node",
|
||||
index_node_hash="existing-hash",
|
||||
status=SegmentStatus.COMPLETED,
|
||||
)
|
||||
if foreign_owner == "dataset":
|
||||
dataset.tenant_id = str(uuid4())
|
||||
elif foreign_owner == "document_tenant":
|
||||
document.tenant_id = str(uuid4())
|
||||
elif foreign_owner == "document_dataset":
|
||||
document.dataset_id = str(uuid4())
|
||||
elif foreign_owner == "upload":
|
||||
upload_file.tenant_id = str(uuid4())
|
||||
sqlite_session.add_all([dataset, document, upload_file, existing_segment])
|
||||
sqlite_session.commit()
|
||||
|
||||
def mock_download(_key: str, file_path: str) -> None:
|
||||
Path(file_path).write_text("content\nfirst\nsecond\nthird\n", encoding="utf-8")
|
||||
|
||||
max_position_queries: list[str] = []
|
||||
|
||||
def count_max_position_query(
|
||||
_connection: object,
|
||||
_cursor: object,
|
||||
statement: str,
|
||||
_parameters: object,
|
||||
_context: object,
|
||||
_executemany: bool,
|
||||
) -> None:
|
||||
normalized_statement = statement.lower()
|
||||
if "max(" in normalized_statement and "document_segments.position" in normalized_statement:
|
||||
max_position_queries.append(statement)
|
||||
|
||||
event.listen(sqlite_engine, "before_cursor_execute", count_max_position_query)
|
||||
try:
|
||||
with (
|
||||
patch("tasks.batch_create_segment_to_index_task.storage.download", side_effect=mock_download) as download,
|
||||
patch("extensions.ext_application_services.application_services") as services,
|
||||
):
|
||||
batch_create_segment_to_index_task(
|
||||
job_id=str(uuid4()),
|
||||
upload_file_id=upload_file.id,
|
||||
dataset_id=dataset.id,
|
||||
document_id=document.id,
|
||||
tenant_id=tenant_id,
|
||||
user_id=user_id,
|
||||
)
|
||||
finally:
|
||||
event.remove(sqlite_engine, "before_cursor_execute", count_max_position_query)
|
||||
|
||||
if foreign_owner is not None:
|
||||
download.assert_not_called()
|
||||
services.assert_not_called()
|
||||
assert not max_position_queries
|
||||
assert sqlite_session.scalars(select(DocumentSegment.id)).all() == [existing_segment.id]
|
||||
return
|
||||
|
||||
positions = sqlite_session.scalars(
|
||||
select(DocumentSegment.position)
|
||||
.where(DocumentSegment.document_id == document.id)
|
||||
.order_by(DocumentSegment.position)
|
||||
).all()
|
||||
assert positions == [7, 8, 9, 10]
|
||||
assert len(max_position_queries) == 1
|
||||
indexing = services.return_value.knowledge.segments.mutations.index_segments
|
||||
indexing.assert_called_once()
|
||||
assert indexing.call_args.args == (DocumentRef(DatasetRef(tenant_id, dataset.id), document.id),)
|
||||
assert len(indexing.call_args.kwargs["segment_ids"]) == 3
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,388 @@
|
||||
"""SQLite-backed resilience tests for ``clean_document_task``.
|
||||
|
||||
The task must continue PostgreSQL cleanup when vector cleanup fails. Each test
|
||||
starts from the production incident shape: the caller has already deleted the
|
||||
``Document`` row while its segments and metadata bindings remain.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
import tasks.clean_document_task as clean_document_task_module
|
||||
from extensions.storage.storage_type import StorageType
|
||||
from models.dataset import (
|
||||
Dataset,
|
||||
DatasetMetadataBinding,
|
||||
Document,
|
||||
DocumentSegment,
|
||||
SegmentAttachmentBinding,
|
||||
)
|
||||
from models.enums import CreatorUserRole, DataSourceType
|
||||
from models.knowledge_fs import KnowledgeFSUpgradeFileLease, KnowledgeFSUpgradeJob
|
||||
from models.model import UploadFile
|
||||
from tasks.clean_document_task import clean_document_task
|
||||
from tests.helpers.unit.model_factories import make_document
|
||||
|
||||
SQLITE_MODELS = (
|
||||
Dataset,
|
||||
Document,
|
||||
DocumentSegment,
|
||||
SegmentAttachmentBinding,
|
||||
UploadFile,
|
||||
DatasetMetadataBinding,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.parametrize("sqlite_session", [SQLITE_MODELS], indirect=True)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def document_id() -> str:
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dataset_id() -> str:
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tenant_id() -> str:
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def bind_task_sessions(monkeypatch: pytest.MonkeyPatch, sqlite_session: Session) -> None:
|
||||
"""Bind every short-lived task transaction to the isolated SQLite engine."""
|
||||
engine = sqlite_session.get_bind()
|
||||
monkeypatch.setattr(
|
||||
clean_document_task_module.session_factory,
|
||||
"create_session",
|
||||
lambda: Session(engine, expire_on_commit=False),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_storage():
|
||||
with patch("tasks.clean_document_task.storage", autospec=True) as mock:
|
||||
mock.delete.return_value = None
|
||||
yield mock
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_index_cleanup(tenant_id: str):
|
||||
"""Keep task resilience tests independent of the external index backend."""
|
||||
with patch("tasks.clean_document_task.clean_document_indexes", return_value=tenant_id) as cleanup:
|
||||
yield cleanup
|
||||
|
||||
|
||||
def _document(*, document_id: str, dataset_id: str, tenant_id: str, created_by: str) -> Document:
|
||||
return make_document(
|
||||
document_id=document_id,
|
||||
tenant_id=tenant_id,
|
||||
dataset_id=dataset_id,
|
||||
name=f"{document_id}.txt",
|
||||
created_by=created_by,
|
||||
)
|
||||
|
||||
|
||||
def _segment(*, segment_id: str, document_id: str, dataset_id: str, tenant_id: str, created_by: str) -> DocumentSegment:
|
||||
segment = DocumentSegment(
|
||||
tenant_id=tenant_id,
|
||||
dataset_id=dataset_id,
|
||||
document_id=document_id,
|
||||
position=1,
|
||||
content="segment content",
|
||||
word_count=2,
|
||||
tokens=2,
|
||||
created_by=created_by,
|
||||
index_node_id=f"node-{segment_id}",
|
||||
)
|
||||
segment.id = segment_id
|
||||
return segment
|
||||
|
||||
|
||||
def _persist_deleted_document_state(
|
||||
session: Session,
|
||||
*,
|
||||
document_id: str,
|
||||
dataset_id: str,
|
||||
tenant_id: str,
|
||||
target_segment_ids: list[str],
|
||||
) -> tuple[str, str]:
|
||||
"""Persist target children after deleting their document, plus scoped control rows."""
|
||||
created_by = str(uuid.uuid4())
|
||||
other_document_id = str(uuid.uuid4())
|
||||
survivor_segment_id = str(uuid.uuid4())
|
||||
dataset = Dataset(
|
||||
id=dataset_id,
|
||||
tenant_id=tenant_id,
|
||||
name="Cleanup dataset",
|
||||
data_source_type=DataSourceType.UPLOAD_FILE,
|
||||
created_by=created_by,
|
||||
)
|
||||
target_document = _document(
|
||||
document_id=document_id,
|
||||
dataset_id=dataset_id,
|
||||
tenant_id=tenant_id,
|
||||
created_by=created_by,
|
||||
)
|
||||
other_document = _document(
|
||||
document_id=other_document_id,
|
||||
dataset_id=dataset_id,
|
||||
tenant_id=tenant_id,
|
||||
created_by=created_by,
|
||||
)
|
||||
segments = [
|
||||
_segment(
|
||||
segment_id=segment_id,
|
||||
document_id=document_id,
|
||||
dataset_id=dataset_id,
|
||||
tenant_id=tenant_id,
|
||||
created_by=created_by,
|
||||
)
|
||||
for segment_id in target_segment_ids
|
||||
]
|
||||
segments.append(
|
||||
_segment(
|
||||
segment_id=survivor_segment_id,
|
||||
document_id=other_document_id,
|
||||
dataset_id=dataset_id,
|
||||
tenant_id=tenant_id,
|
||||
created_by=created_by,
|
||||
)
|
||||
)
|
||||
metadata_bindings = [
|
||||
DatasetMetadataBinding(
|
||||
tenant_id=tenant_id,
|
||||
dataset_id=dataset_id,
|
||||
metadata_id=str(uuid.uuid4()),
|
||||
document_id=current_document_id,
|
||||
created_by=created_by,
|
||||
)
|
||||
for current_document_id in (document_id, other_document_id)
|
||||
]
|
||||
|
||||
session.add_all([dataset, target_document, other_document, *segments, *metadata_bindings])
|
||||
session.commit()
|
||||
session.delete(target_document)
|
||||
session.commit()
|
||||
return other_document_id, survivor_segment_id
|
||||
|
||||
|
||||
def _assert_relational_cleanup(
|
||||
session: Session,
|
||||
*,
|
||||
document_id: str,
|
||||
other_document_id: str,
|
||||
survivor_segment_id: str,
|
||||
) -> None:
|
||||
session.expire_all()
|
||||
assert session.get(Document, document_id) is None
|
||||
remaining_segments = session.scalars(select(DocumentSegment)).all()
|
||||
assert [(segment.id, segment.document_id) for segment in remaining_segments] == [
|
||||
(survivor_segment_id, other_document_id)
|
||||
]
|
||||
remaining_binding_document_ids = set(session.scalars(select(DatasetMetadataBinding.document_id)).all())
|
||||
assert remaining_binding_document_ids == {other_document_id}
|
||||
|
||||
|
||||
class TestVectorCleanupResilience:
|
||||
"""Vector/index failures must not abort relational cleanup."""
|
||||
|
||||
def test_billing_failure_during_vector_cleanup_does_not_skip_pg_cleanup(
|
||||
self,
|
||||
document_id: str,
|
||||
dataset_id: str,
|
||||
tenant_id: str,
|
||||
sqlite_session: Session,
|
||||
bind_task_sessions: None,
|
||||
mock_storage,
|
||||
mock_index_cleanup,
|
||||
) -> None:
|
||||
"""A transient billing failure leaves only the unrelated document's rows."""
|
||||
other_document_id, survivor_segment_id = _persist_deleted_document_state(
|
||||
sqlite_session,
|
||||
document_id=document_id,
|
||||
dataset_id=dataset_id,
|
||||
tenant_id=tenant_id,
|
||||
target_segment_ids=["seg-1", "seg-2"],
|
||||
)
|
||||
mock_index_cleanup.side_effect = ValueError(
|
||||
"Unable to retrieve billing information. Please try again later or contact support."
|
||||
)
|
||||
|
||||
# Act — must not raise out of the task even though clean() raises.
|
||||
with patch("tasks.clean_document_task.schedule_billing_vector_space_refresh") as schedule_refresh:
|
||||
clean_document_task(
|
||||
document_id=document_id,
|
||||
dataset_id=dataset_id,
|
||||
doc_form="paragraph",
|
||||
file_id=None,
|
||||
)
|
||||
|
||||
mock_index_cleanup.assert_called_once()
|
||||
_assert_relational_cleanup(
|
||||
sqlite_session,
|
||||
document_id=document_id,
|
||||
other_document_id=other_document_id,
|
||||
survivor_segment_id=survivor_segment_id,
|
||||
)
|
||||
schedule_refresh.assert_not_called()
|
||||
|
||||
|
||||
def test_active_upgrade_lease_keeps_the_legacy_source_file(
|
||||
document_id: str,
|
||||
dataset_id: str,
|
||||
tenant_id: str,
|
||||
sqlite_session: Session,
|
||||
bind_task_sessions: None,
|
||||
mock_storage,
|
||||
mock_index_cleanup,
|
||||
) -> None:
|
||||
del bind_task_sessions, mock_index_cleanup
|
||||
_persist_deleted_document_state(
|
||||
sqlite_session,
|
||||
document_id=document_id,
|
||||
dataset_id=dataset_id,
|
||||
tenant_id=tenant_id,
|
||||
target_segment_ids=[],
|
||||
)
|
||||
account_id = str(uuid.uuid4())
|
||||
upload_file = UploadFile(
|
||||
tenant_id=tenant_id,
|
||||
storage_type=StorageType.LOCAL,
|
||||
key=f"upload_files/{tenant_id}/source.txt",
|
||||
name="source.txt",
|
||||
size=12,
|
||||
extension="txt",
|
||||
mime_type="text/plain",
|
||||
created_by_role=CreatorUserRole.ACCOUNT,
|
||||
created_by=account_id,
|
||||
created_at=datetime(2026, 8, 17, tzinfo=UTC),
|
||||
used=False,
|
||||
)
|
||||
now = datetime.now(UTC).replace(tzinfo=None)
|
||||
job = KnowledgeFSUpgradeJob(
|
||||
tenant_id=tenant_id,
|
||||
old_dataset_id=dataset_id,
|
||||
requested_by_account_id=account_id,
|
||||
owner_account_id=account_id,
|
||||
idempotency_key="cleanup-lease-test",
|
||||
snapshot_at=now,
|
||||
config_snapshot={},
|
||||
permission_snapshot={},
|
||||
app_binding_snapshot=[],
|
||||
tag_ids_snapshot=[],
|
||||
)
|
||||
sqlite_session.add_all([upload_file, job])
|
||||
sqlite_session.flush()
|
||||
lease = KnowledgeFSUpgradeFileLease(
|
||||
job_id=job.id,
|
||||
old_upload_file_id=upload_file.id,
|
||||
expires_at=now + timedelta(hours=1),
|
||||
)
|
||||
sqlite_session.add(lease)
|
||||
sqlite_session.commit()
|
||||
|
||||
clean_document_task(
|
||||
document_id=document_id,
|
||||
dataset_id=dataset_id,
|
||||
doc_form="paragraph",
|
||||
file_id=upload_file.id,
|
||||
)
|
||||
|
||||
sqlite_session.expire_all()
|
||||
assert sqlite_session.get(UploadFile, upload_file.id) is not None
|
||||
persisted_lease = sqlite_session.get(KnowledgeFSUpgradeFileLease, lease.id)
|
||||
assert persisted_lease is not None
|
||||
assert persisted_lease.cleanup_requested_at is not None
|
||||
mock_storage.delete.assert_not_called()
|
||||
|
||||
|
||||
class TestVectorCleanupSuccessPaths:
|
||||
def test_vector_cleanup_success_path_remains_unaffected(
|
||||
self,
|
||||
document_id: str,
|
||||
dataset_id: str,
|
||||
tenant_id: str,
|
||||
sqlite_session: Session,
|
||||
bind_task_sessions: None,
|
||||
mock_storage,
|
||||
mock_index_cleanup,
|
||||
) -> None:
|
||||
"""The happy path calls the index boundary and completes scoped cleanup."""
|
||||
other_document_id, survivor_segment_id = _persist_deleted_document_state(
|
||||
sqlite_session,
|
||||
document_id=document_id,
|
||||
dataset_id=dataset_id,
|
||||
tenant_id=tenant_id,
|
||||
target_segment_ids=["seg-1"],
|
||||
)
|
||||
|
||||
with patch("tasks.clean_document_task.schedule_billing_vector_space_refresh") as schedule_refresh:
|
||||
clean_document_task(
|
||||
document_id=document_id,
|
||||
dataset_id=dataset_id,
|
||||
doc_form="paragraph",
|
||||
file_id=None,
|
||||
)
|
||||
|
||||
mock_index_cleanup.assert_called_once()
|
||||
_, kwargs = mock_index_cleanup.call_args
|
||||
new_session = kwargs.pop("new_session")
|
||||
with new_session() as cleanup_session:
|
||||
assert cleanup_session.get_bind() is sqlite_session.get_bind()
|
||||
assert kwargs == {
|
||||
"dataset_id": dataset_id,
|
||||
"document_ids": [document_id],
|
||||
"doc_form": "paragraph",
|
||||
}
|
||||
_assert_relational_cleanup(
|
||||
sqlite_session,
|
||||
document_id=document_id,
|
||||
other_document_id=other_document_id,
|
||||
survivor_segment_id=survivor_segment_id,
|
||||
)
|
||||
schedule_refresh.assert_called_once_with(tenant_id)
|
||||
|
||||
def test_no_index_state_does_not_refresh_billing(
|
||||
self,
|
||||
document_id: str,
|
||||
dataset_id: str,
|
||||
tenant_id: str,
|
||||
sqlite_session: Session,
|
||||
bind_task_sessions: None,
|
||||
mock_storage,
|
||||
mock_index_cleanup,
|
||||
) -> None:
|
||||
"""Cleanup checks for orphan summaries even when no segments remain."""
|
||||
mock_index_cleanup.return_value = None
|
||||
other_document_id, survivor_segment_id = _persist_deleted_document_state(
|
||||
sqlite_session,
|
||||
document_id=document_id,
|
||||
dataset_id=dataset_id,
|
||||
tenant_id=tenant_id,
|
||||
target_segment_ids=[],
|
||||
)
|
||||
|
||||
with patch("tasks.clean_document_task.schedule_billing_vector_space_refresh") as schedule_refresh:
|
||||
clean_document_task(
|
||||
document_id=document_id,
|
||||
dataset_id=dataset_id,
|
||||
doc_form="paragraph",
|
||||
file_id=None,
|
||||
)
|
||||
|
||||
mock_index_cleanup.assert_called_once()
|
||||
_assert_relational_cleanup(
|
||||
sqlite_session,
|
||||
document_id=document_id,
|
||||
other_document_id=other_document_id,
|
||||
survivor_segment_id=survivor_segment_id,
|
||||
)
|
||||
schedule_refresh.assert_not_called()
|
||||
@@ -0,0 +1,207 @@
|
||||
from typing import Protocol, cast
|
||||
|
||||
import pytest
|
||||
|
||||
from services.agent.deletion_service import AgentDeletionService
|
||||
from services.agent.home_snapshot_service import AgentHomeSnapshotService
|
||||
from services.agent.workspace_service import AgentWorkspaceService
|
||||
from tasks.collect_agent_resources_task import (
|
||||
collect_agent_resources,
|
||||
enqueue_agent_resource_collection,
|
||||
)
|
||||
|
||||
|
||||
class _TaskWithQueue(Protocol):
|
||||
queue: str
|
||||
|
||||
|
||||
def test_collection_task_uses_retention_queue() -> None:
|
||||
task = cast(_TaskWithQueue, collect_agent_resources)
|
||||
assert task.queue == "retention"
|
||||
|
||||
|
||||
def test_enqueue_deduplicates_ids_and_skips_empty_input(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
delay_calls: list[dict[str, object]] = []
|
||||
|
||||
def delay(**kwargs: object) -> None:
|
||||
delay_calls.append(kwargs)
|
||||
|
||||
monkeypatch.setattr(collect_agent_resources, "delay", delay)
|
||||
|
||||
enqueue_agent_resource_collection(tenant_id="tenant-1")
|
||||
enqueue_agent_resource_collection(
|
||||
tenant_id="tenant-1",
|
||||
binding_ids=["binding-2", "binding-1", "binding-2"],
|
||||
workspace_ids=["workspace-1"],
|
||||
purge_agent_ids=["agent-2", "", "agent-1", "agent-2"],
|
||||
)
|
||||
|
||||
assert delay_calls == [
|
||||
{
|
||||
"tenant_id": "tenant-1",
|
||||
"binding_ids": ["binding-1", "binding-2"],
|
||||
"workspace_ids": ["workspace-1"],
|
||||
"home_snapshot_ids": [],
|
||||
"purge_agent_ids": ["agent-1", "agent-2"],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_collection_runs_in_workspace_binding_snapshot_order(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
calls: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
AgentWorkspaceService,
|
||||
"collect_retired_workspace",
|
||||
lambda **_kwargs: calls.append("workspace"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
AgentWorkspaceService,
|
||||
"collect_retired_binding",
|
||||
lambda **_kwargs: calls.append("binding"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
AgentHomeSnapshotService,
|
||||
"collect_retired_home_snapshot",
|
||||
lambda **_kwargs: calls.append("home"),
|
||||
)
|
||||
purge_calls: list[tuple[str, list[str]]] = []
|
||||
|
||||
def purge(*, tenant_id: str, agent_ids: list[str]) -> None:
|
||||
purge_calls.append((tenant_id, agent_ids))
|
||||
calls.append("purge")
|
||||
|
||||
monkeypatch.setattr(AgentDeletionService, "purge_archived_agents", purge)
|
||||
|
||||
collect_agent_resources.run(
|
||||
tenant_id="tenant-1",
|
||||
workspace_ids=["workspace-1"],
|
||||
binding_ids=["binding-1"],
|
||||
home_snapshot_ids=["home-1"],
|
||||
purge_agent_ids=["agent-1"],
|
||||
)
|
||||
|
||||
assert calls == ["workspace", "binding", "home", "purge"]
|
||||
assert purge_calls == [("tenant-1", ["agent-1"])]
|
||||
|
||||
|
||||
def test_collection_failure_propagates_after_attempting_remaining_resources(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
calls: list[str] = []
|
||||
first_error = RuntimeError("workspace-1 failed")
|
||||
errors = {
|
||||
"workspace-1": first_error,
|
||||
"workspace-2": RuntimeError("workspace-2 failed"),
|
||||
"binding-1": RuntimeError("binding-1 failed"),
|
||||
"home-2": RuntimeError("home-2 failed"),
|
||||
}
|
||||
logged_exceptions: list[tuple[str, dict[str, object]]] = []
|
||||
|
||||
def log_exception(message: str, *, extra: dict[str, object]) -> None:
|
||||
logged_exceptions.append((message, extra))
|
||||
|
||||
def collect_workspace(*, workspace_id: str, **_kwargs: object) -> None:
|
||||
calls.append(f"workspace:{workspace_id}")
|
||||
if error := errors.get(workspace_id):
|
||||
raise error
|
||||
|
||||
def collect_binding(*, binding_id: str, **_kwargs: object) -> None:
|
||||
calls.append(f"binding:{binding_id}")
|
||||
if error := errors.get(binding_id):
|
||||
raise error
|
||||
|
||||
def collect_home(*, home_snapshot_id: str, **_kwargs: object) -> None:
|
||||
calls.append(f"home:{home_snapshot_id}")
|
||||
if error := errors.get(home_snapshot_id):
|
||||
raise error
|
||||
|
||||
monkeypatch.setattr(AgentWorkspaceService, "collect_retired_workspace", collect_workspace)
|
||||
monkeypatch.setattr(AgentWorkspaceService, "collect_retired_binding", collect_binding)
|
||||
monkeypatch.setattr(AgentHomeSnapshotService, "collect_retired_home_snapshot", collect_home)
|
||||
monkeypatch.setattr("tasks.collect_agent_resources_task.logger.exception", log_exception)
|
||||
purge_calls: list[tuple[str, list[str]]] = []
|
||||
|
||||
def purge(*, tenant_id: str, agent_ids: list[str]) -> None:
|
||||
purge_calls.append((tenant_id, agent_ids))
|
||||
|
||||
monkeypatch.setattr(AgentDeletionService, "purge_archived_agents", purge)
|
||||
|
||||
with pytest.raises(RuntimeError) as exc_info:
|
||||
collect_agent_resources.run(
|
||||
tenant_id="tenant-1",
|
||||
workspace_ids=["workspace-1", "workspace-2", "workspace-3"],
|
||||
binding_ids=["binding-1", "binding-2"],
|
||||
home_snapshot_ids=["home-1", "home-2", "home-3"],
|
||||
purge_agent_ids=["agent-1"],
|
||||
)
|
||||
|
||||
assert exc_info.value.__cause__ is first_error
|
||||
assert str(exc_info.value) == (
|
||||
"Failed to collect 4 retired Agent resource(s): "
|
||||
"workspace:workspace-1, workspace:workspace-2, binding:binding-1, home_snapshot:home-2"
|
||||
)
|
||||
assert calls == [
|
||||
"workspace:workspace-1",
|
||||
"workspace:workspace-2",
|
||||
"workspace:workspace-3",
|
||||
"binding:binding-1",
|
||||
"binding:binding-2",
|
||||
"home:home-1",
|
||||
"home:home-2",
|
||||
"home:home-3",
|
||||
]
|
||||
assert purge_calls == []
|
||||
assert logged_exceptions == [
|
||||
(
|
||||
"Failed to collect retired Agent resource",
|
||||
{
|
||||
"tenant_id": "tenant-1",
|
||||
"resource_type": resource_type,
|
||||
"resource_id": resource_id,
|
||||
},
|
||||
)
|
||||
for resource_type, resource_id in (
|
||||
("workspace", "workspace-1"),
|
||||
("workspace", "workspace-2"),
|
||||
("binding", "binding-1"),
|
||||
("home_snapshot", "home-2"),
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_enqueue_failure_propagates(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
error = RuntimeError("queue unavailable")
|
||||
delay_calls: list[dict[str, object]] = []
|
||||
logged_exceptions: list[tuple[str, dict[str, object]]] = []
|
||||
|
||||
def delay(**kwargs: object) -> None:
|
||||
delay_calls.append(kwargs)
|
||||
raise error
|
||||
|
||||
def log_exception(message: str, *, extra: dict[str, object]) -> None:
|
||||
logged_exceptions.append((message, extra))
|
||||
|
||||
monkeypatch.setattr(collect_agent_resources, "delay", delay)
|
||||
monkeypatch.setattr("tasks.collect_agent_resources_task.logger.exception", log_exception)
|
||||
|
||||
with pytest.raises(RuntimeError) as exc_info:
|
||||
enqueue_agent_resource_collection(
|
||||
tenant_id="tenant-1",
|
||||
binding_ids=["binding-1"],
|
||||
workspace_ids=["workspace-1"],
|
||||
home_snapshot_ids=["home-1"],
|
||||
purge_agent_ids=["agent-2", "agent-1", "agent-2"],
|
||||
)
|
||||
|
||||
assert exc_info.value is error
|
||||
payload = {
|
||||
"binding_ids": ["binding-1"],
|
||||
"workspace_ids": ["workspace-1"],
|
||||
"home_snapshot_ids": ["home-1"],
|
||||
"purge_agent_ids": ["agent-1", "agent-2"],
|
||||
}
|
||||
assert delay_calls == [{"tenant_id": "tenant-1", **payload}]
|
||||
assert logged_exceptions == [
|
||||
(
|
||||
"Failed to enqueue retired Agent resource collection",
|
||||
{"tenant_id": "tenant-1", **payload},
|
||||
)
|
||||
]
|
||||
@@ -0,0 +1,50 @@
|
||||
import logging
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.engine import Engine
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from tasks import community_telemetry_task
|
||||
|
||||
|
||||
def _bind_task_to_sqlite(monkeypatch: pytest.MonkeyPatch, sqlite_engine: Engine) -> None:
|
||||
"""Bind the task-local sessionmaker to the isolated SQLite database."""
|
||||
monkeypatch.setattr(community_telemetry_task, "db", SimpleNamespace(engine=sqlite_engine))
|
||||
|
||||
|
||||
def test_send_community_telemetry_heartbeat_reports_with_a_database_session(
|
||||
monkeypatch: pytest.MonkeyPatch, sqlite_engine: Engine
|
||||
) -> None:
|
||||
_bind_task_to_sqlite(monkeypatch, sqlite_engine)
|
||||
received_sessions: list[Session] = []
|
||||
|
||||
def report_heartbeat(*, session: Session) -> None:
|
||||
received_sessions.append(session)
|
||||
assert session.get_bind() is sqlite_engine
|
||||
|
||||
monkeypatch.setattr(community_telemetry_task.CommunityTelemetryService, "report_heartbeat", report_heartbeat)
|
||||
|
||||
community_telemetry_task.send_community_telemetry_heartbeat.run()
|
||||
|
||||
assert len(received_sessions) == 1
|
||||
assert isinstance(received_sessions[0], Session)
|
||||
|
||||
|
||||
def test_send_community_telemetry_heartbeat_swallows_report_errors(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
sqlite_engine: Engine,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
_bind_task_to_sqlite(monkeypatch, sqlite_engine)
|
||||
monkeypatch.setattr(
|
||||
community_telemetry_task.CommunityTelemetryService,
|
||||
"report_heartbeat",
|
||||
Mock(side_effect=RuntimeError("telemetry unavailable")),
|
||||
)
|
||||
caplog.set_level(logging.DEBUG, logger=community_telemetry_task.logger.name)
|
||||
|
||||
community_telemetry_task.send_community_telemetry_heartbeat.run()
|
||||
|
||||
assert "Failed to process community telemetry heartbeat" in caplog.text
|
||||
@@ -0,0 +1,285 @@
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from decimal import Decimal
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from core.workflow.human_input_adapter import DeliveryMethodType
|
||||
from core.workflow.nodes.human_input.enums import HumanInputFormKind, HumanInputFormStatus
|
||||
from graphon.file import FileTransferMethod, FileType
|
||||
from models import (
|
||||
AgentDebugConversation,
|
||||
Conversation,
|
||||
ConversationVariable,
|
||||
HumanInputForm,
|
||||
HumanInputFormUploadFile,
|
||||
HumanInputFormUploadToken,
|
||||
Message,
|
||||
MessageAgentThought,
|
||||
MessageAnnotation,
|
||||
MessageChain,
|
||||
MessageFeedback,
|
||||
MessageFile,
|
||||
PinnedConversation,
|
||||
SavedMessage,
|
||||
)
|
||||
from models.agent import AgentConfigDraftType
|
||||
from models.enums import (
|
||||
ConversationFromSource,
|
||||
ConversationStatus,
|
||||
CreatorUserRole,
|
||||
FeedbackFromSource,
|
||||
FeedbackRating,
|
||||
InvokeFrom,
|
||||
MessageChainType,
|
||||
)
|
||||
from models.human_input import HumanInputDelivery, HumanInputFormRecipient, RecipientType
|
||||
from models.tools import ToolConversationVariables, ToolFile
|
||||
from tasks.delete_conversation_task import _cleanup_conversation_related_data, sweep_deleted_conversations
|
||||
from tests.helpers.unit.model_factories import make_conversation, make_message
|
||||
|
||||
TENANT_ID = "11111111-1111-1111-1111-111111111111"
|
||||
APP_ID = "22222222-2222-2222-2222-222222222222"
|
||||
ACCOUNT_ID = "33333333-3333-3333-3333-333333333333"
|
||||
CONVERSATION_ID = "44444444-4444-4444-4444-444444444444"
|
||||
OTHER_CONVERSATION_ID = "55555555-5555-5555-5555-555555555555"
|
||||
MESSAGE_ID = "66666666-6666-6666-6666-666666666666"
|
||||
AGENT_ID = "77777777-7777-7777-7777-777777777777"
|
||||
|
||||
|
||||
def _conversation(conversation_id: str, *, deleted: bool) -> Conversation:
|
||||
return make_conversation(
|
||||
conversation_id=conversation_id,
|
||||
app_id=APP_ID,
|
||||
name="Test conversation",
|
||||
inputs={},
|
||||
status=ConversationStatus.NORMAL,
|
||||
from_source=ConversationFromSource.CONSOLE,
|
||||
from_account_id=ACCOUNT_ID,
|
||||
is_deleted=deleted,
|
||||
)
|
||||
|
||||
|
||||
def _message() -> Message:
|
||||
return make_message(
|
||||
message_id=MESSAGE_ID,
|
||||
app_id=APP_ID,
|
||||
conversation_id=CONVERSATION_ID,
|
||||
inputs={},
|
||||
query="hello",
|
||||
message={"role": "user", "content": "hello"},
|
||||
answer="world",
|
||||
message_unit_price=Decimal(0),
|
||||
answer_unit_price=Decimal(0),
|
||||
currency="USD",
|
||||
invoke_from=InvokeFrom.WEB_APP,
|
||||
from_source=ConversationFromSource.CONSOLE,
|
||||
from_account_id=ACCOUNT_ID,
|
||||
)
|
||||
|
||||
|
||||
def _tool_file(*, name: str, conversation_id: str | None = CONVERSATION_ID) -> ToolFile:
|
||||
return ToolFile(
|
||||
user_id=ACCOUNT_ID,
|
||||
tenant_id=TENANT_ID,
|
||||
conversation_id=conversation_id,
|
||||
file_key=f"tools/{TENANT_ID}/{name}",
|
||||
mimetype="text/plain",
|
||||
name=name,
|
||||
size=5,
|
||||
)
|
||||
|
||||
|
||||
def test_cleanup_removes_owned_resources(sqlite_session: Session) -> None:
|
||||
conversation = _conversation(CONVERSATION_ID, deleted=True)
|
||||
other_conversation = _conversation(OTHER_CONVERSATION_ID, deleted=False)
|
||||
message = _message()
|
||||
owned_file = _tool_file(name="owned.txt")
|
||||
other_file = _tool_file(name="other.txt", conversation_id=OTHER_CONVERSATION_ID)
|
||||
sqlite_session.add_all([conversation, other_conversation, message, owned_file, other_file])
|
||||
sqlite_session.flush()
|
||||
|
||||
message_chain = MessageChain(message_id=MESSAGE_ID, type=MessageChainType.SYSTEM, input=None, output=None)
|
||||
form = HumanInputForm(
|
||||
tenant_id=TENANT_ID,
|
||||
app_id=APP_ID,
|
||||
workflow_run_id=None,
|
||||
conversation_id=CONVERSATION_ID,
|
||||
form_kind=HumanInputFormKind.RUNTIME,
|
||||
node_id="ask-human",
|
||||
form_definition="{}",
|
||||
rendered_content="form",
|
||||
status=HumanInputFormStatus.WAITING,
|
||||
expiration_time=datetime.now(UTC) + timedelta(hours=1),
|
||||
)
|
||||
sqlite_session.add_all([message_chain, form])
|
||||
sqlite_session.flush()
|
||||
|
||||
delivery = HumanInputDelivery(
|
||||
form_id=form.id,
|
||||
delivery_method_type=DeliveryMethodType.WEBAPP,
|
||||
delivery_config_id=None,
|
||||
channel_payload="{}",
|
||||
)
|
||||
sqlite_session.add(delivery)
|
||||
sqlite_session.flush()
|
||||
|
||||
upload_token = HumanInputFormUploadToken(
|
||||
tenant_id=TENANT_ID,
|
||||
app_id=APP_ID,
|
||||
form_id=form.id,
|
||||
recipient_id="88888888-8888-8888-8888-888888888888",
|
||||
token="upload-token",
|
||||
)
|
||||
sqlite_session.add(upload_token)
|
||||
sqlite_session.flush()
|
||||
|
||||
related_rows = [
|
||||
MessageAgentThought(
|
||||
message_id=MESSAGE_ID,
|
||||
position=1,
|
||||
created_by_role=CreatorUserRole.ACCOUNT,
|
||||
created_by=ACCOUNT_ID,
|
||||
message_chain_id=message_chain.id,
|
||||
),
|
||||
MessageFile(
|
||||
message_id=MESSAGE_ID,
|
||||
type=FileType.DOCUMENT,
|
||||
transfer_method=FileTransferMethod.REMOTE_URL,
|
||||
created_by_role=CreatorUserRole.ACCOUNT,
|
||||
created_by=ACCOUNT_ID,
|
||||
url="https://example.com/file.txt",
|
||||
),
|
||||
SavedMessage(
|
||||
app_id=APP_ID,
|
||||
message_id=MESSAGE_ID,
|
||||
created_by_role=CreatorUserRole.ACCOUNT,
|
||||
created_by=ACCOUNT_ID,
|
||||
),
|
||||
MessageAnnotation(
|
||||
app_id=APP_ID,
|
||||
question="hello",
|
||||
content="world",
|
||||
account_id=ACCOUNT_ID,
|
||||
conversation_id=CONVERSATION_ID,
|
||||
message_id=MESSAGE_ID,
|
||||
),
|
||||
MessageFeedback(
|
||||
app_id=APP_ID,
|
||||
conversation_id=CONVERSATION_ID,
|
||||
message_id=MESSAGE_ID,
|
||||
rating=FeedbackRating.LIKE,
|
||||
from_source=FeedbackFromSource.ADMIN,
|
||||
from_account_id=ACCOUNT_ID,
|
||||
),
|
||||
ToolConversationVariables(
|
||||
user_id=ACCOUNT_ID,
|
||||
tenant_id=TENANT_ID,
|
||||
conversation_id=CONVERSATION_ID,
|
||||
variables_str="{}",
|
||||
),
|
||||
ConversationVariable(
|
||||
id="99999999-9999-9999-9999-999999999999",
|
||||
conversation_id=CONVERSATION_ID,
|
||||
app_id=APP_ID,
|
||||
data="{}",
|
||||
),
|
||||
PinnedConversation(
|
||||
app_id=APP_ID,
|
||||
conversation_id=CONVERSATION_ID,
|
||||
created_by_role=CreatorUserRole.ACCOUNT,
|
||||
created_by=ACCOUNT_ID,
|
||||
),
|
||||
AgentDebugConversation(
|
||||
tenant_id=TENANT_ID,
|
||||
agent_id=AGENT_ID,
|
||||
app_id=APP_ID,
|
||||
account_id=ACCOUNT_ID,
|
||||
draft_type=AgentConfigDraftType.DEBUG_BUILD,
|
||||
conversation_id=CONVERSATION_ID,
|
||||
),
|
||||
HumanInputFormRecipient(
|
||||
form_id=form.id,
|
||||
delivery_id=delivery.id,
|
||||
recipient_type=RecipientType.CONSOLE,
|
||||
recipient_payload="{}",
|
||||
),
|
||||
HumanInputFormUploadFile(
|
||||
tenant_id=TENANT_ID,
|
||||
app_id=APP_ID,
|
||||
form_id=form.id,
|
||||
upload_file_id="aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
|
||||
upload_token_id=upload_token.id,
|
||||
),
|
||||
]
|
||||
sqlite_session.add_all(related_rows)
|
||||
sqlite_session.commit()
|
||||
form_id = form.id
|
||||
owned_file_id = owned_file.id
|
||||
owned_file_key = owned_file.file_key
|
||||
other_file_id = other_file.id
|
||||
|
||||
with patch("tasks.delete_conversation_task.storage") as storage_mock:
|
||||
assert _cleanup_conversation_related_data(CONVERSATION_ID) is True
|
||||
|
||||
storage_mock.delete.assert_called_once_with(owned_file_key)
|
||||
sqlite_session.expire_all()
|
||||
assert sqlite_session.get(Conversation, CONVERSATION_ID) is None
|
||||
assert sqlite_session.get(Message, MESSAGE_ID) is None
|
||||
assert (
|
||||
sqlite_session.scalar(select(MessageAgentThought).where(MessageAgentThought.message_id == MESSAGE_ID)) is None
|
||||
)
|
||||
assert sqlite_session.scalar(select(HumanInputForm).where(HumanInputForm.id == form_id)) is None
|
||||
assert sqlite_session.get(ToolFile, owned_file_id) is None
|
||||
assert sqlite_session.get(ToolFile, other_file_id) is not None
|
||||
assert sqlite_session.get(Conversation, OTHER_CONVERSATION_ID) is not None
|
||||
|
||||
|
||||
def test_cleanup_storage_failure_retains_marker_and_file_key(sqlite_session: Session) -> None:
|
||||
conversation = _conversation(CONVERSATION_ID, deleted=True)
|
||||
tool_file = _tool_file(name="retry.txt")
|
||||
sqlite_session.add_all([conversation, tool_file])
|
||||
sqlite_session.commit()
|
||||
|
||||
with patch("tasks.delete_conversation_task.storage") as storage_mock:
|
||||
storage_mock.delete.side_effect = RuntimeError("storage unavailable")
|
||||
storage_mock.exists.return_value = True
|
||||
with pytest.raises(RuntimeError, match="storage unavailable"):
|
||||
_cleanup_conversation_related_data(CONVERSATION_ID)
|
||||
|
||||
sqlite_session.expire_all()
|
||||
persisted_conversation = sqlite_session.get(Conversation, CONVERSATION_ID)
|
||||
assert persisted_conversation is not None
|
||||
assert persisted_conversation.is_deleted is True
|
||||
assert sqlite_session.get(ToolFile, tool_file.id) is not None
|
||||
|
||||
|
||||
def test_cleanup_skips_active_conversation(sqlite_session: Session) -> None:
|
||||
conversation = _conversation(CONVERSATION_ID, deleted=False)
|
||||
tool_file = _tool_file(name="active.txt")
|
||||
sqlite_session.add_all([conversation, tool_file])
|
||||
sqlite_session.commit()
|
||||
|
||||
with patch("tasks.delete_conversation_task.storage") as storage_mock:
|
||||
assert _cleanup_conversation_related_data(CONVERSATION_ID) is False
|
||||
storage_mock.delete.assert_not_called()
|
||||
|
||||
assert sqlite_session.get(Conversation, CONVERSATION_ID) is not None
|
||||
assert sqlite_session.get(ToolFile, tool_file.id) is not None
|
||||
|
||||
|
||||
def test_sweeper_dispatches_only_soft_deleted_conversations(sqlite_session: Session) -> None:
|
||||
sqlite_session.add_all(
|
||||
[
|
||||
_conversation(CONVERSATION_ID, deleted=True),
|
||||
_conversation(OTHER_CONVERSATION_ID, deleted=False),
|
||||
]
|
||||
)
|
||||
sqlite_session.commit()
|
||||
|
||||
with patch("tasks.delete_conversation_task.delete_conversation_related_data.delay", MagicMock()) as delay:
|
||||
assert sweep_deleted_conversations.run() == 1
|
||||
|
||||
delay.assert_called_once_with(CONVERSATION_ID)
|
||||
@@ -0,0 +1,321 @@
|
||||
import uuid
|
||||
from collections.abc import Generator
|
||||
from contextlib import contextmanager
|
||||
from datetime import UTC, datetime
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import event
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from core.rag.index_processor.constant.index_type import IndexStructureType
|
||||
from extensions.storage.storage_type import StorageType
|
||||
from models.dataset import Dataset, Document, DocumentSegment, SegmentAttachmentBinding
|
||||
from models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, IndexingStatus, SegmentStatus
|
||||
from models.model import UploadFile
|
||||
from tasks.delete_segment_from_index_task import delete_segment_from_index_task
|
||||
from tasks.disable_segment_from_index_task import disable_segment_from_index_task
|
||||
from tasks.disable_segments_from_index_task import disable_segments_from_index_task
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def indexed_segment(sqlite_session: Session) -> tuple[Dataset, Document, DocumentSegment]:
|
||||
"""Persist the complete owner chain consumed by segment cleanup tasks."""
|
||||
tenant_id = str(uuid.uuid4())
|
||||
created_by = str(uuid.uuid4())
|
||||
dataset = Dataset(
|
||||
id=str(uuid.uuid4()),
|
||||
tenant_id=tenant_id,
|
||||
name="Cleanup dataset",
|
||||
data_source_type=DataSourceType.UPLOAD_FILE,
|
||||
created_by=created_by,
|
||||
is_multimodal=False,
|
||||
)
|
||||
document = Document(
|
||||
id=str(uuid.uuid4()),
|
||||
tenant_id=tenant_id,
|
||||
dataset_id=dataset.id,
|
||||
position=1,
|
||||
data_source_type=DataSourceType.UPLOAD_FILE,
|
||||
batch="batch-1",
|
||||
name="document.txt",
|
||||
created_from=DocumentCreatedFrom.WEB,
|
||||
created_by=created_by,
|
||||
indexing_status=IndexingStatus.COMPLETED,
|
||||
doc_form=IndexStructureType.PARAGRAPH_INDEX,
|
||||
)
|
||||
segment = DocumentSegment(
|
||||
tenant_id=tenant_id,
|
||||
dataset_id=dataset.id,
|
||||
document_id=document.id,
|
||||
position=1,
|
||||
content="content",
|
||||
word_count=1,
|
||||
tokens=1,
|
||||
created_by=created_by,
|
||||
index_node_id="node-1",
|
||||
index_node_hash="hash-1",
|
||||
disabled_by=created_by,
|
||||
status=SegmentStatus.COMPLETED,
|
||||
)
|
||||
sqlite_session.add_all([dataset, document, segment])
|
||||
sqlite_session.commit()
|
||||
return dataset, document, segment
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _record_transaction_events(
|
||||
sqlite_session_factory: sessionmaker[Session], phase_events: list[str]
|
||||
) -> Generator[None]:
|
||||
"""Record real commits made by the task-owned SQLite session."""
|
||||
session_type = sqlite_session_factory.class_
|
||||
|
||||
def after_commit(_session: Session) -> None:
|
||||
phase_events.append("commit")
|
||||
|
||||
event.listen(session_type, "after_commit", after_commit)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
event.remove(session_type, "after_commit", after_commit)
|
||||
|
||||
|
||||
def test_disable_segment_commits_index_cleanup(
|
||||
indexed_segment: tuple[Dataset, Document, DocumentSegment],
|
||||
sqlite_session_factory: sessionmaker[Session],
|
||||
) -> None:
|
||||
dataset, _document, segment = indexed_segment
|
||||
phase_events: list[str] = []
|
||||
processor = MagicMock()
|
||||
processor.clean.side_effect = lambda *_args, **_kwargs: phase_events.append("clean")
|
||||
disable_summaries = MagicMock(side_effect=lambda *_args, **_kwargs: phase_events.append("summary"))
|
||||
|
||||
with (
|
||||
_record_transaction_events(sqlite_session_factory, phase_events),
|
||||
patch("tasks.disable_segment_from_index_task.IndexProcessorFactory") as processor_factory,
|
||||
patch(
|
||||
"services.knowledge.summaries.adapters.SummaryIndexAdapter.disable_summaries_for_segments",
|
||||
disable_summaries,
|
||||
),
|
||||
patch("tasks.disable_segment_from_index_task.redis_client.delete"),
|
||||
):
|
||||
processor_factory.return_value.init_index_processor.return_value = processor
|
||||
disable_segment_from_index_task.run(segment.id)
|
||||
|
||||
assert phase_events == ["clean", "commit", "summary"]
|
||||
disable_summaries.assert_called_once()
|
||||
assert disable_summaries.call_args.kwargs["dataset"].id == dataset.id
|
||||
assert disable_summaries.call_args.kwargs["segment_ids"] == [segment.id]
|
||||
assert disable_summaries.call_args.kwargs["disabled_by"] == segment.disabled_by
|
||||
|
||||
|
||||
def test_disable_segments_commits_index_cleanup(
|
||||
indexed_segment: tuple[Dataset, Document, DocumentSegment],
|
||||
sqlite_session_factory: sessionmaker[Session],
|
||||
) -> None:
|
||||
dataset, document, segment = indexed_segment
|
||||
phase_events: list[str] = []
|
||||
processor = MagicMock()
|
||||
processor.clean.side_effect = lambda *_args, **_kwargs: phase_events.append("clean")
|
||||
disable_summaries = MagicMock(side_effect=lambda *_args, **_kwargs: phase_events.append("summary"))
|
||||
|
||||
with (
|
||||
_record_transaction_events(sqlite_session_factory, phase_events),
|
||||
patch("tasks.disable_segments_from_index_task.IndexProcessorFactory") as processor_factory,
|
||||
patch(
|
||||
"services.knowledge.summaries.adapters.SummaryIndexAdapter.disable_summaries_for_segments",
|
||||
disable_summaries,
|
||||
),
|
||||
patch("tasks.disable_segments_from_index_task.redis_client.delete"),
|
||||
):
|
||||
processor_factory.return_value.init_index_processor.return_value = processor
|
||||
disable_segments_from_index_task.run([segment.id], dataset.id, document.id)
|
||||
|
||||
assert phase_events == ["clean", "commit", "summary"]
|
||||
disable_summaries.assert_called_once()
|
||||
assert disable_summaries.call_args.kwargs["dataset"].id == dataset.id
|
||||
assert disable_summaries.call_args.kwargs["segment_ids"] == [segment.id]
|
||||
assert disable_summaries.call_args.kwargs["disabled_by"] == segment.disabled_by
|
||||
|
||||
|
||||
def test_delete_segment_commits_index_cleanup_without_attachments(
|
||||
indexed_segment: tuple[Dataset, Document, DocumentSegment],
|
||||
sqlite_session_factory: sessionmaker[Session],
|
||||
) -> None:
|
||||
dataset, document, segment = indexed_segment
|
||||
phase_events: list[str] = []
|
||||
processor = MagicMock()
|
||||
processor.clean.side_effect = lambda *_args, **_kwargs: phase_events.append("clean")
|
||||
|
||||
with (
|
||||
_record_transaction_events(sqlite_session_factory, phase_events),
|
||||
patch("tasks.delete_segment_from_index_task.IndexProcessorFactory") as processor_factory,
|
||||
):
|
||||
processor_factory.return_value.init_index_processor.return_value = processor
|
||||
delete_segment_from_index_task.run(["node-1"], dataset.id, document.id, [segment.id])
|
||||
|
||||
assert phase_events == ["clean", "commit"]
|
||||
|
||||
|
||||
def test_delete_segment_removes_attachment_blobs_from_storage(
|
||||
indexed_segment: tuple[Dataset, Document, DocumentSegment],
|
||||
sqlite_session: Session,
|
||||
) -> None:
|
||||
dataset, document, segment = indexed_segment
|
||||
dataset.is_multimodal = True
|
||||
attachment = UploadFile(
|
||||
tenant_id=dataset.tenant_id,
|
||||
storage_type=StorageType.LOCAL,
|
||||
key="attachments/segment-image.png",
|
||||
name="segment-image.png",
|
||||
size=10,
|
||||
extension="png",
|
||||
mime_type="image/png",
|
||||
created_by_role=CreatorUserRole.ACCOUNT,
|
||||
created_by=segment.created_by,
|
||||
created_at=datetime.now(UTC),
|
||||
used=True,
|
||||
)
|
||||
binding = SegmentAttachmentBinding(
|
||||
tenant_id=dataset.tenant_id,
|
||||
dataset_id=dataset.id,
|
||||
document_id=document.id,
|
||||
segment_id=segment.id,
|
||||
attachment_id=attachment.id,
|
||||
)
|
||||
sqlite_session.add_all([dataset, attachment, binding])
|
||||
sqlite_session.commit()
|
||||
attachment_id = attachment.id
|
||||
attachment_key = attachment.key
|
||||
binding_id = binding.id
|
||||
|
||||
with (
|
||||
patch("tasks.delete_segment_from_index_task.IndexProcessorFactory"),
|
||||
patch("tasks.delete_segment_from_index_task.storage.delete") as storage_delete,
|
||||
):
|
||||
delete_segment_from_index_task.run(["node-1"], dataset.id, document.id, [segment.id])
|
||||
|
||||
storage_delete.assert_called_once_with(attachment_key)
|
||||
sqlite_session.expire_all()
|
||||
assert sqlite_session.get(SegmentAttachmentBinding, binding_id) is None
|
||||
assert sqlite_session.get(UploadFile, attachment_id) is None
|
||||
|
||||
|
||||
def test_delete_segment_preserves_attachment_shared_by_another_segment(
|
||||
indexed_segment: tuple[Dataset, Document, DocumentSegment],
|
||||
sqlite_session: Session,
|
||||
) -> None:
|
||||
dataset, document, segment = indexed_segment
|
||||
dataset.is_multimodal = True
|
||||
other_segment = DocumentSegment(
|
||||
tenant_id=dataset.tenant_id,
|
||||
dataset_id=dataset.id,
|
||||
document_id=document.id,
|
||||
position=2,
|
||||
content="other content",
|
||||
word_count=1,
|
||||
tokens=1,
|
||||
created_by=segment.created_by,
|
||||
index_node_id="node-2",
|
||||
index_node_hash="hash-2",
|
||||
disabled_by=segment.disabled_by,
|
||||
status=SegmentStatus.COMPLETED,
|
||||
)
|
||||
attachment = UploadFile(
|
||||
tenant_id=dataset.tenant_id,
|
||||
storage_type=StorageType.LOCAL,
|
||||
key="attachments/shared-segment-image.png",
|
||||
name="shared-segment-image.png",
|
||||
size=10,
|
||||
extension="png",
|
||||
mime_type="image/png",
|
||||
created_by_role=CreatorUserRole.ACCOUNT,
|
||||
created_by=segment.created_by,
|
||||
created_at=datetime.now(UTC),
|
||||
used=True,
|
||||
)
|
||||
binding = SegmentAttachmentBinding(
|
||||
tenant_id=dataset.tenant_id,
|
||||
dataset_id=dataset.id,
|
||||
document_id=document.id,
|
||||
segment_id=segment.id,
|
||||
attachment_id=attachment.id,
|
||||
)
|
||||
shared_binding = SegmentAttachmentBinding(
|
||||
tenant_id=dataset.tenant_id,
|
||||
dataset_id=dataset.id,
|
||||
document_id=document.id,
|
||||
segment_id=other_segment.id,
|
||||
attachment_id=attachment.id,
|
||||
)
|
||||
sqlite_session.add_all([dataset, other_segment, attachment, binding, shared_binding])
|
||||
sqlite_session.commit()
|
||||
attachment_id = attachment.id
|
||||
binding_id = binding.id
|
||||
shared_binding_id = shared_binding.id
|
||||
|
||||
with (
|
||||
patch("tasks.delete_segment_from_index_task.IndexProcessorFactory") as processor_factory,
|
||||
patch("tasks.delete_segment_from_index_task.storage.delete") as storage_delete,
|
||||
):
|
||||
delete_segment_from_index_task.run(["node-1"], dataset.id, document.id, [segment.id])
|
||||
|
||||
processor = processor_factory.return_value.init_index_processor.return_value
|
||||
assert processor.clean.call_count == 1
|
||||
assert processor.clean.call_args.args[1] == ["node-1"]
|
||||
assert processor.clean.call_args.kwargs["with_keywords"] is True
|
||||
storage_delete.assert_not_called()
|
||||
sqlite_session.expire_all()
|
||||
assert sqlite_session.get(SegmentAttachmentBinding, binding_id) is None
|
||||
assert sqlite_session.get(SegmentAttachmentBinding, shared_binding_id) is not None
|
||||
assert sqlite_session.get(UploadFile, attachment_id) is not None
|
||||
|
||||
|
||||
def test_delete_segment_keeps_database_cleanup_when_attachment_storage_delete_fails(
|
||||
indexed_segment: tuple[Dataset, Document, DocumentSegment],
|
||||
sqlite_session: Session,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
dataset, document, segment = indexed_segment
|
||||
dataset.is_multimodal = True
|
||||
attachment = UploadFile(
|
||||
tenant_id=dataset.tenant_id,
|
||||
storage_type=StorageType.LOCAL,
|
||||
key="attachments/failing-segment-image.png",
|
||||
name="failing-segment-image.png",
|
||||
size=10,
|
||||
extension="png",
|
||||
mime_type="image/png",
|
||||
created_by_role=CreatorUserRole.ACCOUNT,
|
||||
created_by=segment.created_by,
|
||||
created_at=datetime.now(UTC),
|
||||
used=True,
|
||||
)
|
||||
binding = SegmentAttachmentBinding(
|
||||
tenant_id=dataset.tenant_id,
|
||||
dataset_id=dataset.id,
|
||||
document_id=document.id,
|
||||
segment_id=segment.id,
|
||||
attachment_id=attachment.id,
|
||||
)
|
||||
sqlite_session.add_all([dataset, attachment, binding])
|
||||
sqlite_session.commit()
|
||||
attachment_id = attachment.id
|
||||
attachment_key = attachment.key
|
||||
binding_id = binding.id
|
||||
|
||||
with (
|
||||
patch("tasks.delete_segment_from_index_task.IndexProcessorFactory"),
|
||||
patch(
|
||||
"tasks.delete_segment_from_index_task.storage.delete",
|
||||
side_effect=RuntimeError("storage unavailable"),
|
||||
) as storage_delete,
|
||||
caplog.at_level("ERROR", logger="tasks.delete_segment_from_index_task"),
|
||||
):
|
||||
delete_segment_from_index_task.run(["node-1"], dataset.id, document.id, [segment.id])
|
||||
|
||||
storage_delete.assert_called_once_with(attachment_key)
|
||||
assert "Failed to delete segment attachment from storage" in caplog.text
|
||||
sqlite_session.expire_all()
|
||||
assert sqlite_session.get(SegmentAttachmentBinding, binding_id) is None
|
||||
assert sqlite_session.get(UploadFile, attachment_id) is None
|
||||
@@ -0,0 +1,284 @@
|
||||
"""Unit tests for document sync persistence and external collaborator wiring.
|
||||
|
||||
The task's Dataset and Document lookups use real SQLite transactions. Datasource,
|
||||
Notion extraction, index cleanup, and indexing remain mocked I/O boundaries.
|
||||
"""
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from unittest.mock import ANY, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from core.db import session_factory as session_factory_module
|
||||
from core.rag.index_processor.constant.index_type import IndexStructureType
|
||||
from models.dataset import Dataset, Document, DocumentSegment
|
||||
from models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus
|
||||
from tasks.document_indexing_sync_task import document_indexing_sync_task
|
||||
|
||||
pytestmark = pytest.mark.parametrize(
|
||||
"sqlite_session",
|
||||
[(Dataset, Document, DocumentSegment)],
|
||||
indirect=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dataset_id() -> str:
|
||||
"""Generate a dataset id."""
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def document_id() -> str:
|
||||
"""Generate a document id."""
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def notion_workspace_id() -> str:
|
||||
"""Generate a notion workspace id."""
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def notion_page_id() -> str:
|
||||
"""Generate a notion page id."""
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def credential_id() -> str:
|
||||
"""Generate a credential id."""
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tenant_id() -> str:
|
||||
"""Generate the tenant that owns the persisted dataset and document."""
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dataset(sqlite_session: Session, dataset_id: str, tenant_id: str) -> Dataset:
|
||||
"""Persist the dataset resolved by the task's initial and cleanup transactions."""
|
||||
dataset = Dataset(
|
||||
id=dataset_id,
|
||||
tenant_id=tenant_id,
|
||||
name="Notion dataset",
|
||||
data_source_type=DataSourceType.NOTION_IMPORT,
|
||||
created_by=str(uuid.uuid4()),
|
||||
)
|
||||
sqlite_session.add(dataset)
|
||||
sqlite_session.commit()
|
||||
return dataset
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def document(
|
||||
sqlite_session: Session,
|
||||
dataset: Dataset,
|
||||
document_id: str,
|
||||
tenant_id: str,
|
||||
notion_workspace_id: str,
|
||||
notion_page_id: str,
|
||||
credential_id: str,
|
||||
) -> Document:
|
||||
"""Persist a completed Notion document for collaborator and update assertions."""
|
||||
document = Document(
|
||||
id=document_id,
|
||||
tenant_id=tenant_id,
|
||||
dataset_id=dataset.id,
|
||||
position=1,
|
||||
data_source_type=DataSourceType.NOTION_IMPORT,
|
||||
data_source_info=json.dumps(
|
||||
{
|
||||
"notion_workspace_id": notion_workspace_id,
|
||||
"notion_page_id": notion_page_id,
|
||||
"type": "page",
|
||||
"last_edited_time": "2024-01-01T00:00:00Z",
|
||||
"credential_id": credential_id,
|
||||
}
|
||||
),
|
||||
batch="batch-1",
|
||||
name="Notion page",
|
||||
created_from=DocumentCreatedFrom.API,
|
||||
created_by=str(uuid.uuid4()),
|
||||
indexing_status=IndexingStatus.COMPLETED,
|
||||
doc_form=IndexStructureType.PARAGRAPH_INDEX,
|
||||
)
|
||||
sqlite_session.add(document)
|
||||
sqlite_session.commit()
|
||||
return document
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _bind_sqlite_session_factory(monkeypatch: pytest.MonkeyPatch, sqlite_session: Session) -> None:
|
||||
"""Bind each task-owned session to the test's isolated SQLite database."""
|
||||
monkeypatch.setattr(
|
||||
session_factory_module,
|
||||
"_session_maker",
|
||||
sessionmaker(bind=sqlite_session.get_bind(), expire_on_commit=False),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_datasource_provider_service():
|
||||
"""Mock datasource credential provider."""
|
||||
with patch("tasks.document_indexing_sync_task.build_data_source_credentials", autospec=True) as mock_service_class:
|
||||
mock_service = MagicMock()
|
||||
mock_service.get_datasource_credentials.return_value = {"integration_secret": "test_token"}
|
||||
mock_service_class.return_value.providers = mock_service
|
||||
yield mock_service
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_notion_extractor():
|
||||
"""Mock notion extractor class and instance."""
|
||||
with patch("tasks.document_indexing_sync_task.NotionExtractor", autospec=True) as mock_extractor_class:
|
||||
mock_extractor = MagicMock()
|
||||
mock_extractor.get_notion_last_edited_time.return_value = "2024-01-01T00:00:00Z"
|
||||
mock_extractor_class.return_value = mock_extractor
|
||||
yield {"class": mock_extractor_class, "instance": mock_extractor}
|
||||
|
||||
|
||||
class TestDocumentIndexingSyncTaskCollaboratorParams:
|
||||
"""Unit tests for collaborator parameter passing in document_indexing_sync_task."""
|
||||
|
||||
def test_notion_extractor_initialized_with_correct_params(
|
||||
self,
|
||||
mock_datasource_provider_service,
|
||||
mock_notion_extractor,
|
||||
document: Document,
|
||||
dataset_id: str,
|
||||
document_id: str,
|
||||
notion_workspace_id: str,
|
||||
notion_page_id: str,
|
||||
):
|
||||
"""Test that NotionExtractor is initialized with expected arguments."""
|
||||
# Arrange
|
||||
expected_token = "test_token"
|
||||
|
||||
# Act
|
||||
document_indexing_sync_task(dataset_id, document_id)
|
||||
|
||||
# Assert
|
||||
mock_notion_extractor["class"].assert_called_once_with(
|
||||
notion_token_loader=ANY,
|
||||
notion_workspace_id=notion_workspace_id,
|
||||
notion_obj_id=notion_page_id,
|
||||
notion_page_type="page",
|
||||
notion_access_token=expected_token,
|
||||
tenant_id=document.tenant_id,
|
||||
)
|
||||
|
||||
def test_datasource_credentials_requested_correctly(
|
||||
self,
|
||||
mock_datasource_provider_service,
|
||||
mock_notion_extractor,
|
||||
document: Document,
|
||||
dataset_id: str,
|
||||
document_id: str,
|
||||
credential_id: str,
|
||||
):
|
||||
"""Test that datasource credentials are requested with expected identifiers."""
|
||||
# Arrange
|
||||
expected_tenant_id = document.tenant_id
|
||||
|
||||
# Act
|
||||
document_indexing_sync_task(dataset_id, document_id)
|
||||
|
||||
# Assert
|
||||
mock_datasource_provider_service.get_datasource_credentials.assert_called_once_with(
|
||||
tenant_id=expected_tenant_id,
|
||||
credential_id=credential_id,
|
||||
provider="notion_datasource",
|
||||
plugin_id="langgenius/notion_datasource",
|
||||
)
|
||||
|
||||
def test_credential_id_missing_uses_none(
|
||||
self,
|
||||
mock_datasource_provider_service,
|
||||
mock_notion_extractor,
|
||||
document: Document,
|
||||
sqlite_session: Session,
|
||||
dataset_id: str,
|
||||
document_id: str,
|
||||
):
|
||||
"""Test that missing credential_id is forwarded as None."""
|
||||
# Arrange
|
||||
document.data_source_info = json.dumps(
|
||||
{
|
||||
"notion_workspace_id": "workspace-id",
|
||||
"notion_page_id": "page-id",
|
||||
"type": "page",
|
||||
"last_edited_time": "2024-01-01T00:00:00Z",
|
||||
}
|
||||
)
|
||||
sqlite_session.commit()
|
||||
|
||||
# Act
|
||||
document_indexing_sync_task(dataset_id, document_id)
|
||||
|
||||
# Assert
|
||||
mock_datasource_provider_service.get_datasource_credentials.assert_called_once_with(
|
||||
tenant_id=document.tenant_id,
|
||||
credential_id=None,
|
||||
provider="notion_datasource",
|
||||
plugin_id="langgenius/notion_datasource",
|
||||
)
|
||||
|
||||
|
||||
class TestDataSourceInfoSerialization:
|
||||
"""Regression test: data_source_info must be written as a JSON string, not a raw dict.
|
||||
|
||||
See https://github.com/langgenius/dify/issues/32705
|
||||
psycopg2 raises ``ProgrammingError: can't adapt type 'dict'`` when a Python
|
||||
dict is passed directly to a text/LongText column.
|
||||
"""
|
||||
|
||||
def test_data_source_info_serialized_as_json_string(
|
||||
self,
|
||||
document: Document,
|
||||
sqlite_session: Session,
|
||||
dataset_id: str,
|
||||
document_id: str,
|
||||
):
|
||||
"""data_source_info must be serialized with json.dumps before DB write."""
|
||||
with (
|
||||
patch("tasks.document_indexing_sync_task.build_data_source_credentials") as mock_service_class,
|
||||
patch("tasks.document_indexing_sync_task.NotionExtractor") as mock_extractor_class,
|
||||
patch("tasks.document_indexing_sync_task.IndexProcessorFactory") as mock_ipf,
|
||||
patch("tasks.document_indexing_sync_task.build_document_indexing_service") as mock_runner_class,
|
||||
):
|
||||
# External collaborators
|
||||
mock_service = MagicMock()
|
||||
mock_service.get_datasource_credentials.return_value = {"integration_secret": "token"}
|
||||
mock_service_class.return_value.providers = mock_service
|
||||
|
||||
mock_extractor = MagicMock()
|
||||
# Return a *different* timestamp so the task enters the sync/update branch
|
||||
mock_extractor.get_notion_last_edited_time.return_value = "2024-02-01T00:00:00Z"
|
||||
mock_extractor_class.return_value = mock_extractor
|
||||
|
||||
mock_ip = MagicMock()
|
||||
mock_ipf.return_value.init_index_processor.return_value = mock_ip
|
||||
|
||||
mock_runner = MagicMock()
|
||||
mock_runner_class.return_value = mock_runner
|
||||
|
||||
# Act
|
||||
document_indexing_sync_task(dataset_id, document_id)
|
||||
|
||||
# Assert: data_source_info must be a JSON *string*, not a dict
|
||||
sqlite_session.expire_all()
|
||||
stored_document = sqlite_session.get(Document, document.id)
|
||||
assert stored_document is not None
|
||||
assert isinstance(stored_document.data_source_info, str), (
|
||||
f"data_source_info should be a JSON string, got {type(stored_document.data_source_info).__name__}"
|
||||
)
|
||||
parsed = json.loads(stored_document.data_source_info)
|
||||
assert parsed["last_edited_time"] == "2024-02-01T00:00:00Z"
|
||||
assert stored_document.indexing_status == IndexingStatus.PARSING
|
||||
assert stored_document.processing_started_at is not None
|
||||
@@ -0,0 +1,605 @@
|
||||
"""SQLite-backed tests for document indexing tasks.
|
||||
|
||||
The indexing task deliberately uses separate transactions for validation,
|
||||
status persistence, indexing, and summary dispatch. These tests persist real
|
||||
ORM rows so each phase observes only committed database state.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import ANY, MagicMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType
|
||||
from enums import CloudPlan, DeploymentEdition
|
||||
from extensions.ext_redis import redis_client
|
||||
from models.dataset import Dataset, Document
|
||||
from models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus
|
||||
from services.document_indexing_proxy.document_indexing_task_proxy import DocumentIndexingTaskProxy
|
||||
from services.knowledge.indexing.errors import DocumentIsPausedError
|
||||
from services.knowledge.resource_scope import DocumentRef
|
||||
from tasks.document_indexing_task import (
|
||||
_document_indexing,
|
||||
_document_indexing_with_tenant_queue,
|
||||
document_indexing_task,
|
||||
normal_document_indexing_task,
|
||||
priority_document_indexing_task,
|
||||
)
|
||||
from tasks.recover_document_indexing_task import recover_document_indexing_task
|
||||
from tests.helpers.unit.config_override import apply_config_overrides, config_overrides_context
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("status", "entry"),
|
||||
[
|
||||
("waiting", "run"),
|
||||
("parsing", "run"),
|
||||
("cleaning", "run"),
|
||||
("splitting", "run_in_splitting_status"),
|
||||
("indexing", "run_in_indexing_status"),
|
||||
("completed", None),
|
||||
],
|
||||
)
|
||||
def test_recovery_dispatches_detached_owner_reference(
|
||||
sqlite_session, tenant_id, dataset_id, document_ids, status, entry
|
||||
):
|
||||
_persist_indexing_rows(sqlite_session, tenant_id=tenant_id, dataset_id=dataset_id, document_ids=document_ids[:1])
|
||||
row = sqlite_session.get(Document, document_ids[0])
|
||||
row.indexing_status = status
|
||||
sqlite_session.commit()
|
||||
with patch("tasks.recover_document_indexing_task.build_document_indexing_service") as factory:
|
||||
recover_document_indexing_task.run(dataset_id, document_ids[0])
|
||||
service = factory.return_value
|
||||
if entry is None:
|
||||
assert service.mock_calls == []
|
||||
else:
|
||||
method = {
|
||||
"run": service.run,
|
||||
"run_in_splitting_status": service.run_in_splitting_status,
|
||||
"run_in_indexing_status": service.run_in_indexing_status,
|
||||
}[entry]
|
||||
method.assert_called_once()
|
||||
(argument,) = method.call_args.args
|
||||
ref = argument[0] if entry == "run" else argument
|
||||
assert isinstance(ref, DocumentRef)
|
||||
assert (ref.dataset.tenant_id, ref.dataset.dataset_id, ref.document_id) == (
|
||||
tenant_id,
|
||||
dataset_id,
|
||||
document_ids[0],
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tenant_id() -> str:
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dataset_id() -> str:
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def document_ids() -> list[str]:
|
||||
return [str(uuid.uuid4()) for _ in range(3)]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_redis() -> MagicMock:
|
||||
"""Reset the external Redis boundary used by tenant-isolated queues."""
|
||||
redis_client.reset_mock()
|
||||
redis_client.get.return_value = None
|
||||
redis_client.setex.return_value = True
|
||||
redis_client.delete.return_value = True
|
||||
redis_client.lpush.return_value = 1
|
||||
redis_client.rpop.return_value = None
|
||||
return redis_client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def indexing_runner(monkeypatch: pytest.MonkeyPatch) -> MagicMock:
|
||||
runner = MagicMock()
|
||||
runner_class = MagicMock(return_value=runner)
|
||||
monkeypatch.setattr("tasks.document_indexing_task.build_document_indexing_service", runner_class)
|
||||
runner._constructor_mock = runner_class
|
||||
return runner
|
||||
|
||||
|
||||
def _features(
|
||||
*,
|
||||
plan: CloudPlan = CloudPlan.PROFESSIONAL,
|
||||
vector_limit: int = 1000,
|
||||
vector_size: int = 0,
|
||||
) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
billing=SimpleNamespace(subscription=SimpleNamespace(plan=plan)),
|
||||
vector_space=SimpleNamespace(limit=vector_limit, size=vector_size),
|
||||
)
|
||||
|
||||
|
||||
def _patch_features(monkeypatch: pytest.MonkeyPatch, features: SimpleNamespace) -> MagicMock:
|
||||
get_features = MagicMock(return_value=features)
|
||||
monkeypatch.setattr("tasks.document_indexing_task.FeatureService.get_features", get_features)
|
||||
return get_features
|
||||
|
||||
|
||||
def _persist_indexing_rows(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
dataset_id: str,
|
||||
document_ids: list[str],
|
||||
indexing_technique: IndexTechniqueType = IndexTechniqueType.HIGH_QUALITY,
|
||||
summary_index_setting: dict[str, bool] | None = None,
|
||||
document_forms: list[IndexStructureType] | None = None,
|
||||
need_summary: list[bool] | None = None,
|
||||
) -> tuple[Dataset, list[Document]]:
|
||||
"""Persist one tenant-owned dataset and the requested document rows."""
|
||||
created_by = str(uuid.uuid4())
|
||||
dataset = Dataset(
|
||||
id=dataset_id,
|
||||
tenant_id=tenant_id,
|
||||
name="Indexing dataset",
|
||||
data_source_type=DataSourceType.UPLOAD_FILE,
|
||||
indexing_technique=indexing_technique,
|
||||
embedding_model_provider="openai",
|
||||
embedding_model="text-embedding-3-small",
|
||||
summary_index_setting=summary_index_setting,
|
||||
created_by=created_by,
|
||||
)
|
||||
documents = [
|
||||
Document(
|
||||
id=document_id,
|
||||
tenant_id=tenant_id,
|
||||
dataset_id=dataset_id,
|
||||
position=position,
|
||||
data_source_type=DataSourceType.UPLOAD_FILE,
|
||||
batch="batch-1",
|
||||
name=f"document-{position}.txt",
|
||||
created_from=DocumentCreatedFrom.WEB,
|
||||
created_by=created_by,
|
||||
indexing_status=IndexingStatus.WAITING,
|
||||
doc_form=(document_forms or [IndexStructureType.PARAGRAPH_INDEX] * len(document_ids))[position - 1],
|
||||
need_summary=(need_summary or [False] * len(document_ids))[position - 1],
|
||||
)
|
||||
for position, document_id in enumerate(document_ids, start=1)
|
||||
]
|
||||
session.add_all([dataset, *documents])
|
||||
session.commit()
|
||||
return dataset, documents
|
||||
|
||||
|
||||
def _persisted_documents(session: Session, document_ids: list[str]) -> list[Document]:
|
||||
session.expire_all()
|
||||
return [document for document_id in document_ids if (document := session.get(Document, document_id)) is not None]
|
||||
|
||||
|
||||
class TestTaskEnqueuing:
|
||||
@config_overrides_context(DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY)
|
||||
def test_self_hosted_dispatches_directly_to_priority_task(
|
||||
self, tenant_id: str, dataset_id: str, document_ids: list[str], mock_redis: MagicMock
|
||||
) -> None:
|
||||
with (
|
||||
patch.object(DocumentIndexingTaskProxy, "features"),
|
||||
patch.object(DocumentIndexingTaskProxy, "PRIORITY_TASK_FUNC", Mock()) as task,
|
||||
):
|
||||
DocumentIndexingTaskProxy(tenant_id, dataset_id, document_ids).delay()
|
||||
|
||||
task.delay.assert_called_once_with(
|
||||
tenant_id=tenant_id,
|
||||
dataset_id=dataset_id,
|
||||
document_ids=document_ids,
|
||||
)
|
||||
|
||||
@config_overrides_context(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD)
|
||||
@pytest.mark.parametrize(
|
||||
("plan", "task_attribute"),
|
||||
[
|
||||
(CloudPlan.SANDBOX, "NORMAL_TASK_FUNC"),
|
||||
(CloudPlan.PROFESSIONAL, "PRIORITY_TASK_FUNC"),
|
||||
],
|
||||
)
|
||||
def test_cloud_dispatches_first_task_through_tenant_queue(
|
||||
self,
|
||||
tenant_id: str,
|
||||
dataset_id: str,
|
||||
document_ids: list[str],
|
||||
mock_redis: MagicMock,
|
||||
plan: CloudPlan,
|
||||
task_attribute: str,
|
||||
) -> None:
|
||||
with (
|
||||
patch.object(DocumentIndexingTaskProxy, "features") as features,
|
||||
patch.object(DocumentIndexingTaskProxy, task_attribute, Mock()) as task,
|
||||
):
|
||||
features.billing.subscription.plan = plan
|
||||
DocumentIndexingTaskProxy(tenant_id, dataset_id, document_ids).delay()
|
||||
|
||||
mock_redis.setex.assert_called()
|
||||
task.delay.assert_called_once()
|
||||
|
||||
@config_overrides_context(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD)
|
||||
def test_running_tenant_task_queues_followup_work(
|
||||
self, tenant_id: str, dataset_id: str, document_ids: list[str], mock_redis: MagicMock
|
||||
) -> None:
|
||||
mock_redis.get.return_value = b"1"
|
||||
with (
|
||||
patch.object(DocumentIndexingTaskProxy, "features") as features,
|
||||
patch.object(DocumentIndexingTaskProxy, "PRIORITY_TASK_FUNC", Mock()) as task,
|
||||
):
|
||||
features.billing.subscription.plan = CloudPlan.PROFESSIONAL
|
||||
DocumentIndexingTaskProxy(tenant_id, dataset_id, document_ids).delay()
|
||||
|
||||
mock_redis.lpush.assert_called_once()
|
||||
task.delay.assert_not_called()
|
||||
|
||||
|
||||
class TestDocumentIndexing:
|
||||
def test_legacy_task_persists_parsing_before_running(
|
||||
self,
|
||||
sqlite_session: Session,
|
||||
tenant_id: str,
|
||||
dataset_id: str,
|
||||
document_ids: list[str],
|
||||
indexing_runner: MagicMock,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_persist_indexing_rows(
|
||||
sqlite_session,
|
||||
tenant_id=tenant_id,
|
||||
dataset_id=dataset_id,
|
||||
document_ids=document_ids,
|
||||
)
|
||||
_patch_features(monkeypatch, _features())
|
||||
|
||||
def assert_committed_parsing(refs: list[DocumentRef]) -> None:
|
||||
with Session(sqlite_session.get_bind()) as observer:
|
||||
documents = [observer.get(Document, ref.document_id) for ref in refs]
|
||||
assert all(document.indexing_status == IndexingStatus.PARSING for document in documents)
|
||||
assert all(document.processing_started_at is not None for document in documents)
|
||||
|
||||
indexing_runner.run.side_effect = assert_committed_parsing
|
||||
document_indexing_task.run(dataset_id, document_ids)
|
||||
|
||||
persisted = _persisted_documents(sqlite_session, document_ids)
|
||||
assert [document.indexing_status for document in persisted] == [IndexingStatus.PARSING] * 3
|
||||
indexing_runner._constructor_mock.assert_called_once_with(
|
||||
session_factory=ANY, enforce_vector_space_admission=True
|
||||
)
|
||||
indexing_runner.run.assert_called_once()
|
||||
assert len(indexing_runner.run.call_args.args) == 1
|
||||
|
||||
def test_only_existing_documents_are_processed(
|
||||
self,
|
||||
sqlite_session: Session,
|
||||
tenant_id: str,
|
||||
dataset_id: str,
|
||||
document_ids: list[str],
|
||||
indexing_runner: MagicMock,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
existing_ids = [document_ids[0], document_ids[2]]
|
||||
_persist_indexing_rows(
|
||||
sqlite_session,
|
||||
tenant_id=tenant_id,
|
||||
dataset_id=dataset_id,
|
||||
document_ids=existing_ids,
|
||||
)
|
||||
_patch_features(monkeypatch, _features())
|
||||
|
||||
_document_indexing(dataset_id, document_ids)
|
||||
|
||||
processed = indexing_runner.run.call_args.args[0]
|
||||
assert {ref.document_id for ref in processed} == set(existing_ids)
|
||||
assert sqlite_session.get(Document, document_ids[1]) is None
|
||||
|
||||
def test_empty_batch_still_reaches_runner(
|
||||
self,
|
||||
sqlite_session: Session,
|
||||
tenant_id: str,
|
||||
dataset_id: str,
|
||||
indexing_runner: MagicMock,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_persist_indexing_rows(
|
||||
sqlite_session,
|
||||
tenant_id=tenant_id,
|
||||
dataset_id=dataset_id,
|
||||
document_ids=[],
|
||||
)
|
||||
_patch_features(monkeypatch, _features())
|
||||
|
||||
_document_indexing(dataset_id, [])
|
||||
|
||||
assert indexing_runner.run.call_args.args[0] == []
|
||||
assert len(indexing_runner.run.call_args.args) == 1
|
||||
|
||||
def test_missing_dataset_returns_before_feature_lookup(
|
||||
self, dataset_id: str, document_ids: list[str], monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
get_features = _patch_features(monkeypatch, _features())
|
||||
runner_class = MagicMock()
|
||||
monkeypatch.setattr("tasks.document_indexing_task.build_document_indexing_service", runner_class)
|
||||
|
||||
_document_indexing(dataset_id, document_ids)
|
||||
|
||||
get_features.assert_not_called()
|
||||
runner_class.assert_not_called()
|
||||
|
||||
@config_overrides_context(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD)
|
||||
@pytest.mark.parametrize(
|
||||
("features", "batch_limit", "message"),
|
||||
[
|
||||
(_features(), 1, "batch upload limit"),
|
||||
(_features(plan=CloudPlan.SANDBOX), 100, "does not support batch upload"),
|
||||
(_features(vector_limit=100, vector_size=100), 100, "over the limit"),
|
||||
],
|
||||
)
|
||||
def test_validation_failure_marks_every_scoped_document_error(
|
||||
self,
|
||||
sqlite_session: Session,
|
||||
tenant_id: str,
|
||||
dataset_id: str,
|
||||
document_ids: list[str],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
features: SimpleNamespace,
|
||||
batch_limit: int,
|
||||
message: str,
|
||||
) -> None:
|
||||
_persist_indexing_rows(
|
||||
sqlite_session,
|
||||
tenant_id=tenant_id,
|
||||
dataset_id=dataset_id,
|
||||
document_ids=document_ids,
|
||||
)
|
||||
control_dataset_id = str(uuid.uuid4())
|
||||
control_document_id = str(uuid.uuid4())
|
||||
_persist_indexing_rows(
|
||||
sqlite_session,
|
||||
tenant_id=str(uuid.uuid4()),
|
||||
dataset_id=control_dataset_id,
|
||||
document_ids=[control_document_id],
|
||||
)
|
||||
_patch_features(monkeypatch, features)
|
||||
apply_config_overrides(monkeypatch, BATCH_UPLOAD_LIMIT=str(batch_limit))
|
||||
|
||||
_document_indexing(dataset_id, document_ids)
|
||||
|
||||
persisted = _persisted_documents(sqlite_session, document_ids)
|
||||
assert all(document.indexing_status == IndexingStatus.ERROR for document in persisted)
|
||||
assert all(document.error and message in document.error for document in persisted)
|
||||
assert all(document.stopped_at is not None for document in persisted)
|
||||
control = sqlite_session.get(Document, control_document_id)
|
||||
assert control is not None
|
||||
assert control.indexing_status == IndexingStatus.WAITING
|
||||
|
||||
@pytest.mark.parametrize("error", [DocumentIsPausedError("paused"), RuntimeError("boom")])
|
||||
def test_runner_failure_stops_before_summary_dispatch(
|
||||
self,
|
||||
sqlite_session: Session,
|
||||
tenant_id: str,
|
||||
dataset_id: str,
|
||||
document_ids: list[str],
|
||||
indexing_runner: MagicMock,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
error: Exception,
|
||||
) -> None:
|
||||
_persist_indexing_rows(
|
||||
sqlite_session,
|
||||
tenant_id=tenant_id,
|
||||
dataset_id=dataset_id,
|
||||
document_ids=document_ids,
|
||||
summary_index_setting={"enable": True},
|
||||
need_summary=[True] * len(document_ids),
|
||||
)
|
||||
_patch_features(monkeypatch, _features())
|
||||
indexing_runner.run.side_effect = error
|
||||
summary_delay = MagicMock()
|
||||
monkeypatch.setattr("tasks.document_indexing_task.generate_summary_index_task.delay", summary_delay)
|
||||
|
||||
_document_indexing(dataset_id, document_ids)
|
||||
|
||||
summary_delay.assert_not_called()
|
||||
persisted = _persisted_documents(sqlite_session, document_ids)
|
||||
assert all(document.indexing_status == IndexingStatus.PARSING for document in persisted)
|
||||
|
||||
|
||||
class TestSummaryDispatch:
|
||||
def test_only_eligible_completed_documents_queue_summaries(
|
||||
self,
|
||||
sqlite_session: Session,
|
||||
tenant_id: str,
|
||||
dataset_id: str,
|
||||
document_ids: list[str],
|
||||
indexing_runner: MagicMock,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_persist_indexing_rows(
|
||||
sqlite_session,
|
||||
tenant_id=tenant_id,
|
||||
dataset_id=dataset_id,
|
||||
document_ids=document_ids,
|
||||
summary_index_setting={"enable": True},
|
||||
document_forms=[
|
||||
IndexStructureType.PARAGRAPH_INDEX,
|
||||
IndexStructureType.QA_INDEX,
|
||||
IndexStructureType.PARAGRAPH_INDEX,
|
||||
],
|
||||
need_summary=[True, True, True],
|
||||
)
|
||||
_patch_features(monkeypatch, _features())
|
||||
|
||||
def finish_documents(refs: list[DocumentRef]) -> None:
|
||||
with Session(sqlite_session.get_bind()) as writer, writer.begin():
|
||||
for ref, status in zip(
|
||||
refs, [IndexingStatus.COMPLETED, IndexingStatus.COMPLETED, IndexingStatus.INDEXING], strict=True
|
||||
):
|
||||
writer.get(Document, ref.document_id).indexing_status = status
|
||||
|
||||
indexing_runner.run.side_effect = finish_documents
|
||||
summary_delay = MagicMock()
|
||||
monkeypatch.setattr("tasks.document_indexing_task.generate_summary_index_task.delay", summary_delay)
|
||||
|
||||
_document_indexing(dataset_id, document_ids)
|
||||
|
||||
summary_delay.assert_called_once_with(dataset_id, document_ids[0], None)
|
||||
|
||||
def test_summary_queue_failure_does_not_fail_indexing(
|
||||
self,
|
||||
sqlite_session: Session,
|
||||
tenant_id: str,
|
||||
dataset_id: str,
|
||||
indexing_runner: MagicMock,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
document_id = str(uuid.uuid4())
|
||||
_persist_indexing_rows(
|
||||
sqlite_session,
|
||||
tenant_id=tenant_id,
|
||||
dataset_id=dataset_id,
|
||||
document_ids=[document_id],
|
||||
summary_index_setting={"enable": True},
|
||||
need_summary=[True],
|
||||
)
|
||||
_patch_features(monkeypatch, _features())
|
||||
|
||||
def complete_documents(refs: list[DocumentRef]) -> None:
|
||||
with Session(sqlite_session.get_bind()) as writer, writer.begin():
|
||||
writer.get(Document, refs[0].document_id).indexing_status = IndexingStatus.COMPLETED
|
||||
|
||||
indexing_runner.run.side_effect = complete_documents
|
||||
summary_delay = MagicMock(side_effect=RuntimeError("queue unavailable"))
|
||||
monkeypatch.setattr("tasks.document_indexing_task.generate_summary_index_task.delay", summary_delay)
|
||||
|
||||
_document_indexing(dataset_id, [document_id])
|
||||
|
||||
summary_delay.assert_called_once_with(dataset_id, document_id, None)
|
||||
persisted = _persisted_documents(sqlite_session, [document_id])[0]
|
||||
assert persisted.indexing_status == IndexingStatus.COMPLETED
|
||||
|
||||
def test_economy_indexing_skips_summary_generation(
|
||||
self,
|
||||
sqlite_session: Session,
|
||||
tenant_id: str,
|
||||
dataset_id: str,
|
||||
indexing_runner: MagicMock,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
document_id = str(uuid.uuid4())
|
||||
_persist_indexing_rows(
|
||||
sqlite_session,
|
||||
tenant_id=tenant_id,
|
||||
dataset_id=dataset_id,
|
||||
document_ids=[document_id],
|
||||
indexing_technique=IndexTechniqueType.ECONOMY,
|
||||
summary_index_setting={"enable": True},
|
||||
need_summary=[True],
|
||||
)
|
||||
_patch_features(monkeypatch, _features())
|
||||
|
||||
def complete_documents(refs: list[DocumentRef]) -> None:
|
||||
with Session(sqlite_session.get_bind()) as writer, writer.begin():
|
||||
writer.get(Document, refs[0].document_id).indexing_status = IndexingStatus.COMPLETED
|
||||
|
||||
indexing_runner.run.side_effect = complete_documents
|
||||
summary_delay = MagicMock()
|
||||
monkeypatch.setattr("tasks.document_indexing_task.generate_summary_index_task.delay", summary_delay)
|
||||
|
||||
_document_indexing(dataset_id, [document_id])
|
||||
|
||||
summary_delay.assert_not_called()
|
||||
|
||||
def test_dataset_removed_by_runner_is_absent_from_summary_phase(
|
||||
self,
|
||||
sqlite_session: Session,
|
||||
tenant_id: str,
|
||||
dataset_id: str,
|
||||
indexing_runner: MagicMock,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
document_id = str(uuid.uuid4())
|
||||
_persist_indexing_rows(
|
||||
sqlite_session,
|
||||
tenant_id=tenant_id,
|
||||
dataset_id=dataset_id,
|
||||
document_ids=[document_id],
|
||||
summary_index_setting={"enable": True},
|
||||
need_summary=[True],
|
||||
)
|
||||
_patch_features(monkeypatch, _features())
|
||||
|
||||
def remove_dataset(_refs: list[DocumentRef]) -> None:
|
||||
with Session(sqlite_session.get_bind()) as writer, writer.begin():
|
||||
dataset = writer.get(Dataset, dataset_id)
|
||||
assert dataset is not None
|
||||
writer.delete(dataset)
|
||||
|
||||
indexing_runner.run.side_effect = remove_dataset
|
||||
summary_delay = MagicMock()
|
||||
monkeypatch.setattr("tasks.document_indexing_task.generate_summary_index_task.delay", summary_delay)
|
||||
|
||||
_document_indexing(dataset_id, [document_id])
|
||||
|
||||
sqlite_session.expire_all()
|
||||
assert sqlite_session.get(Dataset, dataset_id) is None
|
||||
summary_delay.assert_not_called()
|
||||
|
||||
|
||||
class TestTenantQueue:
|
||||
def test_followup_tasks_are_dispatched_with_one_shared_producer(
|
||||
self, tenant_id: str, dataset_id: str, document_ids: list[str], monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
next_documents = [str(uuid.uuid4())]
|
||||
queue = MagicMock()
|
||||
queue.pull_tasks.return_value = [
|
||||
{"tenant_id": tenant_id, "dataset_id": dataset_id, "document_ids": next_documents}
|
||||
]
|
||||
monkeypatch.setattr("tasks.document_indexing_task.TenantIsolatedTaskQueue", MagicMock(return_value=queue))
|
||||
monkeypatch.setattr("tasks.document_indexing_task._document_indexing", MagicMock())
|
||||
producer = object()
|
||||
monkeypatch.setattr(
|
||||
"tasks.document_indexing_task.current_app.producer_or_acquire",
|
||||
MagicMock(return_value=nullcontext(producer)),
|
||||
)
|
||||
task = MagicMock()
|
||||
|
||||
_document_indexing_with_tenant_queue(tenant_id, dataset_id, document_ids, task)
|
||||
|
||||
task.apply_async.assert_called_once_with(
|
||||
kwargs={"tenant_id": tenant_id, "dataset_id": dataset_id, "document_ids": next_documents},
|
||||
producer=producer,
|
||||
)
|
||||
queue.set_task_waiting_time.assert_called_once()
|
||||
queue.delete_task_key.assert_not_called()
|
||||
|
||||
def test_queue_cleanup_runs_when_indexing_fails(
|
||||
self, tenant_id: str, dataset_id: str, document_ids: list[str], monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
queue = MagicMock()
|
||||
queue.pull_tasks.return_value = []
|
||||
monkeypatch.setattr("tasks.document_indexing_task.TenantIsolatedTaskQueue", MagicMock(return_value=queue))
|
||||
indexing = MagicMock(side_effect=RuntimeError("indexing failed"))
|
||||
monkeypatch.setattr("tasks.document_indexing_task._document_indexing", indexing)
|
||||
|
||||
_document_indexing_with_tenant_queue(tenant_id, dataset_id, document_ids, MagicMock())
|
||||
|
||||
queue.delete_task_key.assert_called_once()
|
||||
|
||||
@pytest.mark.parametrize("task", [normal_document_indexing_task, priority_document_indexing_task])
|
||||
def test_celery_entrypoints_delegate_to_tenant_queue(
|
||||
self,
|
||||
task: object,
|
||||
tenant_id: str,
|
||||
dataset_id: str,
|
||||
document_ids: list[str],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
delegate = MagicMock()
|
||||
monkeypatch.setattr("tasks.document_indexing_task._document_indexing_with_tenant_queue", delegate)
|
||||
|
||||
task.run(tenant_id, dataset_id, document_ids) # type: ignore[attr-defined]
|
||||
|
||||
delegate.assert_called_once()
|
||||
@@ -0,0 +1,458 @@
|
||||
"""SQLite-backed tests for document update indexing and summary generation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime
|
||||
from functools import partial
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
import tasks.document_indexing_update_task as task_module
|
||||
from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType
|
||||
from extensions.storage.storage_type import StorageType
|
||||
from models.dataset import Dataset, Document, DocumentSegment, SegmentAttachmentBinding
|
||||
from models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, IndexingStatus
|
||||
from models.model import UploadFile
|
||||
from services.knowledge.indexing.errors import DocumentIsPausedError
|
||||
from tasks.document_indexing_update_task import document_indexing_update_task
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def task_harness(
|
||||
sqlite_session: Session,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> tuple[MagicMock, MagicMock]:
|
||||
"""Bind task-owned sessions to SQLite and keep only external boundaries mocked."""
|
||||
engine = sqlite_session.get_bind()
|
||||
monkeypatch.setattr(
|
||||
task_module.session_factory,
|
||||
"create_session",
|
||||
lambda: Session(engine, expire_on_commit=False),
|
||||
)
|
||||
runner = MagicMock()
|
||||
processor = MagicMock()
|
||||
monkeypatch.setattr(task_module, "build_document_indexing_service", MagicMock(return_value=runner))
|
||||
monkeypatch.setattr(
|
||||
task_module,
|
||||
"IndexProcessorFactory",
|
||||
MagicMock(return_value=MagicMock(init_index_processor=MagicMock(return_value=processor))),
|
||||
)
|
||||
return runner, processor
|
||||
|
||||
|
||||
def _persist_rows(
|
||||
session: Session,
|
||||
*,
|
||||
indexing_technique: IndexTechniqueType = IndexTechniqueType.HIGH_QUALITY,
|
||||
summary_index_setting: dict | None = None,
|
||||
doc_form: IndexStructureType = IndexStructureType.PARAGRAPH_INDEX,
|
||||
need_summary: bool = True,
|
||||
with_segment: bool = False,
|
||||
) -> tuple[Dataset, Document]:
|
||||
tenant_id = str(uuid.uuid4())
|
||||
dataset_id = str(uuid.uuid4())
|
||||
document_id = str(uuid.uuid4())
|
||||
created_by = str(uuid.uuid4())
|
||||
dataset = Dataset(
|
||||
id=dataset_id,
|
||||
tenant_id=tenant_id,
|
||||
name="Update dataset",
|
||||
data_source_type=DataSourceType.UPLOAD_FILE,
|
||||
created_by=created_by,
|
||||
indexing_technique=indexing_technique,
|
||||
summary_index_setting=summary_index_setting,
|
||||
)
|
||||
document = Document(
|
||||
id=document_id,
|
||||
tenant_id=tenant_id,
|
||||
dataset_id=dataset_id,
|
||||
position=1,
|
||||
data_source_type=DataSourceType.UPLOAD_FILE,
|
||||
batch="batch-1",
|
||||
name="document.txt",
|
||||
created_from=DocumentCreatedFrom.WEB,
|
||||
created_by=created_by,
|
||||
indexing_status=IndexingStatus.WAITING,
|
||||
doc_form=doc_form,
|
||||
need_summary=need_summary,
|
||||
)
|
||||
rows: list[object] = [dataset, document]
|
||||
if with_segment:
|
||||
rows.append(
|
||||
DocumentSegment(
|
||||
tenant_id=tenant_id,
|
||||
dataset_id=dataset_id,
|
||||
document_id=document_id,
|
||||
position=1,
|
||||
content="segment",
|
||||
word_count=1,
|
||||
tokens=1,
|
||||
created_by=created_by,
|
||||
index_node_id="node-1",
|
||||
)
|
||||
)
|
||||
session.add_all(rows)
|
||||
session.commit()
|
||||
return dataset, document
|
||||
|
||||
|
||||
def _complete_indexing(refs, *, bind) -> None:
|
||||
with Session(bind) as writer, writer.begin():
|
||||
for ref in refs:
|
||||
writer.get(Document, ref.document_id).indexing_status = IndexingStatus.COMPLETED
|
||||
|
||||
|
||||
def _persist_attachment(
|
||||
session: Session,
|
||||
*,
|
||||
dataset: Dataset,
|
||||
document: Document,
|
||||
segment: DocumentSegment,
|
||||
key: str,
|
||||
) -> tuple[UploadFile, SegmentAttachmentBinding]:
|
||||
attachment = UploadFile(
|
||||
tenant_id=dataset.tenant_id,
|
||||
storage_type=StorageType.LOCAL,
|
||||
key=key,
|
||||
name="image.png",
|
||||
size=10,
|
||||
extension="png",
|
||||
mime_type="image/png",
|
||||
created_by_role=CreatorUserRole.ACCOUNT,
|
||||
created_by=document.created_by,
|
||||
created_at=datetime.now(UTC),
|
||||
used=True,
|
||||
)
|
||||
binding = SegmentAttachmentBinding(
|
||||
tenant_id=dataset.tenant_id,
|
||||
dataset_id=dataset.id,
|
||||
document_id=document.id,
|
||||
segment_id=segment.id,
|
||||
attachment_id=attachment.id,
|
||||
)
|
||||
session.add_all([attachment, binding])
|
||||
session.commit()
|
||||
return attachment, binding
|
||||
|
||||
|
||||
def test_queues_summary_when_all_persisted_conditions_match(
|
||||
sqlite_session: Session,
|
||||
task_harness: tuple[MagicMock, MagicMock],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
runner, _processor = task_harness
|
||||
dataset, document = _persist_rows(sqlite_session, summary_index_setting={"enable": True})
|
||||
runner.run.side_effect = partial(_complete_indexing, bind=sqlite_session.get_bind())
|
||||
delay = MagicMock()
|
||||
monkeypatch.setattr(task_module.generate_summary_index_task, "delay", delay)
|
||||
|
||||
document_indexing_update_task(dataset.id, document.id)
|
||||
|
||||
delay.assert_called_once_with(dataset.id, document.id, None)
|
||||
sqlite_session.expire_all()
|
||||
assert sqlite_session.get(Document, document.id).indexing_status == IndexingStatus.COMPLETED # type: ignore[union-attr]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("dataset_changes", "document_changes"),
|
||||
[
|
||||
({"indexing_technique": IndexTechniqueType.ECONOMY}, {}),
|
||||
({"summary_index_setting": None}, {}),
|
||||
({"summary_index_setting": {"enable": False}}, {}),
|
||||
({"summary_index_setting": {"enable": True}}, {"need_summary": False}),
|
||||
(
|
||||
{"summary_index_setting": {"enable": True}},
|
||||
{"doc_form": IndexStructureType.QA_INDEX},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_skips_summary_when_persisted_eligibility_does_not_match(
|
||||
sqlite_session: Session,
|
||||
task_harness: tuple[MagicMock, MagicMock],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
dataset_changes: dict,
|
||||
document_changes: dict,
|
||||
) -> None:
|
||||
runner, _processor = task_harness
|
||||
dataset, document = _persist_rows(
|
||||
sqlite_session,
|
||||
summary_index_setting={"enable": True},
|
||||
)
|
||||
for key, value in dataset_changes.items():
|
||||
setattr(dataset, key, value)
|
||||
for key, value in document_changes.items():
|
||||
setattr(document, key, value)
|
||||
sqlite_session.commit()
|
||||
runner.run.side_effect = partial(_complete_indexing, bind=sqlite_session.get_bind())
|
||||
delay = MagicMock()
|
||||
monkeypatch.setattr(task_module.generate_summary_index_task, "delay", delay)
|
||||
|
||||
document_indexing_update_task(dataset.id, document.id)
|
||||
|
||||
delay.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"error_factory",
|
||||
[
|
||||
lambda _document_id: RuntimeError("indexing failed"),
|
||||
lambda document_id: DocumentIsPausedError(f"{document_id} is paused"),
|
||||
],
|
||||
)
|
||||
def test_skips_summary_when_indexing_fails_or_is_paused(
|
||||
sqlite_session: Session,
|
||||
task_harness: tuple[MagicMock, MagicMock],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
error_factory: Callable[[str], Exception],
|
||||
) -> None:
|
||||
runner, _processor = task_harness
|
||||
dataset, document = _persist_rows(sqlite_session, summary_index_setting={"enable": True})
|
||||
runner.run.side_effect = error_factory(document.id)
|
||||
delay = MagicMock()
|
||||
monkeypatch.setattr(task_module.generate_summary_index_task, "delay", delay)
|
||||
|
||||
document_indexing_update_task(dataset.id, document.id)
|
||||
|
||||
delay.assert_not_called()
|
||||
|
||||
|
||||
def test_returns_without_opening_external_boundaries_when_document_is_missing(
|
||||
task_harness: tuple[MagicMock, MagicMock],
|
||||
) -> None:
|
||||
runner, processor = task_harness
|
||||
|
||||
document_indexing_update_task(str(uuid.uuid4()), str(uuid.uuid4()))
|
||||
|
||||
runner.run.assert_not_called()
|
||||
processor.clean.assert_not_called()
|
||||
|
||||
|
||||
def test_skips_summary_when_dataset_is_removed_after_indexing(
|
||||
sqlite_session: Session,
|
||||
task_harness: tuple[MagicMock, MagicMock],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
runner, _processor = task_harness
|
||||
dataset, document = _persist_rows(sqlite_session, summary_index_setting={"enable": True})
|
||||
|
||||
def complete_and_remove(documents: list[Document], session: Session) -> None:
|
||||
_complete_indexing(documents, session)
|
||||
persisted_dataset = session.get(Dataset, dataset.id)
|
||||
assert persisted_dataset is not None
|
||||
session.delete(persisted_dataset)
|
||||
|
||||
runner.run.side_effect = complete_and_remove
|
||||
delay = MagicMock()
|
||||
monkeypatch.setattr(task_module.generate_summary_index_task, "delay", delay)
|
||||
|
||||
document_indexing_update_task(dataset.id, document.id)
|
||||
|
||||
delay.assert_not_called()
|
||||
|
||||
|
||||
def test_skips_summary_when_runner_leaves_document_incomplete(
|
||||
sqlite_session: Session,
|
||||
task_harness: tuple[MagicMock, MagicMock],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
runner, _processor = task_harness
|
||||
dataset, document = _persist_rows(sqlite_session, summary_index_setting={"enable": True})
|
||||
runner.run.return_value = None
|
||||
delay = MagicMock()
|
||||
monkeypatch.setattr(task_module.generate_summary_index_task, "delay", delay)
|
||||
|
||||
document_indexing_update_task(dataset.id, document.id)
|
||||
|
||||
delay.assert_not_called()
|
||||
|
||||
|
||||
def test_queue_failure_is_swallowed_after_successful_indexing(
|
||||
sqlite_session: Session,
|
||||
task_harness: tuple[MagicMock, MagicMock],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
runner, _processor = task_harness
|
||||
dataset, document = _persist_rows(sqlite_session, summary_index_setting={"enable": True})
|
||||
runner.run.side_effect = partial(_complete_indexing, bind=sqlite_session.get_bind())
|
||||
monkeypatch.setattr(
|
||||
task_module.generate_summary_index_task,
|
||||
"delay",
|
||||
MagicMock(side_effect=RuntimeError("queue unavailable")),
|
||||
)
|
||||
|
||||
document_indexing_update_task(dataset.id, document.id)
|
||||
|
||||
sqlite_session.expire_all()
|
||||
assert sqlite_session.get(Document, document.id).indexing_status == IndexingStatus.COMPLETED # type: ignore[union-attr]
|
||||
|
||||
|
||||
def test_cleans_and_deletes_persisted_segments_with_real_session(
|
||||
sqlite_session: Session,
|
||||
task_harness: tuple[MagicMock, MagicMock],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
runner, processor = task_harness
|
||||
dataset, document = _persist_rows(
|
||||
sqlite_session,
|
||||
summary_index_setting={"enable": True},
|
||||
with_segment=True,
|
||||
)
|
||||
runner.run.side_effect = partial(_complete_indexing, bind=sqlite_session.get_bind())
|
||||
delay = MagicMock()
|
||||
monkeypatch.setattr(task_module.generate_summary_index_task, "delay", delay)
|
||||
|
||||
document_indexing_update_task(dataset.id, document.id)
|
||||
|
||||
processor.clean.assert_called_once()
|
||||
assert isinstance(processor.clean.call_args.kwargs["session"], Session)
|
||||
assert sqlite_session.scalars(select(DocumentSegment).where(DocumentSegment.document_id == document.id)).all() == []
|
||||
delay.assert_called_once_with(dataset.id, document.id, None)
|
||||
|
||||
|
||||
def test_removes_orphaned_multimodal_attachments_during_reindex(
|
||||
sqlite_session: Session,
|
||||
task_harness: tuple[MagicMock, MagicMock],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
runner, processor = task_harness
|
||||
dataset, document = _persist_rows(sqlite_session, with_segment=True)
|
||||
dataset.is_multimodal = True
|
||||
segment = sqlite_session.scalar(select(DocumentSegment).where(DocumentSegment.document_id == document.id))
|
||||
assert segment is not None
|
||||
attachment, binding = _persist_attachment(
|
||||
sqlite_session,
|
||||
dataset=dataset,
|
||||
document=document,
|
||||
segment=segment,
|
||||
key="attachments/orphaned-image.png",
|
||||
)
|
||||
attachment_id = attachment.id
|
||||
binding_id = binding.id
|
||||
storage_delete = MagicMock()
|
||||
monkeypatch.setattr(task_module.storage, "delete", storage_delete)
|
||||
|
||||
document_indexing_update_task(dataset.id, document.id)
|
||||
|
||||
assert processor.clean.call_count == 2
|
||||
assert processor.clean.call_args_list[0].args[1] == ["node-1"]
|
||||
assert processor.clean.call_args_list[1].kwargs["node_ids"] == [attachment_id]
|
||||
assert processor.clean.call_args_list[1].kwargs["with_keywords"] is False
|
||||
storage_delete.assert_called_once_with("attachments/orphaned-image.png")
|
||||
sqlite_session.expire_all()
|
||||
assert sqlite_session.get(SegmentAttachmentBinding, binding_id) is None
|
||||
assert sqlite_session.get(UploadFile, attachment_id) is None
|
||||
assert sqlite_session.scalars(select(DocumentSegment).where(DocumentSegment.document_id == document.id)).all() == []
|
||||
runner.run.assert_called_once()
|
||||
|
||||
|
||||
def test_preserves_multimodal_attachment_referenced_by_another_document_during_reindex(
|
||||
sqlite_session: Session,
|
||||
task_harness: tuple[MagicMock, MagicMock],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
runner, processor = task_harness
|
||||
dataset, document = _persist_rows(sqlite_session, with_segment=True)
|
||||
dataset.is_multimodal = True
|
||||
segment = sqlite_session.scalar(select(DocumentSegment).where(DocumentSegment.document_id == document.id))
|
||||
assert segment is not None
|
||||
attachment, binding = _persist_attachment(
|
||||
sqlite_session,
|
||||
dataset=dataset,
|
||||
document=document,
|
||||
segment=segment,
|
||||
key="attachments/shared-image.png",
|
||||
)
|
||||
|
||||
other_document = Document(
|
||||
id=str(uuid.uuid4()),
|
||||
tenant_id=dataset.tenant_id,
|
||||
dataset_id=dataset.id,
|
||||
position=2,
|
||||
data_source_type=DataSourceType.UPLOAD_FILE,
|
||||
batch="batch-2",
|
||||
name="other-document.txt",
|
||||
created_from=DocumentCreatedFrom.WEB,
|
||||
created_by=document.created_by,
|
||||
indexing_status=IndexingStatus.COMPLETED,
|
||||
doc_form=IndexStructureType.PARAGRAPH_INDEX,
|
||||
)
|
||||
sqlite_session.add(other_document)
|
||||
sqlite_session.flush()
|
||||
other_segment = DocumentSegment(
|
||||
tenant_id=dataset.tenant_id,
|
||||
dataset_id=dataset.id,
|
||||
document_id=other_document.id,
|
||||
position=1,
|
||||
content="other segment",
|
||||
word_count=2,
|
||||
tokens=2,
|
||||
created_by=document.created_by,
|
||||
index_node_id="node-2",
|
||||
)
|
||||
sqlite_session.add(other_segment)
|
||||
sqlite_session.flush()
|
||||
shared_binding = SegmentAttachmentBinding(
|
||||
tenant_id=dataset.tenant_id,
|
||||
dataset_id=dataset.id,
|
||||
document_id=other_document.id,
|
||||
segment_id=other_segment.id,
|
||||
attachment_id=attachment.id,
|
||||
)
|
||||
sqlite_session.add(shared_binding)
|
||||
sqlite_session.commit()
|
||||
attachment_id = attachment.id
|
||||
binding_id = binding.id
|
||||
shared_binding_id = shared_binding.id
|
||||
storage_delete = MagicMock()
|
||||
monkeypatch.setattr(task_module.storage, "delete", storage_delete)
|
||||
|
||||
document_indexing_update_task(dataset.id, document.id)
|
||||
|
||||
processor.clean.assert_called_once()
|
||||
assert processor.clean.call_args.args[1] == ["node-1"]
|
||||
storage_delete.assert_not_called()
|
||||
sqlite_session.expire_all()
|
||||
assert sqlite_session.get(SegmentAttachmentBinding, binding_id) is None
|
||||
assert sqlite_session.get(SegmentAttachmentBinding, shared_binding_id) is not None
|
||||
assert sqlite_session.get(UploadFile, attachment_id) is not None
|
||||
runner.run.assert_called_once()
|
||||
|
||||
|
||||
def test_keeps_database_cleanup_when_reindex_attachment_storage_delete_fails(
|
||||
sqlite_session: Session,
|
||||
task_harness: tuple[MagicMock, MagicMock],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
runner, _processor = task_harness
|
||||
dataset, document = _persist_rows(sqlite_session, with_segment=True)
|
||||
dataset.is_multimodal = True
|
||||
segment = sqlite_session.scalar(select(DocumentSegment).where(DocumentSegment.document_id == document.id))
|
||||
assert segment is not None
|
||||
attachment, binding = _persist_attachment(
|
||||
sqlite_session,
|
||||
dataset=dataset,
|
||||
document=document,
|
||||
segment=segment,
|
||||
key="attachments/failing-image.png",
|
||||
)
|
||||
attachment_id = attachment.id
|
||||
binding_id = binding.id
|
||||
storage_delete = MagicMock(side_effect=RuntimeError("storage unavailable"))
|
||||
monkeypatch.setattr(task_module.storage, "delete", storage_delete)
|
||||
|
||||
with caplog.at_level("ERROR", logger="tasks.document_indexing_update_task"):
|
||||
document_indexing_update_task(dataset.id, document.id)
|
||||
|
||||
storage_delete.assert_called_once_with("attachments/failing-image.png")
|
||||
assert "Failed to delete document attachment from storage during re-indexing" in caplog.text
|
||||
sqlite_session.expire_all()
|
||||
assert sqlite_session.get(SegmentAttachmentBinding, binding_id) is None
|
||||
assert sqlite_session.get(UploadFile, attachment_id) is None
|
||||
assert sqlite_session.scalars(select(DocumentSegment).where(DocumentSegment.document_id == document.id)).all() == []
|
||||
runner.run.assert_called_once()
|
||||
@@ -0,0 +1,256 @@
|
||||
"""Unit tests for queue/wrapper behaviors in duplicate document indexing tasks (non-database logic)."""
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from unittest.mock import MagicMock, Mock, create_autospec, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from tasks.duplicate_document_indexing_task import (
|
||||
_duplicate_document_indexing_task,
|
||||
_duplicate_document_indexing_task_with_tenant_queue,
|
||||
duplicate_document_indexing_task,
|
||||
normal_duplicate_document_indexing_task,
|
||||
priority_duplicate_document_indexing_task,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tenant_id() -> str:
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dataset_id() -> str:
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def document_ids() -> list[str]:
|
||||
return [str(uuid.uuid4()) for _ in range(3)]
|
||||
|
||||
|
||||
class TestDuplicateDocumentIndexingTask:
|
||||
"""Tests for the deprecated duplicate_document_indexing_task function."""
|
||||
|
||||
@patch("tasks.duplicate_document_indexing_task._duplicate_document_indexing_task", autospec=True)
|
||||
def test_duplicate_document_indexing_task_calls_core_function(
|
||||
self, mock_core_func: MagicMock, dataset_id: str, document_ids: list[str]
|
||||
) -> None:
|
||||
"""Test that duplicate_document_indexing_task calls the core _duplicate_document_indexing_task function."""
|
||||
# Act
|
||||
duplicate_document_indexing_task(dataset_id, document_ids)
|
||||
|
||||
# Assert
|
||||
mock_core_func.assert_called_once_with(dataset_id, document_ids)
|
||||
|
||||
@patch("tasks.duplicate_document_indexing_task._duplicate_document_indexing_task", autospec=True)
|
||||
def test_duplicate_document_indexing_task_with_empty_document_ids(
|
||||
self, mock_core_func: MagicMock, dataset_id: str
|
||||
) -> None:
|
||||
"""Test duplicate_document_indexing_task with empty document_ids list."""
|
||||
# Arrange
|
||||
document_ids = []
|
||||
|
||||
# Act
|
||||
duplicate_document_indexing_task(dataset_id, document_ids)
|
||||
|
||||
# Assert
|
||||
mock_core_func.assert_called_once_with(dataset_id, document_ids)
|
||||
|
||||
|
||||
class TestDuplicateDocumentIndexingTaskWithTenantQueue:
|
||||
"""Tests for _duplicate_document_indexing_task_with_tenant_queue function."""
|
||||
|
||||
@patch("tasks.duplicate_document_indexing_task._duplicate_document_indexing_task", autospec=True)
|
||||
@pytest.mark.usefixtures("tenant_queue_commands")
|
||||
def test_tenant_queue_wrapper_calls_core_function(
|
||||
self,
|
||||
mock_core_func: MagicMock,
|
||||
tenant_id: str,
|
||||
dataset_id: str,
|
||||
document_ids: list[str],
|
||||
) -> None:
|
||||
"""Test that tenant queue wrapper calls the core function."""
|
||||
# Arrange
|
||||
mock_task_func = Mock()
|
||||
|
||||
# Act
|
||||
_duplicate_document_indexing_task_with_tenant_queue(tenant_id, dataset_id, document_ids, mock_task_func)
|
||||
|
||||
# Assert
|
||||
mock_core_func.assert_called_once_with(dataset_id, document_ids)
|
||||
|
||||
@patch(
|
||||
"tasks.duplicate_document_indexing_task._duplicate_document_indexing_task",
|
||||
new=create_autospec(_duplicate_document_indexing_task),
|
||||
)
|
||||
def test_tenant_queue_wrapper_deletes_key_when_no_tasks(
|
||||
self,
|
||||
tenant_queue_commands: MagicMock,
|
||||
tenant_id: str,
|
||||
dataset_id: str,
|
||||
document_ids: list[str],
|
||||
) -> None:
|
||||
"""Test that tenant queue wrapper deletes task key when no more tasks."""
|
||||
# Arrange
|
||||
mock_task_func = Mock()
|
||||
tenant_queue_commands.return_value = None
|
||||
|
||||
# Act
|
||||
_duplicate_document_indexing_task_with_tenant_queue(tenant_id, dataset_id, document_ids, mock_task_func)
|
||||
|
||||
# Assert
|
||||
tenant_queue_commands.assert_any_call("DEL", f"tenant_duplicate_document_indexing_task:{tenant_id}")
|
||||
|
||||
@patch(
|
||||
"tasks.duplicate_document_indexing_task._duplicate_document_indexing_task",
|
||||
new=create_autospec(_duplicate_document_indexing_task),
|
||||
)
|
||||
def test_tenant_queue_wrapper_processes_next_tasks(
|
||||
self,
|
||||
tenant_queue_commands: MagicMock,
|
||||
tenant_id: str,
|
||||
dataset_id: str,
|
||||
document_ids: list[str],
|
||||
) -> None:
|
||||
"""Test that tenant queue wrapper processes next tasks from queue."""
|
||||
# Arrange
|
||||
mock_task_func = Mock()
|
||||
next_task = {
|
||||
"tenant_id": tenant_id,
|
||||
"dataset_id": dataset_id,
|
||||
"document_ids": document_ids,
|
||||
}
|
||||
tenant_queue_commands.side_effect = [json.dumps({"data": next_task}), None, None]
|
||||
|
||||
# Act
|
||||
_duplicate_document_indexing_task_with_tenant_queue(tenant_id, dataset_id, document_ids, mock_task_func)
|
||||
|
||||
# Assert
|
||||
tenant_queue_commands.assert_any_call("SETEX", f"tenant_duplicate_document_indexing_task:{tenant_id}", 3600, 1)
|
||||
mock_task_func.delay.assert_called_once_with(
|
||||
tenant_id=tenant_id,
|
||||
dataset_id=dataset_id,
|
||||
document_ids=document_ids,
|
||||
)
|
||||
|
||||
@patch("tasks.duplicate_document_indexing_task._duplicate_document_indexing_task", autospec=True)
|
||||
def test_tenant_queue_wrapper_handles_core_function_error(
|
||||
self,
|
||||
mock_core_func: MagicMock,
|
||||
tenant_queue_commands: MagicMock,
|
||||
tenant_id: str,
|
||||
dataset_id: str,
|
||||
document_ids: list[str],
|
||||
) -> None:
|
||||
"""Test that tenant queue wrapper handles errors from core function."""
|
||||
# Arrange
|
||||
mock_task_func = Mock()
|
||||
mock_core_func.side_effect = Exception("Core function error")
|
||||
|
||||
# Act
|
||||
_duplicate_document_indexing_task_with_tenant_queue(tenant_id, dataset_id, document_ids, mock_task_func)
|
||||
|
||||
# Assert
|
||||
# Should still check for next tasks even after error
|
||||
tenant_queue_commands.assert_any_call("RPOP", f"tenant_self_duplicate_document_indexing_task_queue:{tenant_id}")
|
||||
|
||||
|
||||
class TestNormalDuplicateDocumentIndexingTask:
|
||||
"""Tests for normal_duplicate_document_indexing_task function."""
|
||||
|
||||
@patch("tasks.duplicate_document_indexing_task._duplicate_document_indexing_task_with_tenant_queue", autospec=True)
|
||||
def test_normal_task_calls_tenant_queue_wrapper(
|
||||
self,
|
||||
mock_wrapper_func: MagicMock,
|
||||
tenant_id: str,
|
||||
dataset_id: str,
|
||||
document_ids: list[str],
|
||||
) -> None:
|
||||
"""Test that normal task calls tenant queue wrapper."""
|
||||
# Act
|
||||
normal_duplicate_document_indexing_task(tenant_id, dataset_id, document_ids)
|
||||
|
||||
# Assert
|
||||
mock_wrapper_func.assert_called_once_with(
|
||||
tenant_id, dataset_id, document_ids, normal_duplicate_document_indexing_task
|
||||
)
|
||||
|
||||
@patch("tasks.duplicate_document_indexing_task._duplicate_document_indexing_task_with_tenant_queue", autospec=True)
|
||||
def test_normal_task_with_empty_document_ids(
|
||||
self,
|
||||
mock_wrapper_func: MagicMock,
|
||||
tenant_id: str,
|
||||
dataset_id: str,
|
||||
) -> None:
|
||||
"""Test normal task with empty document_ids list."""
|
||||
# Arrange
|
||||
document_ids = []
|
||||
|
||||
# Act
|
||||
normal_duplicate_document_indexing_task(tenant_id, dataset_id, document_ids)
|
||||
|
||||
# Assert
|
||||
mock_wrapper_func.assert_called_once_with(
|
||||
tenant_id, dataset_id, document_ids, normal_duplicate_document_indexing_task
|
||||
)
|
||||
|
||||
|
||||
class TestPriorityDuplicateDocumentIndexingTask:
|
||||
"""Tests for priority_duplicate_document_indexing_task function."""
|
||||
|
||||
@patch("tasks.duplicate_document_indexing_task._duplicate_document_indexing_task_with_tenant_queue", autospec=True)
|
||||
def test_priority_task_calls_tenant_queue_wrapper(
|
||||
self,
|
||||
mock_wrapper_func: MagicMock,
|
||||
tenant_id: str,
|
||||
dataset_id: str,
|
||||
document_ids: list[str],
|
||||
) -> None:
|
||||
"""Test that priority task calls tenant queue wrapper."""
|
||||
# Act
|
||||
priority_duplicate_document_indexing_task(tenant_id, dataset_id, document_ids)
|
||||
|
||||
# Assert
|
||||
mock_wrapper_func.assert_called_once_with(
|
||||
tenant_id, dataset_id, document_ids, priority_duplicate_document_indexing_task
|
||||
)
|
||||
|
||||
@patch("tasks.duplicate_document_indexing_task._duplicate_document_indexing_task_with_tenant_queue", autospec=True)
|
||||
def test_priority_task_with_single_document(
|
||||
self,
|
||||
mock_wrapper_func: MagicMock,
|
||||
tenant_id: str,
|
||||
dataset_id: str,
|
||||
) -> None:
|
||||
"""Test priority task with single document."""
|
||||
# Arrange
|
||||
document_ids = ["doc-1"]
|
||||
|
||||
# Act
|
||||
priority_duplicate_document_indexing_task(tenant_id, dataset_id, document_ids)
|
||||
|
||||
# Assert
|
||||
mock_wrapper_func.assert_called_once_with(
|
||||
tenant_id, dataset_id, document_ids, priority_duplicate_document_indexing_task
|
||||
)
|
||||
|
||||
@patch("tasks.duplicate_document_indexing_task._duplicate_document_indexing_task_with_tenant_queue", autospec=True)
|
||||
def test_priority_task_with_large_batch(
|
||||
self,
|
||||
mock_wrapper_func: MagicMock,
|
||||
tenant_id: str,
|
||||
dataset_id: str,
|
||||
) -> None:
|
||||
"""Test priority task with large batch of documents."""
|
||||
# Arrange
|
||||
document_ids = [f"doc-{i}" for i in range(100)]
|
||||
|
||||
# Act
|
||||
priority_duplicate_document_indexing_task(tenant_id, dataset_id, document_ids)
|
||||
|
||||
# Assert
|
||||
mock_wrapper_func.assert_called_once_with(
|
||||
tenant_id, dataset_id, document_ids, priority_duplicate_document_indexing_task
|
||||
)
|
||||
@@ -0,0 +1,236 @@
|
||||
import uuid
|
||||
from collections.abc import Generator
|
||||
from contextlib import contextmanager
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import event
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from core.rag.index_processor.constant.index_type import IndexStructureType
|
||||
from models.dataset import Dataset, Document, DocumentSegment
|
||||
from models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus, SegmentStatus
|
||||
from tasks.enable_segment_to_index_task import enable_segment_to_index_task
|
||||
from tasks.enable_segments_to_index_task import enable_segments_to_index_task
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def indexed_segment(sqlite_session: Session) -> tuple[Dataset, Document, DocumentSegment]:
|
||||
"""Persist the complete owner chain consumed by segment indexing tasks."""
|
||||
tenant_id = str(uuid.uuid4())
|
||||
created_by = str(uuid.uuid4())
|
||||
dataset = Dataset(
|
||||
id=str(uuid.uuid4()),
|
||||
tenant_id=tenant_id,
|
||||
name="Indexing dataset",
|
||||
data_source_type=DataSourceType.UPLOAD_FILE,
|
||||
created_by=created_by,
|
||||
is_multimodal=False,
|
||||
)
|
||||
document = Document(
|
||||
id=str(uuid.uuid4()),
|
||||
tenant_id=tenant_id,
|
||||
dataset_id=dataset.id,
|
||||
position=1,
|
||||
data_source_type=DataSourceType.UPLOAD_FILE,
|
||||
batch="batch-1",
|
||||
name="document.txt",
|
||||
created_from=DocumentCreatedFrom.WEB,
|
||||
created_by=created_by,
|
||||
indexing_status=IndexingStatus.COMPLETED,
|
||||
doc_form=IndexStructureType.PARAGRAPH_INDEX,
|
||||
)
|
||||
segment = DocumentSegment(
|
||||
tenant_id=tenant_id,
|
||||
dataset_id=dataset.id,
|
||||
document_id=document.id,
|
||||
position=1,
|
||||
content="content",
|
||||
word_count=1,
|
||||
tokens=1,
|
||||
created_by=created_by,
|
||||
index_node_id="node-1",
|
||||
index_node_hash="hash-1",
|
||||
status=SegmentStatus.COMPLETED,
|
||||
)
|
||||
sqlite_session.add_all([dataset, document, segment])
|
||||
sqlite_session.commit()
|
||||
return dataset, document, segment
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _record_transaction_events(
|
||||
sqlite_session_factory: sessionmaker[Session], phase_events: list[str]
|
||||
) -> Generator[None]:
|
||||
"""Record real transaction boundaries from task-owned SQLite sessions."""
|
||||
session_type = sqlite_session_factory.class_
|
||||
|
||||
def after_commit(_session: Session) -> None:
|
||||
phase_events.append("commit")
|
||||
|
||||
def after_rollback(_session: Session) -> None:
|
||||
phase_events.append("rollback")
|
||||
|
||||
event.listen(session_type, "after_commit", after_commit)
|
||||
event.listen(session_type, "after_rollback", after_rollback)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
event.remove(session_type, "after_commit", after_commit)
|
||||
event.remove(session_type, "after_rollback", after_rollback)
|
||||
|
||||
|
||||
def test_enable_segment_commits_index_rows_after_loading(
|
||||
indexed_segment: tuple[Dataset, Document, DocumentSegment],
|
||||
sqlite_session_factory: sessionmaker[Session],
|
||||
) -> None:
|
||||
dataset, _document, segment = indexed_segment
|
||||
phase_events: list[str] = []
|
||||
index_processor = MagicMock()
|
||||
index_processor.load.side_effect = lambda *_args, **_kwargs: phase_events.append("load")
|
||||
enable_summaries = MagicMock(side_effect=lambda *_args, **_kwargs: phase_events.append("summary"))
|
||||
|
||||
with (
|
||||
_record_transaction_events(sqlite_session_factory, phase_events),
|
||||
patch("tasks.enable_segment_to_index_task.IndexProcessorFactory") as processor_factory,
|
||||
patch(
|
||||
"services.knowledge.summaries.adapters.SummaryIndexAdapter.enable_summaries_for_segments",
|
||||
enable_summaries,
|
||||
),
|
||||
patch("tasks.enable_segment_to_index_task.redis_client.delete"),
|
||||
):
|
||||
processor_factory.return_value.init_index_processor.return_value = index_processor
|
||||
enable_segment_to_index_task.run(segment.id)
|
||||
|
||||
assert phase_events == ["load", "commit", "summary"]
|
||||
enable_summaries.assert_called_once()
|
||||
assert enable_summaries.call_args.kwargs["dataset"].id == dataset.id
|
||||
assert enable_summaries.call_args.kwargs["segment_ids"] == [segment.id]
|
||||
|
||||
|
||||
def test_enable_segment_rolls_back_before_error_compensation(
|
||||
sqlite_session: Session,
|
||||
indexed_segment: tuple[Dataset, Document, DocumentSegment],
|
||||
sqlite_session_factory: sessionmaker[Session],
|
||||
) -> None:
|
||||
_dataset, _document, segment = indexed_segment
|
||||
phase_events: list[str] = []
|
||||
index_processor = MagicMock()
|
||||
|
||||
def fail_load(*_args: object, **_kwargs: object) -> None:
|
||||
phase_events.append("load")
|
||||
raise RuntimeError("load failed")
|
||||
|
||||
index_processor.load.side_effect = fail_load
|
||||
|
||||
with (
|
||||
_record_transaction_events(sqlite_session_factory, phase_events),
|
||||
patch("tasks.enable_segment_to_index_task.IndexProcessorFactory") as processor_factory,
|
||||
patch(
|
||||
"services.knowledge.summaries.adapters.SummaryIndexAdapter.enable_summaries_for_segments"
|
||||
) as enable_summaries,
|
||||
patch("tasks.enable_segment_to_index_task.redis_client.delete"),
|
||||
):
|
||||
processor_factory.return_value.init_index_processor.return_value = index_processor
|
||||
enable_segment_to_index_task.run(segment.id)
|
||||
|
||||
sqlite_session.expire_all()
|
||||
persisted_segment = sqlite_session.get(DocumentSegment, segment.id)
|
||||
assert persisted_segment is not None
|
||||
assert persisted_segment.enabled is False
|
||||
assert persisted_segment.status == SegmentStatus.ERROR
|
||||
assert persisted_segment.error == "load failed"
|
||||
assert persisted_segment.disabled_at is not None
|
||||
assert phase_events == ["load", "rollback", "commit"]
|
||||
enable_summaries.assert_not_called()
|
||||
|
||||
|
||||
def test_enable_segments_commits_index_rows_after_loading(
|
||||
indexed_segment: tuple[Dataset, Document, DocumentSegment],
|
||||
sqlite_session_factory: sessionmaker[Session],
|
||||
) -> None:
|
||||
dataset, document, segment = indexed_segment
|
||||
phase_events: list[str] = []
|
||||
index_processor = MagicMock()
|
||||
index_processor.load.side_effect = lambda *_args, **_kwargs: phase_events.append("load")
|
||||
enable_summaries = MagicMock(side_effect=lambda *_args, **_kwargs: phase_events.append("summary"))
|
||||
|
||||
with (
|
||||
_record_transaction_events(sqlite_session_factory, phase_events),
|
||||
patch("tasks.enable_segments_to_index_task.IndexProcessorFactory") as processor_factory,
|
||||
patch(
|
||||
"services.knowledge.summaries.adapters.SummaryIndexAdapter.enable_summaries_for_segments",
|
||||
enable_summaries,
|
||||
),
|
||||
patch("tasks.enable_segments_to_index_task.redis_client.delete"),
|
||||
):
|
||||
processor_factory.return_value.init_index_processor.return_value = index_processor
|
||||
enable_segments_to_index_task.run([segment.id], dataset.id, document.id)
|
||||
|
||||
assert phase_events == ["load", "commit", "summary"]
|
||||
enable_summaries.assert_called_once()
|
||||
assert enable_summaries.call_args.kwargs["dataset"].id == dataset.id
|
||||
assert enable_summaries.call_args.kwargs["segment_ids"] == [segment.id]
|
||||
|
||||
|
||||
def test_enable_segments_skips_segments_that_are_not_completed(
|
||||
sqlite_session: Session,
|
||||
indexed_segment: tuple[Dataset, Document, DocumentSegment],
|
||||
) -> None:
|
||||
"""A batch enable must not push a segment that never finished indexing."""
|
||||
dataset, document, segment = indexed_segment
|
||||
segment.status = SegmentStatus.ERROR
|
||||
segment.enabled = False
|
||||
sqlite_session.commit()
|
||||
|
||||
index_processor = MagicMock()
|
||||
|
||||
with (
|
||||
patch("tasks.enable_segments_to_index_task.IndexProcessorFactory") as processor_factory,
|
||||
patch(
|
||||
"services.knowledge.summaries.adapters.SummaryIndexAdapter.enable_summaries_for_segments"
|
||||
) as enable_summaries,
|
||||
patch("tasks.enable_segments_to_index_task.redis_client.delete"),
|
||||
):
|
||||
processor_factory.return_value.init_index_processor.return_value = index_processor
|
||||
enable_segments_to_index_task.run([segment.id], dataset.id, document.id)
|
||||
|
||||
index_processor.load.assert_not_called()
|
||||
enable_summaries.assert_not_called()
|
||||
|
||||
|
||||
def test_enable_segments_rolls_back_before_error_compensation(
|
||||
sqlite_session: Session,
|
||||
indexed_segment: tuple[Dataset, Document, DocumentSegment],
|
||||
sqlite_session_factory: sessionmaker[Session],
|
||||
) -> None:
|
||||
dataset, document, segment = indexed_segment
|
||||
phase_events: list[str] = []
|
||||
index_processor = MagicMock()
|
||||
|
||||
def fail_load(*_args: object, **_kwargs: object) -> None:
|
||||
phase_events.append("load")
|
||||
raise RuntimeError("load failed")
|
||||
|
||||
index_processor.load.side_effect = fail_load
|
||||
|
||||
with (
|
||||
_record_transaction_events(sqlite_session_factory, phase_events),
|
||||
patch("tasks.enable_segments_to_index_task.IndexProcessorFactory") as processor_factory,
|
||||
patch(
|
||||
"services.knowledge.summaries.adapters.SummaryIndexAdapter.enable_summaries_for_segments"
|
||||
) as enable_summaries,
|
||||
patch("tasks.enable_segments_to_index_task.redis_client.delete"),
|
||||
):
|
||||
processor_factory.return_value.init_index_processor.return_value = index_processor
|
||||
enable_segments_to_index_task.run([segment.id], dataset.id, document.id)
|
||||
|
||||
sqlite_session.expire_all()
|
||||
persisted_segment = sqlite_session.get(DocumentSegment, segment.id)
|
||||
assert persisted_segment is not None
|
||||
assert persisted_segment.enabled is False
|
||||
assert persisted_segment.status == SegmentStatus.ERROR
|
||||
assert persisted_segment.error == "load failed"
|
||||
assert persisted_segment.disabled_at is not None
|
||||
assert phase_events == ["load", "rollback", "commit"]
|
||||
enable_summaries.assert_not_called()
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Unit tests for enterprise telemetry Celery task."""
|
||||
|
||||
import json
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from enterprise.telemetry.contracts import TelemetryCase, TelemetryEnvelope
|
||||
from tasks.enterprise_telemetry_task import process_enterprise_telemetry
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_envelope_json() -> str:
|
||||
envelope = TelemetryEnvelope(
|
||||
case=TelemetryCase.APP_CREATED,
|
||||
tenant_id="test-tenant",
|
||||
event_id="test-event-123",
|
||||
payload={"app_id": "app-123"},
|
||||
)
|
||||
return envelope.model_dump_json()
|
||||
|
||||
|
||||
def test_process_enterprise_telemetry_success(sample_envelope_json: str) -> None:
|
||||
with patch("tasks.enterprise_telemetry_task.EnterpriseMetricHandler") as mock_handler_class:
|
||||
mock_handler = MagicMock()
|
||||
mock_handler_class.return_value = mock_handler
|
||||
|
||||
process_enterprise_telemetry(sample_envelope_json)
|
||||
|
||||
mock_handler.handle.assert_called_once()
|
||||
call_args = mock_handler.handle.call_args[0][0]
|
||||
assert isinstance(call_args, TelemetryEnvelope)
|
||||
assert call_args.case == TelemetryCase.APP_CREATED
|
||||
assert call_args.tenant_id == "test-tenant"
|
||||
assert call_args.event_id == "test-event-123"
|
||||
|
||||
|
||||
def test_process_enterprise_telemetry_invalid_json(caplog: pytest.LogCaptureFixture) -> None:
|
||||
invalid_json = "not valid json"
|
||||
|
||||
process_enterprise_telemetry(invalid_json)
|
||||
|
||||
assert "Failed to process enterprise telemetry envelope" in caplog.text
|
||||
|
||||
|
||||
def test_process_enterprise_telemetry_handler_exception(
|
||||
sample_envelope_json: str, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
with patch("tasks.enterprise_telemetry_task.EnterpriseMetricHandler") as mock_handler_class:
|
||||
mock_handler = MagicMock()
|
||||
mock_handler.handle.side_effect = Exception("Handler error")
|
||||
mock_handler_class.return_value = mock_handler
|
||||
|
||||
process_enterprise_telemetry(sample_envelope_json)
|
||||
|
||||
assert "Failed to process enterprise telemetry envelope" in caplog.text
|
||||
|
||||
|
||||
def test_process_enterprise_telemetry_validation_error(caplog: pytest.LogCaptureFixture) -> None:
|
||||
invalid_envelope = json.dumps(
|
||||
{
|
||||
"case": "INVALID_CASE",
|
||||
"tenant_id": "test-tenant",
|
||||
"event_id": "test-event",
|
||||
"payload": {},
|
||||
}
|
||||
)
|
||||
|
||||
process_enterprise_telemetry(invalid_envelope)
|
||||
|
||||
assert "Failed to process enterprise telemetry envelope" in caplog.text
|
||||
@@ -0,0 +1,78 @@
|
||||
"""
|
||||
Unit tests for summary index task queue isolation.
|
||||
|
||||
These tasks must NOT run on the shared 'dataset' queue because they invoke LLMs
|
||||
for each document segment and can occupy all worker slots for hours, blocking
|
||||
document indexing tasks.
|
||||
"""
|
||||
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
import pytest
|
||||
|
||||
from tasks.generate_summary_index_task import generate_summary_index_task
|
||||
from tasks.regenerate_summary_index_task import regenerate_summary_index_task
|
||||
|
||||
SUMMARY_QUEUE = "dataset_summary"
|
||||
INDEXING_QUEUE = "dataset"
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class _Queue(Protocol):
|
||||
name: str
|
||||
|
||||
|
||||
class _Router(Protocol):
|
||||
def route(
|
||||
self,
|
||||
options: dict[str, object],
|
||||
name: str,
|
||||
args: tuple[object, ...],
|
||||
kwargs: dict[str, object],
|
||||
) -> dict[str, object]: ...
|
||||
|
||||
|
||||
class _Amqp(Protocol):
|
||||
router: _Router
|
||||
|
||||
|
||||
class _CeleryApp(Protocol):
|
||||
amqp: _Amqp
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class _RoutableTask(Protocol):
|
||||
name: str
|
||||
app: _CeleryApp
|
||||
|
||||
def _get_exec_options(self) -> dict[str, object]: ...
|
||||
|
||||
|
||||
def _routed_queue(task: object) -> str | None:
|
||||
assert isinstance(task, _RoutableTask), "Expected a Celery task with routing metadata"
|
||||
options = task._get_exec_options()
|
||||
routed = task.app.amqp.router.route(options, task.name, args=(), kwargs={})
|
||||
queue = routed.get("queue")
|
||||
if queue is None:
|
||||
return None
|
||||
assert isinstance(queue, _Queue), "Expected a routed queue with a name"
|
||||
return queue.name
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("task", "task_name"),
|
||||
[
|
||||
(generate_summary_index_task, "generate_summary_index_task"),
|
||||
(regenerate_summary_index_task, "regenerate_summary_index_task"),
|
||||
],
|
||||
)
|
||||
def test_summary_task_routes_to_dedicated_queue(task: object, task_name: str) -> None:
|
||||
"""Summary tasks must route to the dataset_summary queue, not the shared dataset queue.
|
||||
|
||||
Summary generation is LLM-heavy and will block document indexing if placed
|
||||
on the shared queue.
|
||||
"""
|
||||
assert _routed_queue(task) == SUMMARY_QUEUE, (
|
||||
f"{task_name} must route to '{SUMMARY_QUEUE}' queue (not '{INDEXING_QUEUE}'). "
|
||||
"Summary generation is LLM-heavy and will block document indexing if placed on the shared queue."
|
||||
)
|
||||
@@ -0,0 +1,252 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.engine import Engine
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
import core.db.session_factory as session_factory_module
|
||||
from core.repositories.human_input_repository import HumanInputFormSubmissionRepository
|
||||
from core.workflow.nodes.human_input.entities import FormDefinition
|
||||
from core.workflow.nodes.human_input.enums import HumanInputFormKind, HumanInputFormStatus
|
||||
from models.human_input import HumanInputForm
|
||||
from tasks import human_input_timeout_tasks as task_module
|
||||
from tests.helpers.unit.config_override import apply_config_overrides
|
||||
|
||||
|
||||
class _FakeService:
|
||||
def __init__(self):
|
||||
self.enqueued: list[str] = []
|
||||
self.agent_app_resumed: list[tuple[str, str]] = []
|
||||
|
||||
def enqueue_resume(self, workflow_run_id: str | None) -> None:
|
||||
if workflow_run_id is not None:
|
||||
self.enqueued.append(workflow_run_id)
|
||||
|
||||
def enqueue_agent_app_resume(self, *, conversation_id: str, form_id: str) -> None:
|
||||
self.agent_app_resumed.append((conversation_id, form_id))
|
||||
|
||||
|
||||
def _build_form(
|
||||
*,
|
||||
form_id: str,
|
||||
form_kind: HumanInputFormKind,
|
||||
created_at: datetime,
|
||||
expiration_time: datetime,
|
||||
workflow_run_id: str | None,
|
||||
node_id: str,
|
||||
conversation_id: str | None = None,
|
||||
) -> HumanInputForm:
|
||||
form_definition = FormDefinition(
|
||||
form_content="",
|
||||
rendered_content="",
|
||||
expiration_time=expiration_time,
|
||||
)
|
||||
return HumanInputForm(
|
||||
id=form_id,
|
||||
tenant_id="tenant-1",
|
||||
app_id="app-1",
|
||||
form_kind=form_kind,
|
||||
created_at=created_at,
|
||||
expiration_time=expiration_time,
|
||||
workflow_run_id=workflow_run_id,
|
||||
conversation_id=conversation_id,
|
||||
node_id=node_id,
|
||||
form_definition=form_definition.model_dump_json(),
|
||||
rendered_content="",
|
||||
status=HumanInputFormStatus.WAITING,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sqlite_task_database(
|
||||
sqlite_engine: Engine,
|
||||
sqlite_session: Session,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
repository_session_maker = sessionmaker(bind=sqlite_engine, expire_on_commit=False)
|
||||
monkeypatch.setattr(session_factory_module, "_session_maker", repository_session_maker)
|
||||
monkeypatch.setattr(task_module, "db", SimpleNamespace(engine=sqlite_engine))
|
||||
|
||||
|
||||
def test_is_global_timeout_uses_created_at():
|
||||
now = datetime(2025, 1, 1, 12, 0, 0)
|
||||
form = _build_form(
|
||||
form_id="form-1",
|
||||
form_kind=HumanInputFormKind.RUNTIME,
|
||||
created_at=now - timedelta(seconds=61),
|
||||
expiration_time=now + timedelta(hours=1),
|
||||
workflow_run_id="run-1",
|
||||
node_id="node-1",
|
||||
)
|
||||
|
||||
assert task_module._is_global_timeout(form, 60, now=now) is True
|
||||
|
||||
form.workflow_run_id = None
|
||||
assert task_module._is_global_timeout(form, 60, now=now) is False
|
||||
|
||||
form.workflow_run_id = "run-1"
|
||||
form.created_at = now - timedelta(seconds=59)
|
||||
assert task_module._is_global_timeout(form, 60, now=now) is False
|
||||
|
||||
assert task_module._is_global_timeout(form, 0, now=now) is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sqlite_session", [(HumanInputForm,)], indirect=True)
|
||||
def test_check_and_handle_human_input_timeouts_marks_and_routes(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
sqlite_task_database: None,
|
||||
sqlite_engine: Engine,
|
||||
sqlite_session: Session,
|
||||
):
|
||||
now = datetime(2025, 1, 1, 12, 0, 0)
|
||||
monkeypatch.setattr(task_module, "naive_utc_now", lambda: now)
|
||||
apply_config_overrides(monkeypatch, HUMAN_INPUT_GLOBAL_TIMEOUT_SECONDS=3600)
|
||||
|
||||
forms = [
|
||||
_build_form(
|
||||
form_id="form-global",
|
||||
form_kind=HumanInputFormKind.RUNTIME,
|
||||
created_at=now - timedelta(hours=2),
|
||||
expiration_time=now + timedelta(hours=1),
|
||||
workflow_run_id="run-global",
|
||||
node_id="node-global",
|
||||
),
|
||||
_build_form(
|
||||
form_id="form-node",
|
||||
form_kind=HumanInputFormKind.RUNTIME,
|
||||
created_at=now - timedelta(minutes=5),
|
||||
expiration_time=now - timedelta(seconds=1),
|
||||
workflow_run_id="run-node",
|
||||
node_id="node-node",
|
||||
),
|
||||
_build_form(
|
||||
form_id="form-delivery",
|
||||
form_kind=HumanInputFormKind.DELIVERY_TEST,
|
||||
created_at=now - timedelta(minutes=1),
|
||||
expiration_time=now - timedelta(seconds=1),
|
||||
workflow_run_id=None,
|
||||
node_id="node-delivery",
|
||||
),
|
||||
]
|
||||
sqlite_session.add_all(forms)
|
||||
sqlite_session.commit()
|
||||
|
||||
repo = HumanInputFormSubmissionRepository()
|
||||
mark_timeout_spy = MagicMock(wraps=repo.mark_timeout)
|
||||
monkeypatch.setattr(repo, "mark_timeout", mark_timeout_spy)
|
||||
service = _FakeService()
|
||||
service_factory = MagicMock(return_value=service)
|
||||
global_timeout_handler = MagicMock()
|
||||
|
||||
monkeypatch.setattr(task_module, "HumanInputFormSubmissionRepository", lambda: repo)
|
||||
monkeypatch.setattr(task_module, "HumanInputService", service_factory)
|
||||
monkeypatch.setattr(task_module, "_handle_global_timeout", global_timeout_handler)
|
||||
|
||||
task_module.check_and_handle_human_input_timeouts(limit=100)
|
||||
|
||||
assert {
|
||||
(call.kwargs["form_id"], call.kwargs["timeout_status"], call.kwargs["reason"])
|
||||
for call in mark_timeout_spy.call_args_list
|
||||
} == {
|
||||
("form-global", HumanInputFormStatus.EXPIRED, "global_timeout"),
|
||||
("form-node", HumanInputFormStatus.TIMEOUT, "node_timeout"),
|
||||
("form-delivery", HumanInputFormStatus.TIMEOUT, "delivery_test_timeout"),
|
||||
}
|
||||
assert service.enqueued == ["run-node"]
|
||||
global_timeout_handler.assert_called_once()
|
||||
global_timeout_call = global_timeout_handler.call_args.kwargs
|
||||
assert global_timeout_call["form_id"] == "form-global"
|
||||
assert global_timeout_call["workflow_run_id"] == "run-global"
|
||||
assert global_timeout_call["node_id"] == "node-global"
|
||||
task_session_maker = global_timeout_call["session_factory"]
|
||||
assert isinstance(task_session_maker, sessionmaker)
|
||||
assert task_session_maker.kw["bind"] is sqlite_engine
|
||||
service_factory.assert_called_once_with(task_session_maker, form_repository=repo)
|
||||
|
||||
sqlite_session.expire_all()
|
||||
assert sqlite_session.get(HumanInputForm, "form-global").status == HumanInputFormStatus.EXPIRED
|
||||
assert sqlite_session.get(HumanInputForm, "form-node").status == HumanInputFormStatus.TIMEOUT
|
||||
assert sqlite_session.get(HumanInputForm, "form-delivery").status == HumanInputFormStatus.TIMEOUT
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sqlite_session", [(HumanInputForm,)], indirect=True)
|
||||
def test_check_and_handle_human_input_timeouts_orders_by_id_before_limit(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
sqlite_task_database: None,
|
||||
sqlite_session: Session,
|
||||
):
|
||||
now = datetime(2025, 1, 1, 12, 0, 0)
|
||||
monkeypatch.setattr(task_module, "naive_utc_now", lambda: now)
|
||||
apply_config_overrides(monkeypatch, HUMAN_INPUT_GLOBAL_TIMEOUT_SECONDS=0)
|
||||
|
||||
forms = [
|
||||
_build_form(
|
||||
form_id=form_id,
|
||||
form_kind=HumanInputFormKind.DELIVERY_TEST,
|
||||
created_at=now - timedelta(minutes=1),
|
||||
expiration_time=now - timedelta(seconds=1),
|
||||
workflow_run_id=None,
|
||||
node_id=f"node-{form_id}",
|
||||
)
|
||||
for form_id in ("form-b", "form-a")
|
||||
]
|
||||
sqlite_session.add_all(forms)
|
||||
sqlite_session.commit()
|
||||
|
||||
repo = HumanInputFormSubmissionRepository()
|
||||
mark_timeout_spy = MagicMock(wraps=repo.mark_timeout)
|
||||
monkeypatch.setattr(repo, "mark_timeout", mark_timeout_spy)
|
||||
monkeypatch.setattr(task_module, "HumanInputFormSubmissionRepository", lambda: repo)
|
||||
monkeypatch.setattr(task_module, "HumanInputService", MagicMock(return_value=_FakeService()))
|
||||
|
||||
task_module.check_and_handle_human_input_timeouts(limit=1)
|
||||
|
||||
mark_timeout_spy.assert_called_once_with(
|
||||
form_id="form-a",
|
||||
timeout_status=HumanInputFormStatus.TIMEOUT,
|
||||
reason="delivery_test_timeout",
|
||||
)
|
||||
sqlite_session.expire_all()
|
||||
assert sqlite_session.get(HumanInputForm, "form-a").status == HumanInputFormStatus.TIMEOUT
|
||||
assert sqlite_session.get(HumanInputForm, "form-b").status == HumanInputFormStatus.WAITING
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sqlite_session", [(HumanInputForm,)], indirect=True)
|
||||
def test_check_and_handle_human_input_timeouts_omits_global_filter_when_disabled(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
sqlite_task_database: None,
|
||||
sqlite_session: Session,
|
||||
):
|
||||
now = datetime(2025, 1, 1, 12, 0, 0)
|
||||
monkeypatch.setattr(task_module, "naive_utc_now", lambda: now)
|
||||
apply_config_overrides(monkeypatch, HUMAN_INPUT_GLOBAL_TIMEOUT_SECONDS=0)
|
||||
|
||||
old_unexpired_form = _build_form(
|
||||
form_id="form-old",
|
||||
form_kind=HumanInputFormKind.RUNTIME,
|
||||
created_at=now - timedelta(hours=2),
|
||||
expiration_time=now + timedelta(hours=1),
|
||||
workflow_run_id="run-old",
|
||||
node_id="node-old",
|
||||
)
|
||||
sqlite_session.add(old_unexpired_form)
|
||||
sqlite_session.commit()
|
||||
|
||||
repo = HumanInputFormSubmissionRepository()
|
||||
mark_timeout_spy = MagicMock(wraps=repo.mark_timeout)
|
||||
monkeypatch.setattr(repo, "mark_timeout", mark_timeout_spy)
|
||||
monkeypatch.setattr(task_module, "HumanInputFormSubmissionRepository", lambda: repo)
|
||||
monkeypatch.setattr(task_module, "HumanInputService", MagicMock(return_value=_FakeService()))
|
||||
global_timeout_handler = MagicMock()
|
||||
monkeypatch.setattr(task_module, "_handle_global_timeout", global_timeout_handler)
|
||||
|
||||
task_module.check_and_handle_human_input_timeouts(limit=1)
|
||||
|
||||
mark_timeout_spy.assert_not_called()
|
||||
global_timeout_handler.assert_not_called()
|
||||
sqlite_session.refresh(old_unexpired_form)
|
||||
assert old_unexpired_form.status == HumanInputFormStatus.WAITING
|
||||
@@ -0,0 +1,137 @@
|
||||
"""Tests for Celery-owned IM callback execution and retry."""
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import pytest
|
||||
from celery.exceptions import Retry
|
||||
from sqlalchemy.engine import Engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from core.human_input_v2.entities import IMProvider
|
||||
from core.human_input_v2.im_integration.adapters import AuthenticatedIMEvent, IMEventIngressKind
|
||||
from core.human_input_v2.im_message_inbox import IMInboxRecord, IMInboxRecordId
|
||||
from dify_app import DifyApp
|
||||
from repositories.human_input_v2.im_channel_repository import IMChannelId
|
||||
from services.human_input_v2.im_message_inbox.processor import IMInboxProcessor
|
||||
from services.human_input_v2.im_message_inbox.storage import IMInboxStorage
|
||||
from tasks.im_message_inbox_tasks import (
|
||||
IMInboxRuntimeNotConfiguredError,
|
||||
IMInboxTaskRetryError,
|
||||
configure_im_inbox_task_runtime,
|
||||
process_im_message_inbox_record,
|
||||
)
|
||||
|
||||
_NOW = datetime(2026, 8, 2, 8, tzinfo=UTC)
|
||||
|
||||
|
||||
class _RecordProcessor:
|
||||
record_ids: list[IMInboxRecordId]
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.record_ids = []
|
||||
|
||||
def process(self, record_id: IMInboxRecordId) -> None:
|
||||
self.record_ids.append(record_id)
|
||||
|
||||
|
||||
class _FailingRecordProcessor:
|
||||
def process(self, record_id: IMInboxRecordId) -> None:
|
||||
raise RuntimeError(f"payload must-not-log for {record_id}")
|
||||
|
||||
|
||||
class _Consumer:
|
||||
records: list[IMInboxRecord]
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.records = []
|
||||
|
||||
def consume(self, record: IMInboxRecord) -> None:
|
||||
self.records.append(record)
|
||||
|
||||
|
||||
def _event() -> AuthenticatedIMEvent:
|
||||
return AuthenticatedIMEvent(
|
||||
provider=IMProvider.FEISHU,
|
||||
provider_tenant_id="tenant-1",
|
||||
event_id="event-1",
|
||||
event_type="card.action",
|
||||
occurred_at=None,
|
||||
received_at=datetime(2026, 8, 2, 8),
|
||||
ingress_kind=IMEventIngressKind.WEBHOOK,
|
||||
payload='{"secret":"must-not-log"}',
|
||||
)
|
||||
|
||||
|
||||
def test_processing_task_uses_human_input_delivery_queue(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
dispatches: list[dict[str, object]] = []
|
||||
|
||||
def capture_dispatch(*_args: object, **options: object) -> None:
|
||||
dispatches.append(options)
|
||||
|
||||
monkeypatch.setitem(process_im_message_inbox_record.app.conf, "task_always_eager", False)
|
||||
monkeypatch.setattr(process_im_message_inbox_record.app, "send_task", capture_dispatch)
|
||||
|
||||
process_im_message_inbox_record.apply_async(args=("record-1",))
|
||||
|
||||
assert len(dispatches) == 1
|
||||
assert dispatches[0]["queue"] == "human_input_delivery"
|
||||
|
||||
|
||||
def test_processing_task_fails_closed_when_runtime_is_not_configured() -> None:
|
||||
app = DifyApp(__name__)
|
||||
|
||||
with app.app_context(), pytest.raises(IMInboxRuntimeNotConfiguredError):
|
||||
process_im_message_inbox_record.run("record-1")
|
||||
|
||||
|
||||
def test_processing_task_passes_typed_record_id_to_processor() -> None:
|
||||
app = DifyApp(__name__)
|
||||
processor = _RecordProcessor()
|
||||
configure_im_inbox_task_runtime(app, processor_factory=lambda: processor)
|
||||
|
||||
with app.app_context():
|
||||
result = process_im_message_inbox_record.run("record-1")
|
||||
|
||||
assert result is None
|
||||
assert processor.record_ids == [IMInboxRecordId("record-1")]
|
||||
|
||||
|
||||
def test_processing_task_retries_sanitized_processor_failure(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
app = DifyApp(__name__)
|
||||
configure_im_inbox_task_runtime(app, processor_factory=_FailingRecordProcessor)
|
||||
retry_errors: list[Exception] = []
|
||||
|
||||
def retry(*, exc: Exception, **_: object) -> None:
|
||||
retry_errors.append(exc)
|
||||
raise Retry()
|
||||
|
||||
monkeypatch.setattr(process_im_message_inbox_record, "retry", retry)
|
||||
|
||||
with app.app_context(), pytest.raises(Retry):
|
||||
process_im_message_inbox_record.run("record-1")
|
||||
|
||||
assert len(retry_errors) == 1
|
||||
assert isinstance(retry_errors[0], IMInboxTaskRetryError)
|
||||
assert "must-not-log" not in caplog.text
|
||||
assert "must-not-log" not in str(retry_errors[0])
|
||||
|
||||
|
||||
def test_processing_task_loads_callback_and_records_processed_fact(sqlite_engine: Engine) -> None:
|
||||
repository = IMInboxStorage(sessionmaker(bind=sqlite_engine, expire_on_commit=False))
|
||||
accepted = repository.insert_or_resolve(IMChannelId("channel-1"), _event(), now=_NOW)
|
||||
consumer = _Consumer()
|
||||
processor = IMInboxProcessor(repository=repository, consumer=consumer, clock=lambda: _NOW)
|
||||
app = DifyApp(__name__)
|
||||
configure_im_inbox_task_runtime(app, processor_factory=lambda: processor)
|
||||
|
||||
with app.app_context():
|
||||
process_im_message_inbox_record.run(str(accepted.record_id))
|
||||
|
||||
assert len(consumer.records) == 1
|
||||
assert consumer.records[0].event == _event()
|
||||
stored = repository.get(accepted.record_id)
|
||||
assert stored is not None
|
||||
assert stored.processed_at == datetime(2026, 8, 2, 8)
|
||||
+161
@@ -0,0 +1,161 @@
|
||||
"""Task behavior with real workspace queries and an explicit RBAC transport."""
|
||||
|
||||
from collections.abc import Callable, Iterator
|
||||
from typing import Protocol, cast
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
|
||||
import tasks.initialize_created_app_rbac_access_task as task_module
|
||||
from enums import DeploymentEdition
|
||||
from enums.account import TenantAccountRole
|
||||
from extensions.ext_application_services import build_application_services
|
||||
from extensions.ext_redis import redis_client
|
||||
from models import TenantAccountJoin
|
||||
from services.rbac import contracts as dto
|
||||
from tasks.initialize_created_app_rbac_access_task import (
|
||||
initialize_created_app_rbac_access_task,
|
||||
sync_joined_workspace_member_rbac_access_task,
|
||||
)
|
||||
from tests.helpers.unit.rbac_fakes import RBACDomain
|
||||
|
||||
|
||||
class _TaskWithQueue(Protocol):
|
||||
queue: str
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def workspace_services(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
app: Flask,
|
||||
rbac_domain: RBACDomain,
|
||||
config_overrides: Callable[..., None],
|
||||
) -> None:
|
||||
config_overrides(RBAC_ENABLED=True)
|
||||
services = build_application_services(
|
||||
database_client=rbac_domain.sessions,
|
||||
deployment_edition=DeploymentEdition.COMMUNITY,
|
||||
initialization_password="",
|
||||
redis=redis_client,
|
||||
)
|
||||
monkeypatch.setitem(app.extensions, "application_services", services)
|
||||
|
||||
|
||||
def _seed_members(domain: RBACDomain, members: tuple[str, ...]) -> None:
|
||||
with domain.sessions.begin() as session:
|
||||
for account_id in members:
|
||||
membership = TenantAccountJoin(tenant_id="tenant-1", account_id=account_id, role=TenantAccountRole.NORMAL)
|
||||
membership.id = account_id
|
||||
session.add(membership)
|
||||
session.add(TenantAccountJoin(tenant_id="other", account_id="foreign", role=TenantAccountRole.NORMAL))
|
||||
|
||||
|
||||
def test_initialize_created_app_rbac_access_task_uses_rbac_queue() -> None:
|
||||
task = cast(_TaskWithQueue, initialize_created_app_rbac_access_task)
|
||||
assert task.queue == "app_rbac"
|
||||
|
||||
|
||||
def test_sync_joined_workspace_member_rbac_access_task_uses_rbac_queue() -> None:
|
||||
task = cast(_TaskWithQueue, sync_joined_workspace_member_rbac_access_task)
|
||||
assert task.queue == "app_rbac"
|
||||
|
||||
|
||||
def test_initialize_created_app_rbac_access_task_batches_workspace_members(
|
||||
monkeypatch: pytest.MonkeyPatch, rbac_domain: RBACDomain
|
||||
) -> None:
|
||||
_seed_members(rbac_domain, ("acct-1", "acct-2", "acct-3"))
|
||||
monkeypatch.setattr(task_module, "APP_RBAC_ACCOUNT_POLICY_BATCH_SIZE", 2)
|
||||
|
||||
initialize_created_app_rbac_access_task.run("tenant-1", "actor-1", "app-1")
|
||||
|
||||
first, second = rbac_domain.transport.requests
|
||||
for request, account_ids in ((first, ["acct-1", "acct-2"]), (second, ["acct-3"])):
|
||||
assert request.method == "POST"
|
||||
assert request.endpoint == "/rbac/apps/whitelist/members/batch"
|
||||
assert (request.tenant_id, request.account_id) == ("tenant-1", "actor-1")
|
||||
assert request.params is None
|
||||
assert request.json == {"data": [{"app_id": "app-1", "account_ids": account_ids, "policy_id": "default"}]}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["app", "dataset", "agent"])
|
||||
def test_initialize_created_app_rbac_access_task_targets_the_resource_that_was_passed(
|
||||
rbac_domain: RBACDomain, kind: str
|
||||
) -> None:
|
||||
_seed_members(rbac_domain, ("acct-1",))
|
||||
resource_id = f"{kind}-1"
|
||||
|
||||
initialize_created_app_rbac_access_task.run("tenant-1", "actor-1", **{f"{kind}_id": resource_id})
|
||||
|
||||
request = rbac_domain.transport.only_request
|
||||
assert (request.method, request.endpoint) == ("POST", f"/rbac/{kind}s/whitelist/members/batch")
|
||||
assert (request.tenant_id, request.account_id) == ("tenant-1", "actor-1")
|
||||
assert request.params is None
|
||||
assert request.json == {"data": [{f"{kind}_id": resource_id, "account_ids": ["acct-1"], "policy_id": "default"}]}
|
||||
|
||||
|
||||
def test_initialize_created_app_rbac_access_task_retries_on_failure(
|
||||
monkeypatch: pytest.MonkeyPatch, rbac_domain: RBACDomain
|
||||
) -> None:
|
||||
_seed_members(rbac_domain, ("acct-1",))
|
||||
failure = ConnectionError("RBAC unavailable")
|
||||
rbac_domain.transport.failure = failure
|
||||
retries: list[Exception] = []
|
||||
|
||||
def retry(*, exc: Exception) -> RuntimeError:
|
||||
retries.append(exc)
|
||||
return RuntimeError("retry requested")
|
||||
|
||||
monkeypatch.setattr(initialize_created_app_rbac_access_task, "retry", retry)
|
||||
|
||||
with pytest.raises(RuntimeError, match="retry requested"):
|
||||
initialize_created_app_rbac_access_task.run("tenant-1", "actor-1", "app-1")
|
||||
|
||||
assert retries == [failure]
|
||||
assert rbac_domain.transport.only_request.endpoint == "/rbac/apps/whitelist/members/batch"
|
||||
|
||||
|
||||
def test_sync_joined_workspace_member_rbac_access_task_appends_auto_included_resources(
|
||||
monkeypatch: pytest.MonkeyPatch, rbac_domain: RBACDomain
|
||||
) -> None:
|
||||
resources = [
|
||||
dto.ResourceWhitelistConfigResource(resource_type=kind, resource_id=resource_id)
|
||||
for kind, resource_id in (
|
||||
(dto.RBACResourceType.APP, "app-1"),
|
||||
(dto.RBACResourceType.DATASET, "dataset-1"),
|
||||
(dto.RBACResourceType.APP, "app-2"),
|
||||
(dto.RBACResourceType.AGENT, "agent-1"),
|
||||
(dto.RBACResourceType.AGENT, "agent-2"),
|
||||
)
|
||||
]
|
||||
|
||||
def resource_batches(tenant_id: str, batch_size: int) -> Iterator[list[dto.ResourceWhitelistConfigResource]]:
|
||||
assert tenant_id == "tenant-1"
|
||||
assert batch_size == task_module.APP_RBAC_RESOURCE_CONFIG_BATCH_SIZE
|
||||
yield resources
|
||||
|
||||
configs = dto.ResourceWhitelistConfigsResponse(
|
||||
data=[
|
||||
dto.ResourceWhitelistConfigItem(
|
||||
resource_type=resource.resource_type,
|
||||
resource_id=resource.resource_id,
|
||||
automatic_include_workspace_members=resource.resource_id.endswith("-1"),
|
||||
)
|
||||
for resource in resources
|
||||
]
|
||||
)
|
||||
rbac_domain.transport.responses["/rbac/whitelist/configs"] = configs.model_dump(mode="json")
|
||||
monkeypatch.setattr(task_module, "_iter_resource_config_batches", resource_batches)
|
||||
|
||||
sync_joined_workspace_member_rbac_access_task.run("tenant-1", "member-1", "actor-1")
|
||||
|
||||
lookup, *appends = rbac_domain.transport.requests
|
||||
assert (lookup.method, lookup.endpoint) == ("POST", "/rbac/whitelist/configs")
|
||||
assert lookup.json == {"resources": [resource.model_dump(mode="json") for resource in resources]}
|
||||
assert len(appends) == 3
|
||||
for kind, request in zip(("app", "dataset", "agent"), appends, strict=True):
|
||||
assert (request.method, request.endpoint) == ("POST", f"/rbac/{kind}s/whitelist/members/batch")
|
||||
assert request.json == {
|
||||
"data": [{f"{kind}_id": f"{kind}-1", "account_ids": ["member-1"], "policy_id": "default"}]
|
||||
}
|
||||
for request in (lookup, *appends):
|
||||
assert (request.tenant_id, request.account_id) == ("tenant-1", "actor-1")
|
||||
@@ -0,0 +1,203 @@
|
||||
import logging
|
||||
from types import SimpleNamespace
|
||||
from typing import Protocol, cast
|
||||
from unittest.mock import MagicMock, call, patch
|
||||
|
||||
import pytest
|
||||
from celery.exceptions import Retry
|
||||
|
||||
from core.plugin.entities.plugin_daemon import PluginInstallTaskStartResponse
|
||||
from tests.helpers.unit.config_override import apply_config_overrides
|
||||
|
||||
|
||||
class _TaskWithQueue(Protocol):
|
||||
queue: str
|
||||
|
||||
|
||||
def test_install_default_plugins_task_uses_plugin_queue() -> None:
|
||||
from tasks.install_default_plugins_task import install_default_plugins_task
|
||||
|
||||
assert cast(_TaskWithQueue, install_default_plugins_task).queue == "plugin"
|
||||
|
||||
|
||||
def test_configure_default_models_task_uses_plugin_queue() -> None:
|
||||
from tasks.install_default_plugins_task import configure_default_models_task
|
||||
|
||||
assert cast(_TaskWithQueue, configure_default_models_task).queue == "plugin"
|
||||
|
||||
|
||||
def test_install_default_plugins_task_installs_latest_identifiers(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import tasks.install_default_plugins_task as task_module
|
||||
from tasks.install_default_plugins_task import install_default_plugins_task
|
||||
|
||||
plugin_ids = ["langgenius/openai", "langgenius/gemini"]
|
||||
plugin_identifiers = ["langgenius/openai:1.0.0@aaa", "langgenius/gemini:0.9.1@bbb"]
|
||||
fetch = MagicMock(
|
||||
return_value=[
|
||||
SimpleNamespace(plugin_id=plugin_ids[1], latest_package_identifier=plugin_identifiers[1]),
|
||||
SimpleNamespace(plugin_id=plugin_ids[0], latest_package_identifier=plugin_identifiers[0]),
|
||||
]
|
||||
)
|
||||
install = MagicMock()
|
||||
monkeypatch.setattr(task_module.marketplace, "batch_fetch_plugin_manifests", fetch)
|
||||
monkeypatch.setattr(task_module.PluginService, "install_from_marketplace_pkg", install)
|
||||
|
||||
with patch.object(task_module.configure_default_models_task, "delay") as configure:
|
||||
install_default_plugins_task.run("tenant-1", plugin_ids)
|
||||
|
||||
fetch.assert_called_once_with(plugin_ids)
|
||||
install.assert_called_once_with("tenant-1", plugin_identifiers)
|
||||
configure.assert_called_once_with("tenant-1", None)
|
||||
|
||||
|
||||
def test_install_default_plugins_task_skips_missing_plugins(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
import tasks.install_default_plugins_task as task_module
|
||||
from tasks.install_default_plugins_task import install_default_plugins_task
|
||||
|
||||
fetch = MagicMock(
|
||||
return_value=[
|
||||
SimpleNamespace(plugin_id="langgenius/openai", latest_package_identifier="langgenius/openai:1.0.0@aaa")
|
||||
]
|
||||
)
|
||||
install = MagicMock()
|
||||
monkeypatch.setattr(task_module.marketplace, "batch_fetch_plugin_manifests", fetch)
|
||||
monkeypatch.setattr(task_module.PluginService, "install_from_marketplace_pkg", install)
|
||||
|
||||
with (
|
||||
caplog.at_level(logging.WARNING, logger=task_module.logger.name),
|
||||
patch.object(task_module.configure_default_models_task, "delay"),
|
||||
):
|
||||
install_default_plugins_task.run("tenant-1", ["langgenius/openai", "missing/plugin"])
|
||||
|
||||
install.assert_called_once_with("tenant-1", ["langgenius/openai:1.0.0@aaa"])
|
||||
assert "Default plugins not found in marketplace: missing/plugin" in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("configured_models", ["", "llm:provider:model"])
|
||||
@pytest.mark.parametrize("all_installed", [False, True])
|
||||
def test_install_default_plugins_task_queues_model_configuration_after_daemon_install(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
configured_models: str,
|
||||
all_installed: bool,
|
||||
) -> None:
|
||||
import tasks.install_default_plugins_task as task_module
|
||||
from tasks.install_default_plugins_task import install_default_plugins_task
|
||||
|
||||
plugin_id = "langgenius/openai"
|
||||
plugin_identifier = "langgenius/openai:1.0.0@aaa"
|
||||
apply_config_overrides(monkeypatch, NEW_USER_DEFAULT_MODELS=configured_models)
|
||||
monkeypatch.setattr(
|
||||
task_module.marketplace,
|
||||
"batch_fetch_plugin_manifests",
|
||||
MagicMock(return_value=[SimpleNamespace(plugin_id=plugin_id, latest_package_identifier=plugin_identifier)]),
|
||||
)
|
||||
with (
|
||||
patch.object(
|
||||
task_module.PluginService,
|
||||
"install_from_marketplace_pkg",
|
||||
return_value=PluginInstallTaskStartResponse(all_installed=all_installed, task_id="install-task-1"),
|
||||
),
|
||||
patch.object(task_module.configure_default_models_task, "delay") as delay,
|
||||
):
|
||||
install_default_plugins_task.run("tenant-1", [plugin_id])
|
||||
|
||||
delay.assert_called_once_with("tenant-1", None if all_installed else "install-task-1")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("configured_models", ["", "llm:provider:model"])
|
||||
def test_configure_default_models_task_retries_while_plugins_are_installing(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
configured_models: str,
|
||||
) -> None:
|
||||
import tasks.install_default_plugins_task as task_module
|
||||
from tasks.install_default_plugins_task import configure_default_models_task
|
||||
|
||||
apply_config_overrides(monkeypatch, NEW_USER_DEFAULT_MODELS=configured_models)
|
||||
monkeypatch.setattr(
|
||||
task_module.PluginService,
|
||||
"fetch_install_task",
|
||||
MagicMock(return_value=SimpleNamespace(status=task_module.PluginInstallTaskStatus.Running)),
|
||||
)
|
||||
retry = MagicMock(side_effect=Retry())
|
||||
monkeypatch.setattr(configure_default_models_task, "retry", retry)
|
||||
|
||||
with pytest.raises(Retry):
|
||||
configure_default_models_task.run("tenant-1", "install-task-1")
|
||||
|
||||
retry.assert_called_once_with()
|
||||
|
||||
|
||||
def test_configure_default_models_task_sets_each_explicit_model(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import tasks.install_default_plugins_task as task_module
|
||||
from tasks.install_default_plugins_task import configure_default_models_task
|
||||
|
||||
apply_config_overrides(
|
||||
monkeypatch,
|
||||
NEW_USER_DEFAULT_MODELS=(
|
||||
"llm:langgenius/openai/openai:gpt-4o-mini,text-embedding:langgenius/openai/openai:text-embedding-3-small"
|
||||
),
|
||||
)
|
||||
fetch_install_task = MagicMock(
|
||||
return_value=SimpleNamespace(
|
||||
status=task_module.PluginInstallTaskStatus.Success,
|
||||
plugins=[],
|
||||
)
|
||||
)
|
||||
monkeypatch.setattr(task_module.PluginService, "fetch_install_task", fetch_install_task)
|
||||
model_provider_service = MagicMock()
|
||||
monkeypatch.setattr(task_module, "ModelProviderService", MagicMock(return_value=model_provider_service))
|
||||
|
||||
configure_default_models_task.run("tenant-1", "install-task-1")
|
||||
|
||||
fetch_install_task.assert_called_once_with("tenant-1", "install-task-1")
|
||||
assert model_provider_service.update_default_model_of_model_type.call_args_list == [
|
||||
call(
|
||||
tenant_id="tenant-1",
|
||||
model_type="llm",
|
||||
provider="langgenius/openai/openai",
|
||||
model="gpt-4o-mini",
|
||||
),
|
||||
call(
|
||||
tenant_id="tenant-1",
|
||||
model_type="text-embedding",
|
||||
provider="langgenius/openai/openai",
|
||||
model="text-embedding-3-small",
|
||||
),
|
||||
]
|
||||
model_provider_service.initialize_default_llm.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("configured_models", ["", "text-embedding:provider:embedding-model"])
|
||||
def test_configure_default_models_task_initializes_llm_without_an_explicit_llm(
|
||||
monkeypatch: pytest.MonkeyPatch, configured_models: str
|
||||
) -> None:
|
||||
import tasks.install_default_plugins_task as task_module
|
||||
|
||||
apply_config_overrides(monkeypatch, NEW_USER_DEFAULT_MODELS=configured_models)
|
||||
with (
|
||||
patch.object(task_module.ModelProviderService, "initialize_default_llm") as initialize,
|
||||
patch.object(task_module.ModelProviderService, "update_default_model_of_model_type") as update,
|
||||
):
|
||||
task_module.configure_default_models_task.run("tenant-1", None)
|
||||
|
||||
initialize.assert_called_once_with("tenant-1")
|
||||
if configured_models:
|
||||
update.assert_called_once_with(
|
||||
tenant_id="tenant-1", model_type="text-embedding", provider="provider", model="embedding-model"
|
||||
)
|
||||
else:
|
||||
update.assert_not_called()
|
||||
|
||||
|
||||
def test_configure_default_models_task_reports_initialization_failure(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import tasks.install_default_plugins_task as task_module
|
||||
|
||||
apply_config_overrides(monkeypatch, NEW_USER_DEFAULT_MODELS="")
|
||||
with patch.object(
|
||||
task_module.ModelProviderService, "initialize_default_llm", side_effect=RuntimeError("unavailable")
|
||||
):
|
||||
with pytest.raises(RuntimeError, match="Failed to initialize defaults for tenant tenant-1; model types: llm"):
|
||||
task_module.configure_default_models_task.run("tenant-1", None)
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Literal
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from celery.exceptions import Retry
|
||||
from dify_agent.protocol.knowledge_fs import KnowledgeFsError
|
||||
from dify_agent.protocol.knowledge_investigation import KnowledgeInvestigationPayload
|
||||
|
||||
from services.knowledge_fs.product_remote import KnowledgeFSProductRequestRejectedError
|
||||
from tasks import knowledge_fs_agent_investigation_tasks as module
|
||||
from tests.helpers.unit.tasks.knowledge_fs_task_fakes import RecordingRetry
|
||||
from tests.helpers.unit.tasks.task_options import task_options
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Gateway:
|
||||
"""Records investigation captures; raises ``error`` when one is configured."""
|
||||
|
||||
error: Exception | None = None
|
||||
captures: list[tuple[KnowledgeInvestigationPayload, str]] = field(default_factory=list)
|
||||
|
||||
def capture_investigation(self, report: KnowledgeInvestigationPayload, control_space_id: str) -> None:
|
||||
self.captures.append((report, control_space_id))
|
||||
if self.error is not None:
|
||||
raise self.error
|
||||
|
||||
|
||||
def report() -> tuple[dict[str, object], str]:
|
||||
space = str(uuid4())
|
||||
attempts: list[dict[str, object]] = []
|
||||
return {
|
||||
"investigation_id": str(uuid4()),
|
||||
"execution_context": {"tenant_id": "tenant", "agent_mode": "agent_app", "invoke_from": "published"},
|
||||
"bindings": [{"id": "docs", "name": "Docs", "control_space_id": space}],
|
||||
"query": "refund",
|
||||
"status": "completed",
|
||||
"attempts": attempts,
|
||||
}, space
|
||||
|
||||
|
||||
def test_dataset_worker_reconstructs_report_for_each_delivery(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
gateway = _Gateway()
|
||||
monkeypatch.setattr(module, "AgentKnowledgeGateway", lambda: gateway)
|
||||
payload, space = report()
|
||||
task = module.capture_agent_knowledge_investigation_task
|
||||
task.run(report=payload, control_space_id=space)
|
||||
task.run(report=payload, control_space_id=space)
|
||||
assert task_options(task)["queue"] == "dataset"
|
||||
first, second = gateway.captures
|
||||
assert first == second
|
||||
assert first[1] == space
|
||||
assert str(first[0].investigation_id) == payload["investigation_id"]
|
||||
|
||||
|
||||
def test_revoked_authority_stops_publication_and_transient_failures_retry(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
gateway = _Gateway(error=KnowledgeFsError("KNOWLEDGE_SCOPE_CHANGED", "revoked", 403))
|
||||
monkeypatch.setattr(module, "AgentKnowledgeGateway", lambda: gateway)
|
||||
payload, space = report()
|
||||
task = module.capture_agent_knowledge_investigation_task
|
||||
retry = RecordingRetry()
|
||||
monkeypatch.setattr(task, "retry", retry)
|
||||
task.run(report=payload, control_space_id=space)
|
||||
assert retry.calls == []
|
||||
gateway.error = OSError("offline")
|
||||
with pytest.raises(Retry):
|
||||
task.run(report=payload, control_space_id=space)
|
||||
assert len(retry.calls) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status_code", [400, 403, 409, 413, 422])
|
||||
def test_permanent_remote_rejections_do_not_retry(
|
||||
monkeypatch: pytest.MonkeyPatch, status_code: Literal[400, 403, 409, 413, 422]
|
||||
) -> None:
|
||||
gateway = _Gateway(error=KnowledgeFSProductRequestRejectedError(status_code=status_code))
|
||||
monkeypatch.setattr(module, "AgentKnowledgeGateway", lambda: gateway)
|
||||
task = module.capture_agent_knowledge_investigation_task
|
||||
retry = RecordingRetry()
|
||||
monkeypatch.setattr(task, "retry", retry)
|
||||
payload, space = report()
|
||||
|
||||
task.run(report=payload, control_space_id=space)
|
||||
|
||||
assert len(gateway.captures) == 1
|
||||
assert retry.calls == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("retries", "countdown"), [(0, 30), (1, 60), (2, 120)])
|
||||
def test_rate_limited_report_retries_with_backoff(
|
||||
monkeypatch: pytest.MonkeyPatch, retries: int, countdown: int
|
||||
) -> None:
|
||||
error = KnowledgeFSProductRequestRejectedError(status_code=429)
|
||||
gateway = _Gateway(error=error)
|
||||
monkeypatch.setattr(module, "AgentKnowledgeGateway", lambda: gateway)
|
||||
task = module.capture_agent_knowledge_investigation_task
|
||||
retry = RecordingRetry()
|
||||
monkeypatch.setattr(task, "retry", retry)
|
||||
monkeypatch.setattr(task.request, "retries", retries)
|
||||
payload, space = report()
|
||||
|
||||
with pytest.raises(Retry):
|
||||
task.run(report=payload, control_space_id=space)
|
||||
|
||||
assert retry.calls == [{"exc": error, "countdown": countdown}]
|
||||
|
||||
|
||||
def test_malformed_queued_report_is_rejected_before_gateway_publication(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
gateway = _Gateway()
|
||||
monkeypatch.setattr(module, "AgentKnowledgeGateway", lambda: gateway)
|
||||
task = module.capture_agent_knowledge_investigation_task
|
||||
retry = RecordingRetry()
|
||||
monkeypatch.setattr(task, "retry", retry)
|
||||
|
||||
task.run(report={"investigation_id": "invalid"}, control_space_id=str(uuid4()))
|
||||
|
||||
assert gateway.captures == []
|
||||
assert retry.calls == []
|
||||
@@ -0,0 +1,66 @@
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from celery.app.task import Context
|
||||
from celery.exceptions import Reject, Retry
|
||||
|
||||
from tasks.knowledge_fs_background_tasks import execute_delivery, execute_operation
|
||||
|
||||
JOB: dict[str, object] = {
|
||||
"id": "018f0d60-7a49-7cc2-9c1b-5b36f18f8a01",
|
||||
"type": "document.compile",
|
||||
"payload": {"attemptId": "018f0d60-7a49-7cc2-9c1b-5b36f18f8a02"},
|
||||
"attempts": 1,
|
||||
}
|
||||
|
||||
|
||||
def test_delivery_uses_late_ack_and_passes_broker_retry_count_to_engine() -> None:
|
||||
assert execute_delivery.acks_late is True
|
||||
assert execute_delivery.reject_on_worker_lost is True
|
||||
execute_delivery.request_stack.push(Context(id=JOB["id"], retries=2))
|
||||
try:
|
||||
with patch(
|
||||
"tasks.knowledge_fs_background_tasks.background_engine.execute", return_value={"outcome": "completed"}
|
||||
) as engine:
|
||||
assert execute_delivery.run(delivery=JOB) == {"outcome": "completed"}
|
||||
engine.assert_called_once_with("delivery", {**JOB, "attempts": 3})
|
||||
finally:
|
||||
execute_delivery.request_stack.pop()
|
||||
|
||||
|
||||
def test_bad_delivery_identity_never_enters_engine() -> None:
|
||||
execute_delivery.request_stack.push(Context(id="different"))
|
||||
try:
|
||||
with patch("tasks.knowledge_fs_background_tasks.background_engine.execute") as engine:
|
||||
with pytest.raises(Reject):
|
||||
execute_delivery.run(delivery=JOB)
|
||||
engine.assert_not_called()
|
||||
finally:
|
||||
execute_delivery.request_stack.pop()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("response", [{"outcome": "retry", "runAfter": 0}, OSError("engine died")])
|
||||
def test_early_delivery_and_engine_loss_retry_without_acknowledging(response: object) -> None:
|
||||
execute_delivery.request_stack.push(Context(id=JOB["id"], retries=0))
|
||||
try:
|
||||
with (
|
||||
patch("tasks.knowledge_fs_background_tasks.background_engine.execute") as engine,
|
||||
patch.object(execute_delivery, "retry", side_effect=Retry()) as retry,
|
||||
):
|
||||
if isinstance(response, Exception):
|
||||
engine.side_effect = response
|
||||
else:
|
||||
engine.return_value = response
|
||||
with pytest.raises(Retry):
|
||||
execute_delivery.run(delivery=JOB)
|
||||
retry.assert_called_once()
|
||||
assert 1 <= retry.call_args.kwargs["countdown"] <= 300
|
||||
finally:
|
||||
execute_delivery.request_stack.pop()
|
||||
|
||||
|
||||
def test_unknown_operation_is_not_executable() -> None:
|
||||
with patch("tasks.knowledge_fs_background_tasks.background_engine.execute") as engine:
|
||||
with pytest.raises(Reject):
|
||||
execute_operation.run(operation="shell")
|
||||
engine.assert_not_called()
|
||||
+186
@@ -0,0 +1,186 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Literal
|
||||
|
||||
import pytest
|
||||
from celery.app.task import Context
|
||||
from celery.exceptions import Retry
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
from services.knowledge_fs.app_admission_service import KnowledgeFSAppAdmissionError
|
||||
from services.knowledge_fs.product_dto import (
|
||||
KnowledgeFSWorkflowFailedRetrievalCapturePayload,
|
||||
KnowledgeFSWorkflowFailedRetrievalCaptureResponse,
|
||||
)
|
||||
from services.knowledge_fs.product_remote import KnowledgeFSProductRequestRejectedError
|
||||
from services.knowledge_fs.resource import KnowledgeResourceRef
|
||||
from tasks import knowledge_fs_failed_retrieval_tasks as task_module
|
||||
from tests.helpers.unit.tasks.knowledge_fs_task_fakes import RecordingRetry
|
||||
from tests.helpers.unit.tasks.task_options import task_options
|
||||
|
||||
EVENT_ID = "019fac9f-bfb0-75ee-9af5-252ebafbac1e"
|
||||
|
||||
|
||||
class FailedRetrievalTaskArguments(TypedDict, closed=True):
|
||||
event_id: str
|
||||
tenant_id: str
|
||||
app_id: str
|
||||
control_space_id: str
|
||||
query: str
|
||||
mode: Literal["fast", "deep", "research"]
|
||||
retrieval_trace_id: str
|
||||
|
||||
|
||||
def _task_kwargs() -> FailedRetrievalTaskArguments:
|
||||
return {
|
||||
"event_id": EVENT_ID,
|
||||
"tenant_id": "tenant-1",
|
||||
"app_id": "app-1",
|
||||
"control_space_id": "control-1",
|
||||
"query": "missing answer",
|
||||
"mode": "fast",
|
||||
"retrieval_trace_id": "retrieval-trace-1",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _CaptureCall:
|
||||
tenant_id: str
|
||||
app_id: str
|
||||
resource: KnowledgeResourceRef
|
||||
payload: KnowledgeFSWorkflowFailedRetrievalCapturePayload
|
||||
|
||||
|
||||
@dataclass
|
||||
class _AppCapabilities:
|
||||
outcome: KnowledgeFSWorkflowFailedRetrievalCaptureResponse | Exception
|
||||
calls: list[_CaptureCall] = field(default_factory=list)
|
||||
|
||||
def capture_workflow_failed_retrieval(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
app_id: str,
|
||||
resource: KnowledgeResourceRef,
|
||||
payload: KnowledgeFSWorkflowFailedRetrievalCapturePayload,
|
||||
) -> KnowledgeFSWorkflowFailedRetrievalCaptureResponse:
|
||||
self.calls.append(_CaptureCall(tenant_id=tenant_id, app_id=app_id, resource=resource, payload=payload))
|
||||
if isinstance(self.outcome, Exception):
|
||||
raise self.outcome
|
||||
return self.outcome
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Runtime:
|
||||
app_capabilities: _AppCapabilities
|
||||
|
||||
|
||||
def _install_capability(
|
||||
monkeypatch: pytest.MonkeyPatch, outcome: KnowledgeFSWorkflowFailedRetrievalCaptureResponse | Exception
|
||||
) -> _AppCapabilities:
|
||||
capability = _AppCapabilities(outcome)
|
||||
monkeypatch.setattr(task_module, "get_knowledge_fs_runtime", lambda _session_maker: _Runtime(capability))
|
||||
monkeypatch.setattr(task_module.session_factory, "get_session_maker", lambda: object())
|
||||
return capability
|
||||
|
||||
|
||||
def _run_task_with_retries(retries: int) -> None:
|
||||
task = task_module.capture_workflow_failed_retrieval_task
|
||||
task.request_stack.push(Context(retries=retries))
|
||||
try:
|
||||
task.run(**_task_kwargs())
|
||||
finally:
|
||||
task.pop_request()
|
||||
|
||||
|
||||
def test_failed_retrieval_task_runs_on_dataset_queue_and_reauthorizes_app(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
capability = _install_capability(
|
||||
monkeypatch,
|
||||
KnowledgeFSWorkflowFailedRetrievalCaptureResponse.model_construct(
|
||||
failed_query_id="failed-query-1",
|
||||
verdict="retrieval-miss",
|
||||
bad_case_id="bad-case-1",
|
||||
),
|
||||
)
|
||||
|
||||
task_module.capture_workflow_failed_retrieval_task.run(**_task_kwargs())
|
||||
|
||||
assert task_options(task_module.capture_workflow_failed_retrieval_task)["queue"] == "dataset"
|
||||
assert len(capability.calls) == 1
|
||||
call = capability.calls[0]
|
||||
assert call.tenant_id == "tenant-1"
|
||||
assert call.app_id == "app-1"
|
||||
assert call.resource.control_space_id == "control-1"
|
||||
assert str(call.payload.event_id) == EVENT_ID
|
||||
assert call.payload.query == "missing answer"
|
||||
assert call.payload.retrieval_trace_id == "retrieval-trace-1"
|
||||
|
||||
|
||||
def test_enqueue_is_idempotency_aware_and_never_propagates_broker_failure(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
delayed: list[dict[str, object]] = []
|
||||
monkeypatch.setattr(
|
||||
task_module.capture_workflow_failed_retrieval_task, "delay", lambda **kwargs: delayed.append(kwargs)
|
||||
)
|
||||
|
||||
task_module.enqueue_workflow_failed_retrieval_capture(**_task_kwargs())
|
||||
|
||||
assert delayed == [_task_kwargs()]
|
||||
|
||||
def unavailable_broker(**kwargs: object) -> None:
|
||||
delayed.append(kwargs)
|
||||
raise RuntimeError("broker unavailable")
|
||||
|
||||
monkeypatch.setattr(task_module.capture_workflow_failed_retrieval_task, "delay", unavailable_broker)
|
||||
task_module.enqueue_workflow_failed_retrieval_capture(**_task_kwargs())
|
||||
assert len(delayed) == 2
|
||||
|
||||
|
||||
def test_terminal_admission_rejection_is_swallowed(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
capability = _install_capability(monkeypatch, KnowledgeFSAppAdmissionError("not admitted"))
|
||||
|
||||
task_module.capture_workflow_failed_retrieval_task.run(**_task_kwargs())
|
||||
|
||||
assert len(capability.calls) == 1
|
||||
|
||||
|
||||
def test_rate_limit_rejection_retries_with_initial_backoff(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
error = KnowledgeFSProductRequestRejectedError(status_code=429)
|
||||
capability = _install_capability(monkeypatch, error)
|
||||
retry = RecordingRetry()
|
||||
monkeypatch.setattr(task_module.capture_workflow_failed_retrieval_task, "retry", retry)
|
||||
|
||||
with pytest.raises(Retry):
|
||||
_run_task_with_retries(0)
|
||||
|
||||
assert len(capability.calls) == 1
|
||||
assert retry.calls == [{"exc": error, "countdown": 30}]
|
||||
|
||||
|
||||
def test_transient_failure_retries_with_exponential_backoff(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
error = RuntimeError("KnowledgeFS temporarily unavailable")
|
||||
capability = _install_capability(monkeypatch, error)
|
||||
retry = RecordingRetry()
|
||||
monkeypatch.setattr(task_module.capture_workflow_failed_retrieval_task, "retry", retry)
|
||||
|
||||
with pytest.raises(Retry):
|
||||
_run_task_with_retries(1)
|
||||
|
||||
assert len(capability.calls) == 1
|
||||
assert retry.calls == [{"exc": error, "countdown": 60}]
|
||||
|
||||
|
||||
def test_retry_budget_exhaustion_raises_original_failure(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
error = RuntimeError("KnowledgeFS remained unavailable")
|
||||
capability = _install_capability(monkeypatch, error)
|
||||
retry = RecordingRetry()
|
||||
monkeypatch.setattr(task_module.capture_workflow_failed_retrieval_task, "retry", retry)
|
||||
|
||||
with pytest.raises(RuntimeError, match="KnowledgeFS remained unavailable") as exc_info:
|
||||
_run_task_with_retries(3)
|
||||
|
||||
assert exc_info.value is error
|
||||
assert len(capability.calls) == 1
|
||||
assert retry.calls == []
|
||||
+183
@@ -0,0 +1,183 @@
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Never
|
||||
from unittest.mock import patch
|
||||
|
||||
from sqlalchemy import inspect
|
||||
|
||||
from core.db.session_factory import session_factory
|
||||
from models.account import Account, AccountStatus, Tenant, TenantAccountJoin, TenantAccountRole
|
||||
from services.knowledge_fs.initial_source_preview import KnowledgeFSInitialSourcePreviewCanceledError
|
||||
from services.knowledge_fs.product_dto import (
|
||||
KnowledgeFSInitialSourcePreviewJobResponse,
|
||||
KnowledgeFSInitialSourcePreviewResponse,
|
||||
)
|
||||
from tasks.knowledge_fs_initial_source_preview_tasks import run_knowledge_fs_initial_source_preview
|
||||
from tests.helpers.unit.tasks.task_options import task_options
|
||||
|
||||
_JOB_KEY = {"tenant_id": "tenant-1", "account_id": "account-1", "job_id": "job-1"}
|
||||
|
||||
|
||||
@dataclass
|
||||
class _PreviewJobService:
|
||||
"""Records preview job transitions; ``transition_results`` are consumed in order."""
|
||||
|
||||
transition_results: list[bool]
|
||||
job_status: str = "running"
|
||||
transitions: list[dict[str, object]] = field(default_factory=list)
|
||||
stored: list[dict[str, object]] = field(default_factory=list)
|
||||
released: list[dict[str, str]] = field(default_factory=list)
|
||||
lookups: list[dict[str, str]] = field(default_factory=list)
|
||||
|
||||
def transition_status(self, **kwargs: object) -> bool:
|
||||
self.transitions.append(kwargs)
|
||||
return self.transition_results.pop(0)
|
||||
|
||||
def store_content(self, **kwargs: object) -> None:
|
||||
self.stored.append(kwargs)
|
||||
|
||||
def release_active_job(self, *, tenant_id: str, account_id: str, job_id: str) -> None:
|
||||
self.released.append({"tenant_id": tenant_id, "account_id": account_id, "job_id": job_id})
|
||||
|
||||
def get(self, *, tenant_id: str, account_id: str, job_id: str) -> KnowledgeFSInitialSourcePreviewJobResponse:
|
||||
self.lookups.append({"tenant_id": tenant_id, "account_id": account_id, "job_id": job_id})
|
||||
return KnowledgeFSInitialSourcePreviewJobResponse.model_construct(status=self.job_status)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _PreviewService:
|
||||
run: Callable[..., KnowledgeFSInitialSourcePreviewResponse]
|
||||
calls: list[dict[str, object]] = field(default_factory=list)
|
||||
|
||||
def preview(self, **kwargs: object) -> KnowledgeFSInitialSourcePreviewResponse:
|
||||
self.calls.append(kwargs)
|
||||
return self.run(**kwargs)
|
||||
|
||||
|
||||
def _payload() -> dict[str, object]:
|
||||
return {
|
||||
"credentialId": "credential-1",
|
||||
"datasource": "crawl",
|
||||
"kind": "website_crawl",
|
||||
"parameters": {"url": "https://docs.dify.ai"},
|
||||
"pluginId": "langgenius/firecrawl_datasource",
|
||||
"provider": "firecrawl",
|
||||
}
|
||||
|
||||
|
||||
def _seed_account(*, member: bool) -> None:
|
||||
tenant = Tenant(name="Preview tenant")
|
||||
tenant.id = "tenant-1"
|
||||
account = Account(name="Preview user", email="preview-user@example.com", status=AccountStatus.ACTIVE)
|
||||
account.id = "account-1"
|
||||
with session_factory.create_session() as session:
|
||||
session.add_all([tenant, account])
|
||||
if member:
|
||||
session.add(
|
||||
TenantAccountJoin(
|
||||
tenant_id=tenant.id,
|
||||
account_id=account.id,
|
||||
current=True,
|
||||
role=TenantAccountRole.OWNER,
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
|
||||
def _run_preview(job_service: _PreviewJobService, preview_service: _PreviewService | None) -> list[dict[str, object]]:
|
||||
"""Run the task and return the cleanup schedules it requested."""
|
||||
scheduled_cleanups: list[dict[str, object]] = []
|
||||
with (
|
||||
patch(
|
||||
"tasks.knowledge_fs_initial_source_preview_tasks.KnowledgeFSInitialSourcePreviewJobService",
|
||||
return_value=job_service,
|
||||
),
|
||||
patch(
|
||||
"tasks.knowledge_fs_initial_source_preview_tasks.KnowledgeFSInitialSourcePreviewService",
|
||||
return_value=preview_service,
|
||||
) as preview_service_type,
|
||||
patch(
|
||||
"tasks.knowledge_fs_initial_source_preview_tasks.cleanup_knowledge_fs_initial_source_preview.apply_async",
|
||||
side_effect=lambda **kwargs: scheduled_cleanups.append(kwargs),
|
||||
),
|
||||
):
|
||||
run_knowledge_fs_initial_source_preview.run(
|
||||
tenant_id="tenant-1",
|
||||
account_id="account-1",
|
||||
job_id="job-1",
|
||||
payload=_payload(),
|
||||
)
|
||||
if preview_service is None:
|
||||
preview_service_type.assert_not_called()
|
||||
return scheduled_cleanups
|
||||
|
||||
|
||||
def test_preview_task_uses_the_dataset_queue() -> None:
|
||||
assert task_options(run_knowledge_fs_initial_source_preview)["queue"] == "dataset"
|
||||
|
||||
|
||||
def test_preview_task_persists_running_and_completed_states() -> None:
|
||||
_seed_account(member=True)
|
||||
job_service = _PreviewJobService(transition_results=[True, True])
|
||||
result = KnowledgeFSInitialSourcePreviewResponse(kind="website_crawl")
|
||||
preview_service = _PreviewService(run=lambda **_: result)
|
||||
|
||||
scheduled_cleanups = _run_preview(job_service, preview_service)
|
||||
|
||||
assert job_service.transitions == [
|
||||
{**_JOB_KEY, "status": "running", "allowed_from": ("pending",)},
|
||||
{**_JOB_KEY, "status": "completed", "allowed_from": ("running",), "result": result},
|
||||
]
|
||||
assert job_service.stored == [{**_JOB_KEY, "result": result}]
|
||||
assert scheduled_cleanups == [
|
||||
{"kwargs": {"account_id": "account-1", "job_id": "job-1", "tenant_id": "tenant-1"}, "countdown": 3600}
|
||||
]
|
||||
assert job_service.released == [_JOB_KEY]
|
||||
assert len(preview_service.calls) == 1
|
||||
account = preview_service.calls[0]["account"]
|
||||
assert isinstance(account, Account)
|
||||
assert account.id == "account-1"
|
||||
# The tenant was selected through the task's session before the account was detached from it.
|
||||
assert account.current_tenant_id == "tenant-1"
|
||||
assert inspect(account).detached
|
||||
|
||||
|
||||
def test_preview_task_does_not_start_a_canceled_job() -> None:
|
||||
job_service = _PreviewJobService(transition_results=[False])
|
||||
|
||||
_run_preview(job_service, preview_service=None)
|
||||
|
||||
assert job_service.transitions == [{**_JOB_KEY, "status": "running", "allowed_from": ("pending",)}]
|
||||
assert job_service.released == [_JOB_KEY]
|
||||
|
||||
|
||||
def test_preview_task_keeps_a_cooperatively_canceled_job_terminal() -> None:
|
||||
_seed_account(member=True)
|
||||
job_service = _PreviewJobService(transition_results=[True], job_status="canceled")
|
||||
|
||||
def preview(**kwargs: object) -> Never:
|
||||
is_canceled = kwargs["is_canceled"]
|
||||
assert callable(is_canceled)
|
||||
assert is_canceled() is True
|
||||
raise KnowledgeFSInitialSourcePreviewCanceledError("canceled")
|
||||
|
||||
preview_service = _PreviewService(run=preview)
|
||||
|
||||
_run_preview(job_service, preview_service)
|
||||
|
||||
assert job_service.transitions == [{**_JOB_KEY, "status": "running", "allowed_from": ("pending",)}]
|
||||
assert len(preview_service.calls) == 1
|
||||
assert job_service.lookups == [_JOB_KEY]
|
||||
|
||||
|
||||
def test_preview_task_rejects_an_account_removed_from_the_tenant() -> None:
|
||||
_seed_account(member=False)
|
||||
job_service = _PreviewJobService(transition_results=[True, True])
|
||||
|
||||
_run_preview(job_service, preview_service=None)
|
||||
|
||||
assert job_service.transitions[-1] == {
|
||||
**_JOB_KEY,
|
||||
"status": "failed",
|
||||
"allowed_from": ("pending", "running"),
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from services.knowledge_fs.initial_source_import import KnowledgeFSInitialSourceNotReadyError
|
||||
from services.knowledge_fs.product_dto import (
|
||||
KnowledgeFSInitialOnlineDocumentSourcePayload,
|
||||
KnowledgeFSInitialOnlineDriveSourcePayload,
|
||||
KnowledgeFSInitialWebsiteSourcePayload,
|
||||
)
|
||||
from services.knowledge_fs.product_remote import (
|
||||
KnowledgeFSProductRemoteError,
|
||||
KnowledgeFSProductRequestRejectedError,
|
||||
KnowledgeFSProductResourceNotFoundError,
|
||||
)
|
||||
from tasks.knowledge_fs_initial_source_tasks import import_initial_source, import_initial_website_source
|
||||
|
||||
|
||||
def _payload(sync_policy: str = "daily") -> KnowledgeFSInitialWebsiteSourcePayload:
|
||||
return KnowledgeFSInitialWebsiteSourcePayload.model_validate(
|
||||
{
|
||||
"kind": "website_crawl",
|
||||
"name": "Dify docs",
|
||||
"provider": "firecrawl",
|
||||
"providerDisplayName": "Firecrawl",
|
||||
"parameters": {"url": "https://docs.dify.ai", "limit": 25},
|
||||
"root_url": "https://docs.dify.ai",
|
||||
"crawl_options": {"include_subpages": True, "limit": 25},
|
||||
"selection": [
|
||||
{"source_url": "https://docs.dify.ai/a", "title": "A"},
|
||||
{"source_url": "https://docs.dify.ai/b", "title": "B"},
|
||||
],
|
||||
**({"custom_interval_seconds": 129_600} if sync_policy == "custom" else {}),
|
||||
"sync_policy": sync_policy,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _legacy_payload() -> KnowledgeFSInitialWebsiteSourcePayload:
|
||||
return KnowledgeFSInitialWebsiteSourcePayload.model_validate(
|
||||
{
|
||||
"kind": "website_crawl",
|
||||
"name": "Dify docs",
|
||||
"provider": "firecrawl",
|
||||
"providerDisplayName": "Firecrawl",
|
||||
"root_url": "https://docs.dify.ai",
|
||||
"crawl_options": {"include_subpages": False, "limit": 25},
|
||||
"selection": [{"source_url": "https://docs.dify.ai/a", "title": "A"}],
|
||||
"sync_policy": "manual",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _document_payload() -> KnowledgeFSInitialOnlineDocumentSourcePayload:
|
||||
return KnowledgeFSInitialOnlineDocumentSourcePayload.model_validate(
|
||||
{
|
||||
"kind": "online_document",
|
||||
"name": "Product wiki",
|
||||
"pluginId": "langgenius/notion_datasource",
|
||||
"provider": "notion",
|
||||
"providerDisplayName": "Notion",
|
||||
"datasource": "pages",
|
||||
"credentialId": "notion-credential-1",
|
||||
"selection": [
|
||||
{
|
||||
"lastEditedTime": "2026-08-10T00:00:00Z",
|
||||
"name": "Roadmap",
|
||||
"pageId": "page-1",
|
||||
"providerItemId": "notion:page-1",
|
||||
"type": "page",
|
||||
"workspaceId": "workspace-1",
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _drive_payload() -> KnowledgeFSInitialOnlineDriveSourcePayload:
|
||||
return KnowledgeFSInitialOnlineDriveSourcePayload.model_validate(
|
||||
{
|
||||
"kind": "online_drive",
|
||||
"name": "Team drive",
|
||||
"pluginId": "langgenius/google_drive",
|
||||
"provider": "google_drive",
|
||||
"providerDisplayName": "Google Drive",
|
||||
"datasource": "google_drive",
|
||||
"credentialId": "drive-credential-1",
|
||||
"selection": [
|
||||
{
|
||||
"id": "file-1",
|
||||
"mimeType": "application/pdf",
|
||||
"name": "Plan.pdf",
|
||||
"providerItemId": "google-drive:file-1",
|
||||
}
|
||||
],
|
||||
"sync_policy": "manual",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_initial_website_source_task_returns_result_and_retries_not_ready_error() -> None:
|
||||
serialized_payload = _payload().model_dump(mode="json")
|
||||
with patch(
|
||||
"tasks.knowledge_fs_initial_source_tasks.start_initial_website_source_import",
|
||||
return_value="workflow-1",
|
||||
):
|
||||
assert (
|
||||
import_initial_website_source.run(
|
||||
tenant_id="tenant-1",
|
||||
account_id="account-1",
|
||||
control_space_id="control-1",
|
||||
operation_id="operation-1",
|
||||
payload=serialized_payload,
|
||||
)
|
||||
== "workflow-1"
|
||||
)
|
||||
|
||||
retry_error = RuntimeError("retry requested")
|
||||
with (
|
||||
patch(
|
||||
"tasks.knowledge_fs_initial_source_tasks.start_initial_website_source_import",
|
||||
side_effect=KnowledgeFSInitialSourceNotReadyError("not ready"),
|
||||
),
|
||||
patch.object(import_initial_website_source, "retry", side_effect=retry_error) as retry,
|
||||
pytest.raises(RuntimeError, match="retry requested"),
|
||||
):
|
||||
import_initial_website_source.run(
|
||||
tenant_id="tenant-1",
|
||||
account_id="account-1",
|
||||
control_space_id="control-1",
|
||||
operation_id="operation-1",
|
||||
payload=serialized_payload,
|
||||
)
|
||||
retry.assert_called_once()
|
||||
|
||||
|
||||
def test_initial_website_source_task_retries_with_workflow_id() -> None:
|
||||
serialized_payload = _payload().model_dump(mode="json")
|
||||
retry_error = RuntimeError("retry requested")
|
||||
with (
|
||||
patch(
|
||||
"tasks.knowledge_fs_initial_source_tasks.start_initial_website_source_import",
|
||||
side_effect=KnowledgeFSInitialSourceNotReadyError(
|
||||
"not ready",
|
||||
workflow_id="workflow-1",
|
||||
),
|
||||
),
|
||||
patch.object(import_initial_website_source, "retry", side_effect=retry_error) as retry,
|
||||
pytest.raises(RuntimeError, match="retry requested"),
|
||||
):
|
||||
import_initial_website_source.run(
|
||||
tenant_id="tenant-1",
|
||||
account_id="account-1",
|
||||
control_space_id="control-1",
|
||||
operation_id="operation-1",
|
||||
payload=serialized_payload,
|
||||
)
|
||||
|
||||
retry.assert_called_once_with(
|
||||
exc=retry.call_args.kwargs["exc"],
|
||||
kwargs={
|
||||
"tenant_id": "tenant-1",
|
||||
"account_id": "account-1",
|
||||
"control_space_id": "control-1",
|
||||
"operation_id": "operation-1",
|
||||
"payload": serialized_payload,
|
||||
"workflow_id": "workflow-1",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_initial_source_task_validates_discriminated_connector_payload() -> None:
|
||||
serialized_payload = _document_payload().model_dump(mode="json", by_alias=True)
|
||||
with patch(
|
||||
"tasks.knowledge_fs_initial_source_tasks.start_initial_source_import",
|
||||
return_value="workflow-1",
|
||||
) as start_import:
|
||||
assert (
|
||||
import_initial_source.run(
|
||||
tenant_id="tenant-1",
|
||||
account_id="account-1",
|
||||
control_space_id="control-1",
|
||||
operation_id="operation-1",
|
||||
payload=serialized_payload,
|
||||
)
|
||||
== "workflow-1"
|
||||
)
|
||||
|
||||
parsed_payload = start_import.call_args.kwargs["payload"]
|
||||
assert isinstance(parsed_payload, KnowledgeFSInitialOnlineDocumentSourcePayload)
|
||||
assert parsed_payload.credential_id == "notion-credential-1"
|
||||
|
||||
|
||||
def test_initial_source_task_retries_transient_remote_error() -> None:
|
||||
serialized_payload = _drive_payload().model_dump(mode="json", by_alias=True)
|
||||
remote_error = KnowledgeFSProductRemoteError("temporary outage")
|
||||
retry_error = RuntimeError("retry requested")
|
||||
with (
|
||||
patch(
|
||||
"tasks.knowledge_fs_initial_source_tasks.start_initial_source_import",
|
||||
side_effect=remote_error,
|
||||
),
|
||||
patch.object(import_initial_source, "retry", side_effect=retry_error) as retry,
|
||||
pytest.raises(RuntimeError, match="retry requested"),
|
||||
):
|
||||
import_initial_source.run(
|
||||
tenant_id="tenant-1",
|
||||
account_id="account-1",
|
||||
control_space_id="control-1",
|
||||
operation_id="operation-1",
|
||||
payload=serialized_payload,
|
||||
workflow_id="workflow-1",
|
||||
)
|
||||
|
||||
retry.assert_called_once_with(exc=remote_error)
|
||||
|
||||
|
||||
def test_initial_source_task_retries_rate_limited_request() -> None:
|
||||
serialized_payload = _payload().model_dump(mode="json", by_alias=True)
|
||||
rejected = KnowledgeFSProductRequestRejectedError(status_code=429)
|
||||
retry_error = RuntimeError("retry requested")
|
||||
with (
|
||||
patch(
|
||||
"tasks.knowledge_fs_initial_source_tasks.start_initial_source_import",
|
||||
side_effect=rejected,
|
||||
),
|
||||
patch.object(import_initial_source, "retry", side_effect=retry_error) as retry,
|
||||
pytest.raises(RuntimeError, match="retry requested"),
|
||||
):
|
||||
import_initial_source.run(
|
||||
tenant_id="tenant-1",
|
||||
account_id="account-1",
|
||||
control_space_id="control-1",
|
||||
operation_id="operation-1",
|
||||
payload=serialized_payload,
|
||||
workflow_id=None,
|
||||
)
|
||||
|
||||
retry.assert_called_once_with(exc=rejected)
|
||||
|
||||
|
||||
def test_initial_source_task_does_not_retry_authoritative_missing_resource() -> None:
|
||||
serialized_payload = _drive_payload().model_dump(mode="json", by_alias=True)
|
||||
missing_resource = KnowledgeFSProductResourceNotFoundError("workflow was not found")
|
||||
with (
|
||||
patch(
|
||||
"tasks.knowledge_fs_initial_source_tasks.start_initial_source_import",
|
||||
side_effect=missing_resource,
|
||||
),
|
||||
patch.object(import_initial_source, "retry") as retry,
|
||||
pytest.raises(KnowledgeFSProductResourceNotFoundError, match="workflow was not found"),
|
||||
):
|
||||
import_initial_source.run(
|
||||
tenant_id="tenant-1",
|
||||
account_id="account-1",
|
||||
control_space_id="control-1",
|
||||
operation_id="operation-1",
|
||||
payload=serialized_payload,
|
||||
workflow_id="workflow-1",
|
||||
)
|
||||
|
||||
retry.assert_not_called()
|
||||
@@ -0,0 +1,157 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import timedelta
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from services.knowledge_fs.lifecycle_readiness import KnowledgeFSLifecycleWorkerReadiness
|
||||
from services.knowledge_fs.lifecycle_saga import KnowledgeFSLifecycleDispatchResult
|
||||
from services.knowledge_fs.orphan_reconciler import KnowledgeFSOrphanReport
|
||||
from tasks.knowledge_fs_lifecycle_tasks import (
|
||||
cleanup_knowledge_fs_staged_uploads,
|
||||
run_knowledge_fs_lifecycle_worker,
|
||||
)
|
||||
from tests.helpers.unit.config_override import apply_config_overrides
|
||||
|
||||
|
||||
@dataclass
|
||||
class _SagaRunner:
|
||||
results: list[KnowledgeFSLifecycleDispatchResult]
|
||||
dispatches: list[dict[str, object]] = field(default_factory=list)
|
||||
|
||||
def dispatch_one(self, **kwargs: object) -> KnowledgeFSLifecycleDispatchResult:
|
||||
self.dispatches.append(kwargs)
|
||||
return self.results.pop(0)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _OrphanReconciler:
|
||||
report: KnowledgeFSOrphanReport
|
||||
calls: list[dict[str, object]] = field(default_factory=list)
|
||||
|
||||
def reconcile(self, *, limit: int, apply_repairs: bool) -> KnowledgeFSOrphanReport:
|
||||
self.calls.append({"limit": limit, "apply_repairs": apply_repairs})
|
||||
return self.report
|
||||
|
||||
|
||||
@dataclass
|
||||
class _StagedUploads:
|
||||
expired: int
|
||||
limits: list[int] = field(default_factory=list)
|
||||
|
||||
def cleanup_expired(self, *, limit: int) -> int:
|
||||
self.limits.append(limit)
|
||||
return self.expired
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Runtime:
|
||||
facade: str
|
||||
|
||||
|
||||
def test_worker_self_check_returns_before_assembling_remote_when_rollout_is_not_ready() -> None:
|
||||
with (
|
||||
patch(
|
||||
"tasks.knowledge_fs_lifecycle_tasks.get_configured_knowledge_fs_lifecycle_worker_readiness",
|
||||
return_value=KnowledgeFSLifecycleWorkerReadiness(False, ("legacy_acl_freeze",)),
|
||||
),
|
||||
patch("tasks.knowledge_fs_lifecycle_tasks.get_knowledge_fs_lifecycle_remote") as remote,
|
||||
):
|
||||
result = run_knowledge_fs_lifecycle_worker.run()
|
||||
|
||||
assert result == {"status": "disabled", "dispatched": 0, "completed": 0, "reconciled": 0}
|
||||
remote.assert_not_called()
|
||||
|
||||
|
||||
def test_ready_worker_dispatches_a_bounded_batch_then_repairs_orphans(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
runner = _SagaRunner(
|
||||
results=[
|
||||
KnowledgeFSLifecycleDispatchResult(claimed=True, completed=True, outbox_id="outbox-1"),
|
||||
KnowledgeFSLifecycleDispatchResult(claimed=True, completed=False, outbox_id="outbox-2"),
|
||||
KnowledgeFSLifecycleDispatchResult(claimed=False, completed=False, outbox_id=None),
|
||||
]
|
||||
)
|
||||
reconciler = _OrphanReconciler(
|
||||
report=KnowledgeFSOrphanReport(
|
||||
repaired_control_space_ids=("control-1",),
|
||||
remote_orphan_space_ids=(),
|
||||
local_missing_remote_ids=(),
|
||||
cleanup_control_space_ids=("control-2", "control-3"),
|
||||
)
|
||||
)
|
||||
apply_config_overrides(
|
||||
monkeypatch,
|
||||
KNOWLEDGE_FS_LIFECYCLE_BATCH_SIZE=5,
|
||||
KNOWLEDGE_FS_LIFECYCLE_LEASE_SECONDS=30,
|
||||
KNOWLEDGE_FS_ENABLED=True,
|
||||
)
|
||||
with (
|
||||
patch(
|
||||
"tasks.knowledge_fs_lifecycle_tasks.get_configured_knowledge_fs_lifecycle_worker_readiness",
|
||||
return_value=KnowledgeFSLifecycleWorkerReadiness(True, ()),
|
||||
),
|
||||
patch("tasks.knowledge_fs_lifecycle_tasks.get_knowledge_fs_lifecycle_remote", return_value="remote-1"),
|
||||
patch("tasks.knowledge_fs_lifecycle_tasks.session_factory.get_session_maker", return_value="maker-1"),
|
||||
patch("tasks.knowledge_fs_lifecycle_tasks.KnowledgeFSLifecycleSagaRunner", return_value=runner),
|
||||
patch("tasks.knowledge_fs_lifecycle_tasks.KnowledgeFSOrphanReconciler", return_value=reconciler),
|
||||
patch("tasks.knowledge_fs_lifecycle_tasks.socket.gethostname", return_value="worker-host"),
|
||||
patch("tasks.knowledge_fs_lifecycle_tasks.naive_utc_now", return_value="now-1"),
|
||||
):
|
||||
result = run_knowledge_fs_lifecycle_worker.run()
|
||||
|
||||
assert result == {"status": "ok", "dispatched": 2, "completed": 1, "reconciled": 3}
|
||||
assert len(runner.dispatches) == 3
|
||||
assert runner.dispatches[-1] == {
|
||||
"worker_id": "worker-host:manual",
|
||||
"now": "now-1",
|
||||
"lease_duration": timedelta(seconds=30),
|
||||
"product_enabled": True,
|
||||
}
|
||||
assert reconciler.calls == [{"limit": 5, "apply_repairs": True}]
|
||||
|
||||
|
||||
def test_staged_upload_cleanup_returns_before_runtime_assembly_when_not_ready() -> None:
|
||||
with (
|
||||
patch(
|
||||
"tasks.knowledge_fs_lifecycle_tasks.get_configured_knowledge_fs_lifecycle_worker_readiness",
|
||||
return_value=KnowledgeFSLifecycleWorkerReadiness(False, ("worker_disabled",)),
|
||||
),
|
||||
patch("tasks.knowledge_fs_lifecycle_tasks.get_knowledge_fs_runtime") as runtime,
|
||||
):
|
||||
result = cleanup_knowledge_fs_staged_uploads.run()
|
||||
|
||||
assert result == 0
|
||||
runtime.assert_not_called()
|
||||
|
||||
|
||||
def test_staged_upload_cleanup_uses_the_configured_batch_limit(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
staged_uploads = _StagedUploads(expired=4)
|
||||
apply_config_overrides(monkeypatch, KNOWLEDGE_FS_LIFECYCLE_BATCH_SIZE=25)
|
||||
with (
|
||||
patch(
|
||||
"tasks.knowledge_fs_lifecycle_tasks.get_configured_knowledge_fs_lifecycle_worker_readiness",
|
||||
return_value=KnowledgeFSLifecycleWorkerReadiness(True, ()),
|
||||
),
|
||||
patch(
|
||||
"tasks.knowledge_fs_lifecycle_tasks.session_factory.get_session_maker",
|
||||
return_value="maker-1",
|
||||
),
|
||||
patch(
|
||||
"tasks.knowledge_fs_lifecycle_tasks.get_knowledge_fs_runtime",
|
||||
return_value=_Runtime(facade="facade-1"),
|
||||
) as runtime,
|
||||
patch(
|
||||
"tasks.knowledge_fs_lifecycle_tasks.KnowledgeFSStagedUploadService",
|
||||
return_value=staged_uploads,
|
||||
) as service,
|
||||
):
|
||||
result = cleanup_knowledge_fs_staged_uploads.run()
|
||||
|
||||
assert result == 4
|
||||
runtime.assert_called_once_with("maker-1")
|
||||
service.assert_called_once_with("maker-1", facade="facade-1")
|
||||
assert staged_uploads.limits == [25]
|
||||
@@ -0,0 +1,327 @@
|
||||
from typing import Literal, Never
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from celery.exceptions import Retry
|
||||
|
||||
from services.knowledge_fs.product_dto import (
|
||||
KnowledgeFSSourceResponse,
|
||||
KnowledgeFSSourceSyncPolicyPayload,
|
||||
KnowledgeFSSourceUpdatePayload,
|
||||
KnowledgeFSSourceWorkflowResponse,
|
||||
)
|
||||
from services.knowledge_fs.product_remote import (
|
||||
KnowledgeFSProductRequestRejectedError,
|
||||
KnowledgeFSProductResourceNotFoundError,
|
||||
)
|
||||
from tasks import knowledge_fs_source_import_tasks as task_module
|
||||
from tasks.knowledge_fs_source_import_tasks import (
|
||||
KnowledgeFSSourceImportNotReadyError,
|
||||
finalize_source_import_once,
|
||||
)
|
||||
from tests.helpers.unit.tasks.knowledge_fs_task_fakes import (
|
||||
FacadeRuntime,
|
||||
RecordingKnowledgeFSFacade,
|
||||
RecordingRetry,
|
||||
json_object,
|
||||
source_response,
|
||||
source_workflow_response,
|
||||
)
|
||||
|
||||
|
||||
def _source(*, status: str = "syncing") -> KnowledgeFSSourceResponse:
|
||||
return source_response(
|
||||
metadata={
|
||||
"parameters": {"limit": 99},
|
||||
"preview": False,
|
||||
"pendingImport": {
|
||||
"kind": "crawl-preview-selection",
|
||||
"previewWorkflowId": "preview-1",
|
||||
"workflowId": "import-1",
|
||||
"syncPolicy": {"enabled": False, "mode": "manual"},
|
||||
},
|
||||
},
|
||||
status=status,
|
||||
version=4,
|
||||
)
|
||||
|
||||
|
||||
def _completed_workflow() -> KnowledgeFSSourceWorkflowResponse:
|
||||
return source_workflow_response(workflow_id="import-1", state="completed")
|
||||
|
||||
|
||||
def _facade(workflow: KnowledgeFSSourceWorkflowResponse) -> RecordingKnowledgeFSFacade:
|
||||
facade = RecordingKnowledgeFSFacade()
|
||||
facade.script("get_source_workflow", workflow)
|
||||
facade.script("get_source_sync_policy", KnowledgeFSProductResourceNotFoundError("missing"))
|
||||
facade.script("update_source_sync_policy", None)
|
||||
return facade
|
||||
|
||||
|
||||
def _run(facade: RecordingKnowledgeFSFacade) -> str:
|
||||
with patch(
|
||||
"tasks.knowledge_fs_source_import_tasks.get_knowledge_fs_runtime",
|
||||
return_value=FacadeRuntime(facade),
|
||||
):
|
||||
return finalize_source_import_once(
|
||||
tenant_id="tenant-1",
|
||||
account_id="account-1",
|
||||
control_space_id="control-1",
|
||||
source_id="source-1",
|
||||
workflow_id="import-1",
|
||||
)
|
||||
|
||||
|
||||
def _raise(error: Exception) -> Never:
|
||||
raise error
|
||||
|
||||
|
||||
def test_finalize_source_import_waits_for_terminal_workflow() -> None:
|
||||
facade = _facade(source_workflow_response(workflow_id="import-1", state="importing"))
|
||||
|
||||
with pytest.raises(KnowledgeFSSourceImportNotReadyError):
|
||||
_run(facade)
|
||||
|
||||
assert facade.call_count("get_source") == 0
|
||||
|
||||
|
||||
def test_finalize_source_import_activates_source_then_applies_policy_for_new_version() -> None:
|
||||
facade = _facade(_completed_workflow())
|
||||
facade.script("get_source", _source())
|
||||
facade.script(
|
||||
"update_source",
|
||||
source_response(
|
||||
metadata={
|
||||
"lastImport": {
|
||||
"kind": "crawl-preview-selection",
|
||||
"previewWorkflowId": "preview-1",
|
||||
"state": "completed",
|
||||
"syncPolicy": {"enabled": False, "mode": "manual"},
|
||||
"workflowId": "import-1",
|
||||
},
|
||||
"preview": False,
|
||||
},
|
||||
status="active",
|
||||
version=5,
|
||||
),
|
||||
)
|
||||
|
||||
assert _run(facade) == "import-1"
|
||||
|
||||
policy = facade.payload("update_source_sync_policy", KnowledgeFSSourceSyncPolicyPayload)
|
||||
assert policy.enabled is False
|
||||
assert policy.mode == "manual"
|
||||
assert policy.expected_revision == 0
|
||||
assert policy.expected_source_version == 5
|
||||
update = facade.payload("update_source", KnowledgeFSSourceUpdatePayload)
|
||||
update_metadata = json_object(update.metadata)
|
||||
assert update.status == "active"
|
||||
assert "parameters" not in update_metadata
|
||||
assert update_metadata["pendingImport"] is None
|
||||
assert json_object(update_metadata["lastImport"])["state"] == "completed"
|
||||
|
||||
|
||||
def test_finalize_source_import_retries_policy_after_source_activation() -> None:
|
||||
facade = _facade(_completed_workflow())
|
||||
facade.script(
|
||||
"get_source",
|
||||
source_response(
|
||||
metadata={
|
||||
"lastImport": {
|
||||
"kind": "crawl-preview-selection",
|
||||
"previewWorkflowId": "preview-1",
|
||||
"state": "completed",
|
||||
"syncPolicy": {"enabled": True, "mode": "interval"},
|
||||
"workflowId": "import-1",
|
||||
},
|
||||
"preview": False,
|
||||
},
|
||||
status="active",
|
||||
version=5,
|
||||
),
|
||||
)
|
||||
|
||||
assert _run(facade) == "import-1"
|
||||
|
||||
assert facade.call_count("update_source") == 0
|
||||
policy = facade.payload("update_source_sync_policy", KnowledgeFSSourceSyncPolicyPayload)
|
||||
assert policy.expected_source_version == 5
|
||||
|
||||
|
||||
def test_finalize_source_import_recovers_a_stale_failed_marker_after_workflow_completion() -> None:
|
||||
facade = _facade(_completed_workflow())
|
||||
facade.script(
|
||||
"get_source",
|
||||
source_response(
|
||||
metadata={
|
||||
"lastImport": {
|
||||
"errorCode": "SOURCE_OPERATION_FAILED",
|
||||
"errorMessage": "provider failed",
|
||||
"kind": "crawl-preview-selection",
|
||||
"previewWorkflowId": "preview-1",
|
||||
"state": "failed",
|
||||
"syncPolicy": {"enabled": False, "mode": "manual"},
|
||||
"workflowId": "import-1",
|
||||
},
|
||||
"pendingImport": None,
|
||||
"preview": False,
|
||||
},
|
||||
status="error",
|
||||
version=5,
|
||||
),
|
||||
)
|
||||
facade.script(
|
||||
"update_source",
|
||||
source_response(
|
||||
metadata={
|
||||
"lastImport": {
|
||||
"kind": "crawl-preview-selection",
|
||||
"previewWorkflowId": "preview-1",
|
||||
"state": "completed",
|
||||
"syncPolicy": {"enabled": False, "mode": "manual"},
|
||||
"workflowId": "import-1",
|
||||
},
|
||||
"pendingImport": None,
|
||||
"preview": False,
|
||||
},
|
||||
status="active",
|
||||
version=6,
|
||||
),
|
||||
)
|
||||
|
||||
assert _run(facade) == "import-1"
|
||||
|
||||
update = facade.payload("update_source", KnowledgeFSSourceUpdatePayload)
|
||||
|
||||
update_metadata = json_object(update.metadata)
|
||||
assert update.status == "active"
|
||||
assert json_object(update_metadata["lastImport"])["state"] == "completed"
|
||||
assert update_metadata["pendingImport"] is None
|
||||
policy = facade.payload("update_source_sync_policy", KnowledgeFSSourceSyncPolicyPayload)
|
||||
assert policy.expected_source_version == 6
|
||||
|
||||
|
||||
def test_finalize_source_import_persists_failure_on_visible_source() -> None:
|
||||
facade = _facade(
|
||||
source_workflow_response(
|
||||
workflow_id="import-1",
|
||||
state="failed",
|
||||
failure_message="provider failed",
|
||||
last_error_code="PROVIDER_FAILED",
|
||||
)
|
||||
)
|
||||
facade.script("get_source", _source())
|
||||
facade.script("update_source", _source(status="error"))
|
||||
|
||||
assert _run(facade) == "import-1"
|
||||
|
||||
update = facade.payload("update_source", KnowledgeFSSourceUpdatePayload)
|
||||
|
||||
update_metadata = json_object(update.metadata)
|
||||
assert update.status == "error"
|
||||
assert "parameters" not in update_metadata
|
||||
assert update_metadata["lastImport"] == {
|
||||
"errorCode": "PROVIDER_FAILED",
|
||||
"errorMessage": "provider failed",
|
||||
"kind": "crawl-preview-selection",
|
||||
"previewWorkflowId": "preview-1",
|
||||
"state": "failed",
|
||||
"syncPolicy": {"enabled": False, "mode": "manual"},
|
||||
"workflowId": "import-1",
|
||||
}
|
||||
assert update_metadata["pendingImport"] is None
|
||||
assert facade.call_count("update_source_sync_policy") == 0
|
||||
|
||||
|
||||
def test_finalize_website_source_import_waits_for_policy_write_fence() -> None:
|
||||
facade = _facade(_completed_workflow())
|
||||
facade.script(
|
||||
"get_source",
|
||||
source_response(
|
||||
metadata={
|
||||
"lastImport": {
|
||||
"kind": "website-crawl-import",
|
||||
"state": "completed",
|
||||
"syncPolicy": {"enabled": True, "mode": "interval"},
|
||||
"workflowId": "import-1",
|
||||
},
|
||||
"preview": False,
|
||||
},
|
||||
status="active",
|
||||
version=5,
|
||||
),
|
||||
)
|
||||
facade.script("update_source_sync_policy", KnowledgeFSProductRequestRejectedError(status_code=400))
|
||||
|
||||
with pytest.raises(KnowledgeFSSourceImportNotReadyError):
|
||||
_run(facade)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("import_kind", ["online-document-import", "online-drive-import"])
|
||||
def test_finalize_non_website_source_import_does_not_reclassify_a_bad_request(
|
||||
import_kind: str,
|
||||
) -> None:
|
||||
facade = _facade(_completed_workflow())
|
||||
facade.script(
|
||||
"get_source",
|
||||
source_response(
|
||||
metadata={
|
||||
"lastImport": {
|
||||
"kind": import_kind,
|
||||
"state": "completed",
|
||||
"syncPolicy": {"enabled": True, "mode": "interval"},
|
||||
"workflowId": "import-1",
|
||||
},
|
||||
"preview": False,
|
||||
},
|
||||
status="active",
|
||||
version=5,
|
||||
),
|
||||
)
|
||||
rejection = KnowledgeFSProductRequestRejectedError(status_code=400)
|
||||
facade.script("update_source_sync_policy", rejection)
|
||||
|
||||
with pytest.raises(KnowledgeFSProductRequestRejectedError) as raised:
|
||||
_run(facade)
|
||||
|
||||
assert raised.value is rejection
|
||||
|
||||
|
||||
def test_finalize_source_import_retries_a_source_revision_conflict(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
error = KnowledgeFSProductRequestRejectedError(status_code=409)
|
||||
monkeypatch.setattr(task_module, "finalize_source_import_once", lambda **_: _raise(error))
|
||||
retry = RecordingRetry()
|
||||
monkeypatch.setattr(task_module.finalize_source_import, "retry", retry)
|
||||
|
||||
with pytest.raises(Retry):
|
||||
task_module.finalize_source_import.run(
|
||||
tenant_id="tenant-1",
|
||||
account_id="account-1",
|
||||
control_space_id="control-1",
|
||||
source_id="source-1",
|
||||
workflow_id="import-1",
|
||||
)
|
||||
|
||||
assert retry.calls == [{"exc": error}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status_code", [400, 422])
|
||||
def test_finalize_source_import_does_not_retry_a_permanent_rejection(
|
||||
monkeypatch: pytest.MonkeyPatch, status_code: Literal[400, 422]
|
||||
) -> None:
|
||||
error = KnowledgeFSProductRequestRejectedError(status_code=status_code)
|
||||
monkeypatch.setattr(task_module, "finalize_source_import_once", lambda **_: _raise(error))
|
||||
retry = RecordingRetry()
|
||||
monkeypatch.setattr(task_module.finalize_source_import, "retry", retry)
|
||||
|
||||
with pytest.raises(KnowledgeFSProductRequestRejectedError) as raised:
|
||||
task_module.finalize_source_import.run(
|
||||
tenant_id="tenant-1",
|
||||
account_id="account-1",
|
||||
control_space_id="control-1",
|
||||
source_id="source-1",
|
||||
workflow_id="import-1",
|
||||
)
|
||||
|
||||
assert raised.value is error
|
||||
assert retry.calls == []
|
||||
@@ -0,0 +1,185 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from celery.app.task import Context, Task
|
||||
|
||||
from services.dataset_knowledge_fs_upgrade_service import (
|
||||
UPGRADE_CHECKPOINT_TIME_LIMIT_SECONDS,
|
||||
UPGRADE_DELIVERY_TIMEOUT,
|
||||
KnowledgeFSUpgradeBusyError,
|
||||
KnowledgeFSUpgradeNotReadyError,
|
||||
)
|
||||
from tasks.knowledge_fs_upgrade_tasks import (
|
||||
KNOWLEDGE_FS_UPGRADE_QUEUE,
|
||||
cleanup_deferred_knowledge_fs_upgrade_files,
|
||||
reconcile_knowledge_fs_upgrade_documents,
|
||||
recover_stalled_knowledge_fs_upgrades,
|
||||
run_knowledge_fs_upgrade,
|
||||
)
|
||||
from tests.helpers.unit.tasks.task_options import task_options
|
||||
|
||||
|
||||
@dataclass
|
||||
class _UpgradeRunner:
|
||||
"""Returns ``outcome`` from ``run_next`` or raises it, and records failure reports."""
|
||||
|
||||
outcome: bool | None | Exception
|
||||
runs: list[dict[str, object]] = field(default_factory=list)
|
||||
failures: list[dict[str, object]] = field(default_factory=list)
|
||||
releases: list[tuple[str, str | None]] = field(default_factory=list)
|
||||
released_tenant_id: str | None = "tenant-1"
|
||||
|
||||
def run_next(self, **kwargs: object) -> bool | None:
|
||||
self.runs.append(kwargs)
|
||||
if isinstance(self.outcome, Exception):
|
||||
raise self.outcome
|
||||
return self.outcome
|
||||
|
||||
def fail(self, *, job_id: str, celery_task_id: str, error: Exception) -> None:
|
||||
self.failures.append({"job_id": job_id, "celery_task_id": celery_task_id, "error": error})
|
||||
|
||||
def release_for_retry(self, *, job_id: str, celery_task_id: str) -> None:
|
||||
self.releases.append((job_id, celery_task_id))
|
||||
|
||||
def release_for_dispatch(self, *, job_id: str, celery_task_id: str | None) -> str | None:
|
||||
self.releases.append((job_id, celery_task_id))
|
||||
return self.released_tenant_id
|
||||
|
||||
|
||||
def test_upgrade_tasks_are_pinned_to_the_dedicated_queue() -> None:
|
||||
assert task_options(run_knowledge_fs_upgrade)["queue"] == KNOWLEDGE_FS_UPGRADE_QUEUE
|
||||
assert task_options(reconcile_knowledge_fs_upgrade_documents)["queue"] == KNOWLEDGE_FS_UPGRADE_QUEUE
|
||||
assert task_options(cleanup_deferred_knowledge_fs_upgrade_files)["queue"] == KNOWLEDGE_FS_UPGRADE_QUEUE
|
||||
|
||||
|
||||
def test_deferred_file_cleanup_uses_the_upgrade_session_factory() -> None:
|
||||
with (
|
||||
patch("tasks.knowledge_fs_upgrade_tasks.session_factory.get_session_maker", return_value="maker"),
|
||||
patch("tasks.knowledge_fs_upgrade_tasks.cleanup_deferred_upgrade_files", return_value=3) as cleanup,
|
||||
):
|
||||
assert cleanup_deferred_knowledge_fs_upgrade_files.run() == 3
|
||||
|
||||
cleanup.assert_called_once_with("maker")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("has_more", [True, False])
|
||||
def test_worker_enqueues_only_the_expected_follow_up(has_more: bool) -> None:
|
||||
runner = _UpgradeRunner(outcome=has_more)
|
||||
with (
|
||||
patch("tasks.knowledge_fs_upgrade_tasks.session_factory.get_session_maker", return_value="maker"),
|
||||
patch("tasks.knowledge_fs_upgrade_tasks.KnowledgeFSUpgradeRunner", return_value=runner),
|
||||
patch("services.knowledge.datasets.upgrades.enqueue_upgrade_job") as continue_upgrade,
|
||||
patch("tasks.knowledge_fs_upgrade_tasks._dispatch_reconciliations") as reconcile,
|
||||
):
|
||||
run_knowledge_fs_upgrade.run(job_id="job-1")
|
||||
|
||||
assert len(runner.runs) == 1
|
||||
assert runner.runs[0]["job_id"] == "job-1"
|
||||
if has_more:
|
||||
assert runner.releases == [("job-1", None)]
|
||||
continue_upgrade.assert_called_once()
|
||||
assert continue_upgrade.call_args.args[1:] == ("tenant-1", "job-1")
|
||||
reconcile.assert_not_called()
|
||||
else:
|
||||
assert runner.releases == []
|
||||
continue_upgrade.assert_not_called()
|
||||
reconcile.assert_called_once_with(job_id="job-1")
|
||||
|
||||
|
||||
def test_worker_skips_dispatch_when_the_job_moved_on() -> None:
|
||||
runner = _UpgradeRunner(outcome=True, released_tenant_id=None)
|
||||
with (
|
||||
patch("tasks.knowledge_fs_upgrade_tasks.session_factory.get_session_maker", return_value="maker"),
|
||||
patch("tasks.knowledge_fs_upgrade_tasks.KnowledgeFSUpgradeRunner", return_value=runner),
|
||||
patch("services.knowledge.datasets.upgrades.enqueue_upgrade_job") as continue_upgrade,
|
||||
):
|
||||
run_knowledge_fs_upgrade.run(job_id="job-1")
|
||||
|
||||
continue_upgrade.assert_not_called()
|
||||
|
||||
|
||||
def test_worker_marks_parent_failed_when_a_checkpoint_raises() -> None:
|
||||
error = RuntimeError("ordinary document upload failed")
|
||||
runner = _UpgradeRunner(outcome=error)
|
||||
with (
|
||||
patch("tasks.knowledge_fs_upgrade_tasks.session_factory.get_session_maker", return_value="maker"),
|
||||
patch("tasks.knowledge_fs_upgrade_tasks.KnowledgeFSUpgradeRunner", return_value=runner),
|
||||
pytest.raises(RuntimeError, match="ordinary document upload failed"),
|
||||
):
|
||||
run_knowledge_fs_upgrade.run(job_id="job-1")
|
||||
|
||||
assert runner.failures == [{"job_id": "job-1", "celery_task_id": None, "error": error}]
|
||||
|
||||
|
||||
def test_worker_marks_parent_failed_when_provisioning_retries_are_exhausted() -> None:
|
||||
error = KnowledgeFSUpgradeNotReadyError("Space is still provisioning")
|
||||
runner = _UpgradeRunner(outcome=error)
|
||||
run_knowledge_fs_upgrade.request_stack.push(Context(retries=run_knowledge_fs_upgrade.max_retries))
|
||||
try:
|
||||
with (
|
||||
patch("tasks.knowledge_fs_upgrade_tasks.session_factory.get_session_maker", return_value="maker"),
|
||||
patch("tasks.knowledge_fs_upgrade_tasks.KnowledgeFSUpgradeRunner", return_value=runner),
|
||||
pytest.raises(KnowledgeFSUpgradeNotReadyError, match="still provisioning"),
|
||||
):
|
||||
run_knowledge_fs_upgrade.run(job_id="job-1")
|
||||
finally:
|
||||
run_knowledge_fs_upgrade.pop_request()
|
||||
|
||||
assert runner.failures == [{"job_id": "job-1", "celery_task_id": None, "error": error}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("task", [run_knowledge_fs_upgrade, reconcile_knowledge_fs_upgrade_documents])
|
||||
def test_upgrade_checkpoint_is_redelivered_on_worker_loss_and_bounded_before_recovery(task: Task[..., None]) -> None:
|
||||
assert task.acks_late is True
|
||||
assert task.reject_on_worker_lost is True
|
||||
options = task_options(task)
|
||||
assert options["time_limit"] == UPGRADE_CHECKPOINT_TIME_LIMIT_SECONDS
|
||||
assert options["soft_time_limit"] == UPGRADE_CHECKPOINT_TIME_LIMIT_SECONDS - 30
|
||||
assert UPGRADE_DELIVERY_TIMEOUT.total_seconds() > UPGRADE_CHECKPOINT_TIME_LIMIT_SECONDS
|
||||
|
||||
|
||||
def test_superseded_delivery_does_not_schedule_any_follow_up() -> None:
|
||||
runner = _UpgradeRunner(outcome=None)
|
||||
with (
|
||||
patch("tasks.knowledge_fs_upgrade_tasks.KnowledgeFSUpgradeRunner", return_value=runner),
|
||||
patch("services.knowledge.datasets.upgrades.enqueue_upgrade_job") as dispatch,
|
||||
patch.object(reconcile_knowledge_fs_upgrade_documents, "apply_async") as reconcile,
|
||||
):
|
||||
run_knowledge_fs_upgrade.run(job_id="job-1")
|
||||
dispatch.assert_not_called()
|
||||
reconcile.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("busy", [False, True])
|
||||
def test_waiting_delivery_preserves_live_execution_and_releases_provisioning_retry(busy: bool) -> None:
|
||||
error = KnowledgeFSUpgradeBusyError("busy") if busy else KnowledgeFSUpgradeNotReadyError("provisioning")
|
||||
runner = _UpgradeRunner(outcome=error)
|
||||
run_knowledge_fs_upgrade.request_stack.push(Context(id="task-1", retries=0))
|
||||
try:
|
||||
with (
|
||||
patch("tasks.knowledge_fs_upgrade_tasks.KnowledgeFSUpgradeRunner", return_value=runner),
|
||||
patch.object(run_knowledge_fs_upgrade, "retry", side_effect=RuntimeError("retry sent")) as retry,
|
||||
pytest.raises(RuntimeError, match="retry sent"),
|
||||
):
|
||||
run_knowledge_fs_upgrade.run(job_id="job-1")
|
||||
assert runner.releases == ([] if busy else [("job-1", "task-1")])
|
||||
assert runner.failures == []
|
||||
retry.assert_called_once()
|
||||
finally:
|
||||
run_knowledge_fs_upgrade.pop_request()
|
||||
|
||||
|
||||
def test_recovery_sweep_dispatches_each_abandoned_job_even_if_one_publish_fails() -> None:
|
||||
with (
|
||||
patch("tasks.knowledge_fs_upgrade_tasks._dispatch_reconciliations", return_value=0),
|
||||
patch("tasks.knowledge_fs_upgrade_tasks.KnowledgeFSUpgradeSnapshotService") as snapshots,
|
||||
patch(
|
||||
"services.knowledge.datasets.upgrades.enqueue_upgrade_job", side_effect=[OSError("broker down"), None]
|
||||
) as dispatch,
|
||||
):
|
||||
snapshots.return_value.recover_stalled_jobs.return_value = [("tenant-1", "job-1"), ("tenant-2", "job-2")]
|
||||
assert recover_stalled_knowledge_fs_upgrades.run() == 2
|
||||
assert [call.args[1:] for call in dispatch.call_args_list] == [("tenant-1", "job-1"), ("tenant-2", "job-2")]
|
||||
@@ -0,0 +1,143 @@
|
||||
from collections.abc import Sequence
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.engine import Engine
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from graphon.runtime import VariablePool
|
||||
from models.human_input import HumanInputDelivery, HumanInputForm, HumanInputFormRecipient
|
||||
from services.human_input_form_delivery_provider import (
|
||||
HumanInputFormDeliveryContext,
|
||||
HumanInputFormDeliveryDispatcher,
|
||||
HumanInputFormDeliveryProviderRegistry,
|
||||
)
|
||||
from tasks import mail_human_input_delivery_task as task_module
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"sqlite_session", [(HumanInputForm, HumanInputDelivery, HumanInputFormRecipient)], indirect=True
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("email_only", "mail_inited", "feature_enabled", "feature_error", "expected_methods"),
|
||||
[
|
||||
(True, True, True, False, (task_module.DeliveryMethodType.EMAIL,)),
|
||||
(True, True, False, False, ()),
|
||||
(True, False, True, False, ()),
|
||||
(False, True, True, False, task_module.FORM_DELIVERY_METHOD_TYPES),
|
||||
(False, False, True, False, (task_module.DeliveryMethodType.IM,)),
|
||||
(False, True, False, False, (task_module.DeliveryMethodType.IM,)),
|
||||
(False, True, True, True, (task_module.DeliveryMethodType.IM,)),
|
||||
],
|
||||
ids=[
|
||||
"legacy-email",
|
||||
"legacy-disabled",
|
||||
"legacy-unconfigured",
|
||||
"all-channels",
|
||||
"im-without-mail",
|
||||
"im-without-feature",
|
||||
"im-after-feature-error",
|
||||
],
|
||||
)
|
||||
def test_delivery_task_dispatches_available_persisted_channels(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
sqlite_engine: Engine,
|
||||
sqlite_session: Session,
|
||||
email_only: bool,
|
||||
mail_inited: bool,
|
||||
feature_enabled: bool,
|
||||
feature_error: bool,
|
||||
expected_methods: tuple[task_module.DeliveryMethodType, ...],
|
||||
) -> None:
|
||||
form = HumanInputForm(
|
||||
tenant_id=str(uuid4()),
|
||||
app_id=str(uuid4()),
|
||||
workflow_run_id=None,
|
||||
conversation_id=None,
|
||||
node_id="human-input",
|
||||
form_definition="{}",
|
||||
rendered_content="content",
|
||||
expiration_time=datetime.now(UTC) + timedelta(hours=1),
|
||||
)
|
||||
sqlite_session.add(form)
|
||||
sqlite_session.flush()
|
||||
deliveries = [
|
||||
HumanInputDelivery(form_id=form.id, delivery_method_type=method, channel_payload="{}")
|
||||
for method in task_module.FORM_DELIVERY_METHOD_TYPES
|
||||
]
|
||||
sqlite_session.add_all(deliveries)
|
||||
sqlite_session.commit()
|
||||
pool = VariablePool()
|
||||
sent: list[HumanInputFormDeliveryContext] = []
|
||||
loading_sessions: list[Session] = []
|
||||
feature_tenants: list[str] = []
|
||||
load_contexts = HumanInputFormDeliveryDispatcher.load_form_contexts
|
||||
|
||||
def load(
|
||||
dispatcher: HumanInputFormDeliveryDispatcher,
|
||||
*,
|
||||
session: Session,
|
||||
form: HumanInputForm,
|
||||
variable_pool: VariablePool | None = None,
|
||||
delivery_method_types: Sequence[task_module.DeliveryMethodType] | None = None,
|
||||
) -> tuple[HumanInputFormDeliveryContext, ...]:
|
||||
loading_sessions.append(session)
|
||||
assert session.bind is sqlite_engine
|
||||
assert variable_pool is pool
|
||||
assert delivery_method_types == (
|
||||
(task_module.DeliveryMethodType.EMAIL,) if email_only else task_module.FORM_DELIVERY_METHOD_TYPES
|
||||
)
|
||||
return load_contexts(
|
||||
dispatcher,
|
||||
session=session,
|
||||
form=form,
|
||||
variable_pool=variable_pool,
|
||||
delivery_method_types=delivery_method_types,
|
||||
)
|
||||
|
||||
def dispatch(
|
||||
_registry: HumanInputFormDeliveryProviderRegistry,
|
||||
*,
|
||||
context: HumanInputFormDeliveryContext,
|
||||
continue_on_error: bool = True,
|
||||
) -> bool:
|
||||
assert len(loading_sessions) == 1
|
||||
assert not loading_sessions[0].in_transaction()
|
||||
assert not loading_sessions[0].identity_map
|
||||
assert continue_on_error
|
||||
assert context.variable_pool is pool
|
||||
sent.append(context)
|
||||
return True
|
||||
|
||||
from services.entities.feature_entities import FeatureModel
|
||||
from services.feature_service import FeatureService
|
||||
|
||||
def features(tenant_id: str, **_kwargs: object) -> FeatureModel:
|
||||
feature_tenants.append(tenant_id)
|
||||
if feature_error:
|
||||
raise RuntimeError("feature service unavailable")
|
||||
return FeatureModel(human_input_email_delivery_enabled=feature_enabled)
|
||||
|
||||
monkeypatch.setattr(task_module.mail, "is_inited", lambda: mail_inited)
|
||||
monkeypatch.setattr(FeatureService, "get_features", features)
|
||||
monkeypatch.setattr(task_module, "_load_variable_pool", lambda _run_id: pool)
|
||||
monkeypatch.setattr(HumanInputFormDeliveryProviderRegistry, "dispatch", dispatch)
|
||||
monkeypatch.setattr(HumanInputFormDeliveryDispatcher, "load_form_contexts", load)
|
||||
task = (
|
||||
task_module.dispatch_human_input_email_task
|
||||
if email_only
|
||||
else task_module.dispatch_human_input_form_delivery_task
|
||||
)
|
||||
task(
|
||||
form_id=form.id, node_title="Approve", session_factory=sessionmaker(bind=sqlite_engine, expire_on_commit=False)
|
||||
)
|
||||
assert tuple(context.delivery_method_type for context in sent) == expected_methods
|
||||
assert len(loading_sessions) == (0 if email_only and (not mail_inited or not feature_enabled) else 1)
|
||||
assert feature_tenants == ([form.tenant_id] if mail_inited else [])
|
||||
for context in sent:
|
||||
assert context.form_id == form.id
|
||||
assert context.tenant_id == form.tenant_id
|
||||
assert context.app_id == form.app_id
|
||||
assert context.rendered_content == "content"
|
||||
assert context.delivery_id in {delivery.id for delivery in deliveries}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,37 @@
|
||||
import pytest
|
||||
|
||||
from tasks import ops_trace_task
|
||||
|
||||
|
||||
class _Storage:
|
||||
def __init__(self) -> None:
|
||||
self.files: dict[str, bytes] = {}
|
||||
|
||||
def save(self, key: str, data: bytes) -> None:
|
||||
self.files[key] = data
|
||||
|
||||
def delete(self, key: str) -> None:
|
||||
self.files.pop(key, None)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def storage(monkeypatch: pytest.MonkeyPatch) -> _Storage:
|
||||
fake = _Storage()
|
||||
monkeypatch.setattr(ops_trace_task, "storage", fake)
|
||||
return fake
|
||||
|
||||
|
||||
def test_legacy_trace_tasks_are_drained(storage: _Storage) -> None:
|
||||
storage.save("ops_trace/app-1/file-1.json", b"{}")
|
||||
|
||||
ops_trace_task.process_trace_tasks.run({"app_id": "app-1", "file_id": "file-1"})
|
||||
|
||||
assert storage.files == {}
|
||||
|
||||
|
||||
def test_legacy_trace_tasks_never_delete_outside_their_folder(storage: _Storage) -> None:
|
||||
storage.save("ops_trace/app-1/file-1.json", b"{}")
|
||||
|
||||
ops_trace_task.process_trace_tasks.run({"app_id": "..", "file_id": "../secrets"})
|
||||
|
||||
assert list(storage.files) == ["ops_trace/app-1/file-1.json"]
|
||||
+431
@@ -0,0 +1,431 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from core.plugin.entities.marketplace import MarketplacePluginSnapshot
|
||||
from core.plugin.entities.plugin import PluginCategory, PluginInstallationSource
|
||||
from models.account import (
|
||||
TenantPluginAutoUpgradeCategory,
|
||||
TenantPluginAutoUpgradeMode,
|
||||
TenantPluginAutoUpgradeStrategySetting,
|
||||
)
|
||||
|
||||
MODULE = "tasks.process_tenant_plugin_autoupgrade_check_task"
|
||||
|
||||
|
||||
def _make_plugin(
|
||||
plugin_id: str,
|
||||
version: str,
|
||||
source: PluginInstallationSource = PluginInstallationSource.Marketplace,
|
||||
category: PluginCategory = PluginCategory.Tool,
|
||||
) -> SimpleNamespace:
|
||||
"""Build a minimal stand-in for a PluginInstallation entry returned by manager.list_plugins."""
|
||||
return SimpleNamespace(
|
||||
plugin_id=plugin_id,
|
||||
version=version,
|
||||
plugin_unique_identifier=f"{plugin_id}:{version}@deadbeef",
|
||||
source=source,
|
||||
declaration=SimpleNamespace(category=category),
|
||||
)
|
||||
|
||||
|
||||
def _make_manifest(plugin_id: str, latest_version: str) -> MarketplacePluginSnapshot:
|
||||
org, name = plugin_id.split("/", 1)
|
||||
return MarketplacePluginSnapshot(
|
||||
org=org,
|
||||
name=name,
|
||||
latest_version=latest_version,
|
||||
latest_package_identifier=f"{plugin_id}:{latest_version}@cafe1234",
|
||||
latest_package_url=f"https://marketplace.example/{plugin_id}/{latest_version}.difypkg",
|
||||
)
|
||||
|
||||
|
||||
def _run_task(
|
||||
*,
|
||||
plugins: list[SimpleNamespace],
|
||||
manifests: list[MarketplacePluginSnapshot],
|
||||
strategy_setting: TenantPluginAutoUpgradeStrategySetting = TenantPluginAutoUpgradeStrategySetting.LATEST,
|
||||
upgrade_mode: TenantPluginAutoUpgradeMode = TenantPluginAutoUpgradeMode.ALL,
|
||||
exclude_plugins: list[str] | None = None,
|
||||
include_plugins: list[str] | None = None,
|
||||
category: PluginCategory | str | None = None,
|
||||
) -> tuple[MagicMock, list[tuple[str, str, str]]]:
|
||||
"""
|
||||
Execute the celery task synchronously with mocks for the plugin manager,
|
||||
the marketplace cache and PluginService.upgrade_plugin_with_marketplace.
|
||||
Returns the upgrade-call recorder so each test can assert on it.
|
||||
"""
|
||||
fake_manager = MagicMock()
|
||||
fake_manager.list_plugins.return_value = plugins
|
||||
|
||||
upgrade_calls: list[tuple[str, str, str]] = []
|
||||
|
||||
def _record_upgrade(tenant_id: str, original: str, new: str) -> None:
|
||||
upgrade_calls.append((tenant_id, original, new))
|
||||
|
||||
with (
|
||||
patch(f"{MODULE}.PluginInstaller", return_value=fake_manager),
|
||||
patch(f"{MODULE}.marketplace_batch_fetch_plugin_manifests", return_value=manifests),
|
||||
patch(
|
||||
f"{MODULE}.PluginService.upgrade_plugin_with_marketplace",
|
||||
side_effect=_record_upgrade,
|
||||
) as upgrade_mock,
|
||||
):
|
||||
from tasks.process_tenant_plugin_autoupgrade_check_task import (
|
||||
process_tenant_plugin_autoupgrade_check_task,
|
||||
)
|
||||
|
||||
process_tenant_plugin_autoupgrade_check_task(
|
||||
"tenant-1",
|
||||
strategy_setting,
|
||||
0,
|
||||
upgrade_mode,
|
||||
exclude_plugins or [],
|
||||
include_plugins or [],
|
||||
category,
|
||||
)
|
||||
|
||||
return upgrade_mock, upgrade_calls
|
||||
|
||||
|
||||
class TestUpgradeCallsMarketplaceService:
|
||||
"""
|
||||
Regression test for the bug where the auto-upgrade task called
|
||||
manager.upgrade_plugin directly, which skipped downloading the new package
|
||||
from marketplace and uploading it to the daemon. The daemon then failed with
|
||||
"package file not found" and the upgrade silently never completed.
|
||||
"""
|
||||
|
||||
def test_upgrade_routes_through_plugin_service(self) -> None:
|
||||
plugin = _make_plugin("acme/foo", "1.0.0")
|
||||
manifest = _make_manifest("acme/foo", "1.0.1")
|
||||
|
||||
upgrade_mock, calls = _run_task(plugins=[plugin], manifests=[manifest])
|
||||
|
||||
upgrade_mock.assert_called_once()
|
||||
assert calls == [("tenant-1", plugin.plugin_unique_identifier, manifest.latest_package_identifier)]
|
||||
|
||||
def test_does_not_call_manager_upgrade_plugin_directly(self) -> None:
|
||||
"""Locks in that we never go back to the broken path that bypassed download/upload."""
|
||||
plugin = _make_plugin("acme/foo", "1.0.0")
|
||||
manifest = _make_manifest("acme/foo", "1.0.1")
|
||||
|
||||
fake_manager = MagicMock()
|
||||
fake_manager.list_plugins.return_value = [plugin]
|
||||
|
||||
with (
|
||||
patch(f"{MODULE}.PluginInstaller", return_value=fake_manager),
|
||||
patch(f"{MODULE}.marketplace_batch_fetch_plugin_manifests", return_value=[manifest]),
|
||||
patch(f"{MODULE}.PluginService.upgrade_plugin_with_marketplace"),
|
||||
):
|
||||
from tasks.process_tenant_plugin_autoupgrade_check_task import (
|
||||
process_tenant_plugin_autoupgrade_check_task,
|
||||
)
|
||||
|
||||
process_tenant_plugin_autoupgrade_check_task(
|
||||
"tenant-1",
|
||||
TenantPluginAutoUpgradeStrategySetting.LATEST,
|
||||
0,
|
||||
TenantPluginAutoUpgradeMode.ALL,
|
||||
[],
|
||||
[],
|
||||
)
|
||||
|
||||
fake_manager.upgrade_plugin.assert_not_called()
|
||||
|
||||
|
||||
class TestStrategySetting:
|
||||
def test_disabled_strategy_skips_everything(self) -> None:
|
||||
upgrade_mock, _ = _run_task(
|
||||
plugins=[_make_plugin("acme/foo", "1.0.0")],
|
||||
manifests=[_make_manifest("acme/foo", "1.0.1")],
|
||||
strategy_setting=TenantPluginAutoUpgradeStrategySetting.DISABLED,
|
||||
)
|
||||
upgrade_mock.assert_not_called()
|
||||
|
||||
def test_fix_only_upgrades_patch_version(self) -> None:
|
||||
upgrade_mock, calls = _run_task(
|
||||
plugins=[_make_plugin("acme/foo", "1.0.0")],
|
||||
manifests=[_make_manifest("acme/foo", "1.0.5")],
|
||||
strategy_setting=TenantPluginAutoUpgradeStrategySetting.FIX_ONLY,
|
||||
)
|
||||
upgrade_mock.assert_called_once()
|
||||
assert calls[0][2].endswith(":1.0.5@cafe1234")
|
||||
|
||||
def test_fix_only_skips_minor_bump(self) -> None:
|
||||
upgrade_mock, _ = _run_task(
|
||||
plugins=[_make_plugin("acme/foo", "1.0.0")],
|
||||
manifests=[_make_manifest("acme/foo", "1.1.0")],
|
||||
strategy_setting=TenantPluginAutoUpgradeStrategySetting.FIX_ONLY,
|
||||
)
|
||||
upgrade_mock.assert_not_called()
|
||||
|
||||
def test_fix_only_skips_major_bump(self) -> None:
|
||||
upgrade_mock, _ = _run_task(
|
||||
plugins=[_make_plugin("acme/foo", "1.0.0")],
|
||||
manifests=[_make_manifest("acme/foo", "2.0.0")],
|
||||
strategy_setting=TenantPluginAutoUpgradeStrategySetting.FIX_ONLY,
|
||||
)
|
||||
upgrade_mock.assert_not_called()
|
||||
|
||||
def test_latest_strategy_skips_when_versions_equal(self) -> None:
|
||||
upgrade_mock, _ = _run_task(
|
||||
plugins=[_make_plugin("acme/foo", "1.0.0")],
|
||||
manifests=[_make_manifest("acme/foo", "1.0.0")],
|
||||
strategy_setting=TenantPluginAutoUpgradeStrategySetting.LATEST,
|
||||
)
|
||||
upgrade_mock.assert_not_called()
|
||||
|
||||
|
||||
class TestUpgradeMode:
|
||||
def test_mode_all_upgrades_every_marketplace_plugin(self) -> None:
|
||||
plugins = [
|
||||
_make_plugin("acme/foo", "1.0.0"),
|
||||
_make_plugin("acme/bar", "2.0.0"),
|
||||
]
|
||||
manifests = [
|
||||
_make_manifest("acme/foo", "1.0.1"),
|
||||
_make_manifest("acme/bar", "2.0.1"),
|
||||
]
|
||||
|
||||
upgrade_mock, calls = _run_task(
|
||||
plugins=plugins,
|
||||
manifests=manifests,
|
||||
upgrade_mode=TenantPluginAutoUpgradeMode.ALL,
|
||||
)
|
||||
|
||||
assert upgrade_mock.call_count == 2
|
||||
upgraded_ids = sorted(c[1] for c in calls)
|
||||
assert upgraded_ids == sorted(p.plugin_unique_identifier for p in plugins)
|
||||
|
||||
def test_mode_all_skips_non_marketplace_sources(self) -> None:
|
||||
plugins = [
|
||||
_make_plugin("acme/foo", "1.0.0"),
|
||||
_make_plugin("acme/bar", "2.0.0", source=PluginInstallationSource.Github),
|
||||
]
|
||||
manifests = [
|
||||
_make_manifest("acme/foo", "1.0.1"),
|
||||
_make_manifest("acme/bar", "2.0.1"),
|
||||
]
|
||||
|
||||
upgrade_mock, calls = _run_task(
|
||||
plugins=plugins,
|
||||
manifests=manifests,
|
||||
upgrade_mode=TenantPluginAutoUpgradeMode.ALL,
|
||||
)
|
||||
|
||||
assert upgrade_mock.call_count == 1
|
||||
assert calls[0][1] == plugins[0].plugin_unique_identifier
|
||||
|
||||
def test_mode_partial_only_upgrades_included_plugins(self) -> None:
|
||||
plugins = [
|
||||
_make_plugin("acme/foo", "1.0.0"),
|
||||
_make_plugin("acme/bar", "2.0.0"),
|
||||
]
|
||||
manifests = [
|
||||
_make_manifest("acme/foo", "1.0.1"),
|
||||
_make_manifest("acme/bar", "2.0.1"),
|
||||
]
|
||||
|
||||
upgrade_mock, calls = _run_task(
|
||||
plugins=plugins,
|
||||
manifests=manifests,
|
||||
upgrade_mode=TenantPluginAutoUpgradeMode.PARTIAL,
|
||||
include_plugins=["acme/foo"],
|
||||
)
|
||||
|
||||
assert upgrade_mock.call_count == 1
|
||||
assert calls[0][1] == plugins[0].plugin_unique_identifier
|
||||
|
||||
def test_mode_exclude_skips_excluded_plugins(self) -> None:
|
||||
plugins = [
|
||||
_make_plugin("acme/foo", "1.0.0"),
|
||||
_make_plugin("acme/bar", "2.0.0"),
|
||||
]
|
||||
manifests = [
|
||||
_make_manifest("acme/foo", "1.0.1"),
|
||||
_make_manifest("acme/bar", "2.0.1"),
|
||||
]
|
||||
|
||||
upgrade_mock, calls = _run_task(
|
||||
plugins=plugins,
|
||||
manifests=manifests,
|
||||
upgrade_mode=TenantPluginAutoUpgradeMode.EXCLUDE,
|
||||
exclude_plugins=["acme/bar"],
|
||||
)
|
||||
|
||||
assert upgrade_mock.call_count == 1
|
||||
assert calls[0][1] == plugins[0].plugin_unique_identifier
|
||||
|
||||
def test_category_strategy_only_upgrades_matching_category(self) -> None:
|
||||
plugins = [
|
||||
_make_plugin("acme/model-provider", "1.0.0", category=PluginCategory.Model),
|
||||
_make_plugin("acme/tool-provider", "1.0.0", category=PluginCategory.Tool),
|
||||
]
|
||||
manifests = [
|
||||
_make_manifest("acme/model-provider", "1.0.1"),
|
||||
_make_manifest("acme/tool-provider", "1.0.1"),
|
||||
]
|
||||
|
||||
upgrade_mock, calls = _run_task(
|
||||
plugins=plugins,
|
||||
manifests=manifests,
|
||||
upgrade_mode=TenantPluginAutoUpgradeMode.ALL,
|
||||
category=TenantPluginAutoUpgradeCategory.MODEL,
|
||||
)
|
||||
|
||||
upgrade_mock.assert_called_once()
|
||||
assert calls[0][1] == plugins[0].plugin_unique_identifier
|
||||
|
||||
|
||||
class TestErrorIsolation:
|
||||
def test_one_plugin_failure_does_not_block_others(self) -> None:
|
||||
plugins = [
|
||||
_make_plugin("acme/foo", "1.0.0"),
|
||||
_make_plugin("acme/bar", "2.0.0"),
|
||||
]
|
||||
manifests = [
|
||||
_make_manifest("acme/foo", "1.0.1"),
|
||||
_make_manifest("acme/bar", "2.0.1"),
|
||||
]
|
||||
fake_manager = MagicMock()
|
||||
fake_manager.list_plugins.return_value = plugins
|
||||
|
||||
seen: list[str] = []
|
||||
|
||||
def _upgrade(tenant_id: str, original: str, new: str) -> None:
|
||||
seen.append(original)
|
||||
if "foo" in original:
|
||||
raise RuntimeError("boom")
|
||||
|
||||
with (
|
||||
patch(f"{MODULE}.PluginInstaller", return_value=fake_manager),
|
||||
patch(f"{MODULE}.marketplace_batch_fetch_plugin_manifests", return_value=manifests),
|
||||
patch(f"{MODULE}.PluginService.upgrade_plugin_with_marketplace", side_effect=_upgrade),
|
||||
):
|
||||
from tasks.process_tenant_plugin_autoupgrade_check_task import (
|
||||
process_tenant_plugin_autoupgrade_check_task,
|
||||
)
|
||||
|
||||
process_tenant_plugin_autoupgrade_check_task(
|
||||
"tenant-1",
|
||||
TenantPluginAutoUpgradeStrategySetting.LATEST,
|
||||
0,
|
||||
TenantPluginAutoUpgradeMode.ALL,
|
||||
[],
|
||||
[],
|
||||
)
|
||||
|
||||
assert any("foo" in s for s in seen)
|
||||
assert any("bar" in s for s in seen)
|
||||
|
||||
|
||||
def _make_declaration(plugin_id: str, latest_version: str) -> SimpleNamespace:
|
||||
"""Minimal stand-in for a MarketplacePluginDeclaration returned by the batch API."""
|
||||
org, name = plugin_id.split("/", 1)
|
||||
return SimpleNamespace(
|
||||
plugin_id=plugin_id,
|
||||
org=org,
|
||||
name=name,
|
||||
latest_version=latest_version,
|
||||
latest_package_identifier=f"{plugin_id}:{latest_version}@cafe1234",
|
||||
)
|
||||
|
||||
|
||||
class TestManifestCacheFallback:
|
||||
"""
|
||||
The global snapshot is a best-effort optimisation. When it does not cover a plugin,
|
||||
the check must fall back to the batch API instead of silently skipping the upgrade.
|
||||
"""
|
||||
|
||||
def _fetch(
|
||||
self,
|
||||
plugin_ids: list[str],
|
||||
*,
|
||||
cached: dict[str, str],
|
||||
declarations: list[SimpleNamespace] | None = None,
|
||||
batch_error: Exception | None = None,
|
||||
) -> tuple[list[MarketplacePluginSnapshot], MagicMock, MagicMock]:
|
||||
from tasks.process_tenant_plugin_autoupgrade_check_task import (
|
||||
marketplace_batch_fetch_plugin_manifests,
|
||||
)
|
||||
|
||||
redis_mock = MagicMock()
|
||||
redis_mock.get.side_effect = lambda key: cached.get(key)
|
||||
|
||||
batch_mock = MagicMock()
|
||||
if batch_error is not None:
|
||||
batch_mock.side_effect = batch_error
|
||||
else:
|
||||
batch_mock.return_value = declarations or []
|
||||
|
||||
with (
|
||||
patch(f"{MODULE}.redis_client", redis_mock),
|
||||
patch(f"{MODULE}.batch_fetch_plugin_manifests", batch_mock),
|
||||
):
|
||||
result = marketplace_batch_fetch_plugin_manifests(plugin_ids)
|
||||
|
||||
return result, batch_mock, redis_mock
|
||||
|
||||
def test_cache_hits_do_not_touch_the_marketplace(self) -> None:
|
||||
"""The whole point of the snapshot: a warm cache must not generate any request."""
|
||||
key = "plugin_autoupgrade_check_task:cached_plugin_snapshot:langgenius/foo"
|
||||
cached = {key: _make_manifest("langgenius/foo", "1.1.0").model_dump_json()}
|
||||
|
||||
result, batch_mock, _ = self._fetch(["langgenius/foo"], cached=cached)
|
||||
|
||||
assert [r.latest_version for r in result] == ["1.1.0"]
|
||||
batch_mock.assert_not_called()
|
||||
|
||||
def test_empty_snapshot_falls_back_to_batch_api(self) -> None:
|
||||
"""Regression: an empty global snapshot used to skip every plugin silently."""
|
||||
result, batch_mock, _ = self._fetch(
|
||||
["langgenius/foo"],
|
||||
cached={},
|
||||
declarations=[_make_declaration("langgenius/foo", "1.2.0")],
|
||||
)
|
||||
|
||||
batch_mock.assert_called_once_with(["langgenius/foo"])
|
||||
assert [r.plugin_id for r in result] == ["langgenius/foo"]
|
||||
assert [r.latest_version for r in result] == ["1.2.0"]
|
||||
|
||||
def test_fallback_results_are_cached(self) -> None:
|
||||
_, _, redis_mock = self._fetch(
|
||||
["langgenius/foo"],
|
||||
cached={},
|
||||
declarations=[_make_declaration("langgenius/foo", "1.2.0")],
|
||||
)
|
||||
|
||||
cached_keys = [call.args[0] for call in redis_mock.setex.call_args_list]
|
||||
assert "plugin_autoupgrade_check_task:cached_plugin_snapshot:langgenius/foo" in cached_keys
|
||||
|
||||
def test_plugins_unknown_to_the_marketplace_are_negatively_cached(self) -> None:
|
||||
"""Avoid re-asking about a plugin the marketplace does not serve on every cycle."""
|
||||
_, _, redis_mock = self._fetch(["langgenius/ghost"], cached={}, declarations=[])
|
||||
|
||||
redis_mock.setex.assert_called_once()
|
||||
key, _ttl, value = redis_mock.setex.call_args.args
|
||||
assert key == "plugin_autoupgrade_check_task:cached_plugin_snapshot:langgenius/ghost"
|
||||
assert value == "null"
|
||||
|
||||
def test_negatively_cached_plugins_are_not_refetched(self) -> None:
|
||||
key = "plugin_autoupgrade_check_task:cached_plugin_snapshot:langgenius/ghost"
|
||||
|
||||
result, batch_mock, _ = self._fetch(["langgenius/ghost"], cached={key: "null"})
|
||||
|
||||
assert result == []
|
||||
batch_mock.assert_not_called()
|
||||
|
||||
def test_marketplace_failure_keeps_cached_results(self) -> None:
|
||||
"""A failing batch call must not lose the plugins the snapshot did cover."""
|
||||
key = "plugin_autoupgrade_check_task:cached_plugin_snapshot:langgenius/foo"
|
||||
cached = {key: _make_manifest("langgenius/foo", "1.1.0").model_dump_json()}
|
||||
|
||||
result, _, _ = self._fetch(
|
||||
["langgenius/foo", "langgenius/bar"],
|
||||
cached=cached,
|
||||
batch_error=RuntimeError("marketplace down"),
|
||||
)
|
||||
|
||||
assert [r.plugin_id for r in result] == ["langgenius/foo"]
|
||||
@@ -0,0 +1,49 @@
|
||||
from collections.abc import Callable
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from enums import DeploymentEdition
|
||||
from tasks.refresh_billing_vector_space_task import (
|
||||
refresh_billing_vector_space_task,
|
||||
schedule_billing_vector_space_refresh,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _cloud_edition(config_overrides: Callable[..., None]) -> None:
|
||||
config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD)
|
||||
|
||||
|
||||
def test_refresh_invalidates_vector_space_cache() -> None:
|
||||
with (
|
||||
patch(
|
||||
"tasks.refresh_billing_vector_space_task.BillingService.invalidate_vector_space_cache"
|
||||
) as invalidate_cache,
|
||||
):
|
||||
refresh_billing_vector_space_task.run("tenant-1")
|
||||
|
||||
invalidate_cache.assert_called_once_with("tenant-1")
|
||||
|
||||
|
||||
def test_refresh_failure_schedules_retry() -> None:
|
||||
error = RuntimeError("billing unavailable")
|
||||
|
||||
with (
|
||||
patch(
|
||||
"tasks.refresh_billing_vector_space_task.BillingService.invalidate_vector_space_cache",
|
||||
side_effect=error,
|
||||
),
|
||||
patch.object(refresh_billing_vector_space_task, "retry", side_effect=RuntimeError("retry scheduled")) as retry,
|
||||
pytest.raises(RuntimeError, match="retry scheduled"),
|
||||
):
|
||||
refresh_billing_vector_space_task.run("tenant-1")
|
||||
|
||||
retry.assert_called_once_with(exc=error, countdown=30)
|
||||
|
||||
|
||||
def test_dispatch_failure_does_not_propagate() -> None:
|
||||
with (
|
||||
patch.object(refresh_billing_vector_space_task, "delay", side_effect=RuntimeError("broker unavailable")),
|
||||
):
|
||||
schedule_billing_vector_space_refresh("tenant-1")
|
||||
@@ -0,0 +1,627 @@
|
||||
import logging
|
||||
from collections.abc import Generator, Iterator, Sequence
|
||||
from contextlib import AbstractContextManager, contextmanager
|
||||
from datetime import UTC, datetime
|
||||
from unittest.mock import MagicMock, call, patch
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from pytest_mock import MockerFixture
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
import tasks.remove_app_and_related_data_task as remove_app_task_module
|
||||
from enums import DeploymentEdition
|
||||
from extensions.ext_application_services import ApplicationServices
|
||||
from graphon.enums import WorkflowExecutionStatus
|
||||
from graphon.variables.segments import StringSegment
|
||||
from libs.archive_storage import ArchiveStorageNotConfiguredError
|
||||
from models import AppStar
|
||||
from models.agent import WorkflowAgentBindingType, WorkflowAgentNodeBinding
|
||||
from models.enums import CreatorUserRole, WorkflowRunTriggeredFrom
|
||||
from models.workflow import WorkflowArchiveLog, WorkflowDraftVariable
|
||||
from services.network_access_group_service import NetworkAccessGroupUpstreamError
|
||||
from tasks.remove_app_and_related_data_task import (
|
||||
_delete_app_stars,
|
||||
_delete_app_workflow_archive_logs,
|
||||
_delete_archived_workflow_run_files,
|
||||
_delete_draft_variable_offload_data,
|
||||
_delete_draft_variables,
|
||||
_delete_workflow_agent_node_bindings,
|
||||
delete_draft_variables_batch,
|
||||
)
|
||||
from tests.helpers.unit.config_override import apply_config_overrides
|
||||
|
||||
|
||||
def test_delete_workflow_agent_node_bindings_is_scoped_to_tenant_and_app(sqlite_session: Session) -> None:
|
||||
target = WorkflowAgentNodeBinding(
|
||||
tenant_id="tenant-1",
|
||||
app_id="app-1",
|
||||
workflow_id="workflow-1",
|
||||
workflow_version="draft",
|
||||
node_id="node-1",
|
||||
binding_type=WorkflowAgentBindingType.INLINE_AGENT,
|
||||
agent_id="agent-1",
|
||||
current_snapshot_id="snapshot-1",
|
||||
node_job_config={},
|
||||
)
|
||||
kept = WorkflowAgentNodeBinding(
|
||||
tenant_id="tenant-1",
|
||||
app_id="app-2",
|
||||
workflow_id="workflow-2",
|
||||
workflow_version="draft",
|
||||
node_id="node-2",
|
||||
binding_type=WorkflowAgentBindingType.INLINE_AGENT,
|
||||
agent_id="agent-2",
|
||||
current_snapshot_id="snapshot-2",
|
||||
node_job_config={},
|
||||
)
|
||||
other_tenant = WorkflowAgentNodeBinding(
|
||||
tenant_id="tenant-2",
|
||||
app_id="app-1",
|
||||
workflow_id="workflow-3",
|
||||
workflow_version="draft",
|
||||
node_id="node-3",
|
||||
binding_type=WorkflowAgentBindingType.INLINE_AGENT,
|
||||
agent_id="agent-3",
|
||||
current_snapshot_id="snapshot-3",
|
||||
node_job_config={},
|
||||
)
|
||||
sqlite_session.add_all([target, kept, other_tenant])
|
||||
sqlite_session.commit()
|
||||
target_id = target.id
|
||||
kept_id = kept.id
|
||||
other_tenant_id = other_tenant.id
|
||||
|
||||
_delete_workflow_agent_node_bindings("tenant-1", "app-1")
|
||||
|
||||
sqlite_session.expire_all()
|
||||
assert sqlite_session.get(WorkflowAgentNodeBinding, target_id) is None
|
||||
assert sqlite_session.get(WorkflowAgentNodeBinding, kept_id) is not None
|
||||
assert sqlite_session.get(WorkflowAgentNodeBinding, other_tenant_id) is not None
|
||||
|
||||
|
||||
def test_app_cleanup_removes_agent_bindings_before_workflows(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
events: list[str] = []
|
||||
apply_config_overrides(monkeypatch, DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY)
|
||||
other_cleanup_names = (
|
||||
"cleanup_app_storage",
|
||||
"_delete_app_model_configs",
|
||||
"_delete_app_site",
|
||||
"_delete_app_mcp_servers",
|
||||
"_delete_app_api_tokens",
|
||||
"_delete_installed_apps",
|
||||
"_delete_app_stars",
|
||||
"_delete_recommended_apps",
|
||||
"_delete_app_annotation_data",
|
||||
"_delete_app_dataset_joins",
|
||||
"_delete_app_workflow_runs",
|
||||
"_delete_app_workflow_node_executions",
|
||||
"_delete_app_workflow_app_logs",
|
||||
"_delete_app_conversations",
|
||||
"_delete_app_messages",
|
||||
"_delete_workflow_tool_providers",
|
||||
"_delete_app_tag_bindings",
|
||||
"_delete_end_users",
|
||||
"_delete_trace_app_configs",
|
||||
"_delete_conversation_variables",
|
||||
"_delete_draft_variables",
|
||||
"_delete_app_triggers",
|
||||
"_delete_workflow_plugin_triggers",
|
||||
"_delete_workflow_webhook_triggers",
|
||||
"_delete_workflow_schedule_plans",
|
||||
"_delete_workflow_trigger_logs",
|
||||
)
|
||||
for name in other_cleanup_names:
|
||||
monkeypatch.setattr(remove_app_task_module, name, MagicMock())
|
||||
|
||||
delete_bindings = MagicMock(side_effect=lambda *_args: events.append("bindings"))
|
||||
delete_workflows = MagicMock(side_effect=lambda *_args: events.append("workflows"))
|
||||
monkeypatch.setattr(remove_app_task_module, "_delete_workflow_agent_node_bindings", delete_bindings)
|
||||
monkeypatch.setattr(remove_app_task_module, "_delete_app_workflows", delete_workflows)
|
||||
monkeypatch.setattr(
|
||||
remove_app_task_module, "_delete_app_workflow_node_executions", lambda *_args: events.append("executions")
|
||||
)
|
||||
monkeypatch.setattr(remove_app_task_module, "_delete_app_workflow_runs", lambda *_args: events.append("runs"))
|
||||
|
||||
remove_app_task_module.remove_app_and_related_data_task.run(tenant_id="tenant-1", app_id="app-1")
|
||||
|
||||
assert events == ["bindings", "workflows", "executions", "runs"]
|
||||
delete_bindings.assert_called_once_with("tenant-1", "app-1")
|
||||
delete_workflows.assert_called_once_with("tenant-1", "app-1")
|
||||
|
||||
|
||||
def test_cloud_app_cleanup_schedules_independent_binding_cleanup(
|
||||
monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture
|
||||
) -> None:
|
||||
apply_config_overrides(monkeypatch, DEPLOYMENT_EDITION=DeploymentEdition.CLOUD)
|
||||
apply_async = mocker.patch.object(
|
||||
remove_app_task_module.cleanup_app_network_access_group_binding_task,
|
||||
"apply_async",
|
||||
)
|
||||
|
||||
remove_app_task_module._schedule_app_network_access_group_binding_cleanup("tenant-1", "app-1")
|
||||
|
||||
apply_async.assert_called_once_with(
|
||||
kwargs={"tenant_id": "tenant-1", "app_id": "app-1"},
|
||||
countdown=10,
|
||||
)
|
||||
|
||||
|
||||
def test_community_app_cleanup_does_not_schedule_binding_cleanup(
|
||||
monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture
|
||||
) -> None:
|
||||
apply_config_overrides(monkeypatch, DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY)
|
||||
apply_async = mocker.patch.object(
|
||||
remove_app_task_module.cleanup_app_network_access_group_binding_task,
|
||||
"apply_async",
|
||||
)
|
||||
|
||||
remove_app_task_module._schedule_app_network_access_group_binding_cleanup("tenant-1", "app-1")
|
||||
|
||||
apply_async.assert_not_called()
|
||||
|
||||
|
||||
def test_independent_binding_cleanup_task_succeeds(
|
||||
monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture, account_application_services: ApplicationServices
|
||||
) -> None:
|
||||
cleanup_binding = mocker.patch.object(
|
||||
account_application_services.network_access_groups, "cleanup_app_binding", return_value=True
|
||||
)
|
||||
apply_config_overrides(monkeypatch, DEPLOYMENT_EDITION=DeploymentEdition.CLOUD)
|
||||
|
||||
with patch(
|
||||
"extensions.ext_application_services.application_services",
|
||||
return_value=account_application_services,
|
||||
):
|
||||
remove_app_task_module.cleanup_app_network_access_group_binding_task.run(
|
||||
tenant_id="tenant-1",
|
||||
app_id="app-1",
|
||||
)
|
||||
|
||||
cleanup_binding.assert_called_once_with(workspace_id="tenant-1", app_id="app-1")
|
||||
assert remove_app_task_module.cleanup_app_network_access_group_binding_task.max_retries == 24
|
||||
|
||||
|
||||
def test_independent_binding_cleanup_task_retries_with_exponential_backoff(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
mocker: MockerFixture,
|
||||
account_application_services: ApplicationServices,
|
||||
) -> None:
|
||||
upstream_error = NetworkAccessGroupUpstreamError(unavailable=False)
|
||||
retry_error = RuntimeError("retry scheduled")
|
||||
mocker.patch.object(
|
||||
account_application_services.network_access_groups, "cleanup_app_binding", side_effect=upstream_error
|
||||
)
|
||||
apply_config_overrides(monkeypatch, DEPLOYMENT_EDITION=DeploymentEdition.CLOUD)
|
||||
retry = mocker.patch.object(
|
||||
remove_app_task_module.cleanup_app_network_access_group_binding_task, "retry", side_effect=retry_error
|
||||
)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"extensions.ext_application_services.application_services",
|
||||
return_value=account_application_services,
|
||||
),
|
||||
pytest.raises(RuntimeError, match="retry scheduled"),
|
||||
):
|
||||
remove_app_task_module.cleanup_app_network_access_group_binding_task.run(
|
||||
tenant_id="tenant-1",
|
||||
app_id="app-1",
|
||||
)
|
||||
|
||||
retry.assert_called_once_with(exc=upstream_error, countdown=15)
|
||||
assert remove_app_task_module._network_access_binding_cleanup_retry_delay(5) == 480
|
||||
assert remove_app_task_module._network_access_binding_cleanup_retry_delay(6) == 600
|
||||
assert remove_app_task_module._network_access_binding_cleanup_retry_delay(24) == 600
|
||||
|
||||
|
||||
def test_parent_local_cleanup_failure_happens_after_binding_cleanup_is_scheduled(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
local_error = RuntimeError("local cleanup failed")
|
||||
retry_error = RuntimeError("parent retry scheduled")
|
||||
apply_config_overrides(monkeypatch, DEPLOYMENT_EDITION=DeploymentEdition.CLOUD)
|
||||
apply_async = mocker.patch.object(
|
||||
remove_app_task_module.cleanup_app_network_access_group_binding_task,
|
||||
"apply_async",
|
||||
)
|
||||
mocker.patch.object(remove_app_task_module, "_delete_app_model_configs", side_effect=local_error)
|
||||
retry = mocker.patch.object(
|
||||
remove_app_task_module.remove_app_and_related_data_task, "retry", side_effect=retry_error
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="parent retry scheduled"):
|
||||
remove_app_task_module.remove_app_and_related_data_task.run(tenant_id="tenant-1", app_id="app-1")
|
||||
|
||||
apply_async.assert_called_once_with(
|
||||
kwargs={"tenant_id": "tenant-1", "app_id": "app-1"},
|
||||
countdown=10,
|
||||
)
|
||||
retry.assert_called_once_with(exc=local_error, countdown=60)
|
||||
|
||||
|
||||
def test_parent_retries_before_local_cleanup_when_binding_schedule_fails(
|
||||
monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture
|
||||
) -> None:
|
||||
broker_error = RuntimeError("broker unavailable")
|
||||
retry_error = RuntimeError("parent retry scheduled")
|
||||
apply_config_overrides(monkeypatch, DEPLOYMENT_EDITION=DeploymentEdition.CLOUD)
|
||||
mocker.patch.object(
|
||||
remove_app_task_module.cleanup_app_network_access_group_binding_task,
|
||||
"apply_async",
|
||||
side_effect=broker_error,
|
||||
)
|
||||
first_local_cleanup = mocker.patch.object(remove_app_task_module, "_delete_app_model_configs")
|
||||
retry = mocker.patch.object(
|
||||
remove_app_task_module.remove_app_and_related_data_task, "retry", side_effect=retry_error
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="parent retry scheduled"):
|
||||
remove_app_task_module.remove_app_and_related_data_task.run(tenant_id="tenant-1", app_id="app-1")
|
||||
|
||||
retry.assert_called_once_with(exc=broker_error, countdown=60)
|
||||
first_local_cleanup.assert_not_called()
|
||||
|
||||
|
||||
class TestDeleteDraftVariablesBatch:
|
||||
def test_delete_draft_variables_batch_invalid_batch_size(self) -> None:
|
||||
"""Test that invalid batch size raises ValueError."""
|
||||
app_id = "test-app-id"
|
||||
|
||||
with pytest.raises(ValueError, match="batch_size must be positive"):
|
||||
delete_draft_variables_batch(app_id, -1)
|
||||
|
||||
with pytest.raises(ValueError, match="batch_size must be positive"):
|
||||
delete_draft_variables_batch(app_id, 0)
|
||||
|
||||
@patch("tasks.remove_app_and_related_data_task.delete_draft_variables_batch")
|
||||
def test_delete_draft_variables_calls_batch_function(self, mock_batch_delete: MagicMock) -> None:
|
||||
"""Test that _delete_draft_variables calls the batch function correctly."""
|
||||
app_id = "test-app-id"
|
||||
expected_return = 42
|
||||
mock_batch_delete.return_value = expected_return
|
||||
|
||||
result = _delete_draft_variables(app_id)
|
||||
|
||||
assert result == expected_return
|
||||
mock_batch_delete.assert_called_once_with(app_id, batch_size=1000)
|
||||
|
||||
|
||||
class _FakeResult:
|
||||
"""Minimal stand-in for a SQLAlchemy ``Result``."""
|
||||
|
||||
def __init__(self, rows: Sequence[Sequence[object]] = (), rowcount: int = 0) -> None:
|
||||
self._rows = [tuple(row) for row in rows]
|
||||
self.rowcount = rowcount
|
||||
|
||||
def __iter__(self) -> Iterator[Sequence[object]]:
|
||||
return iter(self._rows)
|
||||
|
||||
|
||||
class _FakeSessionFactory:
|
||||
"""Session factory stub that counts open sessions and hands out queued results.
|
||||
|
||||
Lets the tests assert that object storage cleanup runs with no session open.
|
||||
"""
|
||||
|
||||
def __init__(self, results: list[_FakeResult] | None = None) -> None:
|
||||
self._results = list(results or [])
|
||||
self.open_count = 0
|
||||
self.open_count_at_storage_delete: list[int] = []
|
||||
|
||||
def create_session(self) -> AbstractContextManager[MagicMock]:
|
||||
factory = self
|
||||
|
||||
class _TransactionContext:
|
||||
def __init__(self, session: MagicMock) -> None:
|
||||
self._session = session
|
||||
|
||||
def __enter__(self) -> MagicMock:
|
||||
factory.open_count += 1
|
||||
return self._session
|
||||
|
||||
def __exit__(self, *_exc_info: object) -> bool:
|
||||
factory.open_count -= 1
|
||||
return False
|
||||
|
||||
class _SessionContext:
|
||||
def __enter__(self) -> MagicMock:
|
||||
factory.open_count += 1
|
||||
session = MagicMock()
|
||||
session.execute.side_effect = factory.next_result
|
||||
session.begin.return_value = _TransactionContext(session)
|
||||
return session
|
||||
|
||||
def __exit__(self, *_exc_info: object) -> bool:
|
||||
factory.open_count -= 1
|
||||
return False
|
||||
|
||||
return _SessionContext()
|
||||
|
||||
def next_result(self, *_args: object, **_kwargs: object) -> _FakeResult:
|
||||
if self._results:
|
||||
return self._results.pop(0)
|
||||
return _FakeResult([])
|
||||
|
||||
|
||||
class TestDeleteDraftVariableOffloadData:
|
||||
"""Test the Offload data cleanup functionality."""
|
||||
|
||||
def test_delete_draft_variable_offload_data_empty_file_ids(self) -> None:
|
||||
"""An empty file_ids list must not open a session or touch object storage."""
|
||||
with (
|
||||
patch("tasks.remove_app_and_related_data_task.session_factory") as mock_session_factory,
|
||||
patch("extensions.ext_storage.storage") as mock_storage,
|
||||
):
|
||||
result = _delete_draft_variable_offload_data([])
|
||||
|
||||
assert result == 0
|
||||
mock_session_factory.create_session.assert_not_called()
|
||||
mock_storage.delete.assert_not_called()
|
||||
|
||||
def test_delete_draft_variable_offload_data_database_failure(self, caplog: pytest.LogCaptureFixture) -> None:
|
||||
"""Test handling of database operation failures."""
|
||||
mock_session_factory = MagicMock()
|
||||
mock_session_factory.create_session.return_value.__enter__.return_value.execute.side_effect = Exception(
|
||||
"Database error"
|
||||
)
|
||||
|
||||
with patch("tasks.remove_app_and_related_data_task.session_factory", mock_session_factory):
|
||||
with caplog.at_level(logging.ERROR):
|
||||
result = _delete_draft_variable_offload_data(["file-1"])
|
||||
|
||||
assert result == 0
|
||||
assert "Error deleting draft variable offload data:" in caplog.text
|
||||
|
||||
def test_storage_delete_runs_with_no_open_transaction(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Object storage cleanup must not run while a database transaction is open."""
|
||||
factory = _FakeSessionFactory([_FakeResult([("vf-1", "key-1", "uf-1"), ("vf-2", "key-2", "uf-2")])])
|
||||
mock_storage = MagicMock()
|
||||
mock_storage.delete.side_effect = lambda _key: factory.open_count_at_storage_delete.append(factory.open_count)
|
||||
|
||||
monkeypatch.setattr(remove_app_task_module, "session_factory", factory)
|
||||
monkeypatch.setattr("extensions.ext_storage.storage", mock_storage)
|
||||
|
||||
result = _delete_draft_variable_offload_data(["vf-1", "vf-2"])
|
||||
|
||||
assert result == 2
|
||||
assert mock_storage.delete.call_count == 2
|
||||
assert factory.open_count_at_storage_delete == [0, 0]
|
||||
assert factory.open_count == 0
|
||||
|
||||
def test_storage_failure_is_logged_and_not_counted(
|
||||
self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
"""A failing object is logged and not counted, and the DB cleanup still runs."""
|
||||
factory = _FakeSessionFactory([_FakeResult([("vf-1", "key-1", "uf-1"), ("vf-2", "key-2", "uf-2")])])
|
||||
mock_storage = MagicMock()
|
||||
mock_storage.delete.side_effect = [Exception("Storage error"), None]
|
||||
|
||||
monkeypatch.setattr(remove_app_task_module, "session_factory", factory)
|
||||
monkeypatch.setattr("extensions.ext_storage.storage", mock_storage)
|
||||
|
||||
with caplog.at_level(logging.ERROR):
|
||||
result = _delete_draft_variable_offload_data(["vf-1", "vf-2"])
|
||||
|
||||
assert result == 1
|
||||
assert "Failed to delete storage object key-1" in caplog.text
|
||||
assert factory.open_count == 0
|
||||
|
||||
def test_file_ids_with_no_matching_rows_delete_nothing(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""file_ids that no longer resolve to rows must not raise or touch storage."""
|
||||
factory = _FakeSessionFactory([_FakeResult([])])
|
||||
mock_storage = MagicMock()
|
||||
|
||||
monkeypatch.setattr(remove_app_task_module, "session_factory", factory)
|
||||
monkeypatch.setattr("extensions.ext_storage.storage", mock_storage)
|
||||
|
||||
result = _delete_draft_variable_offload_data(["vf-gone"])
|
||||
|
||||
assert result == 0
|
||||
mock_storage.delete.assert_not_called()
|
||||
assert factory.open_count == 0
|
||||
|
||||
|
||||
class TestDeleteDraftVariablesBatchTransactionScope:
|
||||
"""The batch loop must not hold a transaction while object storage is cleaned up."""
|
||||
|
||||
def test_offload_cleanup_is_called_with_no_session_open(
|
||||
self, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, sqlite_session_factory: sessionmaker[Session]
|
||||
) -> None:
|
||||
variable = WorkflowDraftVariable.new_node_variable(
|
||||
app_id="app-1",
|
||||
user_id="user-1",
|
||||
node_id="node-1",
|
||||
name="value",
|
||||
value=StringSegment(value="truncated"),
|
||||
node_execution_id="exec-1",
|
||||
file_id="vf-1",
|
||||
)
|
||||
other = WorkflowDraftVariable.new_node_variable(
|
||||
app_id="app-2",
|
||||
user_id="user-1",
|
||||
node_id="node-1",
|
||||
name="value",
|
||||
value=StringSegment(value="untouched"),
|
||||
node_execution_id="exec-2",
|
||||
)
|
||||
sqlite_session.add_all([variable, other])
|
||||
sqlite_session.commit()
|
||||
variable_id, other_id = variable.id, other.id
|
||||
open_sessions: list[Session] = []
|
||||
|
||||
@contextmanager
|
||||
def tracked_session() -> Generator[Session]:
|
||||
with sqlite_session_factory() as session:
|
||||
open_sessions.append(session)
|
||||
try:
|
||||
yield session
|
||||
finally:
|
||||
open_sessions.remove(session)
|
||||
|
||||
cleaned: list[list[str]] = []
|
||||
|
||||
def _record_cleanup(file_ids: list[str]) -> int:
|
||||
assert not open_sessions
|
||||
cleaned.append(file_ids)
|
||||
return len(file_ids)
|
||||
|
||||
monkeypatch.setattr(remove_app_task_module.session_factory, "create_session", tracked_session)
|
||||
monkeypatch.setattr(remove_app_task_module, "_delete_draft_variable_offload_data", _record_cleanup)
|
||||
|
||||
total_deleted = delete_draft_variables_batch("app-1", batch_size=10)
|
||||
|
||||
assert total_deleted == 1
|
||||
assert cleaned == [["vf-1"]]
|
||||
assert not open_sessions
|
||||
sqlite_session.expire_all()
|
||||
assert sqlite_session.get(WorkflowDraftVariable, variable_id) is None
|
||||
assert sqlite_session.get(WorkflowDraftVariable, other_id) is not None
|
||||
|
||||
def test_rows_without_offloaded_files_skip_cleanup(
|
||||
self, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session
|
||||
) -> None:
|
||||
"""A batch whose rows carry no file_id must not invoke the offload cleanup."""
|
||||
variable = WorkflowDraftVariable.new_node_variable(
|
||||
app_id="app-1",
|
||||
user_id="user-1",
|
||||
node_id="node-1",
|
||||
name="value",
|
||||
value=StringSegment(value="plain"),
|
||||
node_execution_id="exec-1",
|
||||
)
|
||||
sqlite_session.add(variable)
|
||||
sqlite_session.commit()
|
||||
variable_id = variable.id
|
||||
cleanup = MagicMock(return_value=0)
|
||||
|
||||
monkeypatch.setattr(remove_app_task_module, "_delete_draft_variable_offload_data", cleanup)
|
||||
|
||||
total_deleted = delete_draft_variables_batch("app-1", batch_size=10)
|
||||
|
||||
assert total_deleted == 1
|
||||
cleanup.assert_not_called()
|
||||
sqlite_session.expire_all()
|
||||
assert sqlite_session.get(WorkflowDraftVariable, variable_id) is None
|
||||
|
||||
|
||||
class TestDeleteWorkflowArchiveLogs:
|
||||
@pytest.mark.parametrize("sqlite_session", [(WorkflowArchiveLog,)], indirect=True)
|
||||
@patch("tasks.remove_app_and_related_data_task._delete_records")
|
||||
@patch("tasks.remove_app_and_related_data_task.db")
|
||||
def test_delete_app_workflow_archive_logs_calls_delete_records(
|
||||
self, mock_db: MagicMock, mock_delete_records: MagicMock, sqlite_session: Session
|
||||
) -> None:
|
||||
tenant_id = "tenant-1"
|
||||
app_id = "app-1"
|
||||
|
||||
_delete_app_workflow_archive_logs(tenant_id, app_id)
|
||||
|
||||
mock_delete_records.assert_called_once()
|
||||
query_sql, params, delete_func, name = mock_delete_records.call_args[0]
|
||||
assert "workflow_archive_logs" in query_sql
|
||||
assert params == {"tenant_id": tenant_id, "app_id": app_id}
|
||||
assert name == "workflow archive log"
|
||||
|
||||
archive_log = WorkflowArchiveLog(
|
||||
tenant_id=str(uuid4()),
|
||||
app_id=str(uuid4()),
|
||||
workflow_id=str(uuid4()),
|
||||
workflow_run_id=str(uuid4()),
|
||||
created_by_role=CreatorUserRole.ACCOUNT,
|
||||
created_by=str(uuid4()),
|
||||
log_id=None,
|
||||
log_created_at=None,
|
||||
log_created_from=None,
|
||||
run_version="1",
|
||||
run_status=WorkflowExecutionStatus.SUCCEEDED,
|
||||
run_triggered_from=WorkflowRunTriggeredFrom.APP_RUN,
|
||||
run_error=None,
|
||||
run_elapsed_time=0,
|
||||
run_total_tokens=0,
|
||||
run_total_steps=1,
|
||||
run_created_at=datetime.now(UTC),
|
||||
run_finished_at=datetime.now(UTC),
|
||||
run_exceptions_count=0,
|
||||
trigger_metadata=None,
|
||||
)
|
||||
sqlite_session.add(archive_log)
|
||||
sqlite_session.commit()
|
||||
|
||||
delete_func(sqlite_session, archive_log.id)
|
||||
sqlite_session.commit()
|
||||
sqlite_session.expunge_all()
|
||||
|
||||
assert sqlite_session.get(WorkflowArchiveLog, archive_log.id) is None
|
||||
|
||||
|
||||
class TestDeleteAppStars:
|
||||
@pytest.mark.parametrize("sqlite_session", [(AppStar,)], indirect=True)
|
||||
@patch("tasks.remove_app_and_related_data_task._delete_records")
|
||||
def test_delete_app_stars_calls_delete_records(
|
||||
self, mock_delete_records: MagicMock, sqlite_session: Session
|
||||
) -> None:
|
||||
tenant_id = "tenant-1"
|
||||
app_id = "app-1"
|
||||
|
||||
_delete_app_stars(tenant_id, app_id)
|
||||
|
||||
mock_delete_records.assert_called_once()
|
||||
query_sql, params, delete_func, name = mock_delete_records.call_args[0]
|
||||
assert "app_stars" in query_sql
|
||||
assert params == {"tenant_id": tenant_id, "app_id": app_id}
|
||||
assert name == "app star"
|
||||
|
||||
app_star = AppStar(tenant_id=str(uuid4()), app_id=str(uuid4()), account_id=str(uuid4()))
|
||||
sqlite_session.add(app_star)
|
||||
sqlite_session.commit()
|
||||
|
||||
delete_func(sqlite_session, app_star.id)
|
||||
sqlite_session.commit()
|
||||
sqlite_session.expunge_all()
|
||||
|
||||
assert sqlite_session.get(AppStar, app_star.id) is None
|
||||
|
||||
|
||||
class TestDeleteArchivedWorkflowRunFiles:
|
||||
@patch("tasks.remove_app_and_related_data_task.get_archive_storage")
|
||||
def test_delete_archived_workflow_run_files_not_configured(
|
||||
self, mock_get_storage: MagicMock, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
mock_get_storage.side_effect = ArchiveStorageNotConfiguredError("missing config")
|
||||
|
||||
with caplog.at_level(logging.INFO, logger="tasks.remove_app_and_related_data_task"):
|
||||
_delete_archived_workflow_run_files("tenant-1", "app-1")
|
||||
|
||||
assert caplog.text.count("Archive storage not configured") == 1
|
||||
|
||||
@patch("tasks.remove_app_and_related_data_task.get_archive_storage")
|
||||
def test_delete_archived_workflow_run_files_list_failure(
|
||||
self, mock_get_storage: MagicMock, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
storage = MagicMock()
|
||||
storage.list_objects.side_effect = Exception("list failed")
|
||||
mock_get_storage.return_value = storage
|
||||
|
||||
with caplog.at_level(logging.ERROR, logger="tasks.remove_app_and_related_data_task"):
|
||||
_delete_archived_workflow_run_files("tenant-1", "app-1")
|
||||
|
||||
storage.list_objects.assert_called_once_with("tenant-1/app_id=app-1/")
|
||||
storage.delete_object.assert_not_called()
|
||||
assert "Failed to list archive files for app app-1" in caplog.text
|
||||
|
||||
@patch("tasks.remove_app_and_related_data_task.get_archive_storage")
|
||||
def test_delete_archived_workflow_run_files_success(
|
||||
self, mock_get_storage: MagicMock, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
storage = MagicMock()
|
||||
storage.list_objects.return_value = ["key-1", "key-2"]
|
||||
mock_get_storage.return_value = storage
|
||||
|
||||
with caplog.at_level(logging.INFO, logger="tasks.remove_app_and_related_data_task"):
|
||||
_delete_archived_workflow_run_files("tenant-1", "app-1")
|
||||
|
||||
storage.list_objects.assert_called_once_with("tenant-1/app_id=app-1/")
|
||||
storage.delete_object.assert_has_calls([call("key-1"), call("key-2")], any_order=False)
|
||||
assert "Deleted 2 archive objects for app app-1" in caplog.text
|
||||
@@ -0,0 +1,80 @@
|
||||
from unittest.mock import ANY, MagicMock, patch
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from core.app.apps.pipeline.pipeline_generator import PipelineGenerator
|
||||
from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType
|
||||
from enums import DeploymentEdition
|
||||
from models import Account, Tenant, TenantAccountJoin
|
||||
from models.account import TenantAccountRole
|
||||
from models.dataset import Dataset, Document
|
||||
from models.enums import DatasetRuntimeMode, DataSourceType, DocumentCreatedFrom, IndexingStatus
|
||||
from tasks.retry_document_indexing_task import retry_document_indexing_task
|
||||
from tests.helpers.unit.config_override import config_overrides_context
|
||||
|
||||
|
||||
@config_overrides_context(DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY)
|
||||
@pytest.mark.parametrize("runtime_mode", [DatasetRuntimeMode.GENERAL, DatasetRuntimeMode.RAG_PIPELINE])
|
||||
def test_retry_uses_dataset_runtime(sqlite_session: Session, runtime_mode: DatasetRuntimeMode) -> None:
|
||||
tenant = Tenant(name="Retry tenant")
|
||||
user = Account(name="Retry user", email=f"retry-{uuid4()}@example.com")
|
||||
membership = TenantAccountJoin(
|
||||
tenant_id=tenant.id,
|
||||
account_id=user.id,
|
||||
current=True,
|
||||
role=TenantAccountRole.OWNER,
|
||||
)
|
||||
dataset = Dataset(
|
||||
id=str(uuid4()),
|
||||
tenant_id=tenant.id,
|
||||
name="Retry dataset",
|
||||
created_by=user.id,
|
||||
data_source_type=DataSourceType.UPLOAD_FILE,
|
||||
indexing_technique=IndexTechniqueType.ECONOMY,
|
||||
chunk_structure=IndexStructureType.PARAGRAPH_INDEX,
|
||||
runtime_mode=runtime_mode,
|
||||
)
|
||||
document = Document(
|
||||
id=str(uuid4()),
|
||||
tenant_id=tenant.id,
|
||||
dataset_id=dataset.id,
|
||||
position=1,
|
||||
data_source_type=DataSourceType.UPLOAD_FILE,
|
||||
data_source_info="{}",
|
||||
batch="retry-batch",
|
||||
name="Retry document",
|
||||
created_from=DocumentCreatedFrom.WEB,
|
||||
created_by=user.id,
|
||||
indexing_status=IndexingStatus.COMPLETED,
|
||||
enabled=True,
|
||||
archived=False,
|
||||
doc_form=IndexStructureType.PARAGRAPH_INDEX,
|
||||
)
|
||||
sqlite_session.add_all([tenant, user, membership, dataset, document])
|
||||
sqlite_session.commit()
|
||||
features = MagicMock()
|
||||
|
||||
with (
|
||||
patch("tasks.retry_document_indexing_task.FeatureService.get_features", return_value=features),
|
||||
patch("tasks.retry_document_indexing_task.IndexProcessorFactory"),
|
||||
patch("tasks.retry_document_indexing_task.build_document_indexing_service") as indexing_runner,
|
||||
patch("tasks.retry_document_indexing_task.RagPipelineService") as pipeline_service,
|
||||
patch("tasks.retry_document_indexing_task.redis_client"),
|
||||
):
|
||||
retry_document_indexing_task.run(dataset.id, [document.id], user.id)
|
||||
|
||||
if runtime_mode == DatasetRuntimeMode.GENERAL:
|
||||
pipeline_service.assert_not_called()
|
||||
indexing_runner.assert_called_once_with(session_factory=ANY, enforce_vector_space_admission=True)
|
||||
(run_documents,) = indexing_runner.return_value.run.call_args.args
|
||||
assert [item.document_id for item in run_documents] == [document.id]
|
||||
assert run_documents[0].dataset.tenant_id == dataset.tenant_id
|
||||
else:
|
||||
indexing_runner.assert_not_called()
|
||||
retry = pipeline_service.return_value.retry_error_document
|
||||
retry.assert_called_once()
|
||||
retry_dataset, retry_document, retry_user = retry.call_args.args
|
||||
assert (retry_dataset.id, retry_document.id, retry_user.id) == (dataset.id, document.id, user.id)
|
||||
assert isinstance(retry.call_args.kwargs["generator"], PipelineGenerator)
|
||||
@@ -0,0 +1,98 @@
|
||||
import uuid
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from core.rag.index_processor.constant.index_type import IndexStructureType
|
||||
from enums import DeploymentEdition
|
||||
from models.dataset import Dataset, Document, DocumentSegment
|
||||
from models.enums import DataSourceType
|
||||
from tasks.sync_website_document_indexing_task import sync_website_document_indexing_task
|
||||
from tests.helpers.unit.config_override import config_overrides_context
|
||||
from tests.helpers.unit.model_factories import make_dataset, make_document
|
||||
|
||||
|
||||
def _dataset(tenant_id: str) -> Dataset:
|
||||
return make_dataset(
|
||||
dataset_id=str(uuid.uuid4()),
|
||||
tenant_id=tenant_id,
|
||||
name="Website dataset",
|
||||
data_source_type=DataSourceType.WEBSITE_CRAWL,
|
||||
created_by=str(uuid.uuid4()),
|
||||
)
|
||||
|
||||
|
||||
def _document(dataset: Dataset) -> Document:
|
||||
return make_document(
|
||||
document_id=str(uuid.uuid4()),
|
||||
tenant_id=dataset.tenant_id,
|
||||
dataset_id=dataset.id,
|
||||
data_source_type=DataSourceType.WEBSITE_CRAWL,
|
||||
name="Website document",
|
||||
created_by=str(uuid.uuid4()),
|
||||
doc_form=IndexStructureType.PARAGRAPH_INDEX,
|
||||
)
|
||||
|
||||
|
||||
def _segment(*, tenant_id: str, dataset_id: str, document_id: str) -> DocumentSegment:
|
||||
return DocumentSegment(
|
||||
tenant_id=tenant_id,
|
||||
dataset_id=dataset_id,
|
||||
document_id=document_id,
|
||||
position=1,
|
||||
content="content",
|
||||
word_count=1,
|
||||
tokens=1,
|
||||
created_by=str(uuid.uuid4()),
|
||||
)
|
||||
|
||||
|
||||
def test_rejects_document_outside_dataset_before_side_effects(sqlite_session: Session) -> None:
|
||||
tenant_id = str(uuid.uuid4())
|
||||
requested_dataset = _dataset(tenant_id)
|
||||
foreign_dataset = _dataset(tenant_id)
|
||||
foreign_document = _document(foreign_dataset)
|
||||
sqlite_session.add_all([requested_dataset, foreign_dataset, foreign_document])
|
||||
sqlite_session.commit()
|
||||
|
||||
with (
|
||||
patch("tasks.sync_website_document_indexing_task.FeatureService") as feature_service,
|
||||
patch("tasks.sync_website_document_indexing_task.IndexProcessorFactory") as processor_factory,
|
||||
):
|
||||
sync_website_document_indexing_task(requested_dataset.id, foreign_document.id)
|
||||
|
||||
feature_service.get_features.assert_not_called()
|
||||
processor_factory.assert_not_called()
|
||||
|
||||
|
||||
@config_overrides_context(DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY)
|
||||
def test_cleanup_is_owner_scoped_and_skips_empty_vector_ids(sqlite_session: Session) -> None:
|
||||
tenant_id = str(uuid.uuid4())
|
||||
dataset = _dataset(tenant_id)
|
||||
document = _document(dataset)
|
||||
owned_segment = _segment(tenant_id=tenant_id, dataset_id=dataset.id, document_id=document.id)
|
||||
other_dataset = _dataset(tenant_id)
|
||||
decoy_segments = [
|
||||
_segment(tenant_id=tenant_id, dataset_id=dataset.id, document_id=str(uuid.uuid4())),
|
||||
_segment(tenant_id=tenant_id, dataset_id=other_dataset.id, document_id=document.id),
|
||||
_segment(tenant_id=str(uuid.uuid4()), dataset_id=dataset.id, document_id=document.id),
|
||||
]
|
||||
for index, segment in enumerate(decoy_segments):
|
||||
segment.index_node_id = f"decoy-node-{index}"
|
||||
sqlite_session.add_all([dataset, document, owned_segment, other_dataset, *decoy_segments])
|
||||
sqlite_session.commit()
|
||||
|
||||
features = MagicMock()
|
||||
with (
|
||||
patch("tasks.sync_website_document_indexing_task.FeatureService.get_features", return_value=features),
|
||||
patch("tasks.sync_website_document_indexing_task.IndexProcessorFactory") as processor_factory,
|
||||
patch("tasks.sync_website_document_indexing_task.build_document_indexing_service") as indexing_runner,
|
||||
patch("tasks.sync_website_document_indexing_task.redis_client"),
|
||||
):
|
||||
sync_website_document_indexing_task(dataset.id, document.id)
|
||||
|
||||
processor_factory.return_value.init_index_processor.return_value.clean.assert_not_called()
|
||||
indexing_runner.return_value.run.assert_called_once()
|
||||
sqlite_session.expire_all()
|
||||
assert set(sqlite_session.scalars(select(DocumentSegment.id))) == {segment.id for segment in decoy_segments}
|
||||
@@ -0,0 +1,276 @@
|
||||
from dataclasses import replace
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import override
|
||||
|
||||
import pytest
|
||||
from celery.exceptions import Retry
|
||||
from dify_trace_sdk.events import MessageCompleted, TraceCorrelation, TraceEvent
|
||||
from dify_trace_sdk.testing import RecordingEgress, message_trace
|
||||
from flask import Flask
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from enums import DeploymentEdition
|
||||
from extensions.ext_application_services import build_application_services
|
||||
from extensions.ext_redis import redis_client
|
||||
from services.tracing.adapters import payload_key
|
||||
from services.tracing.canonical import CanonicalTraceBuilder
|
||||
from services.tracing.delivery import (
|
||||
ExportJob,
|
||||
ExportRetryQueue,
|
||||
StoredTraceDestination,
|
||||
TraceCollector,
|
||||
TraceDeliveryService,
|
||||
TraceDestinationQuery,
|
||||
TraceRecordSource,
|
||||
)
|
||||
from services.tracing.errors import SourceNotFoundError, SourceNotReadyError
|
||||
from services.tracing.providers import ProviderConfigCodec
|
||||
from services.tracing.records import BaseTraceInfo, GenerateNameTraceInfo, WorkflowTraceInfo
|
||||
from tasks import trace_delivery_task
|
||||
from tasks.trace_delivery_task import deliver_trace_event, retry_trace_export, sweep_trace_payloads
|
||||
from tests.helpers.unit.services.tracing_ports import InstalledTraceProviders, WorkspaceBoundCipher
|
||||
|
||||
|
||||
class _Storage:
|
||||
def __init__(self, files: dict[str, bytes]) -> None:
|
||||
self.files = dict(files)
|
||||
|
||||
def load(self, key: str) -> bytes:
|
||||
return self.files[key]
|
||||
|
||||
def save(self, key: str, data: bytes) -> None:
|
||||
self.files[key] = data
|
||||
|
||||
def delete(self, key: str) -> None:
|
||||
self.files.pop(key, None)
|
||||
|
||||
def scan(self, path: str, files: bool = True) -> list[str]:
|
||||
keys: list[str] = [key for key in self.files if key.startswith(path)] if files else []
|
||||
if not keys:
|
||||
raise FileNotFoundError(path)
|
||||
return keys
|
||||
|
||||
|
||||
class _Records(TraceRecordSource):
|
||||
"""Answers every assembly with the scripted record or error after recording the event."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.result: BaseTraceInfo | Exception = SourceNotFoundError()
|
||||
self.events: list[TraceEvent] = []
|
||||
|
||||
@override
|
||||
def assemble(self, event: TraceEvent) -> BaseTraceInfo:
|
||||
self.events.append(event)
|
||||
if isinstance(self.result, Exception):
|
||||
raise self.result
|
||||
return self.result
|
||||
|
||||
|
||||
class _NoDestinations(TraceDestinationQuery):
|
||||
def __init__(self) -> None:
|
||||
self.lookups: list[tuple[str, str]] = []
|
||||
|
||||
@override
|
||||
def active_destination(self, *, tenant_id: str, app_id: str) -> StoredTraceDestination | None:
|
||||
self.lookups.append((tenant_id, app_id))
|
||||
return None
|
||||
|
||||
|
||||
class _NoRetries(ExportRetryQueue):
|
||||
@override
|
||||
def schedule(self, job: ExportJob, *, delay_seconds: int) -> None:
|
||||
raise AssertionError("no send is attempted")
|
||||
|
||||
|
||||
class _UnavailableCollector(TraceCollector):
|
||||
@override
|
||||
def capture_terminal(self, *, tenant_id: str, info: BaseTraceInfo) -> bool:
|
||||
raise RuntimeError("ledger unavailable")
|
||||
|
||||
@override
|
||||
def link_managed_parent(self, *, tenant_id: str, info: WorkflowTraceInfo) -> WorkflowTraceInfo:
|
||||
return info
|
||||
|
||||
|
||||
class _Delivery:
|
||||
def __init__(self) -> None:
|
||||
self.records = _Records()
|
||||
self.destinations = _NoDestinations()
|
||||
|
||||
def service(self, collector: TraceCollector | None) -> TraceDeliveryService:
|
||||
return TraceDeliveryService(
|
||||
records=self.records,
|
||||
destinations=self.destinations,
|
||||
providers=InstalledTraceProviders(),
|
||||
codec=ProviderConfigCodec(WorkspaceBoundCipher()),
|
||||
builder=CanonicalTraceBuilder(),
|
||||
egress=RecordingEgress([], grpc_target=None),
|
||||
retries=_NoRetries(),
|
||||
enterprise_sink=None,
|
||||
collector=collector,
|
||||
max_export_attempts=3,
|
||||
)
|
||||
|
||||
|
||||
class _RetryRequests:
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[dict[str, object]] = []
|
||||
|
||||
def __call__(self, **kwargs: object) -> Retry:
|
||||
self.calls.append(kwargs)
|
||||
raise Retry("redelivery scheduled", None)
|
||||
|
||||
|
||||
def _event() -> TraceEvent:
|
||||
return TraceEvent(
|
||||
event_id="event-1",
|
||||
tenant_id="tenant-1",
|
||||
app_id="app-1",
|
||||
user_id=None,
|
||||
occurred_at=datetime.now(UTC),
|
||||
correlation=TraceCorrelation(
|
||||
external_trace_id=None, session_id=None, parent_workflow_run_id=None, parent_node_execution_id=None
|
||||
),
|
||||
payload=MessageCompleted(message_id="message-1", conversation_id=None),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def storage(monkeypatch: pytest.MonkeyPatch) -> _Storage:
|
||||
fake = _Storage({})
|
||||
monkeypatch.setattr(trace_delivery_task, "storage", fake)
|
||||
return fake
|
||||
|
||||
|
||||
def _install(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
app: Flask,
|
||||
sessions: sessionmaker[Session],
|
||||
delivery: _Delivery,
|
||||
collector: TraceCollector | None,
|
||||
) -> None:
|
||||
services = build_application_services(
|
||||
database_client=sessions,
|
||||
deployment_edition=DeploymentEdition.COMMUNITY,
|
||||
initialization_password="",
|
||||
redis=redis_client,
|
||||
)
|
||||
tracing = replace(services.tracing, delivery=delivery.service(collector))
|
||||
monkeypatch.setitem(app.extensions, "application_services", replace(services, tracing=tracing))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def delivery(monkeypatch: pytest.MonkeyPatch, app: Flask, sqlite_session_factory: sessionmaker[Session]) -> _Delivery:
|
||||
delivery = _Delivery()
|
||||
_install(monkeypatch, app, sqlite_session_factory, delivery, None)
|
||||
return delivery
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("storage")
|
||||
def test_inline_events_are_delivered(delivery: _Delivery) -> None:
|
||||
deliver_trace_event.run({"event": _event().model_dump_json()})
|
||||
|
||||
assert [event.event_id for event in delivery.records.events] == ["event-1"]
|
||||
|
||||
|
||||
def test_stored_events_are_loaded_and_their_payload_removed(delivery: _Delivery, storage: _Storage) -> None:
|
||||
key = payload_key(datetime.now(UTC), "events", "event-1")
|
||||
storage.save(key, _event().model_dump_json().encode())
|
||||
|
||||
deliver_trace_event.run({"payload_key": key})
|
||||
|
||||
assert len(delivery.records.events) == 1
|
||||
assert key not in storage.files
|
||||
|
||||
|
||||
def test_unreadable_events_are_dropped(delivery: _Delivery, storage: _Storage) -> None:
|
||||
storage.save("trace_payloads/2026-01-01/events/bad.json", b"not json")
|
||||
|
||||
deliver_trace_event.run({"payload_key": "trace_payloads/2026-01-01/events/bad.json"})
|
||||
|
||||
assert delivery.records.events == []
|
||||
assert storage.files == {}
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("storage")
|
||||
def test_missing_sources_are_dropped(delivery: _Delivery) -> None:
|
||||
deliver_trace_event.run({"event": _event().model_dump_json()})
|
||||
|
||||
assert len(delivery.records.events) == 1
|
||||
|
||||
|
||||
def test_sources_not_ready_are_redelivered_and_keep_their_payload(
|
||||
monkeypatch: pytest.MonkeyPatch, delivery: _Delivery, storage: _Storage
|
||||
) -> None:
|
||||
delivery.records.result = SourceNotReadyError()
|
||||
retries = _RetryRequests()
|
||||
monkeypatch.setattr(deliver_trace_event, "retry", retries)
|
||||
key = payload_key(datetime.now(UTC), "events", "event-1")
|
||||
storage.save(key, _event().model_dump_json().encode())
|
||||
|
||||
with pytest.raises(Retry):
|
||||
deliver_trace_event.run({"payload_key": key})
|
||||
|
||||
assert retries.calls == [{"args": ({"payload_key": key},), "countdown": 5}]
|
||||
assert key in storage.files
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("storage")
|
||||
def test_collector_outages_redeliver_without_repeating_enterprise_telemetry(
|
||||
monkeypatch: pytest.MonkeyPatch, app: Flask, sqlite_session_factory: sessionmaker[Session]
|
||||
) -> None:
|
||||
delivery = _Delivery()
|
||||
delivery.records.result = GenerateNameTraceInfo(
|
||||
message_id=None,
|
||||
message_data=None,
|
||||
inputs="hello",
|
||||
outputs="Greeting",
|
||||
start_time=datetime(2026, 1, 1),
|
||||
end_time=datetime(2026, 1, 1),
|
||||
metadata={"trace_event_id": "event-1"},
|
||||
conversation_id=None,
|
||||
tenant_id="tenant-1",
|
||||
)
|
||||
_install(monkeypatch, app, sqlite_session_factory, delivery, _UnavailableCollector())
|
||||
retries = _RetryRequests()
|
||||
monkeypatch.setattr(deliver_trace_event, "retry", retries)
|
||||
envelope = {"event": _event().model_dump_json()}
|
||||
|
||||
with pytest.raises(Retry):
|
||||
deliver_trace_event.run(envelope)
|
||||
|
||||
assert retries.calls[0]["args"] == ({**envelope, "enterprise_delivered": True},)
|
||||
|
||||
|
||||
def test_export_retries_run_the_stored_job_once(delivery: _Delivery, storage: _Storage) -> None:
|
||||
job = ExportJob(tenant_id="tenant-1", app_id="app-1", provider_id="langfuse", attempt=2, trace=message_trace())
|
||||
key = payload_key(datetime.now(UTC), "exports", "job-1")
|
||||
storage.save(key, job.model_dump_json().encode())
|
||||
|
||||
retry_trace_export.run(key)
|
||||
|
||||
assert delivery.destinations.lookups == [("tenant-1", "app-1")]
|
||||
assert key not in storage.files
|
||||
|
||||
|
||||
def test_unreadable_export_jobs_are_dropped(delivery: _Delivery, storage: _Storage) -> None:
|
||||
storage.save("trace_payloads/2026-01-01/exports/bad.json", b"{}")
|
||||
|
||||
retry_trace_export.run("trace_payloads/2026-01-01/exports/bad.json")
|
||||
|
||||
assert delivery.destinations.lookups == []
|
||||
assert storage.files == {}
|
||||
|
||||
|
||||
def test_sweep_preserves_legacy_payloads_while_old_workers_drain(storage: _Storage) -> None:
|
||||
today = datetime.now(UTC)
|
||||
fresh = payload_key(today, "events", "fresh")
|
||||
expired = payload_key(today - timedelta(days=3), "events", "expired")
|
||||
for key in (fresh, expired, "trace_payloads/not-a-date/x.json", "ops_trace/app-1/legacy.json", "other/keep.json"):
|
||||
storage.save(key, b"{}")
|
||||
|
||||
sweep_trace_payloads.run()
|
||||
|
||||
assert sorted(storage.files) == sorted(
|
||||
[fresh, "trace_payloads/not-a-date/x.json", "ops_trace/app-1/legacy.json", "other/keep.json"]
|
||||
)
|
||||
@@ -0,0 +1,267 @@
|
||||
from collections.abc import Generator, Mapping
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
import tasks.trigger_processing_tasks as trigger_processing_tasks_module
|
||||
from core.plugin.entities.plugin_daemon import CredentialType
|
||||
from models.enums import EndUserType
|
||||
from models.model import EndUser
|
||||
from models.trigger import TriggerSubscription, WorkflowPluginTrigger
|
||||
from models.workflow import Workflow
|
||||
from services.errors.app import QuotaExceededError
|
||||
from tasks.trigger_processing_tasks import dispatch_triggered_workflow
|
||||
from tests.helpers.unit.model_factories import make_end_user, make_workflow
|
||||
|
||||
|
||||
def _workflow(*, app_id: str = "app-123") -> Workflow:
|
||||
return make_workflow(
|
||||
workflow_id="workflow-123",
|
||||
tenant_id="tenant-123",
|
||||
app_id=app_id,
|
||||
version="published",
|
||||
graph={
|
||||
"nodes": [
|
||||
{
|
||||
"id": "node-123",
|
||||
"data": {"type": trigger_processing_tasks_module.TRIGGER_PLUGIN_NODE_TYPE},
|
||||
}
|
||||
],
|
||||
"edges": [],
|
||||
},
|
||||
created_by="user-123",
|
||||
)
|
||||
|
||||
|
||||
def _end_user() -> EndUser:
|
||||
return make_end_user(
|
||||
end_user_id="end-user-123",
|
||||
tenant_id="tenant-123",
|
||||
app_id="app-123",
|
||||
end_user_type=EndUserType.TRIGGER,
|
||||
session_id="trigger-session",
|
||||
)
|
||||
|
||||
|
||||
class _EndUserServiceStub:
|
||||
def __init__(self) -> None:
|
||||
self.result: Mapping[str, EndUser] = {}
|
||||
self.calls: list[tuple[EndUserType, str, list[str], str]] = []
|
||||
|
||||
def create_end_user_batch(
|
||||
self,
|
||||
type: EndUserType,
|
||||
tenant_id: str,
|
||||
app_ids: list[str],
|
||||
user_id: str,
|
||||
) -> Mapping[str, EndUser]:
|
||||
self.calls.append((type, tenant_id, app_ids, user_id))
|
||||
return self.result
|
||||
|
||||
|
||||
class _DispatchMocks(TypedDict, closed=True):
|
||||
get_workflows: MagicMock
|
||||
reserve: MagicMock
|
||||
quota_charge: MagicMock
|
||||
mark_rate_limited: MagicMock
|
||||
invoke_trigger_event: MagicMock
|
||||
invoke_response: MagicMock
|
||||
end_users: _EndUserServiceStub
|
||||
trigger_workflow_async: MagicMock
|
||||
|
||||
|
||||
class TestDispatchTriggeredWorkflow:
|
||||
"""Unit tests covering branch behaviours of ``dispatch_triggered_workflow``.
|
||||
|
||||
The covered branches are:
|
||||
- workflow missing for ``plugin_trigger.app_id`` → log + ``continue``
|
||||
- ``QuotaService.reserve`` raising ``QuotaExceededError`` →
|
||||
``mark_tenant_triggers_rate_limited`` + early ``return``
|
||||
- ``trigger_workflow_async`` succeeds →
|
||||
``quota_charge.commit()`` + ``dispatched_count`` increments
|
||||
"""
|
||||
|
||||
@pytest.fixture
|
||||
def subscription(self) -> TriggerSubscription:
|
||||
subscription = TriggerSubscription(
|
||||
tenant_id="tenant-123",
|
||||
user_id="user-123",
|
||||
name="Test Subscription",
|
||||
endpoint_id="endpoint-123",
|
||||
provider_id="langgenius/test_plugin/test_plugin",
|
||||
parameters={},
|
||||
properties={},
|
||||
credentials={},
|
||||
credential_type=CredentialType.API_KEY,
|
||||
)
|
||||
subscription.id = "subscription-123"
|
||||
return subscription
|
||||
|
||||
@pytest.fixture
|
||||
def plugin_trigger(self) -> WorkflowPluginTrigger:
|
||||
trigger = WorkflowPluginTrigger(
|
||||
app_id="app-123",
|
||||
node_id="node-123",
|
||||
tenant_id="tenant-123",
|
||||
provider_id="langgenius/test_plugin/test_plugin",
|
||||
event_name="test_event",
|
||||
subscription_id="subscription-123",
|
||||
)
|
||||
trigger.id = "plugin-trigger-123"
|
||||
return trigger
|
||||
|
||||
@pytest.fixture
|
||||
def provider_controller(self) -> MagicMock:
|
||||
controller = MagicMock()
|
||||
controller.plugin_unique_identifier = "langgenius/test_plugin:0.0.1"
|
||||
controller.entity.identity.name = "Test Plugin"
|
||||
controller.entity.identity.icon = "icon.svg"
|
||||
controller.entity.identity.icon_dark = "icon_dark.svg"
|
||||
return controller
|
||||
|
||||
@pytest.fixture
|
||||
def dispatch_mocks(
|
||||
self, plugin_trigger: WorkflowPluginTrigger, provider_controller: MagicMock
|
||||
) -> Generator[_DispatchMocks]:
|
||||
"""Patch all external dependencies reached by ``dispatch_triggered_workflow``.
|
||||
|
||||
Defaults are configured so the code flow can reach the final async
|
||||
trigger block (line ~385); each test overrides specific handles
|
||||
(``get_workflows``, ``reserve``, ``end_users``, ...) to
|
||||
drive the path it targets.
|
||||
"""
|
||||
invoke_response = MagicMock()
|
||||
invoke_response.cancelled = False
|
||||
invoke_response.variables = dict[str, object]()
|
||||
|
||||
quota_charge = MagicMock()
|
||||
end_users = _EndUserServiceStub()
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
trigger_processing_tasks_module.TriggerHttpRequestCachingService,
|
||||
"get_request",
|
||||
return_value=MagicMock(),
|
||||
),
|
||||
patch.object(
|
||||
trigger_processing_tasks_module.TriggerHttpRequestCachingService,
|
||||
"get_payload",
|
||||
return_value=MagicMock(),
|
||||
),
|
||||
patch.object(
|
||||
trigger_processing_tasks_module.TriggerSubscriptionOperatorService,
|
||||
"get_subscriber_triggers",
|
||||
return_value=[plugin_trigger],
|
||||
),
|
||||
patch.object(
|
||||
trigger_processing_tasks_module.TriggerManager,
|
||||
"get_trigger_provider",
|
||||
return_value=provider_controller,
|
||||
),
|
||||
patch.object(
|
||||
trigger_processing_tasks_module.TriggerManager,
|
||||
"invoke_trigger_event",
|
||||
return_value=invoke_response,
|
||||
) as invoke_trigger_event,
|
||||
patch.object(
|
||||
trigger_processing_tasks_module.TriggerEventNodeData,
|
||||
"model_validate",
|
||||
return_value=MagicMock(),
|
||||
),
|
||||
patch.object(
|
||||
trigger_processing_tasks_module,
|
||||
"_get_published_workflows_by_app_ids",
|
||||
) as get_workflows,
|
||||
patch.object(
|
||||
trigger_processing_tasks_module.QuotaService,
|
||||
"reserve",
|
||||
return_value=quota_charge,
|
||||
) as reserve,
|
||||
patch.object(
|
||||
trigger_processing_tasks_module.AppTriggerService,
|
||||
"mark_tenant_triggers_rate_limited",
|
||||
) as mark_rate_limited,
|
||||
patch.object(
|
||||
trigger_processing_tasks_module.AsyncWorkflowService,
|
||||
"trigger_workflow_async",
|
||||
) as trigger_workflow_async,
|
||||
):
|
||||
yield {
|
||||
"get_workflows": get_workflows,
|
||||
"reserve": reserve,
|
||||
"quota_charge": quota_charge,
|
||||
"mark_rate_limited": mark_rate_limited,
|
||||
"invoke_trigger_event": invoke_trigger_event,
|
||||
"invoke_response": invoke_response,
|
||||
"end_users": end_users,
|
||||
"trigger_workflow_async": trigger_workflow_async,
|
||||
}
|
||||
|
||||
def test_dispatch_skips_when_workflow_missing(
|
||||
self, subscription: TriggerSubscription, dispatch_mocks: _DispatchMocks
|
||||
) -> None:
|
||||
"""Covers missing workflow → log + ``continue``."""
|
||||
dispatch_mocks["get_workflows"].return_value = dict[str, object]()
|
||||
|
||||
dispatched = dispatch_triggered_workflow(
|
||||
user_id="user-123",
|
||||
subscription=subscription,
|
||||
event_name="test_event",
|
||||
request_id="request-123",
|
||||
end_users=dispatch_mocks["end_users"],
|
||||
)
|
||||
|
||||
assert dispatched == 0
|
||||
dispatch_mocks["reserve"].assert_not_called()
|
||||
dispatch_mocks["invoke_trigger_event"].assert_not_called()
|
||||
dispatch_mocks["mark_rate_limited"].assert_not_called()
|
||||
|
||||
def test_dispatch_marks_rate_limited_when_quota_exceeded(
|
||||
self, subscription: TriggerSubscription, plugin_trigger: WorkflowPluginTrigger, dispatch_mocks: _DispatchMocks
|
||||
) -> None:
|
||||
"""Covers QuotaExceededError → mark rate-limited + early return."""
|
||||
workflow = _workflow()
|
||||
dispatch_mocks["get_workflows"].return_value = {plugin_trigger.app_id: workflow}
|
||||
dispatch_mocks["reserve"].side_effect = QuotaExceededError(
|
||||
feature="trigger", tenant_id=subscription.tenant_id, required=1
|
||||
)
|
||||
|
||||
dispatched = dispatch_triggered_workflow(
|
||||
user_id="user-123",
|
||||
subscription=subscription,
|
||||
event_name="test_event",
|
||||
request_id="request-123",
|
||||
end_users=dispatch_mocks["end_users"],
|
||||
)
|
||||
|
||||
assert dispatched == 0
|
||||
dispatch_mocks["reserve"].assert_called_once()
|
||||
dispatch_mocks["mark_rate_limited"].assert_called_once_with(subscription.tenant_id)
|
||||
dispatch_mocks["invoke_trigger_event"].assert_not_called()
|
||||
|
||||
def test_dispatch_commits_quota_and_counts_when_workflow_triggered(
|
||||
self, subscription: TriggerSubscription, plugin_trigger: WorkflowPluginTrigger, dispatch_mocks: _DispatchMocks
|
||||
) -> None:
|
||||
"""Happy path: end user exists and async trigger succeeds."""
|
||||
workflow = _workflow()
|
||||
dispatch_mocks["get_workflows"].return_value = {plugin_trigger.app_id: workflow}
|
||||
|
||||
end_user = _end_user()
|
||||
dispatch_mocks["end_users"].result = {plugin_trigger.app_id: end_user}
|
||||
|
||||
dispatched = dispatch_triggered_workflow(
|
||||
user_id="user-123",
|
||||
subscription=subscription,
|
||||
event_name="test_event",
|
||||
request_id="request-123",
|
||||
end_users=dispatch_mocks["end_users"],
|
||||
)
|
||||
|
||||
assert dispatched == 1
|
||||
dispatch_mocks["trigger_workflow_async"].assert_called_once()
|
||||
_, kwargs = dispatch_mocks["trigger_workflow_async"].call_args
|
||||
assert kwargs["user"] is end_user
|
||||
dispatch_mocks["quota_charge"].commit.assert_called_once()
|
||||
dispatch_mocks["quota_charge"].refund.assert_not_called()
|
||||
dispatch_mocks["mark_rate_limited"].assert_not_called()
|
||||
@@ -0,0 +1,96 @@
|
||||
from datetime import datetime
|
||||
from unittest.mock import patch
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from core.repositories.celery_workflow_execution_repository import CeleryWorkflowExecutionRepository
|
||||
from graphon.enums import WorkflowExecutionStatus, WorkflowType
|
||||
from models import Account, CreatorUserRole, WorkflowRun
|
||||
from models.enums import WorkflowRunTriggeredFrom
|
||||
from repositories.entities.workflow_execution import WorkflowExecution
|
||||
from tasks.workflow_execution_tasks import save_workflow_execution_task
|
||||
|
||||
TABLES = (WorkflowRun,)
|
||||
|
||||
|
||||
def test_legacy_celery_payload_preserves_timestamp_based_duration(
|
||||
sqlite_session_factory: sessionmaker[Session],
|
||||
) -> None:
|
||||
execution_id = str(uuid4())
|
||||
graph: dict[str, object] = {}
|
||||
inputs: dict[str, object] = {}
|
||||
payload = {
|
||||
"id_": execution_id,
|
||||
"workflow_id": str(uuid4()),
|
||||
"workflow_version": "1",
|
||||
"workflow_type": WorkflowType.WORKFLOW.value,
|
||||
"graph": graph,
|
||||
"inputs": inputs,
|
||||
"status": WorkflowExecutionStatus.SUCCEEDED.value,
|
||||
"started_at": "2026-01-01T00:00:00",
|
||||
"finished_at": "2026-01-01T00:00:15",
|
||||
}
|
||||
|
||||
assert (
|
||||
save_workflow_execution_task.run(
|
||||
execution_data=payload,
|
||||
tenant_id=str(uuid4()),
|
||||
app_id=str(uuid4()),
|
||||
triggered_from=WorkflowRunTriggeredFrom.APP_RUN.value,
|
||||
creator_user_id=str(uuid4()),
|
||||
creator_user_role=CreatorUserRole.ACCOUNT.value,
|
||||
)
|
||||
is True
|
||||
)
|
||||
|
||||
with sqlite_session_factory() as session:
|
||||
stored = session.get(WorkflowRun, execution_id)
|
||||
assert stored is not None
|
||||
assert stored.elapsed_time == 15.0
|
||||
|
||||
|
||||
def test_celery_roundtrip_preserves_accumulated_duration_on_create_update_and_redelivery(
|
||||
sqlite_session_factory: sessionmaker[Session],
|
||||
) -> None:
|
||||
account = Account(name="Test", email="test@example.com")
|
||||
account.id = str(uuid4())
|
||||
repository = CeleryWorkflowExecutionRepository(
|
||||
session_factory=sqlite_session_factory,
|
||||
tenant_id=str(uuid4()),
|
||||
user=account,
|
||||
app_id=str(uuid4()),
|
||||
triggered_from=WorkflowRunTriggeredFrom.APP_RUN,
|
||||
)
|
||||
execution = WorkflowExecution(
|
||||
id_=str(uuid4()),
|
||||
workflow_id=str(uuid4()),
|
||||
workflow_version="1",
|
||||
workflow_type=WorkflowType.WORKFLOW,
|
||||
graph={},
|
||||
inputs={},
|
||||
started_at=datetime(2026, 1, 1),
|
||||
accumulated_elapsed_time=5.0,
|
||||
)
|
||||
|
||||
for status, duration in [
|
||||
(WorkflowExecutionStatus.RUNNING, 5.0),
|
||||
(WorkflowExecutionStatus.PAUSED, 7.0),
|
||||
(WorkflowExecutionStatus.SUCCEEDED, 9.0),
|
||||
]:
|
||||
execution.status = status
|
||||
execution.accumulated_elapsed_time = duration
|
||||
if status == WorkflowExecutionStatus.SUCCEEDED:
|
||||
execution.finished_at = datetime(2026, 1, 2)
|
||||
with patch("core.repositories.celery_workflow_execution_repository.save_workflow_execution_task") as task:
|
||||
repository.save(execution)
|
||||
payload = task.delay.call_args.kwargs
|
||||
assert payload["execution_data"]["accumulated_elapsed_time"] == duration
|
||||
assert payload["creator_user_role"] == CreatorUserRole.ACCOUNT
|
||||
for _ in range(2):
|
||||
assert save_workflow_execution_task.run(**payload) is True
|
||||
with sqlite_session_factory() as session:
|
||||
stored = session.get(WorkflowRun, execution.id_)
|
||||
assert stored is not None
|
||||
assert stored.elapsed_time == duration
|
||||
assert stored.created_at == execution.started_at
|
||||
@@ -0,0 +1,490 @@
|
||||
import json
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from decimal import Decimal
|
||||
from typing import Protocol, cast
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepository
|
||||
from graphon.entities import WorkflowNodeExecution
|
||||
from graphon.entities.workflow_node_execution import WorkflowNodeExecutionMetadataKey
|
||||
from graphon.enums import BuiltinNodeTypes, WorkflowExecutionStatus, WorkflowNodeExecutionStatus, WorkflowType
|
||||
from models import Account
|
||||
from models.enums import CreatorUserRole, WorkflowRunTriggeredFrom
|
||||
from models.workflow import WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom, WorkflowRun
|
||||
from models.workflow import WorkflowType as ModelWorkflowType
|
||||
from tasks.workflow_node_execution_tasks import (
|
||||
_create_node_execution_from_domain,
|
||||
_update_node_execution_from_domain,
|
||||
save_workflow_node_execution_task,
|
||||
save_workflow_node_executions_task,
|
||||
)
|
||||
|
||||
|
||||
def test_celery_update_preserves_workflow_agent_binding_identity() -> None:
|
||||
stored = WorkflowNodeExecutionModel(
|
||||
process_data='{"workflow_agent_binding_id": "workflow-binding-1"}',
|
||||
)
|
||||
incoming = WorkflowNodeExecution(
|
||||
id="execution-1",
|
||||
workflow_id="workflow-1",
|
||||
node_id="node-1",
|
||||
node_type="agent",
|
||||
title="Agent",
|
||||
index=1,
|
||||
created_at=datetime.now(UTC),
|
||||
process_data={"retry": True},
|
||||
)
|
||||
|
||||
_update_node_execution_from_domain(stored, incoming)
|
||||
|
||||
assert stored.process_data_dict == {
|
||||
"retry": True,
|
||||
"workflow_agent_binding_id": "workflow-binding-1",
|
||||
}
|
||||
|
||||
|
||||
"""SQLite-backed tests for asynchronous workflow node-execution persistence."""
|
||||
|
||||
|
||||
TENANT_ID = "00000000-0000-0000-0000-000000000010"
|
||||
APP_ID = "00000000-0000-0000-0000-000000000020"
|
||||
WORKFLOW_ID = "00000000-0000-0000-0000-000000000030"
|
||||
WORKFLOW_RUN_ID = "00000000-0000-0000-0000-000000000040"
|
||||
ACCOUNT_ID = "00000000-0000-0000-0000-000000000050"
|
||||
EXECUTION_ID = "00000000-0000-0000-0000-000000000060"
|
||||
|
||||
|
||||
class _ExecutionTaskKwargs(TypedDict, closed=True):
|
||||
tenant_id: str
|
||||
app_id: str
|
||||
triggered_from: str
|
||||
creator_user_id: str
|
||||
creator_user_role: str
|
||||
|
||||
|
||||
class _TaskWithRequestContext(Protocol):
|
||||
def push_request(self, *, retries: int) -> None: ...
|
||||
|
||||
def pop_request(self) -> None: ...
|
||||
|
||||
|
||||
def _execution(
|
||||
*,
|
||||
execution_id: str = EXECUTION_ID,
|
||||
status: WorkflowNodeExecutionStatus = WorkflowNodeExecutionStatus.RUNNING,
|
||||
inputs: dict[str, object] | None = None,
|
||||
process_data: dict[str, object] | None = None,
|
||||
outputs: dict[str, object] | None = None,
|
||||
error: str | None = None,
|
||||
elapsed_time: float = 0,
|
||||
created_at: datetime | None = None,
|
||||
finished_at: datetime | None = None,
|
||||
) -> WorkflowNodeExecution:
|
||||
return WorkflowNodeExecution(
|
||||
id=execution_id,
|
||||
node_execution_id="runtime-node-execution-1",
|
||||
workflow_id=WORKFLOW_ID,
|
||||
workflow_execution_id=WORKFLOW_RUN_ID,
|
||||
index=3,
|
||||
predecessor_node_id="previous-node",
|
||||
node_id="llm-node",
|
||||
node_type=BuiltinNodeTypes.LLM,
|
||||
title="Generate answer",
|
||||
inputs=inputs if inputs is not None else {"question": "hello"},
|
||||
process_data=process_data if process_data is not None else {"attempt": 1},
|
||||
outputs=outputs,
|
||||
status=status,
|
||||
error=error,
|
||||
elapsed_time=elapsed_time,
|
||||
metadata={
|
||||
WorkflowNodeExecutionMetadataKey.TOTAL_TOKENS: 12,
|
||||
WorkflowNodeExecutionMetadataKey.TOTAL_PRICE: Decimal("0.01"),
|
||||
},
|
||||
created_at=created_at or datetime(2026, 8, 12, 1, tzinfo=UTC),
|
||||
finished_at=finished_at,
|
||||
)
|
||||
|
||||
|
||||
def _stored_execution(
|
||||
*,
|
||||
execution_id: str = EXECUTION_ID,
|
||||
tenant_id: str = TENANT_ID,
|
||||
app_id: str = APP_ID,
|
||||
process_data: dict[str, object] | None = None,
|
||||
) -> WorkflowNodeExecutionModel:
|
||||
run = WorkflowNodeExecutionModel(
|
||||
id=execution_id,
|
||||
tenant_id=tenant_id,
|
||||
app_id=app_id,
|
||||
workflow_id=WORKFLOW_ID,
|
||||
triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,
|
||||
workflow_run_id=WORKFLOW_RUN_ID,
|
||||
index=1,
|
||||
predecessor_node_id=None,
|
||||
node_execution_id="old-runtime-id",
|
||||
node_id="llm-node",
|
||||
node_type=BuiltinNodeTypes.LLM,
|
||||
title="Old title",
|
||||
agent_workspace_binding_id="00000000-0000-0000-0000-000000000070",
|
||||
inputs=json.dumps({"old": "input"}),
|
||||
process_data=json.dumps(process_data or {"workflow_agent_binding_id": "workflow-binding-1"}),
|
||||
outputs=json.dumps({"old": "output"}),
|
||||
status=WorkflowNodeExecutionStatus.RUNNING,
|
||||
error=None,
|
||||
elapsed_time=0,
|
||||
execution_metadata="{}",
|
||||
created_by_role=CreatorUserRole.ACCOUNT,
|
||||
created_by=ACCOUNT_ID,
|
||||
finished_at=None,
|
||||
)
|
||||
run.created_at = datetime(2026, 8, 12, 1)
|
||||
return run
|
||||
|
||||
|
||||
def test_create_helper_builds_real_mapped_model_with_serialized_runtime_values() -> None:
|
||||
execution = _execution(outputs={"answer": "world"})
|
||||
|
||||
model = _create_node_execution_from_domain(
|
||||
execution,
|
||||
tenant_id=TENANT_ID,
|
||||
app_id=APP_ID,
|
||||
triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,
|
||||
creator_user_id=ACCOUNT_ID,
|
||||
creator_user_role=CreatorUserRole.ACCOUNT,
|
||||
)
|
||||
|
||||
assert isinstance(model, WorkflowNodeExecutionModel)
|
||||
assert model.id == EXECUTION_ID
|
||||
assert model.tenant_id == TENANT_ID
|
||||
assert model.workflow_run_id == WORKFLOW_RUN_ID
|
||||
assert model.node_type == BuiltinNodeTypes.LLM
|
||||
assert model.inputs_dict == {"question": "hello"}
|
||||
assert model.process_data_dict == {"attempt": 1}
|
||||
assert model.outputs_dict == {"answer": "world"}
|
||||
assert json.loads(model.execution_metadata or "{}") == {"total_tokens": 12, "total_price": 0.01}
|
||||
|
||||
|
||||
def test_update_helper_preserves_binding_identity_and_immutable_ownership() -> None:
|
||||
stored = _stored_execution()
|
||||
incoming = _execution(
|
||||
status=WorkflowNodeExecutionStatus.SUCCEEDED,
|
||||
process_data={"attempt": 2},
|
||||
outputs={"answer": "updated"},
|
||||
elapsed_time=1.25,
|
||||
finished_at=datetime(2026, 8, 12, 1, 0, 2, tzinfo=UTC),
|
||||
)
|
||||
|
||||
_update_node_execution_from_domain(stored, incoming)
|
||||
|
||||
assert stored.tenant_id == TENANT_ID
|
||||
assert stored.app_id == APP_ID
|
||||
assert stored.workflow_id == WORKFLOW_ID
|
||||
assert stored.created_by == ACCOUNT_ID
|
||||
assert stored.node_execution_id == "old-runtime-id"
|
||||
assert stored.process_data_dict == {
|
||||
"attempt": 2,
|
||||
"workflow_agent_binding_id": "workflow-binding-1",
|
||||
}
|
||||
assert stored.outputs_dict == {"answer": "updated"}
|
||||
assert stored.status is WorkflowNodeExecutionStatus.SUCCEEDED
|
||||
assert stored.elapsed_time == 1.25
|
||||
|
||||
|
||||
def test_task_creates_and_commits_node_execution(
|
||||
sqlite_session_factory: sessionmaker[Session],
|
||||
) -> None:
|
||||
execution = _execution(outputs={"answer": "created"})
|
||||
|
||||
result = save_workflow_node_execution_task.run(
|
||||
execution_data=execution.model_dump(),
|
||||
tenant_id=TENANT_ID,
|
||||
app_id=APP_ID,
|
||||
triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN.value,
|
||||
creator_user_id=ACCOUNT_ID,
|
||||
creator_user_role=CreatorUserRole.ACCOUNT.value,
|
||||
)
|
||||
|
||||
assert result is True
|
||||
with sqlite_session_factory() as observer:
|
||||
persisted = observer.get(WorkflowNodeExecutionModel, EXECUTION_ID)
|
||||
assert persisted is not None
|
||||
assert persisted.tenant_id == TENANT_ID
|
||||
assert persisted.app_id == APP_ID
|
||||
assert persisted.outputs_dict == {"answer": "created"}
|
||||
assert persisted.created_by_role is CreatorUserRole.ACCOUNT
|
||||
assert persisted.created_by == ACCOUNT_ID
|
||||
|
||||
|
||||
def test_task_updates_only_mutable_execution_state(
|
||||
sqlite_session_factory: sessionmaker[Session],
|
||||
) -> None:
|
||||
with sqlite_session_factory.begin() as seed_session:
|
||||
seed_session.add(_stored_execution())
|
||||
finished_at = datetime(2026, 8, 12, 1, 0, 3, tzinfo=UTC)
|
||||
execution = _execution(
|
||||
status=WorkflowNodeExecutionStatus.FAILED,
|
||||
inputs={"question": "updated"},
|
||||
process_data={"attempt": 3},
|
||||
outputs={"partial": True},
|
||||
error="provider failed",
|
||||
elapsed_time=2.5,
|
||||
created_at=datetime(2026, 8, 12, 2, tzinfo=UTC),
|
||||
finished_at=finished_at,
|
||||
)
|
||||
|
||||
result = save_workflow_node_execution_task.run(
|
||||
execution_data=execution.model_dump(),
|
||||
tenant_id=TENANT_ID,
|
||||
app_id=APP_ID,
|
||||
triggered_from=WorkflowNodeExecutionTriggeredFrom.SINGLE_STEP.value,
|
||||
creator_user_id="different-account",
|
||||
creator_user_role=CreatorUserRole.END_USER.value,
|
||||
)
|
||||
|
||||
assert result is True
|
||||
with sqlite_session_factory() as observer:
|
||||
persisted = observer.get(WorkflowNodeExecutionModel, EXECUTION_ID)
|
||||
assert persisted is not None
|
||||
assert persisted.triggered_from is WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN
|
||||
assert persisted.created_by_role is CreatorUserRole.ACCOUNT
|
||||
assert persisted.created_by == ACCOUNT_ID
|
||||
assert persisted.created_at == datetime(2026, 8, 12, 1)
|
||||
assert persisted.inputs_dict == {"question": "updated"}
|
||||
assert persisted.process_data_dict == {
|
||||
"attempt": 3,
|
||||
"workflow_agent_binding_id": "workflow-binding-1",
|
||||
}
|
||||
assert persisted.outputs_dict == {"partial": True}
|
||||
assert persisted.status is WorkflowNodeExecutionStatus.FAILED
|
||||
assert persisted.error == "provider failed"
|
||||
assert persisted.elapsed_time == 2.5
|
||||
assert persisted.finished_at == finished_at.replace(tzinfo=None)
|
||||
|
||||
|
||||
def test_task_does_not_overwrite_same_id_from_another_tenant(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
sqlite_session_factory: sessionmaker[Session],
|
||||
) -> None:
|
||||
other_tenant_id = "00000000-0000-0000-0000-000000000099"
|
||||
with sqlite_session_factory.begin() as seed_session:
|
||||
seed_session.add(_stored_execution(tenant_id=other_tenant_id))
|
||||
retry = MagicMock(side_effect=RuntimeError("retry requested"))
|
||||
monkeypatch.setattr(save_workflow_node_execution_task, "retry", retry)
|
||||
|
||||
with pytest.raises(RuntimeError, match="retry requested"):
|
||||
save_workflow_node_execution_task.run(
|
||||
execution_data=_execution(outputs={"unsafe": True}).model_dump(),
|
||||
tenant_id=TENANT_ID,
|
||||
app_id=APP_ID,
|
||||
triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN.value,
|
||||
creator_user_id=ACCOUNT_ID,
|
||||
creator_user_role=CreatorUserRole.ACCOUNT.value,
|
||||
)
|
||||
|
||||
retry.assert_called_once()
|
||||
assert retry.call_args.kwargs["countdown"] == 60
|
||||
with sqlite_session_factory() as observer:
|
||||
persisted = observer.get(WorkflowNodeExecutionModel, EXECUTION_ID)
|
||||
assert persisted is not None
|
||||
assert persisted.tenant_id == other_tenant_id
|
||||
assert persisted.outputs_dict == {"old": "output"}
|
||||
|
||||
|
||||
def test_task_retries_invalid_payload_without_committing(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
sqlite_session_factory: sessionmaker[Session],
|
||||
) -> None:
|
||||
retry = MagicMock(side_effect=RuntimeError("retry requested"))
|
||||
monkeypatch.setattr(save_workflow_node_execution_task, "retry", retry)
|
||||
invalid_payload = _execution().model_dump()
|
||||
invalid_payload["node_id"] = None
|
||||
|
||||
with pytest.raises(RuntimeError, match="retry requested"):
|
||||
save_workflow_node_execution_task.run(
|
||||
execution_data=invalid_payload,
|
||||
tenant_id=TENANT_ID,
|
||||
app_id=APP_ID,
|
||||
triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN.value,
|
||||
creator_user_id=ACCOUNT_ID,
|
||||
creator_user_role=CreatorUserRole.ACCOUNT.value,
|
||||
)
|
||||
|
||||
retry.assert_called_once()
|
||||
with sqlite_session_factory() as observer:
|
||||
assert observer.get(WorkflowNodeExecutionModel, EXECUTION_ID) is None
|
||||
|
||||
|
||||
def test_task_uses_exponential_retry_delay_for_redelivery(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
retry = MagicMock(side_effect=RuntimeError("retry requested"))
|
||||
monkeypatch.setattr(save_workflow_node_execution_task, "retry", retry)
|
||||
task = cast(_TaskWithRequestContext, save_workflow_node_execution_task)
|
||||
|
||||
task.push_request(retries=2)
|
||||
try:
|
||||
with pytest.raises(RuntimeError, match="retry requested"):
|
||||
save_workflow_node_execution_task.run(
|
||||
execution_data={"id": EXECUTION_ID},
|
||||
tenant_id=TENANT_ID,
|
||||
app_id=APP_ID,
|
||||
triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN.value,
|
||||
creator_user_id=ACCOUNT_ID,
|
||||
creator_user_role=CreatorUserRole.ACCOUNT.value,
|
||||
)
|
||||
finally:
|
||||
task.pop_request()
|
||||
|
||||
assert retry.call_args.kwargs["countdown"] == 240
|
||||
|
||||
|
||||
def test_task_repeated_delivery_updates_one_row(
|
||||
sqlite_session_factory: sessionmaker[Session],
|
||||
) -> None:
|
||||
created_at = datetime(2026, 8, 12, 1, tzinfo=UTC)
|
||||
first = _execution(created_at=created_at, outputs={"delivery": 1})
|
||||
second = _execution(
|
||||
created_at=created_at + timedelta(minutes=1),
|
||||
status=WorkflowNodeExecutionStatus.SUCCEEDED,
|
||||
outputs={"delivery": 2},
|
||||
finished_at=created_at + timedelta(seconds=5),
|
||||
)
|
||||
task_kwargs: _ExecutionTaskKwargs = {
|
||||
"tenant_id": TENANT_ID,
|
||||
"app_id": APP_ID,
|
||||
"triggered_from": WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN.value,
|
||||
"creator_user_id": ACCOUNT_ID,
|
||||
"creator_user_role": CreatorUserRole.ACCOUNT.value,
|
||||
}
|
||||
|
||||
assert save_workflow_node_execution_task.run(execution_data=first.model_dump(), **task_kwargs)
|
||||
assert save_workflow_node_execution_task.run(execution_data=second.model_dump(), **task_kwargs)
|
||||
|
||||
with sqlite_session_factory() as observer:
|
||||
assert observer.scalar(select(func.count()).select_from(WorkflowNodeExecutionModel)) == 1
|
||||
persisted = observer.get(WorkflowNodeExecutionModel, EXECUTION_ID)
|
||||
assert persisted is not None
|
||||
assert persisted.outputs_dict == {"delivery": 2}
|
||||
assert persisted.created_at == created_at.replace(tzinfo=None)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("writer", ["celery", "celery-batch", "sql", "sql-batch", "sql-data"])
|
||||
@pytest.mark.parametrize("existing", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"status",
|
||||
[WorkflowNodeExecutionStatus.RUNNING, WorkflowNodeExecutionStatus.PAUSED, WorkflowNodeExecutionStatus.RETRY],
|
||||
)
|
||||
def test_delayed_nonterminal_node_write_cannot_revive_or_insert_a_live_node_after_stop(
|
||||
sqlite_session_factory: sessionmaker[Session], writer: str, existing: bool, status: WorkflowNodeExecutionStatus
|
||||
) -> None:
|
||||
finished_at = datetime(2026, 8, 12, 1, 1)
|
||||
with sqlite_session_factory.begin() as session:
|
||||
session.add(
|
||||
WorkflowRun(
|
||||
id=WORKFLOW_RUN_ID,
|
||||
tenant_id=TENANT_ID,
|
||||
app_id=APP_ID,
|
||||
workflow_id=WORKFLOW_ID,
|
||||
type=ModelWorkflowType(WorkflowType.WORKFLOW),
|
||||
triggered_from=WorkflowRunTriggeredFrom.APP_RUN,
|
||||
version="1",
|
||||
status=WorkflowExecutionStatus.STOPPED,
|
||||
error="User requested stop",
|
||||
stop_requested_at=finished_at,
|
||||
finished_at=finished_at,
|
||||
created_by_role=CreatorUserRole.ACCOUNT,
|
||||
created_by=ACCOUNT_ID,
|
||||
)
|
||||
)
|
||||
if existing:
|
||||
node = _stored_execution()
|
||||
node.status = WorkflowNodeExecutionStatus.FAILED
|
||||
node.finished_at = finished_at
|
||||
node.error = "User requested stop"
|
||||
session.add(node)
|
||||
execution = _execution(status=status)
|
||||
if writer.startswith("celery"):
|
||||
kwargs: _ExecutionTaskKwargs = {
|
||||
"tenant_id": TENANT_ID,
|
||||
"app_id": APP_ID,
|
||||
"triggered_from": WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN.value,
|
||||
"creator_user_id": ACCOUNT_ID,
|
||||
"creator_user_role": CreatorUserRole.ACCOUNT.value,
|
||||
}
|
||||
if writer == "celery":
|
||||
assert save_workflow_node_execution_task.run(execution_data=execution.model_dump(), **kwargs)
|
||||
else:
|
||||
assert save_workflow_node_executions_task.run(executions_data=[execution.model_dump()], **kwargs)
|
||||
else:
|
||||
user = Account(name="Test", email="test@example.com")
|
||||
user.id = ACCOUNT_ID
|
||||
repository = SQLAlchemyWorkflowNodeExecutionRepository(
|
||||
session_factory=sqlite_session_factory,
|
||||
tenant_id=TENANT_ID,
|
||||
app_id=APP_ID,
|
||||
user=user,
|
||||
triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,
|
||||
)
|
||||
if writer == "sql":
|
||||
repository.save(execution)
|
||||
elif writer == "sql-batch":
|
||||
repository.save_many([execution])
|
||||
else:
|
||||
repository.save_execution_data(execution)
|
||||
with sqlite_session_factory() as session:
|
||||
stored = session.get(WorkflowNodeExecutionModel, EXECUTION_ID)
|
||||
assert stored is not None
|
||||
assert stored.status == WorkflowNodeExecutionStatus.FAILED
|
||||
assert stored.finished_at == finished_at
|
||||
assert stored.error == "User requested stop"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"status",
|
||||
[WorkflowNodeExecutionStatus.RUNNING, WorkflowNodeExecutionStatus.PAUSED, WorkflowNodeExecutionStatus.RETRY],
|
||||
)
|
||||
def test_delayed_write_preserves_node_cleanup_before_workflow_summary_finishes(
|
||||
sqlite_session_factory: sessionmaker[Session], status: WorkflowNodeExecutionStatus
|
||||
) -> None:
|
||||
with sqlite_session_factory.begin() as session:
|
||||
session.add(
|
||||
WorkflowRun(
|
||||
id=WORKFLOW_RUN_ID,
|
||||
tenant_id=TENANT_ID,
|
||||
app_id=APP_ID,
|
||||
workflow_id=WORKFLOW_ID,
|
||||
type=ModelWorkflowType(WorkflowType.WORKFLOW),
|
||||
triggered_from=WorkflowRunTriggeredFrom.APP_RUN,
|
||||
version="1",
|
||||
status=WorkflowExecutionStatus.RUNNING,
|
||||
created_by_role=CreatorUserRole.ACCOUNT,
|
||||
created_by=ACCOUNT_ID,
|
||||
)
|
||||
)
|
||||
node = _stored_execution()
|
||||
node.status = WorkflowNodeExecutionStatus.FAILED
|
||||
node.finished_at = datetime(2026, 8, 12, 1, 1)
|
||||
node.elapsed_time = 60
|
||||
node.error = "Execution aborted"
|
||||
session.add(node)
|
||||
session.flush()
|
||||
original = dict(session.execute(select(WorkflowNodeExecutionModel.__table__)).mappings().one())
|
||||
|
||||
assert save_workflow_node_execution_task.run(
|
||||
execution_data=_execution(status=status).model_dump(),
|
||||
tenant_id=TENANT_ID,
|
||||
app_id=APP_ID,
|
||||
triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN.value,
|
||||
creator_user_id=ACCOUNT_ID,
|
||||
creator_user_role=CreatorUserRole.ACCOUNT.value,
|
||||
)
|
||||
|
||||
with sqlite_session_factory() as session:
|
||||
assert dict(session.execute(select(WorkflowNodeExecutionModel.__table__)).mappings().one()) == original
|
||||
run = session.get(WorkflowRun, WORKFLOW_RUN_ID)
|
||||
assert run is not None
|
||||
assert run.status == WorkflowExecutionStatus.RUNNING
|
||||
assert run.stop_requested_at is None
|
||||
Reference in new issue
Block a user