chore: 纳入本地 Dify 1.19.0 源码及品牌与免登改动
演示目录只保留可重建的源码。运行密钥、数据卷、依赖和带口令的本地配置不进入仓库。 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
60158935f4
commit
6519682885
15831 files changed
+3063428
No files matched your search
@@ -0,0 +1,657 @@
|
||||
import json
|
||||
from collections.abc import Callable, Generator, Iterator
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
from redis.exceptions import LockNotOwnedError
|
||||
from sqlalchemy import Engine, event, select
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from controllers.console.datasets.datasets_document import DocumentDetailResponse, DocumentWithSegmentsListResponse
|
||||
from core.model_manager import ModelInstance
|
||||
from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType
|
||||
from enums import CloudPlan
|
||||
from machinery.context import RequestContext
|
||||
from models.account import Account, Tenant, TenantAccountJoin, TenantAccountRole
|
||||
from models.dataset import Dataset, DatasetProcessRule, Document, DocumentPipelineExecutionLog, DocumentSegment
|
||||
from models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus, ProcessRuleMode, SegmentStatus
|
||||
from services.document_indexing_proxy.batch_indexing_base import BatchDocumentIndexingProxy
|
||||
from services.document_indexing_proxy.document_indexing_task_proxy import DocumentIndexingTaskProxy
|
||||
from services.document_indexing_proxy.duplicate_document_indexing_task_proxy import DuplicateDocumentIndexingTaskProxy
|
||||
from services.entities.feature_entities import FeatureModel
|
||||
from services.errors.file import FileNotExistsError
|
||||
from services.knowledge.documents.adapters import SQLAlchemyDocumentOperations
|
||||
from services.knowledge.documents.application import DocumentListFilter, DocumentNotFoundError
|
||||
from services.knowledge.resource_scope import DatasetRef
|
||||
from tests.helpers.unit.model_factories import make_upload_file
|
||||
|
||||
_DOCUMENT_ADAPTERS_REF = DatasetRef("tenant-1", "dataset-1")
|
||||
|
||||
|
||||
def _dataset(**values: object) -> Dataset:
|
||||
row = Dataset(
|
||||
id="dataset-1",
|
||||
tenant_id="tenant-1",
|
||||
name="Dataset",
|
||||
created_by="actor-1",
|
||||
maintainer="actor-1",
|
||||
indexing_technique="economy",
|
||||
chunk_structure="text_model",
|
||||
)
|
||||
for key, value in values.items():
|
||||
setattr(row, key, value)
|
||||
return row
|
||||
|
||||
|
||||
def _document(**values: object) -> Document:
|
||||
row = Document(
|
||||
tenant_id="tenant-1",
|
||||
dataset_id="dataset-1",
|
||||
position=1,
|
||||
data_source_type=DataSourceType("local_file"),
|
||||
batch="batch-1",
|
||||
name="Document",
|
||||
created_from=DocumentCreatedFrom("web"),
|
||||
created_by="actor-1",
|
||||
)
|
||||
row.id = "document-1"
|
||||
row.data_source_info = json.dumps({"file_path": "/path/to/file"})
|
||||
row.indexing_status = IndexingStatus.COMPLETED
|
||||
row.doc_form = IndexStructureType.PARAGRAPH_INDEX
|
||||
row.enabled = True
|
||||
row.archived = False
|
||||
row.word_count = 10
|
||||
row.created_at = datetime(2024, 1, 1)
|
||||
for key, value in values.items():
|
||||
setattr(row, key, value)
|
||||
return row
|
||||
|
||||
|
||||
def _segment(
|
||||
segment_id: str,
|
||||
*,
|
||||
tenant_id: str = "tenant-1",
|
||||
dataset_id: str = "dataset-1",
|
||||
document_id: str = "document-1",
|
||||
status: SegmentStatus = SegmentStatus.COMPLETED,
|
||||
completed_at: datetime | None = datetime(2024, 1, 1),
|
||||
hit_count: int = 0,
|
||||
) -> DocumentSegment:
|
||||
segment = DocumentSegment(
|
||||
tenant_id=tenant_id,
|
||||
dataset_id=dataset_id,
|
||||
document_id=document_id,
|
||||
position=1,
|
||||
content="content",
|
||||
word_count=7,
|
||||
tokens=2,
|
||||
created_by="actor-1",
|
||||
status=status,
|
||||
completed_at=completed_at,
|
||||
hit_count=hit_count,
|
||||
)
|
||||
segment.id = segment_id
|
||||
return segment
|
||||
|
||||
|
||||
class TestDocumentAdaptersCases:
|
||||
@staticmethod
|
||||
@pytest.fixture
|
||||
def operations(sqlite_session_factory: sessionmaker[Session]) -> SQLAlchemyDocumentOperations:
|
||||
with sqlite_session_factory.begin() as session:
|
||||
session.add_all([_dataset(), _document()])
|
||||
return SQLAlchemyDocumentOperations(session_factory=sqlite_session_factory)
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.parametrize("foreign", [{"tenant_id": "other"}, {"dataset_id": "other"}])
|
||||
def test_document_reads_and_writes_reject_incorrect_owner(
|
||||
operations: SQLAlchemyDocumentOperations, sqlite_session_factory: sessionmaker[Session], foreign: dict[str, str]
|
||||
) -> None:
|
||||
with sqlite_session_factory.begin() as session:
|
||||
session.add(_document(id="foreign", **foreign))
|
||||
ref = _DOCUMENT_ADAPTERS_REF.document("foreign")
|
||||
assert operations.get_state(ref) is None
|
||||
with pytest.raises(DocumentNotFoundError):
|
||||
operations.get_detail(ref, metadata_only=False)
|
||||
with pytest.raises(DocumentNotFoundError):
|
||||
operations.update_document(ref, {"name": "changed"})
|
||||
with pytest.raises(DocumentNotFoundError):
|
||||
operations.rename_document(ref, "changed")
|
||||
with sqlite_session_factory() as session:
|
||||
document = session.get(Document, "foreign")
|
||||
assert document is not None
|
||||
assert document.name == "Document"
|
||||
|
||||
@staticmethod
|
||||
def test_document_lookup_uses_active_workspace(operations: SQLAlchemyDocumentOperations) -> None:
|
||||
assert operations.find_document(workspace_id="other", document_id="document-1") is None
|
||||
assert operations.find_document(
|
||||
workspace_id="tenant-1", document_id="document-1"
|
||||
) == _DOCUMENT_ADAPTERS_REF.document("document-1")
|
||||
|
||||
@staticmethod
|
||||
def test_listing_filters_scope_and_materializes_values(
|
||||
operations: SQLAlchemyDocumentOperations, sqlite_session_factory: sessionmaker[Session]
|
||||
) -> None:
|
||||
with sqlite_session_factory.begin() as session:
|
||||
session.add_all(
|
||||
[
|
||||
_document(id="matching", name="Needle", position=2),
|
||||
_document(id="disabled", name="Needle", enabled=False),
|
||||
_document(id="foreign", name="Needle", tenant_id="other"),
|
||||
_document(id="other-dataset", name="Needle", dataset_id="other"),
|
||||
]
|
||||
)
|
||||
result = operations.list_documents(
|
||||
_DOCUMENT_ADAPTERS_REF, DocumentListFilter(search="Needle", status="available", fetch=True)
|
||||
)
|
||||
response = DocumentWithSegmentsListResponse.model_validate(result).model_dump(mode="json")
|
||||
assert response["total"] == 1
|
||||
assert response["data"][0]["id"] == "matching"
|
||||
assert response["data"][0]["data_source_info"] == {"file_path": "/path/to/file"}
|
||||
assert response["data"][0]["completed_segments"] == 0
|
||||
assert response["data"][0]["total_segments"] == 0
|
||||
|
||||
@staticmethod
|
||||
def test_listing_hit_count_sort_and_exact_last_page(
|
||||
operations: SQLAlchemyDocumentOperations, sqlite_session_factory: sessionmaker[Session]
|
||||
) -> None:
|
||||
with sqlite_session_factory.begin() as session:
|
||||
session.add_all(
|
||||
[
|
||||
_document(id="popular", position=2),
|
||||
_segment("segment-1", hit_count=1),
|
||||
_segment("segment-2", document_id="popular", hit_count=10),
|
||||
_segment("foreign", tenant_id="other", hit_count=1000),
|
||||
]
|
||||
)
|
||||
first = operations.list_documents(_DOCUMENT_ADAPTERS_REF, DocumentListFilter(sort="-hit_count", limit=1))
|
||||
last = operations.list_documents(_DOCUMENT_ADAPTERS_REF, DocumentListFilter(sort="-hit_count", limit=1, page=2))
|
||||
assert [row["id"] for row in first["data"]] == ["popular"]
|
||||
assert first["has_more"] is True
|
||||
assert last["has_more"] is False
|
||||
assert last["total"] == 2
|
||||
|
||||
@staticmethod
|
||||
def test_status_counts_use_one_aggregate_and_complete_owner_chain(
|
||||
operations: SQLAlchemyDocumentOperations, sqlite_session_factory: sessionmaker[Session], sqlite_engine: Engine
|
||||
) -> None:
|
||||
with sqlite_session_factory.begin() as session:
|
||||
session.add_all(
|
||||
[
|
||||
_segment("complete"),
|
||||
_segment("waiting", completed_at=None),
|
||||
_segment("excluded", status=SegmentStatus.RE_SEGMENT),
|
||||
_segment("foreign", tenant_id="other"),
|
||||
_segment("wrong-dataset", dataset_id="other"),
|
||||
_document(id="second", is_paused=True, indexing_status=IndexingStatus.INDEXING),
|
||||
]
|
||||
)
|
||||
statements: list[str] = []
|
||||
|
||||
def record(
|
||||
_conn: object, _cursor: object, statement: str, _parameters: object, _context: object, _many: bool
|
||||
) -> None:
|
||||
statements.append(statement)
|
||||
|
||||
event.listen(sqlite_engine, "before_cursor_execute", record)
|
||||
try:
|
||||
result = operations.get_batch_indexing_status(_DOCUMENT_ADAPTERS_REF, "batch-1")
|
||||
finally:
|
||||
event.remove(sqlite_engine, "before_cursor_execute", record)
|
||||
rows = {row["id"]: row for row in result["data"]}
|
||||
assert rows["document-1"]["completed_segments"] == 1
|
||||
assert rows["document-1"]["total_segments"] == 2
|
||||
assert rows["second"]["indexing_status"] == "paused"
|
||||
assert sum("FROM document_segments" in statement for statement in statements) == 1
|
||||
|
||||
@staticmethod
|
||||
def test_unknown_batch_is_not_found(operations: SQLAlchemyDocumentOperations) -> None:
|
||||
with pytest.raises(DocumentNotFoundError):
|
||||
operations.get_batch_indexing_status(_DOCUMENT_ADAPTERS_REF, "missing")
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.parametrize("rules", [None, {"segmentation": {"delimiter": "\\n", "max_tokens": 500}}])
|
||||
def test_process_rule_preserves_null_and_legacy_delimiter(
|
||||
operations: SQLAlchemyDocumentOperations,
|
||||
sqlite_session_factory: sessionmaker[Session],
|
||||
rules: dict[str, object] | None,
|
||||
) -> None:
|
||||
with sqlite_session_factory.begin() as session:
|
||||
session.add(
|
||||
DatasetProcessRule(
|
||||
dataset_id="dataset-1",
|
||||
mode=ProcessRuleMode.CUSTOM,
|
||||
rules=json.dumps(rules) if rules else None,
|
||||
created_by="actor-1",
|
||||
)
|
||||
)
|
||||
result = operations.get_process_rule(_DOCUMENT_ADAPTERS_REF)
|
||||
assert result["mode"] == "custom"
|
||||
assert result["rules"] == rules
|
||||
|
||||
@staticmethod
|
||||
def test_detail_is_serializable_after_session_closes(operations: SQLAlchemyDocumentOperations) -> None:
|
||||
result = operations.get_detail(_DOCUMENT_ADAPTERS_REF.document("document-1"), metadata_only=False)
|
||||
response = DocumentDetailResponse.model_validate(result).model_dump(mode="json")
|
||||
assert response["id"] == "document-1"
|
||||
assert response["data_source_info"] == {"file_path": "/path/to/file"}
|
||||
assert response["data_source_detail_dict"] == {}
|
||||
assert response["created_at"] == int(datetime(2024, 1, 1).timestamp())
|
||||
|
||||
@staticmethod
|
||||
def test_metadata_only_avoids_loading_full_details(operations: SQLAlchemyDocumentOperations) -> None:
|
||||
with patch("services.knowledge.documents.adapters.get_document_source_detail") as source:
|
||||
result = operations.get_detail(_DOCUMENT_ADAPTERS_REF.document("document-1"), metadata_only=True)
|
||||
assert set(result) == {"id", "doc_type", "doc_metadata"}
|
||||
source.assert_not_called()
|
||||
|
||||
@staticmethod
|
||||
def test_update_is_committed_before_returning(
|
||||
operations: SQLAlchemyDocumentOperations, sqlite_session_factory: sessionmaker[Session]
|
||||
) -> None:
|
||||
operations.update_document(
|
||||
_DOCUMENT_ADAPTERS_REF.document("document-1"), {"is_paused": True, "paused_by": "actor-1"}
|
||||
)
|
||||
with sqlite_session_factory() as session:
|
||||
document = session.get(Document, "document-1")
|
||||
assert document is not None
|
||||
assert document.is_paused is True
|
||||
assert document.paused_by == "actor-1"
|
||||
|
||||
@staticmethod
|
||||
def test_latest_pipeline_log_and_empty_result(
|
||||
operations: SQLAlchemyDocumentOperations, sqlite_session_factory: sessionmaker[Session]
|
||||
) -> None:
|
||||
assert operations.get_execution_log(_DOCUMENT_ADAPTERS_REF.document("document-1")) == {}
|
||||
with sqlite_session_factory.begin() as session:
|
||||
for year in [2024, 2025]:
|
||||
row = DocumentPipelineExecutionLog(
|
||||
pipeline_id="pipeline-1",
|
||||
document_id="document-1",
|
||||
datasource_type="local_file",
|
||||
datasource_info=json.dumps({"year": year}),
|
||||
datasource_node_id="node-1",
|
||||
input_data={"year": year},
|
||||
created_by="actor-1",
|
||||
)
|
||||
row.created_at = datetime(year, 1, 1)
|
||||
session.add(row)
|
||||
result = operations.get_execution_log(_DOCUMENT_ADAPTERS_REF.document("document-1"))
|
||||
assert result["datasource_info"] == {"year": 2025}
|
||||
assert result["input_data"] == {"year": 2025}
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.parametrize("settings", [None, {"enable": False}])
|
||||
def test_summary_guard_rejects_disabled_summary(
|
||||
operations: SQLAlchemyDocumentOperations,
|
||||
sqlite_session_factory: sessionmaker[Session],
|
||||
settings: dict[str, object] | None,
|
||||
) -> None:
|
||||
with sqlite_session_factory.begin() as session:
|
||||
dataset = session.get(Dataset, "dataset-1")
|
||||
assert dataset is not None
|
||||
dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY
|
||||
dataset.summary_index_setting = settings
|
||||
with pytest.raises(ValueError, match="not enabled"):
|
||||
operations.require_summary_enabled(_DOCUMENT_ADAPTERS_REF)
|
||||
|
||||
@staticmethod
|
||||
def test_enable_summary_only_updates_requested_owner(
|
||||
operations: SQLAlchemyDocumentOperations, sqlite_session_factory: sessionmaker[Session]
|
||||
) -> None:
|
||||
with sqlite_session_factory.begin() as session:
|
||||
session.add(_document(id="foreign", tenant_id="other", need_summary=False))
|
||||
operations.enable_summary(_DOCUMENT_ADAPTERS_REF, ["document-1", "foreign"])
|
||||
with sqlite_session_factory() as session:
|
||||
document = session.get(Document, "document-1")
|
||||
foreign = session.get(Document, "foreign")
|
||||
assert document is not None
|
||||
assert foreign is not None
|
||||
assert document.need_summary is True
|
||||
assert foreign.need_summary is False
|
||||
|
||||
|
||||
"""Exercise document creation with real SQL and assertions at external boundaries."""
|
||||
|
||||
|
||||
CONTEXT = RequestContext("request", None, "actor", "tenant")
|
||||
_DOCUMENT_CREATION_REF = DatasetRef("tenant", "dataset")
|
||||
|
||||
|
||||
def settings(**values: object) -> dict[str, object]:
|
||||
return {
|
||||
"indexing_technique": "economy",
|
||||
"process_rule": {"mode": "automatic"},
|
||||
"data_source": {"info_list": {"data_source_type": "upload_file", "file_info_list": {"file_ids": ["file"]}}},
|
||||
**values,
|
||||
}
|
||||
|
||||
|
||||
class TestDocumentCreationCases:
|
||||
@staticmethod
|
||||
@pytest.fixture
|
||||
def operations(
|
||||
sqlite_session_factory: sessionmaker[Session], config_overrides: Callable[..., None]
|
||||
) -> SQLAlchemyDocumentOperations:
|
||||
config_overrides(DEPLOYMENT_EDITION="CLOUD", RBAC_ENABLED=False)
|
||||
with sqlite_session_factory.begin() as session:
|
||||
actor = Account(name="Actor", email="actor@example.com")
|
||||
actor.id = "actor"
|
||||
tenant = Tenant(name="Tenant")
|
||||
tenant.id = "tenant"
|
||||
session.add_all(
|
||||
[
|
||||
actor,
|
||||
tenant,
|
||||
TenantAccountJoin(tenant_id="tenant", account_id="actor", role=TenantAccountRole.OWNER),
|
||||
Dataset(
|
||||
id="dataset",
|
||||
tenant_id="tenant",
|
||||
name="Dataset",
|
||||
created_by="actor",
|
||||
indexing_technique="economy",
|
||||
),
|
||||
make_upload_file(file_id="file", tenant_id="tenant", name="example.txt", created_by="actor"),
|
||||
make_upload_file(file_id="foreign-file", tenant_id="foreign", name="foreign.txt"),
|
||||
]
|
||||
)
|
||||
return SQLAlchemyDocumentOperations(session_factory=sqlite_session_factory)
|
||||
|
||||
@staticmethod
|
||||
@pytest.fixture
|
||||
def outside_transaction(sqlite_engine: Engine) -> Iterator[Callable[[], None]]:
|
||||
active: set[int] = set()
|
||||
|
||||
def begin(connection: object) -> None:
|
||||
active.add(id(connection))
|
||||
|
||||
def end(connection: object) -> None:
|
||||
active.discard(id(connection))
|
||||
|
||||
def check() -> None:
|
||||
assert not active, "External I/O must not hold an open database transaction"
|
||||
|
||||
listeners = [("begin", begin), ("commit", end), ("rollback", end)]
|
||||
for name, callback in listeners:
|
||||
event.listen(sqlite_engine, name, callback)
|
||||
yield check
|
||||
for name, callback in listeners:
|
||||
event.remove(sqlite_engine, name, callback)
|
||||
|
||||
@staticmethod
|
||||
@pytest.fixture
|
||||
def external_calls(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
outside_transaction: Callable[[], None],
|
||||
sqlite_session_factory: sessionmaker[Session],
|
||||
) -> list[tuple[str, tuple[str, ...]]]:
|
||||
calls: list[tuple[str, tuple[str, ...]]] = []
|
||||
|
||||
def features(_tenant_id: str, *, exclude_vector_space: bool) -> FeatureModel:
|
||||
outside_transaction()
|
||||
assert exclude_vector_space is True
|
||||
result = FeatureModel()
|
||||
result.billing.subscription.plan = CloudPlan.PROFESSIONAL
|
||||
return result
|
||||
|
||||
@contextmanager
|
||||
def lock(_name: str, *, timeout: int) -> Generator[None]:
|
||||
outside_transaction()
|
||||
assert timeout == 600
|
||||
yield
|
||||
outside_transaction()
|
||||
|
||||
def dispatched(kind: str, dataset_id: str, document_ids: tuple[str, ...]) -> None:
|
||||
outside_transaction()
|
||||
with sqlite_session_factory() as session:
|
||||
for document_id in document_ids:
|
||||
document = session.get(Document, document_id)
|
||||
assert document is not None
|
||||
assert document.dataset_id == dataset_id
|
||||
if kind != "clean":
|
||||
assert document.indexing_status == IndexingStatus.WAITING
|
||||
calls.append((kind, document_ids))
|
||||
|
||||
def create(proxy: BatchDocumentIndexingProxy) -> None:
|
||||
dispatched("create", proxy._dataset_id, tuple(proxy._document_ids))
|
||||
|
||||
def duplicate(proxy: BatchDocumentIndexingProxy) -> None:
|
||||
dispatched("duplicate", proxy._dataset_id, tuple(proxy._document_ids))
|
||||
|
||||
monkeypatch.setattr("services.knowledge.dataset_service.FeatureService.get_features", features)
|
||||
monkeypatch.setattr("services.knowledge.documents.adapters.redis_client.lock", lock)
|
||||
monkeypatch.setattr(DocumentIndexingTaskProxy, "delay", create)
|
||||
monkeypatch.setattr(DuplicateDocumentIndexingTaskProxy, "delay", duplicate)
|
||||
monkeypatch.setattr(
|
||||
"services.knowledge.dataset_service.document_indexing_update_task.delay",
|
||||
lambda dataset_id, document_id: dispatched("update", dataset_id, (document_id,)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"services.knowledge.dataset_service.clean_notion_document_task.delay",
|
||||
lambda document_ids, dataset_id: dispatched("clean", dataset_id, tuple(document_ids)),
|
||||
)
|
||||
return calls
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.parametrize("initialize", [False, True])
|
||||
def test_upload_preserves_response_and_commits_before_dispatch(
|
||||
operations: SQLAlchemyDocumentOperations,
|
||||
external_calls: list[tuple[str, tuple[str, ...]]],
|
||||
sqlite_session_factory: sessionmaker[Session],
|
||||
initialize: bool,
|
||||
) -> None:
|
||||
result = (
|
||||
operations.initialize_dataset(CONTEXT, settings())
|
||||
if initialize
|
||||
else operations.create_documents(CONTEXT, _DOCUMENT_CREATION_REF, settings())
|
||||
)
|
||||
(document,) = result["documents"]
|
||||
assert document["name"] == "example.txt"
|
||||
assert document["data_source_info_dict"] == {"upload_file_id": "file"}
|
||||
assert document["indexing_status"] == "waiting"
|
||||
assert result["batch"]
|
||||
assert external_calls == [("create", (document["id"],))]
|
||||
with sqlite_session_factory() as session:
|
||||
dataset = session.get(Dataset, result["dataset"]["id"])
|
||||
assert dataset is not None
|
||||
assert dataset.tenant_id == "tenant"
|
||||
assert dataset.name == ("example.txt..." if initialize else "Dataset")
|
||||
assert dataset.permission == "only_me"
|
||||
|
||||
@staticmethod
|
||||
def test_duplicate_upload_reuses_document_and_dispatches_duplicate_job(
|
||||
operations: SQLAlchemyDocumentOperations, external_calls: list[tuple[str, tuple[str, ...]]]
|
||||
) -> None:
|
||||
first = operations.create_documents(CONTEXT, _DOCUMENT_CREATION_REF, settings())
|
||||
second = operations.create_documents(CONTEXT, _DOCUMENT_CREATION_REF, settings())
|
||||
first_id = first["documents"][0]["id"]
|
||||
assert second["documents"][0]["id"] == first_id
|
||||
assert external_calls == [("create", (first_id,)), ("duplicate", (first_id,))]
|
||||
|
||||
@staticmethod
|
||||
def test_reprocess_checks_model_outside_transaction_and_preserves_segment_reset(
|
||||
operations: SQLAlchemyDocumentOperations,
|
||||
external_calls: list[tuple[str, tuple[str, ...]]],
|
||||
outside_transaction: Callable[[], None],
|
||||
sqlite_session_factory: sessionmaker[Session],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
created = operations.create_documents(CONTEXT, _DOCUMENT_CREATION_REF, settings())
|
||||
document_id = created["documents"][0]["id"]
|
||||
with sqlite_session_factory.begin() as session:
|
||||
document = session.get(Document, document_id)
|
||||
assert document is not None
|
||||
document.indexing_status = IndexingStatus.COMPLETED
|
||||
document.completed_at = datetime(2026, 1, 1)
|
||||
dataset = session.get(Dataset, _DOCUMENT_CREATION_REF.dataset_id)
|
||||
assert dataset is not None
|
||||
dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY
|
||||
dataset.embedding_model = "embedding"
|
||||
dataset.embedding_model_provider = "provider"
|
||||
session.add(
|
||||
DocumentSegment(
|
||||
tenant_id="tenant",
|
||||
dataset_id="dataset",
|
||||
document_id=document_id,
|
||||
position=1,
|
||||
content="content",
|
||||
word_count=1,
|
||||
tokens=1,
|
||||
created_by="actor",
|
||||
status=SegmentStatus.COMPLETED,
|
||||
)
|
||||
)
|
||||
manager = Mock()
|
||||
manager.get_model_instance.side_effect = lambda **_kwargs: outside_transaction()
|
||||
monkeypatch.setattr("services.knowledge.dataset_service.ModelManager.for_tenant", lambda **_kwargs: manager)
|
||||
result = operations.create_documents(
|
||||
CONTEXT, _DOCUMENT_CREATION_REF, settings(original_document_id=document_id, name="Renamed")
|
||||
)
|
||||
assert result["documents"][0]["id"] == document_id
|
||||
assert result["documents"][0]["name"] == "Renamed"
|
||||
assert result["batch"] == created["batch"]
|
||||
assert external_calls[-1] == ("update", (document_id,))
|
||||
manager.get_model_instance.assert_called_once()
|
||||
with sqlite_session_factory() as session:
|
||||
segment = session.scalar(select(DocumentSegment).where(DocumentSegment.document_id == document_id))
|
||||
assert segment is not None
|
||||
assert segment.status == SegmentStatus.RE_SEGMENT
|
||||
|
||||
@staticmethod
|
||||
def test_first_high_quality_upload_resolves_default_model_without_transaction(
|
||||
operations: SQLAlchemyDocumentOperations,
|
||||
external_calls: list[tuple[str, tuple[str, ...]]],
|
||||
outside_transaction: Callable[[], None],
|
||||
sqlite_session_factory: sessionmaker[Session],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
with sqlite_session_factory.begin() as session:
|
||||
dataset = session.get(Dataset, _DOCUMENT_CREATION_REF.dataset_id)
|
||||
assert dataset is not None
|
||||
dataset.indexing_technique = None
|
||||
|
||||
def model(**_kwargs: object) -> ModelInstance:
|
||||
outside_transaction()
|
||||
instance = object.__new__(ModelInstance)
|
||||
instance.model_name = "default-embedding"
|
||||
instance.provider = "default-provider"
|
||||
return instance
|
||||
|
||||
manager = Mock()
|
||||
manager.get_default_model_instance.side_effect = model
|
||||
monkeypatch.setattr("services.knowledge.dataset_service.ModelManager.for_tenant", lambda **_kwargs: manager)
|
||||
operations.create_documents(CONTEXT, _DOCUMENT_CREATION_REF, settings(indexing_technique="high_quality"))
|
||||
with sqlite_session_factory() as session:
|
||||
dataset = session.get(Dataset, _DOCUMENT_CREATION_REF.dataset_id)
|
||||
assert dataset is not None
|
||||
assert dataset.embedding_model == "default-embedding"
|
||||
assert dataset.embedding_model_provider == "default-provider"
|
||||
assert dataset.collection_binding_id is not None
|
||||
assert len(external_calls) == 1
|
||||
|
||||
@staticmethod
|
||||
def test_foreign_upload_is_rejected_without_dispatch(
|
||||
operations: SQLAlchemyDocumentOperations, external_calls: list[tuple[str, tuple[str, ...]]]
|
||||
) -> None:
|
||||
config = settings(
|
||||
data_source={
|
||||
"info_list": {"data_source_type": "upload_file", "file_info_list": {"file_ids": ["foreign-file"]}}
|
||||
}
|
||||
)
|
||||
with pytest.raises(FileNotExistsError):
|
||||
operations.create_documents(CONTEXT, _DOCUMENT_CREATION_REF, config)
|
||||
assert external_calls == []
|
||||
|
||||
@staticmethod
|
||||
def test_notion_removal_is_dispatched_after_new_documents_commit(
|
||||
operations: SQLAlchemyDocumentOperations, external_calls: list[tuple[str, tuple[str, ...]]]
|
||||
) -> None:
|
||||
def notion(page_id: str) -> dict[str, object]:
|
||||
return settings(
|
||||
data_source={
|
||||
"info_list": {
|
||||
"data_source_type": "notion_import",
|
||||
"notion_info_list": [
|
||||
{
|
||||
"credential_id": "credential",
|
||||
"workspace_id": "workspace",
|
||||
"pages": [{"page_id": page_id, "page_name": page_id, "type": "page"}],
|
||||
}
|
||||
],
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
first = operations.create_documents(CONTEXT, _DOCUMENT_CREATION_REF, notion("old-page"))
|
||||
old_id = first["documents"][0]["id"]
|
||||
operations.create_documents(CONTEXT, _DOCUMENT_CREATION_REF, notion("new-page"))
|
||||
assert external_calls[-2] == ("clean", (old_id,))
|
||||
assert external_calls[-1][0] == "create"
|
||||
assert old_id not in external_calls[-1][1]
|
||||
|
||||
@staticmethod
|
||||
def test_cloud_quota_rejection_does_not_create_documents_or_dispatch(
|
||||
operations: SQLAlchemyDocumentOperations,
|
||||
external_calls: list[tuple[str, tuple[str, ...]]],
|
||||
sqlite_session_factory: sessionmaker[Session],
|
||||
config_overrides: Callable[..., None],
|
||||
) -> None:
|
||||
config_overrides(BATCH_UPLOAD_LIMIT=0)
|
||||
with pytest.raises(ValueError, match="batch upload limit"):
|
||||
operations.create_documents(CONTEXT, _DOCUMENT_CREATION_REF, settings())
|
||||
assert external_calls == []
|
||||
with sqlite_session_factory() as session:
|
||||
assert session.scalar(select(Document.id)) is None
|
||||
|
||||
@staticmethod
|
||||
def test_response_read_failure_still_dispatches_already_committed_documents(
|
||||
operations: SQLAlchemyDocumentOperations,
|
||||
external_calls: list[tuple[str, tuple[str, ...]]],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
response = Mock(side_effect=ValueError("response read failed"))
|
||||
monkeypatch.setattr("services.knowledge.documents.adapters._created_response", response)
|
||||
with pytest.raises(ValueError, match="response read failed"):
|
||||
operations.create_documents(CONTEXT, _DOCUMENT_CREATION_REF, settings())
|
||||
assert len(external_calls) == 1
|
||||
assert external_calls[0][0] == "create"
|
||||
|
||||
@staticmethod
|
||||
def test_lost_creation_lock_keeps_empty_batch_response(
|
||||
operations: SQLAlchemyDocumentOperations,
|
||||
external_calls: list[tuple[str, tuple[str, ...]]],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
@contextmanager
|
||||
def lock(_name: str, **_kwargs: object) -> Generator[None]:
|
||||
raise LockNotOwnedError("lock lost")
|
||||
yield # pragma: no cover
|
||||
|
||||
monkeypatch.setattr("services.knowledge.documents.adapters.redis_client.lock", lock)
|
||||
result = operations.create_documents(CONTEXT, _DOCUMENT_CREATION_REF, settings())
|
||||
assert result["documents"] == []
|
||||
assert result["batch"]
|
||||
assert result["dataset"]["id"] == _DOCUMENT_CREATION_REF.dataset_id
|
||||
assert external_calls == []
|
||||
|
||||
@staticmethod
|
||||
def test_billing_http_boundary_does_not_hold_a_transaction(
|
||||
operations: SQLAlchemyDocumentOperations,
|
||||
outside_transaction: Callable[[], None],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
class StopBeforeNetworkError(Exception):
|
||||
pass
|
||||
|
||||
def request(_method: str, url: str, **_kwargs: object) -> None:
|
||||
assert url.endswith("/subscription/info")
|
||||
outside_transaction()
|
||||
raise StopBeforeNetworkError()
|
||||
|
||||
monkeypatch.setattr("services.billing_client.transport._http_client.request", request)
|
||||
with pytest.raises(StopBeforeNetworkError):
|
||||
operations.create_documents(CONTEXT, _DOCUMENT_CREATION_REF, settings())
|
||||
outside_transaction()
|
||||
@@ -0,0 +1,252 @@
|
||||
from collections.abc import Callable
|
||||
from dataclasses import replace
|
||||
from typing import Literal
|
||||
from unittest.mock import MagicMock, create_autospec
|
||||
|
||||
import pytest
|
||||
|
||||
from machinery.context import RequestContext
|
||||
from services.knowledge.dataset_access import AccessibleDataset, DatasetAccess, DatasetAccessDeniedError
|
||||
from services.knowledge.documents.application import (
|
||||
DatasetDocumentApplicationService,
|
||||
DocumentArchivedError,
|
||||
DocumentIndexingStateError,
|
||||
DocumentInvalidActionError,
|
||||
DocumentNotFoundError,
|
||||
DocumentOperations,
|
||||
DocumentState,
|
||||
)
|
||||
from services.knowledge.resource_scope import DatasetRef
|
||||
|
||||
CONTEXT = RequestContext("request-1", None, "actor-1", "tenant-1")
|
||||
DATASET = DatasetRef("tenant-1", "dataset-1")
|
||||
DOCUMENT = DATASET.document("document-1")
|
||||
STATE = DocumentState("document-1", False, "indexing", False, "website_crawl", "text_model", False)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def operations() -> MagicMock:
|
||||
result = create_autospec(DocumentOperations, instance=True, spec_set=True)
|
||||
result.get_state.return_value = STATE
|
||||
return result
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def access() -> MagicMock:
|
||||
result = create_autospec(DatasetAccess, instance=True, spec_set=True)
|
||||
result.require_accessible.return_value = AccessibleDataset("dataset-1", "tenant-1")
|
||||
return result
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def service(access: MagicMock, operations: MagicMock) -> DatasetDocumentApplicationService:
|
||||
return DatasetDocumentApplicationService(
|
||||
dataset_access=access, operations=operations, metadata_schema={"book": {"title": str}, "others": {}}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"operation",
|
||||
[
|
||||
DatasetDocumentApplicationService.get_document,
|
||||
DatasetDocumentApplicationService.delete_document,
|
||||
DatasetDocumentApplicationService.pause_document,
|
||||
DatasetDocumentApplicationService.recover_document,
|
||||
DatasetDocumentApplicationService.sync_website,
|
||||
],
|
||||
)
|
||||
def test_denied_dataset_never_loads_or_mutates_documents(
|
||||
service: DatasetDocumentApplicationService,
|
||||
access: MagicMock,
|
||||
operations: MagicMock,
|
||||
operation: Callable[..., object],
|
||||
) -> None:
|
||||
access.require_accessible.side_effect = DatasetAccessDeniedError()
|
||||
with pytest.raises(DatasetAccessDeniedError):
|
||||
operation(service, CONTEXT, dataset_id="dataset-1", document_id="document-1")
|
||||
assert operations.mock_calls == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"operation",
|
||||
[
|
||||
DatasetDocumentApplicationService.get_document,
|
||||
DatasetDocumentApplicationService.delete_document,
|
||||
DatasetDocumentApplicationService.pause_document,
|
||||
DatasetDocumentApplicationService.recover_document,
|
||||
DatasetDocumentApplicationService.sync_website,
|
||||
],
|
||||
)
|
||||
def test_missing_document_stops_before_any_side_effect(
|
||||
service: DatasetDocumentApplicationService,
|
||||
access: MagicMock,
|
||||
operations: MagicMock,
|
||||
operation: Callable[..., object],
|
||||
) -> None:
|
||||
operations.get_state.return_value = None
|
||||
with pytest.raises(DocumentNotFoundError):
|
||||
operation(service, CONTEXT, dataset_id="dataset-1", document_id="document-1")
|
||||
access.require_accessible.assert_called_once_with(CONTEXT, "dataset-1")
|
||||
operations.get_state.assert_called_once_with(DOCUMENT)
|
||||
assert len(operations.mock_calls) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"operation",
|
||||
[
|
||||
DatasetDocumentApplicationService.pause_document,
|
||||
DatasetDocumentApplicationService.recover_document,
|
||||
DatasetDocumentApplicationService.sync_website,
|
||||
],
|
||||
)
|
||||
def test_archived_documents_cannot_be_reprocessed(
|
||||
service: DatasetDocumentApplicationService, operations: MagicMock, operation: Callable[..., object]
|
||||
) -> None:
|
||||
operations.get_state.return_value = replace(STATE, archived=True)
|
||||
with pytest.raises(DocumentArchivedError):
|
||||
operation(service, CONTEXT, dataset_id="dataset-1", document_id="document-1")
|
||||
assert len(operations.mock_calls) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("action", "status", "paused"),
|
||||
[("pause", "indexing", True), ("resume", "paused", False), ("resume", "error", False)],
|
||||
)
|
||||
def test_processing_transition_records_explicit_actor(
|
||||
service: DatasetDocumentApplicationService,
|
||||
operations: MagicMock,
|
||||
action: Literal["pause", "resume"],
|
||||
status: str,
|
||||
paused: bool,
|
||||
) -> None:
|
||||
operations.get_state.return_value = replace(STATE, indexing_status=status)
|
||||
service.update_processing(CONTEXT, dataset_id="dataset-1", document_id="document-1", action=action)
|
||||
ref, values = operations.update_document.call_args.args
|
||||
assert ref == DOCUMENT
|
||||
assert values["is_paused"] is paused
|
||||
assert values["paused_by"] == (CONTEXT.account_id if paused else None)
|
||||
assert (values["paused_at"] is not None) is paused
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("action", "status"), [("pause", "completed"), ("resume", "indexing"), ("invalid", "indexing")]
|
||||
)
|
||||
def test_invalid_processing_transition_does_not_write(
|
||||
service: DatasetDocumentApplicationService, operations: MagicMock, action: Literal["pause", "resume"], status: str
|
||||
) -> None:
|
||||
operations.get_state.return_value = replace(STATE, indexing_status=status)
|
||||
with pytest.raises(DocumentInvalidActionError):
|
||||
service.update_processing(CONTEXT, dataset_id="dataset-1", document_id="document-1", action=action)
|
||||
operations.update_document.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("doc_type", "expected"), [("book", {"title": "Title"}), ("others", {"title": "Title", "unknown": 12})]
|
||||
)
|
||||
def test_metadata_policy_filters_schema_but_preserves_custom_fields(
|
||||
service: DatasetDocumentApplicationService, operations: MagicMock, doc_type: str | None, expected: dict[str, object]
|
||||
) -> None:
|
||||
service.update_metadata(
|
||||
CONTEXT,
|
||||
dataset_id="dataset-1",
|
||||
document_id="document-1",
|
||||
doc_type=doc_type,
|
||||
doc_metadata={"title": "Title", "unknown": 12},
|
||||
)
|
||||
ref, values = operations.update_document.call_args.args
|
||||
assert ref == DOCUMENT
|
||||
assert values["doc_metadata"] == expected
|
||||
assert values["doc_type"] == doc_type
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("doc_type", "metadata"), [(None, {}), ("book", None), ("invalid", {})])
|
||||
def test_invalid_metadata_never_writes(
|
||||
service: DatasetDocumentApplicationService,
|
||||
operations: MagicMock,
|
||||
doc_type: str | None,
|
||||
metadata: dict[str, object] | None,
|
||||
) -> None:
|
||||
with pytest.raises(ValueError, match="(?:Both doc_type and doc_metadata must be provided\\.|Invalid doc_type\\.)"):
|
||||
service.update_metadata(
|
||||
CONTEXT, dataset_id="dataset-1", document_id="document-1", doc_type=doc_type, doc_metadata=metadata
|
||||
)
|
||||
operations.update_document.assert_not_called()
|
||||
|
||||
|
||||
def test_retry_filters_missing_archived_completed_and_duplicate_documents(
|
||||
service: DatasetDocumentApplicationService, operations: MagicMock
|
||||
) -> None:
|
||||
operations.get_states.return_value = [
|
||||
replace(STATE, id="error", indexing_status="error"),
|
||||
replace(STATE, id="archived", archived=True),
|
||||
replace(STATE, id="completed", indexing_status="completed"),
|
||||
]
|
||||
service.retry_documents(
|
||||
CONTEXT, dataset_id="dataset-1", document_ids=["archived", "missing", "error", "error", "completed"]
|
||||
)
|
||||
operations.retry_documents.assert_called_once_with(DATASET, ["error"], actor_id="actor-1")
|
||||
|
||||
|
||||
def test_summary_skips_qa_and_enables_summary_before_dispatch(
|
||||
service: DatasetDocumentApplicationService, operations: MagicMock
|
||||
) -> None:
|
||||
operations.get_states.return_value = [
|
||||
STATE,
|
||||
replace(STATE, id="qa", doc_form="qa_model"),
|
||||
replace(STATE, id="ready", need_summary=True),
|
||||
]
|
||||
service.generate_summary(CONTEXT, dataset_id="dataset-1", document_ids=["document-1", "qa", "ready"])
|
||||
operations.enable_summary.assert_called_once_with(DATASET, ["document-1"])
|
||||
assert [call.args[0] for call in operations.dispatch_summary.call_args_list] == [
|
||||
DOCUMENT,
|
||||
DATASET.document("ready"),
|
||||
]
|
||||
names = [call[0] for call in operations.mock_calls]
|
||||
assert names.index("enable_summary") < names.index("dispatch_summary")
|
||||
|
||||
|
||||
def test_summary_missing_document_cannot_partially_schedule(
|
||||
service: DatasetDocumentApplicationService, operations: MagicMock
|
||||
) -> None:
|
||||
operations.get_states.return_value = [STATE]
|
||||
with pytest.raises(DocumentNotFoundError, match="missing"):
|
||||
service.generate_summary(CONTEXT, dataset_id="dataset-1", document_ids=["document-1", "missing"])
|
||||
operations.enable_summary.assert_not_called()
|
||||
operations.dispatch_summary.assert_not_called()
|
||||
|
||||
|
||||
def test_process_rule_resolves_document_inside_active_workspace(
|
||||
service: DatasetDocumentApplicationService, operations: MagicMock, access: MagicMock
|
||||
) -> None:
|
||||
operations.find_document.return_value = DOCUMENT
|
||||
service.get_process_rule(CONTEXT, document_id="document-1")
|
||||
operations.find_document.assert_called_once_with(workspace_id="tenant-1", document_id="document-1")
|
||||
access.require_accessible.assert_called_once_with(CONTEXT, "dataset-1")
|
||||
operations.get_process_rule.assert_called_once_with(DATASET)
|
||||
|
||||
|
||||
def test_default_process_rule_does_not_load_a_dataset(
|
||||
service: DatasetDocumentApplicationService, operations: MagicMock, access: MagicMock
|
||||
) -> None:
|
||||
service.get_process_rule(CONTEXT, document_id=None)
|
||||
access.require_accessible.assert_not_called()
|
||||
operations.get_process_rule.assert_called_once_with(None)
|
||||
|
||||
|
||||
def test_pause_requires_indexing_state_and_recover_requires_pause(
|
||||
service: DatasetDocumentApplicationService, operations: MagicMock
|
||||
) -> None:
|
||||
operations.get_state.return_value = replace(STATE, indexing_status="completed")
|
||||
with pytest.raises(DocumentIndexingStateError):
|
||||
service.pause_document(CONTEXT, dataset_id="dataset-1", document_id="document-1")
|
||||
with pytest.raises(DocumentIndexingStateError):
|
||||
service.recover_document(CONTEXT, dataset_id="dataset-1", document_id="document-1")
|
||||
operations.pause_document.assert_not_called()
|
||||
operations.recover_document.assert_not_called()
|
||||
|
||||
|
||||
def test_website_sync_rejects_other_sources(service: DatasetDocumentApplicationService, operations: MagicMock) -> None:
|
||||
operations.get_state.return_value = replace(STATE, data_source_type="upload_file")
|
||||
with pytest.raises(ValueError, match="not a website"):
|
||||
service.sync_website(CONTEXT, dataset_id="dataset-1", document_id="document-1")
|
||||
operations.sync_website.assert_not_called()
|
||||
Reference in new issue
Block a user