chore: 纳入本地 Dify 1.19.0 源码及品牌与免登改动
演示目录只保留可重建的源码。运行密钥、数据卷、依赖和带口令的本地配置不进入仓库。 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
60158935f4
commit
6519682885
15831 files changed
+3063428
No files matched your search
@@ -0,0 +1 @@
|
||||
"""Knowledge-domain application services and adapters."""
|
||||
@@ -0,0 +1,110 @@
|
||||
"""Manage workspace knowledge API keys and keys restricted to one knowledge base."""
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import replace
|
||||
from datetime import datetime
|
||||
from typing import Protocol
|
||||
|
||||
from machinery.context import RequestContext
|
||||
from services.auth.api_key_contracts import ApiKeyCache, ApiKeyRecord, ApiKeyResourceNotFoundError
|
||||
from services.errors.base import NoPermissionError
|
||||
from services.knowledge.dataset_access import (
|
||||
DatasetAccess,
|
||||
DatasetAccessDeniedError,
|
||||
DatasetNotFoundError,
|
||||
)
|
||||
|
||||
|
||||
class DatasetApiKeyStore(Protocol):
|
||||
def list_keys(self, workspace_id: str, dataset_id: str) -> tuple[ApiKeyRecord, ...]: ...
|
||||
|
||||
def create_key(self, workspace_id: str, dataset_id: str, *, max_keys: int, prefix: str) -> ApiKeyRecord: ...
|
||||
|
||||
def delete_key(self, workspace_id: str, dataset_id: str, key_id: str) -> ApiKeyRecord: ...
|
||||
|
||||
def list_workspace_keys(self, workspace_id: str) -> tuple[ApiKeyRecord, ...]: ...
|
||||
|
||||
def create_workspace_key(
|
||||
self,
|
||||
workspace_id: str,
|
||||
dataset_ids: tuple[str, ...],
|
||||
*,
|
||||
knowledge_space_ids: tuple[str, ...] = (),
|
||||
max_keys: int,
|
||||
prefix: str,
|
||||
) -> ApiKeyRecord: ...
|
||||
|
||||
def delete_workspace_key(self, workspace_id: str, key_id: str) -> ApiKeyRecord: ...
|
||||
|
||||
|
||||
class UnknownDatasetIdsError(Exception):
|
||||
def __init__(self, dataset_ids: list[str]) -> None:
|
||||
super().__init__(f"Unknown knowledge base id(s): {', '.join(dataset_ids)}")
|
||||
|
||||
|
||||
class UnknownKnowledgeSpaceIdsError(Exception):
|
||||
def __init__(self, space_ids: list[str]) -> None:
|
||||
super().__init__(f"Unknown knowledge space id(s): {', '.join(space_ids)}")
|
||||
|
||||
|
||||
class DatasetApiKeyService:
|
||||
MAX_KEYS = 10
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
keys: DatasetApiKeyStore,
|
||||
cache: ApiKeyCache,
|
||||
access: DatasetAccess,
|
||||
rbac_enabled: Callable[[], bool],
|
||||
last_used_at: Callable[[str, str | None, datetime | None], datetime | None],
|
||||
) -> None:
|
||||
self._keys = keys
|
||||
self._cache = cache
|
||||
self._access = access
|
||||
self._rbac_enabled = rbac_enabled
|
||||
self._last_used_at = last_used_at
|
||||
|
||||
def list_keys(self, context: RequestContext, dataset_id: str) -> tuple[ApiKeyRecord, ...]:
|
||||
self._check_access(context, dataset_id)
|
||||
return self._keys.list_keys(context.active_workspace_id, dataset_id)
|
||||
|
||||
def create_key(self, context: RequestContext, dataset_id: str) -> ApiKeyRecord:
|
||||
self._check_access(context, dataset_id)
|
||||
return self._keys.create_key(context.active_workspace_id, dataset_id, max_keys=self.MAX_KEYS, prefix="ds-")
|
||||
|
||||
def delete_key(self, context: RequestContext, dataset_id: str, key_id: str) -> None:
|
||||
self._check_access(context, dataset_id)
|
||||
key = self._keys.delete_key(context.active_workspace_id, dataset_id, key_id)
|
||||
self._cache.delete(key.token, key.type)
|
||||
|
||||
def _check_access(self, context: RequestContext, dataset_id: str) -> None:
|
||||
# Enterprise RBAC is enforced at admission; community deployments use the dataset ACL.
|
||||
if not self._rbac_enabled():
|
||||
try:
|
||||
self._access.require_accessible(context, dataset_id)
|
||||
except DatasetNotFoundError as error:
|
||||
raise ApiKeyResourceNotFoundError("Dataset not found.") from error
|
||||
except DatasetAccessDeniedError as error:
|
||||
raise NoPermissionError("You do not have permission to access this dataset.") from error
|
||||
|
||||
def list_workspace_keys(self, context: RequestContext) -> tuple[ApiKeyRecord, ...]:
|
||||
return tuple(
|
||||
replace(key, last_used_at=self._last_used_at(key.token, key.type, key.last_used_at))
|
||||
for key in self._keys.list_workspace_keys(context.active_workspace_id)
|
||||
)
|
||||
|
||||
def create_workspace_key(
|
||||
self, context: RequestContext, dataset_ids: tuple[str, ...], *, knowledge_space_ids: tuple[str, ...] = ()
|
||||
) -> ApiKeyRecord:
|
||||
return self._keys.create_workspace_key(
|
||||
context.active_workspace_id,
|
||||
tuple(dict.fromkeys(dataset_ids)),
|
||||
knowledge_space_ids=tuple(dict.fromkeys(knowledge_space_ids)),
|
||||
max_keys=self.MAX_KEYS,
|
||||
prefix="dataset-",
|
||||
)
|
||||
|
||||
def delete_workspace_key(self, context: RequestContext, key_id: str) -> None:
|
||||
key = self._keys.delete_workspace_key(context.active_workspace_id, key_id)
|
||||
self._cache.delete(key.token, key.type)
|
||||
@@ -0,0 +1,123 @@
|
||||
"""Dataset access use case, read model, errors, and consumer-owned ports."""
|
||||
|
||||
from typing import NamedTuple, Protocol
|
||||
|
||||
from machinery.context import RequestContext
|
||||
|
||||
_OWNER_ROLE = "owner"
|
||||
_ALL_TEAM_MEMBERS = "all_team_members"
|
||||
_PARTIAL_MEMBERS = "partial_members"
|
||||
|
||||
|
||||
class DatasetAccessRecord(NamedTuple):
|
||||
"""Dataset state consumed only by the access policy."""
|
||||
|
||||
id: str
|
||||
workspace_id: str
|
||||
maintainer_id: str | None
|
||||
permission: str
|
||||
|
||||
|
||||
class AccessibleDataset(NamedTuple):
|
||||
"""Minimal result proving that the request may operate on a dataset."""
|
||||
|
||||
id: str
|
||||
workspace_id: str
|
||||
|
||||
|
||||
class DatasetAccessSnapshot(NamedTuple):
|
||||
"""Dataset state plus the requesting account's partial-member grant."""
|
||||
|
||||
dataset: DatasetAccessRecord
|
||||
actor_has_partial_access: bool
|
||||
|
||||
|
||||
class DatasetAccessReader(Protocol):
|
||||
"""Load tenant-scoped dataset state needed by the access policy."""
|
||||
|
||||
def get_access_snapshot(
|
||||
self,
|
||||
*,
|
||||
workspace_id: str,
|
||||
dataset_id: str,
|
||||
actor_id: str,
|
||||
) -> DatasetAccessSnapshot | None: ...
|
||||
|
||||
|
||||
class WorkspaceRoleReader(Protocol):
|
||||
"""Read the account's legacy role in a workspace."""
|
||||
|
||||
def get_legacy_role(self, *, workspace_id: str, account_id: str) -> str | None: ...
|
||||
|
||||
|
||||
class DatasetAccessError(Exception):
|
||||
"""Base class for framework-neutral dataset access failures."""
|
||||
|
||||
|
||||
class DatasetNotFoundError(DatasetAccessError):
|
||||
def __init__(self) -> None:
|
||||
super().__init__("Dataset not found")
|
||||
|
||||
|
||||
class DatasetAccessDeniedError(DatasetAccessError):
|
||||
def __init__(self) -> None:
|
||||
super().__init__("You do not have permission to access this dataset")
|
||||
|
||||
|
||||
def can_access_dataset(snapshot: DatasetAccessSnapshot, *, actor_id: str, workspace_role: str | None) -> bool:
|
||||
"""Evaluate legacy dataset visibility without database or framework access."""
|
||||
|
||||
if workspace_role is None:
|
||||
return False
|
||||
dataset = snapshot.dataset
|
||||
if workspace_role == _OWNER_ROLE or dataset.maintainer_id == actor_id:
|
||||
return True
|
||||
if dataset.permission == _ALL_TEAM_MEMBERS:
|
||||
return True
|
||||
if dataset.permission == _PARTIAL_MEMBERS:
|
||||
return snapshot.actor_has_partial_access
|
||||
return False
|
||||
|
||||
|
||||
class DatasetAccessService:
|
||||
"""Resolve an owned dataset and enforce legacy permissions when configured."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
datasets: DatasetAccessReader,
|
||||
workspace_roles: WorkspaceRoleReader,
|
||||
legacy_permissions_enabled: bool,
|
||||
) -> None:
|
||||
self._datasets = datasets
|
||||
self._workspace_roles = workspace_roles
|
||||
self._legacy_permissions_enabled = legacy_permissions_enabled
|
||||
|
||||
def require_accessible(self, context: RequestContext, dataset_id: str) -> AccessibleDataset:
|
||||
snapshot = self._datasets.get_access_snapshot(
|
||||
workspace_id=context.active_workspace_id,
|
||||
dataset_id=dataset_id,
|
||||
actor_id=context.account_id,
|
||||
)
|
||||
if snapshot is None:
|
||||
raise DatasetNotFoundError()
|
||||
return self.check_access(context, snapshot)
|
||||
|
||||
def check_access(self, context: RequestContext, snapshot: DatasetAccessSnapshot) -> AccessibleDataset:
|
||||
"""Authorize already-loaded state without querying the dataset again."""
|
||||
if snapshot.dataset.workspace_id != context.active_workspace_id:
|
||||
raise DatasetAccessDeniedError()
|
||||
if self._legacy_permissions_enabled:
|
||||
workspace_role = self._workspace_roles.get_legacy_role(
|
||||
workspace_id=context.active_workspace_id,
|
||||
account_id=context.account_id,
|
||||
)
|
||||
if not can_access_dataset(snapshot, actor_id=context.account_id, workspace_role=workspace_role):
|
||||
raise DatasetAccessDeniedError()
|
||||
return AccessibleDataset(id=snapshot.dataset.id, workspace_id=snapshot.dataset.workspace_id)
|
||||
|
||||
|
||||
class DatasetAccess(Protocol):
|
||||
def require_accessible(self, context: RequestContext, dataset_id: str) -> AccessibleDataset: ...
|
||||
|
||||
def check_access(self, context: RequestContext, snapshot: DatasetAccessSnapshot) -> AccessibleDataset: ...
|
||||
@@ -0,0 +1,329 @@
|
||||
"""Load knowledge details and materialize data before response serialization."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import Any, cast
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from core.rag.index_processor.constant.built_in_field import BuiltInField, MetadataDataSource
|
||||
from core.rag.index_processor.constant.query_type import QueryType
|
||||
from core.tools.signature import sign_upload_file_preview_url
|
||||
from models.dataset import Dataset, DatasetMetadata, DatasetQuery, DocMetadataDetailItem, Document, DocumentSegment
|
||||
from models.model import UploadFile
|
||||
from repositories.knowledge.dataset_read_repository import (
|
||||
DatasetDetailPrefetch,
|
||||
DocumentReadBatch,
|
||||
SegmentReadBatch,
|
||||
build_dataset_detail_prefetch,
|
||||
get_document_dataset,
|
||||
get_document_metadata_rows,
|
||||
get_document_upload_file,
|
||||
get_document_uploader,
|
||||
get_query_upload_files,
|
||||
load_document_read_batch,
|
||||
load_segment_read_batch,
|
||||
)
|
||||
from repositories.knowledge.segment_read_adapter import (
|
||||
format_segment_attachments,
|
||||
sign_segment_content,
|
||||
)
|
||||
from services.knowledge.entities.datasets import DatasetDetailRecord
|
||||
|
||||
|
||||
def load_dataset_detail(
|
||||
dataset: Dataset, *, session: Session, prefetch: DatasetDetailPrefetch | None = None
|
||||
) -> DatasetDetailRecord:
|
||||
values = prefetch if prefetch is not None else build_dataset_detail_prefetch([dataset], session=session)
|
||||
return _dataset_detail_values(dataset, values)
|
||||
|
||||
|
||||
def load_dataset_details(datasets: Sequence[Dataset], *, session: Session) -> list[DatasetDetailRecord]:
|
||||
"""Read a page in one batch and return values with no ORM or session references."""
|
||||
prefetch = build_dataset_detail_prefetch(datasets, session=session)
|
||||
return [_dataset_detail_values(dataset, prefetch) for dataset in datasets]
|
||||
|
||||
|
||||
def _dataset_detail_values(dataset: Dataset, prefetch: DatasetDetailPrefetch) -> DatasetDetailRecord:
|
||||
owner = (dataset.id, dataset.tenant_id)
|
||||
return {
|
||||
"id": dataset.id,
|
||||
"name": dataset.name,
|
||||
"description": dataset.description,
|
||||
"provider": dataset.provider,
|
||||
"permission": dataset.permission,
|
||||
"data_source_type": dataset.data_source_type,
|
||||
"indexing_technique": dataset.indexing_technique,
|
||||
"keyword_number": dataset.keyword_number,
|
||||
"app_count": prefetch.app_counts.get(dataset.id, 0),
|
||||
"document_count": prefetch.document_counts.get(dataset.id, 0),
|
||||
"word_count": prefetch.word_counts.get(dataset.id, 0),
|
||||
"created_by": dataset.created_by,
|
||||
"author_name": prefetch.author_names.get(dataset.created_by),
|
||||
"created_at": dataset.created_at,
|
||||
"updated_by": dataset.updated_by,
|
||||
"updated_at": dataset.updated_at,
|
||||
"embedding_model": dataset.embedding_model,
|
||||
"embedding_model_provider": dataset.embedding_model_provider,
|
||||
"embedding_available": vars(dataset).get("embedding_available"),
|
||||
"retrieval_model_dict": dataset.retrieval_model_dict,
|
||||
"summary_index_setting": dataset.summary_index_setting,
|
||||
"tags": [{"id": tag.id, "name": tag.name, "type": tag.type} for tag in prefetch.tags.get(owner, [])],
|
||||
"doc_form": dataset.chunk_structure or prefetch.doc_forms.get(owner),
|
||||
"external_knowledge_info": prefetch.external_knowledge_infos.get(owner)
|
||||
if dataset.provider == "external"
|
||||
else None,
|
||||
"external_retrieval_model": dataset.external_retrieval_model,
|
||||
"doc_metadata": dataset.build_doc_metadata(prefetch.doc_metadata.get(dataset.id, [])),
|
||||
"built_in_field_enabled": dataset.built_in_field_enabled,
|
||||
"pipeline_id": dataset.pipeline_id,
|
||||
"runtime_mode": dataset.runtime_mode,
|
||||
"chunk_structure": dataset.chunk_structure,
|
||||
"icon_info": dataset.icon_info,
|
||||
"is_published": prefetch.pipeline_published.get(dataset.pipeline_id, False) if dataset.pipeline_id else False,
|
||||
"total_documents": prefetch.document_counts.get(dataset.id, 0),
|
||||
"total_available_documents": prefetch.available_document_counts.get(dataset.id, 0),
|
||||
"enable_api": dataset.enable_api,
|
||||
"is_multimodal": dataset.is_multimodal,
|
||||
"permission_keys": vars(dataset).get("permission_keys", []),
|
||||
"maintainer": dataset.maintainer,
|
||||
}
|
||||
|
||||
|
||||
def load_document_detail(document: Document, *, session: Session) -> dict[str, Any]:
|
||||
return load_document_details([document], session=session)[0]
|
||||
|
||||
|
||||
def load_document_details(documents: Iterable[Document], *, session: Session) -> list[dict[str, Any]]:
|
||||
"""Materialize document attributes and relations while the caller's session is open."""
|
||||
items = list(documents)
|
||||
batch = load_document_read_batch(items, session=session)
|
||||
return [
|
||||
{
|
||||
"id": document.id,
|
||||
"position": document.position,
|
||||
"data_source_type": document.data_source_type,
|
||||
"data_source_info_dict": document.data_source_info_dict,
|
||||
"dataset_process_rule_id": document.dataset_process_rule_id,
|
||||
"name": document.name,
|
||||
"created_from": document.created_from,
|
||||
"created_by": document.created_by,
|
||||
"created_at": document.created_at,
|
||||
"tokens": document.tokens,
|
||||
"indexing_status": document.indexing_status,
|
||||
"error": document.error,
|
||||
"enabled": document.enabled,
|
||||
"disabled_at": document.disabled_at,
|
||||
"disabled_by": document.disabled_by,
|
||||
"archived": document.archived,
|
||||
"display_status": document.display_status,
|
||||
"word_count": document.word_count,
|
||||
"doc_form": document.doc_form,
|
||||
"summary_index_status": vars(document).get("summary_index_status"),
|
||||
"need_summary": document.need_summary,
|
||||
"completed_segments": vars(document).get("completed_segments"),
|
||||
"total_segments": vars(document).get("total_segments"),
|
||||
**document_response_values(document, batch),
|
||||
}
|
||||
for document in items
|
||||
]
|
||||
|
||||
|
||||
def load_segment_detail(
|
||||
segment: DocumentSegment, summary: str | None, *, session: Session, batch: SegmentReadBatch | None = None
|
||||
) -> dict[str, Any]:
|
||||
"""Load relations and authorize file links before handing plain values to the response model."""
|
||||
if batch is None:
|
||||
batch = load_segment_read_batch([segment], session=session)
|
||||
child_chunks = batch.child_chunks.get(segment.id, [])
|
||||
return {
|
||||
"id": segment.id,
|
||||
"position": segment.position,
|
||||
"document_id": segment.document_id,
|
||||
"content": segment.content,
|
||||
"sign_content": sign_segment_content(segment, session=session),
|
||||
"answer": segment.answer,
|
||||
"word_count": segment.word_count,
|
||||
"tokens": segment.tokens,
|
||||
"keywords": segment.keywords,
|
||||
"index_node_id": segment.index_node_id,
|
||||
"index_node_hash": segment.index_node_hash,
|
||||
"hit_count": segment.hit_count,
|
||||
"enabled": segment.enabled,
|
||||
"disabled_at": segment.disabled_at,
|
||||
"disabled_by": segment.disabled_by,
|
||||
"status": segment.status,
|
||||
"created_by": segment.created_by,
|
||||
"created_at": segment.created_at,
|
||||
"updated_at": segment.updated_at,
|
||||
"updated_by": segment.updated_by,
|
||||
"indexing_at": segment.indexing_at,
|
||||
"completed_at": segment.completed_at,
|
||||
"error": segment.error,
|
||||
"stopped_at": segment.stopped_at,
|
||||
"child_chunks": [
|
||||
{
|
||||
"id": chunk.id,
|
||||
"segment_id": chunk.segment_id,
|
||||
"content": chunk.content,
|
||||
"position": chunk.position,
|
||||
"word_count": chunk.word_count,
|
||||
"type": chunk.type,
|
||||
"created_at": chunk.created_at,
|
||||
"updated_at": chunk.updated_at,
|
||||
}
|
||||
for chunk in child_chunks
|
||||
],
|
||||
"attachments": format_segment_attachments(batch.attachments.get(segment.id, [])),
|
||||
"summary": summary,
|
||||
}
|
||||
|
||||
|
||||
def load_segment_details(
|
||||
segments: Iterable[DocumentSegment], summaries: Mapping[str, str | None], *, session: Session
|
||||
) -> list[dict[str, Any]]:
|
||||
items = list(segments)
|
||||
batch = load_segment_read_batch(items, session=session)
|
||||
return [load_segment_detail(segment, summaries.get(segment.id), session=session, batch=batch) for segment in items]
|
||||
|
||||
|
||||
def format_document_source_detail(document: Document, file: UploadFile | None) -> dict[str, Any]:
|
||||
if document.data_source_type == "upload_file" and file is not None:
|
||||
return {
|
||||
"upload_file": {
|
||||
"id": file.id,
|
||||
"name": file.name,
|
||||
"size": file.size,
|
||||
"extension": file.extension,
|
||||
"mime_type": file.mime_type,
|
||||
"created_by": file.created_by,
|
||||
"created_at": file.created_at.timestamp(),
|
||||
}
|
||||
}
|
||||
if document.data_source_type in {"notion_import", "website_crawl"}:
|
||||
return document.data_source_info_dict
|
||||
return {}
|
||||
|
||||
|
||||
def format_document_metadata(
|
||||
document: Document, metadatas: Sequence[DatasetMetadata], *, built_in_enabled: bool, uploader_name: str | None
|
||||
) -> list[DocMetadataDetailItem] | None:
|
||||
values = document.doc_metadata or {}
|
||||
details: list[DocMetadataDetailItem] = [
|
||||
{"id": metadata.id, "name": metadata.name, "type": metadata.type, "value": values.get(metadata.name)}
|
||||
for metadata in metadatas
|
||||
]
|
||||
if built_in_enabled:
|
||||
details.extend(format_document_built_in_fields(document, uploader_name=uploader_name))
|
||||
return details or None
|
||||
|
||||
|
||||
def document_response_values(document: Document, batch: DocumentReadBatch) -> dict[str, Any]:
|
||||
"""Build response-only values from a loaded snapshot without database access."""
|
||||
file_id = document.data_source_info_dict.get("upload_file_id")
|
||||
process_rule = (
|
||||
batch.process_rules.get(document.dataset_process_rule_id)
|
||||
if document.dataset_process_rule_id is not None
|
||||
else None
|
||||
)
|
||||
if process_rule is not None and process_rule.dataset_id != document.dataset_id:
|
||||
process_rule = None
|
||||
return {
|
||||
"data_source_detail_dict": format_document_source_detail(
|
||||
document, batch.uploads.get((document.tenant_id, file_id)) if isinstance(file_id, str) else None
|
||||
),
|
||||
"hit_count": batch.hit_counts.get(document.id, 0),
|
||||
"doc_metadata_details": format_document_metadata(
|
||||
document,
|
||||
batch.metadatas.get(document.id, []),
|
||||
built_in_enabled=batch.built_in_enabled.get((document.tenant_id, document.dataset_id), False),
|
||||
uploader_name=batch.uploader_names.get(document.created_by),
|
||||
),
|
||||
"process_rule_dict": process_rule.to_dict() if process_rule else None,
|
||||
}
|
||||
|
||||
|
||||
def get_document_source_detail(document: Document, *, session: Session) -> dict[str, Any]:
|
||||
file = get_document_upload_file(document, session=session) if document.data_source_type == "upload_file" else None
|
||||
return format_document_source_detail(document, file)
|
||||
|
||||
|
||||
def get_document_metadata_details(document: Document, *, session: Session) -> list[DocMetadataDetailItem] | None:
|
||||
dataset = get_document_dataset(document, session=session)
|
||||
built_in_enabled = bool(dataset is not None and dataset.built_in_field_enabled)
|
||||
return format_document_metadata(
|
||||
document,
|
||||
get_document_metadata_rows(document, session=session) if document.doc_metadata else [],
|
||||
built_in_enabled=built_in_enabled,
|
||||
uploader_name=get_document_uploader(document, session=session) if built_in_enabled else None,
|
||||
)
|
||||
|
||||
|
||||
def get_document_built_in_fields(document: Document, *, session: Session) -> list[DocMetadataDetailItem]:
|
||||
return format_document_built_in_fields(document, uploader_name=get_document_uploader(document, session=session))
|
||||
|
||||
|
||||
def format_document_built_in_fields(document: Document, *, uploader_name: str | None) -> list[DocMetadataDetailItem]:
|
||||
built_in_fields: list[DocMetadataDetailItem] = []
|
||||
built_in_fields.extend(
|
||||
(
|
||||
{"id": "built-in", "name": BuiltInField.document_name, "type": "string", "value": document.name},
|
||||
{"id": "built-in", "name": BuiltInField.uploader, "type": "string", "value": uploader_name},
|
||||
{
|
||||
"id": "built-in",
|
||||
"name": BuiltInField.upload_date,
|
||||
"type": "time",
|
||||
"value": str(document.created_at.timestamp()),
|
||||
},
|
||||
{
|
||||
"id": "built-in",
|
||||
"name": BuiltInField.last_update_date,
|
||||
"type": "time",
|
||||
"value": str(document.updated_at.timestamp()),
|
||||
},
|
||||
{
|
||||
"id": "built-in",
|
||||
"name": BuiltInField.source,
|
||||
"type": "string",
|
||||
"value": MetadataDataSource[document.data_source_type],
|
||||
},
|
||||
)
|
||||
)
|
||||
return built_in_fields
|
||||
|
||||
|
||||
def get_dataset_queries(query: DatasetQuery, *, session: Session) -> list[dict[str, Any]]:
|
||||
"""Read structured audit content, preserving legacy plain-text queries."""
|
||||
try:
|
||||
decoded = json.loads(query.content)
|
||||
except (ValueError, TypeError):
|
||||
decoded = None
|
||||
raw_records = decoded if isinstance(decoded, list) else [decoded]
|
||||
if not raw_records or not all(
|
||||
isinstance(item, dict)
|
||||
and item.get("content_type") in {QueryType.TEXT_QUERY, QueryType.IMAGE_QUERY}
|
||||
and isinstance(item.get("content"), str)
|
||||
for item in raw_records
|
||||
):
|
||||
return [{"content_type": QueryType.TEXT_QUERY, "content": query.content, "file_info": None}]
|
||||
records = cast(list[dict[str, Any]], raw_records)
|
||||
files = get_query_upload_files(
|
||||
query, {item["content"] for item in records if item["content_type"] == QueryType.IMAGE_QUERY}, session=session
|
||||
)
|
||||
result: list[dict[str, Any]] = []
|
||||
for item in records:
|
||||
record: dict[str, Any] = {"content_type": item["content_type"], "content": item["content"], "file_info": None}
|
||||
file = files.get(item["content"]) if item["content_type"] == QueryType.IMAGE_QUERY else None
|
||||
if file is not None:
|
||||
record["file_info"] = {
|
||||
"id": file.id,
|
||||
"name": file.name,
|
||||
"size": file.size,
|
||||
"extension": file.extension,
|
||||
"mime_type": file.mime_type,
|
||||
"source_url": sign_upload_file_preview_url(file.id, file.extension),
|
||||
}
|
||||
result.append(record)
|
||||
return result
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1 @@
|
||||
"""Dataset administration use cases and infrastructure adapters."""
|
||||
@@ -0,0 +1,344 @@
|
||||
"""Session-owning dataset administration adapters."""
|
||||
|
||||
from collections.abc import Generator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from configs import dify_config
|
||||
from core.plugin.impl.model_runtime_factory import create_plugin_provider_manager
|
||||
from graphon.model_runtime.entities.model_entities import ModelType
|
||||
from libs.pagination import clamp_pagination
|
||||
from machinery.context import RequestContext
|
||||
from models import Account, Dataset, Document
|
||||
from models.dataset import DatasetPermission, DatasetPermissionEnum
|
||||
from models.provider_ids import ModelProviderID
|
||||
from repositories.knowledge.dataset_repository import _get_dataset
|
||||
from services.app.query_service import RelatedApps
|
||||
from services.dataset_knowledge_fs_upgrade_service import KnowledgeFSUpgradeSnapshotService, upgrade_discovery_response
|
||||
from services.enterprise import rbac_service
|
||||
from services.errors.base import NoPermissionError
|
||||
from services.knowledge.dataset_access import DatasetAccessDeniedError, DatasetNotFoundError
|
||||
from services.knowledge.dataset_read_service import get_dataset_queries, load_dataset_detail, load_dataset_details
|
||||
from services.knowledge.dataset_service import DatasetPermissionService, DatasetService, DocumentService
|
||||
from services.knowledge.datasets.application import (
|
||||
DatasetListFilter,
|
||||
DatasetVisibility,
|
||||
)
|
||||
from services.knowledge.entities.datasets import DatasetDetailRecord, DatasetPage
|
||||
from services.knowledge.resource_scope import DatasetRef
|
||||
from services.knowledge_fs.feature_service import get_knowledge_fs_features
|
||||
from services.rbac import contracts as rbac_contracts
|
||||
from services.rbac.members import MemberService
|
||||
from services.tag_application_service import TagTargetQuery
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _translate_permissions() -> Generator[None]:
|
||||
try:
|
||||
yield
|
||||
except NoPermissionError as error:
|
||||
raise DatasetAccessDeniedError() from error
|
||||
|
||||
|
||||
def require_dataset(session: Session, ref: DatasetRef) -> Dataset:
|
||||
dataset = _get_dataset(session, ref)
|
||||
if dataset is None:
|
||||
raise DatasetNotFoundError()
|
||||
return dataset
|
||||
|
||||
|
||||
def load_actor(session: Session, context: RequestContext) -> Account:
|
||||
account = session.get(Account, context.account_id)
|
||||
if account is None:
|
||||
raise DatasetAccessDeniedError()
|
||||
account.set_tenant_id_with_session(context.active_workspace_id, session=session)
|
||||
return account
|
||||
|
||||
|
||||
def _normalize_provider(item: DatasetDetailRecord) -> DatasetDetailRecord:
|
||||
if item["indexing_technique"] == "high_quality" and item["embedding_model_provider"]:
|
||||
item["embedding_model_provider"] = str(ModelProviderID(item["embedding_model_provider"]))
|
||||
return item
|
||||
|
||||
|
||||
def _status(document: Document, counts: tuple[int, int] | None = None) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {
|
||||
"id": document.id,
|
||||
"indexing_status": document.indexing_status,
|
||||
"processing_started_at": document.processing_started_at,
|
||||
"parsing_completed_at": document.parsing_completed_at,
|
||||
"cleaning_completed_at": document.cleaning_completed_at,
|
||||
"splitting_completed_at": document.splitting_completed_at,
|
||||
"completed_at": document.completed_at,
|
||||
"paused_at": document.paused_at,
|
||||
"error": document.error,
|
||||
"stopped_at": document.stopped_at,
|
||||
}
|
||||
if counts is not None:
|
||||
result.update(completed_segments=counts[0], total_segments=counts[1])
|
||||
return result
|
||||
|
||||
|
||||
class SQLAlchemyDatasetOperations:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
session_factory: sessionmaker[Session],
|
||||
tags: TagTargetQuery,
|
||||
app_queries: RelatedApps,
|
||||
members: MemberService,
|
||||
) -> None:
|
||||
self._sessions = session_factory
|
||||
self._tags = tags
|
||||
self._app_queries = app_queries
|
||||
self._members_service = members
|
||||
|
||||
def visibility(self, context: RequestContext) -> DatasetVisibility:
|
||||
permissions = self._members_service.permissions(context.active_workspace_id, context.account_id)
|
||||
scope = (
|
||||
rbac_service.RBACService.DatasetAccess.whitelist_resources(context.active_workspace_id, context.account_id)
|
||||
if dify_config.RBAC_ENABLED
|
||||
else None
|
||||
)
|
||||
return DatasetVisibility(
|
||||
default_permissions=permissions.dataset.default_permission_keys,
|
||||
workspace_permissions=permissions.workspace.permission_keys,
|
||||
overrides={item.resource_id: item.permission_keys for item in permissions.dataset.overrides},
|
||||
unrestricted=scope is None or scope.unrestricted,
|
||||
whitelist_ids=scope.resource_ids if scope is not None else (),
|
||||
)
|
||||
|
||||
def embedding_models(self, workspace_id: str) -> set[str]:
|
||||
manager = create_plugin_provider_manager(tenant_id=workspace_id)
|
||||
models = manager.get_configurations(tenant_id=workspace_id).get_models(
|
||||
model_type=ModelType.TEXT_EMBEDDING, only_active=True
|
||||
)
|
||||
return {f"{model.model}:{model.provider.provider}" for model in models}
|
||||
|
||||
def list_datasets(
|
||||
self, context: RequestContext, query: DatasetListFilter, accessible_ids: list[str] | None
|
||||
) -> DatasetPage:
|
||||
page, limit = clamp_pagination(query.page, query.limit, 100)
|
||||
with self._sessions() as session:
|
||||
account = load_actor(session, context)
|
||||
if query.ids:
|
||||
datasets, total = DatasetService.get_datasets_by_ids(
|
||||
list(query.ids),
|
||||
context.active_workspace_id,
|
||||
user=account,
|
||||
accessible_dataset_ids=accessible_ids,
|
||||
session=session,
|
||||
)
|
||||
else:
|
||||
datasets, total = DatasetService.get_datasets(
|
||||
page,
|
||||
limit,
|
||||
session,
|
||||
context.active_workspace_id,
|
||||
account,
|
||||
query.keyword,
|
||||
list(query.tag_ids),
|
||||
query.include_all,
|
||||
creator_ids=list(query.creator_ids),
|
||||
tags=self._tags,
|
||||
accessible_dataset_ids=accessible_ids,
|
||||
)
|
||||
data = load_dataset_details(datasets, session=session)
|
||||
partial_ids = [item["id"] for item in data if item["permission"] == "partial_members"]
|
||||
members: dict[str, list[str]] = {}
|
||||
if partial_ids:
|
||||
for dataset_id, account_id in session.execute(
|
||||
select(DatasetPermission.dataset_id, DatasetPermission.account_id).where(
|
||||
DatasetPermission.dataset_id.in_(partial_ids),
|
||||
DatasetPermission.tenant_id == context.active_workspace_id,
|
||||
)
|
||||
):
|
||||
members.setdefault(dataset_id, []).append(account_id)
|
||||
upgrade_jobs = KnowledgeFSUpgradeSnapshotService(self._sessions).get_latest_by_dataset_ids(
|
||||
tenant_id=context.active_workspace_id,
|
||||
dataset_ids=[item["id"] for item in data],
|
||||
session=session,
|
||||
)
|
||||
knowledge_fs_enabled = get_knowledge_fs_features().enabled
|
||||
for item in data:
|
||||
_normalize_provider(item)
|
||||
item["partial_member_list"] = members.get(item["id"], [])
|
||||
item["knowledge_fs_upgrade"] = upgrade_discovery_response(
|
||||
upgrade_jobs.get(item["id"]),
|
||||
feature_enabled=knowledge_fs_enabled,
|
||||
dataset_provider=str(item["provider"]),
|
||||
).model_dump(mode="json")
|
||||
return {
|
||||
"data": data,
|
||||
"has_more": not query.ids and page * limit < total,
|
||||
"total": total,
|
||||
"page": page,
|
||||
"limit": limit,
|
||||
}
|
||||
|
||||
def _detail(self, session: Session, context: RequestContext, dataset: Dataset) -> DatasetDetailRecord:
|
||||
result = load_dataset_detail(dataset, session=session)
|
||||
permissions = self._members_service.resource_permissions(
|
||||
context.active_workspace_id, context.account_id, rbac_contracts.RBACResourceType.DATASET, [dataset.id]
|
||||
)
|
||||
result["permission_keys"] = permissions.get(dataset.id, [])
|
||||
return result
|
||||
|
||||
def get_dataset(self, context: RequestContext, ref: DatasetRef) -> DatasetDetailRecord:
|
||||
with self._sessions() as session:
|
||||
dataset = require_dataset(session, ref)
|
||||
result = load_dataset_detail(dataset, session=session)
|
||||
permissions = self._members_service.permissions(
|
||||
context.active_workspace_id, context.account_id, dataset_id=ref.dataset_id
|
||||
)
|
||||
result["permission_keys"] = permissions.dataset.permission_keys_by_resource_ids([dataset.id]).get(
|
||||
dataset.id, []
|
||||
)
|
||||
if dataset.permission == DatasetPermissionEnum.PARTIAL_TEAM:
|
||||
result["partial_member_list"] = self._members(session, ref)
|
||||
return _normalize_provider(result)
|
||||
|
||||
def create_dataset(self, context: RequestContext, values: Mapping[str, Any]) -> DatasetDetailRecord:
|
||||
with _translate_permissions(), self._sessions() as session:
|
||||
dataset = DatasetService.create_empty_dataset(
|
||||
session=session,
|
||||
tenant_id=context.active_workspace_id,
|
||||
account=load_actor(session, context),
|
||||
**dict(values),
|
||||
)
|
||||
result = self._detail(session, context, dataset)
|
||||
result["partial_member_list"] = None
|
||||
dataset_id = dataset.id
|
||||
session.commit()
|
||||
if dify_config.RBAC_ENABLED:
|
||||
rbac_service.RBACService.DatasetAccess.replace_whitelist(
|
||||
context.active_workspace_id,
|
||||
context.account_id,
|
||||
dataset_id,
|
||||
rbac_contracts.ReplaceMemberBindings(automatic_include_workspace_members=False),
|
||||
)
|
||||
return result
|
||||
|
||||
def update_dataset(
|
||||
self, context: RequestContext, ref: DatasetRef, values: Mapping[str, Any]
|
||||
) -> DatasetDetailRecord:
|
||||
data = dict(values)
|
||||
if (
|
||||
data.get("indexing_technique") == "high_quality"
|
||||
and data.get("embedding_model_provider") is not None
|
||||
and data.get("embedding_model") is not None
|
||||
):
|
||||
data["is_multimodal"] = DatasetService.check_is_multimodal_model(
|
||||
ref.tenant_id, data["embedding_model_provider"], data["embedding_model"]
|
||||
)
|
||||
with _translate_permissions(), self._sessions() as session:
|
||||
dataset = require_dataset(session, ref)
|
||||
account = load_actor(session, context)
|
||||
if not dify_config.RBAC_ENABLED:
|
||||
DatasetPermissionService.check_permission(
|
||||
account, dataset, data.get("permission"), data.get("partial_member_list"), session=session
|
||||
)
|
||||
# The legacy updater consumes fields from its input; retain the
|
||||
# member selection for the permission update in this transaction.
|
||||
dataset = DatasetService.update_dataset(ref.dataset_id, dict(data), account, session=session)
|
||||
if dataset is None:
|
||||
raise DatasetNotFoundError()
|
||||
result = self._detail(session, context, dataset)
|
||||
if (
|
||||
data.get("partial_member_list") is not None
|
||||
and data.get("permission") == DatasetPermissionEnum.PARTIAL_TEAM
|
||||
):
|
||||
DatasetPermissionService.update_partial_member_list(
|
||||
ref.tenant_id, ref.dataset_id, data["partial_member_list"], session
|
||||
)
|
||||
elif data.get("permission") in {DatasetPermissionEnum.ONLY_ME, DatasetPermissionEnum.ALL_TEAM}:
|
||||
DatasetPermissionService.clear_partial_member_list(ref.dataset_id, session)
|
||||
result["partial_member_list"] = self._members(session, ref)
|
||||
session.commit()
|
||||
return result
|
||||
|
||||
def delete_dataset(self, context: RequestContext, ref: DatasetRef) -> None:
|
||||
with _translate_permissions(), self._sessions() as session:
|
||||
require_dataset(session, ref)
|
||||
if not DatasetService.delete_dataset(ref.dataset_id, load_actor(session, context), session):
|
||||
raise DatasetNotFoundError()
|
||||
DatasetPermissionService.clear_partial_member_list(ref.dataset_id, session)
|
||||
session.commit()
|
||||
|
||||
def is_in_use(self, ref: DatasetRef) -> bool:
|
||||
with self._sessions() as session:
|
||||
require_dataset(session, ref)
|
||||
return DatasetService.dataset_use_check(ref, session)
|
||||
|
||||
def queries(self, ref: DatasetRef, *, page: int, limit: int) -> dict[str, Any]:
|
||||
page, limit = clamp_pagination(page, limit, 100)
|
||||
with self._sessions() as session:
|
||||
require_dataset(session, ref)
|
||||
queries, total = DatasetService.get_dataset_queries(
|
||||
dataset_id=ref.dataset_id, page=page, per_page=limit, session=session
|
||||
)
|
||||
data = [
|
||||
{
|
||||
"id": query.id,
|
||||
"queries": get_dataset_queries(query, session=session),
|
||||
"source": query.source,
|
||||
"source_app_id": query.source_app_id,
|
||||
"created_by_role": query.created_by_role,
|
||||
"created_by": query.created_by,
|
||||
"created_at": query.created_at,
|
||||
}
|
||||
for query in queries
|
||||
]
|
||||
return {"data": data, "has_more": page * limit < total, "total": total, "page": page, "limit": limit}
|
||||
|
||||
def related_apps(self, ref: DatasetRef) -> dict[str, Any]:
|
||||
with self._sessions() as session:
|
||||
require_dataset(session, ref)
|
||||
joins = DatasetService.get_related_apps(ref.dataset_id, session)
|
||||
app_ids = [join.app_id for join in joins]
|
||||
apps = self._app_queries.related_apps(ref.tenant_id, app_ids)
|
||||
return {"data": apps, "total": len(apps)}
|
||||
|
||||
def indexing_status(self, ref: DatasetRef) -> dict[str, Any]:
|
||||
with self._sessions() as session:
|
||||
require_dataset(session, ref)
|
||||
documents = session.scalars(
|
||||
select(Document).where(Document.dataset_id == ref.dataset_id, Document.tenant_id == ref.tenant_id)
|
||||
).all()
|
||||
counts = DocumentService.get_document_segment_counts(documents, session=session)
|
||||
return {"data": [_status(document, counts.get(document.id, (0, 0))) for document in documents]}
|
||||
|
||||
def error_documents(self, ref: DatasetRef) -> dict[str, Any]:
|
||||
with self._sessions() as session:
|
||||
require_dataset(session, ref)
|
||||
documents = DocumentService.get_error_documents_by_dataset_ref(ref, session)
|
||||
return {"data": [_status(document) for document in documents], "total": len(documents)}
|
||||
|
||||
@staticmethod
|
||||
def _members(session: Session, ref: DatasetRef) -> list[str]:
|
||||
return list(
|
||||
session.scalars(
|
||||
select(DatasetPermission.account_id).where(
|
||||
DatasetPermission.dataset_id == ref.dataset_id, DatasetPermission.tenant_id == ref.tenant_id
|
||||
)
|
||||
).all()
|
||||
)
|
||||
|
||||
def partial_members(self, ref: DatasetRef) -> list[str]:
|
||||
with self._sessions() as session:
|
||||
require_dataset(session, ref)
|
||||
return self._members(session, ref)
|
||||
|
||||
def auto_disable_logs(self, ref: DatasetRef) -> dict[str, Any]:
|
||||
with self._sessions() as session:
|
||||
return dict(DatasetService.get_dataset_auto_disable_logs(ref, session))
|
||||
|
||||
def set_api_enabled(self, context: RequestContext, ref: DatasetRef, enabled: bool) -> None:
|
||||
with self._sessions() as session:
|
||||
DatasetService.update_dataset_api_status(
|
||||
require_dataset(session, ref), enabled, load_actor(session, context), session
|
||||
)
|
||||
session.commit()
|
||||
@@ -0,0 +1,168 @@
|
||||
"""Dataset administration policies independent of HTTP and persistence."""
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Protocol
|
||||
|
||||
from libs.url_utils import normalize_api_base_url
|
||||
from machinery.context import RequestContext
|
||||
from services.knowledge.dataset_access import DatasetAccess
|
||||
from services.knowledge.datasets.retrieval import retrieval_methods
|
||||
from services.knowledge.entities.datasets import DatasetDetailRecord, DatasetPage
|
||||
from services.knowledge.resource_scope import DatasetRef
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DatasetListFilter:
|
||||
page: int = 1
|
||||
limit: int = 20
|
||||
keyword: str | None = None
|
||||
include_all: bool = False
|
||||
ids: Sequence[str] = ()
|
||||
tag_ids: Sequence[str] = ()
|
||||
creator_ids: Sequence[str] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DatasetVisibility:
|
||||
default_permissions: Sequence[str] = ()
|
||||
workspace_permissions: Sequence[str] = ()
|
||||
overrides: Mapping[str, Sequence[str]] = field(default_factory=dict)
|
||||
unrestricted: bool = True
|
||||
whitelist_ids: Sequence[str] = ()
|
||||
|
||||
def list_scope(self, *, rbac_enabled: bool) -> list[str] | None:
|
||||
if not rbac_enabled:
|
||||
return None
|
||||
# Maintainers always retain visibility of their own datasets. The
|
||||
# whitelist takes precedence over grants only for other datasets.
|
||||
if not self.unrestricted:
|
||||
return sorted(set(self.whitelist_ids))
|
||||
read_keys = {"dataset.preview", "dataset.acl.preview", "dataset.full_access"}
|
||||
default_read = bool(
|
||||
read_keys.intersection(self.default_permissions) or read_keys.intersection(self.workspace_permissions)
|
||||
)
|
||||
ids = (
|
||||
None
|
||||
if default_read
|
||||
else sorted(key for key, grants in self.overrides.items() if read_keys.intersection(grants))
|
||||
)
|
||||
return ids
|
||||
|
||||
|
||||
class DatasetOperations(Protocol):
|
||||
"""Materialize owned values and complete writes before returning."""
|
||||
|
||||
def visibility(self, context: RequestContext) -> DatasetVisibility: ...
|
||||
def list_datasets(
|
||||
self, context: RequestContext, query: DatasetListFilter, accessible_ids: list[str] | None
|
||||
) -> DatasetPage: ...
|
||||
def embedding_models(self, workspace_id: str) -> set[str]: ...
|
||||
def get_dataset(self, context: RequestContext, ref: DatasetRef) -> DatasetDetailRecord: ...
|
||||
def create_dataset(self, context: RequestContext, values: Mapping[str, Any]) -> DatasetDetailRecord: ...
|
||||
def update_dataset(
|
||||
self, context: RequestContext, ref: DatasetRef, values: Mapping[str, Any]
|
||||
) -> DatasetDetailRecord: ...
|
||||
def delete_dataset(self, context: RequestContext, ref: DatasetRef) -> None: ...
|
||||
def is_in_use(self, ref: DatasetRef) -> bool: ...
|
||||
def queries(self, ref: DatasetRef, *, page: int, limit: int) -> dict[str, Any]: ...
|
||||
def related_apps(self, ref: DatasetRef) -> dict[str, Any]: ...
|
||||
def indexing_status(self, ref: DatasetRef) -> dict[str, Any]: ...
|
||||
def error_documents(self, ref: DatasetRef) -> dict[str, Any]: ...
|
||||
def partial_members(self, ref: DatasetRef) -> list[str]: ...
|
||||
def auto_disable_logs(self, ref: DatasetRef) -> dict[str, Any]: ...
|
||||
def set_api_enabled(self, context: RequestContext, ref: DatasetRef, enabled: bool) -> None: ...
|
||||
|
||||
|
||||
class DatasetApplicationService:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
dataset_access: DatasetAccess,
|
||||
operations: DatasetOperations,
|
||||
rbac_enabled: bool,
|
||||
service_api_url: str,
|
||||
vector_store: str | None,
|
||||
tidb_fulltext: bool,
|
||||
) -> None:
|
||||
self._dataset_access = dataset_access
|
||||
self._operations = operations
|
||||
self._rbac_enabled = rbac_enabled
|
||||
self._service_api_url = service_api_url
|
||||
self._vector_store = vector_store
|
||||
self._tidb_fulltext = tidb_fulltext
|
||||
|
||||
def _dataset(self, context: RequestContext, dataset_id: str) -> DatasetRef:
|
||||
dataset = self._dataset_access.require_accessible(context, dataset_id)
|
||||
return DatasetRef(dataset.workspace_id, dataset.id)
|
||||
|
||||
@staticmethod
|
||||
def _embedding_available(item: DatasetDetailRecord, models: set[str], *, listing: bool) -> None:
|
||||
high_quality = item["indexing_technique"] == "high_quality"
|
||||
item["embedding_available"] = (
|
||||
not high_quality
|
||||
or (listing and not item["embedding_model_provider"])
|
||||
or f"{item['embedding_model']}:{item['embedding_model_provider']}" in models
|
||||
)
|
||||
|
||||
def list_datasets(self, context: RequestContext, query: DatasetListFilter) -> DatasetPage:
|
||||
visibility = self._operations.visibility(context)
|
||||
ids = visibility.list_scope(rbac_enabled=self._rbac_enabled)
|
||||
result = self._operations.list_datasets(context, query, ids)
|
||||
models = self._operations.embedding_models(context.active_workspace_id)
|
||||
for item in result["data"]:
|
||||
item["permission_keys"] = list(visibility.overrides.get(item["id"], visibility.default_permissions))
|
||||
self._embedding_available(item, models, listing=True)
|
||||
return result
|
||||
|
||||
def get_dataset(self, context: RequestContext, *, dataset_id: str) -> DatasetDetailRecord:
|
||||
result = self._operations.get_dataset(context, self._dataset(context, dataset_id))
|
||||
self._embedding_available(result, self._operations.embedding_models(context.active_workspace_id), listing=False)
|
||||
return result
|
||||
|
||||
def create_dataset(self, context: RequestContext, *, values: Mapping[str, Any]) -> DatasetDetailRecord:
|
||||
settings = dict(values)
|
||||
settings["permission"] = "all_team_members" if self._rbac_enabled else settings.get("permission") or "only_me"
|
||||
return self._operations.create_dataset(context, settings)
|
||||
|
||||
def update_dataset(
|
||||
self, context: RequestContext, *, dataset_id: str, values: Mapping[str, Any]
|
||||
) -> DatasetDetailRecord:
|
||||
return self._operations.update_dataset(context, self._dataset(context, dataset_id), values)
|
||||
|
||||
def delete_dataset(self, context: RequestContext, *, dataset_id: str) -> None:
|
||||
self._operations.delete_dataset(context, self._dataset(context, dataset_id))
|
||||
|
||||
def is_in_use(self, context: RequestContext, *, dataset_id: str) -> bool:
|
||||
return self._operations.is_in_use(self._dataset(context, dataset_id))
|
||||
|
||||
def queries(self, context: RequestContext, *, dataset_id: str, page: int, limit: int) -> dict[str, Any]:
|
||||
return self._operations.queries(self._dataset(context, dataset_id), page=page, limit=limit)
|
||||
|
||||
def related_apps(self, context: RequestContext, *, dataset_id: str) -> dict[str, Any]:
|
||||
return self._operations.related_apps(self._dataset(context, dataset_id))
|
||||
|
||||
def indexing_status(self, context: RequestContext, *, dataset_id: str) -> dict[str, Any]:
|
||||
return self._operations.indexing_status(self._dataset(context, dataset_id))
|
||||
|
||||
def error_documents(self, context: RequestContext, *, dataset_id: str) -> dict[str, Any]:
|
||||
return self._operations.error_documents(self._dataset(context, dataset_id))
|
||||
|
||||
def partial_members(self, context: RequestContext, *, dataset_id: str) -> list[str]:
|
||||
return self._operations.partial_members(self._dataset(context, dataset_id))
|
||||
|
||||
def auto_disable_logs(self, context: RequestContext, *, dataset_id: str) -> dict[str, Any]:
|
||||
return self._operations.auto_disable_logs(self._dataset(context, dataset_id))
|
||||
|
||||
def set_api_enabled(self, context: RequestContext, *, dataset_id: str, status: str) -> None:
|
||||
self._operations.set_api_enabled(context, self._dataset(context, dataset_id), status == "enable")
|
||||
|
||||
def api_base_url(self, context: RequestContext, *, request_base_url: str) -> str:
|
||||
return normalize_api_base_url(self._service_api_url or request_base_url)
|
||||
|
||||
def retrieval_settings(
|
||||
self, context: RequestContext, *, vector_type: str | None = None, is_mock: bool = False
|
||||
) -> dict[str, list[str]]:
|
||||
return retrieval_methods(
|
||||
vector_type if is_mock else self._vector_store, is_mock=is_mock, tidb_fulltext=self._tidb_fulltext
|
||||
)
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Supported dataset retrieval methods for each vector store."""
|
||||
|
||||
from core.rag.datasource.vdb.vector_type import VectorType
|
||||
from core.rag.retrieval.retrieval_methods import RetrievalMethod
|
||||
|
||||
|
||||
def retrieval_methods(
|
||||
vector_type: str | None, *, is_mock: bool = False, tidb_fulltext: bool = False
|
||||
) -> dict[str, list[str]]:
|
||||
"""
|
||||
Get supported retrieval methods based on vector database type.
|
||||
|
||||
Args:
|
||||
vector_type: Vector database type, can be None
|
||||
is_mock: Whether this is a Mock API, affects MILVUS handling
|
||||
tidb_fulltext: Whether TiDB full-text retrieval is enabled
|
||||
|
||||
Returns:
|
||||
Dictionary containing supported retrieval methods
|
||||
|
||||
Raises:
|
||||
ValueError: If vector_type is None or unsupported
|
||||
"""
|
||||
if vector_type is None:
|
||||
raise ValueError("Vector store type is not configured.")
|
||||
|
||||
# Define vector database types that only support semantic search
|
||||
semantic_only_types = {
|
||||
VectorType.RELYT,
|
||||
VectorType.CHROMA,
|
||||
VectorType.PGVECTO_RS,
|
||||
VectorType.VIKINGDB,
|
||||
VectorType.UPSTASH,
|
||||
}
|
||||
|
||||
# Define vector database types that support all retrieval methods
|
||||
full_search_types = {
|
||||
VectorType.QDRANT,
|
||||
VectorType.WEAVIATE,
|
||||
VectorType.OPENSEARCH,
|
||||
VectorType.ANALYTICDB,
|
||||
VectorType.MYSCALE,
|
||||
VectorType.ORACLE,
|
||||
VectorType.ELASTICSEARCH,
|
||||
VectorType.ELASTICSEARCH_JA,
|
||||
VectorType.PGVECTOR,
|
||||
VectorType.VASTBASE,
|
||||
VectorType.TIDB_ON_QDRANT,
|
||||
VectorType.LINDORM,
|
||||
VectorType.COUCHBASE,
|
||||
VectorType.OPENGAUSS,
|
||||
VectorType.OCEANBASE,
|
||||
VectorType.SEEKDB,
|
||||
VectorType.TABLESTORE,
|
||||
VectorType.HUAWEI_CLOUD,
|
||||
VectorType.TENCENT,
|
||||
VectorType.MATRIXONE,
|
||||
VectorType.CLICKZETTA,
|
||||
VectorType.BAIDU,
|
||||
VectorType.ALIBABACLOUD_MYSQL,
|
||||
VectorType.IRIS,
|
||||
VectorType.HOLOGRES,
|
||||
}
|
||||
|
||||
semantic_methods = {"retrieval_method": [RetrievalMethod.SEMANTIC_SEARCH.value]}
|
||||
full_methods = {
|
||||
"retrieval_method": [
|
||||
RetrievalMethod.SEMANTIC_SEARCH.value,
|
||||
RetrievalMethod.FULL_TEXT_SEARCH.value,
|
||||
RetrievalMethod.HYBRID_SEARCH.value,
|
||||
]
|
||||
}
|
||||
|
||||
if vector_type == VectorType.MILVUS:
|
||||
return semantic_methods if is_mock else full_methods
|
||||
|
||||
if vector_type == VectorType.TIDB_VECTOR:
|
||||
return full_methods if tidb_fulltext else semantic_methods
|
||||
|
||||
if vector_type in semantic_only_types:
|
||||
return semantic_methods
|
||||
elif vector_type in full_search_types:
|
||||
return full_methods
|
||||
else:
|
||||
raise ValueError(f"Unsupported vector db type {vector_type}.")
|
||||
@@ -0,0 +1,206 @@
|
||||
"""Classic-to-Agent knowledge base upgrade use cases behind the dataset console endpoints."""
|
||||
|
||||
from collections.abc import Callable
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from graphon.model_runtime.entities.model_entities import ModelType
|
||||
from machinery.context import RequestContext
|
||||
from models import Dataset
|
||||
from models.knowledge_fs import KnowledgeFSUpgradeJob, KnowledgeFSUpgradeJobStatus
|
||||
from models.provider import TenantDefaultModel
|
||||
from services.dataset_knowledge_fs_upgrade_service import (
|
||||
KnowledgeFSUpgradeConflictError,
|
||||
KnowledgeFSUpgradeModelPreferences,
|
||||
KnowledgeFSUpgradeModelResolver,
|
||||
KnowledgeFSUpgradeNotFoundError,
|
||||
KnowledgeFSUpgradeSnapshotService,
|
||||
upgrade_discovery_response,
|
||||
upgrade_job_matches_model_choices,
|
||||
)
|
||||
from services.knowledge.dataset_access import DatasetAccess, DatasetAccessError
|
||||
from services.knowledge.dataset_service import DatasetService
|
||||
from services.knowledge.datasets.adapters import SQLAlchemyDatasetOperations, load_actor
|
||||
from services.knowledge_fs.product_dto import (
|
||||
KnowledgeFSUpgradeModelChoice,
|
||||
KnowledgeFSUpgradeModelsPayload,
|
||||
KnowledgeFSUpgradePreparationResponse,
|
||||
)
|
||||
|
||||
_RECOVERABLE_STATUSES = (
|
||||
KnowledgeFSUpgradeJobStatus.QUEUED,
|
||||
KnowledgeFSUpgradeJobStatus.RUNNING,
|
||||
KnowledgeFSUpgradeJobStatus.FAILED,
|
||||
)
|
||||
|
||||
|
||||
class KnowledgeFSUpgradeUnavailableError(Exception):
|
||||
"""The KnowledgeFS feature is disabled, so upgrades cannot start or retry."""
|
||||
|
||||
|
||||
class KnowledgeUpgradeApplicationService:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
session_factory: sessionmaker[Session],
|
||||
dataset_access: DatasetAccess,
|
||||
datasets: SQLAlchemyDatasetOperations,
|
||||
rbac_enabled: bool,
|
||||
feature_enabled: Callable[[], bool],
|
||||
enqueue: Callable[[KnowledgeFSUpgradeSnapshotService, str, str], None] | None = None,
|
||||
) -> None:
|
||||
self._sessions = session_factory
|
||||
self._dataset_access = dataset_access
|
||||
self._datasets = datasets
|
||||
self._rbac_enabled = rbac_enabled
|
||||
self._feature_enabled = feature_enabled
|
||||
self._enqueue = enqueue or enqueue_upgrade_job
|
||||
|
||||
def _snapshots(self) -> KnowledgeFSUpgradeSnapshotService:
|
||||
return KnowledgeFSUpgradeSnapshotService(self._sessions)
|
||||
|
||||
@staticmethod
|
||||
def _choice(provider: str | None, model: str | None) -> KnowledgeFSUpgradeModelChoice | None:
|
||||
if not provider or not model:
|
||||
return None
|
||||
return KnowledgeFSUpgradeModelChoice(provider=provider, model=model)
|
||||
|
||||
def _load_model_preferences(
|
||||
self, *, tenant_id: str, dataset_id: str
|
||||
) -> tuple[str, KnowledgeFSUpgradeModelPreferences]:
|
||||
"""Read the legacy settings and saved defaults in a short, tenant-scoped session."""
|
||||
with self._sessions() as session:
|
||||
dataset = session.scalar(select(Dataset).where(Dataset.id == dataset_id, Dataset.tenant_id == tenant_id))
|
||||
if dataset is None:
|
||||
raise KnowledgeFSUpgradeNotFoundError("Dataset was not found")
|
||||
defaults: dict[ModelType, KnowledgeFSUpgradeModelChoice] = {}
|
||||
for row in session.scalars(
|
||||
select(TenantDefaultModel).where(
|
||||
TenantDefaultModel.tenant_id == tenant_id,
|
||||
TenantDefaultModel.model_type.in_((ModelType.TEXT_EMBEDDING, ModelType.RERANK, ModelType.LLM)),
|
||||
)
|
||||
):
|
||||
choice = self._choice(row.provider_name, row.model_name)
|
||||
if choice is not None:
|
||||
defaults[row.model_type] = choice
|
||||
retrieval = dataset.retrieval_model if isinstance(dataset.retrieval_model, dict) else {}
|
||||
reranking = retrieval.get("reranking_model") or {}
|
||||
if not isinstance(reranking, dict):
|
||||
reranking = {}
|
||||
preferences = KnowledgeFSUpgradeModelPreferences(
|
||||
embedding=self._choice(dataset.embedding_model_provider, dataset.embedding_model),
|
||||
rerank=self._choice(reranking.get("reranking_provider_name"), reranking.get("reranking_model_name")),
|
||||
defaults=defaults,
|
||||
)
|
||||
return str(dataset.provider or ""), preferences
|
||||
|
||||
def list_recoverable_jobs(self, context: RequestContext) -> list[KnowledgeFSUpgradeJob]:
|
||||
workspace_id = context.active_workspace_id
|
||||
jobs = self._snapshots().list_by_statuses(tenant_id=workspace_id, statuses=_RECOVERABLE_STATUSES)
|
||||
dataset_ids = list(dict.fromkeys(job.old_dataset_id for job in jobs))
|
||||
if not dataset_ids:
|
||||
return []
|
||||
accessible_ids = self._datasets.visibility(context).list_scope(rbac_enabled=self._rbac_enabled)
|
||||
with self._sessions() as session:
|
||||
datasets, _ = DatasetService.get_datasets_by_ids(
|
||||
dataset_ids,
|
||||
workspace_id,
|
||||
user=load_actor(session, context),
|
||||
accessible_dataset_ids=accessible_ids,
|
||||
session=session,
|
||||
)
|
||||
visible_ids = [str(dataset.id) for dataset in datasets]
|
||||
allowed_ids: set[str] = set()
|
||||
for dataset_id in visible_ids:
|
||||
try:
|
||||
self._dataset_access.require_accessible(context, dataset_id)
|
||||
except DatasetAccessError:
|
||||
continue
|
||||
allowed_ids.add(dataset_id)
|
||||
return [job for job in jobs if job.old_dataset_id in allowed_ids]
|
||||
|
||||
def discovery(self, context: RequestContext, *, dataset_id: str) -> KnowledgeFSUpgradePreparationResponse:
|
||||
dataset = self._dataset_access.require_accessible(context, dataset_id)
|
||||
with self._sessions() as session:
|
||||
provider = session.scalar(
|
||||
select(Dataset.provider).where(Dataset.id == dataset.id, Dataset.tenant_id == dataset.workspace_id)
|
||||
)
|
||||
job = self._snapshots().get_latest(tenant_id=dataset.workspace_id, dataset_id=dataset.id)
|
||||
eligibility = upgrade_discovery_response(
|
||||
job, feature_enabled=self._feature_enabled(), dataset_provider=str(provider or "")
|
||||
)
|
||||
recommendations = None
|
||||
if eligibility.can_upgrade:
|
||||
_, preferences = self._load_model_preferences(tenant_id=dataset.workspace_id, dataset_id=dataset.id)
|
||||
recommendations = KnowledgeFSUpgradeModelResolver(tenant_id=dataset.workspace_id).recommend(preferences)
|
||||
return KnowledgeFSUpgradePreparationResponse.model_validate(
|
||||
{**eligibility.model_dump(mode="json"), "models": recommendations}
|
||||
)
|
||||
|
||||
def start(
|
||||
self,
|
||||
context: RequestContext,
|
||||
*,
|
||||
dataset_id: str,
|
||||
idempotency_key: str,
|
||||
models: KnowledgeFSUpgradeModelsPayload | None = None,
|
||||
) -> KnowledgeFSUpgradeJob:
|
||||
if not self._feature_enabled():
|
||||
raise KnowledgeFSUpgradeUnavailableError()
|
||||
dataset = self._dataset_access.require_accessible(context, dataset_id)
|
||||
snapshots = self._snapshots()
|
||||
idempotent = snapshots.get_by_idempotency_key(tenant_id=dataset.workspace_id, idempotency_key=idempotency_key)
|
||||
if idempotent is not None:
|
||||
if idempotent.old_dataset_id != dataset.id:
|
||||
raise KnowledgeFSUpgradeConflictError("Idempotency key belongs to another Dataset upgrade")
|
||||
if models is not None and not upgrade_job_matches_model_choices(idempotent, models):
|
||||
raise KnowledgeFSUpgradeConflictError("Idempotency key belongs to another model selection")
|
||||
self._enqueue(snapshots, dataset.workspace_id, idempotent.id)
|
||||
return idempotent
|
||||
provider, preferences = self._load_model_preferences(tenant_id=dataset.workspace_id, dataset_id=dataset.id)
|
||||
if provider == "external":
|
||||
raise KnowledgeFSUpgradeConflictError("External-provider Datasets cannot be upgraded")
|
||||
resolver = KnowledgeFSUpgradeModelResolver(tenant_id=dataset.workspace_id)
|
||||
resolved_models = (
|
||||
resolver.validate(models) if models is not None else resolver.resolve_recommendations(preferences)
|
||||
)
|
||||
job = snapshots.create(
|
||||
tenant_id=dataset.workspace_id,
|
||||
dataset_id=dataset.id,
|
||||
requested_by_account_id=context.account_id,
|
||||
idempotency_key=idempotency_key,
|
||||
models=resolved_models,
|
||||
)
|
||||
self._enqueue(snapshots, dataset.workspace_id, job.id)
|
||||
return job
|
||||
|
||||
def get_job(self, context: RequestContext, *, dataset_id: str, job_id: str) -> KnowledgeFSUpgradeJob:
|
||||
dataset = self._dataset_access.require_accessible(context, dataset_id)
|
||||
job = self._snapshots().get(tenant_id=dataset.workspace_id, job_id=job_id)
|
||||
if job.old_dataset_id != dataset.id:
|
||||
raise KnowledgeFSUpgradeNotFoundError("Upgrade job was not found")
|
||||
return job
|
||||
|
||||
def retry(self, context: RequestContext, *, dataset_id: str, job_id: str) -> KnowledgeFSUpgradeJob:
|
||||
if not self._feature_enabled():
|
||||
raise KnowledgeFSUpgradeUnavailableError()
|
||||
dataset = self._dataset_access.require_accessible(context, dataset_id)
|
||||
snapshots = self._snapshots()
|
||||
job = snapshots.retry(tenant_id=dataset.workspace_id, dataset_id=dataset.id, job_id=job_id)
|
||||
self._enqueue(snapshots, dataset.workspace_id, job.id)
|
||||
return job
|
||||
|
||||
|
||||
def enqueue_upgrade_job(snapshots: KnowledgeFSUpgradeSnapshotService, tenant_id: str, job_id: str) -> None:
|
||||
from tasks.knowledge_fs_upgrade_tasks import run_knowledge_fs_upgrade
|
||||
|
||||
task_id = str(uuid4())
|
||||
if not snapshots.claim_enqueue(tenant_id=tenant_id, job_id=job_id, task_id=task_id):
|
||||
return
|
||||
try:
|
||||
run_knowledge_fs_upgrade.apply_async(kwargs={"job_id": job_id}, task_id=task_id)
|
||||
except Exception:
|
||||
snapshots.release_enqueue_claim(tenant_id=tenant_id, job_id=job_id, task_id=task_id)
|
||||
raise
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Request-side document synchronization use cases."""
|
||||
|
||||
from typing import NamedTuple, Protocol
|
||||
|
||||
from machinery.context import RequestContext
|
||||
from services.knowledge.dataset_access import DatasetAccess
|
||||
from services.knowledge.resource_scope import DatasetRef, DocumentRef
|
||||
|
||||
|
||||
class SyncDocumentRecord(NamedTuple):
|
||||
"""Document state required to authorize and dispatch a sync request."""
|
||||
|
||||
id: str
|
||||
data_source_type: str
|
||||
|
||||
|
||||
class DocumentSyncReader(Protocol):
|
||||
def get_sync_document(self, document_ref: DocumentRef) -> SyncDocumentRecord | None: ...
|
||||
|
||||
def list_active_notion_refs(self, dataset_ref: DatasetRef) -> tuple[DocumentRef, ...]: ...
|
||||
|
||||
|
||||
class DocumentSyncDispatcher(Protocol):
|
||||
def dispatch(self, *, dataset_id: str, document_id: str) -> None: ...
|
||||
|
||||
|
||||
class DocumentSyncError(Exception):
|
||||
"""Base class for document synchronization failures."""
|
||||
|
||||
|
||||
class SyncDocumentNotFoundError(DocumentSyncError):
|
||||
def __init__(self) -> None:
|
||||
super().__init__("Document not found")
|
||||
|
||||
|
||||
class SyncDocumentSourceError(DocumentSyncError):
|
||||
def __init__(self) -> None:
|
||||
super().__init__("Document is not notion type")
|
||||
|
||||
|
||||
class DocumentSyncApplicationService:
|
||||
"""Authorize synchronization requests and dispatch the worker use case."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
dataset_access: DatasetAccess,
|
||||
documents: DocumentSyncReader,
|
||||
dispatcher: DocumentSyncDispatcher,
|
||||
) -> None:
|
||||
self._dataset_access = dataset_access
|
||||
self._documents = documents
|
||||
self._dispatcher = dispatcher
|
||||
|
||||
def sync_dataset(self, context: RequestContext, dataset_id: str) -> int:
|
||||
dataset = self._dataset_access.require_accessible(context, dataset_id)
|
||||
dataset_ref = DatasetRef(dataset.workspace_id, dataset.id)
|
||||
document_refs = self._documents.list_active_notion_refs(dataset_ref)
|
||||
for document_ref in document_refs:
|
||||
self._dispatcher.dispatch(dataset_id=dataset_ref.dataset_id, document_id=document_ref.document_id)
|
||||
return len(document_refs)
|
||||
|
||||
def sync_document(self, context: RequestContext, dataset_id: str, document_id: str) -> None:
|
||||
dataset = self._dataset_access.require_accessible(context, dataset_id)
|
||||
document_ref = DatasetRef(dataset.workspace_id, dataset.id).document(document_id)
|
||||
document = self._documents.get_sync_document(document_ref)
|
||||
if document is None:
|
||||
raise SyncDocumentNotFoundError()
|
||||
if document.data_source_type != "notion_import":
|
||||
raise SyncDocumentSourceError()
|
||||
self._dispatcher.dispatch(dataset_id=dataset.id, document_id=document.id)
|
||||
@@ -0,0 +1,11 @@
|
||||
"""Infrastructure adapters for document-sync requests."""
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
|
||||
class CeleryDocumentSyncDispatcher:
|
||||
def __init__(self, *, delay: Callable[[str, str], object]) -> None:
|
||||
self._delay = delay
|
||||
|
||||
def dispatch(self, *, dataset_id: str, document_id: str) -> None:
|
||||
self._delay(dataset_id, document_id)
|
||||
@@ -0,0 +1 @@
|
||||
"""Document application use cases and their infrastructure adapters."""
|
||||
@@ -0,0 +1,579 @@
|
||||
"""Session-owning adapters for document queries and existing document operations."""
|
||||
|
||||
import copy
|
||||
import json
|
||||
from collections.abc import Generator, Mapping, Sequence
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from typing import Any, Literal
|
||||
from uuid import uuid4
|
||||
|
||||
from redis.exceptions import LockNotOwnedError
|
||||
from sqlalchemy import asc, desc, func, select, update
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
from werkzeug.exceptions import NotFound
|
||||
|
||||
from configs import dify_config
|
||||
from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
|
||||
from core.model_manager import ModelManager
|
||||
from core.rag.index_processor.constant.built_in_field import BuiltInField
|
||||
from core.rag.index_processor.constant.index_type import IndexTechniqueType
|
||||
from extensions.ext_redis import redis_client
|
||||
from graphon.model_runtime.entities.model_entities import ModelType
|
||||
from graphon.model_runtime.errors.invoke import InvokeAuthorizationError
|
||||
from libs.pagination import paginate_query
|
||||
from machinery.context import RequestContext
|
||||
from models import Account, Dataset, Document, DocumentSegment, UploadFile
|
||||
from models.dataset import DatasetPermissionEnum, DocumentPipelineExecutionLog
|
||||
from models.enums import IndexingStatus
|
||||
from repositories.knowledge.dataset_read_repository import (
|
||||
get_dataset_doc_form,
|
||||
get_document_hit_count,
|
||||
get_document_process_rule,
|
||||
get_document_segment_count,
|
||||
get_latest_dataset_process_rule,
|
||||
)
|
||||
from repositories.knowledge.dataset_repository import _get_dataset
|
||||
from repositories.knowledge.document_repository import _get_document
|
||||
from services.enterprise import rbac_service
|
||||
from services.errors.base import NoPermissionError
|
||||
from services.errors.document import DocumentIndexingError
|
||||
from services.file_service import FileService
|
||||
from services.knowledge.dataset_access import DatasetAccessDeniedError, DatasetNotFoundError
|
||||
from services.knowledge.dataset_read_service import (
|
||||
get_document_metadata_details,
|
||||
get_document_source_detail,
|
||||
load_document_detail,
|
||||
load_document_details,
|
||||
)
|
||||
from services.knowledge.dataset_service import DatasetService, DocumentService
|
||||
from services.knowledge.documents.application import (
|
||||
DocumentIndexingStateError,
|
||||
DocumentInvalidActionError,
|
||||
DocumentListFilter,
|
||||
DocumentNotFoundError,
|
||||
DocumentProviderError,
|
||||
DocumentState,
|
||||
DocumentZip,
|
||||
)
|
||||
from services.knowledge.entities.document_creation import DocumentCreationDataset, DocumentIndexingJobs
|
||||
from services.knowledge.entities.knowledge_entities import KnowledgeConfig
|
||||
from services.knowledge.resource_scope import DatasetRef, DocumentRef
|
||||
from services.knowledge.summaries.adapters import SummaryIndexAdapter
|
||||
from services.rbac import contracts as rbac_contracts
|
||||
from services.vector_space_admission_service import get_vector_space_admission_error_fields
|
||||
from tasks.generate_summary_index_task import generate_summary_index_task
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _translate_errors() -> Generator[None]:
|
||||
try:
|
||||
yield
|
||||
except NotFound as error:
|
||||
raise DocumentNotFoundError(error.description) from error
|
||||
except NoPermissionError as error:
|
||||
raise DatasetAccessDeniedError() from error
|
||||
except DocumentIndexingError as error:
|
||||
raise DocumentIndexingStateError(str(error)) from error
|
||||
except (ProviderTokenNotInitError, InvokeAuthorizationError) as error:
|
||||
raise DocumentProviderError(str(error), kind="uninitialized") from error
|
||||
except QuotaExceededError as error:
|
||||
raise DocumentProviderError(str(error), kind="quota") from error
|
||||
except ModelCurrentlyNotSupportError as error:
|
||||
raise DocumentProviderError(str(error), kind="unsupported") from error
|
||||
|
||||
|
||||
def _require_dataset(session: Session, ref: DatasetRef) -> Dataset:
|
||||
dataset = _get_dataset(session, ref)
|
||||
if dataset is None:
|
||||
raise DatasetNotFoundError()
|
||||
return dataset
|
||||
|
||||
|
||||
def _require_document(session: Session, ref: DocumentRef) -> Document:
|
||||
document = _get_document(session, ref)
|
||||
if document is None:
|
||||
raise DocumentNotFoundError("Document not found.")
|
||||
return document
|
||||
|
||||
|
||||
def _state(document: Document) -> DocumentState:
|
||||
return DocumentState(
|
||||
id=document.id,
|
||||
archived=bool(document.archived),
|
||||
indexing_status=str(document.indexing_status),
|
||||
is_paused=bool(document.is_paused),
|
||||
data_source_type=str(document.data_source_type),
|
||||
doc_form=str(document.doc_form),
|
||||
need_summary=bool(document.need_summary),
|
||||
)
|
||||
|
||||
|
||||
def _created_response(dataset: Dataset, documents: Sequence[Document], batch: str, session: Session) -> dict[str, Any]:
|
||||
return {
|
||||
"dataset": {
|
||||
"id": dataset.id,
|
||||
"name": dataset.name,
|
||||
"description": dataset.description,
|
||||
"permission": dataset.permission,
|
||||
"data_source_type": dataset.data_source_type,
|
||||
"indexing_technique": dataset.indexing_technique,
|
||||
"created_by": dataset.created_by,
|
||||
"created_at": dataset.created_at,
|
||||
},
|
||||
"documents": load_document_details(documents, session=session),
|
||||
"batch": batch,
|
||||
}
|
||||
|
||||
|
||||
def _indexing_status(document: Document, counts: tuple[int, int]) -> dict[str, Any]:
|
||||
return {
|
||||
"id": document.id,
|
||||
"processing_started_at": document.processing_started_at,
|
||||
"parsing_completed_at": document.parsing_completed_at,
|
||||
"cleaning_completed_at": document.cleaning_completed_at,
|
||||
"splitting_completed_at": document.splitting_completed_at,
|
||||
"completed_at": document.completed_at,
|
||||
"paused_at": document.paused_at,
|
||||
"error": document.error,
|
||||
"stopped_at": document.stopped_at,
|
||||
"indexing_status": IndexingStatus.PAUSED if document.is_paused else document.indexing_status,
|
||||
**get_vector_space_admission_error_fields(document.error),
|
||||
"completed_segments": counts[0],
|
||||
"total_segments": counts[1],
|
||||
}
|
||||
|
||||
|
||||
class SQLAlchemyDocumentOperations:
|
||||
def __init__(self, *, session_factory: sessionmaker[Session]) -> None:
|
||||
self._sessions = session_factory
|
||||
|
||||
def find_document(self, *, workspace_id: str, document_id: str) -> DocumentRef | None:
|
||||
with self._sessions() as session:
|
||||
document = session.scalar(
|
||||
select(Document).where(Document.id == document_id, Document.tenant_id == workspace_id)
|
||||
)
|
||||
return DatasetRef(workspace_id, document.dataset_id).document(document.id) if document else None
|
||||
|
||||
def get_state(self, ref: DocumentRef) -> DocumentState | None:
|
||||
with self._sessions() as session:
|
||||
document = _get_document(session, ref)
|
||||
return _state(document) if document else None
|
||||
|
||||
def get_states(self, ref: DatasetRef, document_ids: Sequence[str]) -> Sequence[DocumentState]:
|
||||
with self._sessions() as session:
|
||||
return tuple(
|
||||
_state(document) for document in DocumentService.get_documents_by_ids(ref, document_ids, session)
|
||||
)
|
||||
|
||||
def get_process_rule(self, ref: DatasetRef | None) -> dict[str, Any]:
|
||||
result = copy.deepcopy(DocumentService.DEFAULT_RULES)
|
||||
if ref is not None:
|
||||
with self._sessions() as session:
|
||||
rule = get_latest_dataset_process_rule(_require_dataset(session, ref), session=session)
|
||||
if rule is not None:
|
||||
result.update(mode=rule.mode, rules=rule.rules_dict)
|
||||
return result
|
||||
|
||||
def list_documents(self, ref: DatasetRef, query: DocumentListFilter) -> dict[str, Any]:
|
||||
with self._sessions() as session:
|
||||
dataset = _require_dataset(session, ref)
|
||||
statement = select(Document).where(
|
||||
Document.dataset_id == ref.dataset_id, Document.tenant_id == ref.tenant_id
|
||||
)
|
||||
statement = DocumentService.apply_display_status_filter(statement, query.status)
|
||||
if query.search:
|
||||
statement = statement.where(Document.name.like(f"%{query.search}%"))
|
||||
direction = desc if query.sort.startswith("-") else asc
|
||||
sort = query.sort.removeprefix("-")
|
||||
if sort == "hit_count":
|
||||
hits = (
|
||||
select(DocumentSegment.document_id, func.sum(DocumentSegment.hit_count).label("total_hit_count"))
|
||||
.where(DocumentSegment.dataset_id == ref.dataset_id, DocumentSegment.tenant_id == ref.tenant_id)
|
||||
.group_by(DocumentSegment.document_id)
|
||||
.subquery()
|
||||
)
|
||||
statement = statement.outerjoin(hits, hits.c.document_id == Document.id).order_by(
|
||||
direction(func.coalesce(hits.c.total_hit_count, 0)), direction(Document.position)
|
||||
)
|
||||
elif sort == "created_at":
|
||||
statement = statement.order_by(direction(Document.created_at), direction(Document.position))
|
||||
else:
|
||||
statement = statement.order_by(desc(Document.created_at), desc(Document.position))
|
||||
page = paginate_query(statement, session=session, page=query.page, per_page=query.limit, max_per_page=100)
|
||||
DocumentService.enrich_documents_with_summary_index_status(
|
||||
documents=page.items, dataset=dataset, tenant_id=ref.tenant_id, session=session
|
||||
)
|
||||
if query.fetch:
|
||||
counts = DocumentService.get_document_segment_counts(page.items, session=session)
|
||||
for document in page.items:
|
||||
document.completed_segments, document.total_segments = counts.get(document.id, (0, 0))
|
||||
return {
|
||||
"data": load_document_details(page.items, session=session),
|
||||
"has_more": page.has_next,
|
||||
"limit": page.per_page,
|
||||
"total": page.total,
|
||||
"page": page.page,
|
||||
}
|
||||
|
||||
def get_detail(self, ref: DocumentRef, *, metadata_only: bool) -> dict[str, Any]:
|
||||
with self._sessions() as session:
|
||||
document = _require_document(session, ref)
|
||||
result: dict[str, Any] = {
|
||||
"id": document.id,
|
||||
"doc_type": document.doc_type,
|
||||
"doc_metadata": get_document_metadata_details(document, session=session),
|
||||
}
|
||||
if metadata_only:
|
||||
return result
|
||||
rule = get_document_process_rule(document, session=session)
|
||||
segment_count = get_document_segment_count(document, session=session)
|
||||
result.update(
|
||||
{
|
||||
"position": document.position,
|
||||
"data_source_type": document.data_source_type,
|
||||
"dataset_process_rule_id": document.dataset_process_rule_id,
|
||||
"name": document.name,
|
||||
"created_from": document.created_from,
|
||||
"created_by": document.created_by,
|
||||
"tokens": document.tokens,
|
||||
"indexing_status": document.indexing_status,
|
||||
"indexing_latency": document.indexing_latency,
|
||||
"error": document.error,
|
||||
"enabled": document.enabled,
|
||||
"disabled_by": document.disabled_by,
|
||||
"archived": document.archived,
|
||||
"display_status": document.display_status,
|
||||
"doc_form": document.doc_form,
|
||||
"doc_language": document.doc_language,
|
||||
}
|
||||
)
|
||||
result.update(
|
||||
{
|
||||
"created_at": int(document.created_at.timestamp()) if document.created_at else None,
|
||||
"completed_at": int(document.completed_at.timestamp()) if document.completed_at else None,
|
||||
"updated_at": int(document.updated_at.timestamp()) if document.updated_at else None,
|
||||
"disabled_at": int(document.disabled_at.timestamp()) if document.disabled_at else None,
|
||||
}
|
||||
)
|
||||
result.update(
|
||||
data_source_info=document.data_source_info_dict,
|
||||
data_source_detail_dict=get_document_source_detail(document, session=session),
|
||||
dataset_process_rule=DatasetService.get_process_rules(ref.dataset.dataset_id, session),
|
||||
document_process_rule=dict(rule.to_dict()) if rule else None,
|
||||
segment_count=segment_count,
|
||||
average_segment_length=(document.word_count or 0) // segment_count if segment_count else 0,
|
||||
hit_count=get_document_hit_count(document, session=session),
|
||||
need_summary=bool(document.need_summary),
|
||||
)
|
||||
return result
|
||||
|
||||
def get_indexing_status(self, ref: DocumentRef) -> dict[str, Any]:
|
||||
with self._sessions() as session:
|
||||
document = _require_document(session, ref)
|
||||
counts = DocumentService.get_document_segment_counts([document], session=session)
|
||||
return _indexing_status(document, counts.get(document.id, (0, 0)))
|
||||
|
||||
def get_batch_indexing_status(self, ref: DatasetRef, batch: str) -> dict[str, Any]:
|
||||
with self._sessions() as session:
|
||||
documents = session.scalars(
|
||||
select(Document).where(
|
||||
Document.tenant_id == ref.tenant_id, Document.dataset_id == ref.dataset_id, Document.batch == batch
|
||||
)
|
||||
).all()
|
||||
if not documents:
|
||||
raise DocumentNotFoundError("Documents not found.")
|
||||
counts = DocumentService.get_document_segment_counts(documents, session=session)
|
||||
return {"data": [_indexing_status(document, counts.get(document.id, (0, 0))) for document in documents]}
|
||||
|
||||
def get_execution_log(self, ref: DocumentRef) -> dict[str, Any]:
|
||||
with self._sessions() as session:
|
||||
_require_document(session, ref)
|
||||
log = session.scalar(
|
||||
select(DocumentPipelineExecutionLog)
|
||||
.where(DocumentPipelineExecutionLog.document_id == ref.document_id)
|
||||
.order_by(DocumentPipelineExecutionLog.created_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
if log is None:
|
||||
return {}
|
||||
return {
|
||||
"datasource_info": json.loads(log.datasource_info),
|
||||
"datasource_type": log.datasource_type,
|
||||
"input_data": log.input_data,
|
||||
"datasource_node_id": log.datasource_node_id,
|
||||
}
|
||||
|
||||
def get_summary_status(self, ref: DocumentRef) -> dict[str, Any]:
|
||||
with self._sessions() as session:
|
||||
_require_document(session, ref)
|
||||
return dict(
|
||||
SummaryIndexAdapter.get_document_summary_status_detail(
|
||||
tenant_id=ref.dataset.tenant_id,
|
||||
document_id=ref.document_id,
|
||||
dataset_id=ref.dataset.dataset_id,
|
||||
session=session,
|
||||
)
|
||||
)
|
||||
|
||||
def get_download_url(self, ref: DocumentRef) -> str:
|
||||
with _translate_errors(), self._sessions() as session:
|
||||
return DocumentService.get_document_download_url(_require_document(session, ref), session)
|
||||
|
||||
@contextmanager
|
||||
def build_download_zip(self, ref: DatasetRef, document_ids: Sequence[str]) -> Generator[DocumentZip]:
|
||||
with _translate_errors(), self._sessions() as session:
|
||||
files_by_document = DocumentService._get_upload_files_by_document_id_for_zip_download(
|
||||
dataset_id=ref.dataset_id, document_ids=document_ids, tenant_id=ref.tenant_id, session=session
|
||||
)
|
||||
files = [files_by_document[document_id] for document_id in document_ids]
|
||||
filename = DocumentService._generate_document_batch_download_zip_filename()
|
||||
# The read transaction is closed before reading storage or streaming the response.
|
||||
with FileService.build_upload_files_zip_tempfile(upload_files=files) as path:
|
||||
yield DocumentZip(path=path, filename=filename)
|
||||
|
||||
@staticmethod
|
||||
def _actor(session: Session, context: RequestContext) -> Account:
|
||||
account = session.get(Account, context.account_id)
|
||||
if account is None:
|
||||
raise DatasetAccessDeniedError()
|
||||
account.set_tenant_id_with_session(context.active_workspace_id, session=session)
|
||||
return account
|
||||
|
||||
def _creation_dataset(
|
||||
self, context: RequestContext, ref: DatasetRef, config: KnowledgeConfig
|
||||
) -> DocumentCreationDataset:
|
||||
with self._sessions() as session:
|
||||
dataset = _require_dataset(session, ref)
|
||||
self._actor(session, context)
|
||||
DatasetService.check_doc_form(dataset, config.doc_form, session=session)
|
||||
return DocumentCreationDataset(
|
||||
dataset.tenant_id, dataset.indexing_technique, dataset.embedding_model, dataset.embedding_model_provider
|
||||
)
|
||||
|
||||
def _persist_creation(
|
||||
self, context: RequestContext, ref: DatasetRef, config: KnowledgeConfig, *, initialize: bool
|
||||
) -> tuple[dict[str, Any], DocumentIndexingJobs]:
|
||||
"""Keep ORM values local and materialize the response before closing the session."""
|
||||
jobs: DocumentIndexingJobs | None = None
|
||||
try:
|
||||
with self._sessions() as session:
|
||||
account = self._actor(session, context)
|
||||
dataset = (
|
||||
DocumentService.build_dataset_for_documents(ref, config, account, session)
|
||||
if initialize
|
||||
else _require_dataset(session, ref)
|
||||
)
|
||||
documents, batch, jobs = DocumentService.save_prepared_documents(
|
||||
dataset, config, account, session=session
|
||||
)
|
||||
if initialize:
|
||||
dataset.name = documents[0].name[:18] + "..."
|
||||
dataset.description = "useful for when you want to answer queries about the " + documents[0].name
|
||||
dataset.permission = (
|
||||
DatasetPermissionEnum.ALL_TEAM if dify_config.RBAC_ENABLED else DatasetPermissionEnum.ONLY_ME
|
||||
)
|
||||
session.flush()
|
||||
result = _created_response(dataset, documents, batch, session)
|
||||
session.commit()
|
||||
return result, jobs
|
||||
except Exception:
|
||||
# The persistence helper commits documents before loading response
|
||||
# details. Preserve their indexing even if that later read fails.
|
||||
if jobs is not None:
|
||||
DocumentService.dispatch_document_indexing(jobs)
|
||||
raise
|
||||
|
||||
def _run_creation(
|
||||
self, context: RequestContext, ref: DatasetRef, config: KnowledgeConfig, *, initialize: bool
|
||||
) -> dict[str, Any]:
|
||||
lock = (
|
||||
nullcontext()
|
||||
if config.original_document_id
|
||||
else redis_client.lock(f"add_document_lock_dataset_id_{ref.dataset_id}", timeout=600)
|
||||
)
|
||||
result = None
|
||||
try:
|
||||
with lock:
|
||||
result, jobs = self._persist_creation(context, ref, config, initialize=initialize)
|
||||
# No session or ORM entity crosses the task boundary. Keep the
|
||||
# creation lock until committed documents have been dispatched.
|
||||
DocumentService.dispatch_document_indexing(jobs)
|
||||
except LockNotOwnedError:
|
||||
if config.original_document_id:
|
||||
raise
|
||||
# Preserve successful creation when the Redis lease expires on release.
|
||||
if result is None:
|
||||
if initialize:
|
||||
raise
|
||||
with self._sessions() as session:
|
||||
result = _created_response(
|
||||
_require_dataset(session, ref), [], DocumentService.generate_document_batch(), session
|
||||
)
|
||||
assert result is not None
|
||||
return result
|
||||
|
||||
def create_documents(self, context: RequestContext, ref: DatasetRef, settings: Mapping[str, Any]) -> dict[str, Any]:
|
||||
config = KnowledgeConfig.model_validate(settings)
|
||||
DocumentService.document_create_args_validate(config)
|
||||
with _translate_errors():
|
||||
snapshot = self._creation_dataset(context, ref, config)
|
||||
config = DocumentService.prepare_document_creation(snapshot, config)
|
||||
return self._run_creation(context, ref, config, initialize=False)
|
||||
|
||||
def initialize_dataset(self, context: RequestContext, settings: Mapping[str, Any]) -> dict[str, Any]:
|
||||
config = KnowledgeConfig.model_validate(settings)
|
||||
with _translate_errors():
|
||||
if config.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
|
||||
if config.embedding_model is None or config.embedding_model_provider is None:
|
||||
raise ValueError(
|
||||
"embedding model and embedding model provider are required for high quality indexing."
|
||||
)
|
||||
ModelManager.for_tenant(tenant_id=context.active_workspace_id).get_model_instance(
|
||||
tenant_id=context.active_workspace_id,
|
||||
provider=config.embedding_model_provider,
|
||||
model_type=ModelType.TEXT_EMBEDDING,
|
||||
model=config.embedding_model,
|
||||
)
|
||||
config = config.model_copy(
|
||||
update={
|
||||
"is_multimodal": DatasetService.check_is_multimodal_model(
|
||||
context.active_workspace_id, config.embedding_model_provider, config.embedding_model
|
||||
)
|
||||
}
|
||||
)
|
||||
DocumentService.document_create_args_validate(config)
|
||||
with self._sessions() as session:
|
||||
self._actor(session, context)
|
||||
config = DocumentService.prepare_document_creation(
|
||||
DocumentCreationDataset(
|
||||
context.active_workspace_id,
|
||||
config.indexing_technique,
|
||||
config.embedding_model,
|
||||
config.embedding_model_provider,
|
||||
),
|
||||
config,
|
||||
)
|
||||
dataset_id = str(uuid4())
|
||||
result = self._run_creation(
|
||||
context, DatasetRef(context.active_workspace_id, dataset_id), config, initialize=True
|
||||
)
|
||||
if dify_config.RBAC_ENABLED:
|
||||
rbac_service.RBACService.DatasetAccess.replace_whitelist(
|
||||
context.active_workspace_id,
|
||||
context.account_id,
|
||||
dataset_id,
|
||||
rbac_contracts.ReplaceMemberBindings(automatic_include_workspace_members=False),
|
||||
)
|
||||
rbac_service.try_sync_creator_access_policy_member_bindings(
|
||||
context.active_workspace_id,
|
||||
context.account_id,
|
||||
rbac_contracts.RBACResourceType.DATASET,
|
||||
dataset_id,
|
||||
)
|
||||
return result
|
||||
|
||||
def _check_model(self, ref: DatasetRef) -> None:
|
||||
with self._sessions() as session:
|
||||
dataset = _require_dataset(session, ref)
|
||||
DatasetService.check_dataset_model_setting(dataset)
|
||||
|
||||
def delete_document(self, ref: DocumentRef) -> None:
|
||||
self._check_model(ref.dataset)
|
||||
with _translate_errors(), self._sessions() as session:
|
||||
DocumentService.delete_document(_require_document(session, ref), session)
|
||||
|
||||
def delete_documents(self, ref: DatasetRef, document_ids: Sequence[str]) -> None:
|
||||
with _translate_errors(), self._sessions() as session:
|
||||
dataset = _require_dataset(session, ref)
|
||||
DocumentService.delete_documents(
|
||||
ref, list(document_ids), get_dataset_doc_form(dataset, session=session), session
|
||||
)
|
||||
|
||||
def update_document(self, ref: DocumentRef, values: Mapping[str, object]) -> None:
|
||||
with self._sessions.begin() as session:
|
||||
document = _require_document(session, ref)
|
||||
for name, value in values.items():
|
||||
setattr(document, name, value)
|
||||
|
||||
def rename_document(self, ref: DocumentRef, name: str) -> dict[str, Any]:
|
||||
with self._sessions.begin() as session:
|
||||
dataset = _require_dataset(session, ref.dataset)
|
||||
document = _require_document(session, ref)
|
||||
if dataset.built_in_field_enabled and document.doc_metadata:
|
||||
document.doc_metadata = {**document.doc_metadata, BuiltInField.document_name: name}
|
||||
document.name = name
|
||||
if document.data_source_info_dict and (file_id := document.data_source_info_dict.get("upload_file_id")):
|
||||
session.execute(
|
||||
update(UploadFile)
|
||||
.where(UploadFile.id == file_id, UploadFile.tenant_id == ref.dataset.tenant_id)
|
||||
.values(name=name)
|
||||
)
|
||||
session.flush()
|
||||
return load_document_detail(document, session=session)
|
||||
|
||||
def pause_document(self, ref: DocumentRef, *, actor_id: str) -> None:
|
||||
with _translate_errors(), self._sessions() as session:
|
||||
DocumentService.pause_document(_require_document(session, ref), session, actor_id=actor_id)
|
||||
|
||||
def recover_document(self, ref: DocumentRef) -> None:
|
||||
with _translate_errors(), self._sessions() as session:
|
||||
DocumentService.recover_document(_require_document(session, ref), session)
|
||||
|
||||
def retry_documents(self, ref: DatasetRef, document_ids: Sequence[str], *, actor_id: str) -> None:
|
||||
with _translate_errors(), self._sessions() as session:
|
||||
documents = DocumentService.get_documents_by_ids(ref, document_ids, session)
|
||||
DocumentService.retry_document(ref.dataset_id, list(documents), session, actor_id=actor_id)
|
||||
|
||||
def change_status(
|
||||
self,
|
||||
context: RequestContext,
|
||||
ref: DatasetRef,
|
||||
document_ids: Sequence[str],
|
||||
action: Literal["enable", "disable", "archive", "un_archive"],
|
||||
) -> None:
|
||||
self._check_model(ref)
|
||||
with _translate_errors(), self._sessions() as session:
|
||||
dataset = _require_dataset(session, ref)
|
||||
account = self._actor(session, context)
|
||||
documents = DocumentService.get_documents_by_ids(ref, document_ids, session)
|
||||
try:
|
||||
DocumentService.batch_update_document_status(
|
||||
dataset, [doc.id for doc in documents], action, account, session
|
||||
)
|
||||
except ValueError as error:
|
||||
raise DocumentInvalidActionError(str(error)) from error
|
||||
|
||||
def sync_website(self, ref: DocumentRef) -> None:
|
||||
with _translate_errors(), self._sessions() as session:
|
||||
DocumentService.sync_website_document(
|
||||
_require_dataset(session, ref.dataset), _require_document(session, ref), session
|
||||
)
|
||||
|
||||
def require_summary_enabled(self, ref: DatasetRef) -> None:
|
||||
with self._sessions() as session:
|
||||
dataset = _require_dataset(session, ref)
|
||||
if dataset.indexing_technique != IndexTechniqueType.HIGH_QUALITY:
|
||||
raise ValueError(
|
||||
"Summary generation is only available for 'high_quality' indexing technique. "
|
||||
f"Current indexing technique: {dataset.indexing_technique}"
|
||||
)
|
||||
if not dataset.summary_index_setting or not dataset.summary_index_setting.get("enable"):
|
||||
raise ValueError(
|
||||
"Summary index is not enabled for this dataset. Please enable it in the dataset settings."
|
||||
)
|
||||
|
||||
def enable_summary(self, ref: DatasetRef, document_ids: Sequence[str]) -> None:
|
||||
if document_ids:
|
||||
with self._sessions.begin() as session:
|
||||
session.execute(
|
||||
update(Document)
|
||||
.where(
|
||||
Document.tenant_id == ref.tenant_id,
|
||||
Document.dataset_id == ref.dataset_id,
|
||||
Document.id.in_(document_ids),
|
||||
)
|
||||
.values(need_summary=True)
|
||||
)
|
||||
|
||||
def dispatch_summary(self, ref: DocumentRef) -> None:
|
||||
generate_summary_index_task.delay(ref.dataset.dataset_id, ref.document_id)
|
||||
@@ -0,0 +1,310 @@
|
||||
"""Console document use cases, independent of HTTP and persistence frameworks."""
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from contextlib import AbstractContextManager
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Literal, Protocol
|
||||
|
||||
from libs.datetime_utils import naive_utc_now
|
||||
from machinery.context import RequestContext
|
||||
from services.knowledge.dataset_access import DatasetAccess
|
||||
from services.knowledge.resource_scope import DatasetRef, DocumentRef
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DocumentListFilter:
|
||||
page: int = 1
|
||||
limit: int = 20
|
||||
search: str = ""
|
||||
sort: str = "-created_at"
|
||||
status: str | None = None
|
||||
fetch: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DocumentState:
|
||||
id: str
|
||||
archived: bool
|
||||
indexing_status: str
|
||||
is_paused: bool
|
||||
data_source_type: str
|
||||
doc_form: str
|
||||
need_summary: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DocumentZip:
|
||||
path: str
|
||||
filename: str
|
||||
|
||||
|
||||
class DocumentApplicationError(Exception):
|
||||
"""Base class for document use-case failures mapped by the transport."""
|
||||
|
||||
|
||||
class DocumentNotFoundError(DocumentApplicationError):
|
||||
pass
|
||||
|
||||
|
||||
class DocumentArchivedError(DocumentApplicationError):
|
||||
pass
|
||||
|
||||
|
||||
class DocumentIndexingStateError(DocumentApplicationError):
|
||||
pass
|
||||
|
||||
|
||||
class DocumentInvalidActionError(DocumentApplicationError):
|
||||
pass
|
||||
|
||||
|
||||
class DocumentProviderError(DocumentApplicationError):
|
||||
def __init__(self, description: str, *, kind: Literal["uninitialized", "quota", "unsupported"]) -> None:
|
||||
super().__init__(description)
|
||||
self.kind = kind
|
||||
|
||||
|
||||
class DocumentOperations(Protocol):
|
||||
"""Owned document operations; adapters return materialized values and own their sessions."""
|
||||
|
||||
def find_document(self, *, workspace_id: str, document_id: str) -> DocumentRef | None: ...
|
||||
def get_state(self, ref: DocumentRef) -> DocumentState | None: ...
|
||||
def get_states(self, ref: DatasetRef, document_ids: Sequence[str]) -> Sequence[DocumentState]: ...
|
||||
def list_documents(self, ref: DatasetRef, query: DocumentListFilter) -> dict[str, Any]: ...
|
||||
def get_detail(self, ref: DocumentRef, *, metadata_only: bool) -> dict[str, Any]: ...
|
||||
def get_process_rule(self, ref: DatasetRef | None) -> dict[str, Any]: ...
|
||||
def get_indexing_status(self, ref: DocumentRef) -> dict[str, Any]: ...
|
||||
def get_batch_indexing_status(self, ref: DatasetRef, batch: str) -> dict[str, Any]: ...
|
||||
def get_execution_log(self, ref: DocumentRef) -> dict[str, Any]: ...
|
||||
def get_summary_status(self, ref: DocumentRef) -> dict[str, Any]: ...
|
||||
def get_download_url(self, ref: DocumentRef) -> str: ...
|
||||
def build_download_zip(
|
||||
self, ref: DatasetRef, document_ids: Sequence[str]
|
||||
) -> AbstractContextManager[DocumentZip]: ...
|
||||
def create_documents(
|
||||
self, context: RequestContext, ref: DatasetRef, settings: Mapping[str, Any]
|
||||
) -> dict[str, Any]: ...
|
||||
def initialize_dataset(self, context: RequestContext, settings: Mapping[str, Any]) -> dict[str, Any]: ...
|
||||
def delete_document(self, ref: DocumentRef) -> None: ...
|
||||
def delete_documents(self, ref: DatasetRef, document_ids: Sequence[str]) -> None: ...
|
||||
def update_document(self, ref: DocumentRef, values: Mapping[str, object]) -> None: ...
|
||||
def rename_document(self, ref: DocumentRef, name: str) -> dict[str, Any]: ...
|
||||
def pause_document(self, ref: DocumentRef, *, actor_id: str) -> None: ...
|
||||
def recover_document(self, ref: DocumentRef) -> None: ...
|
||||
def retry_documents(self, ref: DatasetRef, document_ids: Sequence[str], *, actor_id: str) -> None: ...
|
||||
def change_status(
|
||||
self,
|
||||
context: RequestContext,
|
||||
ref: DatasetRef,
|
||||
document_ids: Sequence[str],
|
||||
action: Literal["enable", "disable", "archive", "un_archive"],
|
||||
) -> None: ...
|
||||
def sync_website(self, ref: DocumentRef) -> None: ...
|
||||
def require_summary_enabled(self, ref: DatasetRef) -> None: ...
|
||||
def enable_summary(self, ref: DatasetRef, document_ids: Sequence[str]) -> None: ...
|
||||
def dispatch_summary(self, ref: DocumentRef) -> None: ...
|
||||
|
||||
|
||||
class DatasetDocumentApplicationService:
|
||||
"""Authorize document use cases and apply document state transitions."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
dataset_access: DatasetAccess,
|
||||
operations: DocumentOperations,
|
||||
metadata_schema: Mapping[str, Mapping[str, type]],
|
||||
) -> None:
|
||||
self._dataset_access = dataset_access
|
||||
self._operations = operations
|
||||
self._metadata_schema = metadata_schema
|
||||
|
||||
def _dataset(self, context: RequestContext, dataset_id: str) -> DatasetRef:
|
||||
dataset = self._dataset_access.require_accessible(context, dataset_id)
|
||||
return DatasetRef(tenant_id=dataset.workspace_id, dataset_id=dataset.id)
|
||||
|
||||
def _document(
|
||||
self, context: RequestContext, dataset_id: str, document_id: str
|
||||
) -> tuple[DocumentRef, DocumentState]:
|
||||
ref = self._dataset(context, dataset_id).document(document_id)
|
||||
state = self._operations.get_state(ref)
|
||||
if state is None:
|
||||
raise DocumentNotFoundError("Document not found.")
|
||||
return ref, state
|
||||
|
||||
@staticmethod
|
||||
def _require_mutable(state: DocumentState) -> None:
|
||||
if state.archived:
|
||||
raise DocumentArchivedError()
|
||||
|
||||
def get_process_rule(self, context: RequestContext, *, document_id: str | None) -> dict[str, Any]:
|
||||
ref = None
|
||||
if document_id:
|
||||
document_ref = self._operations.find_document(
|
||||
workspace_id=context.active_workspace_id, document_id=document_id
|
||||
)
|
||||
if document_ref is None:
|
||||
raise DocumentNotFoundError("Document not found.")
|
||||
ref = self._dataset(context, document_ref.dataset.dataset_id)
|
||||
return self._operations.get_process_rule(ref)
|
||||
|
||||
def list_documents(self, context: RequestContext, *, dataset_id: str, query: DocumentListFilter) -> dict[str, Any]:
|
||||
return self._operations.list_documents(self._dataset(context, dataset_id), query)
|
||||
|
||||
def get_document(
|
||||
self, context: RequestContext, *, dataset_id: str, document_id: str, metadata_only: bool = False
|
||||
) -> dict[str, Any]:
|
||||
ref, _ = self._document(context, dataset_id, document_id)
|
||||
return self._operations.get_detail(ref, metadata_only=metadata_only)
|
||||
|
||||
def get_indexing_status(self, context: RequestContext, *, dataset_id: str, document_id: str) -> dict[str, Any]:
|
||||
ref, _ = self._document(context, dataset_id, document_id)
|
||||
return self._operations.get_indexing_status(ref)
|
||||
|
||||
def get_batch_indexing_status(self, context: RequestContext, *, dataset_id: str, batch: str) -> dict[str, Any]:
|
||||
return self._operations.get_batch_indexing_status(self._dataset(context, dataset_id), batch)
|
||||
|
||||
def get_execution_log(self, context: RequestContext, *, dataset_id: str, document_id: str) -> dict[str, Any]:
|
||||
ref, _ = self._document(context, dataset_id, document_id)
|
||||
return self._operations.get_execution_log(ref)
|
||||
|
||||
def get_summary_status(self, context: RequestContext, *, dataset_id: str, document_id: str) -> dict[str, Any]:
|
||||
ref, _ = self._document(context, dataset_id, document_id)
|
||||
return self._operations.get_summary_status(ref)
|
||||
|
||||
def get_download_url(self, context: RequestContext, *, dataset_id: str, document_id: str) -> str:
|
||||
ref, _ = self._document(context, dataset_id, document_id)
|
||||
return self._operations.get_download_url(ref)
|
||||
|
||||
def build_download_zip(
|
||||
self, context: RequestContext, *, dataset_id: str, document_ids: Sequence[str]
|
||||
) -> AbstractContextManager[DocumentZip]:
|
||||
return self._operations.build_download_zip(self._dataset(context, dataset_id), document_ids)
|
||||
|
||||
def create_documents(
|
||||
self, context: RequestContext, *, dataset_id: str, settings: Mapping[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
return self._operations.create_documents(context, self._dataset(context, dataset_id), settings)
|
||||
|
||||
def initialize_dataset(self, context: RequestContext, *, settings: Mapping[str, Any]) -> dict[str, Any]:
|
||||
return self._operations.initialize_dataset(context, settings)
|
||||
|
||||
def delete_document(self, context: RequestContext, *, dataset_id: str, document_id: str) -> None:
|
||||
ref, _ = self._document(context, dataset_id, document_id)
|
||||
self._operations.delete_document(ref)
|
||||
|
||||
def delete_documents(self, context: RequestContext, *, dataset_id: str, document_ids: Sequence[str]) -> None:
|
||||
self._operations.delete_documents(self._dataset(context, dataset_id), document_ids)
|
||||
|
||||
def update_processing(
|
||||
self,
|
||||
context: RequestContext,
|
||||
*,
|
||||
dataset_id: str,
|
||||
document_id: str,
|
||||
action: Literal["pause", "resume"],
|
||||
) -> None:
|
||||
ref, state = self._document(context, dataset_id, document_id)
|
||||
if action == "pause":
|
||||
if state.indexing_status != "indexing":
|
||||
raise DocumentInvalidActionError("Document not in indexing state.")
|
||||
values: dict[str, object] = {
|
||||
"paused_by": context.account_id,
|
||||
"paused_at": naive_utc_now(),
|
||||
"is_paused": True,
|
||||
}
|
||||
elif action == "resume":
|
||||
if state.indexing_status not in {"paused", "error"}:
|
||||
raise DocumentInvalidActionError("Document not in paused or error state.")
|
||||
values = {"paused_by": None, "paused_at": None, "is_paused": False}
|
||||
else:
|
||||
raise DocumentInvalidActionError("Invalid action.")
|
||||
self._operations.update_document(ref, values)
|
||||
|
||||
def update_metadata(
|
||||
self,
|
||||
context: RequestContext,
|
||||
*,
|
||||
dataset_id: str,
|
||||
document_id: str,
|
||||
doc_type: str | None,
|
||||
doc_metadata: dict[str, Any] | None,
|
||||
) -> None:
|
||||
ref, _ = self._document(context, dataset_id, document_id)
|
||||
if doc_type is None or doc_metadata is None:
|
||||
raise ValueError("Both doc_type and doc_metadata must be provided.")
|
||||
if doc_type not in self._metadata_schema:
|
||||
raise ValueError("Invalid doc_type.")
|
||||
metadata = (
|
||||
doc_metadata
|
||||
if doc_type == "others"
|
||||
else {
|
||||
key: doc_metadata[key]
|
||||
for key, value_type in self._metadata_schema[doc_type].items()
|
||||
if doc_metadata.get(key) is not None and isinstance(doc_metadata[key], value_type)
|
||||
}
|
||||
)
|
||||
self._operations.update_document(
|
||||
ref, {"doc_type": doc_type, "doc_metadata": metadata, "updated_at": naive_utc_now()}
|
||||
)
|
||||
|
||||
def change_status(
|
||||
self,
|
||||
context: RequestContext,
|
||||
*,
|
||||
dataset_id: str,
|
||||
document_ids: Sequence[str],
|
||||
action: Literal["enable", "disable", "archive", "un_archive"],
|
||||
) -> None:
|
||||
self._operations.change_status(context, self._dataset(context, dataset_id), document_ids, action)
|
||||
|
||||
def pause_document(self, context: RequestContext, *, dataset_id: str, document_id: str) -> None:
|
||||
ref, state = self._document(context, dataset_id, document_id)
|
||||
self._require_mutable(state)
|
||||
if state.indexing_status not in {"waiting", "parsing", "cleaning", "splitting", "indexing"}:
|
||||
raise DocumentIndexingStateError("Cannot pause completed document.")
|
||||
self._operations.pause_document(ref, actor_id=context.account_id)
|
||||
|
||||
def recover_document(self, context: RequestContext, *, dataset_id: str, document_id: str) -> None:
|
||||
ref, state = self._document(context, dataset_id, document_id)
|
||||
self._require_mutable(state)
|
||||
if not state.is_paused:
|
||||
raise DocumentIndexingStateError("Document is not in paused status.")
|
||||
self._operations.recover_document(ref)
|
||||
|
||||
def retry_documents(self, context: RequestContext, *, dataset_id: str, document_ids: Sequence[str]) -> None:
|
||||
ref = self._dataset(context, dataset_id)
|
||||
states = {state.id: state for state in self._operations.get_states(ref, document_ids)}
|
||||
retry_ids = [
|
||||
document_id
|
||||
for document_id in dict.fromkeys(document_ids)
|
||||
if (state := states.get(document_id)) is not None
|
||||
and not state.archived
|
||||
and state.indexing_status != "completed"
|
||||
]
|
||||
self._operations.retry_documents(ref, retry_ids, actor_id=context.account_id)
|
||||
|
||||
def rename_document(
|
||||
self, context: RequestContext, *, dataset_id: str, document_id: str, name: str
|
||||
) -> dict[str, Any]:
|
||||
ref, _ = self._document(context, dataset_id, document_id)
|
||||
return self._operations.rename_document(ref, name)
|
||||
|
||||
def sync_website(self, context: RequestContext, *, dataset_id: str, document_id: str) -> None:
|
||||
ref, state = self._document(context, dataset_id, document_id)
|
||||
if state.data_source_type != "website_crawl":
|
||||
raise ValueError("Document is not a website document.")
|
||||
self._require_mutable(state)
|
||||
self._operations.sync_website(ref)
|
||||
|
||||
def generate_summary(self, context: RequestContext, *, dataset_id: str, document_ids: Sequence[str]) -> None:
|
||||
ref = self._dataset(context, dataset_id)
|
||||
self._operations.require_summary_enabled(ref)
|
||||
states = self._operations.get_states(ref, document_ids)
|
||||
if len(states) != len(document_ids):
|
||||
missing_ids = set(document_ids) - {state.id for state in states}
|
||||
raise DocumentNotFoundError(f"Some documents not found: {sorted(missing_ids)}")
|
||||
targets = [state for state in states if state.doc_form != "qa_model"]
|
||||
self._operations.enable_summary(ref, [state.id for state in targets if not state.need_summary])
|
||||
for state in targets:
|
||||
self._operations.dispatch_summary(ref.document(state.id))
|
||||
@@ -0,0 +1 @@
|
||||
"""Shared data contracts for knowledge use cases."""
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Materialized dataset read contracts, independent of persistence and HTTP.
|
||||
|
||||
Timestamps remain datetimes until transport serialization. JSON configuration
|
||||
objects retain their stored keys; this boundary does not apply response defaults.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import datetime
|
||||
from typing import NamedTuple, NotRequired, TypedDict
|
||||
|
||||
from pydantic import JsonValue
|
||||
|
||||
|
||||
class DatasetTagRecord(TypedDict):
|
||||
id: str
|
||||
name: str
|
||||
type: str
|
||||
|
||||
|
||||
class DatasetDetailRecord(TypedDict):
|
||||
id: str
|
||||
name: str
|
||||
description: str | None
|
||||
provider: str
|
||||
permission: str
|
||||
data_source_type: str | None
|
||||
indexing_technique: str | None
|
||||
keyword_number: int | None
|
||||
app_count: int
|
||||
document_count: int
|
||||
word_count: int
|
||||
created_by: str
|
||||
author_name: str | None
|
||||
created_at: datetime
|
||||
updated_by: str | None
|
||||
updated_at: datetime
|
||||
embedding_model: str | None
|
||||
embedding_model_provider: str | None
|
||||
# None means availability has not been evaluated by the application service.
|
||||
embedding_available: bool | None
|
||||
retrieval_model_dict: Mapping[str, JsonValue]
|
||||
summary_index_setting: Mapping[str, JsonValue] | None
|
||||
tags: list[DatasetTagRecord]
|
||||
doc_form: str | None
|
||||
external_knowledge_info: Mapping[str, JsonValue] | None
|
||||
external_retrieval_model: Mapping[str, JsonValue] | None
|
||||
doc_metadata: Sequence[Mapping[str, str]]
|
||||
built_in_field_enabled: bool
|
||||
pipeline_id: str | None
|
||||
runtime_mode: str | None
|
||||
chunk_structure: str | None
|
||||
icon_info: Mapping[str, JsonValue] | None
|
||||
is_published: bool
|
||||
total_documents: int
|
||||
total_available_documents: int
|
||||
enable_api: bool
|
||||
is_multimodal: bool
|
||||
permission_keys: list[str]
|
||||
maintainer: str | None
|
||||
# Detail/create responses may omit membership or explicitly return null;
|
||||
# listing and updates materialize a list, including an empty one.
|
||||
partial_member_list: NotRequired[list[str] | None]
|
||||
# Listing attaches Classic-to-Agent knowledge base upgrade availability.
|
||||
knowledge_fs_upgrade: NotRequired[Mapping[str, JsonValue]]
|
||||
|
||||
|
||||
class DatasetPage(TypedDict):
|
||||
data: list[DatasetDetailRecord]
|
||||
has_more: bool
|
||||
total: int
|
||||
page: int
|
||||
limit: int
|
||||
|
||||
|
||||
class DatasetCleanupRecord(NamedTuple):
|
||||
"""Dataset state needed to schedule index cleanup after the row is deleted."""
|
||||
|
||||
id: str
|
||||
workspace_id: str
|
||||
indexing_technique: str | None
|
||||
index_struct: str | None
|
||||
collection_binding_id: str | None
|
||||
# None means neither the dataset nor any of its documents declares a form.
|
||||
doc_form: str | None
|
||||
pipeline_id: str | None
|
||||
@@ -0,0 +1,22 @@
|
||||
"""Owned values passed between document creation's database and I/O phases."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from services.knowledge.resource_scope import DatasetRef
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DocumentCreationDataset:
|
||||
tenant_id: str
|
||||
indexing_technique: str | None
|
||||
embedding_model: str | None
|
||||
embedding_model_provider: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DocumentIndexingJobs:
|
||||
dataset: DatasetRef
|
||||
created: tuple[str, ...] = ()
|
||||
duplicated: tuple[str, ...] = ()
|
||||
removed_notion: tuple[str, ...] = ()
|
||||
updated: str | None = None
|
||||
@@ -0,0 +1,159 @@
|
||||
"""Commands and validation shared by knowledge indexing-estimate use cases."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Annotated, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator
|
||||
|
||||
from core.rag.entities import PreProcessingRuleKey
|
||||
from services.data_source.entities.notion_import import NotionPageType
|
||||
|
||||
|
||||
class EstimateValidationError(ValueError):
|
||||
"""Raised when an estimate process rule is invalid."""
|
||||
|
||||
|
||||
class _PreProcessingRule(BaseModel):
|
||||
id: PreProcessingRuleKey
|
||||
enabled: bool
|
||||
|
||||
|
||||
class _Segmentation(BaseModel):
|
||||
separator: str = Field(min_length=1)
|
||||
max_tokens: int = Field(gt=0)
|
||||
|
||||
|
||||
class _Rules(BaseModel):
|
||||
pre_processing_rules: list[_PreProcessingRule]
|
||||
segmentation: _Segmentation
|
||||
|
||||
@field_validator("pre_processing_rules")
|
||||
@classmethod
|
||||
def _deduplicate(cls, value: list[_PreProcessingRule]) -> list[_PreProcessingRule]:
|
||||
by_id = {rule.id: rule for rule in value}
|
||||
return list(by_id.values())
|
||||
|
||||
|
||||
class _HierarchicalRules(_Rules):
|
||||
parent_mode: Literal["full-doc", "paragraph"] | None = None
|
||||
subchunk_segmentation: _Segmentation | None = None
|
||||
|
||||
|
||||
class _SummaryIndexSettingDisabled(BaseModel):
|
||||
enable: Literal[False] = False
|
||||
|
||||
|
||||
class _SummaryIndexSettingEnabled(BaseModel):
|
||||
enable: Literal[True]
|
||||
model_name: str = Field(min_length=1)
|
||||
model_provider_name: str = Field(min_length=1)
|
||||
|
||||
|
||||
_SummaryIndexSetting = Annotated[
|
||||
_SummaryIndexSettingDisabled | _SummaryIndexSettingEnabled,
|
||||
Field(discriminator="enable"),
|
||||
]
|
||||
|
||||
|
||||
class _ProcessRuleBase(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
summary_index_setting: _SummaryIndexSetting | None = None
|
||||
|
||||
@field_validator("summary_index_setting", mode="before")
|
||||
@classmethod
|
||||
def _normalize_summary_index_setting(cls, value: object) -> object:
|
||||
"""Treat mappings with a missing/None enable flag as absent (#36602)."""
|
||||
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, Mapping) and value.get("enable") is None:
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
class _AutomaticProcessRule(_ProcessRuleBase):
|
||||
mode: Literal["automatic"]
|
||||
|
||||
|
||||
class _CustomProcessRule(_ProcessRuleBase):
|
||||
mode: Literal["custom"]
|
||||
rules: _Rules
|
||||
|
||||
|
||||
class _HierarchicalProcessRule(_ProcessRuleBase):
|
||||
mode: Literal["hierarchical"]
|
||||
rules: _HierarchicalRules
|
||||
|
||||
|
||||
_ProcessRule = Annotated[
|
||||
_AutomaticProcessRule | _CustomProcessRule | _HierarchicalProcessRule,
|
||||
Field(discriminator="mode"),
|
||||
]
|
||||
|
||||
|
||||
class _ProcessRuleEnvelope(BaseModel):
|
||||
process_rule: _ProcessRule
|
||||
|
||||
|
||||
def _validation_message(error: ValidationError) -> str:
|
||||
first = error.errors()[0]
|
||||
original = first.get("ctx", {}).get("error")
|
||||
return str(original) if isinstance(original, ValueError) else str(first["msg"])
|
||||
|
||||
|
||||
def normalize_process_rule(process_rule: Mapping[str, object]) -> dict[str, object]:
|
||||
"""Validate and normalize a process rule without mutating caller-owned input."""
|
||||
|
||||
try:
|
||||
validated = _ProcessRuleEnvelope.model_validate({"process_rule": process_rule})
|
||||
except ValidationError as error:
|
||||
raise EstimateValidationError(_validation_message(error)) from error
|
||||
|
||||
result = validated.process_rule.model_dump(exclude_none=True)
|
||||
if validated.process_rule.mode == "automatic":
|
||||
result["rules"] = {}
|
||||
elif validated.process_rule.mode == "hierarchical":
|
||||
rules = result.get("rules")
|
||||
if isinstance(rules, dict) and not rules.get("parent_mode"):
|
||||
rules["parent_mode"] = "paragraph"
|
||||
return result
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class UploadFileEstimateSource:
|
||||
file_id: str
|
||||
source_type: Literal["upload_file"] = field(default="upload_file", init=False)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class NotionEstimateSource:
|
||||
workspace_id: str
|
||||
page_id: str
|
||||
page_type: NotionPageType
|
||||
credential_id: str
|
||||
source_type: Literal["notion_import"] = field(default="notion_import", init=False)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class WebsiteEstimateSource:
|
||||
provider: str
|
||||
job_id: str
|
||||
url: str
|
||||
mode: str = "crawl"
|
||||
only_main_content: bool = False
|
||||
source_type: Literal["website_crawl"] = field(default="website_crawl", init=False)
|
||||
|
||||
|
||||
type NewEstimateSource = UploadFileEstimateSource | NotionEstimateSource | WebsiteEstimateSource
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class NewSourcesEstimateCommand:
|
||||
sources: tuple[NewEstimateSource, ...]
|
||||
process_rule: Mapping[str, object]
|
||||
doc_form: str = "text_model"
|
||||
doc_language: str = "English"
|
||||
dataset_id: str | None = None
|
||||
indexing_technique: str = "economy"
|
||||
@@ -0,0 +1,294 @@
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
from pydantic import BaseModel, Field, WithJsonSchema, field_validator
|
||||
|
||||
from core.rag.entities import MetadataFilteringCondition, Rule
|
||||
from core.rag.index_processor.constant.index_type import IndexStructureType
|
||||
from core.rag.retrieval.retrieval_methods import RetrievalMethod
|
||||
from libs.helper import UUIDStr
|
||||
from models.enums import ProcessRuleMode
|
||||
|
||||
DocForm = Annotated[
|
||||
str,
|
||||
WithJsonSchema({"enum": ["text_model", "hierarchical_model", "qa_model"], "type": "string"}),
|
||||
]
|
||||
IndexingTechnique = Annotated[
|
||||
str | None,
|
||||
WithJsonSchema({"anyOf": [{"enum": ["high_quality", "economy"], "type": "string"}, {"type": "null"}]}),
|
||||
]
|
||||
KnowledgeProvider = Annotated[
|
||||
str,
|
||||
WithJsonSchema({"enum": ["vendor", "external"], "type": "string"}),
|
||||
]
|
||||
RerankingMode = Annotated[
|
||||
str | None,
|
||||
WithJsonSchema({"anyOf": [{"enum": ["reranking_model", "weighted_score"], "type": "string"}, {"type": "null"}]}),
|
||||
]
|
||||
SummaryIndexSetting = Annotated[
|
||||
dict[str, Any] | None,
|
||||
WithJsonSchema(
|
||||
{
|
||||
"anyOf": [
|
||||
{
|
||||
"properties": {
|
||||
"enable": {"description": "Whether to enable summary indexing.", "type": "boolean"},
|
||||
"model_name": {
|
||||
"description": "Name of the model used for generating summaries.",
|
||||
"type": "string",
|
||||
},
|
||||
"model_provider_name": {
|
||||
"description": "Provider of the summary generation model.",
|
||||
"type": "string",
|
||||
},
|
||||
"summary_prompt": {
|
||||
"description": "Custom prompt template for summary generation.",
|
||||
"type": "string",
|
||||
},
|
||||
},
|
||||
"type": "object",
|
||||
},
|
||||
{"type": "null"},
|
||||
]
|
||||
}
|
||||
),
|
||||
]
|
||||
ExternalRetrievalModel = Annotated[
|
||||
dict[str, Any] | None,
|
||||
WithJsonSchema(
|
||||
{
|
||||
"anyOf": [
|
||||
{
|
||||
"properties": {
|
||||
"top_k": {"description": "Maximum number of results to return.", "type": "integer"},
|
||||
"score_threshold": {
|
||||
"description": "Minimum similarity score threshold for filtering results.",
|
||||
"type": "number",
|
||||
},
|
||||
"score_threshold_enabled": {
|
||||
"description": "Whether score threshold filtering is enabled.",
|
||||
"type": "boolean",
|
||||
},
|
||||
},
|
||||
"type": "object",
|
||||
},
|
||||
{"type": "null"},
|
||||
]
|
||||
}
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
class RerankingModel(BaseModel):
|
||||
reranking_provider_name: str | None = Field(default=None, description="Provider name of the reranking model.")
|
||||
reranking_model_name: str | None = Field(default=None, description="Name of the reranking model.")
|
||||
|
||||
|
||||
class NotionIcon(BaseModel):
|
||||
type: str
|
||||
url: str | None = None
|
||||
emoji: str | None = None
|
||||
|
||||
|
||||
class NotionPage(BaseModel):
|
||||
page_id: str
|
||||
page_name: str
|
||||
page_icon: NotionIcon | None = None
|
||||
type: str
|
||||
|
||||
|
||||
class NotionInfo(BaseModel):
|
||||
credential_id: str
|
||||
workspace_id: str
|
||||
pages: list[NotionPage]
|
||||
|
||||
|
||||
class WebsiteInfo(BaseModel):
|
||||
provider: str
|
||||
job_id: str
|
||||
urls: list[str]
|
||||
only_main_content: bool = True
|
||||
|
||||
|
||||
class FileInfo(BaseModel):
|
||||
file_ids: list[str]
|
||||
|
||||
|
||||
class InfoList(BaseModel):
|
||||
data_source_type: Literal["upload_file", "notion_import", "website_crawl"]
|
||||
notion_info_list: list[NotionInfo] | None = None
|
||||
file_info_list: FileInfo | None = None
|
||||
website_info_list: WebsiteInfo | None = None
|
||||
|
||||
|
||||
class DataSource(BaseModel):
|
||||
info_list: InfoList
|
||||
|
||||
|
||||
class ProcessRule(BaseModel):
|
||||
mode: ProcessRuleMode = Field(
|
||||
description=(
|
||||
"`automatic` uses built-in rules, `custom` allows manual configuration, `hierarchical` enables "
|
||||
"parent-child chunk structure (use with `doc_form: hierarchical_model`)."
|
||||
)
|
||||
)
|
||||
rules: Rule | None = Field(default=None, description="Custom processing rules.")
|
||||
|
||||
|
||||
class WeightVectorSetting(BaseModel):
|
||||
vector_weight: float = Field(description="Weight assigned to semantic vector search results.")
|
||||
embedding_provider_name: str = Field(description="Provider of the embedding model used for vector search.")
|
||||
embedding_model_name: str = Field(description="Name of the embedding model used for vector search.")
|
||||
|
||||
|
||||
class WeightKeywordSetting(BaseModel):
|
||||
keyword_weight: float = Field(description="Weight assigned to keyword search results.")
|
||||
|
||||
|
||||
class WeightModel(BaseModel):
|
||||
weight_type: Literal["semantic_first", "keyword_first", "customized"] | None = Field(
|
||||
default=None,
|
||||
description="Strategy for balancing semantic and keyword search weights.",
|
||||
)
|
||||
vector_setting: WeightVectorSetting | None = Field(default=None, description="Semantic search weight settings.")
|
||||
keyword_setting: WeightKeywordSetting | None = Field(default=None, description="Keyword search weight settings.")
|
||||
|
||||
|
||||
class RetrievalModel(BaseModel):
|
||||
search_method: RetrievalMethod = Field(description="Search method used for retrieval.")
|
||||
reranking_enable: bool = Field(description="Whether reranking is enabled.")
|
||||
reranking_model: RerankingModel | None = Field(default=None, description="Reranking model configuration.")
|
||||
reranking_mode: RerankingMode = Field(
|
||||
default=None,
|
||||
description="Reranking mode. Required when `reranking_enable` is `true`.",
|
||||
)
|
||||
top_k: int = Field(description="Maximum number of results to return.")
|
||||
score_threshold_enabled: bool = Field(description="Whether score threshold filtering is enabled.")
|
||||
score_threshold: float | None = Field(
|
||||
default=None,
|
||||
description="Minimum similarity score for results. Only effective when score threshold filtering is enabled.",
|
||||
)
|
||||
weights: WeightModel | None = Field(default=None, description="Weight configuration for hybrid search.")
|
||||
metadata_filtering_conditions: MetadataFilteringCondition | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Restrict retrieval to chunks whose document metadata matches the given conditions. Conditions are "
|
||||
"evaluated server-side against document metadata fields."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class MetaDataConfig(BaseModel):
|
||||
doc_type: str
|
||||
doc_metadata: dict[str, Any]
|
||||
|
||||
|
||||
class KnowledgeConfig(BaseModel):
|
||||
original_document_id: str | None = Field(default=None, description="Original document ID for replacement updates.")
|
||||
duplicate: bool = Field(default=True, description="Whether duplicate document content is allowed.")
|
||||
indexing_technique: Literal["high_quality", "economy"] = Field(
|
||||
description=(
|
||||
"`high_quality` uses embedding models for precise search; `economy` uses keyword-based indexing. "
|
||||
"Required when adding the first document to a knowledge base; subsequent documents inherit the "
|
||||
"knowledge base's indexing technique if omitted."
|
||||
)
|
||||
)
|
||||
data_source: DataSource | None = Field(default=None, description="Document data source configuration.")
|
||||
process_rule: ProcessRule | None = Field(default=None, description="Processing rules for chunking.")
|
||||
retrieval_model: RetrievalModel | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Retrieval model configuration. Controls how chunks are searched and ranked in this knowledge base."
|
||||
),
|
||||
)
|
||||
summary_index_setting: SummaryIndexSetting = Field(
|
||||
default=None,
|
||||
description="Summary index configuration.",
|
||||
)
|
||||
doc_form: DocForm = Field(
|
||||
default="text_model",
|
||||
description=(
|
||||
"`text_model` for standard text chunking, `hierarchical_model` for parent-child chunk structure, "
|
||||
"`qa_model` for question-answer pair extraction."
|
||||
),
|
||||
)
|
||||
doc_language: str = Field(default="English", description="Language of the document for processing optimization.")
|
||||
embedding_model: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Embedding model name. Use the `model` field from "
|
||||
"[Get Available Models](/api-reference/models/get-available-models) with `model_type=text-embedding`."
|
||||
),
|
||||
)
|
||||
embedding_model_provider: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Embedding model provider. Use the `provider` field from "
|
||||
"[Get Available Models](/api-reference/models/get-available-models) with `model_type=text-embedding`."
|
||||
),
|
||||
)
|
||||
name: str | None = Field(default=None, description="Document name.")
|
||||
is_multimodal: bool = Field(default=False, description="Whether the document uses multimodal indexing.")
|
||||
|
||||
@field_validator("doc_form")
|
||||
@classmethod
|
||||
def validate_doc_form(cls, value: str) -> str:
|
||||
valid_forms = [
|
||||
IndexStructureType.PARAGRAPH_INDEX,
|
||||
IndexStructureType.QA_INDEX,
|
||||
IndexStructureType.PARENT_CHILD_INDEX,
|
||||
]
|
||||
if value not in valid_forms:
|
||||
raise ValueError("Invalid doc_form.")
|
||||
return value
|
||||
|
||||
|
||||
class SegmentCreateArgs(BaseModel):
|
||||
content: str | None = Field(default=None, description="Chunk text content.")
|
||||
answer: str | None = Field(default=None, description="Answer content for QA mode.")
|
||||
keywords: list[str] | None = Field(default=None, description="Keywords for the chunk.")
|
||||
attachment_ids: list[str] | None = Field(default=None, description="Attachment file IDs.")
|
||||
|
||||
|
||||
class MetadataArgs(BaseModel):
|
||||
type: Literal["string", "number", "time"] = Field(
|
||||
description="`string` for text values, `number` for numeric values, `time` for date/time values."
|
||||
)
|
||||
name: str = Field(description="Metadata field name.")
|
||||
|
||||
|
||||
class MetadataUpdateArgs(BaseModel):
|
||||
name: str = Field(description="Metadata field name.")
|
||||
value: str | int | float | None = Field(
|
||||
default=None,
|
||||
description="Metadata value. Can be a string, number, or `null`.",
|
||||
)
|
||||
|
||||
|
||||
class MetadataDetail(BaseModel):
|
||||
id: UUIDStr = Field(description="Metadata field ID.")
|
||||
name: str = Field(description="Metadata field name.")
|
||||
value: str | int | float | None = Field(
|
||||
default=None,
|
||||
description="Metadata value. Can be a string, number, or `null`.",
|
||||
)
|
||||
|
||||
|
||||
class DocumentMetadataOperation(BaseModel):
|
||||
document_id: UUIDStr = Field(description="Document ID whose metadata should be updated.")
|
||||
metadata_list: list[MetadataDetail] = Field(description="Metadata fields to update.")
|
||||
partial_update: bool = Field(
|
||||
default=False,
|
||||
description="Whether to partially update metadata, keeping existing values for unspecified fields.",
|
||||
)
|
||||
|
||||
|
||||
class MetadataOperationData(BaseModel):
|
||||
"""
|
||||
Metadata operation data
|
||||
"""
|
||||
|
||||
operation_data: list[DocumentMetadataOperation] = Field(
|
||||
description=(
|
||||
"Array of document metadata update operations. Each entry maps a document ID to its metadata values."
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Detached knowledge segment values shared by application ports."""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
|
||||
class ChildChunkUpdateArgs(BaseModel):
|
||||
id: str | None = Field(default=None, description="Existing child chunk ID. Omit to create a new child chunk.")
|
||||
content: str = Field(description="Child chunk text content.")
|
||||
|
||||
|
||||
class SegmentAttachmentRecord(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True, frozen=True)
|
||||
|
||||
id: str
|
||||
name: str
|
||||
size: int
|
||||
extension: str
|
||||
mime_type: str | None
|
||||
source_url: str
|
||||
|
||||
|
||||
class ChildChunkRecord(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True, frozen=True)
|
||||
|
||||
id: str
|
||||
segment_id: str
|
||||
content: str
|
||||
position: int
|
||||
word_count: int
|
||||
type: str
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
class SegmentRecord(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True, frozen=True)
|
||||
|
||||
id: str
|
||||
position: int
|
||||
document_id: str
|
||||
content: str
|
||||
sign_content: str
|
||||
answer: str | None
|
||||
word_count: int
|
||||
tokens: int
|
||||
keywords: tuple[str, ...] | None
|
||||
index_node_id: str | None
|
||||
index_node_hash: str | None
|
||||
hit_count: int
|
||||
enabled: bool
|
||||
disabled_at: datetime | None
|
||||
disabled_by: str | None
|
||||
status: str
|
||||
created_by: str
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
updated_by: str | None
|
||||
indexing_at: datetime | None
|
||||
completed_at: datetime | None
|
||||
error: str | None
|
||||
stopped_at: datetime | None
|
||||
child_chunks: tuple[ChildChunkRecord, ...]
|
||||
attachments: tuple[SegmentAttachmentRecord, ...]
|
||||
summary: str | None
|
||||
|
||||
|
||||
class SegmentUpdateArgs(BaseModel):
|
||||
content: str | None = Field(default=None, description="Updated chunk text content.")
|
||||
answer: str | None = Field(default=None, description="Updated answer content for QA mode.")
|
||||
keywords: list[str] | None = Field(default=None, description="Updated keywords for the chunk.")
|
||||
regenerate_child_chunks: bool = Field(
|
||||
default=False,
|
||||
description="Whether to regenerate child chunks after updating a parent chunk.",
|
||||
)
|
||||
enabled: bool | None = Field(default=None, description="Whether the chunk is enabled.")
|
||||
attachment_ids: list[str] | None = Field(default=None, description="Attachment file IDs.")
|
||||
summary: str | None = Field(default=None, description="Summary content for summary index.")
|
||||
@@ -0,0 +1 @@
|
||||
"""External knowledge connection and retrieval use cases."""
|
||||
@@ -0,0 +1,155 @@
|
||||
"""External knowledge persistence and network adapters with bounded sessions."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from core.rag.entities import MetadataFilteringCondition
|
||||
from libs.pagination import clamp_pagination
|
||||
from machinery.context import RequestContext
|
||||
from models.dataset import DatasetQuery, ExternalKnowledgeApis
|
||||
from models.enums import CreatorUserRole, DatasetQuerySource
|
||||
from repositories.knowledge.dataset_read_repository import (
|
||||
get_external_api_bindings_batch,
|
||||
get_external_api_dataset_bindings,
|
||||
)
|
||||
from services.entities.external_knowledge_entities.external_knowledge_entities import ExternalDatasetCreatePayload
|
||||
from services.hit_testing_service import HitTestingService
|
||||
from services.knowledge.dataset_read_service import load_dataset_detail
|
||||
from services.knowledge.datasets.adapters import require_dataset
|
||||
from services.knowledge.entities.datasets import DatasetDetailRecord
|
||||
from services.knowledge.external.application import ExternalTemplateNotFoundError
|
||||
from services.knowledge.external.service import ExternalDatasetService
|
||||
from services.knowledge.resource_scope import DatasetRef
|
||||
from services.rbac.contracts import RBACResourceType
|
||||
from services.rbac.members import MemberService
|
||||
|
||||
|
||||
class SQLAlchemyExternalKnowledgeOperations:
|
||||
def __init__(self, *, session_factory: sessionmaker[Session], members: MemberService) -> None:
|
||||
self._sessions = session_factory
|
||||
self._members = members
|
||||
|
||||
@staticmethod
|
||||
def _template(session: Session, workspace_id: str, template_id: str) -> ExternalKnowledgeApis:
|
||||
template = session.scalar(
|
||||
select(ExternalKnowledgeApis).where(
|
||||
ExternalKnowledgeApis.id == template_id, ExternalKnowledgeApis.tenant_id == workspace_id
|
||||
)
|
||||
)
|
||||
if template is None:
|
||||
raise ExternalTemplateNotFoundError("API template not found.")
|
||||
return template
|
||||
|
||||
@staticmethod
|
||||
def _values(session: Session, template: ExternalKnowledgeApis) -> dict[str, Any]:
|
||||
return dict(template.to_dict(dataset_bindings=get_external_api_dataset_bindings(template, session=session)))
|
||||
|
||||
def list_templates(self, workspace_id: str, *, page: int, limit: int, keyword: str | None) -> dict[str, Any]:
|
||||
page, limit = clamp_pagination(page, limit, 100)
|
||||
with self._sessions() as session:
|
||||
templates, total = ExternalDatasetService.get_external_knowledge_apis(
|
||||
page, limit, workspace_id, keyword, session=session
|
||||
)
|
||||
bindings = get_external_api_bindings_batch(templates, session=session)
|
||||
return {
|
||||
"data": [item.to_dict(dataset_bindings=bindings.get(item.id, [])) for item in templates],
|
||||
"has_more": page * limit < total,
|
||||
"total": total,
|
||||
"page": page,
|
||||
"limit": limit,
|
||||
}
|
||||
|
||||
def get_template(self, workspace_id: str, template_id: str) -> dict[str, Any]:
|
||||
with self._sessions() as session:
|
||||
return self._values(session, self._template(session, workspace_id, template_id))
|
||||
|
||||
def create_template(self, context: RequestContext, *, name: str, settings: Mapping[str, Any]) -> dict[str, Any]:
|
||||
# The legacy creator probes the endpoint before its first SQL statement.
|
||||
# An unopened session keeps that external request outside a transaction.
|
||||
with self._sessions() as session:
|
||||
template = ExternalDatasetService.create_external_knowledge_api(
|
||||
context.active_workspace_id,
|
||||
context.account_id,
|
||||
{"name": name, "settings": dict(settings)},
|
||||
session=session,
|
||||
)
|
||||
result = self._values(session, template)
|
||||
session.commit()
|
||||
return result
|
||||
|
||||
def update_template(
|
||||
self, context: RequestContext, template_id: str, *, name: str, settings: Mapping[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
with self._sessions.begin() as session:
|
||||
self._template(session, context.active_workspace_id, template_id)
|
||||
template = ExternalDatasetService.update_external_knowledge_api(
|
||||
context.active_workspace_id,
|
||||
context.account_id,
|
||||
template_id,
|
||||
{"name": name, "settings": dict(settings)},
|
||||
session=session,
|
||||
)
|
||||
return self._values(session, template)
|
||||
|
||||
def delete_template(self, workspace_id: str, template_id: str) -> None:
|
||||
with self._sessions.begin() as session:
|
||||
session.delete(self._template(session, workspace_id, template_id))
|
||||
|
||||
def template_usage(self, workspace_id: str, template_id: str) -> tuple[bool, int]:
|
||||
with self._sessions() as session:
|
||||
return ExternalDatasetService.external_knowledge_api_use_check(template_id, workspace_id, session=session)
|
||||
|
||||
def create_dataset(self, context: RequestContext, payload: ExternalDatasetCreatePayload) -> DatasetDetailRecord:
|
||||
with self._sessions() as session:
|
||||
self._template(session, context.active_workspace_id, payload.external_knowledge_api_id)
|
||||
dataset = ExternalDatasetService.create_external_dataset(
|
||||
context.active_workspace_id, context.account_id, payload, session=session
|
||||
)
|
||||
permissions = self._members.resource_permissions(
|
||||
context.active_workspace_id, context.account_id, RBACResourceType.DATASET, [dataset.id]
|
||||
)
|
||||
result = load_dataset_detail(dataset, session=session)
|
||||
result["permission_keys"] = permissions.get(dataset.id, [])
|
||||
session.commit()
|
||||
return result
|
||||
|
||||
def retrieve(
|
||||
self,
|
||||
context: RequestContext,
|
||||
ref: DatasetRef,
|
||||
*,
|
||||
query: str,
|
||||
retrieval_model: dict[str, Any] | None,
|
||||
metadata_filters: dict[str, Any] | None,
|
||||
) -> dict[str, Any]:
|
||||
with self._sessions() as session:
|
||||
dataset = require_dataset(session, ref)
|
||||
if dataset.provider != "external":
|
||||
return {"query": {"content": query}, "records": []}
|
||||
request = ExternalDatasetService.prepare_external_knowledge_retrieval(
|
||||
tenant_id=ref.tenant_id,
|
||||
dataset_id=ref.dataset_id,
|
||||
query=HitTestingService.escape_query_for_search(query),
|
||||
external_retrieval_parameters=retrieval_model or {},
|
||||
metadata_condition=MetadataFilteringCondition.model_validate(metadata_filters)
|
||||
if metadata_filters
|
||||
else None,
|
||||
session=session,
|
||||
)
|
||||
documents = ExternalDatasetService.execute_external_knowledge_retrieval(request)
|
||||
with self._sessions.begin() as session:
|
||||
require_dataset(session, ref)
|
||||
session.add(
|
||||
DatasetQuery(
|
||||
dataset_id=ref.dataset_id,
|
||||
content=query,
|
||||
source=DatasetQuerySource.HIT_TESTING,
|
||||
source_app_id=None,
|
||||
created_by_role=CreatorUserRole.ACCOUNT,
|
||||
created_by=context.account_id,
|
||||
)
|
||||
)
|
||||
return dict(HitTestingService.compact_external_retrieve_response(dataset, query, documents))
|
||||
@@ -0,0 +1,104 @@
|
||||
"""External knowledge use cases without transport or persistence dependencies."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, Protocol
|
||||
|
||||
from machinery.context import RequestContext
|
||||
from services.entities.external_knowledge_entities.external_knowledge_entities import ExternalDatasetCreatePayload
|
||||
from services.knowledge.dataset_access import DatasetAccess
|
||||
from services.knowledge.entities.datasets import DatasetDetailRecord
|
||||
from services.knowledge.resource_scope import DatasetRef
|
||||
|
||||
|
||||
class ExternalTemplateNotFoundError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class ExternalHitTestingError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class ExternalKnowledgeOperations(Protocol):
|
||||
def list_templates(self, workspace_id: str, *, page: int, limit: int, keyword: str | None) -> dict[str, Any]: ...
|
||||
def get_template(self, workspace_id: str, template_id: str) -> dict[str, Any]: ...
|
||||
def create_template(self, context: RequestContext, *, name: str, settings: Mapping[str, Any]) -> dict[str, Any]: ...
|
||||
def update_template(
|
||||
self, context: RequestContext, template_id: str, *, name: str, settings: Mapping[str, Any]
|
||||
) -> dict[str, Any]: ...
|
||||
def delete_template(self, workspace_id: str, template_id: str) -> None: ...
|
||||
def template_usage(self, workspace_id: str, template_id: str) -> tuple[bool, int]: ...
|
||||
def create_dataset(self, context: RequestContext, payload: ExternalDatasetCreatePayload) -> DatasetDetailRecord: ...
|
||||
def retrieve(
|
||||
self,
|
||||
context: RequestContext,
|
||||
ref: DatasetRef,
|
||||
*,
|
||||
query: str,
|
||||
retrieval_model: dict[str, Any] | None,
|
||||
metadata_filters: dict[str, Any] | None,
|
||||
) -> dict[str, Any]: ...
|
||||
|
||||
|
||||
class ExternalKnowledgeApplicationService:
|
||||
def __init__(self, *, dataset_access: DatasetAccess, operations: ExternalKnowledgeOperations) -> None:
|
||||
self._dataset_access = dataset_access
|
||||
self._operations = operations
|
||||
|
||||
@staticmethod
|
||||
def _validate_settings(settings: Mapping[str, Any]) -> None:
|
||||
if not settings:
|
||||
raise ValueError("api list is empty")
|
||||
if not settings.get("endpoint"):
|
||||
raise ValueError("endpoint is required")
|
||||
if not settings.get("api_key"):
|
||||
raise ValueError("api_key is required")
|
||||
|
||||
def list_templates(self, context: RequestContext, *, page: int, limit: int, keyword: str | None) -> dict[str, Any]:
|
||||
return self._operations.list_templates(context.active_workspace_id, page=page, limit=limit, keyword=keyword)
|
||||
|
||||
def get_template(self, context: RequestContext, *, template_id: str) -> dict[str, Any]:
|
||||
return self._operations.get_template(context.active_workspace_id, template_id)
|
||||
|
||||
def create_template(self, context: RequestContext, *, name: str, settings: Mapping[str, Any]) -> dict[str, Any]:
|
||||
self._validate_settings(settings)
|
||||
return self._operations.create_template(context, name=name, settings=settings)
|
||||
|
||||
def update_template(
|
||||
self, context: RequestContext, *, template_id: str, name: str, settings: Mapping[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
self._validate_settings(settings)
|
||||
return self._operations.update_template(context, template_id, name=name, settings=settings)
|
||||
|
||||
def delete_template(self, context: RequestContext, *, template_id: str) -> None:
|
||||
self._operations.delete_template(context.active_workspace_id, template_id)
|
||||
|
||||
def template_usage(self, context: RequestContext, *, template_id: str) -> tuple[bool, int]:
|
||||
return self._operations.template_usage(context.active_workspace_id, template_id)
|
||||
|
||||
def create_dataset(self, context: RequestContext, *, payload: ExternalDatasetCreatePayload) -> DatasetDetailRecord:
|
||||
return self._operations.create_dataset(context, payload)
|
||||
|
||||
def hit_testing(
|
||||
self,
|
||||
context: RequestContext,
|
||||
*,
|
||||
dataset_id: str,
|
||||
query: str,
|
||||
retrieval_model: dict[str, Any] | None,
|
||||
metadata_filters: dict[str, Any] | None,
|
||||
) -> dict[str, Any]:
|
||||
dataset = self._dataset_access.require_accessible(context, dataset_id)
|
||||
if not query:
|
||||
raise ValueError("Query or attachment_ids is required")
|
||||
if len(query) > 250:
|
||||
raise ValueError("Query cannot exceed 250 characters")
|
||||
try:
|
||||
return self._operations.retrieve(
|
||||
context,
|
||||
DatasetRef(dataset.workspace_id, dataset.id),
|
||||
query=query,
|
||||
retrieval_model=retrieval_model,
|
||||
metadata_filters=metadata_filters,
|
||||
)
|
||||
except Exception as error:
|
||||
raise ExternalHitTestingError(str(error)) from error
|
||||
@@ -0,0 +1,445 @@
|
||||
import json
|
||||
from copy import deepcopy
|
||||
from typing import Any, cast
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from constants import HIDDEN_VALUE
|
||||
from core.helper import ssrf_proxy
|
||||
from core.rag.entities import MetadataFilteringCondition
|
||||
from extensions.ext_database import db # noqa: F401
|
||||
from graphon.nodes.http_request.exc import InvalidHttpMethodError
|
||||
from libs.datetime_utils import naive_utc_now
|
||||
from libs.pagination import paginate_query
|
||||
from models.dataset import (
|
||||
Dataset,
|
||||
ExternalKnowledgeApis,
|
||||
ExternalKnowledgeBindings,
|
||||
)
|
||||
from services.enterprise import rbac_service as enterprise_rbac_service
|
||||
from services.entities.external_knowledge_entities.external_knowledge_entities import (
|
||||
Authorization,
|
||||
ExternalDatasetCreatePayload,
|
||||
ExternalKnowledgeApiSetting,
|
||||
)
|
||||
from services.errors.dataset import DatasetNameDuplicateError
|
||||
from services.errors.knowledge_retrieval import ExternalKnowledgeRetrievalError
|
||||
from services.rbac import contracts as rbac_contracts
|
||||
|
||||
|
||||
class ExternalDatasetService:
|
||||
@staticmethod
|
||||
def get_external_knowledge_apis(
|
||||
page, per_page, tenant_id, search=None, *, session: Session
|
||||
) -> tuple[list[ExternalKnowledgeApis], int]:
|
||||
query = (
|
||||
select(ExternalKnowledgeApis)
|
||||
.where(ExternalKnowledgeApis.tenant_id == tenant_id)
|
||||
.order_by(ExternalKnowledgeApis.created_at.desc())
|
||||
)
|
||||
if search:
|
||||
from libs.helper import escape_like_pattern
|
||||
|
||||
escaped_search = escape_like_pattern(search)
|
||||
query = query.where(ExternalKnowledgeApis.name.ilike(f"%{escaped_search}%", escape="\\"))
|
||||
|
||||
external_knowledge_apis = paginate_query(query, session=session, page=page, per_page=per_page, max_per_page=100)
|
||||
|
||||
return external_knowledge_apis.items, external_knowledge_apis.total
|
||||
|
||||
@classmethod
|
||||
def validate_api_list(cls, api_settings: dict[str, Any]):
|
||||
if not api_settings:
|
||||
raise ValueError("api list is empty")
|
||||
if not api_settings.get("endpoint"):
|
||||
raise ValueError("endpoint is required")
|
||||
if not api_settings.get("api_key"):
|
||||
raise ValueError("api_key is required")
|
||||
|
||||
@staticmethod
|
||||
def create_external_knowledge_api(
|
||||
tenant_id: str, user_id: str, args: dict[str, Any], *, session: Session
|
||||
) -> ExternalKnowledgeApis:
|
||||
settings = args.get("settings")
|
||||
if settings is None:
|
||||
raise ValueError("settings is required")
|
||||
ExternalDatasetService.check_endpoint_and_api_key(settings)
|
||||
external_knowledge_api = ExternalKnowledgeApis(
|
||||
tenant_id=tenant_id,
|
||||
created_by=user_id,
|
||||
updated_by=user_id,
|
||||
name=str(args.get("name")),
|
||||
description=args.get("description", ""),
|
||||
settings=json.dumps(args.get("settings"), ensure_ascii=False),
|
||||
)
|
||||
|
||||
session.add(external_knowledge_api)
|
||||
session.flush()
|
||||
return external_knowledge_api
|
||||
|
||||
@staticmethod
|
||||
def check_endpoint_and_api_key(settings: dict[str, Any]):
|
||||
if "endpoint" not in settings or not settings["endpoint"]:
|
||||
raise ValueError("endpoint is required")
|
||||
if "api_key" not in settings or not settings["api_key"]:
|
||||
raise ValueError("api_key is required")
|
||||
|
||||
endpoint = f"{settings['endpoint']}/retrieval"
|
||||
api_key = settings["api_key"]
|
||||
|
||||
parsed_url = urlparse(endpoint)
|
||||
if not all([parsed_url.scheme, parsed_url.netloc]):
|
||||
if not endpoint.startswith("http://") and not endpoint.startswith("https://"):
|
||||
raise ValueError(f"invalid endpoint: {endpoint} must start with http:// or https://")
|
||||
else:
|
||||
raise ValueError(f"invalid endpoint: {endpoint}")
|
||||
# Send a minimal body shaped like the External Knowledge API retrieval contract so providers
|
||||
# that require a JSON payload (e.g. RAGFlow) accept the validation probe instead of rejecting
|
||||
# a body-less POST. Mirrors the request built in fetch_external_knowledge_retrieval.
|
||||
validation_payload = {
|
||||
"knowledge_id": "",
|
||||
"query": "",
|
||||
"retrieval_setting": {"top_k": 1, "score_threshold": 0.0},
|
||||
}
|
||||
try:
|
||||
response = ssrf_proxy.post(
|
||||
endpoint,
|
||||
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
|
||||
data=json.dumps(validation_payload),
|
||||
)
|
||||
except Exception as e:
|
||||
raise ValueError(f"failed to connect to the endpoint: {endpoint}") from e
|
||||
if response.status_code == 502:
|
||||
raise ValueError(f"Bad Gateway: failed to connect to the endpoint: {endpoint}")
|
||||
if response.status_code == 404:
|
||||
raise ValueError(f"Not Found: failed to connect to the endpoint: {endpoint}")
|
||||
if response.status_code == 403:
|
||||
raise ValueError("Forbidden: Authorization failed with the provided api_key")
|
||||
|
||||
@staticmethod
|
||||
def get_external_knowledge_api(
|
||||
external_knowledge_api_id: str, tenant_id: str, *, session: Session
|
||||
) -> ExternalKnowledgeApis:
|
||||
external_knowledge_api: ExternalKnowledgeApis | None = session.scalar(
|
||||
select(ExternalKnowledgeApis)
|
||||
.where(ExternalKnowledgeApis.id == external_knowledge_api_id, ExternalKnowledgeApis.tenant_id == tenant_id)
|
||||
.limit(1)
|
||||
)
|
||||
if external_knowledge_api is None:
|
||||
raise ValueError("api template not found")
|
||||
return external_knowledge_api
|
||||
|
||||
@staticmethod
|
||||
def update_external_knowledge_api(
|
||||
tenant_id: str,
|
||||
user_id: str,
|
||||
external_knowledge_api_id: str,
|
||||
args: dict[str, Any],
|
||||
*,
|
||||
session: Session,
|
||||
) -> ExternalKnowledgeApis:
|
||||
external_knowledge_api: ExternalKnowledgeApis | None = session.scalar(
|
||||
select(ExternalKnowledgeApis)
|
||||
.where(ExternalKnowledgeApis.id == external_knowledge_api_id, ExternalKnowledgeApis.tenant_id == tenant_id)
|
||||
.limit(1)
|
||||
)
|
||||
if external_knowledge_api is None:
|
||||
raise ValueError("api template not found")
|
||||
settings = args.get("settings")
|
||||
if settings and settings.get("api_key") == HIDDEN_VALUE and external_knowledge_api.settings_dict:
|
||||
settings["api_key"] = external_knowledge_api.settings_dict.get("api_key")
|
||||
|
||||
external_knowledge_api.name = str(args.get("name"))
|
||||
external_knowledge_api.description = str(args.get("description", ""))
|
||||
external_knowledge_api.settings = json.dumps(settings, ensure_ascii=False)
|
||||
external_knowledge_api.updated_by = user_id
|
||||
external_knowledge_api.updated_at = naive_utc_now()
|
||||
session.flush()
|
||||
|
||||
return external_knowledge_api
|
||||
|
||||
@staticmethod
|
||||
def delete_external_knowledge_api(tenant_id: str, external_knowledge_api_id: str, *, session: Session) -> None:
|
||||
external_knowledge_api = session.scalar(
|
||||
select(ExternalKnowledgeApis)
|
||||
.where(ExternalKnowledgeApis.id == external_knowledge_api_id, ExternalKnowledgeApis.tenant_id == tenant_id)
|
||||
.limit(1)
|
||||
)
|
||||
if external_knowledge_api is None:
|
||||
raise ValueError("api template not found")
|
||||
|
||||
session.delete(external_knowledge_api)
|
||||
session.flush()
|
||||
|
||||
@staticmethod
|
||||
def external_knowledge_api_use_check(
|
||||
external_knowledge_api_id: str, tenant_id: str, *, session: Session
|
||||
) -> tuple[bool, int]:
|
||||
"""
|
||||
Return usage for an external knowledge API within a single tenant.
|
||||
|
||||
The caller already scopes access by tenant, so this query must do the
|
||||
same; otherwise the endpoint becomes a cross-tenant UUID oracle.
|
||||
"""
|
||||
count = (
|
||||
session.scalar(
|
||||
select(func.count(ExternalKnowledgeBindings.id)).where(
|
||||
ExternalKnowledgeBindings.external_knowledge_api_id == external_knowledge_api_id,
|
||||
ExternalKnowledgeBindings.tenant_id == tenant_id,
|
||||
)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
return count > 0, count
|
||||
|
||||
@staticmethod
|
||||
def get_external_knowledge_binding_with_dataset_id(
|
||||
tenant_id: str, dataset_id: str, *, session: Session
|
||||
) -> ExternalKnowledgeBindings:
|
||||
external_knowledge_binding: ExternalKnowledgeBindings | None = session.scalar(
|
||||
select(ExternalKnowledgeBindings)
|
||||
.where(ExternalKnowledgeBindings.dataset_id == dataset_id, ExternalKnowledgeBindings.tenant_id == tenant_id)
|
||||
.limit(1)
|
||||
)
|
||||
if not external_knowledge_binding:
|
||||
raise ValueError("external knowledge binding not found")
|
||||
return external_knowledge_binding
|
||||
|
||||
@staticmethod
|
||||
def document_create_args_validate(
|
||||
tenant_id: str,
|
||||
external_knowledge_api_id: str,
|
||||
process_parameter: dict[str, Any],
|
||||
*,
|
||||
session: Session,
|
||||
) -> None:
|
||||
external_knowledge_api = session.scalar(
|
||||
select(ExternalKnowledgeApis)
|
||||
.where(ExternalKnowledgeApis.id == external_knowledge_api_id, ExternalKnowledgeApis.tenant_id == tenant_id)
|
||||
.limit(1)
|
||||
)
|
||||
if external_knowledge_api is None or external_knowledge_api.settings is None:
|
||||
raise ValueError("api template not found")
|
||||
settings = json.loads(external_knowledge_api.settings)
|
||||
for setting in settings:
|
||||
custom_parameters = setting.get("document_process_setting")
|
||||
if custom_parameters:
|
||||
for parameter in custom_parameters:
|
||||
if parameter.get("required", False) and not process_parameter.get(parameter.get("name")):
|
||||
raise ValueError(f"{parameter.get('name')} is required")
|
||||
|
||||
@staticmethod
|
||||
def process_external_api(settings: ExternalKnowledgeApiSetting, files: dict[str, Any] | None) -> httpx.Response:
|
||||
"""
|
||||
do http request depending on api bundle
|
||||
"""
|
||||
|
||||
kwargs: dict[str, Any] = {
|
||||
"url": settings.url,
|
||||
"headers": settings.headers,
|
||||
"follow_redirects": True,
|
||||
}
|
||||
|
||||
_method_map = {
|
||||
"get": ssrf_proxy.get,
|
||||
"head": ssrf_proxy.head,
|
||||
"post": ssrf_proxy.post,
|
||||
"put": ssrf_proxy.put,
|
||||
"delete": ssrf_proxy.delete,
|
||||
"patch": ssrf_proxy.patch,
|
||||
}
|
||||
method_lc = settings.request_method.lower()
|
||||
if method_lc not in _method_map:
|
||||
raise InvalidHttpMethodError(f"Invalid http method {settings.request_method}")
|
||||
|
||||
response: httpx.Response = _method_map[method_lc](data=json.dumps(settings.params), files=files, **kwargs)
|
||||
return response
|
||||
|
||||
@staticmethod
|
||||
def assembling_headers(authorization: Authorization, headers: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
authorization = deepcopy(authorization)
|
||||
headers = deepcopy(headers) if headers else {}
|
||||
if authorization.type == "api-key":
|
||||
if authorization.config is None:
|
||||
raise ValueError("authorization config is required")
|
||||
|
||||
if authorization.config.api_key is None:
|
||||
raise ValueError("api_key is required")
|
||||
|
||||
if not authorization.config.header:
|
||||
authorization.config.header = "Authorization"
|
||||
|
||||
if authorization.config.type == "bearer":
|
||||
headers[authorization.config.header] = f"Bearer {authorization.config.api_key}"
|
||||
elif authorization.config.type == "basic":
|
||||
headers[authorization.config.header] = f"Basic {authorization.config.api_key}"
|
||||
elif authorization.config.type == "custom":
|
||||
headers[authorization.config.header] = authorization.config.api_key
|
||||
|
||||
return headers
|
||||
|
||||
@staticmethod
|
||||
def get_external_knowledge_api_settings(settings: dict[str, Any]) -> ExternalKnowledgeApiSetting:
|
||||
return ExternalKnowledgeApiSetting.model_validate(settings)
|
||||
|
||||
@staticmethod
|
||||
def create_external_dataset(
|
||||
tenant_id: str, user_id: str, args: ExternalDatasetCreatePayload, *, session: Session
|
||||
) -> Dataset:
|
||||
"""Create a tenant-scoped external dataset and binding in the caller's transaction."""
|
||||
# check if dataset name already exists
|
||||
if session.scalar(select(Dataset).where(Dataset.name == args.name, Dataset.tenant_id == tenant_id).limit(1)):
|
||||
raise DatasetNameDuplicateError(f"Dataset with name {args.name} already exists.")
|
||||
external_knowledge_api = session.scalar(
|
||||
select(ExternalKnowledgeApis)
|
||||
.where(
|
||||
ExternalKnowledgeApis.id == args.external_knowledge_api_id,
|
||||
ExternalKnowledgeApis.tenant_id == tenant_id,
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
|
||||
if external_knowledge_api is None:
|
||||
raise ValueError("api template not found")
|
||||
|
||||
dataset = Dataset(
|
||||
tenant_id=tenant_id,
|
||||
name=args.name,
|
||||
description=args.description or "",
|
||||
provider="external",
|
||||
retrieval_model=args.external_retrieval_model,
|
||||
created_by=user_id,
|
||||
maintainer=user_id,
|
||||
)
|
||||
|
||||
session.add(dataset)
|
||||
session.flush()
|
||||
|
||||
external_knowledge_binding = ExternalKnowledgeBindings(
|
||||
tenant_id=tenant_id,
|
||||
dataset_id=dataset.id,
|
||||
external_knowledge_api_id=args.external_knowledge_api_id or "",
|
||||
external_knowledge_id=args.external_knowledge_id or "",
|
||||
created_by=user_id,
|
||||
)
|
||||
session.add(external_knowledge_binding)
|
||||
|
||||
session.commit()
|
||||
enterprise_rbac_service.try_sync_creator_access_policy_member_bindings(
|
||||
tenant_id,
|
||||
user_id,
|
||||
rbac_contracts.RBACResourceType.DATASET,
|
||||
dataset.id,
|
||||
)
|
||||
|
||||
return dataset
|
||||
|
||||
@staticmethod
|
||||
def fetch_external_knowledge_retrieval(
|
||||
tenant_id: str,
|
||||
dataset_id: str,
|
||||
query: str,
|
||||
external_retrieval_parameters: dict[str, Any],
|
||||
metadata_condition: MetadataFilteringCondition | None = None,
|
||||
*,
|
||||
session: Session,
|
||||
):
|
||||
"""Fetch retrieval records from an external knowledge provider.
|
||||
|
||||
Success requires a tenant-scoped binding plus API template and a ``200``
|
||||
response body shaped like ``{"records": [...]}``. All dependency
|
||||
failures, non-200 responses, and malformed success payloads must be
|
||||
normalized to ``ExternalKnowledgeRetrievalError`` so callers—especially
|
||||
the inner knowledge retrieval API—can consistently expose
|
||||
``502 external_knowledge_failed``.
|
||||
"""
|
||||
request = ExternalDatasetService.prepare_external_knowledge_retrieval(
|
||||
tenant_id, dataset_id, query, external_retrieval_parameters, metadata_condition, session=session
|
||||
)
|
||||
return ExternalDatasetService.execute_external_knowledge_retrieval(request)
|
||||
|
||||
@staticmethod
|
||||
def prepare_external_knowledge_retrieval(
|
||||
tenant_id: str,
|
||||
dataset_id: str,
|
||||
query: str,
|
||||
external_retrieval_parameters: dict[str, Any],
|
||||
metadata_condition: MetadataFilteringCondition | None = None,
|
||||
*,
|
||||
session: Session,
|
||||
) -> ExternalKnowledgeApiSetting:
|
||||
"""Resolve owned configuration into a request that outlives the read session."""
|
||||
external_knowledge_binding = session.scalar(
|
||||
select(ExternalKnowledgeBindings)
|
||||
.where(ExternalKnowledgeBindings.dataset_id == dataset_id, ExternalKnowledgeBindings.tenant_id == tenant_id)
|
||||
.limit(1)
|
||||
)
|
||||
if not external_knowledge_binding:
|
||||
raise ExternalKnowledgeRetrievalError("external knowledge binding not found")
|
||||
|
||||
external_knowledge_api = session.scalar(
|
||||
select(ExternalKnowledgeApis)
|
||||
.where(
|
||||
ExternalKnowledgeApis.id == external_knowledge_binding.external_knowledge_api_id,
|
||||
ExternalKnowledgeApis.tenant_id == tenant_id,
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
if external_knowledge_api is None or external_knowledge_api.settings is None:
|
||||
raise ExternalKnowledgeRetrievalError("external api template not found")
|
||||
|
||||
settings = json.loads(external_knowledge_api.settings)
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if settings.get("api_key"):
|
||||
headers["Authorization"] = f"Bearer {settings.get('api_key')}"
|
||||
score_threshold_enabled = external_retrieval_parameters.get("score_threshold_enabled") or False
|
||||
score_threshold = external_retrieval_parameters.get("score_threshold", 0.0) if score_threshold_enabled else 0.0
|
||||
request_params = {
|
||||
"retrieval_setting": {
|
||||
"top_k": external_retrieval_parameters.get("top_k"),
|
||||
"score_threshold": score_threshold,
|
||||
},
|
||||
"query": query,
|
||||
"knowledge_id": external_knowledge_binding.external_knowledge_id,
|
||||
"metadata_condition": metadata_condition.model_dump() if metadata_condition else None,
|
||||
}
|
||||
|
||||
return ExternalKnowledgeApiSetting(
|
||||
url=f"{settings.get('endpoint')}/retrieval",
|
||||
request_method="post",
|
||||
headers=headers,
|
||||
params=request_params,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def execute_external_knowledge_retrieval(request: ExternalKnowledgeApiSetting) -> list[Any]:
|
||||
"""Execute and validate a prepared request without holding a database session."""
|
||||
try:
|
||||
response = ExternalDatasetService.process_external_api(
|
||||
request,
|
||||
None,
|
||||
)
|
||||
except ExternalKnowledgeRetrievalError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise ExternalKnowledgeRetrievalError(str(exc)) from exc
|
||||
|
||||
if response.status_code == 200:
|
||||
try:
|
||||
response_payload = response.json()
|
||||
except Exception as exc:
|
||||
raise ExternalKnowledgeRetrievalError("invalid external knowledge response") from exc
|
||||
|
||||
if not isinstance(response_payload, dict):
|
||||
raise ExternalKnowledgeRetrievalError("invalid external knowledge response")
|
||||
|
||||
records = response_payload.get("records", [])
|
||||
if not isinstance(records, list):
|
||||
raise ExternalKnowledgeRetrievalError("invalid external knowledge response")
|
||||
|
||||
return cast(list[Any], records)
|
||||
else:
|
||||
raise ExternalKnowledgeRetrievalError(response.text)
|
||||
@@ -0,0 +1 @@
|
||||
"""Use-case-centered stored-document indexing slice."""
|
||||
@@ -0,0 +1 @@
|
||||
"""Infrastructure adapters for the knowledge indexing slice."""
|
||||
@@ -0,0 +1,143 @@
|
||||
"""Delete document indexes between bounded database transactions."""
|
||||
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from configs import dify_config
|
||||
from core.rag.datasource.keyword.jieba.jieba import Jieba
|
||||
from core.rag.datasource.vdb.vector_factory import Vector
|
||||
from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType
|
||||
from extensions.ext_redis import redis_client
|
||||
from models.dataset import ChildChunk, Dataset, DocumentSegment, DocumentSegmentSummary
|
||||
from repositories.knowledge.dataset_read_repository import get_dataset_keyword_table
|
||||
from repositories.knowledge.keyword_table_repository import persist_keyword_table
|
||||
|
||||
|
||||
def delete_index_entries(
|
||||
dataset: Dataset,
|
||||
*,
|
||||
vector_node_ids: Sequence[str],
|
||||
vector_type: str | None,
|
||||
keyword_node_ids: Sequence[str],
|
||||
new_session: Callable[[], Session],
|
||||
) -> None:
|
||||
"""Delete index entries by node id; the single owner of how each index type is removed.
|
||||
|
||||
High-quality datasets delete ``vector_node_ids`` from the vector store; other datasets
|
||||
delete ``keyword_node_ids`` from the keyword table under the dataset keyword lock.
|
||||
"""
|
||||
if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
|
||||
if vector_node_ids:
|
||||
Vector(dataset, session=None, vector_type=vector_type).delete_by_ids(list(vector_node_ids))
|
||||
return
|
||||
if not keyword_node_ids:
|
||||
return
|
||||
|
||||
# The keyword adapter owns its lock and storage I/O; callbacks only do SQL.
|
||||
def read() -> tuple[str, str | None]:
|
||||
with new_session() as session:
|
||||
row = get_dataset_keyword_table(dataset, session=session)
|
||||
return (row.data_source_type, row.keyword_table) if row else (dify_config.KEYWORD_DATA_SOURCE_TYPE, None)
|
||||
|
||||
def write(storage_type: str, data: str, keywords: Mapping[str, Sequence[str]]) -> None:
|
||||
with new_session() as session, session.begin():
|
||||
persist_keyword_table(
|
||||
session,
|
||||
tenant_id=dataset.tenant_id,
|
||||
dataset_id=dataset.id,
|
||||
storage_type=storage_type,
|
||||
data=data,
|
||||
keywords=keywords,
|
||||
)
|
||||
|
||||
Jieba(dataset).update_texts(
|
||||
[],
|
||||
read=read,
|
||||
write=write,
|
||||
lock=redis_client.lock(f"keyword_indexing_lock_{dataset.id}", timeout=600),
|
||||
deleted_ids=keyword_node_ids,
|
||||
)
|
||||
|
||||
|
||||
def clean_document_indexes(
|
||||
*, dataset_id: str, document_ids: Sequence[str], doc_form: str | None, new_session: Callable[[], Session]
|
||||
) -> str | None:
|
||||
"""Clean indexes even when the owning document rows have already been deleted.
|
||||
|
||||
Snapshot the dataset-owned rows, close the read session, remove external
|
||||
indexes, then delete the selected summary and child rows in one transaction.
|
||||
A failed vector deletion leaves those rows available for a retry.
|
||||
Empty document selections never mean deleting the entire dataset index.
|
||||
"""
|
||||
if not document_ids:
|
||||
return None
|
||||
with new_session() as session:
|
||||
dataset = session.scalar(select(Dataset).where(Dataset.id == dataset_id))
|
||||
if dataset is None:
|
||||
return None
|
||||
tenant_id = dataset.tenant_id
|
||||
segments = session.scalars(
|
||||
select(DocumentSegment).where(
|
||||
DocumentSegment.tenant_id == tenant_id,
|
||||
DocumentSegment.dataset_id == dataset_id,
|
||||
DocumentSegment.document_id.in_(document_ids),
|
||||
)
|
||||
).all()
|
||||
summaries = session.scalars(
|
||||
select(DocumentSegmentSummary).where(
|
||||
DocumentSegmentSummary.dataset_id == dataset_id,
|
||||
DocumentSegmentSummary.document_id.in_(document_ids),
|
||||
)
|
||||
).all()
|
||||
children = session.scalars(
|
||||
select(ChildChunk).where(
|
||||
ChildChunk.tenant_id == tenant_id,
|
||||
ChildChunk.dataset_id == dataset_id,
|
||||
ChildChunk.document_id.in_(document_ids),
|
||||
)
|
||||
).all()
|
||||
summary_ids = [summary.id for summary in summaries]
|
||||
child_ids = [child.id for child in children]
|
||||
summary_node_ids = [summary.summary_index_node_id for summary in summaries if summary.summary_index_node_id]
|
||||
body_node_ids = (
|
||||
[child.index_node_id for child in children if child.index_node_id]
|
||||
if doc_form == IndexStructureType.PARENT_CHILD_INDEX
|
||||
else [segment.index_node_id for segment in segments if segment.index_node_id]
|
||||
)
|
||||
high_quality = dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY
|
||||
node_ids = list(dict.fromkeys([*summary_node_ids, *body_node_ids]))
|
||||
vector_type = Vector.resolve_vector_type(dataset, session=session) if high_quality and node_ids else None
|
||||
session.expunge(dataset)
|
||||
|
||||
if not node_ids and not summary_ids and not child_ids:
|
||||
return None
|
||||
|
||||
delete_index_entries(
|
||||
dataset,
|
||||
vector_node_ids=node_ids,
|
||||
vector_type=vector_type,
|
||||
keyword_node_ids=body_node_ids,
|
||||
new_session=new_session,
|
||||
)
|
||||
|
||||
with new_session() as session, session.begin():
|
||||
if summary_ids:
|
||||
session.execute(
|
||||
delete(DocumentSegmentSummary).where(
|
||||
DocumentSegmentSummary.dataset_id == dataset_id,
|
||||
DocumentSegmentSummary.document_id.in_(document_ids),
|
||||
DocumentSegmentSummary.id.in_(summary_ids),
|
||||
)
|
||||
)
|
||||
if child_ids:
|
||||
session.execute(
|
||||
delete(ChildChunk).where(
|
||||
ChildChunk.tenant_id == tenant_id,
|
||||
ChildChunk.dataset_id == dataset_id,
|
||||
ChildChunk.document_id.in_(document_ids),
|
||||
ChildChunk.id.in_(child_ids),
|
||||
)
|
||||
)
|
||||
return tenant_id
|
||||
@@ -0,0 +1,190 @@
|
||||
"""Infrastructure adapters for indexing estimates."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Mapping
|
||||
from typing import cast
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from configs import dify_config
|
||||
from core.credit_usage import CreditUsageCreatedBy
|
||||
from core.entities.knowledge_entities import IndexingEstimate, PreviewDetail, QAPreviewDetail
|
||||
from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
|
||||
from core.model_context import with_credit_usage_created_by
|
||||
from core.model_manager import ModelInstance, ModelManager
|
||||
from core.plugin.impl.exc import PluginDaemonClientSideError
|
||||
from core.rag.extractor.entity.extract_setting import ExtractSetting
|
||||
from core.rag.index_processor.constant.index_type import IndexTechniqueType
|
||||
from core.rag.index_processor.index_processor import IndexProcessorFactory
|
||||
from core.rag.index_processor.index_processor_base import SummaryIndexSettingDict
|
||||
from core.tools.utils.web_reader_tool import get_image_upload_file_ids
|
||||
from enums import DeploymentEdition
|
||||
from extensions.ext_storage import storage
|
||||
from graphon.model_runtime.entities.model_entities import ModelType
|
||||
from models.dataset import Dataset, DatasetProcessRule
|
||||
from repositories.knowledge.dataset_repository import _get_dataset
|
||||
from repositories.knowledge.upload_file_repository import SQLAlchemyKnowledgeUploadRepository
|
||||
from services.credentials.refresh import CredentialRefreshUnavailableError
|
||||
from services.knowledge.indexing.estimate import (
|
||||
IndexingEstimateExecutionError,
|
||||
IndexingEstimateProviderUnavailableError,
|
||||
)
|
||||
from services.knowledge.resource_scope import DatasetRef
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SQLAlchemyProcessRuleReader:
|
||||
"""Load one tenant-owned process rule in a bounded read session."""
|
||||
|
||||
def __init__(self, *, session_factory: sessionmaker[Session]) -> None:
|
||||
self._session_factory = session_factory
|
||||
|
||||
def get_by_id(self, *, dataset_ref: DatasetRef, process_rule_id: str) -> Mapping[str, object] | None:
|
||||
with self._session_factory() as session:
|
||||
process_rule = session.scalar(
|
||||
select(DatasetProcessRule)
|
||||
.join(Dataset, Dataset.id == DatasetProcessRule.dataset_id)
|
||||
.where(
|
||||
DatasetProcessRule.id == process_rule_id,
|
||||
DatasetProcessRule.dataset_id == dataset_ref.dataset_id,
|
||||
Dataset.tenant_id == dataset_ref.tenant_id,
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
if process_rule is None:
|
||||
return None
|
||||
rules = json.loads(process_rule.rules) if process_rule.rules else {}
|
||||
return {"mode": str(process_rule.mode), "rules": rules}
|
||||
|
||||
|
||||
class IndexingEstimateAdapter:
|
||||
"""Extract previews without depending on the document indexing execution flow."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
session_factory: sessionmaker[Session],
|
||||
) -> None:
|
||||
self._session_factory = session_factory
|
||||
self._uploads = SQLAlchemyKnowledgeUploadRepository(session_factory=session_factory)
|
||||
|
||||
@with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)
|
||||
def run(
|
||||
self,
|
||||
tenant_id: str,
|
||||
extract_settings: list[ExtractSetting],
|
||||
tmp_processing_rule: Mapping[str, object],
|
||||
doc_form: str | None = None,
|
||||
doc_language: str = "English",
|
||||
dataset_id: str | None = None,
|
||||
indexing_technique: str = "economy",
|
||||
) -> IndexingEstimate:
|
||||
try:
|
||||
return self._estimate(
|
||||
tenant_id, extract_settings, tmp_processing_rule, doc_form, doc_language, dataset_id, indexing_technique
|
||||
)
|
||||
except (LLMBadRequestError, ProviderTokenNotInitError, PluginDaemonClientSideError) as error:
|
||||
raise IndexingEstimateProviderUnavailableError(error.description) from error
|
||||
except CredentialRefreshUnavailableError:
|
||||
raise
|
||||
except Exception as error:
|
||||
raise IndexingEstimateExecutionError(str(error)) from error
|
||||
|
||||
def _estimate(
|
||||
self,
|
||||
tenant_id: str,
|
||||
extract_settings: list[ExtractSetting],
|
||||
processing_rule: Mapping[str, object],
|
||||
doc_form: str | None,
|
||||
doc_language: str,
|
||||
dataset_id: str | None,
|
||||
indexing_technique: str,
|
||||
) -> IndexingEstimate:
|
||||
if (
|
||||
dify_config.DEPLOYMENT_EDITION == DeploymentEdition.CLOUD
|
||||
and len(extract_settings) > dify_config.BATCH_UPLOAD_LIMIT
|
||||
):
|
||||
raise ValueError(f"You have reached the batch upload limit of {dify_config.BATCH_UPLOAD_LIMIT}.")
|
||||
|
||||
model = self._embedding_model(tenant_id, dataset_id, indexing_technique)
|
||||
processor = IndexProcessorFactory(doc_form).init_index_processor()
|
||||
previews: list[PreviewDetail] = []
|
||||
qa_previews: list[QAPreviewDetail] = []
|
||||
image_ids: set[str] = set()
|
||||
total_segments = 0
|
||||
# Legacy processors need a local session for extraction artifacts. No
|
||||
# document-state transaction is shared with this preview operation.
|
||||
with self._session_factory() as session:
|
||||
for setting in extract_settings:
|
||||
texts = processor.extract(setting, process_rule_mode=processing_rule["mode"], session=session)
|
||||
documents = processor.transform(
|
||||
texts,
|
||||
current_user=None,
|
||||
embedding_model_instance=model,
|
||||
process_rule={"mode": processing_rule["mode"], "rules": processing_rule.get("rules")},
|
||||
tenant_id=tenant_id,
|
||||
doc_language=doc_language,
|
||||
preview=True,
|
||||
session=session,
|
||||
)
|
||||
total_segments += len(documents)
|
||||
for document in documents:
|
||||
if doc_form == "qa_model":
|
||||
if len(qa_previews) < 10:
|
||||
qa_previews.append(
|
||||
QAPreviewDetail(
|
||||
question=document.page_content, answer=document.metadata.get("answer") or ""
|
||||
)
|
||||
)
|
||||
elif len(previews) < 10:
|
||||
preview = PreviewDetail(content=document.page_content)
|
||||
if document.children:
|
||||
preview.child_chunks = [child.page_content for child in document.children]
|
||||
previews.append(preview)
|
||||
image_ids.update(get_image_upload_file_ids(document.page_content))
|
||||
session.commit()
|
||||
|
||||
# Storage I/O takes place after extraction commits and outside the
|
||||
# upload-record deletion transaction. Foreign tenant files are excluded.
|
||||
uploads = self._uploads.get_files(workspace_id=tenant_id, file_ids=tuple(image_ids))
|
||||
for upload in uploads.values():
|
||||
try:
|
||||
storage.delete(upload.key)
|
||||
except Exception:
|
||||
logger.exception("Delete image file failed while estimating indexing, upload_file_id: %s", upload.id)
|
||||
self._uploads.delete_files(workspace_id=tenant_id, file_ids=tuple(uploads))
|
||||
|
||||
if doc_form == "qa_model":
|
||||
return IndexingEstimate(total_segments=total_segments * 20, qa_preview=qa_previews, preview=[])
|
||||
summary = processing_rule.get("summary_index_setting")
|
||||
if isinstance(summary, Mapping) and summary.get("enable") and previews:
|
||||
with self._session_factory() as session:
|
||||
previews = processor.generate_summary_preview(
|
||||
tenant_id, previews, cast(SummaryIndexSettingDict, summary), doc_language, session=session
|
||||
)
|
||||
return IndexingEstimate(total_segments=total_segments, preview=previews)
|
||||
|
||||
def _embedding_model(self, tenant_id: str, dataset_id: str | None, technique: str) -> ModelInstance | None:
|
||||
provider = None
|
||||
model_name = None
|
||||
if dataset_id:
|
||||
with self._session_factory() as session:
|
||||
dataset = _get_dataset(session, DatasetRef(tenant_id, dataset_id))
|
||||
if dataset is None:
|
||||
raise ValueError("Dataset not found.")
|
||||
if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
|
||||
technique = IndexTechniqueType.HIGH_QUALITY
|
||||
provider, model_name = dataset.embedding_model_provider, dataset.embedding_model
|
||||
if technique != IndexTechniqueType.HIGH_QUALITY:
|
||||
return None
|
||||
manager = ModelManager.for_tenant(tenant_id=tenant_id)
|
||||
if provider:
|
||||
if not model_name:
|
||||
raise ValueError("Embedding model is not configured.")
|
||||
return manager.get_model_instance(
|
||||
tenant_id=tenant_id, provider=provider, model_type=ModelType.TEXT_EMBEDDING, model=model_name
|
||||
)
|
||||
return manager.get_default_model_instance(tenant_id=tenant_id, model_type=ModelType.TEXT_EMBEDDING)
|
||||
@@ -0,0 +1,280 @@
|
||||
"""Adapt legacy processors to committed document-indexing phases."""
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import cast
|
||||
|
||||
from flask import Flask, current_app
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
from werkzeug.exceptions import HTTPException
|
||||
from werkzeug.local import LocalProxy
|
||||
|
||||
from core.errors.error import (
|
||||
AppInvokeQuotaExceededError,
|
||||
InvokeRateLimitError,
|
||||
LLMError,
|
||||
ModelCurrentlyNotSupportError,
|
||||
ProviderTokenNotInitError,
|
||||
QuotaExceededError,
|
||||
)
|
||||
from core.model_manager import ModelManager
|
||||
from core.plugin.impl.exc import PluginDaemonError
|
||||
from core.rag.datasource.keyword.jieba.jieba import Jieba
|
||||
from core.rag.datasource.vdb.vector_factory import Vector
|
||||
from core.rag.embedding.token_counter import calculate_segment_token_counts
|
||||
from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType
|
||||
from core.rag.index_processor.index_processor import IndexProcessorFactory
|
||||
from core.rag.models.document import Document
|
||||
from extensions.application_services.data_sources import build_data_source_credentials
|
||||
from extensions.ext_redis import redis_client
|
||||
from extensions.otel import propagate_context
|
||||
from graphon.model_runtime.entities.model_entities import ModelType
|
||||
from libs import helper
|
||||
from models.enums import DataSourceType
|
||||
from repositories.knowledge.document_repository import SQLAlchemyDocumentRepository
|
||||
from repositories.knowledge.segment_repository import SQLAlchemySegmentRepository
|
||||
from repositories.knowledge.upload_file_repository import SQLAlchemyKnowledgeUploadRepository
|
||||
from services.knowledge.indexing.adapters.cleanup import delete_index_entries
|
||||
from services.knowledge.indexing.adapters.sources import (
|
||||
CompositeStoredSourceResolver,
|
||||
FileSourceAdapter,
|
||||
NotionSourceResolver,
|
||||
WebsiteSourceAdapter,
|
||||
)
|
||||
from services.knowledge.indexing.errors import DocumentIsPausedError
|
||||
from services.knowledge.indexing.execution import DocumentIndexingService, IndexingDocument
|
||||
from services.knowledge.resource_scope import DocumentRef
|
||||
from services.vector_space_admission_service import VectorSpaceAdmissionService
|
||||
|
||||
|
||||
class IndexingExecutionAdapter:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
session_factory: sessionmaker[Session],
|
||||
documents: SQLAlchemyDocumentRepository,
|
||||
segments: SQLAlchemySegmentRepository,
|
||||
uploads: SQLAlchemyKnowledgeUploadRepository,
|
||||
sources: CompositeStoredSourceResolver,
|
||||
) -> None:
|
||||
self._session_factory = session_factory
|
||||
self._documents = documents
|
||||
self._segments = segments
|
||||
self._uploads = uploads
|
||||
self._sources = sources
|
||||
|
||||
def describe_error(self, error: Exception) -> str:
|
||||
"""Preserve provider and extractor descriptions without their transport prefixes."""
|
||||
if isinstance(
|
||||
error,
|
||||
(
|
||||
LLMError,
|
||||
ProviderTokenNotInitError,
|
||||
QuotaExceededError,
|
||||
AppInvokeQuotaExceededError,
|
||||
ModelCurrentlyNotSupportError,
|
||||
InvokeRateLimitError,
|
||||
PluginDaemonError,
|
||||
HTTPException,
|
||||
),
|
||||
):
|
||||
return str(error.description)
|
||||
return str(error)
|
||||
|
||||
def check_paused(self, ref: DocumentRef) -> None:
|
||||
if redis_client.get(f"document_{ref.document_id}_is_paused"):
|
||||
raise DocumentIsPausedError(f"Document paused, document id: {ref.document_id}")
|
||||
|
||||
def extract(self, document: IndexingDocument) -> list[Document]:
|
||||
if not document.processing_rule:
|
||||
raise ValueError("no process rule found")
|
||||
setting = self._sources.resolve_for_indexing(document.source)
|
||||
processor = IndexProcessorFactory(document.doc_form).init_index_processor()
|
||||
# Legacy extractors persist downloaded images and Notion source metadata.
|
||||
# Their session is independent of document-state write transactions.
|
||||
with self._session_factory() as session:
|
||||
texts = processor.extract(setting, process_rule_mode=document.processing_rule["mode"], session=session)
|
||||
session.commit()
|
||||
for text in texts:
|
||||
text.metadata.update(document_id=document.ref.document_id, dataset_id=document.ref.dataset.dataset_id)
|
||||
return texts
|
||||
|
||||
def transform(self, document: IndexingDocument, texts: list[Document]) -> list[Document]:
|
||||
dataset, _ = self._documents.get_indexing_models(document.ref)
|
||||
user = self._documents.get_indexing_user(document.ref)
|
||||
model = None
|
||||
if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
|
||||
manager = ModelManager.for_tenant(tenant_id=dataset.tenant_id)
|
||||
if dataset.embedding_model_provider:
|
||||
model = manager.get_model_instance(
|
||||
tenant_id=dataset.tenant_id,
|
||||
provider=dataset.embedding_model_provider,
|
||||
model_type=ModelType.TEXT_EMBEDDING,
|
||||
model=dataset.embedding_model,
|
||||
)
|
||||
else:
|
||||
model = manager.get_default_model_instance(
|
||||
tenant_id=dataset.tenant_id, model_type=ModelType.TEXT_EMBEDDING
|
||||
)
|
||||
processor = IndexProcessorFactory(document.doc_form).init_index_processor()
|
||||
with self._session_factory() as session:
|
||||
chunks = processor.transform(
|
||||
texts,
|
||||
user,
|
||||
embedding_model_instance=model,
|
||||
process_rule=document.processing_rule,
|
||||
tenant_id=dataset.tenant_id,
|
||||
doc_language=document.doc_language,
|
||||
session=session,
|
||||
)
|
||||
session.commit()
|
||||
return chunks
|
||||
|
||||
def count_tokens(self, document: IndexingDocument, chunks: list[Document]) -> list[int]:
|
||||
dataset, _ = self._documents.get_indexing_models(document.ref)
|
||||
return calculate_segment_token_counts(dataset=dataset, documents=chunks)
|
||||
|
||||
def ensure_admission(self, document: IndexingDocument, chunks: list[Document]) -> None:
|
||||
dataset, _ = self._documents.get_indexing_models(document.ref)
|
||||
with self._session_factory() as session:
|
||||
VectorSpaceAdmissionService().ensure_document_can_be_indexed(
|
||||
dataset=dataset,
|
||||
document_id=document.ref.document_id,
|
||||
doc_form=document.doc_form,
|
||||
documents=chunks,
|
||||
include_summaries=document.need_summary,
|
||||
session=session,
|
||||
)
|
||||
|
||||
def load(self, document: IndexingDocument, chunks: list[Document]) -> None:
|
||||
dataset, _ = self._documents.get_indexing_models(document.ref)
|
||||
app = cast(LocalProxy[Flask], current_app)._get_current_object()
|
||||
if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
|
||||
groups: list[list[Document]] = [[] for _ in range(10)]
|
||||
for chunk in chunks:
|
||||
groups[int(helper.generate_text_hash(chunk.page_content), 16) % 10].append(chunk)
|
||||
with ThreadPoolExecutor(max_workers=10) as executor:
|
||||
futures = [
|
||||
executor.submit(
|
||||
propagate_context(self._process_chunk),
|
||||
app,
|
||||
document.ref,
|
||||
group,
|
||||
False,
|
||||
)
|
||||
for group in groups
|
||||
if group
|
||||
]
|
||||
for future in futures:
|
||||
future.result()
|
||||
elif (
|
||||
dataset.indexing_technique == IndexTechniqueType.ECONOMY
|
||||
and document.doc_form != IndexStructureType.PARENT_CHILD_INDEX
|
||||
):
|
||||
# A future propagates keyword failures; Thread.join() used to hide
|
||||
# them and incorrectly let the document become completed.
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
executor.submit(propagate_context(self._process_chunk), app, document.ref, chunks, True).result()
|
||||
|
||||
def discard(self, document: IndexingDocument, chunks: list[Document]) -> None:
|
||||
"""Remove the index entries ``load`` writes for ``chunks`` through ``delete_index_entries``.
|
||||
|
||||
Mirrors ``_process_chunk``: child vectors for parent-child documents, body vectors
|
||||
otherwise, and body keywords for economy datasets. Some ids may never have been
|
||||
written; each store handles that as it does when document cleanup issues the same call.
|
||||
"""
|
||||
dataset = self._documents.get_indexing_dataset(document.ref.dataset)
|
||||
if dataset is None:
|
||||
return
|
||||
if document.doc_form == IndexStructureType.PARENT_CHILD_INDEX:
|
||||
node_ids = [child.metadata["doc_id"] for chunk in chunks for child in chunk.children or []]
|
||||
else:
|
||||
node_ids = [chunk.metadata["doc_id"] for chunk in chunks]
|
||||
if not node_ids:
|
||||
return
|
||||
vector_type = None
|
||||
if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
|
||||
with self._session_factory() as session:
|
||||
vector_type = Vector.resolve_vector_type(dataset, session=session)
|
||||
writes_keywords = (
|
||||
dataset.indexing_technique == IndexTechniqueType.ECONOMY
|
||||
and document.doc_form != IndexStructureType.PARENT_CHILD_INDEX
|
||||
)
|
||||
delete_index_entries(
|
||||
dataset,
|
||||
vector_node_ids=node_ids,
|
||||
vector_type=vector_type,
|
||||
keyword_node_ids=node_ids if writes_keywords else [],
|
||||
new_session=self._session_factory,
|
||||
)
|
||||
|
||||
def _process_chunk(self, app: Flask, ref: DocumentRef, chunks: list[Document], keywords: bool) -> None:
|
||||
with app.app_context():
|
||||
self.check_paused(ref)
|
||||
# Workers load their own inputs and own their sessions. No ORM
|
||||
# instance or session crosses the executor boundary.
|
||||
dataset, document = self._documents.get_indexing_models(ref)
|
||||
if keywords:
|
||||
# Keep the read/modify/write lock while each repository call
|
||||
# closes its transaction before keyword extraction or storage I/O.
|
||||
Jieba(dataset).update_texts(
|
||||
chunks,
|
||||
read=lambda: self._segments.get_keyword_table(ref.dataset),
|
||||
write=lambda storage_type, data, selected: self._segments.save_keyword_table(
|
||||
ref.dataset, storage_type=storage_type, data=data, keywords=selected
|
||||
),
|
||||
lock=redis_client.lock(f"keyword_indexing_lock_{dataset.id}", timeout=600),
|
||||
)
|
||||
elif dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
|
||||
with self._session_factory() as session:
|
||||
vector_type = Vector.resolve_vector_type(dataset, session=session)
|
||||
attachments = (
|
||||
[attachment for chunk in chunks for attachment in chunk.attachments or []]
|
||||
if dataset.is_multimodal
|
||||
else []
|
||||
)
|
||||
files = self._uploads.get_files(
|
||||
workspace_id=ref.dataset.tenant_id,
|
||||
file_ids=[attachment.metadata["doc_id"] for attachment in attachments],
|
||||
)
|
||||
# All database inputs are materialized before initializing the
|
||||
# vector backend, embedding text/images, or reading object storage.
|
||||
vector = Vector(dataset, session=None, vector_type=vector_type)
|
||||
if document.doc_form == IndexStructureType.PARENT_CHILD_INDEX:
|
||||
for chunk in chunks:
|
||||
if chunk.children:
|
||||
vector.create([Document.model_validate(child.model_dump()) for child in chunk.children])
|
||||
else:
|
||||
vector.create(chunks)
|
||||
if attachments:
|
||||
vector.create_multimodal(attachments, upload_files=files)
|
||||
self.check_paused(ref)
|
||||
self._segments.complete_indexing_segments(ref, [chunk.metadata["doc_id"] for chunk in chunks])
|
||||
|
||||
|
||||
def build_document_indexing_service(
|
||||
*,
|
||||
session_factory: sessionmaker[Session],
|
||||
enforce_vector_space_admission: bool = False,
|
||||
) -> DocumentIndexingService:
|
||||
"""Assemble a fresh indexing use case from explicit database dependencies."""
|
||||
documents = SQLAlchemyDocumentRepository(session_factory=session_factory)
|
||||
segments = SQLAlchemySegmentRepository(session_factory=session_factory)
|
||||
uploads = SQLAlchemyKnowledgeUploadRepository(session_factory=session_factory)
|
||||
credentials = build_data_source_credentials(database_client=session_factory)
|
||||
sources = CompositeStoredSourceResolver(
|
||||
adapters={
|
||||
DataSourceType.UPLOAD_FILE: FileSourceAdapter(uploads=uploads),
|
||||
DataSourceType.NOTION_IMPORT: NotionSourceResolver(
|
||||
actor_credentials=credentials.actor, stored_credentials=credentials.stored
|
||||
),
|
||||
DataSourceType.WEBSITE_CRAWL: WebsiteSourceAdapter(),
|
||||
}
|
||||
)
|
||||
return DocumentIndexingService(
|
||||
documents=documents,
|
||||
segments=segments,
|
||||
backend=IndexingExecutionAdapter(
|
||||
session_factory=session_factory, documents=documents, segments=segments, uploads=uploads, sources=sources
|
||||
),
|
||||
enforce_vector_space_admission=enforce_vector_space_admission,
|
||||
)
|
||||
@@ -0,0 +1,282 @@
|
||||
"""Datasource-specific extraction input resolvers."""
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Protocol
|
||||
|
||||
from configs import dify_config
|
||||
from core.rag.extractor.entity.datasource_type import DatasourceType
|
||||
from core.rag.extractor.entity.extract_setting import (
|
||||
ExtractSetting,
|
||||
NotionInfo,
|
||||
StoredDocumentExtractionInput,
|
||||
UploadFileExtractionInput,
|
||||
WebsiteInfo,
|
||||
)
|
||||
from services.data_source.credential_gateway import (
|
||||
ActorDatasourceCredentialResolver,
|
||||
NotionCredentialUnavailableError,
|
||||
StoredDatasourceCredentialResolver,
|
||||
resolve_stored_notion_access_token,
|
||||
)
|
||||
from services.data_source.entities.notion_import import NotionPageType
|
||||
from services.knowledge.indexing.errors import (
|
||||
IndexingInputSourceError,
|
||||
SourceCredentialUnavailableError,
|
||||
UnsupportedStoredSourceError,
|
||||
)
|
||||
from services.knowledge.indexing.estimate import StoredSource
|
||||
|
||||
_NOTION_PROVIDER = "notion_datasource"
|
||||
_NOTION_PLUGIN = "langgenius/notion_datasource"
|
||||
|
||||
|
||||
class StoredSourceAdapter(Protocol):
|
||||
"""Resolve one persisted datasource kind."""
|
||||
|
||||
def resolve(self, source: StoredSource) -> ExtractSetting: ...
|
||||
|
||||
|
||||
class KnowledgeUploadCatalog(Protocol):
|
||||
def get_by_ids(self, *, workspace_id: str, file_ids: Sequence[str]) -> Mapping[str, UploadFileExtractionInput]: ...
|
||||
|
||||
|
||||
class FileSourceAdapter:
|
||||
def __init__(self, *, uploads: KnowledgeUploadCatalog) -> None:
|
||||
self._uploads = uploads
|
||||
|
||||
def resolve(self, source: StoredSource) -> ExtractSetting:
|
||||
info = _required_info(source, "no upload file found")
|
||||
file_id = _required_string(info, "upload_file_id", "no upload file found")
|
||||
return self.resolve_selection(
|
||||
workspace_id=source.document_ref.dataset.tenant_id,
|
||||
file_id=file_id,
|
||||
document_model=source.document_model,
|
||||
)
|
||||
|
||||
def resolve_selection(
|
||||
self,
|
||||
*,
|
||||
workspace_id: str,
|
||||
file_id: str,
|
||||
document_model: str,
|
||||
) -> ExtractSetting:
|
||||
return self.resolve_selections(workspace_id=workspace_id, file_ids=(file_id,), document_model=document_model)[
|
||||
file_id
|
||||
]
|
||||
|
||||
def resolve_selections(
|
||||
self, *, workspace_id: str, file_ids: Sequence[str], document_model: str
|
||||
) -> dict[str, ExtractSetting]:
|
||||
if not file_ids:
|
||||
return {}
|
||||
uploads = self._uploads.get_by_ids(workspace_id=workspace_id, file_ids=file_ids)
|
||||
settings: dict[str, ExtractSetting] = {}
|
||||
for file_id in file_ids:
|
||||
upload = uploads.get(file_id)
|
||||
if upload is None:
|
||||
raise IndexingInputSourceError(f"no upload file found: {file_id}")
|
||||
settings[file_id] = ExtractSetting(
|
||||
datasource_type=DatasourceType.FILE, upload_file=upload, document_model=document_model
|
||||
)
|
||||
return settings
|
||||
|
||||
|
||||
class WebsiteSourceAdapter:
|
||||
def resolve(self, source: StoredSource) -> ExtractSetting:
|
||||
info = _required_info(source, "no website crawl info found")
|
||||
return self.resolve_selection(
|
||||
workspace_id=source.document_ref.dataset.tenant_id,
|
||||
provider=_required_string(info, "provider"),
|
||||
job_id=_required_string(info, "job_id"),
|
||||
url=_required_string(info, "url"),
|
||||
mode=_required_string(info, "mode"),
|
||||
only_main_content=bool(info.get("only_main_content", False)),
|
||||
document_model=source.document_model,
|
||||
)
|
||||
|
||||
def resolve_selection(
|
||||
self,
|
||||
*,
|
||||
workspace_id: str,
|
||||
provider: str,
|
||||
job_id: str,
|
||||
url: str,
|
||||
mode: str,
|
||||
only_main_content: bool,
|
||||
document_model: str,
|
||||
) -> ExtractSetting:
|
||||
return ExtractSetting(
|
||||
datasource_type=DatasourceType.WEBSITE,
|
||||
website_info=WebsiteInfo(
|
||||
provider=provider,
|
||||
job_id=job_id,
|
||||
url=url,
|
||||
mode=mode,
|
||||
tenant_id=workspace_id,
|
||||
only_main_content=only_main_content,
|
||||
),
|
||||
document_model=document_model,
|
||||
)
|
||||
|
||||
|
||||
class NotionSourceResolver:
|
||||
"""Resolve actor-selected and trusted-stored Notion extraction inputs."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
actor_credentials: ActorDatasourceCredentialResolver,
|
||||
stored_credentials: StoredDatasourceCredentialResolver,
|
||||
integration_token: str | None = None,
|
||||
) -> None:
|
||||
self._actor_credentials = actor_credentials
|
||||
self._stored_credentials = stored_credentials
|
||||
self._integration_token = integration_token
|
||||
|
||||
def resolve(self, source: StoredSource) -> ExtractSetting:
|
||||
info = _required_info(source, "no notion import info found")
|
||||
document_ref = source.document_ref
|
||||
credential_id = _string_value(info, "credential_id")
|
||||
token = self._stored_token(source, credential_id)
|
||||
return self._prepared(
|
||||
tenant_id=document_ref.dataset.tenant_id,
|
||||
notion_workspace_id=_required_string(info, "notion_workspace_id"),
|
||||
page_id=_required_string(info, "notion_page_id"),
|
||||
page_type=_notion_page_type(info),
|
||||
document_model=source.document_model,
|
||||
credential_id=credential_id,
|
||||
token=token,
|
||||
)
|
||||
|
||||
def _stored_token(self, source: StoredSource, credential_id: str | None) -> str:
|
||||
document_ref = source.document_ref
|
||||
|
||||
def load_token(saved_id: str) -> str:
|
||||
credentials = self._stored_credentials.resolve_for_document(
|
||||
workspace_id=document_ref.dataset.tenant_id,
|
||||
dataset_id=document_ref.dataset.dataset_id,
|
||||
document_id=document_ref.document_id,
|
||||
credential_id=saved_id,
|
||||
provider=_NOTION_PROVIDER,
|
||||
plugin_id=_NOTION_PLUGIN,
|
||||
)
|
||||
return self._required_token(credentials)
|
||||
|
||||
try:
|
||||
return resolve_stored_notion_access_token(
|
||||
credential_id=credential_id,
|
||||
load_token=load_token,
|
||||
integration_token=self._integration_token or dify_config.NOTION_INTEGRATION_TOKEN,
|
||||
)
|
||||
except NotionCredentialUnavailableError as error:
|
||||
raise SourceCredentialUnavailableError("Notion credential is unavailable") from error
|
||||
|
||||
def resolve_selection(
|
||||
self,
|
||||
*,
|
||||
workspace_id: str,
|
||||
actor_id: str,
|
||||
credential_id: str,
|
||||
notion_workspace_id: str,
|
||||
page_id: str,
|
||||
page_type: NotionPageType,
|
||||
document_model: str,
|
||||
) -> ExtractSetting:
|
||||
credentials = self._actor_credentials.resolve(
|
||||
workspace_id=workspace_id,
|
||||
actor_id=actor_id,
|
||||
credential_id=credential_id,
|
||||
provider=_NOTION_PROVIDER,
|
||||
plugin_id=_NOTION_PLUGIN,
|
||||
)
|
||||
return self._prepared(
|
||||
tenant_id=workspace_id,
|
||||
notion_workspace_id=notion_workspace_id,
|
||||
page_id=page_id,
|
||||
page_type=page_type,
|
||||
document_model=document_model,
|
||||
credential_id=credential_id,
|
||||
token=self._required_token(credentials),
|
||||
)
|
||||
|
||||
def _prepared(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
notion_workspace_id: str,
|
||||
page_id: str,
|
||||
page_type: NotionPageType,
|
||||
document_model: str,
|
||||
credential_id: str | None,
|
||||
token: str,
|
||||
) -> ExtractSetting:
|
||||
return ExtractSetting(
|
||||
datasource_type=DatasourceType.NOTION,
|
||||
notion_info=NotionInfo(
|
||||
credential_id=credential_id,
|
||||
notion_workspace_id=notion_workspace_id,
|
||||
notion_obj_id=page_id,
|
||||
notion_page_type=page_type.value,
|
||||
tenant_id=tenant_id,
|
||||
notion_access_token=token,
|
||||
),
|
||||
document_model=document_model,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _required_token(credentials: Mapping[str, object]) -> str:
|
||||
token = credentials.get("integration_secret")
|
||||
if isinstance(token, str) and token:
|
||||
return token
|
||||
raise SourceCredentialUnavailableError("Notion credential is unavailable")
|
||||
|
||||
|
||||
class CompositeStoredSourceResolver:
|
||||
"""Route persisted sources to datasource-specific adapters."""
|
||||
|
||||
def __init__(self, *, adapters: Mapping[str, StoredSourceAdapter]) -> None:
|
||||
self._adapters = dict(adapters)
|
||||
|
||||
def resolve(self, source: StoredSource) -> ExtractSetting:
|
||||
adapter = self._adapters.get(source.source_type)
|
||||
if adapter is None:
|
||||
raise UnsupportedStoredSourceError(source.source_type)
|
||||
return adapter.resolve(source)
|
||||
|
||||
def resolve_for_indexing(self, source: StoredSource) -> ExtractSetting:
|
||||
setting = self.resolve(source)
|
||||
if setting.notion_info is not None:
|
||||
ref = source.document_ref
|
||||
setting.notion_info.document = StoredDocumentExtractionInput(
|
||||
id=ref.document_id,
|
||||
tenant_id=ref.dataset.tenant_id,
|
||||
dataset_id=ref.dataset.dataset_id,
|
||||
data_source_info_dict=dict(source.source_info) if source.source_info is not None else None,
|
||||
)
|
||||
return setting
|
||||
|
||||
|
||||
def _required_info(source: StoredSource, message: str) -> Mapping[str, object]:
|
||||
if source.source_info is None:
|
||||
raise IndexingInputSourceError(message)
|
||||
return source.source_info
|
||||
|
||||
|
||||
def _string_value(info: Mapping[str, object], key: str) -> str | None:
|
||||
value = info.get(key)
|
||||
return value if isinstance(value, str) and value else None
|
||||
|
||||
|
||||
def _required_string(info: Mapping[str, object], key: str, message: str | None = None) -> str:
|
||||
value = _string_value(info, key)
|
||||
if value is None:
|
||||
raise IndexingInputSourceError(message or f"invalid data source field: {key}")
|
||||
return value
|
||||
|
||||
|
||||
def _notion_page_type(info: Mapping[str, object]) -> NotionPageType:
|
||||
value = _required_string(info, "type")
|
||||
try:
|
||||
return NotionPageType(value)
|
||||
except ValueError as error:
|
||||
raise IndexingInputSourceError("invalid data source field: type") from error
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Application failures raised while preparing or executing indexing."""
|
||||
|
||||
|
||||
class IndexingInputError(Exception):
|
||||
"""Base class for failures while resolving stored extraction inputs."""
|
||||
|
||||
|
||||
class IndexingInputSourceError(IndexingInputError):
|
||||
"""Raised when a persisted source cannot form a valid extraction input."""
|
||||
|
||||
|
||||
class UnsupportedStoredSourceError(IndexingInputSourceError):
|
||||
def __init__(self, source_type: str) -> None:
|
||||
self.source_type = source_type
|
||||
super().__init__(f"unsupported data source type: {source_type}")
|
||||
|
||||
|
||||
class SourceCredentialUnavailableError(IndexingInputSourceError):
|
||||
"""Raised when a datasource cannot resolve its required credential policy."""
|
||||
|
||||
|
||||
class DocumentIsPausedError(Exception):
|
||||
"""Indexing stopped because the owned document was paused."""
|
||||
|
||||
|
||||
class DocumentIsDeletedPausedError(Exception):
|
||||
"""Indexing stopped because its document owner chain no longer exists."""
|
||||
@@ -0,0 +1,346 @@
|
||||
"""Indexing-estimate application service and its consumer-owned ports."""
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import NamedTuple, Never, Protocol
|
||||
|
||||
from core.entities.knowledge_entities import IndexingEstimate
|
||||
from core.rag.extractor.entity.extract_setting import ExtractSetting
|
||||
from machinery.context import RequestContext
|
||||
from services.credentials.refresh import CredentialRefreshUnavailableError
|
||||
from services.data_source.credential_gateway import DatasourceCredentialError
|
||||
from services.data_source.entities.notion_import import NotionPageType
|
||||
from services.knowledge.dataset_access import DatasetAccess, DatasetNotFoundError
|
||||
from services.knowledge.entities.indexing_estimate import (
|
||||
NewEstimateSource,
|
||||
NewSourcesEstimateCommand,
|
||||
NotionEstimateSource,
|
||||
UploadFileEstimateSource,
|
||||
WebsiteEstimateSource,
|
||||
normalize_process_rule,
|
||||
)
|
||||
from services.knowledge.indexing.errors import (
|
||||
IndexingInputSourceError,
|
||||
SourceCredentialUnavailableError,
|
||||
UnsupportedStoredSourceError,
|
||||
)
|
||||
from services.knowledge.resource_scope import DatasetRef, DocumentRef
|
||||
|
||||
|
||||
class DatasetEstimateRecord(NamedTuple):
|
||||
"""Dataset state required to estimate stored documents."""
|
||||
|
||||
id: str
|
||||
workspace_id: str
|
||||
indexing_technique: str | None
|
||||
|
||||
|
||||
class EstimateDocumentRecord(NamedTuple):
|
||||
"""Document state required by the stored-document estimate use case."""
|
||||
|
||||
id: str
|
||||
workspace_id: str
|
||||
dataset_id: str
|
||||
data_source_type: str
|
||||
data_source_info: Mapping[str, object] | None
|
||||
indexing_status: str
|
||||
doc_form: str
|
||||
doc_language: str | None
|
||||
dataset_process_rule_id: str | None
|
||||
|
||||
@property
|
||||
def ref(self) -> DocumentRef:
|
||||
return DatasetRef(self.workspace_id, self.dataset_id).document(self.id)
|
||||
|
||||
|
||||
class StoredSource(NamedTuple):
|
||||
"""A persisted extraction source bound to its owning document."""
|
||||
|
||||
document_ref: DocumentRef
|
||||
source_type: str
|
||||
source_info: Mapping[str, object] | None
|
||||
document_model: str
|
||||
|
||||
@classmethod
|
||||
def from_document(cls, document: EstimateDocumentRecord) -> "StoredSource":
|
||||
return cls(
|
||||
document_ref=document.ref,
|
||||
source_type=document.data_source_type,
|
||||
source_info=document.data_source_info,
|
||||
document_model=document.doc_form,
|
||||
)
|
||||
|
||||
|
||||
class EstimateDocumentReader(Protocol):
|
||||
def get_estimate_document(self, document_ref: DocumentRef) -> EstimateDocumentRecord | None: ...
|
||||
|
||||
def list_estimate_documents_by_batch(
|
||||
self, dataset_ref: DatasetRef, batch: str
|
||||
) -> tuple[EstimateDocumentRecord, ...]: ...
|
||||
|
||||
|
||||
class EstimateDatasetReader(Protocol):
|
||||
def get_estimate_record(self, dataset_ref: DatasetRef) -> DatasetEstimateRecord | None: ...
|
||||
|
||||
|
||||
class UploadEstimateSourceResolver(Protocol):
|
||||
def resolve_selections(
|
||||
self, *, workspace_id: str, file_ids: Sequence[str], document_model: str
|
||||
) -> Mapping[str, ExtractSetting]: ...
|
||||
|
||||
|
||||
class NotionEstimateSourceResolver(Protocol):
|
||||
def resolve_selection(
|
||||
self,
|
||||
*,
|
||||
workspace_id: str,
|
||||
actor_id: str,
|
||||
credential_id: str,
|
||||
notion_workspace_id: str,
|
||||
page_id: str,
|
||||
page_type: NotionPageType,
|
||||
document_model: str,
|
||||
) -> ExtractSetting: ...
|
||||
|
||||
|
||||
class WebsiteEstimateSourceResolver(Protocol):
|
||||
def resolve_selection(
|
||||
self,
|
||||
*,
|
||||
workspace_id: str,
|
||||
provider: str,
|
||||
job_id: str,
|
||||
url: str,
|
||||
mode: str,
|
||||
only_main_content: bool,
|
||||
document_model: str,
|
||||
) -> ExtractSetting: ...
|
||||
|
||||
|
||||
class StoredEstimateSourceResolver(Protocol):
|
||||
def resolve(self, source: StoredSource) -> ExtractSetting: ...
|
||||
|
||||
|
||||
class ProcessRuleReader(Protocol):
|
||||
def get_by_id(self, *, dataset_ref: DatasetRef, process_rule_id: str) -> Mapping[str, object] | None: ...
|
||||
|
||||
|
||||
class IndexingEstimateRunner(Protocol):
|
||||
def run(
|
||||
self,
|
||||
tenant_id: str,
|
||||
extract_settings: list[ExtractSetting],
|
||||
tmp_processing_rule: Mapping[str, object],
|
||||
doc_form: str | None = None,
|
||||
doc_language: str = "English",
|
||||
dataset_id: str | None = None,
|
||||
indexing_technique: str = "economy",
|
||||
) -> IndexingEstimate: ...
|
||||
|
||||
|
||||
class IndexingEstimateApplicationError(Exception):
|
||||
"""Base class for framework-neutral estimate failures."""
|
||||
|
||||
|
||||
class EstimateSourceNotFoundError(IndexingEstimateApplicationError):
|
||||
def __init__(self, source_id: str) -> None:
|
||||
super().__init__(f"Estimate source not found: {source_id}")
|
||||
|
||||
|
||||
class UnsupportedEstimateSourceError(IndexingEstimateApplicationError):
|
||||
def __init__(self, source_type: str) -> None:
|
||||
super().__init__(f"Data source type not supported: {source_type}")
|
||||
|
||||
|
||||
class IndexingEstimateCredentialUnavailableError(IndexingEstimateApplicationError):
|
||||
"""Raised when a datasource credential required by an estimate is unavailable."""
|
||||
|
||||
|
||||
class IndexingEstimateProviderUnavailableError(IndexingEstimateApplicationError):
|
||||
"""Raised when a configured model or plugin provider is unavailable."""
|
||||
|
||||
|
||||
class IndexingEstimateExecutionError(IndexingEstimateApplicationError):
|
||||
"""Raised when the estimate processor cannot complete an estimate."""
|
||||
|
||||
|
||||
class EstimateDocumentNotFoundError(IndexingEstimateApplicationError):
|
||||
def __init__(self) -> None:
|
||||
super().__init__("Document not found")
|
||||
|
||||
|
||||
class EstimateDocumentAlreadyFinishedError(IndexingEstimateApplicationError):
|
||||
def __init__(self) -> None:
|
||||
super().__init__("Document already finished")
|
||||
|
||||
|
||||
class IndexingEstimateApplicationService:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
dataset_access: DatasetAccess,
|
||||
datasets: EstimateDatasetReader,
|
||||
documents: EstimateDocumentReader,
|
||||
files: UploadEstimateSourceResolver,
|
||||
notion: NotionEstimateSourceResolver,
|
||||
websites: WebsiteEstimateSourceResolver,
|
||||
stored_sources: StoredEstimateSourceResolver,
|
||||
process_rules: ProcessRuleReader,
|
||||
runner: IndexingEstimateRunner,
|
||||
) -> None:
|
||||
self._dataset_access = dataset_access
|
||||
self._datasets = datasets
|
||||
self._documents = documents
|
||||
self._files = files
|
||||
self._notion = notion
|
||||
self._websites = websites
|
||||
self._stored_sources = stored_sources
|
||||
self._process_rules = process_rules
|
||||
self._runner = runner
|
||||
|
||||
def estimate_new_sources(
|
||||
self,
|
||||
context: RequestContext,
|
||||
command: NewSourcesEstimateCommand,
|
||||
) -> IndexingEstimate:
|
||||
dataset_id = command.dataset_id or None
|
||||
if dataset_id is not None:
|
||||
dataset = self._dataset_access.require_accessible(context, dataset_id)
|
||||
dataset_id = dataset.id
|
||||
process_rule = normalize_process_rule(command.process_rule)
|
||||
|
||||
try:
|
||||
file_settings = self._files.resolve_selections(
|
||||
workspace_id=context.active_workspace_id,
|
||||
file_ids=tuple(
|
||||
dict.fromkeys(
|
||||
source.file_id for source in command.sources if isinstance(source, UploadFileEstimateSource)
|
||||
)
|
||||
),
|
||||
document_model=command.doc_form,
|
||||
)
|
||||
extract_settings = [
|
||||
file_settings[source.file_id]
|
||||
if isinstance(source, UploadFileEstimateSource)
|
||||
else self._resolve_new_source(context, source, document_model=command.doc_form)
|
||||
for source in command.sources
|
||||
]
|
||||
if not extract_settings:
|
||||
return IndexingEstimate(total_segments=0, preview=[])
|
||||
return self._runner.run(
|
||||
tenant_id=context.active_workspace_id,
|
||||
extract_settings=extract_settings,
|
||||
tmp_processing_rule=process_rule,
|
||||
doc_form=command.doc_form,
|
||||
doc_language=command.doc_language,
|
||||
dataset_id=dataset_id,
|
||||
indexing_technique=command.indexing_technique,
|
||||
)
|
||||
except Exception as error:
|
||||
self._translate_error(error)
|
||||
|
||||
def estimate_document(
|
||||
self,
|
||||
context: RequestContext,
|
||||
*,
|
||||
dataset_id: str,
|
||||
document_id: str,
|
||||
) -> IndexingEstimate:
|
||||
accessible = self._dataset_access.require_accessible(context, dataset_id)
|
||||
dataset_ref = DatasetRef(accessible.workspace_id, accessible.id)
|
||||
dataset = self._require_estimate_dataset(dataset_ref)
|
||||
document = self._documents.get_estimate_document(dataset_ref.document(document_id))
|
||||
if document is None:
|
||||
raise EstimateDocumentNotFoundError()
|
||||
return self._estimate_document(context, dataset, document)
|
||||
|
||||
def estimate_batch(self, context: RequestContext, *, dataset_id: str, batch: str) -> IndexingEstimate:
|
||||
accessible = self._dataset_access.require_accessible(context, dataset_id)
|
||||
dataset_ref = DatasetRef(accessible.workspace_id, accessible.id)
|
||||
dataset = self._require_estimate_dataset(dataset_ref)
|
||||
documents = self._documents.list_estimate_documents_by_batch(dataset_ref, batch)
|
||||
if not documents:
|
||||
raise EstimateDocumentNotFoundError()
|
||||
# A batch preview uses its first document, as the existing API does.
|
||||
return self._estimate_document(context, dataset, documents[0])
|
||||
|
||||
def _estimate_document(
|
||||
self,
|
||||
context: RequestContext,
|
||||
dataset: DatasetEstimateRecord,
|
||||
document: EstimateDocumentRecord,
|
||||
) -> IndexingEstimate:
|
||||
if document.indexing_status in {"completed", "error"}:
|
||||
raise EstimateDocumentAlreadyFinishedError()
|
||||
try:
|
||||
extract_setting = self._stored_sources.resolve(StoredSource.from_document(document))
|
||||
process_rule = self._load_process_rule(dataset, document)
|
||||
return self._runner.run(
|
||||
tenant_id=context.active_workspace_id,
|
||||
extract_settings=[extract_setting],
|
||||
tmp_processing_rule=process_rule,
|
||||
doc_form=document.doc_form,
|
||||
doc_language=document.doc_language or "English",
|
||||
dataset_id=dataset.id,
|
||||
indexing_technique=dataset.indexing_technique or "economy",
|
||||
)
|
||||
except Exception as error:
|
||||
self._translate_error(error)
|
||||
|
||||
def _resolve_new_source(
|
||||
self, context: RequestContext, source: NewEstimateSource, *, document_model: str
|
||||
) -> ExtractSetting:
|
||||
if isinstance(source, NotionEstimateSource):
|
||||
return self._notion.resolve_selection(
|
||||
workspace_id=context.active_workspace_id,
|
||||
actor_id=context.account_id,
|
||||
credential_id=source.credential_id,
|
||||
notion_workspace_id=source.workspace_id,
|
||||
page_id=source.page_id,
|
||||
page_type=source.page_type,
|
||||
document_model=document_model,
|
||||
)
|
||||
if isinstance(source, WebsiteEstimateSource):
|
||||
return self._websites.resolve_selection(
|
||||
workspace_id=context.active_workspace_id,
|
||||
provider=source.provider,
|
||||
job_id=source.job_id,
|
||||
url=source.url,
|
||||
mode=source.mode,
|
||||
only_main_content=source.only_main_content,
|
||||
document_model=document_model,
|
||||
)
|
||||
raise UnsupportedEstimateSourceError(source.source_type)
|
||||
|
||||
def _load_process_rule(
|
||||
self,
|
||||
dataset: DatasetEstimateRecord,
|
||||
document: EstimateDocumentRecord,
|
||||
) -> Mapping[str, object]:
|
||||
process_rule_id = document.dataset_process_rule_id
|
||||
if not process_rule_id:
|
||||
return {"mode": "automatic", "rules": {}}
|
||||
process_rule = self._process_rules.get_by_id(
|
||||
dataset_ref=DatasetRef(dataset.workspace_id, dataset.id),
|
||||
process_rule_id=process_rule_id,
|
||||
)
|
||||
if process_rule is None:
|
||||
raise EstimateSourceNotFoundError(process_rule_id)
|
||||
return process_rule
|
||||
|
||||
@staticmethod
|
||||
def _translate_error(error: Exception) -> Never:
|
||||
if isinstance(error, (IndexingEstimateApplicationError, CredentialRefreshUnavailableError)):
|
||||
raise error
|
||||
if isinstance(error, UnsupportedStoredSourceError):
|
||||
raise UnsupportedEstimateSourceError(error.source_type) from error
|
||||
if isinstance(error, (DatasourceCredentialError, SourceCredentialUnavailableError)):
|
||||
raise IndexingEstimateCredentialUnavailableError(str(error)) from error
|
||||
if isinstance(error, IndexingInputSourceError):
|
||||
raise EstimateSourceNotFoundError(str(error)) from error
|
||||
raise IndexingEstimateExecutionError(str(error)) from error
|
||||
|
||||
def _require_estimate_dataset(self, dataset_ref: DatasetRef) -> DatasetEstimateRecord:
|
||||
dataset = self._datasets.get_estimate_record(dataset_ref)
|
||||
if dataset is None:
|
||||
raise DatasetNotFoundError()
|
||||
return dataset
|
||||
@@ -0,0 +1,178 @@
|
||||
"""Document indexing phases and their application-owned persistence and execution ports."""
|
||||
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Literal, Protocol
|
||||
|
||||
from core.credit_usage import CreditUsageCreatedBy
|
||||
from core.model_context import with_credit_usage_created_by
|
||||
from core.rag.models.document import Document
|
||||
from services.knowledge.indexing.errors import DocumentIsDeletedPausedError, DocumentIsPausedError
|
||||
from services.knowledge.indexing.estimate import StoredSource
|
||||
from services.knowledge.resource_scope import DocumentRef
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class IndexingDocument:
|
||||
source: StoredSource
|
||||
processing_rule: Mapping[str, object]
|
||||
doc_language: str
|
||||
need_summary: bool
|
||||
|
||||
@property
|
||||
def ref(self) -> DocumentRef:
|
||||
return self.source.document_ref
|
||||
|
||||
@property
|
||||
def doc_form(self) -> str:
|
||||
return self.source.document_model
|
||||
|
||||
|
||||
class IndexingDocumentStore(Protocol):
|
||||
def get_indexing_document(self, ref: DocumentRef) -> IndexingDocument | None: ...
|
||||
|
||||
def mark_splitting(self, ref: DocumentRef) -> None: ...
|
||||
|
||||
def complete_indexing(self, ref: DocumentRef, *, tokens: int, latency: float) -> None: ...
|
||||
|
||||
def fail_indexing(self, ref: DocumentRef, error: str) -> None: ...
|
||||
|
||||
|
||||
class IndexingSegmentStore(Protocol):
|
||||
def clear_for_indexing(self, document: IndexingDocument) -> None: ...
|
||||
|
||||
def save_for_indexing(
|
||||
self, document: IndexingDocument, chunks: list[Document], token_counts: list[int]
|
||||
) -> None: ...
|
||||
|
||||
def resume_indexing(self, document: IndexingDocument) -> tuple[list[Document], int]: ...
|
||||
|
||||
|
||||
class IndexingBackend(Protocol):
|
||||
def describe_error(self, error: Exception) -> str:
|
||||
"""Translate backend-specific exceptions into the persisted error message."""
|
||||
...
|
||||
|
||||
def check_paused(self, ref: DocumentRef) -> None: ...
|
||||
|
||||
def extract(self, document: IndexingDocument) -> list[Document]: ...
|
||||
|
||||
def transform(self, document: IndexingDocument, texts: list[Document]) -> list[Document]: ...
|
||||
|
||||
def ensure_admission(self, document: IndexingDocument, chunks: list[Document]) -> None: ...
|
||||
|
||||
def count_tokens(self, document: IndexingDocument, chunks: list[Document]) -> list[int]: ...
|
||||
|
||||
def load(self, document: IndexingDocument, chunks: list[Document]) -> None: ...
|
||||
|
||||
def discard(self, document: IndexingDocument, chunks: list[Document]) -> None:
|
||||
"""Remove index entries that ``load`` may have written for a document deleted meanwhile."""
|
||||
...
|
||||
|
||||
|
||||
class DocumentIndexingService:
|
||||
"""Coordinate committed phases; no caller session is held across execution."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
documents: IndexingDocumentStore,
|
||||
segments: IndexingSegmentStore,
|
||||
backend: IndexingBackend,
|
||||
enforce_vector_space_admission: bool = False,
|
||||
clock: Callable[[], float] = time.perf_counter,
|
||||
) -> None:
|
||||
self._documents = documents
|
||||
self._segments = segments
|
||||
self._backend = backend
|
||||
self._enforce_vector_space_admission = enforce_vector_space_admission
|
||||
self._clock = clock
|
||||
|
||||
@with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)
|
||||
def run(self, document_refs: Sequence[DocumentRef]) -> None:
|
||||
for ref in document_refs:
|
||||
self._run(ref, start="parsing")
|
||||
|
||||
@with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)
|
||||
def run_in_splitting_status(self, ref: DocumentRef) -> None:
|
||||
self._run(ref, start="splitting")
|
||||
|
||||
@with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)
|
||||
def run_in_indexing_status(self, ref: DocumentRef) -> None:
|
||||
self._run(ref, start="indexing")
|
||||
|
||||
def _run(self, ref: DocumentRef, *, start: Literal["parsing", "splitting", "indexing"]) -> None:
|
||||
# None: no index write has started, so a deletion leaves nothing behind.
|
||||
# Set: load may have written these chunks; document cleanup snapshots segment
|
||||
# node ids before deleting them and cannot see entries written after its snapshot,
|
||||
# so whichever way this run stops, the entries are discarded if the document is gone.
|
||||
loading: tuple[IndexingDocument, list[Document]] | None = None
|
||||
try:
|
||||
document = self._documents.get_indexing_document(ref)
|
||||
if document is None:
|
||||
logger.info("Document not found, skipping indexing: %s", ref)
|
||||
return
|
||||
self._backend.check_paused(ref)
|
||||
if start == "indexing":
|
||||
chunks, total_tokens = self._segments.resume_indexing(document)
|
||||
else:
|
||||
if start == "splitting":
|
||||
self._segments.clear_for_indexing(document)
|
||||
texts = self._backend.extract(document)
|
||||
self._documents.mark_splitting(ref)
|
||||
chunks = self._backend.transform(document, texts)
|
||||
# Recovery retains its existing admission policy; only normal
|
||||
# indexing callers that explicitly opt in reserve vector space.
|
||||
if start == "parsing" and self._enforce_vector_space_admission:
|
||||
self._backend.ensure_admission(document, chunks)
|
||||
token_counts = self._backend.count_tokens(document, chunks)
|
||||
total_tokens = sum(token_counts)
|
||||
self._segments.save_for_indexing(document, chunks, token_counts)
|
||||
|
||||
started = self._clock()
|
||||
self._backend.check_paused(ref)
|
||||
loading = (document, chunks)
|
||||
self._backend.load(document, chunks)
|
||||
self._backend.check_paused(ref)
|
||||
self._documents.complete_indexing(ref, tokens=total_tokens, latency=self._clock() - started)
|
||||
except DocumentIsPausedError:
|
||||
# The pause flag is checked before document existence, so a paused document
|
||||
# deleted mid-write surfaces here rather than as a deletion.
|
||||
self._discard_if_deleted(ref, loading)
|
||||
raise
|
||||
except DocumentIsDeletedPausedError:
|
||||
logger.info("Document deleted during indexing: %s", ref)
|
||||
self._discard(ref, loading)
|
||||
except Exception as error:
|
||||
logger.exception("Document indexing failed: %s", ref)
|
||||
self._documents.fail_indexing(ref, self._backend.describe_error(error))
|
||||
self._discard_if_deleted(ref, loading)
|
||||
|
||||
def _discard(self, ref: DocumentRef, loading: tuple[IndexingDocument, list[Document]] | None) -> None:
|
||||
"""Best-effort removal of entries ``load`` may have written; failures never mask the outcome."""
|
||||
if loading is None:
|
||||
return
|
||||
try:
|
||||
self._backend.discard(*loading)
|
||||
except Exception:
|
||||
logger.exception("Failed to discard index entries of document deleted during indexing: %s", ref)
|
||||
|
||||
def _discard_if_deleted(self, ref: DocumentRef, loading: tuple[IndexingDocument, list[Document]] | None) -> None:
|
||||
"""Discard written entries when indexing stopped for another reason and the document is gone."""
|
||||
if loading is None:
|
||||
return
|
||||
try:
|
||||
self._documents.get_indexing_document(ref)
|
||||
except DocumentIsDeletedPausedError:
|
||||
logger.info("Document deleted during indexing: %s", ref)
|
||||
self._discard(ref, loading)
|
||||
except DocumentIsPausedError:
|
||||
return
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Could not check whether document still exists after indexing stopped: %s", ref, exc_info=True
|
||||
)
|
||||
Whitespace-only changes.
@@ -0,0 +1,125 @@
|
||||
"""Metadata use cases with explicit ownership and persistence contracts."""
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Protocol, TypedDict
|
||||
|
||||
from core.rag.index_processor.constant.built_in_field import BuiltInField
|
||||
from machinery.context import RequestContext
|
||||
from services.knowledge.dataset_access import DatasetAccess
|
||||
from services.knowledge.resource_scope import DatasetRef
|
||||
|
||||
|
||||
class MetadataInput(Protocol):
|
||||
@property
|
||||
def name(self) -> str: ...
|
||||
@property
|
||||
def type(self) -> str: ...
|
||||
|
||||
|
||||
class MetadataValue(Protocol):
|
||||
@property
|
||||
def id(self) -> str: ...
|
||||
@property
|
||||
def value(self) -> str | int | float | None: ...
|
||||
|
||||
|
||||
class MetadataOperation(Protocol):
|
||||
@property
|
||||
def document_id(self) -> str: ...
|
||||
@property
|
||||
def metadata_list(self) -> Sequence[MetadataValue]: ...
|
||||
@property
|
||||
def partial_update(self) -> bool: ...
|
||||
|
||||
|
||||
class MetadataOperationsInput(Protocol):
|
||||
@property
|
||||
def operation_data(self) -> Sequence[MetadataOperation]: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MetadataRecord:
|
||||
id: str
|
||||
name: str
|
||||
type: str
|
||||
tenant_id: str
|
||||
dataset_id: str
|
||||
created_by: str
|
||||
created_at: datetime
|
||||
updated_by: str | None
|
||||
updated_at: datetime | None
|
||||
|
||||
|
||||
class MetadataFieldSummary(TypedDict):
|
||||
id: str
|
||||
name: str
|
||||
type: str
|
||||
count: int
|
||||
|
||||
|
||||
class MetadataList(TypedDict):
|
||||
doc_metadata: list[MetadataFieldSummary]
|
||||
built_in_field_enabled: bool
|
||||
|
||||
|
||||
class MetadataStore(Protocol):
|
||||
"""Own bounded transactions and recheck the complete dataset owner chain."""
|
||||
|
||||
def create(self, ref: DatasetRef, *, name: str, field_type: str, actor_id: str) -> MetadataRecord: ...
|
||||
def rename(self, ref: DatasetRef, metadata_id: str, *, name: str, actor_id: str) -> MetadataRecord: ...
|
||||
def delete(self, ref: DatasetRef, metadata_id: str) -> MetadataRecord: ...
|
||||
def set_built_in(self, ref: DatasetRef, *, enabled: bool) -> None: ...
|
||||
def update_documents(self, ref: DatasetRef, operations: Sequence[MetadataOperation], *, actor_id: str) -> None: ...
|
||||
def list_fields(self, ref: DatasetRef) -> MetadataList: ...
|
||||
|
||||
|
||||
class MetadataService:
|
||||
def __init__(self, *, store: MetadataStore, dataset_access: DatasetAccess) -> None:
|
||||
self._store = store
|
||||
self._access = dataset_access
|
||||
|
||||
def require_dataset(self, context: RequestContext, dataset_id: str) -> DatasetRef:
|
||||
dataset = self._access.require_accessible(context, dataset_id)
|
||||
return DatasetRef(dataset.workspace_id, dataset.id)
|
||||
|
||||
@staticmethod
|
||||
def _validate_name(name: str) -> None:
|
||||
if len(name) > 255:
|
||||
raise ValueError("Metadata name cannot exceed 255 characters.")
|
||||
if name in BuiltInField:
|
||||
raise ValueError("Metadata name already exists in Built-in fields.")
|
||||
|
||||
def create_metadata(self, ref: DatasetRef, args: MetadataInput, *, actor_id: str) -> MetadataRecord:
|
||||
self._validate_name(args.name)
|
||||
return self._store.create(ref, name=args.name, field_type=args.type, actor_id=actor_id)
|
||||
|
||||
def update_metadata_name(self, ref: DatasetRef, metadata_id: str, name: str, *, actor_id: str) -> MetadataRecord:
|
||||
self._validate_name(name)
|
||||
return self._store.rename(ref, metadata_id, name=name, actor_id=actor_id)
|
||||
|
||||
def delete_metadata(self, ref: DatasetRef, metadata_id: str) -> MetadataRecord:
|
||||
return self._store.delete(ref, metadata_id)
|
||||
|
||||
@staticmethod
|
||||
def get_built_in_fields() -> list[dict[str, str]]:
|
||||
return [
|
||||
{
|
||||
"name": field.value,
|
||||
"type": "time" if field in (BuiltInField.upload_date, BuiltInField.last_update_date) else "string",
|
||||
}
|
||||
for field in BuiltInField
|
||||
]
|
||||
|
||||
def enable_built_in_field(self, ref: DatasetRef) -> None:
|
||||
self._store.set_built_in(ref, enabled=True)
|
||||
|
||||
def disable_built_in_field(self, ref: DatasetRef) -> None:
|
||||
self._store.set_built_in(ref, enabled=False)
|
||||
|
||||
def update_documents_metadata(self, ref: DatasetRef, args: MetadataOperationsInput, *, actor_id: str) -> None:
|
||||
self._store.update_documents(ref, args.operation_data, actor_id=actor_id)
|
||||
|
||||
def get_dataset_metadatas(self, ref: DatasetRef) -> MetadataList:
|
||||
return self._store.list_fields(ref)
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Typed owner chains used by knowledge application and persistence ports."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import NamedTuple
|
||||
|
||||
|
||||
class DatasetRef(NamedTuple):
|
||||
"""Identify a dataset inside its owning workspace."""
|
||||
|
||||
tenant_id: str
|
||||
dataset_id: str
|
||||
|
||||
def document(self, document_id: str) -> DocumentRef:
|
||||
return DocumentRef(dataset=self, document_id=document_id)
|
||||
|
||||
|
||||
class DocumentRef(NamedTuple):
|
||||
"""Identify a document through its complete dataset owner chain."""
|
||||
|
||||
dataset: DatasetRef
|
||||
document_id: str
|
||||
|
||||
def segment(self, segment_id: str) -> SegmentRef:
|
||||
return SegmentRef(document=self, segment_id=segment_id)
|
||||
|
||||
|
||||
class SegmentRef(NamedTuple):
|
||||
"""Identify a segment through its complete document owner chain."""
|
||||
|
||||
document: DocumentRef
|
||||
segment_id: str
|
||||
@@ -0,0 +1 @@
|
||||
"""Knowledge retrieval authorization and application operations."""
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Grant execution-local file access only for validated knowledge retrieval results."""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from core.app.file_access import (
|
||||
get_current_file_access_scope,
|
||||
grant_retriever_segment_access,
|
||||
grant_upload_file_access,
|
||||
)
|
||||
from models.dataset import DocumentSegment
|
||||
from models.model import UploadFile
|
||||
from repositories.knowledge.dataset_read_repository import get_segment_attachment_files, is_retrieved_segment_owned
|
||||
|
||||
|
||||
def authorize_retrieved_segment(
|
||||
segment: DocumentSegment, *, tenant_id: str, dataset_ids: Sequence[str], session: Session
|
||||
) -> Sequence[UploadFile] | None:
|
||||
"""Authorize the owner chain and attachments before content links are signed.
|
||||
|
||||
A valid segment with no attachments returns an empty sequence; an invalid
|
||||
retrieval result returns None and must not be included in the response.
|
||||
"""
|
||||
scope = get_current_file_access_scope()
|
||||
if scope is not None and scope.tenant_id != tenant_id:
|
||||
return None
|
||||
if not is_retrieved_segment_owned(segment, tenant_id=tenant_id, dataset_ids=dataset_ids, session=session):
|
||||
return None
|
||||
attachments = get_segment_attachment_files(segment, session=session)
|
||||
grant_retriever_segment_access([segment.id])
|
||||
grant_upload_file_access(file.id for file in attachments)
|
||||
return attachments
|
||||
@@ -0,0 +1 @@
|
||||
"""Dataset segment use cases."""
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Infrastructure adapters for dataset segment use cases."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any, Protocol
|
||||
|
||||
from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
|
||||
from core.model_manager import ModelManager
|
||||
from graphon.model_runtime.entities.model_entities import ModelType
|
||||
from services.knowledge.segments.application import (
|
||||
SegmentDatasetRecord,
|
||||
SegmentModelProviderError,
|
||||
)
|
||||
|
||||
|
||||
class RedisSegmentClient(Protocol):
|
||||
def get(self, name: str | bytes) -> Any: ...
|
||||
|
||||
def setnx(self, name: str | bytes, value: Any) -> Any: ...
|
||||
def setex(self, name: str, time: int, value: Any) -> Any: ...
|
||||
def lock(self, name: str, timeout: int) -> Any: ...
|
||||
|
||||
|
||||
class ModelManagerSegmentGuard:
|
||||
def check(self, dataset: SegmentDatasetRecord) -> None:
|
||||
try:
|
||||
ModelManager.for_tenant(tenant_id=dataset.workspace_id).get_model_instance(
|
||||
tenant_id=dataset.workspace_id,
|
||||
provider=dataset.embedding_model_provider or "",
|
||||
model_type=ModelType.TEXT_EMBEDDING,
|
||||
model=dataset.embedding_model or "",
|
||||
)
|
||||
except LLMBadRequestError as error:
|
||||
raise SegmentModelProviderError(kind="bad_request", description=str(error)) from error
|
||||
except ProviderTokenNotInitError as error:
|
||||
raise SegmentModelProviderError(kind="token", description=error.description) from error
|
||||
|
||||
|
||||
class RedisSegmentIndexingState:
|
||||
def __init__(self, redis: RedisSegmentClient) -> None:
|
||||
self._redis = redis
|
||||
|
||||
def lock(self, name: str, *, timeout: int):
|
||||
return self._redis.lock(name, timeout=timeout)
|
||||
|
||||
def is_segment_indexing(self, segment_id: str, *, deleting: bool = False) -> bool:
|
||||
suffix = "delete_indexing" if deleting else "indexing"
|
||||
return self._redis.get(f"segment_{segment_id}_{suffix}") is not None
|
||||
|
||||
def mark_segment_indexing(self, segment_id: str, *, deleting: bool = False) -> None:
|
||||
suffix = "delete_indexing" if deleting else "indexing"
|
||||
self._redis.setex(f"segment_{segment_id}_{suffix}", 600, 1)
|
||||
|
||||
def is_document_indexing(self, document_id: str) -> bool:
|
||||
return self._redis.get(f"document_{document_id}_indexing") is not None
|
||||
|
||||
def set_batch_waiting(self, job_id: str) -> None:
|
||||
self._redis.setnx(f"segment_batch_import_{job_id}", "waiting")
|
||||
|
||||
def get_batch_status(self, job_id: str) -> str | None:
|
||||
value = self._redis.get(f"segment_batch_import_{job_id}")
|
||||
if value is None:
|
||||
return None
|
||||
return value.decode() if isinstance(value, bytes) else value
|
||||
|
||||
|
||||
class CelerySegmentBatchImportDispatcher:
|
||||
def __init__(self, *, delay: Callable[..., object]) -> None:
|
||||
self._delay = delay
|
||||
|
||||
def dispatch(
|
||||
self,
|
||||
*,
|
||||
job_id: str,
|
||||
upload_file_id: str,
|
||||
dataset_id: str,
|
||||
document_id: str,
|
||||
workspace_id: str,
|
||||
actor_id: str,
|
||||
) -> None:
|
||||
self._delay(job_id, upload_file_id, dataset_id, document_id, workspace_id, actor_id)
|
||||
@@ -0,0 +1,966 @@
|
||||
"""Application service for dataset segment and child-chunk HTTP use cases."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections import Counter
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from contextlib import AbstractContextManager
|
||||
from dataclasses import dataclass, replace
|
||||
from typing import Literal, Protocol
|
||||
from uuid import uuid4
|
||||
|
||||
from libs.datetime_utils import naive_utc_now
|
||||
from machinery.context import RequestContext
|
||||
from services.knowledge.dataset_access import DatasetAccess, DatasetAccessDeniedError, DatasetAccessSnapshot
|
||||
from services.knowledge.entities.segments import (
|
||||
ChildChunkRecord,
|
||||
ChildChunkUpdateArgs,
|
||||
SegmentRecord,
|
||||
SegmentUpdateArgs,
|
||||
)
|
||||
from services.knowledge.resource_scope import DatasetRef, DocumentRef, SegmentRef
|
||||
|
||||
_HIGH_QUALITY = "high_quality"
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SegmentDatasetRecord:
|
||||
id: str
|
||||
workspace_id: str
|
||||
indexing_technique: str
|
||||
embedding_model_provider: str | None
|
||||
embedding_model: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SegmentDocumentRecord:
|
||||
id: str
|
||||
dataset_id: str
|
||||
workspace_id: str
|
||||
doc_form: str
|
||||
|
||||
@property
|
||||
def ref(self) -> DocumentRef:
|
||||
return DatasetRef(self.workspace_id, self.dataset_id).document(self.id)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SegmentScope:
|
||||
dataset: SegmentDatasetRecord | None
|
||||
document: SegmentDocumentRecord | None
|
||||
access_snapshot: DatasetAccessSnapshot | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SegmentListFilter:
|
||||
page: int = 1
|
||||
limit: int = 20
|
||||
statuses: tuple[str, ...] = ()
|
||||
hit_count_gte: int | None = None
|
||||
enabled: str = "all"
|
||||
keyword: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ChildChunkListFilter:
|
||||
page: int = 1
|
||||
limit: int = 20
|
||||
keyword: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SegmentPage:
|
||||
items: tuple[SegmentRecord, ...]
|
||||
total: int
|
||||
total_pages: int
|
||||
page: int
|
||||
limit: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SegmentDetail:
|
||||
data: SegmentRecord
|
||||
doc_form: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ChildChunkPage:
|
||||
items: tuple[ChildChunkRecord, ...]
|
||||
total: int
|
||||
total_pages: int
|
||||
page: int
|
||||
limit: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SegmentBatchImport:
|
||||
job_id: str
|
||||
job_status: str
|
||||
|
||||
|
||||
class SegmentScopeReader(Protocol):
|
||||
def get_segment_scope(
|
||||
self,
|
||||
*,
|
||||
dataset_ref: DatasetRef,
|
||||
document_id: str,
|
||||
actor_id: str | None,
|
||||
) -> SegmentScope: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ChildChunkState:
|
||||
data: ChildChunkRecord
|
||||
index_node_id: str | None
|
||||
index_node_hash: str | None
|
||||
created_by: str
|
||||
updated_by: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SegmentIndexTarget:
|
||||
id: str
|
||||
index_node_id: str | None
|
||||
enabled: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SegmentUpdateState:
|
||||
"""Persisted fields needed to decide a segment update, without presentation data."""
|
||||
|
||||
content: str
|
||||
enabled: bool
|
||||
keywords: tuple[str, ...] | None
|
||||
# The stored QA answer, or None when the segment has none (non-QA documents).
|
||||
answer: str | None
|
||||
|
||||
|
||||
class SegmentStore(Protocol):
|
||||
"""Owner-scoped persistence; each operation owns a short session."""
|
||||
|
||||
def list_segments(self, document_ref: DocumentRef, query: SegmentListFilter) -> SegmentPage: ...
|
||||
def get_segments(self, document_ref: DocumentRef, segment_ids: Sequence[str]) -> tuple[SegmentIndexTarget, ...]: ...
|
||||
def get_segment(self, segment_ref: SegmentRef) -> SegmentDetail | None: ...
|
||||
def get_segment_update_state(self, segment_ref: SegmentRef) -> SegmentUpdateState | None: ...
|
||||
def save_segment(self, segment_ref: SegmentRef, values: Mapping[str, object], *, create: bool = False) -> None: ...
|
||||
def delete_segments(self, document_ref: DocumentRef, segment_ids: Sequence[str]) -> None: ...
|
||||
def list_child_chunks(self, segment_ref: SegmentRef, query: ChildChunkListFilter) -> ChildChunkPage | None: ...
|
||||
def get_children_for_segments(
|
||||
self, document_ref: DocumentRef, segment_ids: Sequence[str]
|
||||
) -> tuple[ChildChunkState, ...]: ...
|
||||
def set_segments(
|
||||
self, document_ref: DocumentRef, segment_ids: Sequence[str], values: Mapping[str, object]
|
||||
) -> None: ...
|
||||
def get_children(self, segment_ref: SegmentRef) -> tuple[ChildChunkState, ...] | None: ...
|
||||
def save_children(
|
||||
self,
|
||||
segment_ref: SegmentRef,
|
||||
*,
|
||||
added: Sequence[ChildChunkState] = (),
|
||||
updated: Sequence[ChildChunkState] = (),
|
||||
deleted: Sequence[ChildChunkState] = (),
|
||||
) -> None: ...
|
||||
|
||||
|
||||
class SegmentIndex(Protocol):
|
||||
def count_tokens(self, dataset: SegmentDatasetRecord, text: str) -> int: ...
|
||||
def create_many(
|
||||
self,
|
||||
document_ref: DocumentRef,
|
||||
*,
|
||||
segment_ids: Sequence[str],
|
||||
keywords_list: Sequence[Sequence[str] | None] | None,
|
||||
) -> None: ...
|
||||
def create(
|
||||
self, segment_ref: SegmentRef, *, keywords: Sequence[str] | None, attachment_ids: Sequence[str]
|
||||
) -> None: ...
|
||||
def update(self, segment_ref: SegmentRef, *, keywords: Sequence[str] | None, regenerate_children: bool) -> None: ...
|
||||
def update_attachments(self, segment_ref: SegmentRef, attachment_ids: Sequence[str]) -> None: ...
|
||||
def update_summary(self, segment_ref: SegmentRef, *, summary: str | None, content_changed: bool) -> None: ...
|
||||
def update_children(
|
||||
self,
|
||||
segment_ref: SegmentRef,
|
||||
*,
|
||||
added: Sequence[ChildChunkState] = (),
|
||||
updated: Sequence[ChildChunkState] = (),
|
||||
deleted: Sequence[ChildChunkState] = (),
|
||||
) -> None: ...
|
||||
def delete(
|
||||
self, document_ref: DocumentRef, segments: Sequence[SegmentIndexTarget], children: Sequence[ChildChunkState]
|
||||
) -> None: ...
|
||||
def change_status(
|
||||
self, document_ref: DocumentRef, segment_ids: Sequence[str], action: Literal["enable", "disable"]
|
||||
) -> None: ...
|
||||
|
||||
|
||||
class SegmentLimits(Protocol):
|
||||
SINGLE_CHUNK_ATTACHMENT_LIMIT: int
|
||||
|
||||
|
||||
class SegmentModelGuard(Protocol):
|
||||
def check(self, dataset: SegmentDatasetRecord) -> None: ...
|
||||
|
||||
|
||||
class SegmentUploadCatalog(Protocol):
|
||||
def get_file_name(self, *, workspace_id: str, upload_file_id: str) -> str | None: ...
|
||||
|
||||
|
||||
class SegmentIndexingState(Protocol):
|
||||
def lock(self, name: str, *, timeout: int) -> AbstractContextManager[object]: ...
|
||||
def is_segment_indexing(self, segment_id: str, *, deleting: bool = False) -> bool: ...
|
||||
def mark_segment_indexing(self, segment_id: str, *, deleting: bool = False) -> None: ...
|
||||
|
||||
def is_document_indexing(self, document_id: str) -> bool: ...
|
||||
|
||||
def set_batch_waiting(self, job_id: str) -> None: ...
|
||||
|
||||
def get_batch_status(self, job_id: str) -> str | None: ...
|
||||
|
||||
|
||||
class SegmentBatchImportDispatcher(Protocol):
|
||||
def dispatch(
|
||||
self,
|
||||
*,
|
||||
job_id: str,
|
||||
upload_file_id: str,
|
||||
dataset_id: str,
|
||||
document_id: str,
|
||||
workspace_id: str,
|
||||
actor_id: str,
|
||||
) -> None: ...
|
||||
|
||||
|
||||
class SegmentApplicationError(Exception):
|
||||
"""Base class for framework-neutral segment use-case errors."""
|
||||
|
||||
|
||||
class SegmentDatasetNotFoundError(SegmentApplicationError):
|
||||
pass
|
||||
|
||||
|
||||
class SegmentDocumentNotFoundError(SegmentApplicationError):
|
||||
pass
|
||||
|
||||
|
||||
class SegmentNotFoundError(SegmentApplicationError):
|
||||
pass
|
||||
|
||||
|
||||
class ChildChunkNotFoundError(SegmentApplicationError):
|
||||
pass
|
||||
|
||||
|
||||
class SegmentUploadFileNotFoundError(SegmentApplicationError):
|
||||
pass
|
||||
|
||||
|
||||
class SegmentPermissionDeniedError(SegmentApplicationError):
|
||||
pass
|
||||
|
||||
|
||||
class SegmentInvalidFileTypeError(SegmentApplicationError):
|
||||
pass
|
||||
|
||||
|
||||
class SegmentDocumentIndexingError(SegmentApplicationError):
|
||||
pass
|
||||
|
||||
|
||||
class SegmentStatusUpdateError(SegmentApplicationError):
|
||||
pass
|
||||
|
||||
|
||||
class SegmentBatchImportDispatchError(SegmentApplicationError):
|
||||
pass
|
||||
|
||||
|
||||
class SegmentBatchImportNotFoundError(SegmentApplicationError):
|
||||
pass
|
||||
|
||||
|
||||
class SegmentModelProviderError(Exception):
|
||||
"""Adapter error raised when the configured embedding model cannot be loaded."""
|
||||
|
||||
def __init__(self, *, kind: Literal["bad_request", "token"], description: str) -> None:
|
||||
super().__init__(description)
|
||||
self.kind = kind
|
||||
self.description = description
|
||||
|
||||
|
||||
class SegmentDatasetModelUnavailableError(SegmentApplicationError):
|
||||
pass
|
||||
|
||||
|
||||
class SegmentEmbeddingModelUnavailableError(SegmentApplicationError):
|
||||
pass
|
||||
|
||||
|
||||
class ChildChunkIndexingApplicationError(SegmentApplicationError):
|
||||
pass
|
||||
|
||||
|
||||
class ChildChunkDeleteIndexApplicationError(SegmentApplicationError):
|
||||
pass
|
||||
|
||||
|
||||
class DatasetSegmentApplicationService:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
dataset_access: DatasetAccess,
|
||||
scopes: SegmentScopeReader,
|
||||
store: SegmentStore,
|
||||
index: SegmentIndex,
|
||||
limits: SegmentLimits,
|
||||
text_hash: Callable[[str], str],
|
||||
uploads: SegmentUploadCatalog,
|
||||
model_guard: SegmentModelGuard,
|
||||
indexing_state: SegmentIndexingState,
|
||||
batch_dispatcher: SegmentBatchImportDispatcher,
|
||||
job_id_factory: Callable[[], str],
|
||||
) -> None:
|
||||
self._dataset_access = dataset_access
|
||||
self._scopes = scopes
|
||||
self._store = store
|
||||
self._index = index
|
||||
self._limits = limits
|
||||
self._uploads = uploads
|
||||
self._model_guard = model_guard
|
||||
self._indexing_state = indexing_state
|
||||
self._batch_dispatcher = batch_dispatcher
|
||||
self._job_id_factory = job_id_factory
|
||||
self.mutations = SegmentMutationService(
|
||||
store=store, index=index, text_hash=text_hash, indexing_state=indexing_state
|
||||
)
|
||||
|
||||
def list_segments(
|
||||
self,
|
||||
context: RequestContext,
|
||||
*,
|
||||
dataset_id: str,
|
||||
document_id: str,
|
||||
query: SegmentListFilter,
|
||||
) -> SegmentPage:
|
||||
scope = self._require_scope(context, dataset_id=dataset_id, document_id=document_id)
|
||||
return self._store.list_segments(scope.document.ref, query)
|
||||
|
||||
def delete_segments(
|
||||
self,
|
||||
context: RequestContext,
|
||||
*,
|
||||
dataset_id: str,
|
||||
document_id: str,
|
||||
segment_ids: Sequence[str],
|
||||
) -> None:
|
||||
scope = self._require_scope(context, dataset_id=dataset_id, document_id=document_id)
|
||||
self._check_dataset_model(scope.dataset)
|
||||
segments = self._store.get_segments(scope.document.ref, segment_ids)
|
||||
children = self._store.get_children_for_segments(scope.document.ref, [segment.id for segment in segments])
|
||||
if segments:
|
||||
self._index.delete(scope.document.ref, segments, children)
|
||||
self._store.delete_segments(scope.document.ref, [segment.id for segment in segments])
|
||||
|
||||
def change_segment_status(
|
||||
self,
|
||||
context: RequestContext,
|
||||
*,
|
||||
dataset_id: str,
|
||||
document_id: str,
|
||||
segment_ids: Sequence[str],
|
||||
action: Literal["enable", "disable"],
|
||||
) -> None:
|
||||
scope = self._require_scope(context, dataset_id=dataset_id, document_id=document_id)
|
||||
self._check_dataset_model(scope.dataset)
|
||||
self._check_embedding_model(scope.dataset)
|
||||
if self._indexing_state.is_document_indexing(scope.document.id):
|
||||
raise SegmentDocumentIndexingError("Document is being indexed, please try again later")
|
||||
try:
|
||||
segments = self._store.get_segments(scope.document.ref, segment_ids)
|
||||
ids = []
|
||||
for segment in segments:
|
||||
if segment.enabled == (action == "enable") or self._indexing_state.is_segment_indexing(segment.id):
|
||||
continue
|
||||
ids.append(segment.id)
|
||||
if ids:
|
||||
self._store.set_segments(
|
||||
scope.document.ref,
|
||||
ids,
|
||||
{
|
||||
"enabled": action == "enable",
|
||||
"disabled_at": naive_utc_now() if action == "disable" else None,
|
||||
"disabled_by": context.account_id if action == "disable" else None,
|
||||
},
|
||||
)
|
||||
self._index.change_status(scope.document.ref, ids, action)
|
||||
except Exception as error:
|
||||
raise SegmentStatusUpdateError(str(error)) from error
|
||||
|
||||
def create_segment(
|
||||
self,
|
||||
context: RequestContext,
|
||||
*,
|
||||
dataset_id: str,
|
||||
document_id: str,
|
||||
values: Mapping[str, object],
|
||||
) -> SegmentDetail:
|
||||
scope = self._require_scope(context, dataset_id=dataset_id, document_id=document_id)
|
||||
self._check_embedding_model(scope.dataset)
|
||||
args = self._validate_values(values, scope.document.doc_form)
|
||||
segment_ref = self.mutations.create_segment(scope, actor_id=context.account_id, args=args)
|
||||
return self._detail(segment_ref)
|
||||
|
||||
def update_segment(
|
||||
self,
|
||||
context: RequestContext,
|
||||
*,
|
||||
dataset_id: str,
|
||||
document_id: str,
|
||||
segment_id: str,
|
||||
values: Mapping[str, object],
|
||||
) -> SegmentDetail:
|
||||
scope = self._require_scope(context, dataset_id=dataset_id, document_id=document_id)
|
||||
self._check_dataset_model(scope.dataset)
|
||||
self._check_embedding_model(scope.dataset)
|
||||
args = self._validate_values(values, scope.document.doc_form)
|
||||
self.mutations.update_segment(scope, actor_id=context.account_id, segment_id=segment_id, args=args)
|
||||
return self._detail(scope.document.ref.segment(segment_id))
|
||||
|
||||
def _detail(self, segment_ref: SegmentRef) -> SegmentDetail:
|
||||
result = self._store.get_segment(segment_ref)
|
||||
if result is None:
|
||||
raise SegmentNotFoundError("Segment not found")
|
||||
return result
|
||||
|
||||
def delete_segment(
|
||||
self,
|
||||
context: RequestContext,
|
||||
*,
|
||||
dataset_id: str,
|
||||
document_id: str,
|
||||
segment_id: str,
|
||||
) -> None:
|
||||
scope = self._require_scope(context, dataset_id=dataset_id, document_id=document_id)
|
||||
self._check_dataset_model(scope.dataset)
|
||||
return self.mutations.delete_segment(scope.document.ref.segment(segment_id))
|
||||
|
||||
def start_batch_import(
|
||||
self,
|
||||
context: RequestContext,
|
||||
*,
|
||||
dataset_id: str,
|
||||
document_id: str,
|
||||
upload_file_id: str,
|
||||
) -> SegmentBatchImport:
|
||||
scope = self._require_scope(
|
||||
context,
|
||||
dataset_id=dataset_id,
|
||||
document_id=document_id,
|
||||
enforce_permission=False,
|
||||
)
|
||||
filename = self._uploads.get_file_name(
|
||||
workspace_id=scope.dataset.workspace_id,
|
||||
upload_file_id=upload_file_id,
|
||||
)
|
||||
if filename is None:
|
||||
raise SegmentUploadFileNotFoundError("UploadFile not found")
|
||||
if not filename.lower().endswith(".csv"):
|
||||
raise SegmentInvalidFileTypeError("Invalid file type. Only CSV files are allowed")
|
||||
|
||||
job_id = self._job_id_factory()
|
||||
try:
|
||||
self._indexing_state.set_batch_waiting(job_id)
|
||||
self._batch_dispatcher.dispatch(
|
||||
job_id=job_id,
|
||||
upload_file_id=upload_file_id,
|
||||
dataset_id=scope.dataset.id,
|
||||
document_id=scope.document.id,
|
||||
workspace_id=scope.dataset.workspace_id,
|
||||
actor_id=context.account_id,
|
||||
)
|
||||
except Exception as error:
|
||||
raise SegmentBatchImportDispatchError(str(error)) from error
|
||||
return SegmentBatchImport(job_id=job_id, job_status="waiting")
|
||||
|
||||
def get_batch_import_status(self, job_id: str) -> SegmentBatchImport:
|
||||
status = self._indexing_state.get_batch_status(job_id)
|
||||
if status is None:
|
||||
raise SegmentBatchImportNotFoundError("The job does not exist.")
|
||||
return SegmentBatchImport(job_id=job_id, job_status=status)
|
||||
|
||||
def create_child_chunk(
|
||||
self,
|
||||
context: RequestContext,
|
||||
*,
|
||||
dataset_id: str,
|
||||
document_id: str,
|
||||
segment_id: str,
|
||||
content: str,
|
||||
) -> ChildChunkRecord:
|
||||
scope = self._require_scope(context, dataset_id=dataset_id, document_id=document_id)
|
||||
self._check_embedding_model(scope.dataset)
|
||||
return self.mutations.create_child_chunk(
|
||||
scope.document.ref.segment(segment_id), content=content, actor_id=context.account_id
|
||||
)
|
||||
|
||||
def list_child_chunks(
|
||||
self,
|
||||
*,
|
||||
workspace_id: str,
|
||||
dataset_id: str,
|
||||
document_id: str,
|
||||
segment_id: str,
|
||||
query: ChildChunkListFilter,
|
||||
) -> ChildChunkPage:
|
||||
scope = self._require_scope_without_actor(
|
||||
workspace_id=workspace_id,
|
||||
dataset_id=dataset_id,
|
||||
document_id=document_id,
|
||||
)
|
||||
self._check_dataset_model(scope.dataset)
|
||||
result = self._store.list_child_chunks(scope.document.ref.segment(segment_id), query)
|
||||
if result is None:
|
||||
raise SegmentNotFoundError("Segment not found")
|
||||
return result
|
||||
|
||||
def update_child_chunks(
|
||||
self,
|
||||
context: RequestContext,
|
||||
*,
|
||||
dataset_id: str,
|
||||
document_id: str,
|
||||
segment_id: str,
|
||||
chunks: Sequence[ChildChunkUpdateArgs],
|
||||
) -> tuple[ChildChunkRecord, ...]:
|
||||
scope = self._require_scope(context, dataset_id=dataset_id, document_id=document_id)
|
||||
self._check_dataset_model(scope.dataset)
|
||||
return self.mutations.update_child_chunks(
|
||||
scope.document.ref.segment(segment_id), chunks=chunks, actor_id=context.account_id
|
||||
)
|
||||
|
||||
def delete_child_chunk(
|
||||
self,
|
||||
context: RequestContext,
|
||||
*,
|
||||
dataset_id: str,
|
||||
document_id: str,
|
||||
segment_id: str,
|
||||
child_chunk_id: str,
|
||||
) -> None:
|
||||
scope = self._require_scope(context, dataset_id=dataset_id, document_id=document_id)
|
||||
self._check_dataset_model(scope.dataset)
|
||||
return self.mutations.delete_child_chunk(scope.document.ref.segment(segment_id), child_chunk_id=child_chunk_id)
|
||||
|
||||
def update_child_chunk(
|
||||
self,
|
||||
context: RequestContext,
|
||||
*,
|
||||
dataset_id: str,
|
||||
document_id: str,
|
||||
segment_id: str,
|
||||
child_chunk_id: str,
|
||||
content: str,
|
||||
) -> ChildChunkRecord:
|
||||
scope = self._require_scope(context, dataset_id=dataset_id, document_id=document_id)
|
||||
self._check_dataset_model(scope.dataset)
|
||||
return self.mutations.update_child_chunk(
|
||||
scope.document.ref.segment(segment_id),
|
||||
child_chunk_id=child_chunk_id,
|
||||
content=content,
|
||||
actor_id=context.account_id,
|
||||
)
|
||||
|
||||
def _validate_values(self, values: Mapping[str, object], doc_form: str) -> SegmentUpdateArgs:
|
||||
return validate_segment_values(
|
||||
values, doc_form=doc_form, attachment_limit=self._limits.SINGLE_CHUNK_ATTACHMENT_LIMIT
|
||||
)
|
||||
|
||||
def _require_scope(
|
||||
self,
|
||||
context: RequestContext,
|
||||
*,
|
||||
dataset_id: str,
|
||||
document_id: str,
|
||||
enforce_permission: bool = True,
|
||||
) -> SegmentMutationScope:
|
||||
scope = self._scopes.get_segment_scope(
|
||||
dataset_ref=DatasetRef(context.active_workspace_id, dataset_id),
|
||||
document_id=document_id,
|
||||
actor_id=context.account_id if enforce_permission else None,
|
||||
)
|
||||
dataset = scope.dataset
|
||||
if dataset is None:
|
||||
raise SegmentDatasetNotFoundError("Dataset not found")
|
||||
if enforce_permission:
|
||||
if scope.access_snapshot is None:
|
||||
raise SegmentPermissionDeniedError("Dataset access state is missing")
|
||||
try:
|
||||
self._dataset_access.check_access(context, scope.access_snapshot)
|
||||
except DatasetAccessDeniedError as error:
|
||||
raise SegmentPermissionDeniedError(str(error)) from error
|
||||
document = scope.document
|
||||
if document is None:
|
||||
raise SegmentDocumentNotFoundError("Document not found")
|
||||
return SegmentMutationScope(dataset=dataset, document=document)
|
||||
|
||||
def _require_scope_without_actor(
|
||||
self,
|
||||
*,
|
||||
workspace_id: str,
|
||||
dataset_id: str,
|
||||
document_id: str,
|
||||
) -> SegmentMutationScope:
|
||||
scope = self._scopes.get_segment_scope(
|
||||
dataset_ref=DatasetRef(workspace_id, dataset_id),
|
||||
document_id=document_id,
|
||||
actor_id=None,
|
||||
)
|
||||
if scope.dataset is None:
|
||||
raise SegmentDatasetNotFoundError("Dataset not found")
|
||||
if scope.document is None:
|
||||
raise SegmentDocumentNotFoundError("Document not found")
|
||||
return SegmentMutationScope(dataset=scope.dataset, document=scope.document)
|
||||
|
||||
def _check_dataset_model(self, dataset: SegmentDatasetRecord) -> None:
|
||||
if dataset.indexing_technique != _HIGH_QUALITY:
|
||||
return
|
||||
try:
|
||||
self._model_guard.check(dataset)
|
||||
except SegmentModelProviderError as error:
|
||||
if error.kind == "token":
|
||||
message = f"The dataset is unavailable, due to: {error.description}"
|
||||
else:
|
||||
message = (
|
||||
"No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
|
||||
)
|
||||
raise SegmentDatasetModelUnavailableError(message) from error
|
||||
|
||||
def _check_embedding_model(self, dataset: SegmentDatasetRecord) -> None:
|
||||
if dataset.indexing_technique != _HIGH_QUALITY:
|
||||
return
|
||||
try:
|
||||
self._model_guard.check(dataset)
|
||||
except SegmentModelProviderError as error:
|
||||
if error.kind == "bad_request":
|
||||
message = (
|
||||
"No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
|
||||
)
|
||||
else:
|
||||
message = error.description
|
||||
raise SegmentEmbeddingModelUnavailableError(message) from error
|
||||
|
||||
|
||||
class SegmentMutationService:
|
||||
"""Shared writes for Console and Service API after their own admission checks.
|
||||
|
||||
Callers supply an owner chain reconstructed from authorized database rows.
|
||||
Persistence ports recheck that chain and own their transactions. Partial
|
||||
update arguments are accepted here; transport validation stays at admission.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
store: SegmentStore,
|
||||
index: SegmentIndex,
|
||||
text_hash: Callable[[str], str],
|
||||
indexing_state: SegmentIndexingState,
|
||||
) -> None:
|
||||
self._store = store
|
||||
self._index = index
|
||||
self._text_hash = text_hash
|
||||
self._indexing_state = indexing_state
|
||||
|
||||
def index_segments(
|
||||
self,
|
||||
document_ref: DocumentRef,
|
||||
*,
|
||||
segment_ids: Sequence[str],
|
||||
keywords_list: Sequence[Sequence[str] | None] | None = None,
|
||||
) -> None:
|
||||
"""Index a committed batch and persist failures through the scoped store.
|
||||
|
||||
Callers must finish their write transaction before invoking this method.
|
||||
The index adapter reloads the complete owner chain before external I/O.
|
||||
"""
|
||||
if keywords_list is not None and len(keywords_list) != len(segment_ids):
|
||||
raise ValueError("Keywords must correspond to the segment batch")
|
||||
if not segment_ids:
|
||||
return
|
||||
try:
|
||||
self._index.create_many(document_ref, segment_ids=segment_ids, keywords_list=keywords_list)
|
||||
except Exception as error:
|
||||
logger.exception("Batch indexing failed for document %s", document_ref.document_id)
|
||||
self._store.set_segments(
|
||||
document_ref,
|
||||
segment_ids,
|
||||
{"enabled": False, "disabled_at": naive_utc_now(), "status": "error", "error": str(error)},
|
||||
)
|
||||
raise
|
||||
|
||||
def create_segment(
|
||||
self,
|
||||
scope: SegmentMutationScope,
|
||||
*,
|
||||
actor_id: str,
|
||||
args: SegmentUpdateArgs,
|
||||
) -> SegmentRef:
|
||||
now = naive_utc_now()
|
||||
segment_ref = scope.document.ref.segment(str(uuid4()))
|
||||
saved = False
|
||||
try:
|
||||
with self._indexing_state.lock(f"add_segment_lock_document_id_{scope.document.id}", timeout=600):
|
||||
self._store.save_segment(
|
||||
segment_ref,
|
||||
{
|
||||
"content": args.content,
|
||||
"answer": args.answer if scope.document.doc_form == "qa_model" else None,
|
||||
"word_count": len(args.content or "")
|
||||
+ (len(args.answer or "") if scope.document.doc_form == "qa_model" else 0),
|
||||
"tokens": self._index.count_tokens(scope.dataset, args.content or ""),
|
||||
"index_node_id": str(uuid4()),
|
||||
"index_node_hash": self._text_hash(args.content or ""),
|
||||
"created_by": actor_id,
|
||||
"status": "completed",
|
||||
"indexing_at": now,
|
||||
"completed_at": now,
|
||||
},
|
||||
create=True,
|
||||
)
|
||||
saved = True
|
||||
except Exception as error:
|
||||
if saved:
|
||||
self._mark_error(segment_ref, error)
|
||||
raise
|
||||
try:
|
||||
self._index.create(segment_ref, keywords=args.keywords, attachment_ids=args.attachment_ids or ())
|
||||
except Exception as error:
|
||||
self._mark_error(segment_ref, error)
|
||||
return segment_ref
|
||||
|
||||
def update_segment(
|
||||
self,
|
||||
scope: SegmentMutationScope,
|
||||
*,
|
||||
actor_id: str,
|
||||
segment_id: str,
|
||||
args: SegmentUpdateArgs,
|
||||
) -> None:
|
||||
segment_ref = scope.document.ref.segment(segment_id)
|
||||
previous = self._store.get_segment_update_state(segment_ref)
|
||||
if previous is None:
|
||||
raise SegmentNotFoundError("Segment not found")
|
||||
if self._indexing_state.is_segment_indexing(segment_id):
|
||||
raise ValueError("Segment is indexing, please try again later")
|
||||
if args.enabled is True and not previous.enabled and args.model_fields_set == {"enabled"}:
|
||||
# Disabled segments have no vectors to update. Reuse the console's load-based enable task.
|
||||
self._store.save_segment(segment_ref, {"enabled": True, "disabled_at": None, "disabled_by": None})
|
||||
self._index.change_status(scope.document.ref, [segment_id], "enable")
|
||||
return
|
||||
if args.enabled is False and previous.enabled:
|
||||
self._store.save_segment(
|
||||
segment_ref, {"enabled": False, "disabled_at": naive_utc_now(), "disabled_by": actor_id}
|
||||
)
|
||||
self._indexing_state.mark_segment_indexing(segment_id)
|
||||
self._index.change_status(scope.document.ref, [segment_id], "disable")
|
||||
return
|
||||
if not previous.enabled and args.enabled is not True:
|
||||
raise ValueError("Can't update disabled segment")
|
||||
content = args.content or previous.content
|
||||
answer = args.answer if "answer" in args.model_fields_set else previous.answer
|
||||
content_changed = content != previous.content
|
||||
keyword_changed = bool(args.keywords) and Counter(previous.keywords or ()) != Counter(args.keywords or ())
|
||||
changes: dict[str, object] = {
|
||||
"content": content,
|
||||
"word_count": len(content) + (len(answer or "") if scope.document.doc_form == "qa_model" else 0),
|
||||
"enabled": True,
|
||||
"disabled_at": None,
|
||||
"disabled_by": None,
|
||||
}
|
||||
if scope.document.doc_form == "qa_model":
|
||||
changes["answer"] = answer
|
||||
if keyword_changed and not content_changed:
|
||||
changes["keywords"] = args.keywords
|
||||
try:
|
||||
if content_changed:
|
||||
now = naive_utc_now()
|
||||
changes.update(
|
||||
index_node_hash=self._text_hash(content),
|
||||
tokens=self._index.count_tokens(
|
||||
scope.dataset,
|
||||
content + (answer or "") if scope.document.doc_form == "qa_model" else content,
|
||||
),
|
||||
status="completed",
|
||||
indexing_at=now,
|
||||
completed_at=now,
|
||||
updated_at=now,
|
||||
updated_by=actor_id,
|
||||
)
|
||||
self._store.save_segment(segment_ref, changes)
|
||||
if scope.document.doc_form == "hierarchical_model":
|
||||
if args.regenerate_child_chunks:
|
||||
self._index.update(segment_ref, keywords=args.keywords, regenerate_children=True)
|
||||
elif content_changed or args.enabled or keyword_changed:
|
||||
self._index.update(segment_ref, keywords=args.keywords, regenerate_children=False)
|
||||
if scope.dataset.indexing_technique == _HIGH_QUALITY:
|
||||
try:
|
||||
self._index.update_summary(segment_ref, summary=args.summary, content_changed=content_changed)
|
||||
except Exception:
|
||||
logger.exception("Failed to update summary for segment %s", segment_id)
|
||||
if args.attachment_ids is not None:
|
||||
self._index.update_attachments(segment_ref, args.attachment_ids)
|
||||
except Exception as error:
|
||||
self._mark_error(segment_ref, error)
|
||||
|
||||
def delete_segment(self, segment_ref: SegmentRef) -> None:
|
||||
segments = self._store.get_segments(segment_ref.document, (segment_ref.segment_id,))
|
||||
if not segments:
|
||||
raise SegmentNotFoundError("Segment not found")
|
||||
segment = segments[0]
|
||||
if self._indexing_state.is_segment_indexing(segment_ref.segment_id, deleting=True):
|
||||
raise ValueError("Segment is deleting.")
|
||||
if segment.enabled:
|
||||
self._indexing_state.mark_segment_indexing(segment_ref.segment_id, deleting=True)
|
||||
self._index.delete(
|
||||
segment_ref.document,
|
||||
(segment,),
|
||||
self._children(segment_ref),
|
||||
)
|
||||
self._store.delete_segments(segment_ref.document, (segment_ref.segment_id,))
|
||||
|
||||
def create_child_chunk(self, segment_ref: SegmentRef, *, content: str, actor_id: str) -> ChildChunkRecord:
|
||||
with self._indexing_state.lock(f"add_child_lock_{segment_ref.segment_id}", timeout=20):
|
||||
children = self._children(segment_ref)
|
||||
child = self._new_child(
|
||||
segment_ref, content, max((c.data.position for c in children), default=0) + 1, actor_id
|
||||
)
|
||||
self._write_children(segment_ref, added=(child,))
|
||||
return child.data
|
||||
|
||||
def update_child_chunks(
|
||||
self, segment_ref: SegmentRef, *, chunks: Sequence[ChildChunkUpdateArgs], actor_id: str
|
||||
) -> tuple[ChildChunkRecord, ...]:
|
||||
existing = self._children(segment_ref)
|
||||
remaining = {child.data.id: child for child in existing}
|
||||
added: list[ChildChunkState] = []
|
||||
updated: list[ChildChunkState] = []
|
||||
for chunk in chunks:
|
||||
if chunk.id:
|
||||
previous = remaining.pop(chunk.id, None)
|
||||
if previous is not None and previous.data.content != chunk.content:
|
||||
updated.append(self._updated_child(previous, chunk.content, actor_id))
|
||||
else:
|
||||
added.append(self._new_child(segment_ref, chunk.content, len(existing) + len(added) + 1, actor_id))
|
||||
self._write_children(segment_ref, added=added, updated=updated, deleted=tuple(remaining.values()))
|
||||
return tuple(child.data for child in sorted(added + updated, key=lambda child: child.data.position))
|
||||
|
||||
def delete_child_chunk(self, segment_ref: SegmentRef, *, child_chunk_id: str) -> None:
|
||||
child = self._child(segment_ref, child_chunk_id)
|
||||
try:
|
||||
self._index.update_children(segment_ref, deleted=(child,))
|
||||
except Exception as error:
|
||||
raise ChildChunkDeleteIndexApplicationError(str(error)) from error
|
||||
self._store.save_children(segment_ref, deleted=(child,))
|
||||
|
||||
def update_child_chunk(
|
||||
self, segment_ref: SegmentRef, *, child_chunk_id: str, content: str, actor_id: str
|
||||
) -> ChildChunkRecord:
|
||||
child = self._updated_child(self._child(segment_ref, child_chunk_id), content, actor_id)
|
||||
self._write_children(segment_ref, updated=(child,))
|
||||
return child.data
|
||||
|
||||
def _mark_error(self, segment_ref: SegmentRef, error: Exception) -> None:
|
||||
logger.exception("Segment indexing failed: %s", segment_ref.segment_id)
|
||||
self._store.save_segment(
|
||||
segment_ref, {"enabled": False, "disabled_at": naive_utc_now(), "status": "error", "error": str(error)}
|
||||
)
|
||||
|
||||
def _children(self, segment_ref: SegmentRef) -> tuple[ChildChunkState, ...]:
|
||||
children = self._store.get_children(segment_ref)
|
||||
if children is None:
|
||||
raise SegmentNotFoundError("Segment not found")
|
||||
return children
|
||||
|
||||
def _child(self, segment_ref: SegmentRef, child_id: str) -> ChildChunkState:
|
||||
for child in self._children(segment_ref):
|
||||
if child.data.id == child_id:
|
||||
return child
|
||||
raise ChildChunkNotFoundError("Child chunk not found")
|
||||
|
||||
def _new_child(self, segment_ref: SegmentRef, content: str, position: int, actor_id: str) -> ChildChunkState:
|
||||
now = naive_utc_now()
|
||||
return ChildChunkState(
|
||||
data=ChildChunkRecord(
|
||||
id=str(uuid4()),
|
||||
segment_id=segment_ref.segment_id,
|
||||
content=content,
|
||||
position=position,
|
||||
word_count=len(content),
|
||||
type="customized",
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
),
|
||||
index_node_id=str(uuid4()),
|
||||
index_node_hash=self._text_hash(content),
|
||||
created_by=actor_id,
|
||||
)
|
||||
|
||||
def _updated_child(self, child: ChildChunkState, content: str, actor_id: str) -> ChildChunkState:
|
||||
return replace(
|
||||
child,
|
||||
data=child.data.model_copy(
|
||||
update={
|
||||
"content": content,
|
||||
"word_count": len(content),
|
||||
"type": "customized",
|
||||
"updated_at": naive_utc_now(),
|
||||
}
|
||||
),
|
||||
updated_by=actor_id,
|
||||
)
|
||||
|
||||
def _write_children(
|
||||
self,
|
||||
segment_ref: SegmentRef,
|
||||
*,
|
||||
added: Sequence[ChildChunkState] = (),
|
||||
updated: Sequence[ChildChunkState] = (),
|
||||
deleted: Sequence[ChildChunkState] = (),
|
||||
) -> None:
|
||||
try:
|
||||
self._index.update_children(segment_ref, added=added, updated=updated, deleted=deleted)
|
||||
self._store.save_children(segment_ref, added=added, updated=updated, deleted=deleted)
|
||||
except Exception as error:
|
||||
raise ChildChunkIndexingApplicationError(str(error)) from error
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SegmentMutationScope:
|
||||
"""Authorized dataset and document values for shared mutation policy."""
|
||||
|
||||
dataset: SegmentDatasetRecord
|
||||
document: SegmentDocumentRecord
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.document.dataset_id != self.dataset.id or self.document.workspace_id != self.dataset.workspace_id:
|
||||
raise ValueError("Document does not belong to the dataset")
|
||||
|
||||
|
||||
def validate_segment_values(values: Mapping[str, object], *, doc_form: str, attachment_limit: int) -> SegmentUpdateArgs:
|
||||
answer = values.get("answer")
|
||||
if doc_form == "qa_model":
|
||||
if not answer:
|
||||
raise ValueError("Answer is required")
|
||||
if not isinstance(answer, str) or not answer.strip():
|
||||
raise ValueError("Answer is empty")
|
||||
content = values.get("content")
|
||||
if not isinstance(content, str) or not content.strip():
|
||||
raise ValueError("Content is empty")
|
||||
attachments = values.get("attachment_ids")
|
||||
if attachments:
|
||||
if not isinstance(attachments, list):
|
||||
raise ValueError("Attachment IDs is invalid")
|
||||
if len(attachments) > attachment_limit:
|
||||
raise ValueError(f"Exceeded maximum attachment limit of {attachment_limit}")
|
||||
return SegmentUpdateArgs.model_validate(values)
|
||||
@@ -0,0 +1,356 @@
|
||||
"""Indexing adapters that materialize database inputs before external I/O."""
|
||||
|
||||
import logging
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from core.credit_usage import CreditUsageCreatedBy
|
||||
from core.model_context import with_credit_usage_created_by
|
||||
from core.model_manager import ModelManager
|
||||
from core.rag.datasource.keyword.jieba.jieba import Jieba
|
||||
from core.rag.datasource.vdb.vector_factory import Vector
|
||||
from core.rag.entities import Rule
|
||||
from core.rag.index_processor.constant.doc_type import DocType
|
||||
from core.rag.index_processor.processor.paragraph_index_processor import ParagraphIndexProcessor
|
||||
from core.rag.index_processor.processor.parent_child_index_processor import ParentChildIndexProcessor
|
||||
from core.rag.models.document import Document
|
||||
from graphon.model_runtime.entities.model_entities import ModelType
|
||||
from libs.datetime_utils import naive_utc_now
|
||||
from models.enums import SegmentType
|
||||
from repositories.knowledge.segment_repository import SegmentIndexingSnapshot, SQLAlchemySegmentRepository
|
||||
from repositories.knowledge.upload_file_repository import SQLAlchemyKnowledgeUploadRepository
|
||||
from services.knowledge.entities.segments import ChildChunkRecord
|
||||
from services.knowledge.resource_scope import DocumentRef, SegmentRef
|
||||
from services.knowledge.segments.adapters import RedisSegmentClient
|
||||
from services.knowledge.segments.application import ChildChunkState, SegmentDatasetRecord, SegmentIndexTarget
|
||||
from services.knowledge.summaries.adapters import SummaryIndexAdapter
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SegmentIndexingGateway:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
segments: SQLAlchemySegmentRepository,
|
||||
uploads: SQLAlchemyKnowledgeUploadRepository,
|
||||
redis: RedisSegmentClient,
|
||||
new_session: Callable[[], Session],
|
||||
delete_task: Callable[..., object],
|
||||
enable_task: Callable[..., object],
|
||||
disable_task: Callable[..., object],
|
||||
) -> None:
|
||||
self._segments = segments
|
||||
self._uploads = uploads
|
||||
self._redis = redis
|
||||
self._new_session = new_session
|
||||
self._delete_task = delete_task
|
||||
self._enable_task = enable_task
|
||||
self._disable_task = disable_task
|
||||
|
||||
def count_tokens(self, dataset: SegmentDatasetRecord, text: str) -> int:
|
||||
if dataset.indexing_technique != "high_quality":
|
||||
return 0
|
||||
model = ModelManager.for_tenant(tenant_id=dataset.workspace_id).get_model_instance(
|
||||
tenant_id=dataset.workspace_id,
|
||||
provider=dataset.embedding_model_provider or "",
|
||||
model_type=ModelType.TEXT_EMBEDDING,
|
||||
model=dataset.embedding_model or "",
|
||||
)
|
||||
return model.get_text_embedding_num_tokens(texts=[text])[0]
|
||||
|
||||
@staticmethod
|
||||
def _vector(snapshot: SegmentIndexingSnapshot) -> Vector:
|
||||
return Vector(snapshot.dataset, session=None, vector_type=snapshot.vector_type)
|
||||
|
||||
@staticmethod
|
||||
def _document(snapshot: SegmentIndexingSnapshot) -> Document:
|
||||
segment = snapshot.segment
|
||||
return Document(
|
||||
page_content=segment.content,
|
||||
metadata={
|
||||
"doc_id": segment.index_node_id,
|
||||
"doc_hash": segment.index_node_hash,
|
||||
"document_id": segment.document_id,
|
||||
"dataset_id": segment.dataset_id,
|
||||
"doc_type": DocType.TEXT,
|
||||
},
|
||||
)
|
||||
|
||||
@with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)
|
||||
def create_many(
|
||||
self,
|
||||
document_ref: DocumentRef,
|
||||
*,
|
||||
segment_ids: Sequence[str],
|
||||
keywords_list: Sequence[Sequence[str] | None] | None,
|
||||
) -> None:
|
||||
"""Materialize the whole scoped batch before indexing any of its records."""
|
||||
snapshots = self._segments.get_indexing_snapshots(document_ref, segment_ids)
|
||||
if not snapshots:
|
||||
return
|
||||
first = snapshots[0]
|
||||
documents = [self._document(snapshot) for snapshot in snapshots]
|
||||
if first.document.doc_form == "hierarchical_model":
|
||||
for snapshot in snapshots:
|
||||
self._regenerate_children(document_ref.segment(snapshot.segment.id), snapshot, replace_existing=False)
|
||||
elif first.dataset.indexing_technique == "high_quality":
|
||||
self._vector(first).create(documents)
|
||||
elif first.document.doc_form != "qa_model":
|
||||
self._write_keywords(document_ref, first, documents, keywords_list, replace_existing=False)
|
||||
if first.dataset.is_multimodal:
|
||||
files = self._uploads.get_files(
|
||||
workspace_id=document_ref.dataset.tenant_id,
|
||||
file_ids=tuple(attachment_id for snapshot in snapshots for attachment_id in snapshot.attachment_ids),
|
||||
)
|
||||
attachments = [
|
||||
document
|
||||
for snapshot in snapshots
|
||||
for document in self._attachment_documents(
|
||||
snapshot, [files[file_id] for file_id in snapshot.attachment_ids if file_id in files]
|
||||
)
|
||||
]
|
||||
if attachments and first.dataset.indexing_technique == "high_quality":
|
||||
self._vector(first).create_multimodal(attachments, upload_files=files)
|
||||
|
||||
@with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)
|
||||
def create(self, segment_ref: SegmentRef, *, keywords: Sequence[str] | None, attachment_ids: Sequence[str]) -> None:
|
||||
snapshot = self._segments.get_indexing_snapshot(segment_ref)
|
||||
files = self._uploads.get_files(workspace_id=segment_ref.document.dataset.tenant_id, file_ids=attachment_ids)
|
||||
self._segments.replace_attachments(segment_ref, tuple(files))
|
||||
if snapshot.document.doc_form == "hierarchical_model":
|
||||
self._regenerate_children(segment_ref, snapshot, replace_existing=False)
|
||||
elif snapshot.dataset.indexing_technique == "high_quality":
|
||||
self._vector(snapshot).create([self._document(snapshot)])
|
||||
else:
|
||||
self._update_keywords(segment_ref, snapshot, keywords, replace_existing=False)
|
||||
if files and snapshot.dataset.is_multimodal:
|
||||
self._vector(snapshot).create_multimodal(
|
||||
self._attachment_documents(snapshot, files.values()), upload_files=files
|
||||
)
|
||||
|
||||
@with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)
|
||||
def update(self, segment_ref: SegmentRef, *, keywords: Sequence[str] | None, regenerate_children: bool) -> None:
|
||||
snapshot = self._segments.get_indexing_snapshot(segment_ref)
|
||||
if regenerate_children:
|
||||
self._regenerate_children(segment_ref, snapshot, replace_existing=True)
|
||||
elif snapshot.dataset.indexing_technique == "high_quality":
|
||||
vector = self._vector(snapshot)
|
||||
assert snapshot.segment.index_node_id is not None
|
||||
vector.delete_by_ids([snapshot.segment.index_node_id])
|
||||
vector.add_texts([self._document(snapshot)], duplicate_check=True)
|
||||
else:
|
||||
self._update_keywords(segment_ref, snapshot, keywords, replace_existing=True)
|
||||
|
||||
def _regenerate_children(
|
||||
self, segment_ref: SegmentRef, snapshot: SegmentIndexingSnapshot, *, replace_existing: bool
|
||||
) -> None:
|
||||
if snapshot.dataset.indexing_technique != "high_quality":
|
||||
raise ValueError("The knowledge base index technique is not high quality!")
|
||||
rule = snapshot.process_rule
|
||||
if rule is None:
|
||||
if replace_existing:
|
||||
return
|
||||
raise ValueError("No processing rule found.")
|
||||
manager = ModelManager.for_tenant(tenant_id=snapshot.dataset.tenant_id)
|
||||
model = (
|
||||
manager.get_model_instance(
|
||||
tenant_id=snapshot.dataset.tenant_id,
|
||||
provider=snapshot.dataset.embedding_model_provider,
|
||||
model_type=ModelType.TEXT_EMBEDDING,
|
||||
model=snapshot.dataset.embedding_model,
|
||||
)
|
||||
if snapshot.dataset.embedding_model_provider
|
||||
else manager.get_default_model_instance(
|
||||
tenant_id=snapshot.dataset.tenant_id, model_type=ModelType.TEXT_EMBEDDING
|
||||
)
|
||||
)
|
||||
children = ParentChildIndexProcessor().split_child_nodes(
|
||||
self._document(snapshot), Rule.model_validate(rule.to_dict()["rules"]), rule.mode, model
|
||||
)
|
||||
previous = self._segments.get_children(segment_ref) or ()
|
||||
vector = self._vector(snapshot)
|
||||
if replace_existing and previous:
|
||||
vector.delete_by_ids([child.index_node_id for child in previous if child.index_node_id])
|
||||
if children:
|
||||
vector.create([Document.model_validate(child.model_dump()) for child in children])
|
||||
now = naive_utc_now()
|
||||
added = tuple(
|
||||
ChildChunkState(
|
||||
data=ChildChunkRecord(
|
||||
id=str(uuid4()),
|
||||
segment_id=segment_ref.segment_id,
|
||||
content=child.page_content,
|
||||
position=position,
|
||||
word_count=len(child.page_content),
|
||||
type=SegmentType.AUTOMATIC,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
),
|
||||
index_node_id=child.metadata["doc_id"],
|
||||
index_node_hash=child.metadata["doc_hash"],
|
||||
created_by=snapshot.document.created_by,
|
||||
)
|
||||
for position, child in enumerate(children, start=1)
|
||||
)
|
||||
self._segments.save_children(segment_ref, added=added, deleted=previous if replace_existing else ())
|
||||
|
||||
@with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)
|
||||
def update_children(
|
||||
self,
|
||||
segment_ref: SegmentRef,
|
||||
*,
|
||||
added: Sequence[ChildChunkState] = (),
|
||||
updated: Sequence[ChildChunkState] = (),
|
||||
deleted: Sequence[ChildChunkState] = (),
|
||||
) -> None:
|
||||
snapshot = self._segments.get_indexing_snapshot(segment_ref)
|
||||
if snapshot.dataset.indexing_technique != "high_quality":
|
||||
return
|
||||
documents = [
|
||||
Document(
|
||||
page_content=child.data.content,
|
||||
metadata={
|
||||
"doc_id": child.index_node_id,
|
||||
"doc_hash": child.index_node_hash,
|
||||
"document_id": segment_ref.document.document_id,
|
||||
"dataset_id": segment_ref.document.dataset.dataset_id,
|
||||
},
|
||||
)
|
||||
for child in (*added, *updated)
|
||||
]
|
||||
vector = self._vector(snapshot)
|
||||
deleted_ids = []
|
||||
for child in (*updated, *deleted):
|
||||
assert child.index_node_id is not None
|
||||
deleted_ids.append(child.index_node_id)
|
||||
if deleted_ids:
|
||||
vector.delete_by_ids(deleted_ids)
|
||||
if documents:
|
||||
vector.add_texts(documents, duplicate_check=True)
|
||||
|
||||
@staticmethod
|
||||
def _attachment_documents(snapshot: SegmentIndexingSnapshot, files) -> list[Document]:
|
||||
return [
|
||||
Document(
|
||||
page_content=file.name,
|
||||
metadata={
|
||||
"doc_id": file.id,
|
||||
"doc_hash": "",
|
||||
"dataset_id": snapshot.dataset.id,
|
||||
"document_id": snapshot.document.id,
|
||||
"doc_type": DocType.IMAGE,
|
||||
},
|
||||
)
|
||||
for file in files
|
||||
]
|
||||
|
||||
@with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)
|
||||
def update_attachments(self, segment_ref: SegmentRef, attachment_ids: Sequence[str]) -> None:
|
||||
snapshot = self._segments.get_indexing_snapshot(segment_ref)
|
||||
if snapshot.dataset.indexing_technique != "high_quality" or set(snapshot.attachment_ids) == set(attachment_ids):
|
||||
return
|
||||
files = self._uploads.get_files(workspace_id=snapshot.dataset.tenant_id, file_ids=attachment_ids)
|
||||
if snapshot.dataset.is_multimodal:
|
||||
vector = self._vector(snapshot)
|
||||
if snapshot.attachment_ids:
|
||||
vector.delete_by_ids(list(snapshot.attachment_ids))
|
||||
if files:
|
||||
vector.create_multimodal(self._attachment_documents(snapshot, files.values()), upload_files=files)
|
||||
self._segments.replace_attachments(segment_ref, tuple(files))
|
||||
|
||||
def update_summary(self, segment_ref: SegmentRef, *, summary: str | None, content_changed: bool) -> None:
|
||||
if summary is None and not content_changed:
|
||||
return
|
||||
snapshot = self._segments.get_indexing_snapshot(segment_ref)
|
||||
if snapshot.dataset.indexing_technique != "high_quality" or snapshot.document.doc_form == "qa_model":
|
||||
return
|
||||
setting = snapshot.dataset.summary_index_setting
|
||||
previous = snapshot.summary
|
||||
if summary is None or (previous is not None and summary == previous.summary_content):
|
||||
if not (content_changed and previous and setting and setting.get("enable") is True):
|
||||
return
|
||||
try:
|
||||
self._segments.save_summary(segment_ref, previous.summary_content or "")
|
||||
summary, _ = ParagraphIndexProcessor.generate_summary_from_inputs(
|
||||
snapshot.dataset.tenant_id,
|
||||
snapshot.segment.content,
|
||||
setting,
|
||||
document_language=snapshot.document.doc_language,
|
||||
image_loader=lambda: self._segments.get_summary_images(segment_ref),
|
||||
)
|
||||
if not summary.strip():
|
||||
raise ValueError("Generated summary is empty")
|
||||
except Exception as error:
|
||||
self._segments.save_summary(segment_ref, previous.summary_content or "", error=str(error))
|
||||
raise
|
||||
if not summary.strip():
|
||||
if previous and previous.summary_index_node_id:
|
||||
try:
|
||||
self._vector(snapshot).delete_by_ids([previous.summary_index_node_id])
|
||||
except Exception:
|
||||
logger.exception("Failed to delete summary vector for %s", segment_ref.segment_id)
|
||||
self._segments.save_summary(segment_ref, None)
|
||||
return
|
||||
record = self._segments.save_summary(segment_ref, summary)
|
||||
assert record is not None
|
||||
SummaryIndexAdapter.vectorize_summary(
|
||||
record,
|
||||
snapshot.segment,
|
||||
snapshot.dataset,
|
||||
new_session=self._new_session,
|
||||
vector_factory=lambda: self._vector(snapshot),
|
||||
)
|
||||
|
||||
def _update_keywords(
|
||||
self,
|
||||
segment_ref: SegmentRef,
|
||||
snapshot: SegmentIndexingSnapshot,
|
||||
keywords: Sequence[str] | None,
|
||||
*,
|
||||
replace_existing: bool,
|
||||
) -> None:
|
||||
assert snapshot.segment.index_node_id is not None
|
||||
self._write_keywords(
|
||||
segment_ref.document, snapshot, [self._document(snapshot)], [keywords], replace_existing=replace_existing
|
||||
)
|
||||
|
||||
def _write_keywords(
|
||||
self,
|
||||
document_ref: DocumentRef,
|
||||
snapshot: SegmentIndexingSnapshot,
|
||||
documents: list[Document],
|
||||
keywords_list: Sequence[Sequence[str] | None] | None,
|
||||
*,
|
||||
replace_existing: bool,
|
||||
) -> None:
|
||||
dataset_ref = document_ref.dataset
|
||||
|
||||
def persist(storage_type: str, data: str, selected: Mapping[str, Sequence[str]]) -> None:
|
||||
self._segments.save_keyword_table(dataset_ref, storage_type=storage_type, data=data, keywords=selected)
|
||||
|
||||
Jieba(snapshot.dataset).update_texts(
|
||||
documents,
|
||||
read=lambda: self._segments.get_keyword_table(dataset_ref),
|
||||
write=persist,
|
||||
lock=self._redis.lock(f"keyword_indexing_lock_{snapshot.dataset.id}", timeout=600),
|
||||
keywords_list=keywords_list,
|
||||
replace_existing=replace_existing,
|
||||
)
|
||||
|
||||
def delete(
|
||||
self, document_ref: DocumentRef, segments: Sequence[SegmentIndexTarget], children: Sequence[ChildChunkState]
|
||||
) -> None:
|
||||
self._delete_task(
|
||||
[segment.index_node_id for segment in segments],
|
||||
document_ref.dataset.dataset_id,
|
||||
document_ref.document_id,
|
||||
[segment.id for segment in segments],
|
||||
[child.index_node_id for child in children if child.index_node_id],
|
||||
)
|
||||
|
||||
def change_status(self, document_ref: DocumentRef, segment_ids: Sequence[str], action: str) -> None:
|
||||
task = self._enable_task if action == "enable" else self._disable_task
|
||||
task(list(segment_ids), document_ref.dataset.dataset_id, document_ref.document_id)
|
||||
Whitespace-only changes.
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,82 @@
|
||||
"""Summary vectorization policy independent of ORM, model providers, and vector stores."""
|
||||
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Protocol
|
||||
|
||||
from services.knowledge.resource_scope import SegmentRef
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SummaryVectorInput:
|
||||
ref: SegmentRef
|
||||
summary_id: str
|
||||
content: str
|
||||
previous_node_id: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SummaryVectorResult:
|
||||
node_id: str
|
||||
content_hash: str
|
||||
tokens: int
|
||||
|
||||
|
||||
class SummaryStore(Protocol):
|
||||
def completed(self, command: SummaryVectorInput, result: SummaryVectorResult) -> None: ...
|
||||
def failed(self, command: SummaryVectorInput, error: str) -> None: ...
|
||||
|
||||
|
||||
class SummaryVectorBackend(Protocol):
|
||||
def delete(self, node_id: str) -> None: ...
|
||||
def count_tokens(self, content: str) -> int: ...
|
||||
def write(self, command: SummaryVectorInput, result: SummaryVectorResult) -> None: ...
|
||||
|
||||
|
||||
class SummaryIndexService:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
store: SummaryStore,
|
||||
vectors: SummaryVectorBackend,
|
||||
new_id: Callable[[], str],
|
||||
text_hash: Callable[[str], str],
|
||||
sleep: Callable[[float], None],
|
||||
) -> None:
|
||||
self._store, self._vectors = store, vectors
|
||||
self._new_id, self._hash, self._sleep = new_id, text_hash, sleep
|
||||
|
||||
def vectorize(self, command: SummaryVectorInput) -> SummaryVectorResult:
|
||||
if not command.content.strip():
|
||||
raise ValueError(f"Summary content is empty for segment {command.ref.segment_id}, cannot vectorize")
|
||||
node_id = command.previous_node_id or self._new_id()
|
||||
if command.previous_node_id:
|
||||
try:
|
||||
self._vectors.delete(command.previous_node_id)
|
||||
except Exception:
|
||||
logger.warning("Failed to delete previous summary vector for %s", command.ref.segment_id, exc_info=True)
|
||||
tokens = 0
|
||||
try:
|
||||
tokens = self._vectors.count_tokens(command.content)
|
||||
except Exception:
|
||||
logger.warning("Failed to calculate summary embedding tokens", exc_info=True)
|
||||
result = SummaryVectorResult(node_id, self._hash(command.content), tokens)
|
||||
for attempt in range(3):
|
||||
try:
|
||||
self._vectors.write(command, result)
|
||||
self._store.completed(command, result)
|
||||
return result
|
||||
except Exception as error:
|
||||
transient = any(
|
||||
word in str(error).lower()
|
||||
for word in ("connection", "disconnected", "timeout", "network", "weaviate")
|
||||
)
|
||||
if transient and attempt < 2:
|
||||
self._sleep(2.0 * 2**attempt)
|
||||
continue
|
||||
self._store.failed(command, f"Vectorization failed: {error}")
|
||||
raise
|
||||
raise AssertionError("Unreachable vectorization retry state")
|
||||
Reference in new issue
Block a user