chore: 纳入本地 Dify 1.19.0 源码及品牌与免登改动

演示目录只保留可重建的源码。运行密钥、数据卷、依赖和带口令的本地配置不进入仓库。

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
wuzilongandCursor committed 2026-11-18 13:59:25 +08:00
1 parent 60158935f4
commit 6519682885
15831 files changed
+3063428

No files matched your search

@@ -0,0 +1 @@
"""Knowledge-domain application services and adapters."""
@@ -0,0 +1,110 @@
"""Manage workspace knowledge API keys and keys restricted to one knowledge base."""
from collections.abc import Callable
from dataclasses import replace
from datetime import datetime
from typing import Protocol
from machinery.context import RequestContext
from services.auth.api_key_contracts import ApiKeyCache, ApiKeyRecord, ApiKeyResourceNotFoundError
from services.errors.base import NoPermissionError
from services.knowledge.dataset_access import (
DatasetAccess,
DatasetAccessDeniedError,
DatasetNotFoundError,
)
class DatasetApiKeyStore(Protocol):
def list_keys(self, workspace_id: str, dataset_id: str) -> tuple[ApiKeyRecord, ...]: ...
def create_key(self, workspace_id: str, dataset_id: str, *, max_keys: int, prefix: str) -> ApiKeyRecord: ...
def delete_key(self, workspace_id: str, dataset_id: str, key_id: str) -> ApiKeyRecord: ...
def list_workspace_keys(self, workspace_id: str) -> tuple[ApiKeyRecord, ...]: ...
def create_workspace_key(
self,
workspace_id: str,
dataset_ids: tuple[str, ...],
*,
knowledge_space_ids: tuple[str, ...] = (),
max_keys: int,
prefix: str,
) -> ApiKeyRecord: ...
def delete_workspace_key(self, workspace_id: str, key_id: str) -> ApiKeyRecord: ...
class UnknownDatasetIdsError(Exception):
def __init__(self, dataset_ids: list[str]) -> None:
super().__init__(f"Unknown knowledge base id(s): {', '.join(dataset_ids)}")
class UnknownKnowledgeSpaceIdsError(Exception):
def __init__(self, space_ids: list[str]) -> None:
super().__init__(f"Unknown knowledge space id(s): {', '.join(space_ids)}")
class DatasetApiKeyService:
MAX_KEYS = 10
def __init__(
self,
*,
keys: DatasetApiKeyStore,
cache: ApiKeyCache,
access: DatasetAccess,
rbac_enabled: Callable[[], bool],
last_used_at: Callable[[str, str | None, datetime | None], datetime | None],
) -> None:
self._keys = keys
self._cache = cache
self._access = access
self._rbac_enabled = rbac_enabled
self._last_used_at = last_used_at
def list_keys(self, context: RequestContext, dataset_id: str) -> tuple[ApiKeyRecord, ...]:
self._check_access(context, dataset_id)
return self._keys.list_keys(context.active_workspace_id, dataset_id)
def create_key(self, context: RequestContext, dataset_id: str) -> ApiKeyRecord:
self._check_access(context, dataset_id)
return self._keys.create_key(context.active_workspace_id, dataset_id, max_keys=self.MAX_KEYS, prefix="ds-")
def delete_key(self, context: RequestContext, dataset_id: str, key_id: str) -> None:
self._check_access(context, dataset_id)
key = self._keys.delete_key(context.active_workspace_id, dataset_id, key_id)
self._cache.delete(key.token, key.type)
def _check_access(self, context: RequestContext, dataset_id: str) -> None:
# Enterprise RBAC is enforced at admission; community deployments use the dataset ACL.
if not self._rbac_enabled():
try:
self._access.require_accessible(context, dataset_id)
except DatasetNotFoundError as error:
raise ApiKeyResourceNotFoundError("Dataset not found.") from error
except DatasetAccessDeniedError as error:
raise NoPermissionError("You do not have permission to access this dataset.") from error
def list_workspace_keys(self, context: RequestContext) -> tuple[ApiKeyRecord, ...]:
return tuple(
replace(key, last_used_at=self._last_used_at(key.token, key.type, key.last_used_at))
for key in self._keys.list_workspace_keys(context.active_workspace_id)
)
def create_workspace_key(
self, context: RequestContext, dataset_ids: tuple[str, ...], *, knowledge_space_ids: tuple[str, ...] = ()
) -> ApiKeyRecord:
return self._keys.create_workspace_key(
context.active_workspace_id,
tuple(dict.fromkeys(dataset_ids)),
knowledge_space_ids=tuple(dict.fromkeys(knowledge_space_ids)),
max_keys=self.MAX_KEYS,
prefix="dataset-",
)
def delete_workspace_key(self, context: RequestContext, key_id: str) -> None:
key = self._keys.delete_workspace_key(context.active_workspace_id, key_id)
self._cache.delete(key.token, key.type)
@@ -0,0 +1,123 @@
"""Dataset access use case, read model, errors, and consumer-owned ports."""
from typing import NamedTuple, Protocol
from machinery.context import RequestContext
_OWNER_ROLE = "owner"
_ALL_TEAM_MEMBERS = "all_team_members"
_PARTIAL_MEMBERS = "partial_members"
class DatasetAccessRecord(NamedTuple):
"""Dataset state consumed only by the access policy."""
id: str
workspace_id: str
maintainer_id: str | None
permission: str
class AccessibleDataset(NamedTuple):
"""Minimal result proving that the request may operate on a dataset."""
id: str
workspace_id: str
class DatasetAccessSnapshot(NamedTuple):
"""Dataset state plus the requesting account's partial-member grant."""
dataset: DatasetAccessRecord
actor_has_partial_access: bool
class DatasetAccessReader(Protocol):
"""Load tenant-scoped dataset state needed by the access policy."""
def get_access_snapshot(
self,
*,
workspace_id: str,
dataset_id: str,
actor_id: str,
) -> DatasetAccessSnapshot | None: ...
class WorkspaceRoleReader(Protocol):
"""Read the account's legacy role in a workspace."""
def get_legacy_role(self, *, workspace_id: str, account_id: str) -> str | None: ...
class DatasetAccessError(Exception):
"""Base class for framework-neutral dataset access failures."""
class DatasetNotFoundError(DatasetAccessError):
def __init__(self) -> None:
super().__init__("Dataset not found")
class DatasetAccessDeniedError(DatasetAccessError):
def __init__(self) -> None:
super().__init__("You do not have permission to access this dataset")
def can_access_dataset(snapshot: DatasetAccessSnapshot, *, actor_id: str, workspace_role: str | None) -> bool:
"""Evaluate legacy dataset visibility without database or framework access."""
if workspace_role is None:
return False
dataset = snapshot.dataset
if workspace_role == _OWNER_ROLE or dataset.maintainer_id == actor_id:
return True
if dataset.permission == _ALL_TEAM_MEMBERS:
return True
if dataset.permission == _PARTIAL_MEMBERS:
return snapshot.actor_has_partial_access
return False
class DatasetAccessService:
"""Resolve an owned dataset and enforce legacy permissions when configured."""
def __init__(
self,
*,
datasets: DatasetAccessReader,
workspace_roles: WorkspaceRoleReader,
legacy_permissions_enabled: bool,
) -> None:
self._datasets = datasets
self._workspace_roles = workspace_roles
self._legacy_permissions_enabled = legacy_permissions_enabled
def require_accessible(self, context: RequestContext, dataset_id: str) -> AccessibleDataset:
snapshot = self._datasets.get_access_snapshot(
workspace_id=context.active_workspace_id,
dataset_id=dataset_id,
actor_id=context.account_id,
)
if snapshot is None:
raise DatasetNotFoundError()
return self.check_access(context, snapshot)
def check_access(self, context: RequestContext, snapshot: DatasetAccessSnapshot) -> AccessibleDataset:
"""Authorize already-loaded state without querying the dataset again."""
if snapshot.dataset.workspace_id != context.active_workspace_id:
raise DatasetAccessDeniedError()
if self._legacy_permissions_enabled:
workspace_role = self._workspace_roles.get_legacy_role(
workspace_id=context.active_workspace_id,
account_id=context.account_id,
)
if not can_access_dataset(snapshot, actor_id=context.account_id, workspace_role=workspace_role):
raise DatasetAccessDeniedError()
return AccessibleDataset(id=snapshot.dataset.id, workspace_id=snapshot.dataset.workspace_id)
class DatasetAccess(Protocol):
def require_accessible(self, context: RequestContext, dataset_id: str) -> AccessibleDataset: ...
def check_access(self, context: RequestContext, snapshot: DatasetAccessSnapshot) -> AccessibleDataset: ...
@@ -0,0 +1,329 @@
"""Load knowledge details and materialize data before response serialization."""
from __future__ import annotations
import json
from collections.abc import Iterable, Mapping, Sequence
from typing import Any, cast
from sqlalchemy.orm import Session
from core.rag.index_processor.constant.built_in_field import BuiltInField, MetadataDataSource
from core.rag.index_processor.constant.query_type import QueryType
from core.tools.signature import sign_upload_file_preview_url
from models.dataset import Dataset, DatasetMetadata, DatasetQuery, DocMetadataDetailItem, Document, DocumentSegment
from models.model import UploadFile
from repositories.knowledge.dataset_read_repository import (
DatasetDetailPrefetch,
DocumentReadBatch,
SegmentReadBatch,
build_dataset_detail_prefetch,
get_document_dataset,
get_document_metadata_rows,
get_document_upload_file,
get_document_uploader,
get_query_upload_files,
load_document_read_batch,
load_segment_read_batch,
)
from repositories.knowledge.segment_read_adapter import (
format_segment_attachments,
sign_segment_content,
)
from services.knowledge.entities.datasets import DatasetDetailRecord
def load_dataset_detail(
dataset: Dataset, *, session: Session, prefetch: DatasetDetailPrefetch | None = None
) -> DatasetDetailRecord:
values = prefetch if prefetch is not None else build_dataset_detail_prefetch([dataset], session=session)
return _dataset_detail_values(dataset, values)
def load_dataset_details(datasets: Sequence[Dataset], *, session: Session) -> list[DatasetDetailRecord]:
"""Read a page in one batch and return values with no ORM or session references."""
prefetch = build_dataset_detail_prefetch(datasets, session=session)
return [_dataset_detail_values(dataset, prefetch) for dataset in datasets]
def _dataset_detail_values(dataset: Dataset, prefetch: DatasetDetailPrefetch) -> DatasetDetailRecord:
owner = (dataset.id, dataset.tenant_id)
return {
"id": dataset.id,
"name": dataset.name,
"description": dataset.description,
"provider": dataset.provider,
"permission": dataset.permission,
"data_source_type": dataset.data_source_type,
"indexing_technique": dataset.indexing_technique,
"keyword_number": dataset.keyword_number,
"app_count": prefetch.app_counts.get(dataset.id, 0),
"document_count": prefetch.document_counts.get(dataset.id, 0),
"word_count": prefetch.word_counts.get(dataset.id, 0),
"created_by": dataset.created_by,
"author_name": prefetch.author_names.get(dataset.created_by),
"created_at": dataset.created_at,
"updated_by": dataset.updated_by,
"updated_at": dataset.updated_at,
"embedding_model": dataset.embedding_model,
"embedding_model_provider": dataset.embedding_model_provider,
"embedding_available": vars(dataset).get("embedding_available"),
"retrieval_model_dict": dataset.retrieval_model_dict,
"summary_index_setting": dataset.summary_index_setting,
"tags": [{"id": tag.id, "name": tag.name, "type": tag.type} for tag in prefetch.tags.get(owner, [])],
"doc_form": dataset.chunk_structure or prefetch.doc_forms.get(owner),
"external_knowledge_info": prefetch.external_knowledge_infos.get(owner)
if dataset.provider == "external"
else None,
"external_retrieval_model": dataset.external_retrieval_model,
"doc_metadata": dataset.build_doc_metadata(prefetch.doc_metadata.get(dataset.id, [])),
"built_in_field_enabled": dataset.built_in_field_enabled,
"pipeline_id": dataset.pipeline_id,
"runtime_mode": dataset.runtime_mode,
"chunk_structure": dataset.chunk_structure,
"icon_info": dataset.icon_info,
"is_published": prefetch.pipeline_published.get(dataset.pipeline_id, False) if dataset.pipeline_id else False,
"total_documents": prefetch.document_counts.get(dataset.id, 0),
"total_available_documents": prefetch.available_document_counts.get(dataset.id, 0),
"enable_api": dataset.enable_api,
"is_multimodal": dataset.is_multimodal,
"permission_keys": vars(dataset).get("permission_keys", []),
"maintainer": dataset.maintainer,
}
def load_document_detail(document: Document, *, session: Session) -> dict[str, Any]:
return load_document_details([document], session=session)[0]
def load_document_details(documents: Iterable[Document], *, session: Session) -> list[dict[str, Any]]:
"""Materialize document attributes and relations while the caller's session is open."""
items = list(documents)
batch = load_document_read_batch(items, session=session)
return [
{
"id": document.id,
"position": document.position,
"data_source_type": document.data_source_type,
"data_source_info_dict": document.data_source_info_dict,
"dataset_process_rule_id": document.dataset_process_rule_id,
"name": document.name,
"created_from": document.created_from,
"created_by": document.created_by,
"created_at": document.created_at,
"tokens": document.tokens,
"indexing_status": document.indexing_status,
"error": document.error,
"enabled": document.enabled,
"disabled_at": document.disabled_at,
"disabled_by": document.disabled_by,
"archived": document.archived,
"display_status": document.display_status,
"word_count": document.word_count,
"doc_form": document.doc_form,
"summary_index_status": vars(document).get("summary_index_status"),
"need_summary": document.need_summary,
"completed_segments": vars(document).get("completed_segments"),
"total_segments": vars(document).get("total_segments"),
**document_response_values(document, batch),
}
for document in items
]
def load_segment_detail(
segment: DocumentSegment, summary: str | None, *, session: Session, batch: SegmentReadBatch | None = None
) -> dict[str, Any]:
"""Load relations and authorize file links before handing plain values to the response model."""
if batch is None:
batch = load_segment_read_batch([segment], session=session)
child_chunks = batch.child_chunks.get(segment.id, [])
return {
"id": segment.id,
"position": segment.position,
"document_id": segment.document_id,
"content": segment.content,
"sign_content": sign_segment_content(segment, session=session),
"answer": segment.answer,
"word_count": segment.word_count,
"tokens": segment.tokens,
"keywords": segment.keywords,
"index_node_id": segment.index_node_id,
"index_node_hash": segment.index_node_hash,
"hit_count": segment.hit_count,
"enabled": segment.enabled,
"disabled_at": segment.disabled_at,
"disabled_by": segment.disabled_by,
"status": segment.status,
"created_by": segment.created_by,
"created_at": segment.created_at,
"updated_at": segment.updated_at,
"updated_by": segment.updated_by,
"indexing_at": segment.indexing_at,
"completed_at": segment.completed_at,
"error": segment.error,
"stopped_at": segment.stopped_at,
"child_chunks": [
{
"id": chunk.id,
"segment_id": chunk.segment_id,
"content": chunk.content,
"position": chunk.position,
"word_count": chunk.word_count,
"type": chunk.type,
"created_at": chunk.created_at,
"updated_at": chunk.updated_at,
}
for chunk in child_chunks
],
"attachments": format_segment_attachments(batch.attachments.get(segment.id, [])),
"summary": summary,
}
def load_segment_details(
segments: Iterable[DocumentSegment], summaries: Mapping[str, str | None], *, session: Session
) -> list[dict[str, Any]]:
items = list(segments)
batch = load_segment_read_batch(items, session=session)
return [load_segment_detail(segment, summaries.get(segment.id), session=session, batch=batch) for segment in items]
def format_document_source_detail(document: Document, file: UploadFile | None) -> dict[str, Any]:
if document.data_source_type == "upload_file" and file is not None:
return {
"upload_file": {
"id": file.id,
"name": file.name,
"size": file.size,
"extension": file.extension,
"mime_type": file.mime_type,
"created_by": file.created_by,
"created_at": file.created_at.timestamp(),
}
}
if document.data_source_type in {"notion_import", "website_crawl"}:
return document.data_source_info_dict
return {}
def format_document_metadata(
document: Document, metadatas: Sequence[DatasetMetadata], *, built_in_enabled: bool, uploader_name: str | None
) -> list[DocMetadataDetailItem] | None:
values = document.doc_metadata or {}
details: list[DocMetadataDetailItem] = [
{"id": metadata.id, "name": metadata.name, "type": metadata.type, "value": values.get(metadata.name)}
for metadata in metadatas
]
if built_in_enabled:
details.extend(format_document_built_in_fields(document, uploader_name=uploader_name))
return details or None
def document_response_values(document: Document, batch: DocumentReadBatch) -> dict[str, Any]:
"""Build response-only values from a loaded snapshot without database access."""
file_id = document.data_source_info_dict.get("upload_file_id")
process_rule = (
batch.process_rules.get(document.dataset_process_rule_id)
if document.dataset_process_rule_id is not None
else None
)
if process_rule is not None and process_rule.dataset_id != document.dataset_id:
process_rule = None
return {
"data_source_detail_dict": format_document_source_detail(
document, batch.uploads.get((document.tenant_id, file_id)) if isinstance(file_id, str) else None
),
"hit_count": batch.hit_counts.get(document.id, 0),
"doc_metadata_details": format_document_metadata(
document,
batch.metadatas.get(document.id, []),
built_in_enabled=batch.built_in_enabled.get((document.tenant_id, document.dataset_id), False),
uploader_name=batch.uploader_names.get(document.created_by),
),
"process_rule_dict": process_rule.to_dict() if process_rule else None,
}
def get_document_source_detail(document: Document, *, session: Session) -> dict[str, Any]:
file = get_document_upload_file(document, session=session) if document.data_source_type == "upload_file" else None
return format_document_source_detail(document, file)
def get_document_metadata_details(document: Document, *, session: Session) -> list[DocMetadataDetailItem] | None:
dataset = get_document_dataset(document, session=session)
built_in_enabled = bool(dataset is not None and dataset.built_in_field_enabled)
return format_document_metadata(
document,
get_document_metadata_rows(document, session=session) if document.doc_metadata else [],
built_in_enabled=built_in_enabled,
uploader_name=get_document_uploader(document, session=session) if built_in_enabled else None,
)
def get_document_built_in_fields(document: Document, *, session: Session) -> list[DocMetadataDetailItem]:
return format_document_built_in_fields(document, uploader_name=get_document_uploader(document, session=session))
def format_document_built_in_fields(document: Document, *, uploader_name: str | None) -> list[DocMetadataDetailItem]:
built_in_fields: list[DocMetadataDetailItem] = []
built_in_fields.extend(
(
{"id": "built-in", "name": BuiltInField.document_name, "type": "string", "value": document.name},
{"id": "built-in", "name": BuiltInField.uploader, "type": "string", "value": uploader_name},
{
"id": "built-in",
"name": BuiltInField.upload_date,
"type": "time",
"value": str(document.created_at.timestamp()),
},
{
"id": "built-in",
"name": BuiltInField.last_update_date,
"type": "time",
"value": str(document.updated_at.timestamp()),
},
{
"id": "built-in",
"name": BuiltInField.source,
"type": "string",
"value": MetadataDataSource[document.data_source_type],
},
)
)
return built_in_fields
def get_dataset_queries(query: DatasetQuery, *, session: Session) -> list[dict[str, Any]]:
"""Read structured audit content, preserving legacy plain-text queries."""
try:
decoded = json.loads(query.content)
except (ValueError, TypeError):
decoded = None
raw_records = decoded if isinstance(decoded, list) else [decoded]
if not raw_records or not all(
isinstance(item, dict)
and item.get("content_type") in {QueryType.TEXT_QUERY, QueryType.IMAGE_QUERY}
and isinstance(item.get("content"), str)
for item in raw_records
):
return [{"content_type": QueryType.TEXT_QUERY, "content": query.content, "file_info": None}]
records = cast(list[dict[str, Any]], raw_records)
files = get_query_upload_files(
query, {item["content"] for item in records if item["content_type"] == QueryType.IMAGE_QUERY}, session=session
)
result: list[dict[str, Any]] = []
for item in records:
record: dict[str, Any] = {"content_type": item["content_type"], "content": item["content"], "file_info": None}
file = files.get(item["content"]) if item["content_type"] == QueryType.IMAGE_QUERY else None
if file is not None:
record["file_info"] = {
"id": file.id,
"name": file.name,
"size": file.size,
"extension": file.extension,
"mime_type": file.mime_type,
"source_url": sign_upload_file_preview_url(file.id, file.extension),
}
result.append(record)
return result
File diff suppressed because it is too large. Load diff
@@ -0,0 +1 @@
"""Dataset administration use cases and infrastructure adapters."""
@@ -0,0 +1,344 @@
"""Session-owning dataset administration adapters."""
from collections.abc import Generator, Mapping
from contextlib import contextmanager
from typing import Any
from sqlalchemy import select
from sqlalchemy.orm import Session, sessionmaker
from configs import dify_config
from core.plugin.impl.model_runtime_factory import create_plugin_provider_manager
from graphon.model_runtime.entities.model_entities import ModelType
from libs.pagination import clamp_pagination
from machinery.context import RequestContext
from models import Account, Dataset, Document
from models.dataset import DatasetPermission, DatasetPermissionEnum
from models.provider_ids import ModelProviderID
from repositories.knowledge.dataset_repository import _get_dataset
from services.app.query_service import RelatedApps
from services.dataset_knowledge_fs_upgrade_service import KnowledgeFSUpgradeSnapshotService, upgrade_discovery_response
from services.enterprise import rbac_service
from services.errors.base import NoPermissionError
from services.knowledge.dataset_access import DatasetAccessDeniedError, DatasetNotFoundError
from services.knowledge.dataset_read_service import get_dataset_queries, load_dataset_detail, load_dataset_details
from services.knowledge.dataset_service import DatasetPermissionService, DatasetService, DocumentService
from services.knowledge.datasets.application import (
DatasetListFilter,
DatasetVisibility,
)
from services.knowledge.entities.datasets import DatasetDetailRecord, DatasetPage
from services.knowledge.resource_scope import DatasetRef
from services.knowledge_fs.feature_service import get_knowledge_fs_features
from services.rbac import contracts as rbac_contracts
from services.rbac.members import MemberService
from services.tag_application_service import TagTargetQuery
@contextmanager
def _translate_permissions() -> Generator[None]:
try:
yield
except NoPermissionError as error:
raise DatasetAccessDeniedError() from error
def require_dataset(session: Session, ref: DatasetRef) -> Dataset:
dataset = _get_dataset(session, ref)
if dataset is None:
raise DatasetNotFoundError()
return dataset
def load_actor(session: Session, context: RequestContext) -> Account:
account = session.get(Account, context.account_id)
if account is None:
raise DatasetAccessDeniedError()
account.set_tenant_id_with_session(context.active_workspace_id, session=session)
return account
def _normalize_provider(item: DatasetDetailRecord) -> DatasetDetailRecord:
if item["indexing_technique"] == "high_quality" and item["embedding_model_provider"]:
item["embedding_model_provider"] = str(ModelProviderID(item["embedding_model_provider"]))
return item
def _status(document: Document, counts: tuple[int, int] | None = None) -> dict[str, Any]:
result: dict[str, Any] = {
"id": document.id,
"indexing_status": document.indexing_status,
"processing_started_at": document.processing_started_at,
"parsing_completed_at": document.parsing_completed_at,
"cleaning_completed_at": document.cleaning_completed_at,
"splitting_completed_at": document.splitting_completed_at,
"completed_at": document.completed_at,
"paused_at": document.paused_at,
"error": document.error,
"stopped_at": document.stopped_at,
}
if counts is not None:
result.update(completed_segments=counts[0], total_segments=counts[1])
return result
class SQLAlchemyDatasetOperations:
def __init__(
self,
*,
session_factory: sessionmaker[Session],
tags: TagTargetQuery,
app_queries: RelatedApps,
members: MemberService,
) -> None:
self._sessions = session_factory
self._tags = tags
self._app_queries = app_queries
self._members_service = members
def visibility(self, context: RequestContext) -> DatasetVisibility:
permissions = self._members_service.permissions(context.active_workspace_id, context.account_id)
scope = (
rbac_service.RBACService.DatasetAccess.whitelist_resources(context.active_workspace_id, context.account_id)
if dify_config.RBAC_ENABLED
else None
)
return DatasetVisibility(
default_permissions=permissions.dataset.default_permission_keys,
workspace_permissions=permissions.workspace.permission_keys,
overrides={item.resource_id: item.permission_keys for item in permissions.dataset.overrides},
unrestricted=scope is None or scope.unrestricted,
whitelist_ids=scope.resource_ids if scope is not None else (),
)
def embedding_models(self, workspace_id: str) -> set[str]:
manager = create_plugin_provider_manager(tenant_id=workspace_id)
models = manager.get_configurations(tenant_id=workspace_id).get_models(
model_type=ModelType.TEXT_EMBEDDING, only_active=True
)
return {f"{model.model}:{model.provider.provider}" for model in models}
def list_datasets(
self, context: RequestContext, query: DatasetListFilter, accessible_ids: list[str] | None
) -> DatasetPage:
page, limit = clamp_pagination(query.page, query.limit, 100)
with self._sessions() as session:
account = load_actor(session, context)
if query.ids:
datasets, total = DatasetService.get_datasets_by_ids(
list(query.ids),
context.active_workspace_id,
user=account,
accessible_dataset_ids=accessible_ids,
session=session,
)
else:
datasets, total = DatasetService.get_datasets(
page,
limit,
session,
context.active_workspace_id,
account,
query.keyword,
list(query.tag_ids),
query.include_all,
creator_ids=list(query.creator_ids),
tags=self._tags,
accessible_dataset_ids=accessible_ids,
)
data = load_dataset_details(datasets, session=session)
partial_ids = [item["id"] for item in data if item["permission"] == "partial_members"]
members: dict[str, list[str]] = {}
if partial_ids:
for dataset_id, account_id in session.execute(
select(DatasetPermission.dataset_id, DatasetPermission.account_id).where(
DatasetPermission.dataset_id.in_(partial_ids),
DatasetPermission.tenant_id == context.active_workspace_id,
)
):
members.setdefault(dataset_id, []).append(account_id)
upgrade_jobs = KnowledgeFSUpgradeSnapshotService(self._sessions).get_latest_by_dataset_ids(
tenant_id=context.active_workspace_id,
dataset_ids=[item["id"] for item in data],
session=session,
)
knowledge_fs_enabled = get_knowledge_fs_features().enabled
for item in data:
_normalize_provider(item)
item["partial_member_list"] = members.get(item["id"], [])
item["knowledge_fs_upgrade"] = upgrade_discovery_response(
upgrade_jobs.get(item["id"]),
feature_enabled=knowledge_fs_enabled,
dataset_provider=str(item["provider"]),
).model_dump(mode="json")
return {
"data": data,
"has_more": not query.ids and page * limit < total,
"total": total,
"page": page,
"limit": limit,
}
def _detail(self, session: Session, context: RequestContext, dataset: Dataset) -> DatasetDetailRecord:
result = load_dataset_detail(dataset, session=session)
permissions = self._members_service.resource_permissions(
context.active_workspace_id, context.account_id, rbac_contracts.RBACResourceType.DATASET, [dataset.id]
)
result["permission_keys"] = permissions.get(dataset.id, [])
return result
def get_dataset(self, context: RequestContext, ref: DatasetRef) -> DatasetDetailRecord:
with self._sessions() as session:
dataset = require_dataset(session, ref)
result = load_dataset_detail(dataset, session=session)
permissions = self._members_service.permissions(
context.active_workspace_id, context.account_id, dataset_id=ref.dataset_id
)
result["permission_keys"] = permissions.dataset.permission_keys_by_resource_ids([dataset.id]).get(
dataset.id, []
)
if dataset.permission == DatasetPermissionEnum.PARTIAL_TEAM:
result["partial_member_list"] = self._members(session, ref)
return _normalize_provider(result)
def create_dataset(self, context: RequestContext, values: Mapping[str, Any]) -> DatasetDetailRecord:
with _translate_permissions(), self._sessions() as session:
dataset = DatasetService.create_empty_dataset(
session=session,
tenant_id=context.active_workspace_id,
account=load_actor(session, context),
**dict(values),
)
result = self._detail(session, context, dataset)
result["partial_member_list"] = None
dataset_id = dataset.id
session.commit()
if dify_config.RBAC_ENABLED:
rbac_service.RBACService.DatasetAccess.replace_whitelist(
context.active_workspace_id,
context.account_id,
dataset_id,
rbac_contracts.ReplaceMemberBindings(automatic_include_workspace_members=False),
)
return result
def update_dataset(
self, context: RequestContext, ref: DatasetRef, values: Mapping[str, Any]
) -> DatasetDetailRecord:
data = dict(values)
if (
data.get("indexing_technique") == "high_quality"
and data.get("embedding_model_provider") is not None
and data.get("embedding_model") is not None
):
data["is_multimodal"] = DatasetService.check_is_multimodal_model(
ref.tenant_id, data["embedding_model_provider"], data["embedding_model"]
)
with _translate_permissions(), self._sessions() as session:
dataset = require_dataset(session, ref)
account = load_actor(session, context)
if not dify_config.RBAC_ENABLED:
DatasetPermissionService.check_permission(
account, dataset, data.get("permission"), data.get("partial_member_list"), session=session
)
# The legacy updater consumes fields from its input; retain the
# member selection for the permission update in this transaction.
dataset = DatasetService.update_dataset(ref.dataset_id, dict(data), account, session=session)
if dataset is None:
raise DatasetNotFoundError()
result = self._detail(session, context, dataset)
if (
data.get("partial_member_list") is not None
and data.get("permission") == DatasetPermissionEnum.PARTIAL_TEAM
):
DatasetPermissionService.update_partial_member_list(
ref.tenant_id, ref.dataset_id, data["partial_member_list"], session
)
elif data.get("permission") in {DatasetPermissionEnum.ONLY_ME, DatasetPermissionEnum.ALL_TEAM}:
DatasetPermissionService.clear_partial_member_list(ref.dataset_id, session)
result["partial_member_list"] = self._members(session, ref)
session.commit()
return result
def delete_dataset(self, context: RequestContext, ref: DatasetRef) -> None:
with _translate_permissions(), self._sessions() as session:
require_dataset(session, ref)
if not DatasetService.delete_dataset(ref.dataset_id, load_actor(session, context), session):
raise DatasetNotFoundError()
DatasetPermissionService.clear_partial_member_list(ref.dataset_id, session)
session.commit()
def is_in_use(self, ref: DatasetRef) -> bool:
with self._sessions() as session:
require_dataset(session, ref)
return DatasetService.dataset_use_check(ref, session)
def queries(self, ref: DatasetRef, *, page: int, limit: int) -> dict[str, Any]:
page, limit = clamp_pagination(page, limit, 100)
with self._sessions() as session:
require_dataset(session, ref)
queries, total = DatasetService.get_dataset_queries(
dataset_id=ref.dataset_id, page=page, per_page=limit, session=session
)
data = [
{
"id": query.id,
"queries": get_dataset_queries(query, session=session),
"source": query.source,
"source_app_id": query.source_app_id,
"created_by_role": query.created_by_role,
"created_by": query.created_by,
"created_at": query.created_at,
}
for query in queries
]
return {"data": data, "has_more": page * limit < total, "total": total, "page": page, "limit": limit}
def related_apps(self, ref: DatasetRef) -> dict[str, Any]:
with self._sessions() as session:
require_dataset(session, ref)
joins = DatasetService.get_related_apps(ref.dataset_id, session)
app_ids = [join.app_id for join in joins]
apps = self._app_queries.related_apps(ref.tenant_id, app_ids)
return {"data": apps, "total": len(apps)}
def indexing_status(self, ref: DatasetRef) -> dict[str, Any]:
with self._sessions() as session:
require_dataset(session, ref)
documents = session.scalars(
select(Document).where(Document.dataset_id == ref.dataset_id, Document.tenant_id == ref.tenant_id)
).all()
counts = DocumentService.get_document_segment_counts(documents, session=session)
return {"data": [_status(document, counts.get(document.id, (0, 0))) for document in documents]}
def error_documents(self, ref: DatasetRef) -> dict[str, Any]:
with self._sessions() as session:
require_dataset(session, ref)
documents = DocumentService.get_error_documents_by_dataset_ref(ref, session)
return {"data": [_status(document) for document in documents], "total": len(documents)}
@staticmethod
def _members(session: Session, ref: DatasetRef) -> list[str]:
return list(
session.scalars(
select(DatasetPermission.account_id).where(
DatasetPermission.dataset_id == ref.dataset_id, DatasetPermission.tenant_id == ref.tenant_id
)
).all()
)
def partial_members(self, ref: DatasetRef) -> list[str]:
with self._sessions() as session:
require_dataset(session, ref)
return self._members(session, ref)
def auto_disable_logs(self, ref: DatasetRef) -> dict[str, Any]:
with self._sessions() as session:
return dict(DatasetService.get_dataset_auto_disable_logs(ref, session))
def set_api_enabled(self, context: RequestContext, ref: DatasetRef, enabled: bool) -> None:
with self._sessions() as session:
DatasetService.update_dataset_api_status(
require_dataset(session, ref), enabled, load_actor(session, context), session
)
session.commit()
@@ -0,0 +1,168 @@
"""Dataset administration policies independent of HTTP and persistence."""
from collections.abc import Mapping, Sequence
from dataclasses import dataclass, field
from typing import Any, Protocol
from libs.url_utils import normalize_api_base_url
from machinery.context import RequestContext
from services.knowledge.dataset_access import DatasetAccess
from services.knowledge.datasets.retrieval import retrieval_methods
from services.knowledge.entities.datasets import DatasetDetailRecord, DatasetPage
from services.knowledge.resource_scope import DatasetRef
@dataclass(frozen=True)
class DatasetListFilter:
page: int = 1
limit: int = 20
keyword: str | None = None
include_all: bool = False
ids: Sequence[str] = ()
tag_ids: Sequence[str] = ()
creator_ids: Sequence[str] = ()
@dataclass(frozen=True)
class DatasetVisibility:
default_permissions: Sequence[str] = ()
workspace_permissions: Sequence[str] = ()
overrides: Mapping[str, Sequence[str]] = field(default_factory=dict)
unrestricted: bool = True
whitelist_ids: Sequence[str] = ()
def list_scope(self, *, rbac_enabled: bool) -> list[str] | None:
if not rbac_enabled:
return None
# Maintainers always retain visibility of their own datasets. The
# whitelist takes precedence over grants only for other datasets.
if not self.unrestricted:
return sorted(set(self.whitelist_ids))
read_keys = {"dataset.preview", "dataset.acl.preview", "dataset.full_access"}
default_read = bool(
read_keys.intersection(self.default_permissions) or read_keys.intersection(self.workspace_permissions)
)
ids = (
None
if default_read
else sorted(key for key, grants in self.overrides.items() if read_keys.intersection(grants))
)
return ids
class DatasetOperations(Protocol):
"""Materialize owned values and complete writes before returning."""
def visibility(self, context: RequestContext) -> DatasetVisibility: ...
def list_datasets(
self, context: RequestContext, query: DatasetListFilter, accessible_ids: list[str] | None
) -> DatasetPage: ...
def embedding_models(self, workspace_id: str) -> set[str]: ...
def get_dataset(self, context: RequestContext, ref: DatasetRef) -> DatasetDetailRecord: ...
def create_dataset(self, context: RequestContext, values: Mapping[str, Any]) -> DatasetDetailRecord: ...
def update_dataset(
self, context: RequestContext, ref: DatasetRef, values: Mapping[str, Any]
) -> DatasetDetailRecord: ...
def delete_dataset(self, context: RequestContext, ref: DatasetRef) -> None: ...
def is_in_use(self, ref: DatasetRef) -> bool: ...
def queries(self, ref: DatasetRef, *, page: int, limit: int) -> dict[str, Any]: ...
def related_apps(self, ref: DatasetRef) -> dict[str, Any]: ...
def indexing_status(self, ref: DatasetRef) -> dict[str, Any]: ...
def error_documents(self, ref: DatasetRef) -> dict[str, Any]: ...
def partial_members(self, ref: DatasetRef) -> list[str]: ...
def auto_disable_logs(self, ref: DatasetRef) -> dict[str, Any]: ...
def set_api_enabled(self, context: RequestContext, ref: DatasetRef, enabled: bool) -> None: ...
class DatasetApplicationService:
def __init__(
self,
*,
dataset_access: DatasetAccess,
operations: DatasetOperations,
rbac_enabled: bool,
service_api_url: str,
vector_store: str | None,
tidb_fulltext: bool,
) -> None:
self._dataset_access = dataset_access
self._operations = operations
self._rbac_enabled = rbac_enabled
self._service_api_url = service_api_url
self._vector_store = vector_store
self._tidb_fulltext = tidb_fulltext
def _dataset(self, context: RequestContext, dataset_id: str) -> DatasetRef:
dataset = self._dataset_access.require_accessible(context, dataset_id)
return DatasetRef(dataset.workspace_id, dataset.id)
@staticmethod
def _embedding_available(item: DatasetDetailRecord, models: set[str], *, listing: bool) -> None:
high_quality = item["indexing_technique"] == "high_quality"
item["embedding_available"] = (
not high_quality
or (listing and not item["embedding_model_provider"])
or f"{item['embedding_model']}:{item['embedding_model_provider']}" in models
)
def list_datasets(self, context: RequestContext, query: DatasetListFilter) -> DatasetPage:
visibility = self._operations.visibility(context)
ids = visibility.list_scope(rbac_enabled=self._rbac_enabled)
result = self._operations.list_datasets(context, query, ids)
models = self._operations.embedding_models(context.active_workspace_id)
for item in result["data"]:
item["permission_keys"] = list(visibility.overrides.get(item["id"], visibility.default_permissions))
self._embedding_available(item, models, listing=True)
return result
def get_dataset(self, context: RequestContext, *, dataset_id: str) -> DatasetDetailRecord:
result = self._operations.get_dataset(context, self._dataset(context, dataset_id))
self._embedding_available(result, self._operations.embedding_models(context.active_workspace_id), listing=False)
return result
def create_dataset(self, context: RequestContext, *, values: Mapping[str, Any]) -> DatasetDetailRecord:
settings = dict(values)
settings["permission"] = "all_team_members" if self._rbac_enabled else settings.get("permission") or "only_me"
return self._operations.create_dataset(context, settings)
def update_dataset(
self, context: RequestContext, *, dataset_id: str, values: Mapping[str, Any]
) -> DatasetDetailRecord:
return self._operations.update_dataset(context, self._dataset(context, dataset_id), values)
def delete_dataset(self, context: RequestContext, *, dataset_id: str) -> None:
self._operations.delete_dataset(context, self._dataset(context, dataset_id))
def is_in_use(self, context: RequestContext, *, dataset_id: str) -> bool:
return self._operations.is_in_use(self._dataset(context, dataset_id))
def queries(self, context: RequestContext, *, dataset_id: str, page: int, limit: int) -> dict[str, Any]:
return self._operations.queries(self._dataset(context, dataset_id), page=page, limit=limit)
def related_apps(self, context: RequestContext, *, dataset_id: str) -> dict[str, Any]:
return self._operations.related_apps(self._dataset(context, dataset_id))
def indexing_status(self, context: RequestContext, *, dataset_id: str) -> dict[str, Any]:
return self._operations.indexing_status(self._dataset(context, dataset_id))
def error_documents(self, context: RequestContext, *, dataset_id: str) -> dict[str, Any]:
return self._operations.error_documents(self._dataset(context, dataset_id))
def partial_members(self, context: RequestContext, *, dataset_id: str) -> list[str]:
return self._operations.partial_members(self._dataset(context, dataset_id))
def auto_disable_logs(self, context: RequestContext, *, dataset_id: str) -> dict[str, Any]:
return self._operations.auto_disable_logs(self._dataset(context, dataset_id))
def set_api_enabled(self, context: RequestContext, *, dataset_id: str, status: str) -> None:
self._operations.set_api_enabled(context, self._dataset(context, dataset_id), status == "enable")
def api_base_url(self, context: RequestContext, *, request_base_url: str) -> str:
return normalize_api_base_url(self._service_api_url or request_base_url)
def retrieval_settings(
self, context: RequestContext, *, vector_type: str | None = None, is_mock: bool = False
) -> dict[str, list[str]]:
return retrieval_methods(
vector_type if is_mock else self._vector_store, is_mock=is_mock, tidb_fulltext=self._tidb_fulltext
)
@@ -0,0 +1,85 @@
"""Supported dataset retrieval methods for each vector store."""
from core.rag.datasource.vdb.vector_type import VectorType
from core.rag.retrieval.retrieval_methods import RetrievalMethod
def retrieval_methods(
vector_type: str | None, *, is_mock: bool = False, tidb_fulltext: bool = False
) -> dict[str, list[str]]:
"""
Get supported retrieval methods based on vector database type.
Args:
vector_type: Vector database type, can be None
is_mock: Whether this is a Mock API, affects MILVUS handling
tidb_fulltext: Whether TiDB full-text retrieval is enabled
Returns:
Dictionary containing supported retrieval methods
Raises:
ValueError: If vector_type is None or unsupported
"""
if vector_type is None:
raise ValueError("Vector store type is not configured.")
# Define vector database types that only support semantic search
semantic_only_types = {
VectorType.RELYT,
VectorType.CHROMA,
VectorType.PGVECTO_RS,
VectorType.VIKINGDB,
VectorType.UPSTASH,
}
# Define vector database types that support all retrieval methods
full_search_types = {
VectorType.QDRANT,
VectorType.WEAVIATE,
VectorType.OPENSEARCH,
VectorType.ANALYTICDB,
VectorType.MYSCALE,
VectorType.ORACLE,
VectorType.ELASTICSEARCH,
VectorType.ELASTICSEARCH_JA,
VectorType.PGVECTOR,
VectorType.VASTBASE,
VectorType.TIDB_ON_QDRANT,
VectorType.LINDORM,
VectorType.COUCHBASE,
VectorType.OPENGAUSS,
VectorType.OCEANBASE,
VectorType.SEEKDB,
VectorType.TABLESTORE,
VectorType.HUAWEI_CLOUD,
VectorType.TENCENT,
VectorType.MATRIXONE,
VectorType.CLICKZETTA,
VectorType.BAIDU,
VectorType.ALIBABACLOUD_MYSQL,
VectorType.IRIS,
VectorType.HOLOGRES,
}
semantic_methods = {"retrieval_method": [RetrievalMethod.SEMANTIC_SEARCH.value]}
full_methods = {
"retrieval_method": [
RetrievalMethod.SEMANTIC_SEARCH.value,
RetrievalMethod.FULL_TEXT_SEARCH.value,
RetrievalMethod.HYBRID_SEARCH.value,
]
}
if vector_type == VectorType.MILVUS:
return semantic_methods if is_mock else full_methods
if vector_type == VectorType.TIDB_VECTOR:
return full_methods if tidb_fulltext else semantic_methods
if vector_type in semantic_only_types:
return semantic_methods
elif vector_type in full_search_types:
return full_methods
else:
raise ValueError(f"Unsupported vector db type {vector_type}.")
@@ -0,0 +1,206 @@
"""Classic-to-Agent knowledge base upgrade use cases behind the dataset console endpoints."""
from collections.abc import Callable
from uuid import uuid4
from sqlalchemy import select
from sqlalchemy.orm import Session, sessionmaker
from graphon.model_runtime.entities.model_entities import ModelType
from machinery.context import RequestContext
from models import Dataset
from models.knowledge_fs import KnowledgeFSUpgradeJob, KnowledgeFSUpgradeJobStatus
from models.provider import TenantDefaultModel
from services.dataset_knowledge_fs_upgrade_service import (
KnowledgeFSUpgradeConflictError,
KnowledgeFSUpgradeModelPreferences,
KnowledgeFSUpgradeModelResolver,
KnowledgeFSUpgradeNotFoundError,
KnowledgeFSUpgradeSnapshotService,
upgrade_discovery_response,
upgrade_job_matches_model_choices,
)
from services.knowledge.dataset_access import DatasetAccess, DatasetAccessError
from services.knowledge.dataset_service import DatasetService
from services.knowledge.datasets.adapters import SQLAlchemyDatasetOperations, load_actor
from services.knowledge_fs.product_dto import (
KnowledgeFSUpgradeModelChoice,
KnowledgeFSUpgradeModelsPayload,
KnowledgeFSUpgradePreparationResponse,
)
_RECOVERABLE_STATUSES = (
KnowledgeFSUpgradeJobStatus.QUEUED,
KnowledgeFSUpgradeJobStatus.RUNNING,
KnowledgeFSUpgradeJobStatus.FAILED,
)
class KnowledgeFSUpgradeUnavailableError(Exception):
"""The KnowledgeFS feature is disabled, so upgrades cannot start or retry."""
class KnowledgeUpgradeApplicationService:
def __init__(
self,
*,
session_factory: sessionmaker[Session],
dataset_access: DatasetAccess,
datasets: SQLAlchemyDatasetOperations,
rbac_enabled: bool,
feature_enabled: Callable[[], bool],
enqueue: Callable[[KnowledgeFSUpgradeSnapshotService, str, str], None] | None = None,
) -> None:
self._sessions = session_factory
self._dataset_access = dataset_access
self._datasets = datasets
self._rbac_enabled = rbac_enabled
self._feature_enabled = feature_enabled
self._enqueue = enqueue or enqueue_upgrade_job
def _snapshots(self) -> KnowledgeFSUpgradeSnapshotService:
return KnowledgeFSUpgradeSnapshotService(self._sessions)
@staticmethod
def _choice(provider: str | None, model: str | None) -> KnowledgeFSUpgradeModelChoice | None:
if not provider or not model:
return None
return KnowledgeFSUpgradeModelChoice(provider=provider, model=model)
def _load_model_preferences(
self, *, tenant_id: str, dataset_id: str
) -> tuple[str, KnowledgeFSUpgradeModelPreferences]:
"""Read the legacy settings and saved defaults in a short, tenant-scoped session."""
with self._sessions() as session:
dataset = session.scalar(select(Dataset).where(Dataset.id == dataset_id, Dataset.tenant_id == tenant_id))
if dataset is None:
raise KnowledgeFSUpgradeNotFoundError("Dataset was not found")
defaults: dict[ModelType, KnowledgeFSUpgradeModelChoice] = {}
for row in session.scalars(
select(TenantDefaultModel).where(
TenantDefaultModel.tenant_id == tenant_id,
TenantDefaultModel.model_type.in_((ModelType.TEXT_EMBEDDING, ModelType.RERANK, ModelType.LLM)),
)
):
choice = self._choice(row.provider_name, row.model_name)
if choice is not None:
defaults[row.model_type] = choice
retrieval = dataset.retrieval_model if isinstance(dataset.retrieval_model, dict) else {}
reranking = retrieval.get("reranking_model") or {}
if not isinstance(reranking, dict):
reranking = {}
preferences = KnowledgeFSUpgradeModelPreferences(
embedding=self._choice(dataset.embedding_model_provider, dataset.embedding_model),
rerank=self._choice(reranking.get("reranking_provider_name"), reranking.get("reranking_model_name")),
defaults=defaults,
)
return str(dataset.provider or ""), preferences
def list_recoverable_jobs(self, context: RequestContext) -> list[KnowledgeFSUpgradeJob]:
workspace_id = context.active_workspace_id
jobs = self._snapshots().list_by_statuses(tenant_id=workspace_id, statuses=_RECOVERABLE_STATUSES)
dataset_ids = list(dict.fromkeys(job.old_dataset_id for job in jobs))
if not dataset_ids:
return []
accessible_ids = self._datasets.visibility(context).list_scope(rbac_enabled=self._rbac_enabled)
with self._sessions() as session:
datasets, _ = DatasetService.get_datasets_by_ids(
dataset_ids,
workspace_id,
user=load_actor(session, context),
accessible_dataset_ids=accessible_ids,
session=session,
)
visible_ids = [str(dataset.id) for dataset in datasets]
allowed_ids: set[str] = set()
for dataset_id in visible_ids:
try:
self._dataset_access.require_accessible(context, dataset_id)
except DatasetAccessError:
continue
allowed_ids.add(dataset_id)
return [job for job in jobs if job.old_dataset_id in allowed_ids]
def discovery(self, context: RequestContext, *, dataset_id: str) -> KnowledgeFSUpgradePreparationResponse:
dataset = self._dataset_access.require_accessible(context, dataset_id)
with self._sessions() as session:
provider = session.scalar(
select(Dataset.provider).where(Dataset.id == dataset.id, Dataset.tenant_id == dataset.workspace_id)
)
job = self._snapshots().get_latest(tenant_id=dataset.workspace_id, dataset_id=dataset.id)
eligibility = upgrade_discovery_response(
job, feature_enabled=self._feature_enabled(), dataset_provider=str(provider or "")
)
recommendations = None
if eligibility.can_upgrade:
_, preferences = self._load_model_preferences(tenant_id=dataset.workspace_id, dataset_id=dataset.id)
recommendations = KnowledgeFSUpgradeModelResolver(tenant_id=dataset.workspace_id).recommend(preferences)
return KnowledgeFSUpgradePreparationResponse.model_validate(
{**eligibility.model_dump(mode="json"), "models": recommendations}
)
def start(
self,
context: RequestContext,
*,
dataset_id: str,
idempotency_key: str,
models: KnowledgeFSUpgradeModelsPayload | None = None,
) -> KnowledgeFSUpgradeJob:
if not self._feature_enabled():
raise KnowledgeFSUpgradeUnavailableError()
dataset = self._dataset_access.require_accessible(context, dataset_id)
snapshots = self._snapshots()
idempotent = snapshots.get_by_idempotency_key(tenant_id=dataset.workspace_id, idempotency_key=idempotency_key)
if idempotent is not None:
if idempotent.old_dataset_id != dataset.id:
raise KnowledgeFSUpgradeConflictError("Idempotency key belongs to another Dataset upgrade")
if models is not None and not upgrade_job_matches_model_choices(idempotent, models):
raise KnowledgeFSUpgradeConflictError("Idempotency key belongs to another model selection")
self._enqueue(snapshots, dataset.workspace_id, idempotent.id)
return idempotent
provider, preferences = self._load_model_preferences(tenant_id=dataset.workspace_id, dataset_id=dataset.id)
if provider == "external":
raise KnowledgeFSUpgradeConflictError("External-provider Datasets cannot be upgraded")
resolver = KnowledgeFSUpgradeModelResolver(tenant_id=dataset.workspace_id)
resolved_models = (
resolver.validate(models) if models is not None else resolver.resolve_recommendations(preferences)
)
job = snapshots.create(
tenant_id=dataset.workspace_id,
dataset_id=dataset.id,
requested_by_account_id=context.account_id,
idempotency_key=idempotency_key,
models=resolved_models,
)
self._enqueue(snapshots, dataset.workspace_id, job.id)
return job
def get_job(self, context: RequestContext, *, dataset_id: str, job_id: str) -> KnowledgeFSUpgradeJob:
dataset = self._dataset_access.require_accessible(context, dataset_id)
job = self._snapshots().get(tenant_id=dataset.workspace_id, job_id=job_id)
if job.old_dataset_id != dataset.id:
raise KnowledgeFSUpgradeNotFoundError("Upgrade job was not found")
return job
def retry(self, context: RequestContext, *, dataset_id: str, job_id: str) -> KnowledgeFSUpgradeJob:
if not self._feature_enabled():
raise KnowledgeFSUpgradeUnavailableError()
dataset = self._dataset_access.require_accessible(context, dataset_id)
snapshots = self._snapshots()
job = snapshots.retry(tenant_id=dataset.workspace_id, dataset_id=dataset.id, job_id=job_id)
self._enqueue(snapshots, dataset.workspace_id, job.id)
return job
def enqueue_upgrade_job(snapshots: KnowledgeFSUpgradeSnapshotService, tenant_id: str, job_id: str) -> None:
from tasks.knowledge_fs_upgrade_tasks import run_knowledge_fs_upgrade
task_id = str(uuid4())
if not snapshots.claim_enqueue(tenant_id=tenant_id, job_id=job_id, task_id=task_id):
return
try:
run_knowledge_fs_upgrade.apply_async(kwargs={"job_id": job_id}, task_id=task_id)
except Exception:
snapshots.release_enqueue_claim(tenant_id=tenant_id, job_id=job_id, task_id=task_id)
raise
@@ -0,0 +1,71 @@
"""Request-side document synchronization use cases."""
from typing import NamedTuple, Protocol
from machinery.context import RequestContext
from services.knowledge.dataset_access import DatasetAccess
from services.knowledge.resource_scope import DatasetRef, DocumentRef
class SyncDocumentRecord(NamedTuple):
"""Document state required to authorize and dispatch a sync request."""
id: str
data_source_type: str
class DocumentSyncReader(Protocol):
def get_sync_document(self, document_ref: DocumentRef) -> SyncDocumentRecord | None: ...
def list_active_notion_refs(self, dataset_ref: DatasetRef) -> tuple[DocumentRef, ...]: ...
class DocumentSyncDispatcher(Protocol):
def dispatch(self, *, dataset_id: str, document_id: str) -> None: ...
class DocumentSyncError(Exception):
"""Base class for document synchronization failures."""
class SyncDocumentNotFoundError(DocumentSyncError):
def __init__(self) -> None:
super().__init__("Document not found")
class SyncDocumentSourceError(DocumentSyncError):
def __init__(self) -> None:
super().__init__("Document is not notion type")
class DocumentSyncApplicationService:
"""Authorize synchronization requests and dispatch the worker use case."""
def __init__(
self,
*,
dataset_access: DatasetAccess,
documents: DocumentSyncReader,
dispatcher: DocumentSyncDispatcher,
) -> None:
self._dataset_access = dataset_access
self._documents = documents
self._dispatcher = dispatcher
def sync_dataset(self, context: RequestContext, dataset_id: str) -> int:
dataset = self._dataset_access.require_accessible(context, dataset_id)
dataset_ref = DatasetRef(dataset.workspace_id, dataset.id)
document_refs = self._documents.list_active_notion_refs(dataset_ref)
for document_ref in document_refs:
self._dispatcher.dispatch(dataset_id=dataset_ref.dataset_id, document_id=document_ref.document_id)
return len(document_refs)
def sync_document(self, context: RequestContext, dataset_id: str, document_id: str) -> None:
dataset = self._dataset_access.require_accessible(context, dataset_id)
document_ref = DatasetRef(dataset.workspace_id, dataset.id).document(document_id)
document = self._documents.get_sync_document(document_ref)
if document is None:
raise SyncDocumentNotFoundError()
if document.data_source_type != "notion_import":
raise SyncDocumentSourceError()
self._dispatcher.dispatch(dataset_id=dataset.id, document_id=document.id)
@@ -0,0 +1,11 @@
"""Infrastructure adapters for document-sync requests."""
from collections.abc import Callable
class CeleryDocumentSyncDispatcher:
def __init__(self, *, delay: Callable[[str, str], object]) -> None:
self._delay = delay
def dispatch(self, *, dataset_id: str, document_id: str) -> None:
self._delay(dataset_id, document_id)
@@ -0,0 +1 @@
"""Document application use cases and their infrastructure adapters."""
@@ -0,0 +1,579 @@
"""Session-owning adapters for document queries and existing document operations."""
import copy
import json
from collections.abc import Generator, Mapping, Sequence
from contextlib import contextmanager, nullcontext
from typing import Any, Literal
from uuid import uuid4
from redis.exceptions import LockNotOwnedError
from sqlalchemy import asc, desc, func, select, update
from sqlalchemy.orm import Session, sessionmaker
from werkzeug.exceptions import NotFound
from configs import dify_config
from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
from core.model_manager import ModelManager
from core.rag.index_processor.constant.built_in_field import BuiltInField
from core.rag.index_processor.constant.index_type import IndexTechniqueType
from extensions.ext_redis import redis_client
from graphon.model_runtime.entities.model_entities import ModelType
from graphon.model_runtime.errors.invoke import InvokeAuthorizationError
from libs.pagination import paginate_query
from machinery.context import RequestContext
from models import Account, Dataset, Document, DocumentSegment, UploadFile
from models.dataset import DatasetPermissionEnum, DocumentPipelineExecutionLog
from models.enums import IndexingStatus
from repositories.knowledge.dataset_read_repository import (
get_dataset_doc_form,
get_document_hit_count,
get_document_process_rule,
get_document_segment_count,
get_latest_dataset_process_rule,
)
from repositories.knowledge.dataset_repository import _get_dataset
from repositories.knowledge.document_repository import _get_document
from services.enterprise import rbac_service
from services.errors.base import NoPermissionError
from services.errors.document import DocumentIndexingError
from services.file_service import FileService
from services.knowledge.dataset_access import DatasetAccessDeniedError, DatasetNotFoundError
from services.knowledge.dataset_read_service import (
get_document_metadata_details,
get_document_source_detail,
load_document_detail,
load_document_details,
)
from services.knowledge.dataset_service import DatasetService, DocumentService
from services.knowledge.documents.application import (
DocumentIndexingStateError,
DocumentInvalidActionError,
DocumentListFilter,
DocumentNotFoundError,
DocumentProviderError,
DocumentState,
DocumentZip,
)
from services.knowledge.entities.document_creation import DocumentCreationDataset, DocumentIndexingJobs
from services.knowledge.entities.knowledge_entities import KnowledgeConfig
from services.knowledge.resource_scope import DatasetRef, DocumentRef
from services.knowledge.summaries.adapters import SummaryIndexAdapter
from services.rbac import contracts as rbac_contracts
from services.vector_space_admission_service import get_vector_space_admission_error_fields
from tasks.generate_summary_index_task import generate_summary_index_task
@contextmanager
def _translate_errors() -> Generator[None]:
try:
yield
except NotFound as error:
raise DocumentNotFoundError(error.description) from error
except NoPermissionError as error:
raise DatasetAccessDeniedError() from error
except DocumentIndexingError as error:
raise DocumentIndexingStateError(str(error)) from error
except (ProviderTokenNotInitError, InvokeAuthorizationError) as error:
raise DocumentProviderError(str(error), kind="uninitialized") from error
except QuotaExceededError as error:
raise DocumentProviderError(str(error), kind="quota") from error
except ModelCurrentlyNotSupportError as error:
raise DocumentProviderError(str(error), kind="unsupported") from error
def _require_dataset(session: Session, ref: DatasetRef) -> Dataset:
dataset = _get_dataset(session, ref)
if dataset is None:
raise DatasetNotFoundError()
return dataset
def _require_document(session: Session, ref: DocumentRef) -> Document:
document = _get_document(session, ref)
if document is None:
raise DocumentNotFoundError("Document not found.")
return document
def _state(document: Document) -> DocumentState:
return DocumentState(
id=document.id,
archived=bool(document.archived),
indexing_status=str(document.indexing_status),
is_paused=bool(document.is_paused),
data_source_type=str(document.data_source_type),
doc_form=str(document.doc_form),
need_summary=bool(document.need_summary),
)
def _created_response(dataset: Dataset, documents: Sequence[Document], batch: str, session: Session) -> dict[str, Any]:
return {
"dataset": {
"id": dataset.id,
"name": dataset.name,
"description": dataset.description,
"permission": dataset.permission,
"data_source_type": dataset.data_source_type,
"indexing_technique": dataset.indexing_technique,
"created_by": dataset.created_by,
"created_at": dataset.created_at,
},
"documents": load_document_details(documents, session=session),
"batch": batch,
}
def _indexing_status(document: Document, counts: tuple[int, int]) -> dict[str, Any]:
return {
"id": document.id,
"processing_started_at": document.processing_started_at,
"parsing_completed_at": document.parsing_completed_at,
"cleaning_completed_at": document.cleaning_completed_at,
"splitting_completed_at": document.splitting_completed_at,
"completed_at": document.completed_at,
"paused_at": document.paused_at,
"error": document.error,
"stopped_at": document.stopped_at,
"indexing_status": IndexingStatus.PAUSED if document.is_paused else document.indexing_status,
**get_vector_space_admission_error_fields(document.error),
"completed_segments": counts[0],
"total_segments": counts[1],
}
class SQLAlchemyDocumentOperations:
def __init__(self, *, session_factory: sessionmaker[Session]) -> None:
self._sessions = session_factory
def find_document(self, *, workspace_id: str, document_id: str) -> DocumentRef | None:
with self._sessions() as session:
document = session.scalar(
select(Document).where(Document.id == document_id, Document.tenant_id == workspace_id)
)
return DatasetRef(workspace_id, document.dataset_id).document(document.id) if document else None
def get_state(self, ref: DocumentRef) -> DocumentState | None:
with self._sessions() as session:
document = _get_document(session, ref)
return _state(document) if document else None
def get_states(self, ref: DatasetRef, document_ids: Sequence[str]) -> Sequence[DocumentState]:
with self._sessions() as session:
return tuple(
_state(document) for document in DocumentService.get_documents_by_ids(ref, document_ids, session)
)
def get_process_rule(self, ref: DatasetRef | None) -> dict[str, Any]:
result = copy.deepcopy(DocumentService.DEFAULT_RULES)
if ref is not None:
with self._sessions() as session:
rule = get_latest_dataset_process_rule(_require_dataset(session, ref), session=session)
if rule is not None:
result.update(mode=rule.mode, rules=rule.rules_dict)
return result
def list_documents(self, ref: DatasetRef, query: DocumentListFilter) -> dict[str, Any]:
with self._sessions() as session:
dataset = _require_dataset(session, ref)
statement = select(Document).where(
Document.dataset_id == ref.dataset_id, Document.tenant_id == ref.tenant_id
)
statement = DocumentService.apply_display_status_filter(statement, query.status)
if query.search:
statement = statement.where(Document.name.like(f"%{query.search}%"))
direction = desc if query.sort.startswith("-") else asc
sort = query.sort.removeprefix("-")
if sort == "hit_count":
hits = (
select(DocumentSegment.document_id, func.sum(DocumentSegment.hit_count).label("total_hit_count"))
.where(DocumentSegment.dataset_id == ref.dataset_id, DocumentSegment.tenant_id == ref.tenant_id)
.group_by(DocumentSegment.document_id)
.subquery()
)
statement = statement.outerjoin(hits, hits.c.document_id == Document.id).order_by(
direction(func.coalesce(hits.c.total_hit_count, 0)), direction(Document.position)
)
elif sort == "created_at":
statement = statement.order_by(direction(Document.created_at), direction(Document.position))
else:
statement = statement.order_by(desc(Document.created_at), desc(Document.position))
page = paginate_query(statement, session=session, page=query.page, per_page=query.limit, max_per_page=100)
DocumentService.enrich_documents_with_summary_index_status(
documents=page.items, dataset=dataset, tenant_id=ref.tenant_id, session=session
)
if query.fetch:
counts = DocumentService.get_document_segment_counts(page.items, session=session)
for document in page.items:
document.completed_segments, document.total_segments = counts.get(document.id, (0, 0))
return {
"data": load_document_details(page.items, session=session),
"has_more": page.has_next,
"limit": page.per_page,
"total": page.total,
"page": page.page,
}
def get_detail(self, ref: DocumentRef, *, metadata_only: bool) -> dict[str, Any]:
with self._sessions() as session:
document = _require_document(session, ref)
result: dict[str, Any] = {
"id": document.id,
"doc_type": document.doc_type,
"doc_metadata": get_document_metadata_details(document, session=session),
}
if metadata_only:
return result
rule = get_document_process_rule(document, session=session)
segment_count = get_document_segment_count(document, session=session)
result.update(
{
"position": document.position,
"data_source_type": document.data_source_type,
"dataset_process_rule_id": document.dataset_process_rule_id,
"name": document.name,
"created_from": document.created_from,
"created_by": document.created_by,
"tokens": document.tokens,
"indexing_status": document.indexing_status,
"indexing_latency": document.indexing_latency,
"error": document.error,
"enabled": document.enabled,
"disabled_by": document.disabled_by,
"archived": document.archived,
"display_status": document.display_status,
"doc_form": document.doc_form,
"doc_language": document.doc_language,
}
)
result.update(
{
"created_at": int(document.created_at.timestamp()) if document.created_at else None,
"completed_at": int(document.completed_at.timestamp()) if document.completed_at else None,
"updated_at": int(document.updated_at.timestamp()) if document.updated_at else None,
"disabled_at": int(document.disabled_at.timestamp()) if document.disabled_at else None,
}
)
result.update(
data_source_info=document.data_source_info_dict,
data_source_detail_dict=get_document_source_detail(document, session=session),
dataset_process_rule=DatasetService.get_process_rules(ref.dataset.dataset_id, session),
document_process_rule=dict(rule.to_dict()) if rule else None,
segment_count=segment_count,
average_segment_length=(document.word_count or 0) // segment_count if segment_count else 0,
hit_count=get_document_hit_count(document, session=session),
need_summary=bool(document.need_summary),
)
return result
def get_indexing_status(self, ref: DocumentRef) -> dict[str, Any]:
with self._sessions() as session:
document = _require_document(session, ref)
counts = DocumentService.get_document_segment_counts([document], session=session)
return _indexing_status(document, counts.get(document.id, (0, 0)))
def get_batch_indexing_status(self, ref: DatasetRef, batch: str) -> dict[str, Any]:
with self._sessions() as session:
documents = session.scalars(
select(Document).where(
Document.tenant_id == ref.tenant_id, Document.dataset_id == ref.dataset_id, Document.batch == batch
)
).all()
if not documents:
raise DocumentNotFoundError("Documents not found.")
counts = DocumentService.get_document_segment_counts(documents, session=session)
return {"data": [_indexing_status(document, counts.get(document.id, (0, 0))) for document in documents]}
def get_execution_log(self, ref: DocumentRef) -> dict[str, Any]:
with self._sessions() as session:
_require_document(session, ref)
log = session.scalar(
select(DocumentPipelineExecutionLog)
.where(DocumentPipelineExecutionLog.document_id == ref.document_id)
.order_by(DocumentPipelineExecutionLog.created_at.desc())
.limit(1)
)
if log is None:
return {}
return {
"datasource_info": json.loads(log.datasource_info),
"datasource_type": log.datasource_type,
"input_data": log.input_data,
"datasource_node_id": log.datasource_node_id,
}
def get_summary_status(self, ref: DocumentRef) -> dict[str, Any]:
with self._sessions() as session:
_require_document(session, ref)
return dict(
SummaryIndexAdapter.get_document_summary_status_detail(
tenant_id=ref.dataset.tenant_id,
document_id=ref.document_id,
dataset_id=ref.dataset.dataset_id,
session=session,
)
)
def get_download_url(self, ref: DocumentRef) -> str:
with _translate_errors(), self._sessions() as session:
return DocumentService.get_document_download_url(_require_document(session, ref), session)
@contextmanager
def build_download_zip(self, ref: DatasetRef, document_ids: Sequence[str]) -> Generator[DocumentZip]:
with _translate_errors(), self._sessions() as session:
files_by_document = DocumentService._get_upload_files_by_document_id_for_zip_download(
dataset_id=ref.dataset_id, document_ids=document_ids, tenant_id=ref.tenant_id, session=session
)
files = [files_by_document[document_id] for document_id in document_ids]
filename = DocumentService._generate_document_batch_download_zip_filename()
# The read transaction is closed before reading storage or streaming the response.
with FileService.build_upload_files_zip_tempfile(upload_files=files) as path:
yield DocumentZip(path=path, filename=filename)
@staticmethod
def _actor(session: Session, context: RequestContext) -> Account:
account = session.get(Account, context.account_id)
if account is None:
raise DatasetAccessDeniedError()
account.set_tenant_id_with_session(context.active_workspace_id, session=session)
return account
def _creation_dataset(
self, context: RequestContext, ref: DatasetRef, config: KnowledgeConfig
) -> DocumentCreationDataset:
with self._sessions() as session:
dataset = _require_dataset(session, ref)
self._actor(session, context)
DatasetService.check_doc_form(dataset, config.doc_form, session=session)
return DocumentCreationDataset(
dataset.tenant_id, dataset.indexing_technique, dataset.embedding_model, dataset.embedding_model_provider
)
def _persist_creation(
self, context: RequestContext, ref: DatasetRef, config: KnowledgeConfig, *, initialize: bool
) -> tuple[dict[str, Any], DocumentIndexingJobs]:
"""Keep ORM values local and materialize the response before closing the session."""
jobs: DocumentIndexingJobs | None = None
try:
with self._sessions() as session:
account = self._actor(session, context)
dataset = (
DocumentService.build_dataset_for_documents(ref, config, account, session)
if initialize
else _require_dataset(session, ref)
)
documents, batch, jobs = DocumentService.save_prepared_documents(
dataset, config, account, session=session
)
if initialize:
dataset.name = documents[0].name[:18] + "..."
dataset.description = "useful for when you want to answer queries about the " + documents[0].name
dataset.permission = (
DatasetPermissionEnum.ALL_TEAM if dify_config.RBAC_ENABLED else DatasetPermissionEnum.ONLY_ME
)
session.flush()
result = _created_response(dataset, documents, batch, session)
session.commit()
return result, jobs
except Exception:
# The persistence helper commits documents before loading response
# details. Preserve their indexing even if that later read fails.
if jobs is not None:
DocumentService.dispatch_document_indexing(jobs)
raise
def _run_creation(
self, context: RequestContext, ref: DatasetRef, config: KnowledgeConfig, *, initialize: bool
) -> dict[str, Any]:
lock = (
nullcontext()
if config.original_document_id
else redis_client.lock(f"add_document_lock_dataset_id_{ref.dataset_id}", timeout=600)
)
result = None
try:
with lock:
result, jobs = self._persist_creation(context, ref, config, initialize=initialize)
# No session or ORM entity crosses the task boundary. Keep the
# creation lock until committed documents have been dispatched.
DocumentService.dispatch_document_indexing(jobs)
except LockNotOwnedError:
if config.original_document_id:
raise
# Preserve successful creation when the Redis lease expires on release.
if result is None:
if initialize:
raise
with self._sessions() as session:
result = _created_response(
_require_dataset(session, ref), [], DocumentService.generate_document_batch(), session
)
assert result is not None
return result
def create_documents(self, context: RequestContext, ref: DatasetRef, settings: Mapping[str, Any]) -> dict[str, Any]:
config = KnowledgeConfig.model_validate(settings)
DocumentService.document_create_args_validate(config)
with _translate_errors():
snapshot = self._creation_dataset(context, ref, config)
config = DocumentService.prepare_document_creation(snapshot, config)
return self._run_creation(context, ref, config, initialize=False)
def initialize_dataset(self, context: RequestContext, settings: Mapping[str, Any]) -> dict[str, Any]:
config = KnowledgeConfig.model_validate(settings)
with _translate_errors():
if config.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
if config.embedding_model is None or config.embedding_model_provider is None:
raise ValueError(
"embedding model and embedding model provider are required for high quality indexing."
)
ModelManager.for_tenant(tenant_id=context.active_workspace_id).get_model_instance(
tenant_id=context.active_workspace_id,
provider=config.embedding_model_provider,
model_type=ModelType.TEXT_EMBEDDING,
model=config.embedding_model,
)
config = config.model_copy(
update={
"is_multimodal": DatasetService.check_is_multimodal_model(
context.active_workspace_id, config.embedding_model_provider, config.embedding_model
)
}
)
DocumentService.document_create_args_validate(config)
with self._sessions() as session:
self._actor(session, context)
config = DocumentService.prepare_document_creation(
DocumentCreationDataset(
context.active_workspace_id,
config.indexing_technique,
config.embedding_model,
config.embedding_model_provider,
),
config,
)
dataset_id = str(uuid4())
result = self._run_creation(
context, DatasetRef(context.active_workspace_id, dataset_id), config, initialize=True
)
if dify_config.RBAC_ENABLED:
rbac_service.RBACService.DatasetAccess.replace_whitelist(
context.active_workspace_id,
context.account_id,
dataset_id,
rbac_contracts.ReplaceMemberBindings(automatic_include_workspace_members=False),
)
rbac_service.try_sync_creator_access_policy_member_bindings(
context.active_workspace_id,
context.account_id,
rbac_contracts.RBACResourceType.DATASET,
dataset_id,
)
return result
def _check_model(self, ref: DatasetRef) -> None:
with self._sessions() as session:
dataset = _require_dataset(session, ref)
DatasetService.check_dataset_model_setting(dataset)
def delete_document(self, ref: DocumentRef) -> None:
self._check_model(ref.dataset)
with _translate_errors(), self._sessions() as session:
DocumentService.delete_document(_require_document(session, ref), session)
def delete_documents(self, ref: DatasetRef, document_ids: Sequence[str]) -> None:
with _translate_errors(), self._sessions() as session:
dataset = _require_dataset(session, ref)
DocumentService.delete_documents(
ref, list(document_ids), get_dataset_doc_form(dataset, session=session), session
)
def update_document(self, ref: DocumentRef, values: Mapping[str, object]) -> None:
with self._sessions.begin() as session:
document = _require_document(session, ref)
for name, value in values.items():
setattr(document, name, value)
def rename_document(self, ref: DocumentRef, name: str) -> dict[str, Any]:
with self._sessions.begin() as session:
dataset = _require_dataset(session, ref.dataset)
document = _require_document(session, ref)
if dataset.built_in_field_enabled and document.doc_metadata:
document.doc_metadata = {**document.doc_metadata, BuiltInField.document_name: name}
document.name = name
if document.data_source_info_dict and (file_id := document.data_source_info_dict.get("upload_file_id")):
session.execute(
update(UploadFile)
.where(UploadFile.id == file_id, UploadFile.tenant_id == ref.dataset.tenant_id)
.values(name=name)
)
session.flush()
return load_document_detail(document, session=session)
def pause_document(self, ref: DocumentRef, *, actor_id: str) -> None:
with _translate_errors(), self._sessions() as session:
DocumentService.pause_document(_require_document(session, ref), session, actor_id=actor_id)
def recover_document(self, ref: DocumentRef) -> None:
with _translate_errors(), self._sessions() as session:
DocumentService.recover_document(_require_document(session, ref), session)
def retry_documents(self, ref: DatasetRef, document_ids: Sequence[str], *, actor_id: str) -> None:
with _translate_errors(), self._sessions() as session:
documents = DocumentService.get_documents_by_ids(ref, document_ids, session)
DocumentService.retry_document(ref.dataset_id, list(documents), session, actor_id=actor_id)
def change_status(
self,
context: RequestContext,
ref: DatasetRef,
document_ids: Sequence[str],
action: Literal["enable", "disable", "archive", "un_archive"],
) -> None:
self._check_model(ref)
with _translate_errors(), self._sessions() as session:
dataset = _require_dataset(session, ref)
account = self._actor(session, context)
documents = DocumentService.get_documents_by_ids(ref, document_ids, session)
try:
DocumentService.batch_update_document_status(
dataset, [doc.id for doc in documents], action, account, session
)
except ValueError as error:
raise DocumentInvalidActionError(str(error)) from error
def sync_website(self, ref: DocumentRef) -> None:
with _translate_errors(), self._sessions() as session:
DocumentService.sync_website_document(
_require_dataset(session, ref.dataset), _require_document(session, ref), session
)
def require_summary_enabled(self, ref: DatasetRef) -> None:
with self._sessions() as session:
dataset = _require_dataset(session, ref)
if dataset.indexing_technique != IndexTechniqueType.HIGH_QUALITY:
raise ValueError(
"Summary generation is only available for 'high_quality' indexing technique. "
f"Current indexing technique: {dataset.indexing_technique}"
)
if not dataset.summary_index_setting or not dataset.summary_index_setting.get("enable"):
raise ValueError(
"Summary index is not enabled for this dataset. Please enable it in the dataset settings."
)
def enable_summary(self, ref: DatasetRef, document_ids: Sequence[str]) -> None:
if document_ids:
with self._sessions.begin() as session:
session.execute(
update(Document)
.where(
Document.tenant_id == ref.tenant_id,
Document.dataset_id == ref.dataset_id,
Document.id.in_(document_ids),
)
.values(need_summary=True)
)
def dispatch_summary(self, ref: DocumentRef) -> None:
generate_summary_index_task.delay(ref.dataset.dataset_id, ref.document_id)
@@ -0,0 +1,310 @@
"""Console document use cases, independent of HTTP and persistence frameworks."""
from collections.abc import Mapping, Sequence
from contextlib import AbstractContextManager
from dataclasses import dataclass
from typing import Any, Literal, Protocol
from libs.datetime_utils import naive_utc_now
from machinery.context import RequestContext
from services.knowledge.dataset_access import DatasetAccess
from services.knowledge.resource_scope import DatasetRef, DocumentRef
@dataclass(frozen=True)
class DocumentListFilter:
page: int = 1
limit: int = 20
search: str = ""
sort: str = "-created_at"
status: str | None = None
fetch: bool = False
@dataclass(frozen=True)
class DocumentState:
id: str
archived: bool
indexing_status: str
is_paused: bool
data_source_type: str
doc_form: str
need_summary: bool
@dataclass(frozen=True)
class DocumentZip:
path: str
filename: str
class DocumentApplicationError(Exception):
"""Base class for document use-case failures mapped by the transport."""
class DocumentNotFoundError(DocumentApplicationError):
pass
class DocumentArchivedError(DocumentApplicationError):
pass
class DocumentIndexingStateError(DocumentApplicationError):
pass
class DocumentInvalidActionError(DocumentApplicationError):
pass
class DocumentProviderError(DocumentApplicationError):
def __init__(self, description: str, *, kind: Literal["uninitialized", "quota", "unsupported"]) -> None:
super().__init__(description)
self.kind = kind
class DocumentOperations(Protocol):
"""Owned document operations; adapters return materialized values and own their sessions."""
def find_document(self, *, workspace_id: str, document_id: str) -> DocumentRef | None: ...
def get_state(self, ref: DocumentRef) -> DocumentState | None: ...
def get_states(self, ref: DatasetRef, document_ids: Sequence[str]) -> Sequence[DocumentState]: ...
def list_documents(self, ref: DatasetRef, query: DocumentListFilter) -> dict[str, Any]: ...
def get_detail(self, ref: DocumentRef, *, metadata_only: bool) -> dict[str, Any]: ...
def get_process_rule(self, ref: DatasetRef | None) -> dict[str, Any]: ...
def get_indexing_status(self, ref: DocumentRef) -> dict[str, Any]: ...
def get_batch_indexing_status(self, ref: DatasetRef, batch: str) -> dict[str, Any]: ...
def get_execution_log(self, ref: DocumentRef) -> dict[str, Any]: ...
def get_summary_status(self, ref: DocumentRef) -> dict[str, Any]: ...
def get_download_url(self, ref: DocumentRef) -> str: ...
def build_download_zip(
self, ref: DatasetRef, document_ids: Sequence[str]
) -> AbstractContextManager[DocumentZip]: ...
def create_documents(
self, context: RequestContext, ref: DatasetRef, settings: Mapping[str, Any]
) -> dict[str, Any]: ...
def initialize_dataset(self, context: RequestContext, settings: Mapping[str, Any]) -> dict[str, Any]: ...
def delete_document(self, ref: DocumentRef) -> None: ...
def delete_documents(self, ref: DatasetRef, document_ids: Sequence[str]) -> None: ...
def update_document(self, ref: DocumentRef, values: Mapping[str, object]) -> None: ...
def rename_document(self, ref: DocumentRef, name: str) -> dict[str, Any]: ...
def pause_document(self, ref: DocumentRef, *, actor_id: str) -> None: ...
def recover_document(self, ref: DocumentRef) -> None: ...
def retry_documents(self, ref: DatasetRef, document_ids: Sequence[str], *, actor_id: str) -> None: ...
def change_status(
self,
context: RequestContext,
ref: DatasetRef,
document_ids: Sequence[str],
action: Literal["enable", "disable", "archive", "un_archive"],
) -> None: ...
def sync_website(self, ref: DocumentRef) -> None: ...
def require_summary_enabled(self, ref: DatasetRef) -> None: ...
def enable_summary(self, ref: DatasetRef, document_ids: Sequence[str]) -> None: ...
def dispatch_summary(self, ref: DocumentRef) -> None: ...
class DatasetDocumentApplicationService:
"""Authorize document use cases and apply document state transitions."""
def __init__(
self,
*,
dataset_access: DatasetAccess,
operations: DocumentOperations,
metadata_schema: Mapping[str, Mapping[str, type]],
) -> None:
self._dataset_access = dataset_access
self._operations = operations
self._metadata_schema = metadata_schema
def _dataset(self, context: RequestContext, dataset_id: str) -> DatasetRef:
dataset = self._dataset_access.require_accessible(context, dataset_id)
return DatasetRef(tenant_id=dataset.workspace_id, dataset_id=dataset.id)
def _document(
self, context: RequestContext, dataset_id: str, document_id: str
) -> tuple[DocumentRef, DocumentState]:
ref = self._dataset(context, dataset_id).document(document_id)
state = self._operations.get_state(ref)
if state is None:
raise DocumentNotFoundError("Document not found.")
return ref, state
@staticmethod
def _require_mutable(state: DocumentState) -> None:
if state.archived:
raise DocumentArchivedError()
def get_process_rule(self, context: RequestContext, *, document_id: str | None) -> dict[str, Any]:
ref = None
if document_id:
document_ref = self._operations.find_document(
workspace_id=context.active_workspace_id, document_id=document_id
)
if document_ref is None:
raise DocumentNotFoundError("Document not found.")
ref = self._dataset(context, document_ref.dataset.dataset_id)
return self._operations.get_process_rule(ref)
def list_documents(self, context: RequestContext, *, dataset_id: str, query: DocumentListFilter) -> dict[str, Any]:
return self._operations.list_documents(self._dataset(context, dataset_id), query)
def get_document(
self, context: RequestContext, *, dataset_id: str, document_id: str, metadata_only: bool = False
) -> dict[str, Any]:
ref, _ = self._document(context, dataset_id, document_id)
return self._operations.get_detail(ref, metadata_only=metadata_only)
def get_indexing_status(self, context: RequestContext, *, dataset_id: str, document_id: str) -> dict[str, Any]:
ref, _ = self._document(context, dataset_id, document_id)
return self._operations.get_indexing_status(ref)
def get_batch_indexing_status(self, context: RequestContext, *, dataset_id: str, batch: str) -> dict[str, Any]:
return self._operations.get_batch_indexing_status(self._dataset(context, dataset_id), batch)
def get_execution_log(self, context: RequestContext, *, dataset_id: str, document_id: str) -> dict[str, Any]:
ref, _ = self._document(context, dataset_id, document_id)
return self._operations.get_execution_log(ref)
def get_summary_status(self, context: RequestContext, *, dataset_id: str, document_id: str) -> dict[str, Any]:
ref, _ = self._document(context, dataset_id, document_id)
return self._operations.get_summary_status(ref)
def get_download_url(self, context: RequestContext, *, dataset_id: str, document_id: str) -> str:
ref, _ = self._document(context, dataset_id, document_id)
return self._operations.get_download_url(ref)
def build_download_zip(
self, context: RequestContext, *, dataset_id: str, document_ids: Sequence[str]
) -> AbstractContextManager[DocumentZip]:
return self._operations.build_download_zip(self._dataset(context, dataset_id), document_ids)
def create_documents(
self, context: RequestContext, *, dataset_id: str, settings: Mapping[str, Any]
) -> dict[str, Any]:
return self._operations.create_documents(context, self._dataset(context, dataset_id), settings)
def initialize_dataset(self, context: RequestContext, *, settings: Mapping[str, Any]) -> dict[str, Any]:
return self._operations.initialize_dataset(context, settings)
def delete_document(self, context: RequestContext, *, dataset_id: str, document_id: str) -> None:
ref, _ = self._document(context, dataset_id, document_id)
self._operations.delete_document(ref)
def delete_documents(self, context: RequestContext, *, dataset_id: str, document_ids: Sequence[str]) -> None:
self._operations.delete_documents(self._dataset(context, dataset_id), document_ids)
def update_processing(
self,
context: RequestContext,
*,
dataset_id: str,
document_id: str,
action: Literal["pause", "resume"],
) -> None:
ref, state = self._document(context, dataset_id, document_id)
if action == "pause":
if state.indexing_status != "indexing":
raise DocumentInvalidActionError("Document not in indexing state.")
values: dict[str, object] = {
"paused_by": context.account_id,
"paused_at": naive_utc_now(),
"is_paused": True,
}
elif action == "resume":
if state.indexing_status not in {"paused", "error"}:
raise DocumentInvalidActionError("Document not in paused or error state.")
values = {"paused_by": None, "paused_at": None, "is_paused": False}
else:
raise DocumentInvalidActionError("Invalid action.")
self._operations.update_document(ref, values)
def update_metadata(
self,
context: RequestContext,
*,
dataset_id: str,
document_id: str,
doc_type: str | None,
doc_metadata: dict[str, Any] | None,
) -> None:
ref, _ = self._document(context, dataset_id, document_id)
if doc_type is None or doc_metadata is None:
raise ValueError("Both doc_type and doc_metadata must be provided.")
if doc_type not in self._metadata_schema:
raise ValueError("Invalid doc_type.")
metadata = (
doc_metadata
if doc_type == "others"
else {
key: doc_metadata[key]
for key, value_type in self._metadata_schema[doc_type].items()
if doc_metadata.get(key) is not None and isinstance(doc_metadata[key], value_type)
}
)
self._operations.update_document(
ref, {"doc_type": doc_type, "doc_metadata": metadata, "updated_at": naive_utc_now()}
)
def change_status(
self,
context: RequestContext,
*,
dataset_id: str,
document_ids: Sequence[str],
action: Literal["enable", "disable", "archive", "un_archive"],
) -> None:
self._operations.change_status(context, self._dataset(context, dataset_id), document_ids, action)
def pause_document(self, context: RequestContext, *, dataset_id: str, document_id: str) -> None:
ref, state = self._document(context, dataset_id, document_id)
self._require_mutable(state)
if state.indexing_status not in {"waiting", "parsing", "cleaning", "splitting", "indexing"}:
raise DocumentIndexingStateError("Cannot pause completed document.")
self._operations.pause_document(ref, actor_id=context.account_id)
def recover_document(self, context: RequestContext, *, dataset_id: str, document_id: str) -> None:
ref, state = self._document(context, dataset_id, document_id)
self._require_mutable(state)
if not state.is_paused:
raise DocumentIndexingStateError("Document is not in paused status.")
self._operations.recover_document(ref)
def retry_documents(self, context: RequestContext, *, dataset_id: str, document_ids: Sequence[str]) -> None:
ref = self._dataset(context, dataset_id)
states = {state.id: state for state in self._operations.get_states(ref, document_ids)}
retry_ids = [
document_id
for document_id in dict.fromkeys(document_ids)
if (state := states.get(document_id)) is not None
and not state.archived
and state.indexing_status != "completed"
]
self._operations.retry_documents(ref, retry_ids, actor_id=context.account_id)
def rename_document(
self, context: RequestContext, *, dataset_id: str, document_id: str, name: str
) -> dict[str, Any]:
ref, _ = self._document(context, dataset_id, document_id)
return self._operations.rename_document(ref, name)
def sync_website(self, context: RequestContext, *, dataset_id: str, document_id: str) -> None:
ref, state = self._document(context, dataset_id, document_id)
if state.data_source_type != "website_crawl":
raise ValueError("Document is not a website document.")
self._require_mutable(state)
self._operations.sync_website(ref)
def generate_summary(self, context: RequestContext, *, dataset_id: str, document_ids: Sequence[str]) -> None:
ref = self._dataset(context, dataset_id)
self._operations.require_summary_enabled(ref)
states = self._operations.get_states(ref, document_ids)
if len(states) != len(document_ids):
missing_ids = set(document_ids) - {state.id for state in states}
raise DocumentNotFoundError(f"Some documents not found: {sorted(missing_ids)}")
targets = [state for state in states if state.doc_form != "qa_model"]
self._operations.enable_summary(ref, [state.id for state in targets if not state.need_summary])
for state in targets:
self._operations.dispatch_summary(ref.document(state.id))
@@ -0,0 +1 @@
"""Shared data contracts for knowledge use cases."""
@@ -0,0 +1,85 @@
"""Materialized dataset read contracts, independent of persistence and HTTP.
Timestamps remain datetimes until transport serialization. JSON configuration
objects retain their stored keys; this boundary does not apply response defaults.
"""
from collections.abc import Mapping, Sequence
from datetime import datetime
from typing import NamedTuple, NotRequired, TypedDict
from pydantic import JsonValue
class DatasetTagRecord(TypedDict):
id: str
name: str
type: str
class DatasetDetailRecord(TypedDict):
id: str
name: str
description: str | None
provider: str
permission: str
data_source_type: str | None
indexing_technique: str | None
keyword_number: int | None
app_count: int
document_count: int
word_count: int
created_by: str
author_name: str | None
created_at: datetime
updated_by: str | None
updated_at: datetime
embedding_model: str | None
embedding_model_provider: str | None
# None means availability has not been evaluated by the application service.
embedding_available: bool | None
retrieval_model_dict: Mapping[str, JsonValue]
summary_index_setting: Mapping[str, JsonValue] | None
tags: list[DatasetTagRecord]
doc_form: str | None
external_knowledge_info: Mapping[str, JsonValue] | None
external_retrieval_model: Mapping[str, JsonValue] | None
doc_metadata: Sequence[Mapping[str, str]]
built_in_field_enabled: bool
pipeline_id: str | None
runtime_mode: str | None
chunk_structure: str | None
icon_info: Mapping[str, JsonValue] | None
is_published: bool
total_documents: int
total_available_documents: int
enable_api: bool
is_multimodal: bool
permission_keys: list[str]
maintainer: str | None
# Detail/create responses may omit membership or explicitly return null;
# listing and updates materialize a list, including an empty one.
partial_member_list: NotRequired[list[str] | None]
# Listing attaches Classic-to-Agent knowledge base upgrade availability.
knowledge_fs_upgrade: NotRequired[Mapping[str, JsonValue]]
class DatasetPage(TypedDict):
data: list[DatasetDetailRecord]
has_more: bool
total: int
page: int
limit: int
class DatasetCleanupRecord(NamedTuple):
"""Dataset state needed to schedule index cleanup after the row is deleted."""
id: str
workspace_id: str
indexing_technique: str | None
index_struct: str | None
collection_binding_id: str | None
# None means neither the dataset nor any of its documents declares a form.
doc_form: str | None
pipeline_id: str | None
@@ -0,0 +1,22 @@
"""Owned values passed between document creation's database and I/O phases."""
from dataclasses import dataclass
from services.knowledge.resource_scope import DatasetRef
@dataclass(frozen=True)
class DocumentCreationDataset:
tenant_id: str
indexing_technique: str | None
embedding_model: str | None
embedding_model_provider: str | None
@dataclass(frozen=True)
class DocumentIndexingJobs:
dataset: DatasetRef
created: tuple[str, ...] = ()
duplicated: tuple[str, ...] = ()
removed_notion: tuple[str, ...] = ()
updated: str | None = None
@@ -0,0 +1,159 @@
"""Commands and validation shared by knowledge indexing-estimate use cases."""
from collections.abc import Mapping
from dataclasses import dataclass, field
from typing import Annotated, Literal
from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator
from core.rag.entities import PreProcessingRuleKey
from services.data_source.entities.notion_import import NotionPageType
class EstimateValidationError(ValueError):
"""Raised when an estimate process rule is invalid."""
class _PreProcessingRule(BaseModel):
id: PreProcessingRuleKey
enabled: bool
class _Segmentation(BaseModel):
separator: str = Field(min_length=1)
max_tokens: int = Field(gt=0)
class _Rules(BaseModel):
pre_processing_rules: list[_PreProcessingRule]
segmentation: _Segmentation
@field_validator("pre_processing_rules")
@classmethod
def _deduplicate(cls, value: list[_PreProcessingRule]) -> list[_PreProcessingRule]:
by_id = {rule.id: rule for rule in value}
return list(by_id.values())
class _HierarchicalRules(_Rules):
parent_mode: Literal["full-doc", "paragraph"] | None = None
subchunk_segmentation: _Segmentation | None = None
class _SummaryIndexSettingDisabled(BaseModel):
enable: Literal[False] = False
class _SummaryIndexSettingEnabled(BaseModel):
enable: Literal[True]
model_name: str = Field(min_length=1)
model_provider_name: str = Field(min_length=1)
_SummaryIndexSetting = Annotated[
_SummaryIndexSettingDisabled | _SummaryIndexSettingEnabled,
Field(discriminator="enable"),
]
class _ProcessRuleBase(BaseModel):
model_config = ConfigDict(extra="allow")
summary_index_setting: _SummaryIndexSetting | None = None
@field_validator("summary_index_setting", mode="before")
@classmethod
def _normalize_summary_index_setting(cls, value: object) -> object:
"""Treat mappings with a missing/None enable flag as absent (#36602)."""
if value is None:
return None
if isinstance(value, Mapping) and value.get("enable") is None:
return None
return value
class _AutomaticProcessRule(_ProcessRuleBase):
mode: Literal["automatic"]
class _CustomProcessRule(_ProcessRuleBase):
mode: Literal["custom"]
rules: _Rules
class _HierarchicalProcessRule(_ProcessRuleBase):
mode: Literal["hierarchical"]
rules: _HierarchicalRules
_ProcessRule = Annotated[
_AutomaticProcessRule | _CustomProcessRule | _HierarchicalProcessRule,
Field(discriminator="mode"),
]
class _ProcessRuleEnvelope(BaseModel):
process_rule: _ProcessRule
def _validation_message(error: ValidationError) -> str:
first = error.errors()[0]
original = first.get("ctx", {}).get("error")
return str(original) if isinstance(original, ValueError) else str(first["msg"])
def normalize_process_rule(process_rule: Mapping[str, object]) -> dict[str, object]:
"""Validate and normalize a process rule without mutating caller-owned input."""
try:
validated = _ProcessRuleEnvelope.model_validate({"process_rule": process_rule})
except ValidationError as error:
raise EstimateValidationError(_validation_message(error)) from error
result = validated.process_rule.model_dump(exclude_none=True)
if validated.process_rule.mode == "automatic":
result["rules"] = {}
elif validated.process_rule.mode == "hierarchical":
rules = result.get("rules")
if isinstance(rules, dict) and not rules.get("parent_mode"):
rules["parent_mode"] = "paragraph"
return result
@dataclass(frozen=True, slots=True)
class UploadFileEstimateSource:
file_id: str
source_type: Literal["upload_file"] = field(default="upload_file", init=False)
@dataclass(frozen=True, slots=True)
class NotionEstimateSource:
workspace_id: str
page_id: str
page_type: NotionPageType
credential_id: str
source_type: Literal["notion_import"] = field(default="notion_import", init=False)
@dataclass(frozen=True, slots=True)
class WebsiteEstimateSource:
provider: str
job_id: str
url: str
mode: str = "crawl"
only_main_content: bool = False
source_type: Literal["website_crawl"] = field(default="website_crawl", init=False)
type NewEstimateSource = UploadFileEstimateSource | NotionEstimateSource | WebsiteEstimateSource
@dataclass(frozen=True, slots=True)
class NewSourcesEstimateCommand:
sources: tuple[NewEstimateSource, ...]
process_rule: Mapping[str, object]
doc_form: str = "text_model"
doc_language: str = "English"
dataset_id: str | None = None
indexing_technique: str = "economy"
@@ -0,0 +1,294 @@
from typing import Annotated, Any, Literal
from pydantic import BaseModel, Field, WithJsonSchema, field_validator
from core.rag.entities import MetadataFilteringCondition, Rule
from core.rag.index_processor.constant.index_type import IndexStructureType
from core.rag.retrieval.retrieval_methods import RetrievalMethod
from libs.helper import UUIDStr
from models.enums import ProcessRuleMode
DocForm = Annotated[
str,
WithJsonSchema({"enum": ["text_model", "hierarchical_model", "qa_model"], "type": "string"}),
]
IndexingTechnique = Annotated[
str | None,
WithJsonSchema({"anyOf": [{"enum": ["high_quality", "economy"], "type": "string"}, {"type": "null"}]}),
]
KnowledgeProvider = Annotated[
str,
WithJsonSchema({"enum": ["vendor", "external"], "type": "string"}),
]
RerankingMode = Annotated[
str | None,
WithJsonSchema({"anyOf": [{"enum": ["reranking_model", "weighted_score"], "type": "string"}, {"type": "null"}]}),
]
SummaryIndexSetting = Annotated[
dict[str, Any] | None,
WithJsonSchema(
{
"anyOf": [
{
"properties": {
"enable": {"description": "Whether to enable summary indexing.", "type": "boolean"},
"model_name": {
"description": "Name of the model used for generating summaries.",
"type": "string",
},
"model_provider_name": {
"description": "Provider of the summary generation model.",
"type": "string",
},
"summary_prompt": {
"description": "Custom prompt template for summary generation.",
"type": "string",
},
},
"type": "object",
},
{"type": "null"},
]
}
),
]
ExternalRetrievalModel = Annotated[
dict[str, Any] | None,
WithJsonSchema(
{
"anyOf": [
{
"properties": {
"top_k": {"description": "Maximum number of results to return.", "type": "integer"},
"score_threshold": {
"description": "Minimum similarity score threshold for filtering results.",
"type": "number",
},
"score_threshold_enabled": {
"description": "Whether score threshold filtering is enabled.",
"type": "boolean",
},
},
"type": "object",
},
{"type": "null"},
]
}
),
]
class RerankingModel(BaseModel):
reranking_provider_name: str | None = Field(default=None, description="Provider name of the reranking model.")
reranking_model_name: str | None = Field(default=None, description="Name of the reranking model.")
class NotionIcon(BaseModel):
type: str
url: str | None = None
emoji: str | None = None
class NotionPage(BaseModel):
page_id: str
page_name: str
page_icon: NotionIcon | None = None
type: str
class NotionInfo(BaseModel):
credential_id: str
workspace_id: str
pages: list[NotionPage]
class WebsiteInfo(BaseModel):
provider: str
job_id: str
urls: list[str]
only_main_content: bool = True
class FileInfo(BaseModel):
file_ids: list[str]
class InfoList(BaseModel):
data_source_type: Literal["upload_file", "notion_import", "website_crawl"]
notion_info_list: list[NotionInfo] | None = None
file_info_list: FileInfo | None = None
website_info_list: WebsiteInfo | None = None
class DataSource(BaseModel):
info_list: InfoList
class ProcessRule(BaseModel):
mode: ProcessRuleMode = Field(
description=(
"`automatic` uses built-in rules, `custom` allows manual configuration, `hierarchical` enables "
"parent-child chunk structure (use with `doc_form: hierarchical_model`)."
)
)
rules: Rule | None = Field(default=None, description="Custom processing rules.")
class WeightVectorSetting(BaseModel):
vector_weight: float = Field(description="Weight assigned to semantic vector search results.")
embedding_provider_name: str = Field(description="Provider of the embedding model used for vector search.")
embedding_model_name: str = Field(description="Name of the embedding model used for vector search.")
class WeightKeywordSetting(BaseModel):
keyword_weight: float = Field(description="Weight assigned to keyword search results.")
class WeightModel(BaseModel):
weight_type: Literal["semantic_first", "keyword_first", "customized"] | None = Field(
default=None,
description="Strategy for balancing semantic and keyword search weights.",
)
vector_setting: WeightVectorSetting | None = Field(default=None, description="Semantic search weight settings.")
keyword_setting: WeightKeywordSetting | None = Field(default=None, description="Keyword search weight settings.")
class RetrievalModel(BaseModel):
search_method: RetrievalMethod = Field(description="Search method used for retrieval.")
reranking_enable: bool = Field(description="Whether reranking is enabled.")
reranking_model: RerankingModel | None = Field(default=None, description="Reranking model configuration.")
reranking_mode: RerankingMode = Field(
default=None,
description="Reranking mode. Required when `reranking_enable` is `true`.",
)
top_k: int = Field(description="Maximum number of results to return.")
score_threshold_enabled: bool = Field(description="Whether score threshold filtering is enabled.")
score_threshold: float | None = Field(
default=None,
description="Minimum similarity score for results. Only effective when score threshold filtering is enabled.",
)
weights: WeightModel | None = Field(default=None, description="Weight configuration for hybrid search.")
metadata_filtering_conditions: MetadataFilteringCondition | None = Field(
default=None,
description=(
"Restrict retrieval to chunks whose document metadata matches the given conditions. Conditions are "
"evaluated server-side against document metadata fields."
),
)
class MetaDataConfig(BaseModel):
doc_type: str
doc_metadata: dict[str, Any]
class KnowledgeConfig(BaseModel):
original_document_id: str | None = Field(default=None, description="Original document ID for replacement updates.")
duplicate: bool = Field(default=True, description="Whether duplicate document content is allowed.")
indexing_technique: Literal["high_quality", "economy"] = Field(
description=(
"`high_quality` uses embedding models for precise search; `economy` uses keyword-based indexing. "
"Required when adding the first document to a knowledge base; subsequent documents inherit the "
"knowledge base's indexing technique if omitted."
)
)
data_source: DataSource | None = Field(default=None, description="Document data source configuration.")
process_rule: ProcessRule | None = Field(default=None, description="Processing rules for chunking.")
retrieval_model: RetrievalModel | None = Field(
default=None,
description=(
"Retrieval model configuration. Controls how chunks are searched and ranked in this knowledge base."
),
)
summary_index_setting: SummaryIndexSetting = Field(
default=None,
description="Summary index configuration.",
)
doc_form: DocForm = Field(
default="text_model",
description=(
"`text_model` for standard text chunking, `hierarchical_model` for parent-child chunk structure, "
"`qa_model` for question-answer pair extraction."
),
)
doc_language: str = Field(default="English", description="Language of the document for processing optimization.")
embedding_model: str | None = Field(
default=None,
description=(
"Embedding model name. Use the `model` field from "
"[Get Available Models](/api-reference/models/get-available-models) with `model_type=text-embedding`."
),
)
embedding_model_provider: str | None = Field(
default=None,
description=(
"Embedding model provider. Use the `provider` field from "
"[Get Available Models](/api-reference/models/get-available-models) with `model_type=text-embedding`."
),
)
name: str | None = Field(default=None, description="Document name.")
is_multimodal: bool = Field(default=False, description="Whether the document uses multimodal indexing.")
@field_validator("doc_form")
@classmethod
def validate_doc_form(cls, value: str) -> str:
valid_forms = [
IndexStructureType.PARAGRAPH_INDEX,
IndexStructureType.QA_INDEX,
IndexStructureType.PARENT_CHILD_INDEX,
]
if value not in valid_forms:
raise ValueError("Invalid doc_form.")
return value
class SegmentCreateArgs(BaseModel):
content: str | None = Field(default=None, description="Chunk text content.")
answer: str | None = Field(default=None, description="Answer content for QA mode.")
keywords: list[str] | None = Field(default=None, description="Keywords for the chunk.")
attachment_ids: list[str] | None = Field(default=None, description="Attachment file IDs.")
class MetadataArgs(BaseModel):
type: Literal["string", "number", "time"] = Field(
description="`string` for text values, `number` for numeric values, `time` for date/time values."
)
name: str = Field(description="Metadata field name.")
class MetadataUpdateArgs(BaseModel):
name: str = Field(description="Metadata field name.")
value: str | int | float | None = Field(
default=None,
description="Metadata value. Can be a string, number, or `null`.",
)
class MetadataDetail(BaseModel):
id: UUIDStr = Field(description="Metadata field ID.")
name: str = Field(description="Metadata field name.")
value: str | int | float | None = Field(
default=None,
description="Metadata value. Can be a string, number, or `null`.",
)
class DocumentMetadataOperation(BaseModel):
document_id: UUIDStr = Field(description="Document ID whose metadata should be updated.")
metadata_list: list[MetadataDetail] = Field(description="Metadata fields to update.")
partial_update: bool = Field(
default=False,
description="Whether to partially update metadata, keeping existing values for unspecified fields.",
)
class MetadataOperationData(BaseModel):
"""
Metadata operation data
"""
operation_data: list[DocumentMetadataOperation] = Field(
description=(
"Array of document metadata update operations. Each entry maps a document ID to its metadata values."
)
)
@@ -0,0 +1,79 @@
"""Detached knowledge segment values shared by application ports."""
from datetime import datetime
from pydantic import BaseModel, ConfigDict, Field
class ChildChunkUpdateArgs(BaseModel):
id: str | None = Field(default=None, description="Existing child chunk ID. Omit to create a new child chunk.")
content: str = Field(description="Child chunk text content.")
class SegmentAttachmentRecord(BaseModel):
model_config = ConfigDict(from_attributes=True, frozen=True)
id: str
name: str
size: int
extension: str
mime_type: str | None
source_url: str
class ChildChunkRecord(BaseModel):
model_config = ConfigDict(from_attributes=True, frozen=True)
id: str
segment_id: str
content: str
position: int
word_count: int
type: str
created_at: datetime
updated_at: datetime
class SegmentRecord(BaseModel):
model_config = ConfigDict(from_attributes=True, frozen=True)
id: str
position: int
document_id: str
content: str
sign_content: str
answer: str | None
word_count: int
tokens: int
keywords: tuple[str, ...] | None
index_node_id: str | None
index_node_hash: str | None
hit_count: int
enabled: bool
disabled_at: datetime | None
disabled_by: str | None
status: str
created_by: str
created_at: datetime
updated_at: datetime
updated_by: str | None
indexing_at: datetime | None
completed_at: datetime | None
error: str | None
stopped_at: datetime | None
child_chunks: tuple[ChildChunkRecord, ...]
attachments: tuple[SegmentAttachmentRecord, ...]
summary: str | None
class SegmentUpdateArgs(BaseModel):
content: str | None = Field(default=None, description="Updated chunk text content.")
answer: str | None = Field(default=None, description="Updated answer content for QA mode.")
keywords: list[str] | None = Field(default=None, description="Updated keywords for the chunk.")
regenerate_child_chunks: bool = Field(
default=False,
description="Whether to regenerate child chunks after updating a parent chunk.",
)
enabled: bool | None = Field(default=None, description="Whether the chunk is enabled.")
attachment_ids: list[str] | None = Field(default=None, description="Attachment file IDs.")
summary: str | None = Field(default=None, description="Summary content for summary index.")
@@ -0,0 +1 @@
"""External knowledge connection and retrieval use cases."""
@@ -0,0 +1,155 @@
"""External knowledge persistence and network adapters with bounded sessions."""
from collections.abc import Mapping
from typing import Any
from sqlalchemy import select
from sqlalchemy.orm import Session, sessionmaker
from core.rag.entities import MetadataFilteringCondition
from libs.pagination import clamp_pagination
from machinery.context import RequestContext
from models.dataset import DatasetQuery, ExternalKnowledgeApis
from models.enums import CreatorUserRole, DatasetQuerySource
from repositories.knowledge.dataset_read_repository import (
get_external_api_bindings_batch,
get_external_api_dataset_bindings,
)
from services.entities.external_knowledge_entities.external_knowledge_entities import ExternalDatasetCreatePayload
from services.hit_testing_service import HitTestingService
from services.knowledge.dataset_read_service import load_dataset_detail
from services.knowledge.datasets.adapters import require_dataset
from services.knowledge.entities.datasets import DatasetDetailRecord
from services.knowledge.external.application import ExternalTemplateNotFoundError
from services.knowledge.external.service import ExternalDatasetService
from services.knowledge.resource_scope import DatasetRef
from services.rbac.contracts import RBACResourceType
from services.rbac.members import MemberService
class SQLAlchemyExternalKnowledgeOperations:
def __init__(self, *, session_factory: sessionmaker[Session], members: MemberService) -> None:
self._sessions = session_factory
self._members = members
@staticmethod
def _template(session: Session, workspace_id: str, template_id: str) -> ExternalKnowledgeApis:
template = session.scalar(
select(ExternalKnowledgeApis).where(
ExternalKnowledgeApis.id == template_id, ExternalKnowledgeApis.tenant_id == workspace_id
)
)
if template is None:
raise ExternalTemplateNotFoundError("API template not found.")
return template
@staticmethod
def _values(session: Session, template: ExternalKnowledgeApis) -> dict[str, Any]:
return dict(template.to_dict(dataset_bindings=get_external_api_dataset_bindings(template, session=session)))
def list_templates(self, workspace_id: str, *, page: int, limit: int, keyword: str | None) -> dict[str, Any]:
page, limit = clamp_pagination(page, limit, 100)
with self._sessions() as session:
templates, total = ExternalDatasetService.get_external_knowledge_apis(
page, limit, workspace_id, keyword, session=session
)
bindings = get_external_api_bindings_batch(templates, session=session)
return {
"data": [item.to_dict(dataset_bindings=bindings.get(item.id, [])) for item in templates],
"has_more": page * limit < total,
"total": total,
"page": page,
"limit": limit,
}
def get_template(self, workspace_id: str, template_id: str) -> dict[str, Any]:
with self._sessions() as session:
return self._values(session, self._template(session, workspace_id, template_id))
def create_template(self, context: RequestContext, *, name: str, settings: Mapping[str, Any]) -> dict[str, Any]:
# The legacy creator probes the endpoint before its first SQL statement.
# An unopened session keeps that external request outside a transaction.
with self._sessions() as session:
template = ExternalDatasetService.create_external_knowledge_api(
context.active_workspace_id,
context.account_id,
{"name": name, "settings": dict(settings)},
session=session,
)
result = self._values(session, template)
session.commit()
return result
def update_template(
self, context: RequestContext, template_id: str, *, name: str, settings: Mapping[str, Any]
) -> dict[str, Any]:
with self._sessions.begin() as session:
self._template(session, context.active_workspace_id, template_id)
template = ExternalDatasetService.update_external_knowledge_api(
context.active_workspace_id,
context.account_id,
template_id,
{"name": name, "settings": dict(settings)},
session=session,
)
return self._values(session, template)
def delete_template(self, workspace_id: str, template_id: str) -> None:
with self._sessions.begin() as session:
session.delete(self._template(session, workspace_id, template_id))
def template_usage(self, workspace_id: str, template_id: str) -> tuple[bool, int]:
with self._sessions() as session:
return ExternalDatasetService.external_knowledge_api_use_check(template_id, workspace_id, session=session)
def create_dataset(self, context: RequestContext, payload: ExternalDatasetCreatePayload) -> DatasetDetailRecord:
with self._sessions() as session:
self._template(session, context.active_workspace_id, payload.external_knowledge_api_id)
dataset = ExternalDatasetService.create_external_dataset(
context.active_workspace_id, context.account_id, payload, session=session
)
permissions = self._members.resource_permissions(
context.active_workspace_id, context.account_id, RBACResourceType.DATASET, [dataset.id]
)
result = load_dataset_detail(dataset, session=session)
result["permission_keys"] = permissions.get(dataset.id, [])
session.commit()
return result
def retrieve(
self,
context: RequestContext,
ref: DatasetRef,
*,
query: str,
retrieval_model: dict[str, Any] | None,
metadata_filters: dict[str, Any] | None,
) -> dict[str, Any]:
with self._sessions() as session:
dataset = require_dataset(session, ref)
if dataset.provider != "external":
return {"query": {"content": query}, "records": []}
request = ExternalDatasetService.prepare_external_knowledge_retrieval(
tenant_id=ref.tenant_id,
dataset_id=ref.dataset_id,
query=HitTestingService.escape_query_for_search(query),
external_retrieval_parameters=retrieval_model or {},
metadata_condition=MetadataFilteringCondition.model_validate(metadata_filters)
if metadata_filters
else None,
session=session,
)
documents = ExternalDatasetService.execute_external_knowledge_retrieval(request)
with self._sessions.begin() as session:
require_dataset(session, ref)
session.add(
DatasetQuery(
dataset_id=ref.dataset_id,
content=query,
source=DatasetQuerySource.HIT_TESTING,
source_app_id=None,
created_by_role=CreatorUserRole.ACCOUNT,
created_by=context.account_id,
)
)
return dict(HitTestingService.compact_external_retrieve_response(dataset, query, documents))
@@ -0,0 +1,104 @@
"""External knowledge use cases without transport or persistence dependencies."""
from collections.abc import Mapping
from typing import Any, Protocol
from machinery.context import RequestContext
from services.entities.external_knowledge_entities.external_knowledge_entities import ExternalDatasetCreatePayload
from services.knowledge.dataset_access import DatasetAccess
from services.knowledge.entities.datasets import DatasetDetailRecord
from services.knowledge.resource_scope import DatasetRef
class ExternalTemplateNotFoundError(Exception):
pass
class ExternalHitTestingError(Exception):
pass
class ExternalKnowledgeOperations(Protocol):
def list_templates(self, workspace_id: str, *, page: int, limit: int, keyword: str | None) -> dict[str, Any]: ...
def get_template(self, workspace_id: str, template_id: str) -> dict[str, Any]: ...
def create_template(self, context: RequestContext, *, name: str, settings: Mapping[str, Any]) -> dict[str, Any]: ...
def update_template(
self, context: RequestContext, template_id: str, *, name: str, settings: Mapping[str, Any]
) -> dict[str, Any]: ...
def delete_template(self, workspace_id: str, template_id: str) -> None: ...
def template_usage(self, workspace_id: str, template_id: str) -> tuple[bool, int]: ...
def create_dataset(self, context: RequestContext, payload: ExternalDatasetCreatePayload) -> DatasetDetailRecord: ...
def retrieve(
self,
context: RequestContext,
ref: DatasetRef,
*,
query: str,
retrieval_model: dict[str, Any] | None,
metadata_filters: dict[str, Any] | None,
) -> dict[str, Any]: ...
class ExternalKnowledgeApplicationService:
def __init__(self, *, dataset_access: DatasetAccess, operations: ExternalKnowledgeOperations) -> None:
self._dataset_access = dataset_access
self._operations = operations
@staticmethod
def _validate_settings(settings: Mapping[str, Any]) -> None:
if not settings:
raise ValueError("api list is empty")
if not settings.get("endpoint"):
raise ValueError("endpoint is required")
if not settings.get("api_key"):
raise ValueError("api_key is required")
def list_templates(self, context: RequestContext, *, page: int, limit: int, keyword: str | None) -> dict[str, Any]:
return self._operations.list_templates(context.active_workspace_id, page=page, limit=limit, keyword=keyword)
def get_template(self, context: RequestContext, *, template_id: str) -> dict[str, Any]:
return self._operations.get_template(context.active_workspace_id, template_id)
def create_template(self, context: RequestContext, *, name: str, settings: Mapping[str, Any]) -> dict[str, Any]:
self._validate_settings(settings)
return self._operations.create_template(context, name=name, settings=settings)
def update_template(
self, context: RequestContext, *, template_id: str, name: str, settings: Mapping[str, Any]
) -> dict[str, Any]:
self._validate_settings(settings)
return self._operations.update_template(context, template_id, name=name, settings=settings)
def delete_template(self, context: RequestContext, *, template_id: str) -> None:
self._operations.delete_template(context.active_workspace_id, template_id)
def template_usage(self, context: RequestContext, *, template_id: str) -> tuple[bool, int]:
return self._operations.template_usage(context.active_workspace_id, template_id)
def create_dataset(self, context: RequestContext, *, payload: ExternalDatasetCreatePayload) -> DatasetDetailRecord:
return self._operations.create_dataset(context, payload)
def hit_testing(
self,
context: RequestContext,
*,
dataset_id: str,
query: str,
retrieval_model: dict[str, Any] | None,
metadata_filters: dict[str, Any] | None,
) -> dict[str, Any]:
dataset = self._dataset_access.require_accessible(context, dataset_id)
if not query:
raise ValueError("Query or attachment_ids is required")
if len(query) > 250:
raise ValueError("Query cannot exceed 250 characters")
try:
return self._operations.retrieve(
context,
DatasetRef(dataset.workspace_id, dataset.id),
query=query,
retrieval_model=retrieval_model,
metadata_filters=metadata_filters,
)
except Exception as error:
raise ExternalHitTestingError(str(error)) from error
@@ -0,0 +1,445 @@
import json
from copy import deepcopy
from typing import Any, cast
from urllib.parse import urlparse
import httpx
from sqlalchemy import func, select
from sqlalchemy.orm import Session
from constants import HIDDEN_VALUE
from core.helper import ssrf_proxy
from core.rag.entities import MetadataFilteringCondition
from extensions.ext_database import db # noqa: F401
from graphon.nodes.http_request.exc import InvalidHttpMethodError
from libs.datetime_utils import naive_utc_now
from libs.pagination import paginate_query
from models.dataset import (
Dataset,
ExternalKnowledgeApis,
ExternalKnowledgeBindings,
)
from services.enterprise import rbac_service as enterprise_rbac_service
from services.entities.external_knowledge_entities.external_knowledge_entities import (
Authorization,
ExternalDatasetCreatePayload,
ExternalKnowledgeApiSetting,
)
from services.errors.dataset import DatasetNameDuplicateError
from services.errors.knowledge_retrieval import ExternalKnowledgeRetrievalError
from services.rbac import contracts as rbac_contracts
class ExternalDatasetService:
@staticmethod
def get_external_knowledge_apis(
page, per_page, tenant_id, search=None, *, session: Session
) -> tuple[list[ExternalKnowledgeApis], int]:
query = (
select(ExternalKnowledgeApis)
.where(ExternalKnowledgeApis.tenant_id == tenant_id)
.order_by(ExternalKnowledgeApis.created_at.desc())
)
if search:
from libs.helper import escape_like_pattern
escaped_search = escape_like_pattern(search)
query = query.where(ExternalKnowledgeApis.name.ilike(f"%{escaped_search}%", escape="\\"))
external_knowledge_apis = paginate_query(query, session=session, page=page, per_page=per_page, max_per_page=100)
return external_knowledge_apis.items, external_knowledge_apis.total
@classmethod
def validate_api_list(cls, api_settings: dict[str, Any]):
if not api_settings:
raise ValueError("api list is empty")
if not api_settings.get("endpoint"):
raise ValueError("endpoint is required")
if not api_settings.get("api_key"):
raise ValueError("api_key is required")
@staticmethod
def create_external_knowledge_api(
tenant_id: str, user_id: str, args: dict[str, Any], *, session: Session
) -> ExternalKnowledgeApis:
settings = args.get("settings")
if settings is None:
raise ValueError("settings is required")
ExternalDatasetService.check_endpoint_and_api_key(settings)
external_knowledge_api = ExternalKnowledgeApis(
tenant_id=tenant_id,
created_by=user_id,
updated_by=user_id,
name=str(args.get("name")),
description=args.get("description", ""),
settings=json.dumps(args.get("settings"), ensure_ascii=False),
)
session.add(external_knowledge_api)
session.flush()
return external_knowledge_api
@staticmethod
def check_endpoint_and_api_key(settings: dict[str, Any]):
if "endpoint" not in settings or not settings["endpoint"]:
raise ValueError("endpoint is required")
if "api_key" not in settings or not settings["api_key"]:
raise ValueError("api_key is required")
endpoint = f"{settings['endpoint']}/retrieval"
api_key = settings["api_key"]
parsed_url = urlparse(endpoint)
if not all([parsed_url.scheme, parsed_url.netloc]):
if not endpoint.startswith("http://") and not endpoint.startswith("https://"):
raise ValueError(f"invalid endpoint: {endpoint} must start with http:// or https://")
else:
raise ValueError(f"invalid endpoint: {endpoint}")
# Send a minimal body shaped like the External Knowledge API retrieval contract so providers
# that require a JSON payload (e.g. RAGFlow) accept the validation probe instead of rejecting
# a body-less POST. Mirrors the request built in fetch_external_knowledge_retrieval.
validation_payload = {
"knowledge_id": "",
"query": "",
"retrieval_setting": {"top_k": 1, "score_threshold": 0.0},
}
try:
response = ssrf_proxy.post(
endpoint,
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
data=json.dumps(validation_payload),
)
except Exception as e:
raise ValueError(f"failed to connect to the endpoint: {endpoint}") from e
if response.status_code == 502:
raise ValueError(f"Bad Gateway: failed to connect to the endpoint: {endpoint}")
if response.status_code == 404:
raise ValueError(f"Not Found: failed to connect to the endpoint: {endpoint}")
if response.status_code == 403:
raise ValueError("Forbidden: Authorization failed with the provided api_key")
@staticmethod
def get_external_knowledge_api(
external_knowledge_api_id: str, tenant_id: str, *, session: Session
) -> ExternalKnowledgeApis:
external_knowledge_api: ExternalKnowledgeApis | None = session.scalar(
select(ExternalKnowledgeApis)
.where(ExternalKnowledgeApis.id == external_knowledge_api_id, ExternalKnowledgeApis.tenant_id == tenant_id)
.limit(1)
)
if external_knowledge_api is None:
raise ValueError("api template not found")
return external_knowledge_api
@staticmethod
def update_external_knowledge_api(
tenant_id: str,
user_id: str,
external_knowledge_api_id: str,
args: dict[str, Any],
*,
session: Session,
) -> ExternalKnowledgeApis:
external_knowledge_api: ExternalKnowledgeApis | None = session.scalar(
select(ExternalKnowledgeApis)
.where(ExternalKnowledgeApis.id == external_knowledge_api_id, ExternalKnowledgeApis.tenant_id == tenant_id)
.limit(1)
)
if external_knowledge_api is None:
raise ValueError("api template not found")
settings = args.get("settings")
if settings and settings.get("api_key") == HIDDEN_VALUE and external_knowledge_api.settings_dict:
settings["api_key"] = external_knowledge_api.settings_dict.get("api_key")
external_knowledge_api.name = str(args.get("name"))
external_knowledge_api.description = str(args.get("description", ""))
external_knowledge_api.settings = json.dumps(settings, ensure_ascii=False)
external_knowledge_api.updated_by = user_id
external_knowledge_api.updated_at = naive_utc_now()
session.flush()
return external_knowledge_api
@staticmethod
def delete_external_knowledge_api(tenant_id: str, external_knowledge_api_id: str, *, session: Session) -> None:
external_knowledge_api = session.scalar(
select(ExternalKnowledgeApis)
.where(ExternalKnowledgeApis.id == external_knowledge_api_id, ExternalKnowledgeApis.tenant_id == tenant_id)
.limit(1)
)
if external_knowledge_api is None:
raise ValueError("api template not found")
session.delete(external_knowledge_api)
session.flush()
@staticmethod
def external_knowledge_api_use_check(
external_knowledge_api_id: str, tenant_id: str, *, session: Session
) -> tuple[bool, int]:
"""
Return usage for an external knowledge API within a single tenant.
The caller already scopes access by tenant, so this query must do the
same; otherwise the endpoint becomes a cross-tenant UUID oracle.
"""
count = (
session.scalar(
select(func.count(ExternalKnowledgeBindings.id)).where(
ExternalKnowledgeBindings.external_knowledge_api_id == external_knowledge_api_id,
ExternalKnowledgeBindings.tenant_id == tenant_id,
)
)
or 0
)
return count > 0, count
@staticmethod
def get_external_knowledge_binding_with_dataset_id(
tenant_id: str, dataset_id: str, *, session: Session
) -> ExternalKnowledgeBindings:
external_knowledge_binding: ExternalKnowledgeBindings | None = session.scalar(
select(ExternalKnowledgeBindings)
.where(ExternalKnowledgeBindings.dataset_id == dataset_id, ExternalKnowledgeBindings.tenant_id == tenant_id)
.limit(1)
)
if not external_knowledge_binding:
raise ValueError("external knowledge binding not found")
return external_knowledge_binding
@staticmethod
def document_create_args_validate(
tenant_id: str,
external_knowledge_api_id: str,
process_parameter: dict[str, Any],
*,
session: Session,
) -> None:
external_knowledge_api = session.scalar(
select(ExternalKnowledgeApis)
.where(ExternalKnowledgeApis.id == external_knowledge_api_id, ExternalKnowledgeApis.tenant_id == tenant_id)
.limit(1)
)
if external_knowledge_api is None or external_knowledge_api.settings is None:
raise ValueError("api template not found")
settings = json.loads(external_knowledge_api.settings)
for setting in settings:
custom_parameters = setting.get("document_process_setting")
if custom_parameters:
for parameter in custom_parameters:
if parameter.get("required", False) and not process_parameter.get(parameter.get("name")):
raise ValueError(f"{parameter.get('name')} is required")
@staticmethod
def process_external_api(settings: ExternalKnowledgeApiSetting, files: dict[str, Any] | None) -> httpx.Response:
"""
do http request depending on api bundle
"""
kwargs: dict[str, Any] = {
"url": settings.url,
"headers": settings.headers,
"follow_redirects": True,
}
_method_map = {
"get": ssrf_proxy.get,
"head": ssrf_proxy.head,
"post": ssrf_proxy.post,
"put": ssrf_proxy.put,
"delete": ssrf_proxy.delete,
"patch": ssrf_proxy.patch,
}
method_lc = settings.request_method.lower()
if method_lc not in _method_map:
raise InvalidHttpMethodError(f"Invalid http method {settings.request_method}")
response: httpx.Response = _method_map[method_lc](data=json.dumps(settings.params), files=files, **kwargs)
return response
@staticmethod
def assembling_headers(authorization: Authorization, headers: dict[str, Any] | None = None) -> dict[str, Any]:
authorization = deepcopy(authorization)
headers = deepcopy(headers) if headers else {}
if authorization.type == "api-key":
if authorization.config is None:
raise ValueError("authorization config is required")
if authorization.config.api_key is None:
raise ValueError("api_key is required")
if not authorization.config.header:
authorization.config.header = "Authorization"
if authorization.config.type == "bearer":
headers[authorization.config.header] = f"Bearer {authorization.config.api_key}"
elif authorization.config.type == "basic":
headers[authorization.config.header] = f"Basic {authorization.config.api_key}"
elif authorization.config.type == "custom":
headers[authorization.config.header] = authorization.config.api_key
return headers
@staticmethod
def get_external_knowledge_api_settings(settings: dict[str, Any]) -> ExternalKnowledgeApiSetting:
return ExternalKnowledgeApiSetting.model_validate(settings)
@staticmethod
def create_external_dataset(
tenant_id: str, user_id: str, args: ExternalDatasetCreatePayload, *, session: Session
) -> Dataset:
"""Create a tenant-scoped external dataset and binding in the caller's transaction."""
# check if dataset name already exists
if session.scalar(select(Dataset).where(Dataset.name == args.name, Dataset.tenant_id == tenant_id).limit(1)):
raise DatasetNameDuplicateError(f"Dataset with name {args.name} already exists.")
external_knowledge_api = session.scalar(
select(ExternalKnowledgeApis)
.where(
ExternalKnowledgeApis.id == args.external_knowledge_api_id,
ExternalKnowledgeApis.tenant_id == tenant_id,
)
.limit(1)
)
if external_knowledge_api is None:
raise ValueError("api template not found")
dataset = Dataset(
tenant_id=tenant_id,
name=args.name,
description=args.description or "",
provider="external",
retrieval_model=args.external_retrieval_model,
created_by=user_id,
maintainer=user_id,
)
session.add(dataset)
session.flush()
external_knowledge_binding = ExternalKnowledgeBindings(
tenant_id=tenant_id,
dataset_id=dataset.id,
external_knowledge_api_id=args.external_knowledge_api_id or "",
external_knowledge_id=args.external_knowledge_id or "",
created_by=user_id,
)
session.add(external_knowledge_binding)
session.commit()
enterprise_rbac_service.try_sync_creator_access_policy_member_bindings(
tenant_id,
user_id,
rbac_contracts.RBACResourceType.DATASET,
dataset.id,
)
return dataset
@staticmethod
def fetch_external_knowledge_retrieval(
tenant_id: str,
dataset_id: str,
query: str,
external_retrieval_parameters: dict[str, Any],
metadata_condition: MetadataFilteringCondition | None = None,
*,
session: Session,
):
"""Fetch retrieval records from an external knowledge provider.
Success requires a tenant-scoped binding plus API template and a ``200``
response body shaped like ``{"records": [...]}``. All dependency
failures, non-200 responses, and malformed success payloads must be
normalized to ``ExternalKnowledgeRetrievalError`` so callers—especially
the inner knowledge retrieval API—can consistently expose
``502 external_knowledge_failed``.
"""
request = ExternalDatasetService.prepare_external_knowledge_retrieval(
tenant_id, dataset_id, query, external_retrieval_parameters, metadata_condition, session=session
)
return ExternalDatasetService.execute_external_knowledge_retrieval(request)
@staticmethod
def prepare_external_knowledge_retrieval(
tenant_id: str,
dataset_id: str,
query: str,
external_retrieval_parameters: dict[str, Any],
metadata_condition: MetadataFilteringCondition | None = None,
*,
session: Session,
) -> ExternalKnowledgeApiSetting:
"""Resolve owned configuration into a request that outlives the read session."""
external_knowledge_binding = session.scalar(
select(ExternalKnowledgeBindings)
.where(ExternalKnowledgeBindings.dataset_id == dataset_id, ExternalKnowledgeBindings.tenant_id == tenant_id)
.limit(1)
)
if not external_knowledge_binding:
raise ExternalKnowledgeRetrievalError("external knowledge binding not found")
external_knowledge_api = session.scalar(
select(ExternalKnowledgeApis)
.where(
ExternalKnowledgeApis.id == external_knowledge_binding.external_knowledge_api_id,
ExternalKnowledgeApis.tenant_id == tenant_id,
)
.limit(1)
)
if external_knowledge_api is None or external_knowledge_api.settings is None:
raise ExternalKnowledgeRetrievalError("external api template not found")
settings = json.loads(external_knowledge_api.settings)
headers = {"Content-Type": "application/json"}
if settings.get("api_key"):
headers["Authorization"] = f"Bearer {settings.get('api_key')}"
score_threshold_enabled = external_retrieval_parameters.get("score_threshold_enabled") or False
score_threshold = external_retrieval_parameters.get("score_threshold", 0.0) if score_threshold_enabled else 0.0
request_params = {
"retrieval_setting": {
"top_k": external_retrieval_parameters.get("top_k"),
"score_threshold": score_threshold,
},
"query": query,
"knowledge_id": external_knowledge_binding.external_knowledge_id,
"metadata_condition": metadata_condition.model_dump() if metadata_condition else None,
}
return ExternalKnowledgeApiSetting(
url=f"{settings.get('endpoint')}/retrieval",
request_method="post",
headers=headers,
params=request_params,
)
@staticmethod
def execute_external_knowledge_retrieval(request: ExternalKnowledgeApiSetting) -> list[Any]:
"""Execute and validate a prepared request without holding a database session."""
try:
response = ExternalDatasetService.process_external_api(
request,
None,
)
except ExternalKnowledgeRetrievalError:
raise
except Exception as exc:
raise ExternalKnowledgeRetrievalError(str(exc)) from exc
if response.status_code == 200:
try:
response_payload = response.json()
except Exception as exc:
raise ExternalKnowledgeRetrievalError("invalid external knowledge response") from exc
if not isinstance(response_payload, dict):
raise ExternalKnowledgeRetrievalError("invalid external knowledge response")
records = response_payload.get("records", [])
if not isinstance(records, list):
raise ExternalKnowledgeRetrievalError("invalid external knowledge response")
return cast(list[Any], records)
else:
raise ExternalKnowledgeRetrievalError(response.text)
@@ -0,0 +1 @@
"""Use-case-centered stored-document indexing slice."""
@@ -0,0 +1 @@
"""Infrastructure adapters for the knowledge indexing slice."""
@@ -0,0 +1,143 @@
"""Delete document indexes between bounded database transactions."""
from collections.abc import Callable, Mapping, Sequence
from sqlalchemy import delete, select
from sqlalchemy.orm import Session
from configs import dify_config
from core.rag.datasource.keyword.jieba.jieba import Jieba
from core.rag.datasource.vdb.vector_factory import Vector
from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType
from extensions.ext_redis import redis_client
from models.dataset import ChildChunk, Dataset, DocumentSegment, DocumentSegmentSummary
from repositories.knowledge.dataset_read_repository import get_dataset_keyword_table
from repositories.knowledge.keyword_table_repository import persist_keyword_table
def delete_index_entries(
dataset: Dataset,
*,
vector_node_ids: Sequence[str],
vector_type: str | None,
keyword_node_ids: Sequence[str],
new_session: Callable[[], Session],
) -> None:
"""Delete index entries by node id; the single owner of how each index type is removed.
High-quality datasets delete ``vector_node_ids`` from the vector store; other datasets
delete ``keyword_node_ids`` from the keyword table under the dataset keyword lock.
"""
if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
if vector_node_ids:
Vector(dataset, session=None, vector_type=vector_type).delete_by_ids(list(vector_node_ids))
return
if not keyword_node_ids:
return
# The keyword adapter owns its lock and storage I/O; callbacks only do SQL.
def read() -> tuple[str, str | None]:
with new_session() as session:
row = get_dataset_keyword_table(dataset, session=session)
return (row.data_source_type, row.keyword_table) if row else (dify_config.KEYWORD_DATA_SOURCE_TYPE, None)
def write(storage_type: str, data: str, keywords: Mapping[str, Sequence[str]]) -> None:
with new_session() as session, session.begin():
persist_keyword_table(
session,
tenant_id=dataset.tenant_id,
dataset_id=dataset.id,
storage_type=storage_type,
data=data,
keywords=keywords,
)
Jieba(dataset).update_texts(
[],
read=read,
write=write,
lock=redis_client.lock(f"keyword_indexing_lock_{dataset.id}", timeout=600),
deleted_ids=keyword_node_ids,
)
def clean_document_indexes(
*, dataset_id: str, document_ids: Sequence[str], doc_form: str | None, new_session: Callable[[], Session]
) -> str | None:
"""Clean indexes even when the owning document rows have already been deleted.
Snapshot the dataset-owned rows, close the read session, remove external
indexes, then delete the selected summary and child rows in one transaction.
A failed vector deletion leaves those rows available for a retry.
Empty document selections never mean deleting the entire dataset index.
"""
if not document_ids:
return None
with new_session() as session:
dataset = session.scalar(select(Dataset).where(Dataset.id == dataset_id))
if dataset is None:
return None
tenant_id = dataset.tenant_id
segments = session.scalars(
select(DocumentSegment).where(
DocumentSegment.tenant_id == tenant_id,
DocumentSegment.dataset_id == dataset_id,
DocumentSegment.document_id.in_(document_ids),
)
).all()
summaries = session.scalars(
select(DocumentSegmentSummary).where(
DocumentSegmentSummary.dataset_id == dataset_id,
DocumentSegmentSummary.document_id.in_(document_ids),
)
).all()
children = session.scalars(
select(ChildChunk).where(
ChildChunk.tenant_id == tenant_id,
ChildChunk.dataset_id == dataset_id,
ChildChunk.document_id.in_(document_ids),
)
).all()
summary_ids = [summary.id for summary in summaries]
child_ids = [child.id for child in children]
summary_node_ids = [summary.summary_index_node_id for summary in summaries if summary.summary_index_node_id]
body_node_ids = (
[child.index_node_id for child in children if child.index_node_id]
if doc_form == IndexStructureType.PARENT_CHILD_INDEX
else [segment.index_node_id for segment in segments if segment.index_node_id]
)
high_quality = dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY
node_ids = list(dict.fromkeys([*summary_node_ids, *body_node_ids]))
vector_type = Vector.resolve_vector_type(dataset, session=session) if high_quality and node_ids else None
session.expunge(dataset)
if not node_ids and not summary_ids and not child_ids:
return None
delete_index_entries(
dataset,
vector_node_ids=node_ids,
vector_type=vector_type,
keyword_node_ids=body_node_ids,
new_session=new_session,
)
with new_session() as session, session.begin():
if summary_ids:
session.execute(
delete(DocumentSegmentSummary).where(
DocumentSegmentSummary.dataset_id == dataset_id,
DocumentSegmentSummary.document_id.in_(document_ids),
DocumentSegmentSummary.id.in_(summary_ids),
)
)
if child_ids:
session.execute(
delete(ChildChunk).where(
ChildChunk.tenant_id == tenant_id,
ChildChunk.dataset_id == dataset_id,
ChildChunk.document_id.in_(document_ids),
ChildChunk.id.in_(child_ids),
)
)
return tenant_id
@@ -0,0 +1,190 @@
"""Infrastructure adapters for indexing estimates."""
import json
import logging
from collections.abc import Mapping
from typing import cast
from sqlalchemy import select
from sqlalchemy.orm import Session, sessionmaker
from configs import dify_config
from core.credit_usage import CreditUsageCreatedBy
from core.entities.knowledge_entities import IndexingEstimate, PreviewDetail, QAPreviewDetail
from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
from core.model_context import with_credit_usage_created_by
from core.model_manager import ModelInstance, ModelManager
from core.plugin.impl.exc import PluginDaemonClientSideError
from core.rag.extractor.entity.extract_setting import ExtractSetting
from core.rag.index_processor.constant.index_type import IndexTechniqueType
from core.rag.index_processor.index_processor import IndexProcessorFactory
from core.rag.index_processor.index_processor_base import SummaryIndexSettingDict
from core.tools.utils.web_reader_tool import get_image_upload_file_ids
from enums import DeploymentEdition
from extensions.ext_storage import storage
from graphon.model_runtime.entities.model_entities import ModelType
from models.dataset import Dataset, DatasetProcessRule
from repositories.knowledge.dataset_repository import _get_dataset
from repositories.knowledge.upload_file_repository import SQLAlchemyKnowledgeUploadRepository
from services.credentials.refresh import CredentialRefreshUnavailableError
from services.knowledge.indexing.estimate import (
IndexingEstimateExecutionError,
IndexingEstimateProviderUnavailableError,
)
from services.knowledge.resource_scope import DatasetRef
logger = logging.getLogger(__name__)
class SQLAlchemyProcessRuleReader:
"""Load one tenant-owned process rule in a bounded read session."""
def __init__(self, *, session_factory: sessionmaker[Session]) -> None:
self._session_factory = session_factory
def get_by_id(self, *, dataset_ref: DatasetRef, process_rule_id: str) -> Mapping[str, object] | None:
with self._session_factory() as session:
process_rule = session.scalar(
select(DatasetProcessRule)
.join(Dataset, Dataset.id == DatasetProcessRule.dataset_id)
.where(
DatasetProcessRule.id == process_rule_id,
DatasetProcessRule.dataset_id == dataset_ref.dataset_id,
Dataset.tenant_id == dataset_ref.tenant_id,
)
.limit(1)
)
if process_rule is None:
return None
rules = json.loads(process_rule.rules) if process_rule.rules else {}
return {"mode": str(process_rule.mode), "rules": rules}
class IndexingEstimateAdapter:
"""Extract previews without depending on the document indexing execution flow."""
def __init__(
self,
*,
session_factory: sessionmaker[Session],
) -> None:
self._session_factory = session_factory
self._uploads = SQLAlchemyKnowledgeUploadRepository(session_factory=session_factory)
@with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)
def run(
self,
tenant_id: str,
extract_settings: list[ExtractSetting],
tmp_processing_rule: Mapping[str, object],
doc_form: str | None = None,
doc_language: str = "English",
dataset_id: str | None = None,
indexing_technique: str = "economy",
) -> IndexingEstimate:
try:
return self._estimate(
tenant_id, extract_settings, tmp_processing_rule, doc_form, doc_language, dataset_id, indexing_technique
)
except (LLMBadRequestError, ProviderTokenNotInitError, PluginDaemonClientSideError) as error:
raise IndexingEstimateProviderUnavailableError(error.description) from error
except CredentialRefreshUnavailableError:
raise
except Exception as error:
raise IndexingEstimateExecutionError(str(error)) from error
def _estimate(
self,
tenant_id: str,
extract_settings: list[ExtractSetting],
processing_rule: Mapping[str, object],
doc_form: str | None,
doc_language: str,
dataset_id: str | None,
indexing_technique: str,
) -> IndexingEstimate:
if (
dify_config.DEPLOYMENT_EDITION == DeploymentEdition.CLOUD
and len(extract_settings) > dify_config.BATCH_UPLOAD_LIMIT
):
raise ValueError(f"You have reached the batch upload limit of {dify_config.BATCH_UPLOAD_LIMIT}.")
model = self._embedding_model(tenant_id, dataset_id, indexing_technique)
processor = IndexProcessorFactory(doc_form).init_index_processor()
previews: list[PreviewDetail] = []
qa_previews: list[QAPreviewDetail] = []
image_ids: set[str] = set()
total_segments = 0
# Legacy processors need a local session for extraction artifacts. No
# document-state transaction is shared with this preview operation.
with self._session_factory() as session:
for setting in extract_settings:
texts = processor.extract(setting, process_rule_mode=processing_rule["mode"], session=session)
documents = processor.transform(
texts,
current_user=None,
embedding_model_instance=model,
process_rule={"mode": processing_rule["mode"], "rules": processing_rule.get("rules")},
tenant_id=tenant_id,
doc_language=doc_language,
preview=True,
session=session,
)
total_segments += len(documents)
for document in documents:
if doc_form == "qa_model":
if len(qa_previews) < 10:
qa_previews.append(
QAPreviewDetail(
question=document.page_content, answer=document.metadata.get("answer") or ""
)
)
elif len(previews) < 10:
preview = PreviewDetail(content=document.page_content)
if document.children:
preview.child_chunks = [child.page_content for child in document.children]
previews.append(preview)
image_ids.update(get_image_upload_file_ids(document.page_content))
session.commit()
# Storage I/O takes place after extraction commits and outside the
# upload-record deletion transaction. Foreign tenant files are excluded.
uploads = self._uploads.get_files(workspace_id=tenant_id, file_ids=tuple(image_ids))
for upload in uploads.values():
try:
storage.delete(upload.key)
except Exception:
logger.exception("Delete image file failed while estimating indexing, upload_file_id: %s", upload.id)
self._uploads.delete_files(workspace_id=tenant_id, file_ids=tuple(uploads))
if doc_form == "qa_model":
return IndexingEstimate(total_segments=total_segments * 20, qa_preview=qa_previews, preview=[])
summary = processing_rule.get("summary_index_setting")
if isinstance(summary, Mapping) and summary.get("enable") and previews:
with self._session_factory() as session:
previews = processor.generate_summary_preview(
tenant_id, previews, cast(SummaryIndexSettingDict, summary), doc_language, session=session
)
return IndexingEstimate(total_segments=total_segments, preview=previews)
def _embedding_model(self, tenant_id: str, dataset_id: str | None, technique: str) -> ModelInstance | None:
provider = None
model_name = None
if dataset_id:
with self._session_factory() as session:
dataset = _get_dataset(session, DatasetRef(tenant_id, dataset_id))
if dataset is None:
raise ValueError("Dataset not found.")
if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
technique = IndexTechniqueType.HIGH_QUALITY
provider, model_name = dataset.embedding_model_provider, dataset.embedding_model
if technique != IndexTechniqueType.HIGH_QUALITY:
return None
manager = ModelManager.for_tenant(tenant_id=tenant_id)
if provider:
if not model_name:
raise ValueError("Embedding model is not configured.")
return manager.get_model_instance(
tenant_id=tenant_id, provider=provider, model_type=ModelType.TEXT_EMBEDDING, model=model_name
)
return manager.get_default_model_instance(tenant_id=tenant_id, model_type=ModelType.TEXT_EMBEDDING)
@@ -0,0 +1,280 @@
"""Adapt legacy processors to committed document-indexing phases."""
from concurrent.futures import ThreadPoolExecutor
from typing import cast
from flask import Flask, current_app
from sqlalchemy.orm import Session, sessionmaker
from werkzeug.exceptions import HTTPException
from werkzeug.local import LocalProxy
from core.errors.error import (
AppInvokeQuotaExceededError,
InvokeRateLimitError,
LLMError,
ModelCurrentlyNotSupportError,
ProviderTokenNotInitError,
QuotaExceededError,
)
from core.model_manager import ModelManager
from core.plugin.impl.exc import PluginDaemonError
from core.rag.datasource.keyword.jieba.jieba import Jieba
from core.rag.datasource.vdb.vector_factory import Vector
from core.rag.embedding.token_counter import calculate_segment_token_counts
from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType
from core.rag.index_processor.index_processor import IndexProcessorFactory
from core.rag.models.document import Document
from extensions.application_services.data_sources import build_data_source_credentials
from extensions.ext_redis import redis_client
from extensions.otel import propagate_context
from graphon.model_runtime.entities.model_entities import ModelType
from libs import helper
from models.enums import DataSourceType
from repositories.knowledge.document_repository import SQLAlchemyDocumentRepository
from repositories.knowledge.segment_repository import SQLAlchemySegmentRepository
from repositories.knowledge.upload_file_repository import SQLAlchemyKnowledgeUploadRepository
from services.knowledge.indexing.adapters.cleanup import delete_index_entries
from services.knowledge.indexing.adapters.sources import (
CompositeStoredSourceResolver,
FileSourceAdapter,
NotionSourceResolver,
WebsiteSourceAdapter,
)
from services.knowledge.indexing.errors import DocumentIsPausedError
from services.knowledge.indexing.execution import DocumentIndexingService, IndexingDocument
from services.knowledge.resource_scope import DocumentRef
from services.vector_space_admission_service import VectorSpaceAdmissionService
class IndexingExecutionAdapter:
def __init__(
self,
*,
session_factory: sessionmaker[Session],
documents: SQLAlchemyDocumentRepository,
segments: SQLAlchemySegmentRepository,
uploads: SQLAlchemyKnowledgeUploadRepository,
sources: CompositeStoredSourceResolver,
) -> None:
self._session_factory = session_factory
self._documents = documents
self._segments = segments
self._uploads = uploads
self._sources = sources
def describe_error(self, error: Exception) -> str:
"""Preserve provider and extractor descriptions without their transport prefixes."""
if isinstance(
error,
(
LLMError,
ProviderTokenNotInitError,
QuotaExceededError,
AppInvokeQuotaExceededError,
ModelCurrentlyNotSupportError,
InvokeRateLimitError,
PluginDaemonError,
HTTPException,
),
):
return str(error.description)
return str(error)
def check_paused(self, ref: DocumentRef) -> None:
if redis_client.get(f"document_{ref.document_id}_is_paused"):
raise DocumentIsPausedError(f"Document paused, document id: {ref.document_id}")
def extract(self, document: IndexingDocument) -> list[Document]:
if not document.processing_rule:
raise ValueError("no process rule found")
setting = self._sources.resolve_for_indexing(document.source)
processor = IndexProcessorFactory(document.doc_form).init_index_processor()
# Legacy extractors persist downloaded images and Notion source metadata.
# Their session is independent of document-state write transactions.
with self._session_factory() as session:
texts = processor.extract(setting, process_rule_mode=document.processing_rule["mode"], session=session)
session.commit()
for text in texts:
text.metadata.update(document_id=document.ref.document_id, dataset_id=document.ref.dataset.dataset_id)
return texts
def transform(self, document: IndexingDocument, texts: list[Document]) -> list[Document]:
dataset, _ = self._documents.get_indexing_models(document.ref)
user = self._documents.get_indexing_user(document.ref)
model = None
if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
manager = ModelManager.for_tenant(tenant_id=dataset.tenant_id)
if dataset.embedding_model_provider:
model = manager.get_model_instance(
tenant_id=dataset.tenant_id,
provider=dataset.embedding_model_provider,
model_type=ModelType.TEXT_EMBEDDING,
model=dataset.embedding_model,
)
else:
model = manager.get_default_model_instance(
tenant_id=dataset.tenant_id, model_type=ModelType.TEXT_EMBEDDING
)
processor = IndexProcessorFactory(document.doc_form).init_index_processor()
with self._session_factory() as session:
chunks = processor.transform(
texts,
user,
embedding_model_instance=model,
process_rule=document.processing_rule,
tenant_id=dataset.tenant_id,
doc_language=document.doc_language,
session=session,
)
session.commit()
return chunks
def count_tokens(self, document: IndexingDocument, chunks: list[Document]) -> list[int]:
dataset, _ = self._documents.get_indexing_models(document.ref)
return calculate_segment_token_counts(dataset=dataset, documents=chunks)
def ensure_admission(self, document: IndexingDocument, chunks: list[Document]) -> None:
dataset, _ = self._documents.get_indexing_models(document.ref)
with self._session_factory() as session:
VectorSpaceAdmissionService().ensure_document_can_be_indexed(
dataset=dataset,
document_id=document.ref.document_id,
doc_form=document.doc_form,
documents=chunks,
include_summaries=document.need_summary,
session=session,
)
def load(self, document: IndexingDocument, chunks: list[Document]) -> None:
dataset, _ = self._documents.get_indexing_models(document.ref)
app = cast(LocalProxy[Flask], current_app)._get_current_object()
if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
groups: list[list[Document]] = [[] for _ in range(10)]
for chunk in chunks:
groups[int(helper.generate_text_hash(chunk.page_content), 16) % 10].append(chunk)
with ThreadPoolExecutor(max_workers=10) as executor:
futures = [
executor.submit(
propagate_context(self._process_chunk),
app,
document.ref,
group,
False,
)
for group in groups
if group
]
for future in futures:
future.result()
elif (
dataset.indexing_technique == IndexTechniqueType.ECONOMY
and document.doc_form != IndexStructureType.PARENT_CHILD_INDEX
):
# A future propagates keyword failures; Thread.join() used to hide
# them and incorrectly let the document become completed.
with ThreadPoolExecutor(max_workers=1) as executor:
executor.submit(propagate_context(self._process_chunk), app, document.ref, chunks, True).result()
def discard(self, document: IndexingDocument, chunks: list[Document]) -> None:
"""Remove the index entries ``load`` writes for ``chunks`` through ``delete_index_entries``.
Mirrors ``_process_chunk``: child vectors for parent-child documents, body vectors
otherwise, and body keywords for economy datasets. Some ids may never have been
written; each store handles that as it does when document cleanup issues the same call.
"""
dataset = self._documents.get_indexing_dataset(document.ref.dataset)
if dataset is None:
return
if document.doc_form == IndexStructureType.PARENT_CHILD_INDEX:
node_ids = [child.metadata["doc_id"] for chunk in chunks for child in chunk.children or []]
else:
node_ids = [chunk.metadata["doc_id"] for chunk in chunks]
if not node_ids:
return
vector_type = None
if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
with self._session_factory() as session:
vector_type = Vector.resolve_vector_type(dataset, session=session)
writes_keywords = (
dataset.indexing_technique == IndexTechniqueType.ECONOMY
and document.doc_form != IndexStructureType.PARENT_CHILD_INDEX
)
delete_index_entries(
dataset,
vector_node_ids=node_ids,
vector_type=vector_type,
keyword_node_ids=node_ids if writes_keywords else [],
new_session=self._session_factory,
)
def _process_chunk(self, app: Flask, ref: DocumentRef, chunks: list[Document], keywords: bool) -> None:
with app.app_context():
self.check_paused(ref)
# Workers load their own inputs and own their sessions. No ORM
# instance or session crosses the executor boundary.
dataset, document = self._documents.get_indexing_models(ref)
if keywords:
# Keep the read/modify/write lock while each repository call
# closes its transaction before keyword extraction or storage I/O.
Jieba(dataset).update_texts(
chunks,
read=lambda: self._segments.get_keyword_table(ref.dataset),
write=lambda storage_type, data, selected: self._segments.save_keyword_table(
ref.dataset, storage_type=storage_type, data=data, keywords=selected
),
lock=redis_client.lock(f"keyword_indexing_lock_{dataset.id}", timeout=600),
)
elif dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
with self._session_factory() as session:
vector_type = Vector.resolve_vector_type(dataset, session=session)
attachments = (
[attachment for chunk in chunks for attachment in chunk.attachments or []]
if dataset.is_multimodal
else []
)
files = self._uploads.get_files(
workspace_id=ref.dataset.tenant_id,
file_ids=[attachment.metadata["doc_id"] for attachment in attachments],
)
# All database inputs are materialized before initializing the
# vector backend, embedding text/images, or reading object storage.
vector = Vector(dataset, session=None, vector_type=vector_type)
if document.doc_form == IndexStructureType.PARENT_CHILD_INDEX:
for chunk in chunks:
if chunk.children:
vector.create([Document.model_validate(child.model_dump()) for child in chunk.children])
else:
vector.create(chunks)
if attachments:
vector.create_multimodal(attachments, upload_files=files)
self.check_paused(ref)
self._segments.complete_indexing_segments(ref, [chunk.metadata["doc_id"] for chunk in chunks])
def build_document_indexing_service(
*,
session_factory: sessionmaker[Session],
enforce_vector_space_admission: bool = False,
) -> DocumentIndexingService:
"""Assemble a fresh indexing use case from explicit database dependencies."""
documents = SQLAlchemyDocumentRepository(session_factory=session_factory)
segments = SQLAlchemySegmentRepository(session_factory=session_factory)
uploads = SQLAlchemyKnowledgeUploadRepository(session_factory=session_factory)
credentials = build_data_source_credentials(database_client=session_factory)
sources = CompositeStoredSourceResolver(
adapters={
DataSourceType.UPLOAD_FILE: FileSourceAdapter(uploads=uploads),
DataSourceType.NOTION_IMPORT: NotionSourceResolver(
actor_credentials=credentials.actor, stored_credentials=credentials.stored
),
DataSourceType.WEBSITE_CRAWL: WebsiteSourceAdapter(),
}
)
return DocumentIndexingService(
documents=documents,
segments=segments,
backend=IndexingExecutionAdapter(
session_factory=session_factory, documents=documents, segments=segments, uploads=uploads, sources=sources
),
enforce_vector_space_admission=enforce_vector_space_admission,
)
@@ -0,0 +1,282 @@
"""Datasource-specific extraction input resolvers."""
from collections.abc import Mapping, Sequence
from typing import Protocol
from configs import dify_config
from core.rag.extractor.entity.datasource_type import DatasourceType
from core.rag.extractor.entity.extract_setting import (
ExtractSetting,
NotionInfo,
StoredDocumentExtractionInput,
UploadFileExtractionInput,
WebsiteInfo,
)
from services.data_source.credential_gateway import (
ActorDatasourceCredentialResolver,
NotionCredentialUnavailableError,
StoredDatasourceCredentialResolver,
resolve_stored_notion_access_token,
)
from services.data_source.entities.notion_import import NotionPageType
from services.knowledge.indexing.errors import (
IndexingInputSourceError,
SourceCredentialUnavailableError,
UnsupportedStoredSourceError,
)
from services.knowledge.indexing.estimate import StoredSource
_NOTION_PROVIDER = "notion_datasource"
_NOTION_PLUGIN = "langgenius/notion_datasource"
class StoredSourceAdapter(Protocol):
"""Resolve one persisted datasource kind."""
def resolve(self, source: StoredSource) -> ExtractSetting: ...
class KnowledgeUploadCatalog(Protocol):
def get_by_ids(self, *, workspace_id: str, file_ids: Sequence[str]) -> Mapping[str, UploadFileExtractionInput]: ...
class FileSourceAdapter:
def __init__(self, *, uploads: KnowledgeUploadCatalog) -> None:
self._uploads = uploads
def resolve(self, source: StoredSource) -> ExtractSetting:
info = _required_info(source, "no upload file found")
file_id = _required_string(info, "upload_file_id", "no upload file found")
return self.resolve_selection(
workspace_id=source.document_ref.dataset.tenant_id,
file_id=file_id,
document_model=source.document_model,
)
def resolve_selection(
self,
*,
workspace_id: str,
file_id: str,
document_model: str,
) -> ExtractSetting:
return self.resolve_selections(workspace_id=workspace_id, file_ids=(file_id,), document_model=document_model)[
file_id
]
def resolve_selections(
self, *, workspace_id: str, file_ids: Sequence[str], document_model: str
) -> dict[str, ExtractSetting]:
if not file_ids:
return {}
uploads = self._uploads.get_by_ids(workspace_id=workspace_id, file_ids=file_ids)
settings: dict[str, ExtractSetting] = {}
for file_id in file_ids:
upload = uploads.get(file_id)
if upload is None:
raise IndexingInputSourceError(f"no upload file found: {file_id}")
settings[file_id] = ExtractSetting(
datasource_type=DatasourceType.FILE, upload_file=upload, document_model=document_model
)
return settings
class WebsiteSourceAdapter:
def resolve(self, source: StoredSource) -> ExtractSetting:
info = _required_info(source, "no website crawl info found")
return self.resolve_selection(
workspace_id=source.document_ref.dataset.tenant_id,
provider=_required_string(info, "provider"),
job_id=_required_string(info, "job_id"),
url=_required_string(info, "url"),
mode=_required_string(info, "mode"),
only_main_content=bool(info.get("only_main_content", False)),
document_model=source.document_model,
)
def resolve_selection(
self,
*,
workspace_id: str,
provider: str,
job_id: str,
url: str,
mode: str,
only_main_content: bool,
document_model: str,
) -> ExtractSetting:
return ExtractSetting(
datasource_type=DatasourceType.WEBSITE,
website_info=WebsiteInfo(
provider=provider,
job_id=job_id,
url=url,
mode=mode,
tenant_id=workspace_id,
only_main_content=only_main_content,
),
document_model=document_model,
)
class NotionSourceResolver:
"""Resolve actor-selected and trusted-stored Notion extraction inputs."""
def __init__(
self,
*,
actor_credentials: ActorDatasourceCredentialResolver,
stored_credentials: StoredDatasourceCredentialResolver,
integration_token: str | None = None,
) -> None:
self._actor_credentials = actor_credentials
self._stored_credentials = stored_credentials
self._integration_token = integration_token
def resolve(self, source: StoredSource) -> ExtractSetting:
info = _required_info(source, "no notion import info found")
document_ref = source.document_ref
credential_id = _string_value(info, "credential_id")
token = self._stored_token(source, credential_id)
return self._prepared(
tenant_id=document_ref.dataset.tenant_id,
notion_workspace_id=_required_string(info, "notion_workspace_id"),
page_id=_required_string(info, "notion_page_id"),
page_type=_notion_page_type(info),
document_model=source.document_model,
credential_id=credential_id,
token=token,
)
def _stored_token(self, source: StoredSource, credential_id: str | None) -> str:
document_ref = source.document_ref
def load_token(saved_id: str) -> str:
credentials = self._stored_credentials.resolve_for_document(
workspace_id=document_ref.dataset.tenant_id,
dataset_id=document_ref.dataset.dataset_id,
document_id=document_ref.document_id,
credential_id=saved_id,
provider=_NOTION_PROVIDER,
plugin_id=_NOTION_PLUGIN,
)
return self._required_token(credentials)
try:
return resolve_stored_notion_access_token(
credential_id=credential_id,
load_token=load_token,
integration_token=self._integration_token or dify_config.NOTION_INTEGRATION_TOKEN,
)
except NotionCredentialUnavailableError as error:
raise SourceCredentialUnavailableError("Notion credential is unavailable") from error
def resolve_selection(
self,
*,
workspace_id: str,
actor_id: str,
credential_id: str,
notion_workspace_id: str,
page_id: str,
page_type: NotionPageType,
document_model: str,
) -> ExtractSetting:
credentials = self._actor_credentials.resolve(
workspace_id=workspace_id,
actor_id=actor_id,
credential_id=credential_id,
provider=_NOTION_PROVIDER,
plugin_id=_NOTION_PLUGIN,
)
return self._prepared(
tenant_id=workspace_id,
notion_workspace_id=notion_workspace_id,
page_id=page_id,
page_type=page_type,
document_model=document_model,
credential_id=credential_id,
token=self._required_token(credentials),
)
def _prepared(
self,
*,
tenant_id: str,
notion_workspace_id: str,
page_id: str,
page_type: NotionPageType,
document_model: str,
credential_id: str | None,
token: str,
) -> ExtractSetting:
return ExtractSetting(
datasource_type=DatasourceType.NOTION,
notion_info=NotionInfo(
credential_id=credential_id,
notion_workspace_id=notion_workspace_id,
notion_obj_id=page_id,
notion_page_type=page_type.value,
tenant_id=tenant_id,
notion_access_token=token,
),
document_model=document_model,
)
@staticmethod
def _required_token(credentials: Mapping[str, object]) -> str:
token = credentials.get("integration_secret")
if isinstance(token, str) and token:
return token
raise SourceCredentialUnavailableError("Notion credential is unavailable")
class CompositeStoredSourceResolver:
"""Route persisted sources to datasource-specific adapters."""
def __init__(self, *, adapters: Mapping[str, StoredSourceAdapter]) -> None:
self._adapters = dict(adapters)
def resolve(self, source: StoredSource) -> ExtractSetting:
adapter = self._adapters.get(source.source_type)
if adapter is None:
raise UnsupportedStoredSourceError(source.source_type)
return adapter.resolve(source)
def resolve_for_indexing(self, source: StoredSource) -> ExtractSetting:
setting = self.resolve(source)
if setting.notion_info is not None:
ref = source.document_ref
setting.notion_info.document = StoredDocumentExtractionInput(
id=ref.document_id,
tenant_id=ref.dataset.tenant_id,
dataset_id=ref.dataset.dataset_id,
data_source_info_dict=dict(source.source_info) if source.source_info is not None else None,
)
return setting
def _required_info(source: StoredSource, message: str) -> Mapping[str, object]:
if source.source_info is None:
raise IndexingInputSourceError(message)
return source.source_info
def _string_value(info: Mapping[str, object], key: str) -> str | None:
value = info.get(key)
return value if isinstance(value, str) and value else None
def _required_string(info: Mapping[str, object], key: str, message: str | None = None) -> str:
value = _string_value(info, key)
if value is None:
raise IndexingInputSourceError(message or f"invalid data source field: {key}")
return value
def _notion_page_type(info: Mapping[str, object]) -> NotionPageType:
value = _required_string(info, "type")
try:
return NotionPageType(value)
except ValueError as error:
raise IndexingInputSourceError("invalid data source field: type") from error
@@ -0,0 +1,27 @@
"""Application failures raised while preparing or executing indexing."""
class IndexingInputError(Exception):
"""Base class for failures while resolving stored extraction inputs."""
class IndexingInputSourceError(IndexingInputError):
"""Raised when a persisted source cannot form a valid extraction input."""
class UnsupportedStoredSourceError(IndexingInputSourceError):
def __init__(self, source_type: str) -> None:
self.source_type = source_type
super().__init__(f"unsupported data source type: {source_type}")
class SourceCredentialUnavailableError(IndexingInputSourceError):
"""Raised when a datasource cannot resolve its required credential policy."""
class DocumentIsPausedError(Exception):
"""Indexing stopped because the owned document was paused."""
class DocumentIsDeletedPausedError(Exception):
"""Indexing stopped because its document owner chain no longer exists."""
@@ -0,0 +1,346 @@
"""Indexing-estimate application service and its consumer-owned ports."""
from collections.abc import Mapping, Sequence
from typing import NamedTuple, Never, Protocol
from core.entities.knowledge_entities import IndexingEstimate
from core.rag.extractor.entity.extract_setting import ExtractSetting
from machinery.context import RequestContext
from services.credentials.refresh import CredentialRefreshUnavailableError
from services.data_source.credential_gateway import DatasourceCredentialError
from services.data_source.entities.notion_import import NotionPageType
from services.knowledge.dataset_access import DatasetAccess, DatasetNotFoundError
from services.knowledge.entities.indexing_estimate import (
NewEstimateSource,
NewSourcesEstimateCommand,
NotionEstimateSource,
UploadFileEstimateSource,
WebsiteEstimateSource,
normalize_process_rule,
)
from services.knowledge.indexing.errors import (
IndexingInputSourceError,
SourceCredentialUnavailableError,
UnsupportedStoredSourceError,
)
from services.knowledge.resource_scope import DatasetRef, DocumentRef
class DatasetEstimateRecord(NamedTuple):
"""Dataset state required to estimate stored documents."""
id: str
workspace_id: str
indexing_technique: str | None
class EstimateDocumentRecord(NamedTuple):
"""Document state required by the stored-document estimate use case."""
id: str
workspace_id: str
dataset_id: str
data_source_type: str
data_source_info: Mapping[str, object] | None
indexing_status: str
doc_form: str
doc_language: str | None
dataset_process_rule_id: str | None
@property
def ref(self) -> DocumentRef:
return DatasetRef(self.workspace_id, self.dataset_id).document(self.id)
class StoredSource(NamedTuple):
"""A persisted extraction source bound to its owning document."""
document_ref: DocumentRef
source_type: str
source_info: Mapping[str, object] | None
document_model: str
@classmethod
def from_document(cls, document: EstimateDocumentRecord) -> "StoredSource":
return cls(
document_ref=document.ref,
source_type=document.data_source_type,
source_info=document.data_source_info,
document_model=document.doc_form,
)
class EstimateDocumentReader(Protocol):
def get_estimate_document(self, document_ref: DocumentRef) -> EstimateDocumentRecord | None: ...
def list_estimate_documents_by_batch(
self, dataset_ref: DatasetRef, batch: str
) -> tuple[EstimateDocumentRecord, ...]: ...
class EstimateDatasetReader(Protocol):
def get_estimate_record(self, dataset_ref: DatasetRef) -> DatasetEstimateRecord | None: ...
class UploadEstimateSourceResolver(Protocol):
def resolve_selections(
self, *, workspace_id: str, file_ids: Sequence[str], document_model: str
) -> Mapping[str, ExtractSetting]: ...
class NotionEstimateSourceResolver(Protocol):
def resolve_selection(
self,
*,
workspace_id: str,
actor_id: str,
credential_id: str,
notion_workspace_id: str,
page_id: str,
page_type: NotionPageType,
document_model: str,
) -> ExtractSetting: ...
class WebsiteEstimateSourceResolver(Protocol):
def resolve_selection(
self,
*,
workspace_id: str,
provider: str,
job_id: str,
url: str,
mode: str,
only_main_content: bool,
document_model: str,
) -> ExtractSetting: ...
class StoredEstimateSourceResolver(Protocol):
def resolve(self, source: StoredSource) -> ExtractSetting: ...
class ProcessRuleReader(Protocol):
def get_by_id(self, *, dataset_ref: DatasetRef, process_rule_id: str) -> Mapping[str, object] | None: ...
class IndexingEstimateRunner(Protocol):
def run(
self,
tenant_id: str,
extract_settings: list[ExtractSetting],
tmp_processing_rule: Mapping[str, object],
doc_form: str | None = None,
doc_language: str = "English",
dataset_id: str | None = None,
indexing_technique: str = "economy",
) -> IndexingEstimate: ...
class IndexingEstimateApplicationError(Exception):
"""Base class for framework-neutral estimate failures."""
class EstimateSourceNotFoundError(IndexingEstimateApplicationError):
def __init__(self, source_id: str) -> None:
super().__init__(f"Estimate source not found: {source_id}")
class UnsupportedEstimateSourceError(IndexingEstimateApplicationError):
def __init__(self, source_type: str) -> None:
super().__init__(f"Data source type not supported: {source_type}")
class IndexingEstimateCredentialUnavailableError(IndexingEstimateApplicationError):
"""Raised when a datasource credential required by an estimate is unavailable."""
class IndexingEstimateProviderUnavailableError(IndexingEstimateApplicationError):
"""Raised when a configured model or plugin provider is unavailable."""
class IndexingEstimateExecutionError(IndexingEstimateApplicationError):
"""Raised when the estimate processor cannot complete an estimate."""
class EstimateDocumentNotFoundError(IndexingEstimateApplicationError):
def __init__(self) -> None:
super().__init__("Document not found")
class EstimateDocumentAlreadyFinishedError(IndexingEstimateApplicationError):
def __init__(self) -> None:
super().__init__("Document already finished")
class IndexingEstimateApplicationService:
def __init__(
self,
*,
dataset_access: DatasetAccess,
datasets: EstimateDatasetReader,
documents: EstimateDocumentReader,
files: UploadEstimateSourceResolver,
notion: NotionEstimateSourceResolver,
websites: WebsiteEstimateSourceResolver,
stored_sources: StoredEstimateSourceResolver,
process_rules: ProcessRuleReader,
runner: IndexingEstimateRunner,
) -> None:
self._dataset_access = dataset_access
self._datasets = datasets
self._documents = documents
self._files = files
self._notion = notion
self._websites = websites
self._stored_sources = stored_sources
self._process_rules = process_rules
self._runner = runner
def estimate_new_sources(
self,
context: RequestContext,
command: NewSourcesEstimateCommand,
) -> IndexingEstimate:
dataset_id = command.dataset_id or None
if dataset_id is not None:
dataset = self._dataset_access.require_accessible(context, dataset_id)
dataset_id = dataset.id
process_rule = normalize_process_rule(command.process_rule)
try:
file_settings = self._files.resolve_selections(
workspace_id=context.active_workspace_id,
file_ids=tuple(
dict.fromkeys(
source.file_id for source in command.sources if isinstance(source, UploadFileEstimateSource)
)
),
document_model=command.doc_form,
)
extract_settings = [
file_settings[source.file_id]
if isinstance(source, UploadFileEstimateSource)
else self._resolve_new_source(context, source, document_model=command.doc_form)
for source in command.sources
]
if not extract_settings:
return IndexingEstimate(total_segments=0, preview=[])
return self._runner.run(
tenant_id=context.active_workspace_id,
extract_settings=extract_settings,
tmp_processing_rule=process_rule,
doc_form=command.doc_form,
doc_language=command.doc_language,
dataset_id=dataset_id,
indexing_technique=command.indexing_technique,
)
except Exception as error:
self._translate_error(error)
def estimate_document(
self,
context: RequestContext,
*,
dataset_id: str,
document_id: str,
) -> IndexingEstimate:
accessible = self._dataset_access.require_accessible(context, dataset_id)
dataset_ref = DatasetRef(accessible.workspace_id, accessible.id)
dataset = self._require_estimate_dataset(dataset_ref)
document = self._documents.get_estimate_document(dataset_ref.document(document_id))
if document is None:
raise EstimateDocumentNotFoundError()
return self._estimate_document(context, dataset, document)
def estimate_batch(self, context: RequestContext, *, dataset_id: str, batch: str) -> IndexingEstimate:
accessible = self._dataset_access.require_accessible(context, dataset_id)
dataset_ref = DatasetRef(accessible.workspace_id, accessible.id)
dataset = self._require_estimate_dataset(dataset_ref)
documents = self._documents.list_estimate_documents_by_batch(dataset_ref, batch)
if not documents:
raise EstimateDocumentNotFoundError()
# A batch preview uses its first document, as the existing API does.
return self._estimate_document(context, dataset, documents[0])
def _estimate_document(
self,
context: RequestContext,
dataset: DatasetEstimateRecord,
document: EstimateDocumentRecord,
) -> IndexingEstimate:
if document.indexing_status in {"completed", "error"}:
raise EstimateDocumentAlreadyFinishedError()
try:
extract_setting = self._stored_sources.resolve(StoredSource.from_document(document))
process_rule = self._load_process_rule(dataset, document)
return self._runner.run(
tenant_id=context.active_workspace_id,
extract_settings=[extract_setting],
tmp_processing_rule=process_rule,
doc_form=document.doc_form,
doc_language=document.doc_language or "English",
dataset_id=dataset.id,
indexing_technique=dataset.indexing_technique or "economy",
)
except Exception as error:
self._translate_error(error)
def _resolve_new_source(
self, context: RequestContext, source: NewEstimateSource, *, document_model: str
) -> ExtractSetting:
if isinstance(source, NotionEstimateSource):
return self._notion.resolve_selection(
workspace_id=context.active_workspace_id,
actor_id=context.account_id,
credential_id=source.credential_id,
notion_workspace_id=source.workspace_id,
page_id=source.page_id,
page_type=source.page_type,
document_model=document_model,
)
if isinstance(source, WebsiteEstimateSource):
return self._websites.resolve_selection(
workspace_id=context.active_workspace_id,
provider=source.provider,
job_id=source.job_id,
url=source.url,
mode=source.mode,
only_main_content=source.only_main_content,
document_model=document_model,
)
raise UnsupportedEstimateSourceError(source.source_type)
def _load_process_rule(
self,
dataset: DatasetEstimateRecord,
document: EstimateDocumentRecord,
) -> Mapping[str, object]:
process_rule_id = document.dataset_process_rule_id
if not process_rule_id:
return {"mode": "automatic", "rules": {}}
process_rule = self._process_rules.get_by_id(
dataset_ref=DatasetRef(dataset.workspace_id, dataset.id),
process_rule_id=process_rule_id,
)
if process_rule is None:
raise EstimateSourceNotFoundError(process_rule_id)
return process_rule
@staticmethod
def _translate_error(error: Exception) -> Never:
if isinstance(error, (IndexingEstimateApplicationError, CredentialRefreshUnavailableError)):
raise error
if isinstance(error, UnsupportedStoredSourceError):
raise UnsupportedEstimateSourceError(error.source_type) from error
if isinstance(error, (DatasourceCredentialError, SourceCredentialUnavailableError)):
raise IndexingEstimateCredentialUnavailableError(str(error)) from error
if isinstance(error, IndexingInputSourceError):
raise EstimateSourceNotFoundError(str(error)) from error
raise IndexingEstimateExecutionError(str(error)) from error
def _require_estimate_dataset(self, dataset_ref: DatasetRef) -> DatasetEstimateRecord:
dataset = self._datasets.get_estimate_record(dataset_ref)
if dataset is None:
raise DatasetNotFoundError()
return dataset
@@ -0,0 +1,178 @@
"""Document indexing phases and their application-owned persistence and execution ports."""
import logging
import time
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass
from typing import Literal, Protocol
from core.credit_usage import CreditUsageCreatedBy
from core.model_context import with_credit_usage_created_by
from core.rag.models.document import Document
from services.knowledge.indexing.errors import DocumentIsDeletedPausedError, DocumentIsPausedError
from services.knowledge.indexing.estimate import StoredSource
from services.knowledge.resource_scope import DocumentRef
logger = logging.getLogger(__name__)
@dataclass(frozen=True)
class IndexingDocument:
source: StoredSource
processing_rule: Mapping[str, object]
doc_language: str
need_summary: bool
@property
def ref(self) -> DocumentRef:
return self.source.document_ref
@property
def doc_form(self) -> str:
return self.source.document_model
class IndexingDocumentStore(Protocol):
def get_indexing_document(self, ref: DocumentRef) -> IndexingDocument | None: ...
def mark_splitting(self, ref: DocumentRef) -> None: ...
def complete_indexing(self, ref: DocumentRef, *, tokens: int, latency: float) -> None: ...
def fail_indexing(self, ref: DocumentRef, error: str) -> None: ...
class IndexingSegmentStore(Protocol):
def clear_for_indexing(self, document: IndexingDocument) -> None: ...
def save_for_indexing(
self, document: IndexingDocument, chunks: list[Document], token_counts: list[int]
) -> None: ...
def resume_indexing(self, document: IndexingDocument) -> tuple[list[Document], int]: ...
class IndexingBackend(Protocol):
def describe_error(self, error: Exception) -> str:
"""Translate backend-specific exceptions into the persisted error message."""
...
def check_paused(self, ref: DocumentRef) -> None: ...
def extract(self, document: IndexingDocument) -> list[Document]: ...
def transform(self, document: IndexingDocument, texts: list[Document]) -> list[Document]: ...
def ensure_admission(self, document: IndexingDocument, chunks: list[Document]) -> None: ...
def count_tokens(self, document: IndexingDocument, chunks: list[Document]) -> list[int]: ...
def load(self, document: IndexingDocument, chunks: list[Document]) -> None: ...
def discard(self, document: IndexingDocument, chunks: list[Document]) -> None:
"""Remove index entries that ``load`` may have written for a document deleted meanwhile."""
...
class DocumentIndexingService:
"""Coordinate committed phases; no caller session is held across execution."""
def __init__(
self,
*,
documents: IndexingDocumentStore,
segments: IndexingSegmentStore,
backend: IndexingBackend,
enforce_vector_space_admission: bool = False,
clock: Callable[[], float] = time.perf_counter,
) -> None:
self._documents = documents
self._segments = segments
self._backend = backend
self._enforce_vector_space_admission = enforce_vector_space_admission
self._clock = clock
@with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)
def run(self, document_refs: Sequence[DocumentRef]) -> None:
for ref in document_refs:
self._run(ref, start="parsing")
@with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)
def run_in_splitting_status(self, ref: DocumentRef) -> None:
self._run(ref, start="splitting")
@with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)
def run_in_indexing_status(self, ref: DocumentRef) -> None:
self._run(ref, start="indexing")
def _run(self, ref: DocumentRef, *, start: Literal["parsing", "splitting", "indexing"]) -> None:
# None: no index write has started, so a deletion leaves nothing behind.
# Set: load may have written these chunks; document cleanup snapshots segment
# node ids before deleting them and cannot see entries written after its snapshot,
# so whichever way this run stops, the entries are discarded if the document is gone.
loading: tuple[IndexingDocument, list[Document]] | None = None
try:
document = self._documents.get_indexing_document(ref)
if document is None:
logger.info("Document not found, skipping indexing: %s", ref)
return
self._backend.check_paused(ref)
if start == "indexing":
chunks, total_tokens = self._segments.resume_indexing(document)
else:
if start == "splitting":
self._segments.clear_for_indexing(document)
texts = self._backend.extract(document)
self._documents.mark_splitting(ref)
chunks = self._backend.transform(document, texts)
# Recovery retains its existing admission policy; only normal
# indexing callers that explicitly opt in reserve vector space.
if start == "parsing" and self._enforce_vector_space_admission:
self._backend.ensure_admission(document, chunks)
token_counts = self._backend.count_tokens(document, chunks)
total_tokens = sum(token_counts)
self._segments.save_for_indexing(document, chunks, token_counts)
started = self._clock()
self._backend.check_paused(ref)
loading = (document, chunks)
self._backend.load(document, chunks)
self._backend.check_paused(ref)
self._documents.complete_indexing(ref, tokens=total_tokens, latency=self._clock() - started)
except DocumentIsPausedError:
# The pause flag is checked before document existence, so a paused document
# deleted mid-write surfaces here rather than as a deletion.
self._discard_if_deleted(ref, loading)
raise
except DocumentIsDeletedPausedError:
logger.info("Document deleted during indexing: %s", ref)
self._discard(ref, loading)
except Exception as error:
logger.exception("Document indexing failed: %s", ref)
self._documents.fail_indexing(ref, self._backend.describe_error(error))
self._discard_if_deleted(ref, loading)
def _discard(self, ref: DocumentRef, loading: tuple[IndexingDocument, list[Document]] | None) -> None:
"""Best-effort removal of entries ``load`` may have written; failures never mask the outcome."""
if loading is None:
return
try:
self._backend.discard(*loading)
except Exception:
logger.exception("Failed to discard index entries of document deleted during indexing: %s", ref)
def _discard_if_deleted(self, ref: DocumentRef, loading: tuple[IndexingDocument, list[Document]] | None) -> None:
"""Discard written entries when indexing stopped for another reason and the document is gone."""
if loading is None:
return
try:
self._documents.get_indexing_document(ref)
except DocumentIsDeletedPausedError:
logger.info("Document deleted during indexing: %s", ref)
self._discard(ref, loading)
except DocumentIsPausedError:
return
except Exception:
logger.warning(
"Could not check whether document still exists after indexing stopped: %s", ref, exc_info=True
)
Whitespace-only changes.
@@ -0,0 +1,125 @@
"""Metadata use cases with explicit ownership and persistence contracts."""
from collections.abc import Sequence
from dataclasses import dataclass
from datetime import datetime
from typing import Protocol, TypedDict
from core.rag.index_processor.constant.built_in_field import BuiltInField
from machinery.context import RequestContext
from services.knowledge.dataset_access import DatasetAccess
from services.knowledge.resource_scope import DatasetRef
class MetadataInput(Protocol):
@property
def name(self) -> str: ...
@property
def type(self) -> str: ...
class MetadataValue(Protocol):
@property
def id(self) -> str: ...
@property
def value(self) -> str | int | float | None: ...
class MetadataOperation(Protocol):
@property
def document_id(self) -> str: ...
@property
def metadata_list(self) -> Sequence[MetadataValue]: ...
@property
def partial_update(self) -> bool: ...
class MetadataOperationsInput(Protocol):
@property
def operation_data(self) -> Sequence[MetadataOperation]: ...
@dataclass(frozen=True)
class MetadataRecord:
id: str
name: str
type: str
tenant_id: str
dataset_id: str
created_by: str
created_at: datetime
updated_by: str | None
updated_at: datetime | None
class MetadataFieldSummary(TypedDict):
id: str
name: str
type: str
count: int
class MetadataList(TypedDict):
doc_metadata: list[MetadataFieldSummary]
built_in_field_enabled: bool
class MetadataStore(Protocol):
"""Own bounded transactions and recheck the complete dataset owner chain."""
def create(self, ref: DatasetRef, *, name: str, field_type: str, actor_id: str) -> MetadataRecord: ...
def rename(self, ref: DatasetRef, metadata_id: str, *, name: str, actor_id: str) -> MetadataRecord: ...
def delete(self, ref: DatasetRef, metadata_id: str) -> MetadataRecord: ...
def set_built_in(self, ref: DatasetRef, *, enabled: bool) -> None: ...
def update_documents(self, ref: DatasetRef, operations: Sequence[MetadataOperation], *, actor_id: str) -> None: ...
def list_fields(self, ref: DatasetRef) -> MetadataList: ...
class MetadataService:
def __init__(self, *, store: MetadataStore, dataset_access: DatasetAccess) -> None:
self._store = store
self._access = dataset_access
def require_dataset(self, context: RequestContext, dataset_id: str) -> DatasetRef:
dataset = self._access.require_accessible(context, dataset_id)
return DatasetRef(dataset.workspace_id, dataset.id)
@staticmethod
def _validate_name(name: str) -> None:
if len(name) > 255:
raise ValueError("Metadata name cannot exceed 255 characters.")
if name in BuiltInField:
raise ValueError("Metadata name already exists in Built-in fields.")
def create_metadata(self, ref: DatasetRef, args: MetadataInput, *, actor_id: str) -> MetadataRecord:
self._validate_name(args.name)
return self._store.create(ref, name=args.name, field_type=args.type, actor_id=actor_id)
def update_metadata_name(self, ref: DatasetRef, metadata_id: str, name: str, *, actor_id: str) -> MetadataRecord:
self._validate_name(name)
return self._store.rename(ref, metadata_id, name=name, actor_id=actor_id)
def delete_metadata(self, ref: DatasetRef, metadata_id: str) -> MetadataRecord:
return self._store.delete(ref, metadata_id)
@staticmethod
def get_built_in_fields() -> list[dict[str, str]]:
return [
{
"name": field.value,
"type": "time" if field in (BuiltInField.upload_date, BuiltInField.last_update_date) else "string",
}
for field in BuiltInField
]
def enable_built_in_field(self, ref: DatasetRef) -> None:
self._store.set_built_in(ref, enabled=True)
def disable_built_in_field(self, ref: DatasetRef) -> None:
self._store.set_built_in(ref, enabled=False)
def update_documents_metadata(self, ref: DatasetRef, args: MetadataOperationsInput, *, actor_id: str) -> None:
self._store.update_documents(ref, args.operation_data, actor_id=actor_id)
def get_dataset_metadatas(self, ref: DatasetRef) -> MetadataList:
return self._store.list_fields(ref)
@@ -0,0 +1,32 @@
"""Typed owner chains used by knowledge application and persistence ports."""
from __future__ import annotations
from typing import NamedTuple
class DatasetRef(NamedTuple):
"""Identify a dataset inside its owning workspace."""
tenant_id: str
dataset_id: str
def document(self, document_id: str) -> DocumentRef:
return DocumentRef(dataset=self, document_id=document_id)
class DocumentRef(NamedTuple):
"""Identify a document through its complete dataset owner chain."""
dataset: DatasetRef
document_id: str
def segment(self, segment_id: str) -> SegmentRef:
return SegmentRef(document=self, segment_id=segment_id)
class SegmentRef(NamedTuple):
"""Identify a segment through its complete document owner chain."""
document: DocumentRef
segment_id: str
@@ -0,0 +1 @@
"""Knowledge retrieval authorization and application operations."""
@@ -0,0 +1,33 @@
"""Grant execution-local file access only for validated knowledge retrieval results."""
from collections.abc import Sequence
from sqlalchemy.orm import Session
from core.app.file_access import (
get_current_file_access_scope,
grant_retriever_segment_access,
grant_upload_file_access,
)
from models.dataset import DocumentSegment
from models.model import UploadFile
from repositories.knowledge.dataset_read_repository import get_segment_attachment_files, is_retrieved_segment_owned
def authorize_retrieved_segment(
segment: DocumentSegment, *, tenant_id: str, dataset_ids: Sequence[str], session: Session
) -> Sequence[UploadFile] | None:
"""Authorize the owner chain and attachments before content links are signed.
A valid segment with no attachments returns an empty sequence; an invalid
retrieval result returns None and must not be included in the response.
"""
scope = get_current_file_access_scope()
if scope is not None and scope.tenant_id != tenant_id:
return None
if not is_retrieved_segment_owned(segment, tenant_id=tenant_id, dataset_ids=dataset_ids, session=session):
return None
attachments = get_segment_attachment_files(segment, session=session)
grant_retriever_segment_access([segment.id])
grant_upload_file_access(file.id for file in attachments)
return attachments
@@ -0,0 +1 @@
"""Dataset segment use cases."""
@@ -0,0 +1,82 @@
"""Infrastructure adapters for dataset segment use cases."""
from __future__ import annotations
from collections.abc import Callable
from typing import Any, Protocol
from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
from core.model_manager import ModelManager
from graphon.model_runtime.entities.model_entities import ModelType
from services.knowledge.segments.application import (
SegmentDatasetRecord,
SegmentModelProviderError,
)
class RedisSegmentClient(Protocol):
def get(self, name: str | bytes) -> Any: ...
def setnx(self, name: str | bytes, value: Any) -> Any: ...
def setex(self, name: str, time: int, value: Any) -> Any: ...
def lock(self, name: str, timeout: int) -> Any: ...
class ModelManagerSegmentGuard:
def check(self, dataset: SegmentDatasetRecord) -> None:
try:
ModelManager.for_tenant(tenant_id=dataset.workspace_id).get_model_instance(
tenant_id=dataset.workspace_id,
provider=dataset.embedding_model_provider or "",
model_type=ModelType.TEXT_EMBEDDING,
model=dataset.embedding_model or "",
)
except LLMBadRequestError as error:
raise SegmentModelProviderError(kind="bad_request", description=str(error)) from error
except ProviderTokenNotInitError as error:
raise SegmentModelProviderError(kind="token", description=error.description) from error
class RedisSegmentIndexingState:
def __init__(self, redis: RedisSegmentClient) -> None:
self._redis = redis
def lock(self, name: str, *, timeout: int):
return self._redis.lock(name, timeout=timeout)
def is_segment_indexing(self, segment_id: str, *, deleting: bool = False) -> bool:
suffix = "delete_indexing" if deleting else "indexing"
return self._redis.get(f"segment_{segment_id}_{suffix}") is not None
def mark_segment_indexing(self, segment_id: str, *, deleting: bool = False) -> None:
suffix = "delete_indexing" if deleting else "indexing"
self._redis.setex(f"segment_{segment_id}_{suffix}", 600, 1)
def is_document_indexing(self, document_id: str) -> bool:
return self._redis.get(f"document_{document_id}_indexing") is not None
def set_batch_waiting(self, job_id: str) -> None:
self._redis.setnx(f"segment_batch_import_{job_id}", "waiting")
def get_batch_status(self, job_id: str) -> str | None:
value = self._redis.get(f"segment_batch_import_{job_id}")
if value is None:
return None
return value.decode() if isinstance(value, bytes) else value
class CelerySegmentBatchImportDispatcher:
def __init__(self, *, delay: Callable[..., object]) -> None:
self._delay = delay
def dispatch(
self,
*,
job_id: str,
upload_file_id: str,
dataset_id: str,
document_id: str,
workspace_id: str,
actor_id: str,
) -> None:
self._delay(job_id, upload_file_id, dataset_id, document_id, workspace_id, actor_id)
@@ -0,0 +1,966 @@
"""Application service for dataset segment and child-chunk HTTP use cases."""
from __future__ import annotations
import logging
from collections import Counter
from collections.abc import Callable, Mapping, Sequence
from contextlib import AbstractContextManager
from dataclasses import dataclass, replace
from typing import Literal, Protocol
from uuid import uuid4
from libs.datetime_utils import naive_utc_now
from machinery.context import RequestContext
from services.knowledge.dataset_access import DatasetAccess, DatasetAccessDeniedError, DatasetAccessSnapshot
from services.knowledge.entities.segments import (
ChildChunkRecord,
ChildChunkUpdateArgs,
SegmentRecord,
SegmentUpdateArgs,
)
from services.knowledge.resource_scope import DatasetRef, DocumentRef, SegmentRef
_HIGH_QUALITY = "high_quality"
logger = logging.getLogger(__name__)
@dataclass(frozen=True, slots=True)
class SegmentDatasetRecord:
id: str
workspace_id: str
indexing_technique: str
embedding_model_provider: str | None
embedding_model: str | None
@dataclass(frozen=True, slots=True)
class SegmentDocumentRecord:
id: str
dataset_id: str
workspace_id: str
doc_form: str
@property
def ref(self) -> DocumentRef:
return DatasetRef(self.workspace_id, self.dataset_id).document(self.id)
@dataclass(frozen=True, slots=True)
class SegmentScope:
dataset: SegmentDatasetRecord | None
document: SegmentDocumentRecord | None
access_snapshot: DatasetAccessSnapshot | None = None
@dataclass(frozen=True, slots=True)
class SegmentListFilter:
page: int = 1
limit: int = 20
statuses: tuple[str, ...] = ()
hit_count_gte: int | None = None
enabled: str = "all"
keyword: str | None = None
@dataclass(frozen=True, slots=True)
class ChildChunkListFilter:
page: int = 1
limit: int = 20
keyword: str | None = None
@dataclass(frozen=True, slots=True)
class SegmentPage:
items: tuple[SegmentRecord, ...]
total: int
total_pages: int
page: int
limit: int
@dataclass(frozen=True, slots=True)
class SegmentDetail:
data: SegmentRecord
doc_form: str
@dataclass(frozen=True, slots=True)
class ChildChunkPage:
items: tuple[ChildChunkRecord, ...]
total: int
total_pages: int
page: int
limit: int
@dataclass(frozen=True, slots=True)
class SegmentBatchImport:
job_id: str
job_status: str
class SegmentScopeReader(Protocol):
def get_segment_scope(
self,
*,
dataset_ref: DatasetRef,
document_id: str,
actor_id: str | None,
) -> SegmentScope: ...
@dataclass(frozen=True, slots=True)
class ChildChunkState:
data: ChildChunkRecord
index_node_id: str | None
index_node_hash: str | None
created_by: str
updated_by: str | None = None
@dataclass(frozen=True, slots=True)
class SegmentIndexTarget:
id: str
index_node_id: str | None
enabled: bool
@dataclass(frozen=True, slots=True)
class SegmentUpdateState:
"""Persisted fields needed to decide a segment update, without presentation data."""
content: str
enabled: bool
keywords: tuple[str, ...] | None
# The stored QA answer, or None when the segment has none (non-QA documents).
answer: str | None
class SegmentStore(Protocol):
"""Owner-scoped persistence; each operation owns a short session."""
def list_segments(self, document_ref: DocumentRef, query: SegmentListFilter) -> SegmentPage: ...
def get_segments(self, document_ref: DocumentRef, segment_ids: Sequence[str]) -> tuple[SegmentIndexTarget, ...]: ...
def get_segment(self, segment_ref: SegmentRef) -> SegmentDetail | None: ...
def get_segment_update_state(self, segment_ref: SegmentRef) -> SegmentUpdateState | None: ...
def save_segment(self, segment_ref: SegmentRef, values: Mapping[str, object], *, create: bool = False) -> None: ...
def delete_segments(self, document_ref: DocumentRef, segment_ids: Sequence[str]) -> None: ...
def list_child_chunks(self, segment_ref: SegmentRef, query: ChildChunkListFilter) -> ChildChunkPage | None: ...
def get_children_for_segments(
self, document_ref: DocumentRef, segment_ids: Sequence[str]
) -> tuple[ChildChunkState, ...]: ...
def set_segments(
self, document_ref: DocumentRef, segment_ids: Sequence[str], values: Mapping[str, object]
) -> None: ...
def get_children(self, segment_ref: SegmentRef) -> tuple[ChildChunkState, ...] | None: ...
def save_children(
self,
segment_ref: SegmentRef,
*,
added: Sequence[ChildChunkState] = (),
updated: Sequence[ChildChunkState] = (),
deleted: Sequence[ChildChunkState] = (),
) -> None: ...
class SegmentIndex(Protocol):
def count_tokens(self, dataset: SegmentDatasetRecord, text: str) -> int: ...
def create_many(
self,
document_ref: DocumentRef,
*,
segment_ids: Sequence[str],
keywords_list: Sequence[Sequence[str] | None] | None,
) -> None: ...
def create(
self, segment_ref: SegmentRef, *, keywords: Sequence[str] | None, attachment_ids: Sequence[str]
) -> None: ...
def update(self, segment_ref: SegmentRef, *, keywords: Sequence[str] | None, regenerate_children: bool) -> None: ...
def update_attachments(self, segment_ref: SegmentRef, attachment_ids: Sequence[str]) -> None: ...
def update_summary(self, segment_ref: SegmentRef, *, summary: str | None, content_changed: bool) -> None: ...
def update_children(
self,
segment_ref: SegmentRef,
*,
added: Sequence[ChildChunkState] = (),
updated: Sequence[ChildChunkState] = (),
deleted: Sequence[ChildChunkState] = (),
) -> None: ...
def delete(
self, document_ref: DocumentRef, segments: Sequence[SegmentIndexTarget], children: Sequence[ChildChunkState]
) -> None: ...
def change_status(
self, document_ref: DocumentRef, segment_ids: Sequence[str], action: Literal["enable", "disable"]
) -> None: ...
class SegmentLimits(Protocol):
SINGLE_CHUNK_ATTACHMENT_LIMIT: int
class SegmentModelGuard(Protocol):
def check(self, dataset: SegmentDatasetRecord) -> None: ...
class SegmentUploadCatalog(Protocol):
def get_file_name(self, *, workspace_id: str, upload_file_id: str) -> str | None: ...
class SegmentIndexingState(Protocol):
def lock(self, name: str, *, timeout: int) -> AbstractContextManager[object]: ...
def is_segment_indexing(self, segment_id: str, *, deleting: bool = False) -> bool: ...
def mark_segment_indexing(self, segment_id: str, *, deleting: bool = False) -> None: ...
def is_document_indexing(self, document_id: str) -> bool: ...
def set_batch_waiting(self, job_id: str) -> None: ...
def get_batch_status(self, job_id: str) -> str | None: ...
class SegmentBatchImportDispatcher(Protocol):
def dispatch(
self,
*,
job_id: str,
upload_file_id: str,
dataset_id: str,
document_id: str,
workspace_id: str,
actor_id: str,
) -> None: ...
class SegmentApplicationError(Exception):
"""Base class for framework-neutral segment use-case errors."""
class SegmentDatasetNotFoundError(SegmentApplicationError):
pass
class SegmentDocumentNotFoundError(SegmentApplicationError):
pass
class SegmentNotFoundError(SegmentApplicationError):
pass
class ChildChunkNotFoundError(SegmentApplicationError):
pass
class SegmentUploadFileNotFoundError(SegmentApplicationError):
pass
class SegmentPermissionDeniedError(SegmentApplicationError):
pass
class SegmentInvalidFileTypeError(SegmentApplicationError):
pass
class SegmentDocumentIndexingError(SegmentApplicationError):
pass
class SegmentStatusUpdateError(SegmentApplicationError):
pass
class SegmentBatchImportDispatchError(SegmentApplicationError):
pass
class SegmentBatchImportNotFoundError(SegmentApplicationError):
pass
class SegmentModelProviderError(Exception):
"""Adapter error raised when the configured embedding model cannot be loaded."""
def __init__(self, *, kind: Literal["bad_request", "token"], description: str) -> None:
super().__init__(description)
self.kind = kind
self.description = description
class SegmentDatasetModelUnavailableError(SegmentApplicationError):
pass
class SegmentEmbeddingModelUnavailableError(SegmentApplicationError):
pass
class ChildChunkIndexingApplicationError(SegmentApplicationError):
pass
class ChildChunkDeleteIndexApplicationError(SegmentApplicationError):
pass
class DatasetSegmentApplicationService:
def __init__(
self,
*,
dataset_access: DatasetAccess,
scopes: SegmentScopeReader,
store: SegmentStore,
index: SegmentIndex,
limits: SegmentLimits,
text_hash: Callable[[str], str],
uploads: SegmentUploadCatalog,
model_guard: SegmentModelGuard,
indexing_state: SegmentIndexingState,
batch_dispatcher: SegmentBatchImportDispatcher,
job_id_factory: Callable[[], str],
) -> None:
self._dataset_access = dataset_access
self._scopes = scopes
self._store = store
self._index = index
self._limits = limits
self._uploads = uploads
self._model_guard = model_guard
self._indexing_state = indexing_state
self._batch_dispatcher = batch_dispatcher
self._job_id_factory = job_id_factory
self.mutations = SegmentMutationService(
store=store, index=index, text_hash=text_hash, indexing_state=indexing_state
)
def list_segments(
self,
context: RequestContext,
*,
dataset_id: str,
document_id: str,
query: SegmentListFilter,
) -> SegmentPage:
scope = self._require_scope(context, dataset_id=dataset_id, document_id=document_id)
return self._store.list_segments(scope.document.ref, query)
def delete_segments(
self,
context: RequestContext,
*,
dataset_id: str,
document_id: str,
segment_ids: Sequence[str],
) -> None:
scope = self._require_scope(context, dataset_id=dataset_id, document_id=document_id)
self._check_dataset_model(scope.dataset)
segments = self._store.get_segments(scope.document.ref, segment_ids)
children = self._store.get_children_for_segments(scope.document.ref, [segment.id for segment in segments])
if segments:
self._index.delete(scope.document.ref, segments, children)
self._store.delete_segments(scope.document.ref, [segment.id for segment in segments])
def change_segment_status(
self,
context: RequestContext,
*,
dataset_id: str,
document_id: str,
segment_ids: Sequence[str],
action: Literal["enable", "disable"],
) -> None:
scope = self._require_scope(context, dataset_id=dataset_id, document_id=document_id)
self._check_dataset_model(scope.dataset)
self._check_embedding_model(scope.dataset)
if self._indexing_state.is_document_indexing(scope.document.id):
raise SegmentDocumentIndexingError("Document is being indexed, please try again later")
try:
segments = self._store.get_segments(scope.document.ref, segment_ids)
ids = []
for segment in segments:
if segment.enabled == (action == "enable") or self._indexing_state.is_segment_indexing(segment.id):
continue
ids.append(segment.id)
if ids:
self._store.set_segments(
scope.document.ref,
ids,
{
"enabled": action == "enable",
"disabled_at": naive_utc_now() if action == "disable" else None,
"disabled_by": context.account_id if action == "disable" else None,
},
)
self._index.change_status(scope.document.ref, ids, action)
except Exception as error:
raise SegmentStatusUpdateError(str(error)) from error
def create_segment(
self,
context: RequestContext,
*,
dataset_id: str,
document_id: str,
values: Mapping[str, object],
) -> SegmentDetail:
scope = self._require_scope(context, dataset_id=dataset_id, document_id=document_id)
self._check_embedding_model(scope.dataset)
args = self._validate_values(values, scope.document.doc_form)
segment_ref = self.mutations.create_segment(scope, actor_id=context.account_id, args=args)
return self._detail(segment_ref)
def update_segment(
self,
context: RequestContext,
*,
dataset_id: str,
document_id: str,
segment_id: str,
values: Mapping[str, object],
) -> SegmentDetail:
scope = self._require_scope(context, dataset_id=dataset_id, document_id=document_id)
self._check_dataset_model(scope.dataset)
self._check_embedding_model(scope.dataset)
args = self._validate_values(values, scope.document.doc_form)
self.mutations.update_segment(scope, actor_id=context.account_id, segment_id=segment_id, args=args)
return self._detail(scope.document.ref.segment(segment_id))
def _detail(self, segment_ref: SegmentRef) -> SegmentDetail:
result = self._store.get_segment(segment_ref)
if result is None:
raise SegmentNotFoundError("Segment not found")
return result
def delete_segment(
self,
context: RequestContext,
*,
dataset_id: str,
document_id: str,
segment_id: str,
) -> None:
scope = self._require_scope(context, dataset_id=dataset_id, document_id=document_id)
self._check_dataset_model(scope.dataset)
return self.mutations.delete_segment(scope.document.ref.segment(segment_id))
def start_batch_import(
self,
context: RequestContext,
*,
dataset_id: str,
document_id: str,
upload_file_id: str,
) -> SegmentBatchImport:
scope = self._require_scope(
context,
dataset_id=dataset_id,
document_id=document_id,
enforce_permission=False,
)
filename = self._uploads.get_file_name(
workspace_id=scope.dataset.workspace_id,
upload_file_id=upload_file_id,
)
if filename is None:
raise SegmentUploadFileNotFoundError("UploadFile not found")
if not filename.lower().endswith(".csv"):
raise SegmentInvalidFileTypeError("Invalid file type. Only CSV files are allowed")
job_id = self._job_id_factory()
try:
self._indexing_state.set_batch_waiting(job_id)
self._batch_dispatcher.dispatch(
job_id=job_id,
upload_file_id=upload_file_id,
dataset_id=scope.dataset.id,
document_id=scope.document.id,
workspace_id=scope.dataset.workspace_id,
actor_id=context.account_id,
)
except Exception as error:
raise SegmentBatchImportDispatchError(str(error)) from error
return SegmentBatchImport(job_id=job_id, job_status="waiting")
def get_batch_import_status(self, job_id: str) -> SegmentBatchImport:
status = self._indexing_state.get_batch_status(job_id)
if status is None:
raise SegmentBatchImportNotFoundError("The job does not exist.")
return SegmentBatchImport(job_id=job_id, job_status=status)
def create_child_chunk(
self,
context: RequestContext,
*,
dataset_id: str,
document_id: str,
segment_id: str,
content: str,
) -> ChildChunkRecord:
scope = self._require_scope(context, dataset_id=dataset_id, document_id=document_id)
self._check_embedding_model(scope.dataset)
return self.mutations.create_child_chunk(
scope.document.ref.segment(segment_id), content=content, actor_id=context.account_id
)
def list_child_chunks(
self,
*,
workspace_id: str,
dataset_id: str,
document_id: str,
segment_id: str,
query: ChildChunkListFilter,
) -> ChildChunkPage:
scope = self._require_scope_without_actor(
workspace_id=workspace_id,
dataset_id=dataset_id,
document_id=document_id,
)
self._check_dataset_model(scope.dataset)
result = self._store.list_child_chunks(scope.document.ref.segment(segment_id), query)
if result is None:
raise SegmentNotFoundError("Segment not found")
return result
def update_child_chunks(
self,
context: RequestContext,
*,
dataset_id: str,
document_id: str,
segment_id: str,
chunks: Sequence[ChildChunkUpdateArgs],
) -> tuple[ChildChunkRecord, ...]:
scope = self._require_scope(context, dataset_id=dataset_id, document_id=document_id)
self._check_dataset_model(scope.dataset)
return self.mutations.update_child_chunks(
scope.document.ref.segment(segment_id), chunks=chunks, actor_id=context.account_id
)
def delete_child_chunk(
self,
context: RequestContext,
*,
dataset_id: str,
document_id: str,
segment_id: str,
child_chunk_id: str,
) -> None:
scope = self._require_scope(context, dataset_id=dataset_id, document_id=document_id)
self._check_dataset_model(scope.dataset)
return self.mutations.delete_child_chunk(scope.document.ref.segment(segment_id), child_chunk_id=child_chunk_id)
def update_child_chunk(
self,
context: RequestContext,
*,
dataset_id: str,
document_id: str,
segment_id: str,
child_chunk_id: str,
content: str,
) -> ChildChunkRecord:
scope = self._require_scope(context, dataset_id=dataset_id, document_id=document_id)
self._check_dataset_model(scope.dataset)
return self.mutations.update_child_chunk(
scope.document.ref.segment(segment_id),
child_chunk_id=child_chunk_id,
content=content,
actor_id=context.account_id,
)
def _validate_values(self, values: Mapping[str, object], doc_form: str) -> SegmentUpdateArgs:
return validate_segment_values(
values, doc_form=doc_form, attachment_limit=self._limits.SINGLE_CHUNK_ATTACHMENT_LIMIT
)
def _require_scope(
self,
context: RequestContext,
*,
dataset_id: str,
document_id: str,
enforce_permission: bool = True,
) -> SegmentMutationScope:
scope = self._scopes.get_segment_scope(
dataset_ref=DatasetRef(context.active_workspace_id, dataset_id),
document_id=document_id,
actor_id=context.account_id if enforce_permission else None,
)
dataset = scope.dataset
if dataset is None:
raise SegmentDatasetNotFoundError("Dataset not found")
if enforce_permission:
if scope.access_snapshot is None:
raise SegmentPermissionDeniedError("Dataset access state is missing")
try:
self._dataset_access.check_access(context, scope.access_snapshot)
except DatasetAccessDeniedError as error:
raise SegmentPermissionDeniedError(str(error)) from error
document = scope.document
if document is None:
raise SegmentDocumentNotFoundError("Document not found")
return SegmentMutationScope(dataset=dataset, document=document)
def _require_scope_without_actor(
self,
*,
workspace_id: str,
dataset_id: str,
document_id: str,
) -> SegmentMutationScope:
scope = self._scopes.get_segment_scope(
dataset_ref=DatasetRef(workspace_id, dataset_id),
document_id=document_id,
actor_id=None,
)
if scope.dataset is None:
raise SegmentDatasetNotFoundError("Dataset not found")
if scope.document is None:
raise SegmentDocumentNotFoundError("Document not found")
return SegmentMutationScope(dataset=scope.dataset, document=scope.document)
def _check_dataset_model(self, dataset: SegmentDatasetRecord) -> None:
if dataset.indexing_technique != _HIGH_QUALITY:
return
try:
self._model_guard.check(dataset)
except SegmentModelProviderError as error:
if error.kind == "token":
message = f"The dataset is unavailable, due to: {error.description}"
else:
message = (
"No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
)
raise SegmentDatasetModelUnavailableError(message) from error
def _check_embedding_model(self, dataset: SegmentDatasetRecord) -> None:
if dataset.indexing_technique != _HIGH_QUALITY:
return
try:
self._model_guard.check(dataset)
except SegmentModelProviderError as error:
if error.kind == "bad_request":
message = (
"No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
)
else:
message = error.description
raise SegmentEmbeddingModelUnavailableError(message) from error
class SegmentMutationService:
"""Shared writes for Console and Service API after their own admission checks.
Callers supply an owner chain reconstructed from authorized database rows.
Persistence ports recheck that chain and own their transactions. Partial
update arguments are accepted here; transport validation stays at admission.
"""
def __init__(
self,
*,
store: SegmentStore,
index: SegmentIndex,
text_hash: Callable[[str], str],
indexing_state: SegmentIndexingState,
) -> None:
self._store = store
self._index = index
self._text_hash = text_hash
self._indexing_state = indexing_state
def index_segments(
self,
document_ref: DocumentRef,
*,
segment_ids: Sequence[str],
keywords_list: Sequence[Sequence[str] | None] | None = None,
) -> None:
"""Index a committed batch and persist failures through the scoped store.
Callers must finish their write transaction before invoking this method.
The index adapter reloads the complete owner chain before external I/O.
"""
if keywords_list is not None and len(keywords_list) != len(segment_ids):
raise ValueError("Keywords must correspond to the segment batch")
if not segment_ids:
return
try:
self._index.create_many(document_ref, segment_ids=segment_ids, keywords_list=keywords_list)
except Exception as error:
logger.exception("Batch indexing failed for document %s", document_ref.document_id)
self._store.set_segments(
document_ref,
segment_ids,
{"enabled": False, "disabled_at": naive_utc_now(), "status": "error", "error": str(error)},
)
raise
def create_segment(
self,
scope: SegmentMutationScope,
*,
actor_id: str,
args: SegmentUpdateArgs,
) -> SegmentRef:
now = naive_utc_now()
segment_ref = scope.document.ref.segment(str(uuid4()))
saved = False
try:
with self._indexing_state.lock(f"add_segment_lock_document_id_{scope.document.id}", timeout=600):
self._store.save_segment(
segment_ref,
{
"content": args.content,
"answer": args.answer if scope.document.doc_form == "qa_model" else None,
"word_count": len(args.content or "")
+ (len(args.answer or "") if scope.document.doc_form == "qa_model" else 0),
"tokens": self._index.count_tokens(scope.dataset, args.content or ""),
"index_node_id": str(uuid4()),
"index_node_hash": self._text_hash(args.content or ""),
"created_by": actor_id,
"status": "completed",
"indexing_at": now,
"completed_at": now,
},
create=True,
)
saved = True
except Exception as error:
if saved:
self._mark_error(segment_ref, error)
raise
try:
self._index.create(segment_ref, keywords=args.keywords, attachment_ids=args.attachment_ids or ())
except Exception as error:
self._mark_error(segment_ref, error)
return segment_ref
def update_segment(
self,
scope: SegmentMutationScope,
*,
actor_id: str,
segment_id: str,
args: SegmentUpdateArgs,
) -> None:
segment_ref = scope.document.ref.segment(segment_id)
previous = self._store.get_segment_update_state(segment_ref)
if previous is None:
raise SegmentNotFoundError("Segment not found")
if self._indexing_state.is_segment_indexing(segment_id):
raise ValueError("Segment is indexing, please try again later")
if args.enabled is True and not previous.enabled and args.model_fields_set == {"enabled"}:
# Disabled segments have no vectors to update. Reuse the console's load-based enable task.
self._store.save_segment(segment_ref, {"enabled": True, "disabled_at": None, "disabled_by": None})
self._index.change_status(scope.document.ref, [segment_id], "enable")
return
if args.enabled is False and previous.enabled:
self._store.save_segment(
segment_ref, {"enabled": False, "disabled_at": naive_utc_now(), "disabled_by": actor_id}
)
self._indexing_state.mark_segment_indexing(segment_id)
self._index.change_status(scope.document.ref, [segment_id], "disable")
return
if not previous.enabled and args.enabled is not True:
raise ValueError("Can't update disabled segment")
content = args.content or previous.content
answer = args.answer if "answer" in args.model_fields_set else previous.answer
content_changed = content != previous.content
keyword_changed = bool(args.keywords) and Counter(previous.keywords or ()) != Counter(args.keywords or ())
changes: dict[str, object] = {
"content": content,
"word_count": len(content) + (len(answer or "") if scope.document.doc_form == "qa_model" else 0),
"enabled": True,
"disabled_at": None,
"disabled_by": None,
}
if scope.document.doc_form == "qa_model":
changes["answer"] = answer
if keyword_changed and not content_changed:
changes["keywords"] = args.keywords
try:
if content_changed:
now = naive_utc_now()
changes.update(
index_node_hash=self._text_hash(content),
tokens=self._index.count_tokens(
scope.dataset,
content + (answer or "") if scope.document.doc_form == "qa_model" else content,
),
status="completed",
indexing_at=now,
completed_at=now,
updated_at=now,
updated_by=actor_id,
)
self._store.save_segment(segment_ref, changes)
if scope.document.doc_form == "hierarchical_model":
if args.regenerate_child_chunks:
self._index.update(segment_ref, keywords=args.keywords, regenerate_children=True)
elif content_changed or args.enabled or keyword_changed:
self._index.update(segment_ref, keywords=args.keywords, regenerate_children=False)
if scope.dataset.indexing_technique == _HIGH_QUALITY:
try:
self._index.update_summary(segment_ref, summary=args.summary, content_changed=content_changed)
except Exception:
logger.exception("Failed to update summary for segment %s", segment_id)
if args.attachment_ids is not None:
self._index.update_attachments(segment_ref, args.attachment_ids)
except Exception as error:
self._mark_error(segment_ref, error)
def delete_segment(self, segment_ref: SegmentRef) -> None:
segments = self._store.get_segments(segment_ref.document, (segment_ref.segment_id,))
if not segments:
raise SegmentNotFoundError("Segment not found")
segment = segments[0]
if self._indexing_state.is_segment_indexing(segment_ref.segment_id, deleting=True):
raise ValueError("Segment is deleting.")
if segment.enabled:
self._indexing_state.mark_segment_indexing(segment_ref.segment_id, deleting=True)
self._index.delete(
segment_ref.document,
(segment,),
self._children(segment_ref),
)
self._store.delete_segments(segment_ref.document, (segment_ref.segment_id,))
def create_child_chunk(self, segment_ref: SegmentRef, *, content: str, actor_id: str) -> ChildChunkRecord:
with self._indexing_state.lock(f"add_child_lock_{segment_ref.segment_id}", timeout=20):
children = self._children(segment_ref)
child = self._new_child(
segment_ref, content, max((c.data.position for c in children), default=0) + 1, actor_id
)
self._write_children(segment_ref, added=(child,))
return child.data
def update_child_chunks(
self, segment_ref: SegmentRef, *, chunks: Sequence[ChildChunkUpdateArgs], actor_id: str
) -> tuple[ChildChunkRecord, ...]:
existing = self._children(segment_ref)
remaining = {child.data.id: child for child in existing}
added: list[ChildChunkState] = []
updated: list[ChildChunkState] = []
for chunk in chunks:
if chunk.id:
previous = remaining.pop(chunk.id, None)
if previous is not None and previous.data.content != chunk.content:
updated.append(self._updated_child(previous, chunk.content, actor_id))
else:
added.append(self._new_child(segment_ref, chunk.content, len(existing) + len(added) + 1, actor_id))
self._write_children(segment_ref, added=added, updated=updated, deleted=tuple(remaining.values()))
return tuple(child.data for child in sorted(added + updated, key=lambda child: child.data.position))
def delete_child_chunk(self, segment_ref: SegmentRef, *, child_chunk_id: str) -> None:
child = self._child(segment_ref, child_chunk_id)
try:
self._index.update_children(segment_ref, deleted=(child,))
except Exception as error:
raise ChildChunkDeleteIndexApplicationError(str(error)) from error
self._store.save_children(segment_ref, deleted=(child,))
def update_child_chunk(
self, segment_ref: SegmentRef, *, child_chunk_id: str, content: str, actor_id: str
) -> ChildChunkRecord:
child = self._updated_child(self._child(segment_ref, child_chunk_id), content, actor_id)
self._write_children(segment_ref, updated=(child,))
return child.data
def _mark_error(self, segment_ref: SegmentRef, error: Exception) -> None:
logger.exception("Segment indexing failed: %s", segment_ref.segment_id)
self._store.save_segment(
segment_ref, {"enabled": False, "disabled_at": naive_utc_now(), "status": "error", "error": str(error)}
)
def _children(self, segment_ref: SegmentRef) -> tuple[ChildChunkState, ...]:
children = self._store.get_children(segment_ref)
if children is None:
raise SegmentNotFoundError("Segment not found")
return children
def _child(self, segment_ref: SegmentRef, child_id: str) -> ChildChunkState:
for child in self._children(segment_ref):
if child.data.id == child_id:
return child
raise ChildChunkNotFoundError("Child chunk not found")
def _new_child(self, segment_ref: SegmentRef, content: str, position: int, actor_id: str) -> ChildChunkState:
now = naive_utc_now()
return ChildChunkState(
data=ChildChunkRecord(
id=str(uuid4()),
segment_id=segment_ref.segment_id,
content=content,
position=position,
word_count=len(content),
type="customized",
created_at=now,
updated_at=now,
),
index_node_id=str(uuid4()),
index_node_hash=self._text_hash(content),
created_by=actor_id,
)
def _updated_child(self, child: ChildChunkState, content: str, actor_id: str) -> ChildChunkState:
return replace(
child,
data=child.data.model_copy(
update={
"content": content,
"word_count": len(content),
"type": "customized",
"updated_at": naive_utc_now(),
}
),
updated_by=actor_id,
)
def _write_children(
self,
segment_ref: SegmentRef,
*,
added: Sequence[ChildChunkState] = (),
updated: Sequence[ChildChunkState] = (),
deleted: Sequence[ChildChunkState] = (),
) -> None:
try:
self._index.update_children(segment_ref, added=added, updated=updated, deleted=deleted)
self._store.save_children(segment_ref, added=added, updated=updated, deleted=deleted)
except Exception as error:
raise ChildChunkIndexingApplicationError(str(error)) from error
@dataclass(frozen=True, slots=True)
class SegmentMutationScope:
"""Authorized dataset and document values for shared mutation policy."""
dataset: SegmentDatasetRecord
document: SegmentDocumentRecord
def __post_init__(self) -> None:
if self.document.dataset_id != self.dataset.id or self.document.workspace_id != self.dataset.workspace_id:
raise ValueError("Document does not belong to the dataset")
def validate_segment_values(values: Mapping[str, object], *, doc_form: str, attachment_limit: int) -> SegmentUpdateArgs:
answer = values.get("answer")
if doc_form == "qa_model":
if not answer:
raise ValueError("Answer is required")
if not isinstance(answer, str) or not answer.strip():
raise ValueError("Answer is empty")
content = values.get("content")
if not isinstance(content, str) or not content.strip():
raise ValueError("Content is empty")
attachments = values.get("attachment_ids")
if attachments:
if not isinstance(attachments, list):
raise ValueError("Attachment IDs is invalid")
if len(attachments) > attachment_limit:
raise ValueError(f"Exceeded maximum attachment limit of {attachment_limit}")
return SegmentUpdateArgs.model_validate(values)
@@ -0,0 +1,356 @@
"""Indexing adapters that materialize database inputs before external I/O."""
import logging
from collections.abc import Callable, Mapping, Sequence
from uuid import uuid4
from sqlalchemy.orm import Session
from core.credit_usage import CreditUsageCreatedBy
from core.model_context import with_credit_usage_created_by
from core.model_manager import ModelManager
from core.rag.datasource.keyword.jieba.jieba import Jieba
from core.rag.datasource.vdb.vector_factory import Vector
from core.rag.entities import Rule
from core.rag.index_processor.constant.doc_type import DocType
from core.rag.index_processor.processor.paragraph_index_processor import ParagraphIndexProcessor
from core.rag.index_processor.processor.parent_child_index_processor import ParentChildIndexProcessor
from core.rag.models.document import Document
from graphon.model_runtime.entities.model_entities import ModelType
from libs.datetime_utils import naive_utc_now
from models.enums import SegmentType
from repositories.knowledge.segment_repository import SegmentIndexingSnapshot, SQLAlchemySegmentRepository
from repositories.knowledge.upload_file_repository import SQLAlchemyKnowledgeUploadRepository
from services.knowledge.entities.segments import ChildChunkRecord
from services.knowledge.resource_scope import DocumentRef, SegmentRef
from services.knowledge.segments.adapters import RedisSegmentClient
from services.knowledge.segments.application import ChildChunkState, SegmentDatasetRecord, SegmentIndexTarget
from services.knowledge.summaries.adapters import SummaryIndexAdapter
logger = logging.getLogger(__name__)
class SegmentIndexingGateway:
def __init__(
self,
*,
segments: SQLAlchemySegmentRepository,
uploads: SQLAlchemyKnowledgeUploadRepository,
redis: RedisSegmentClient,
new_session: Callable[[], Session],
delete_task: Callable[..., object],
enable_task: Callable[..., object],
disable_task: Callable[..., object],
) -> None:
self._segments = segments
self._uploads = uploads
self._redis = redis
self._new_session = new_session
self._delete_task = delete_task
self._enable_task = enable_task
self._disable_task = disable_task
def count_tokens(self, dataset: SegmentDatasetRecord, text: str) -> int:
if dataset.indexing_technique != "high_quality":
return 0
model = ModelManager.for_tenant(tenant_id=dataset.workspace_id).get_model_instance(
tenant_id=dataset.workspace_id,
provider=dataset.embedding_model_provider or "",
model_type=ModelType.TEXT_EMBEDDING,
model=dataset.embedding_model or "",
)
return model.get_text_embedding_num_tokens(texts=[text])[0]
@staticmethod
def _vector(snapshot: SegmentIndexingSnapshot) -> Vector:
return Vector(snapshot.dataset, session=None, vector_type=snapshot.vector_type)
@staticmethod
def _document(snapshot: SegmentIndexingSnapshot) -> Document:
segment = snapshot.segment
return Document(
page_content=segment.content,
metadata={
"doc_id": segment.index_node_id,
"doc_hash": segment.index_node_hash,
"document_id": segment.document_id,
"dataset_id": segment.dataset_id,
"doc_type": DocType.TEXT,
},
)
@with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)
def create_many(
self,
document_ref: DocumentRef,
*,
segment_ids: Sequence[str],
keywords_list: Sequence[Sequence[str] | None] | None,
) -> None:
"""Materialize the whole scoped batch before indexing any of its records."""
snapshots = self._segments.get_indexing_snapshots(document_ref, segment_ids)
if not snapshots:
return
first = snapshots[0]
documents = [self._document(snapshot) for snapshot in snapshots]
if first.document.doc_form == "hierarchical_model":
for snapshot in snapshots:
self._regenerate_children(document_ref.segment(snapshot.segment.id), snapshot, replace_existing=False)
elif first.dataset.indexing_technique == "high_quality":
self._vector(first).create(documents)
elif first.document.doc_form != "qa_model":
self._write_keywords(document_ref, first, documents, keywords_list, replace_existing=False)
if first.dataset.is_multimodal:
files = self._uploads.get_files(
workspace_id=document_ref.dataset.tenant_id,
file_ids=tuple(attachment_id for snapshot in snapshots for attachment_id in snapshot.attachment_ids),
)
attachments = [
document
for snapshot in snapshots
for document in self._attachment_documents(
snapshot, [files[file_id] for file_id in snapshot.attachment_ids if file_id in files]
)
]
if attachments and first.dataset.indexing_technique == "high_quality":
self._vector(first).create_multimodal(attachments, upload_files=files)
@with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)
def create(self, segment_ref: SegmentRef, *, keywords: Sequence[str] | None, attachment_ids: Sequence[str]) -> None:
snapshot = self._segments.get_indexing_snapshot(segment_ref)
files = self._uploads.get_files(workspace_id=segment_ref.document.dataset.tenant_id, file_ids=attachment_ids)
self._segments.replace_attachments(segment_ref, tuple(files))
if snapshot.document.doc_form == "hierarchical_model":
self._regenerate_children(segment_ref, snapshot, replace_existing=False)
elif snapshot.dataset.indexing_technique == "high_quality":
self._vector(snapshot).create([self._document(snapshot)])
else:
self._update_keywords(segment_ref, snapshot, keywords, replace_existing=False)
if files and snapshot.dataset.is_multimodal:
self._vector(snapshot).create_multimodal(
self._attachment_documents(snapshot, files.values()), upload_files=files
)
@with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)
def update(self, segment_ref: SegmentRef, *, keywords: Sequence[str] | None, regenerate_children: bool) -> None:
snapshot = self._segments.get_indexing_snapshot(segment_ref)
if regenerate_children:
self._regenerate_children(segment_ref, snapshot, replace_existing=True)
elif snapshot.dataset.indexing_technique == "high_quality":
vector = self._vector(snapshot)
assert snapshot.segment.index_node_id is not None
vector.delete_by_ids([snapshot.segment.index_node_id])
vector.add_texts([self._document(snapshot)], duplicate_check=True)
else:
self._update_keywords(segment_ref, snapshot, keywords, replace_existing=True)
def _regenerate_children(
self, segment_ref: SegmentRef, snapshot: SegmentIndexingSnapshot, *, replace_existing: bool
) -> None:
if snapshot.dataset.indexing_technique != "high_quality":
raise ValueError("The knowledge base index technique is not high quality!")
rule = snapshot.process_rule
if rule is None:
if replace_existing:
return
raise ValueError("No processing rule found.")
manager = ModelManager.for_tenant(tenant_id=snapshot.dataset.tenant_id)
model = (
manager.get_model_instance(
tenant_id=snapshot.dataset.tenant_id,
provider=snapshot.dataset.embedding_model_provider,
model_type=ModelType.TEXT_EMBEDDING,
model=snapshot.dataset.embedding_model,
)
if snapshot.dataset.embedding_model_provider
else manager.get_default_model_instance(
tenant_id=snapshot.dataset.tenant_id, model_type=ModelType.TEXT_EMBEDDING
)
)
children = ParentChildIndexProcessor().split_child_nodes(
self._document(snapshot), Rule.model_validate(rule.to_dict()["rules"]), rule.mode, model
)
previous = self._segments.get_children(segment_ref) or ()
vector = self._vector(snapshot)
if replace_existing and previous:
vector.delete_by_ids([child.index_node_id for child in previous if child.index_node_id])
if children:
vector.create([Document.model_validate(child.model_dump()) for child in children])
now = naive_utc_now()
added = tuple(
ChildChunkState(
data=ChildChunkRecord(
id=str(uuid4()),
segment_id=segment_ref.segment_id,
content=child.page_content,
position=position,
word_count=len(child.page_content),
type=SegmentType.AUTOMATIC,
created_at=now,
updated_at=now,
),
index_node_id=child.metadata["doc_id"],
index_node_hash=child.metadata["doc_hash"],
created_by=snapshot.document.created_by,
)
for position, child in enumerate(children, start=1)
)
self._segments.save_children(segment_ref, added=added, deleted=previous if replace_existing else ())
@with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)
def update_children(
self,
segment_ref: SegmentRef,
*,
added: Sequence[ChildChunkState] = (),
updated: Sequence[ChildChunkState] = (),
deleted: Sequence[ChildChunkState] = (),
) -> None:
snapshot = self._segments.get_indexing_snapshot(segment_ref)
if snapshot.dataset.indexing_technique != "high_quality":
return
documents = [
Document(
page_content=child.data.content,
metadata={
"doc_id": child.index_node_id,
"doc_hash": child.index_node_hash,
"document_id": segment_ref.document.document_id,
"dataset_id": segment_ref.document.dataset.dataset_id,
},
)
for child in (*added, *updated)
]
vector = self._vector(snapshot)
deleted_ids = []
for child in (*updated, *deleted):
assert child.index_node_id is not None
deleted_ids.append(child.index_node_id)
if deleted_ids:
vector.delete_by_ids(deleted_ids)
if documents:
vector.add_texts(documents, duplicate_check=True)
@staticmethod
def _attachment_documents(snapshot: SegmentIndexingSnapshot, files) -> list[Document]:
return [
Document(
page_content=file.name,
metadata={
"doc_id": file.id,
"doc_hash": "",
"dataset_id": snapshot.dataset.id,
"document_id": snapshot.document.id,
"doc_type": DocType.IMAGE,
},
)
for file in files
]
@with_credit_usage_created_by(CreditUsageCreatedBy.KNOWLEDGE_INDEXING)
def update_attachments(self, segment_ref: SegmentRef, attachment_ids: Sequence[str]) -> None:
snapshot = self._segments.get_indexing_snapshot(segment_ref)
if snapshot.dataset.indexing_technique != "high_quality" or set(snapshot.attachment_ids) == set(attachment_ids):
return
files = self._uploads.get_files(workspace_id=snapshot.dataset.tenant_id, file_ids=attachment_ids)
if snapshot.dataset.is_multimodal:
vector = self._vector(snapshot)
if snapshot.attachment_ids:
vector.delete_by_ids(list(snapshot.attachment_ids))
if files:
vector.create_multimodal(self._attachment_documents(snapshot, files.values()), upload_files=files)
self._segments.replace_attachments(segment_ref, tuple(files))
def update_summary(self, segment_ref: SegmentRef, *, summary: str | None, content_changed: bool) -> None:
if summary is None and not content_changed:
return
snapshot = self._segments.get_indexing_snapshot(segment_ref)
if snapshot.dataset.indexing_technique != "high_quality" or snapshot.document.doc_form == "qa_model":
return
setting = snapshot.dataset.summary_index_setting
previous = snapshot.summary
if summary is None or (previous is not None and summary == previous.summary_content):
if not (content_changed and previous and setting and setting.get("enable") is True):
return
try:
self._segments.save_summary(segment_ref, previous.summary_content or "")
summary, _ = ParagraphIndexProcessor.generate_summary_from_inputs(
snapshot.dataset.tenant_id,
snapshot.segment.content,
setting,
document_language=snapshot.document.doc_language,
image_loader=lambda: self._segments.get_summary_images(segment_ref),
)
if not summary.strip():
raise ValueError("Generated summary is empty")
except Exception as error:
self._segments.save_summary(segment_ref, previous.summary_content or "", error=str(error))
raise
if not summary.strip():
if previous and previous.summary_index_node_id:
try:
self._vector(snapshot).delete_by_ids([previous.summary_index_node_id])
except Exception:
logger.exception("Failed to delete summary vector for %s", segment_ref.segment_id)
self._segments.save_summary(segment_ref, None)
return
record = self._segments.save_summary(segment_ref, summary)
assert record is not None
SummaryIndexAdapter.vectorize_summary(
record,
snapshot.segment,
snapshot.dataset,
new_session=self._new_session,
vector_factory=lambda: self._vector(snapshot),
)
def _update_keywords(
self,
segment_ref: SegmentRef,
snapshot: SegmentIndexingSnapshot,
keywords: Sequence[str] | None,
*,
replace_existing: bool,
) -> None:
assert snapshot.segment.index_node_id is not None
self._write_keywords(
segment_ref.document, snapshot, [self._document(snapshot)], [keywords], replace_existing=replace_existing
)
def _write_keywords(
self,
document_ref: DocumentRef,
snapshot: SegmentIndexingSnapshot,
documents: list[Document],
keywords_list: Sequence[Sequence[str] | None] | None,
*,
replace_existing: bool,
) -> None:
dataset_ref = document_ref.dataset
def persist(storage_type: str, data: str, selected: Mapping[str, Sequence[str]]) -> None:
self._segments.save_keyword_table(dataset_ref, storage_type=storage_type, data=data, keywords=selected)
Jieba(snapshot.dataset).update_texts(
documents,
read=lambda: self._segments.get_keyword_table(dataset_ref),
write=persist,
lock=self._redis.lock(f"keyword_indexing_lock_{snapshot.dataset.id}", timeout=600),
keywords_list=keywords_list,
replace_existing=replace_existing,
)
def delete(
self, document_ref: DocumentRef, segments: Sequence[SegmentIndexTarget], children: Sequence[ChildChunkState]
) -> None:
self._delete_task(
[segment.index_node_id for segment in segments],
document_ref.dataset.dataset_id,
document_ref.document_id,
[segment.id for segment in segments],
[child.index_node_id for child in children if child.index_node_id],
)
def change_status(self, document_ref: DocumentRef, segment_ids: Sequence[str], action: str) -> None:
task = self._enable_task if action == "enable" else self._disable_task
task(list(segment_ids), document_ref.dataset.dataset_id, document_ref.document_id)
Whitespace-only changes.
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,82 @@
"""Summary vectorization policy independent of ORM, model providers, and vector stores."""
import logging
from collections.abc import Callable
from dataclasses import dataclass
from typing import Protocol
from services.knowledge.resource_scope import SegmentRef
logger = logging.getLogger(__name__)
@dataclass(frozen=True)
class SummaryVectorInput:
ref: SegmentRef
summary_id: str
content: str
previous_node_id: str | None
@dataclass(frozen=True)
class SummaryVectorResult:
node_id: str
content_hash: str
tokens: int
class SummaryStore(Protocol):
def completed(self, command: SummaryVectorInput, result: SummaryVectorResult) -> None: ...
def failed(self, command: SummaryVectorInput, error: str) -> None: ...
class SummaryVectorBackend(Protocol):
def delete(self, node_id: str) -> None: ...
def count_tokens(self, content: str) -> int: ...
def write(self, command: SummaryVectorInput, result: SummaryVectorResult) -> None: ...
class SummaryIndexService:
def __init__(
self,
*,
store: SummaryStore,
vectors: SummaryVectorBackend,
new_id: Callable[[], str],
text_hash: Callable[[str], str],
sleep: Callable[[float], None],
) -> None:
self._store, self._vectors = store, vectors
self._new_id, self._hash, self._sleep = new_id, text_hash, sleep
def vectorize(self, command: SummaryVectorInput) -> SummaryVectorResult:
if not command.content.strip():
raise ValueError(f"Summary content is empty for segment {command.ref.segment_id}, cannot vectorize")
node_id = command.previous_node_id or self._new_id()
if command.previous_node_id:
try:
self._vectors.delete(command.previous_node_id)
except Exception:
logger.warning("Failed to delete previous summary vector for %s", command.ref.segment_id, exc_info=True)
tokens = 0
try:
tokens = self._vectors.count_tokens(command.content)
except Exception:
logger.warning("Failed to calculate summary embedding tokens", exc_info=True)
result = SummaryVectorResult(node_id, self._hash(command.content), tokens)
for attempt in range(3):
try:
self._vectors.write(command, result)
self._store.completed(command, result)
return result
except Exception as error:
transient = any(
word in str(error).lower()
for word in ("connection", "disconnected", "timeout", "network", "weaviate")
)
if transient and attempt < 2:
self._sleep(2.0 * 2**attempt)
continue
self._store.failed(command, f"Vectorization failed: {error}")
raise
raise AssertionError("Unreachable vectorization retry state")