chore: 纳入本地 Dify 1.19.0 源码及品牌与免登改动
演示目录只保留可重建的源码。运行密钥、数据卷、依赖和带口令的本地配置不进入仓库。 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
60158935f4
commit
6519682885
15831 files changed
+3063428
No files matched your search
@@ -0,0 +1 @@
|
||||
"""Dataset administration use cases and infrastructure adapters."""
|
||||
@@ -0,0 +1,344 @@
|
||||
"""Session-owning dataset administration adapters."""
|
||||
|
||||
from collections.abc import Generator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from configs import dify_config
|
||||
from core.plugin.impl.model_runtime_factory import create_plugin_provider_manager
|
||||
from graphon.model_runtime.entities.model_entities import ModelType
|
||||
from libs.pagination import clamp_pagination
|
||||
from machinery.context import RequestContext
|
||||
from models import Account, Dataset, Document
|
||||
from models.dataset import DatasetPermission, DatasetPermissionEnum
|
||||
from models.provider_ids import ModelProviderID
|
||||
from repositories.knowledge.dataset_repository import _get_dataset
|
||||
from services.app.query_service import RelatedApps
|
||||
from services.dataset_knowledge_fs_upgrade_service import KnowledgeFSUpgradeSnapshotService, upgrade_discovery_response
|
||||
from services.enterprise import rbac_service
|
||||
from services.errors.base import NoPermissionError
|
||||
from services.knowledge.dataset_access import DatasetAccessDeniedError, DatasetNotFoundError
|
||||
from services.knowledge.dataset_read_service import get_dataset_queries, load_dataset_detail, load_dataset_details
|
||||
from services.knowledge.dataset_service import DatasetPermissionService, DatasetService, DocumentService
|
||||
from services.knowledge.datasets.application import (
|
||||
DatasetListFilter,
|
||||
DatasetVisibility,
|
||||
)
|
||||
from services.knowledge.entities.datasets import DatasetDetailRecord, DatasetPage
|
||||
from services.knowledge.resource_scope import DatasetRef
|
||||
from services.knowledge_fs.feature_service import get_knowledge_fs_features
|
||||
from services.rbac import contracts as rbac_contracts
|
||||
from services.rbac.members import MemberService
|
||||
from services.tag_application_service import TagTargetQuery
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _translate_permissions() -> Generator[None]:
|
||||
try:
|
||||
yield
|
||||
except NoPermissionError as error:
|
||||
raise DatasetAccessDeniedError() from error
|
||||
|
||||
|
||||
def require_dataset(session: Session, ref: DatasetRef) -> Dataset:
|
||||
dataset = _get_dataset(session, ref)
|
||||
if dataset is None:
|
||||
raise DatasetNotFoundError()
|
||||
return dataset
|
||||
|
||||
|
||||
def load_actor(session: Session, context: RequestContext) -> Account:
|
||||
account = session.get(Account, context.account_id)
|
||||
if account is None:
|
||||
raise DatasetAccessDeniedError()
|
||||
account.set_tenant_id_with_session(context.active_workspace_id, session=session)
|
||||
return account
|
||||
|
||||
|
||||
def _normalize_provider(item: DatasetDetailRecord) -> DatasetDetailRecord:
|
||||
if item["indexing_technique"] == "high_quality" and item["embedding_model_provider"]:
|
||||
item["embedding_model_provider"] = str(ModelProviderID(item["embedding_model_provider"]))
|
||||
return item
|
||||
|
||||
|
||||
def _status(document: Document, counts: tuple[int, int] | None = None) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {
|
||||
"id": document.id,
|
||||
"indexing_status": document.indexing_status,
|
||||
"processing_started_at": document.processing_started_at,
|
||||
"parsing_completed_at": document.parsing_completed_at,
|
||||
"cleaning_completed_at": document.cleaning_completed_at,
|
||||
"splitting_completed_at": document.splitting_completed_at,
|
||||
"completed_at": document.completed_at,
|
||||
"paused_at": document.paused_at,
|
||||
"error": document.error,
|
||||
"stopped_at": document.stopped_at,
|
||||
}
|
||||
if counts is not None:
|
||||
result.update(completed_segments=counts[0], total_segments=counts[1])
|
||||
return result
|
||||
|
||||
|
||||
class SQLAlchemyDatasetOperations:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
session_factory: sessionmaker[Session],
|
||||
tags: TagTargetQuery,
|
||||
app_queries: RelatedApps,
|
||||
members: MemberService,
|
||||
) -> None:
|
||||
self._sessions = session_factory
|
||||
self._tags = tags
|
||||
self._app_queries = app_queries
|
||||
self._members_service = members
|
||||
|
||||
def visibility(self, context: RequestContext) -> DatasetVisibility:
|
||||
permissions = self._members_service.permissions(context.active_workspace_id, context.account_id)
|
||||
scope = (
|
||||
rbac_service.RBACService.DatasetAccess.whitelist_resources(context.active_workspace_id, context.account_id)
|
||||
if dify_config.RBAC_ENABLED
|
||||
else None
|
||||
)
|
||||
return DatasetVisibility(
|
||||
default_permissions=permissions.dataset.default_permission_keys,
|
||||
workspace_permissions=permissions.workspace.permission_keys,
|
||||
overrides={item.resource_id: item.permission_keys for item in permissions.dataset.overrides},
|
||||
unrestricted=scope is None or scope.unrestricted,
|
||||
whitelist_ids=scope.resource_ids if scope is not None else (),
|
||||
)
|
||||
|
||||
def embedding_models(self, workspace_id: str) -> set[str]:
|
||||
manager = create_plugin_provider_manager(tenant_id=workspace_id)
|
||||
models = manager.get_configurations(tenant_id=workspace_id).get_models(
|
||||
model_type=ModelType.TEXT_EMBEDDING, only_active=True
|
||||
)
|
||||
return {f"{model.model}:{model.provider.provider}" for model in models}
|
||||
|
||||
def list_datasets(
|
||||
self, context: RequestContext, query: DatasetListFilter, accessible_ids: list[str] | None
|
||||
) -> DatasetPage:
|
||||
page, limit = clamp_pagination(query.page, query.limit, 100)
|
||||
with self._sessions() as session:
|
||||
account = load_actor(session, context)
|
||||
if query.ids:
|
||||
datasets, total = DatasetService.get_datasets_by_ids(
|
||||
list(query.ids),
|
||||
context.active_workspace_id,
|
||||
user=account,
|
||||
accessible_dataset_ids=accessible_ids,
|
||||
session=session,
|
||||
)
|
||||
else:
|
||||
datasets, total = DatasetService.get_datasets(
|
||||
page,
|
||||
limit,
|
||||
session,
|
||||
context.active_workspace_id,
|
||||
account,
|
||||
query.keyword,
|
||||
list(query.tag_ids),
|
||||
query.include_all,
|
||||
creator_ids=list(query.creator_ids),
|
||||
tags=self._tags,
|
||||
accessible_dataset_ids=accessible_ids,
|
||||
)
|
||||
data = load_dataset_details(datasets, session=session)
|
||||
partial_ids = [item["id"] for item in data if item["permission"] == "partial_members"]
|
||||
members: dict[str, list[str]] = {}
|
||||
if partial_ids:
|
||||
for dataset_id, account_id in session.execute(
|
||||
select(DatasetPermission.dataset_id, DatasetPermission.account_id).where(
|
||||
DatasetPermission.dataset_id.in_(partial_ids),
|
||||
DatasetPermission.tenant_id == context.active_workspace_id,
|
||||
)
|
||||
):
|
||||
members.setdefault(dataset_id, []).append(account_id)
|
||||
upgrade_jobs = KnowledgeFSUpgradeSnapshotService(self._sessions).get_latest_by_dataset_ids(
|
||||
tenant_id=context.active_workspace_id,
|
||||
dataset_ids=[item["id"] for item in data],
|
||||
session=session,
|
||||
)
|
||||
knowledge_fs_enabled = get_knowledge_fs_features().enabled
|
||||
for item in data:
|
||||
_normalize_provider(item)
|
||||
item["partial_member_list"] = members.get(item["id"], [])
|
||||
item["knowledge_fs_upgrade"] = upgrade_discovery_response(
|
||||
upgrade_jobs.get(item["id"]),
|
||||
feature_enabled=knowledge_fs_enabled,
|
||||
dataset_provider=str(item["provider"]),
|
||||
).model_dump(mode="json")
|
||||
return {
|
||||
"data": data,
|
||||
"has_more": not query.ids and page * limit < total,
|
||||
"total": total,
|
||||
"page": page,
|
||||
"limit": limit,
|
||||
}
|
||||
|
||||
def _detail(self, session: Session, context: RequestContext, dataset: Dataset) -> DatasetDetailRecord:
|
||||
result = load_dataset_detail(dataset, session=session)
|
||||
permissions = self._members_service.resource_permissions(
|
||||
context.active_workspace_id, context.account_id, rbac_contracts.RBACResourceType.DATASET, [dataset.id]
|
||||
)
|
||||
result["permission_keys"] = permissions.get(dataset.id, [])
|
||||
return result
|
||||
|
||||
def get_dataset(self, context: RequestContext, ref: DatasetRef) -> DatasetDetailRecord:
|
||||
with self._sessions() as session:
|
||||
dataset = require_dataset(session, ref)
|
||||
result = load_dataset_detail(dataset, session=session)
|
||||
permissions = self._members_service.permissions(
|
||||
context.active_workspace_id, context.account_id, dataset_id=ref.dataset_id
|
||||
)
|
||||
result["permission_keys"] = permissions.dataset.permission_keys_by_resource_ids([dataset.id]).get(
|
||||
dataset.id, []
|
||||
)
|
||||
if dataset.permission == DatasetPermissionEnum.PARTIAL_TEAM:
|
||||
result["partial_member_list"] = self._members(session, ref)
|
||||
return _normalize_provider(result)
|
||||
|
||||
def create_dataset(self, context: RequestContext, values: Mapping[str, Any]) -> DatasetDetailRecord:
|
||||
with _translate_permissions(), self._sessions() as session:
|
||||
dataset = DatasetService.create_empty_dataset(
|
||||
session=session,
|
||||
tenant_id=context.active_workspace_id,
|
||||
account=load_actor(session, context),
|
||||
**dict(values),
|
||||
)
|
||||
result = self._detail(session, context, dataset)
|
||||
result["partial_member_list"] = None
|
||||
dataset_id = dataset.id
|
||||
session.commit()
|
||||
if dify_config.RBAC_ENABLED:
|
||||
rbac_service.RBACService.DatasetAccess.replace_whitelist(
|
||||
context.active_workspace_id,
|
||||
context.account_id,
|
||||
dataset_id,
|
||||
rbac_contracts.ReplaceMemberBindings(automatic_include_workspace_members=False),
|
||||
)
|
||||
return result
|
||||
|
||||
def update_dataset(
|
||||
self, context: RequestContext, ref: DatasetRef, values: Mapping[str, Any]
|
||||
) -> DatasetDetailRecord:
|
||||
data = dict(values)
|
||||
if (
|
||||
data.get("indexing_technique") == "high_quality"
|
||||
and data.get("embedding_model_provider") is not None
|
||||
and data.get("embedding_model") is not None
|
||||
):
|
||||
data["is_multimodal"] = DatasetService.check_is_multimodal_model(
|
||||
ref.tenant_id, data["embedding_model_provider"], data["embedding_model"]
|
||||
)
|
||||
with _translate_permissions(), self._sessions() as session:
|
||||
dataset = require_dataset(session, ref)
|
||||
account = load_actor(session, context)
|
||||
if not dify_config.RBAC_ENABLED:
|
||||
DatasetPermissionService.check_permission(
|
||||
account, dataset, data.get("permission"), data.get("partial_member_list"), session=session
|
||||
)
|
||||
# The legacy updater consumes fields from its input; retain the
|
||||
# member selection for the permission update in this transaction.
|
||||
dataset = DatasetService.update_dataset(ref.dataset_id, dict(data), account, session=session)
|
||||
if dataset is None:
|
||||
raise DatasetNotFoundError()
|
||||
result = self._detail(session, context, dataset)
|
||||
if (
|
||||
data.get("partial_member_list") is not None
|
||||
and data.get("permission") == DatasetPermissionEnum.PARTIAL_TEAM
|
||||
):
|
||||
DatasetPermissionService.update_partial_member_list(
|
||||
ref.tenant_id, ref.dataset_id, data["partial_member_list"], session
|
||||
)
|
||||
elif data.get("permission") in {DatasetPermissionEnum.ONLY_ME, DatasetPermissionEnum.ALL_TEAM}:
|
||||
DatasetPermissionService.clear_partial_member_list(ref.dataset_id, session)
|
||||
result["partial_member_list"] = self._members(session, ref)
|
||||
session.commit()
|
||||
return result
|
||||
|
||||
def delete_dataset(self, context: RequestContext, ref: DatasetRef) -> None:
|
||||
with _translate_permissions(), self._sessions() as session:
|
||||
require_dataset(session, ref)
|
||||
if not DatasetService.delete_dataset(ref.dataset_id, load_actor(session, context), session):
|
||||
raise DatasetNotFoundError()
|
||||
DatasetPermissionService.clear_partial_member_list(ref.dataset_id, session)
|
||||
session.commit()
|
||||
|
||||
def is_in_use(self, ref: DatasetRef) -> bool:
|
||||
with self._sessions() as session:
|
||||
require_dataset(session, ref)
|
||||
return DatasetService.dataset_use_check(ref, session)
|
||||
|
||||
def queries(self, ref: DatasetRef, *, page: int, limit: int) -> dict[str, Any]:
|
||||
page, limit = clamp_pagination(page, limit, 100)
|
||||
with self._sessions() as session:
|
||||
require_dataset(session, ref)
|
||||
queries, total = DatasetService.get_dataset_queries(
|
||||
dataset_id=ref.dataset_id, page=page, per_page=limit, session=session
|
||||
)
|
||||
data = [
|
||||
{
|
||||
"id": query.id,
|
||||
"queries": get_dataset_queries(query, session=session),
|
||||
"source": query.source,
|
||||
"source_app_id": query.source_app_id,
|
||||
"created_by_role": query.created_by_role,
|
||||
"created_by": query.created_by,
|
||||
"created_at": query.created_at,
|
||||
}
|
||||
for query in queries
|
||||
]
|
||||
return {"data": data, "has_more": page * limit < total, "total": total, "page": page, "limit": limit}
|
||||
|
||||
def related_apps(self, ref: DatasetRef) -> dict[str, Any]:
|
||||
with self._sessions() as session:
|
||||
require_dataset(session, ref)
|
||||
joins = DatasetService.get_related_apps(ref.dataset_id, session)
|
||||
app_ids = [join.app_id for join in joins]
|
||||
apps = self._app_queries.related_apps(ref.tenant_id, app_ids)
|
||||
return {"data": apps, "total": len(apps)}
|
||||
|
||||
def indexing_status(self, ref: DatasetRef) -> dict[str, Any]:
|
||||
with self._sessions() as session:
|
||||
require_dataset(session, ref)
|
||||
documents = session.scalars(
|
||||
select(Document).where(Document.dataset_id == ref.dataset_id, Document.tenant_id == ref.tenant_id)
|
||||
).all()
|
||||
counts = DocumentService.get_document_segment_counts(documents, session=session)
|
||||
return {"data": [_status(document, counts.get(document.id, (0, 0))) for document in documents]}
|
||||
|
||||
def error_documents(self, ref: DatasetRef) -> dict[str, Any]:
|
||||
with self._sessions() as session:
|
||||
require_dataset(session, ref)
|
||||
documents = DocumentService.get_error_documents_by_dataset_ref(ref, session)
|
||||
return {"data": [_status(document) for document in documents], "total": len(documents)}
|
||||
|
||||
@staticmethod
|
||||
def _members(session: Session, ref: DatasetRef) -> list[str]:
|
||||
return list(
|
||||
session.scalars(
|
||||
select(DatasetPermission.account_id).where(
|
||||
DatasetPermission.dataset_id == ref.dataset_id, DatasetPermission.tenant_id == ref.tenant_id
|
||||
)
|
||||
).all()
|
||||
)
|
||||
|
||||
def partial_members(self, ref: DatasetRef) -> list[str]:
|
||||
with self._sessions() as session:
|
||||
require_dataset(session, ref)
|
||||
return self._members(session, ref)
|
||||
|
||||
def auto_disable_logs(self, ref: DatasetRef) -> dict[str, Any]:
|
||||
with self._sessions() as session:
|
||||
return dict(DatasetService.get_dataset_auto_disable_logs(ref, session))
|
||||
|
||||
def set_api_enabled(self, context: RequestContext, ref: DatasetRef, enabled: bool) -> None:
|
||||
with self._sessions() as session:
|
||||
DatasetService.update_dataset_api_status(
|
||||
require_dataset(session, ref), enabled, load_actor(session, context), session
|
||||
)
|
||||
session.commit()
|
||||
@@ -0,0 +1,168 @@
|
||||
"""Dataset administration policies independent of HTTP and persistence."""
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Protocol
|
||||
|
||||
from libs.url_utils import normalize_api_base_url
|
||||
from machinery.context import RequestContext
|
||||
from services.knowledge.dataset_access import DatasetAccess
|
||||
from services.knowledge.datasets.retrieval import retrieval_methods
|
||||
from services.knowledge.entities.datasets import DatasetDetailRecord, DatasetPage
|
||||
from services.knowledge.resource_scope import DatasetRef
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DatasetListFilter:
|
||||
page: int = 1
|
||||
limit: int = 20
|
||||
keyword: str | None = None
|
||||
include_all: bool = False
|
||||
ids: Sequence[str] = ()
|
||||
tag_ids: Sequence[str] = ()
|
||||
creator_ids: Sequence[str] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DatasetVisibility:
|
||||
default_permissions: Sequence[str] = ()
|
||||
workspace_permissions: Sequence[str] = ()
|
||||
overrides: Mapping[str, Sequence[str]] = field(default_factory=dict)
|
||||
unrestricted: bool = True
|
||||
whitelist_ids: Sequence[str] = ()
|
||||
|
||||
def list_scope(self, *, rbac_enabled: bool) -> list[str] | None:
|
||||
if not rbac_enabled:
|
||||
return None
|
||||
# Maintainers always retain visibility of their own datasets. The
|
||||
# whitelist takes precedence over grants only for other datasets.
|
||||
if not self.unrestricted:
|
||||
return sorted(set(self.whitelist_ids))
|
||||
read_keys = {"dataset.preview", "dataset.acl.preview", "dataset.full_access"}
|
||||
default_read = bool(
|
||||
read_keys.intersection(self.default_permissions) or read_keys.intersection(self.workspace_permissions)
|
||||
)
|
||||
ids = (
|
||||
None
|
||||
if default_read
|
||||
else sorted(key for key, grants in self.overrides.items() if read_keys.intersection(grants))
|
||||
)
|
||||
return ids
|
||||
|
||||
|
||||
class DatasetOperations(Protocol):
|
||||
"""Materialize owned values and complete writes before returning."""
|
||||
|
||||
def visibility(self, context: RequestContext) -> DatasetVisibility: ...
|
||||
def list_datasets(
|
||||
self, context: RequestContext, query: DatasetListFilter, accessible_ids: list[str] | None
|
||||
) -> DatasetPage: ...
|
||||
def embedding_models(self, workspace_id: str) -> set[str]: ...
|
||||
def get_dataset(self, context: RequestContext, ref: DatasetRef) -> DatasetDetailRecord: ...
|
||||
def create_dataset(self, context: RequestContext, values: Mapping[str, Any]) -> DatasetDetailRecord: ...
|
||||
def update_dataset(
|
||||
self, context: RequestContext, ref: DatasetRef, values: Mapping[str, Any]
|
||||
) -> DatasetDetailRecord: ...
|
||||
def delete_dataset(self, context: RequestContext, ref: DatasetRef) -> None: ...
|
||||
def is_in_use(self, ref: DatasetRef) -> bool: ...
|
||||
def queries(self, ref: DatasetRef, *, page: int, limit: int) -> dict[str, Any]: ...
|
||||
def related_apps(self, ref: DatasetRef) -> dict[str, Any]: ...
|
||||
def indexing_status(self, ref: DatasetRef) -> dict[str, Any]: ...
|
||||
def error_documents(self, ref: DatasetRef) -> dict[str, Any]: ...
|
||||
def partial_members(self, ref: DatasetRef) -> list[str]: ...
|
||||
def auto_disable_logs(self, ref: DatasetRef) -> dict[str, Any]: ...
|
||||
def set_api_enabled(self, context: RequestContext, ref: DatasetRef, enabled: bool) -> None: ...
|
||||
|
||||
|
||||
class DatasetApplicationService:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
dataset_access: DatasetAccess,
|
||||
operations: DatasetOperations,
|
||||
rbac_enabled: bool,
|
||||
service_api_url: str,
|
||||
vector_store: str | None,
|
||||
tidb_fulltext: bool,
|
||||
) -> None:
|
||||
self._dataset_access = dataset_access
|
||||
self._operations = operations
|
||||
self._rbac_enabled = rbac_enabled
|
||||
self._service_api_url = service_api_url
|
||||
self._vector_store = vector_store
|
||||
self._tidb_fulltext = tidb_fulltext
|
||||
|
||||
def _dataset(self, context: RequestContext, dataset_id: str) -> DatasetRef:
|
||||
dataset = self._dataset_access.require_accessible(context, dataset_id)
|
||||
return DatasetRef(dataset.workspace_id, dataset.id)
|
||||
|
||||
@staticmethod
|
||||
def _embedding_available(item: DatasetDetailRecord, models: set[str], *, listing: bool) -> None:
|
||||
high_quality = item["indexing_technique"] == "high_quality"
|
||||
item["embedding_available"] = (
|
||||
not high_quality
|
||||
or (listing and not item["embedding_model_provider"])
|
||||
or f"{item['embedding_model']}:{item['embedding_model_provider']}" in models
|
||||
)
|
||||
|
||||
def list_datasets(self, context: RequestContext, query: DatasetListFilter) -> DatasetPage:
|
||||
visibility = self._operations.visibility(context)
|
||||
ids = visibility.list_scope(rbac_enabled=self._rbac_enabled)
|
||||
result = self._operations.list_datasets(context, query, ids)
|
||||
models = self._operations.embedding_models(context.active_workspace_id)
|
||||
for item in result["data"]:
|
||||
item["permission_keys"] = list(visibility.overrides.get(item["id"], visibility.default_permissions))
|
||||
self._embedding_available(item, models, listing=True)
|
||||
return result
|
||||
|
||||
def get_dataset(self, context: RequestContext, *, dataset_id: str) -> DatasetDetailRecord:
|
||||
result = self._operations.get_dataset(context, self._dataset(context, dataset_id))
|
||||
self._embedding_available(result, self._operations.embedding_models(context.active_workspace_id), listing=False)
|
||||
return result
|
||||
|
||||
def create_dataset(self, context: RequestContext, *, values: Mapping[str, Any]) -> DatasetDetailRecord:
|
||||
settings = dict(values)
|
||||
settings["permission"] = "all_team_members" if self._rbac_enabled else settings.get("permission") or "only_me"
|
||||
return self._operations.create_dataset(context, settings)
|
||||
|
||||
def update_dataset(
|
||||
self, context: RequestContext, *, dataset_id: str, values: Mapping[str, Any]
|
||||
) -> DatasetDetailRecord:
|
||||
return self._operations.update_dataset(context, self._dataset(context, dataset_id), values)
|
||||
|
||||
def delete_dataset(self, context: RequestContext, *, dataset_id: str) -> None:
|
||||
self._operations.delete_dataset(context, self._dataset(context, dataset_id))
|
||||
|
||||
def is_in_use(self, context: RequestContext, *, dataset_id: str) -> bool:
|
||||
return self._operations.is_in_use(self._dataset(context, dataset_id))
|
||||
|
||||
def queries(self, context: RequestContext, *, dataset_id: str, page: int, limit: int) -> dict[str, Any]:
|
||||
return self._operations.queries(self._dataset(context, dataset_id), page=page, limit=limit)
|
||||
|
||||
def related_apps(self, context: RequestContext, *, dataset_id: str) -> dict[str, Any]:
|
||||
return self._operations.related_apps(self._dataset(context, dataset_id))
|
||||
|
||||
def indexing_status(self, context: RequestContext, *, dataset_id: str) -> dict[str, Any]:
|
||||
return self._operations.indexing_status(self._dataset(context, dataset_id))
|
||||
|
||||
def error_documents(self, context: RequestContext, *, dataset_id: str) -> dict[str, Any]:
|
||||
return self._operations.error_documents(self._dataset(context, dataset_id))
|
||||
|
||||
def partial_members(self, context: RequestContext, *, dataset_id: str) -> list[str]:
|
||||
return self._operations.partial_members(self._dataset(context, dataset_id))
|
||||
|
||||
def auto_disable_logs(self, context: RequestContext, *, dataset_id: str) -> dict[str, Any]:
|
||||
return self._operations.auto_disable_logs(self._dataset(context, dataset_id))
|
||||
|
||||
def set_api_enabled(self, context: RequestContext, *, dataset_id: str, status: str) -> None:
|
||||
self._operations.set_api_enabled(context, self._dataset(context, dataset_id), status == "enable")
|
||||
|
||||
def api_base_url(self, context: RequestContext, *, request_base_url: str) -> str:
|
||||
return normalize_api_base_url(self._service_api_url or request_base_url)
|
||||
|
||||
def retrieval_settings(
|
||||
self, context: RequestContext, *, vector_type: str | None = None, is_mock: bool = False
|
||||
) -> dict[str, list[str]]:
|
||||
return retrieval_methods(
|
||||
vector_type if is_mock else self._vector_store, is_mock=is_mock, tidb_fulltext=self._tidb_fulltext
|
||||
)
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Supported dataset retrieval methods for each vector store."""
|
||||
|
||||
from core.rag.datasource.vdb.vector_type import VectorType
|
||||
from core.rag.retrieval.retrieval_methods import RetrievalMethod
|
||||
|
||||
|
||||
def retrieval_methods(
|
||||
vector_type: str | None, *, is_mock: bool = False, tidb_fulltext: bool = False
|
||||
) -> dict[str, list[str]]:
|
||||
"""
|
||||
Get supported retrieval methods based on vector database type.
|
||||
|
||||
Args:
|
||||
vector_type: Vector database type, can be None
|
||||
is_mock: Whether this is a Mock API, affects MILVUS handling
|
||||
tidb_fulltext: Whether TiDB full-text retrieval is enabled
|
||||
|
||||
Returns:
|
||||
Dictionary containing supported retrieval methods
|
||||
|
||||
Raises:
|
||||
ValueError: If vector_type is None or unsupported
|
||||
"""
|
||||
if vector_type is None:
|
||||
raise ValueError("Vector store type is not configured.")
|
||||
|
||||
# Define vector database types that only support semantic search
|
||||
semantic_only_types = {
|
||||
VectorType.RELYT,
|
||||
VectorType.CHROMA,
|
||||
VectorType.PGVECTO_RS,
|
||||
VectorType.VIKINGDB,
|
||||
VectorType.UPSTASH,
|
||||
}
|
||||
|
||||
# Define vector database types that support all retrieval methods
|
||||
full_search_types = {
|
||||
VectorType.QDRANT,
|
||||
VectorType.WEAVIATE,
|
||||
VectorType.OPENSEARCH,
|
||||
VectorType.ANALYTICDB,
|
||||
VectorType.MYSCALE,
|
||||
VectorType.ORACLE,
|
||||
VectorType.ELASTICSEARCH,
|
||||
VectorType.ELASTICSEARCH_JA,
|
||||
VectorType.PGVECTOR,
|
||||
VectorType.VASTBASE,
|
||||
VectorType.TIDB_ON_QDRANT,
|
||||
VectorType.LINDORM,
|
||||
VectorType.COUCHBASE,
|
||||
VectorType.OPENGAUSS,
|
||||
VectorType.OCEANBASE,
|
||||
VectorType.SEEKDB,
|
||||
VectorType.TABLESTORE,
|
||||
VectorType.HUAWEI_CLOUD,
|
||||
VectorType.TENCENT,
|
||||
VectorType.MATRIXONE,
|
||||
VectorType.CLICKZETTA,
|
||||
VectorType.BAIDU,
|
||||
VectorType.ALIBABACLOUD_MYSQL,
|
||||
VectorType.IRIS,
|
||||
VectorType.HOLOGRES,
|
||||
}
|
||||
|
||||
semantic_methods = {"retrieval_method": [RetrievalMethod.SEMANTIC_SEARCH.value]}
|
||||
full_methods = {
|
||||
"retrieval_method": [
|
||||
RetrievalMethod.SEMANTIC_SEARCH.value,
|
||||
RetrievalMethod.FULL_TEXT_SEARCH.value,
|
||||
RetrievalMethod.HYBRID_SEARCH.value,
|
||||
]
|
||||
}
|
||||
|
||||
if vector_type == VectorType.MILVUS:
|
||||
return semantic_methods if is_mock else full_methods
|
||||
|
||||
if vector_type == VectorType.TIDB_VECTOR:
|
||||
return full_methods if tidb_fulltext else semantic_methods
|
||||
|
||||
if vector_type in semantic_only_types:
|
||||
return semantic_methods
|
||||
elif vector_type in full_search_types:
|
||||
return full_methods
|
||||
else:
|
||||
raise ValueError(f"Unsupported vector db type {vector_type}.")
|
||||
@@ -0,0 +1,206 @@
|
||||
"""Classic-to-Agent knowledge base upgrade use cases behind the dataset console endpoints."""
|
||||
|
||||
from collections.abc import Callable
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from graphon.model_runtime.entities.model_entities import ModelType
|
||||
from machinery.context import RequestContext
|
||||
from models import Dataset
|
||||
from models.knowledge_fs import KnowledgeFSUpgradeJob, KnowledgeFSUpgradeJobStatus
|
||||
from models.provider import TenantDefaultModel
|
||||
from services.dataset_knowledge_fs_upgrade_service import (
|
||||
KnowledgeFSUpgradeConflictError,
|
||||
KnowledgeFSUpgradeModelPreferences,
|
||||
KnowledgeFSUpgradeModelResolver,
|
||||
KnowledgeFSUpgradeNotFoundError,
|
||||
KnowledgeFSUpgradeSnapshotService,
|
||||
upgrade_discovery_response,
|
||||
upgrade_job_matches_model_choices,
|
||||
)
|
||||
from services.knowledge.dataset_access import DatasetAccess, DatasetAccessError
|
||||
from services.knowledge.dataset_service import DatasetService
|
||||
from services.knowledge.datasets.adapters import SQLAlchemyDatasetOperations, load_actor
|
||||
from services.knowledge_fs.product_dto import (
|
||||
KnowledgeFSUpgradeModelChoice,
|
||||
KnowledgeFSUpgradeModelsPayload,
|
||||
KnowledgeFSUpgradePreparationResponse,
|
||||
)
|
||||
|
||||
_RECOVERABLE_STATUSES = (
|
||||
KnowledgeFSUpgradeJobStatus.QUEUED,
|
||||
KnowledgeFSUpgradeJobStatus.RUNNING,
|
||||
KnowledgeFSUpgradeJobStatus.FAILED,
|
||||
)
|
||||
|
||||
|
||||
class KnowledgeFSUpgradeUnavailableError(Exception):
|
||||
"""The KnowledgeFS feature is disabled, so upgrades cannot start or retry."""
|
||||
|
||||
|
||||
class KnowledgeUpgradeApplicationService:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
session_factory: sessionmaker[Session],
|
||||
dataset_access: DatasetAccess,
|
||||
datasets: SQLAlchemyDatasetOperations,
|
||||
rbac_enabled: bool,
|
||||
feature_enabled: Callable[[], bool],
|
||||
enqueue: Callable[[KnowledgeFSUpgradeSnapshotService, str, str], None] | None = None,
|
||||
) -> None:
|
||||
self._sessions = session_factory
|
||||
self._dataset_access = dataset_access
|
||||
self._datasets = datasets
|
||||
self._rbac_enabled = rbac_enabled
|
||||
self._feature_enabled = feature_enabled
|
||||
self._enqueue = enqueue or enqueue_upgrade_job
|
||||
|
||||
def _snapshots(self) -> KnowledgeFSUpgradeSnapshotService:
|
||||
return KnowledgeFSUpgradeSnapshotService(self._sessions)
|
||||
|
||||
@staticmethod
|
||||
def _choice(provider: str | None, model: str | None) -> KnowledgeFSUpgradeModelChoice | None:
|
||||
if not provider or not model:
|
||||
return None
|
||||
return KnowledgeFSUpgradeModelChoice(provider=provider, model=model)
|
||||
|
||||
def _load_model_preferences(
|
||||
self, *, tenant_id: str, dataset_id: str
|
||||
) -> tuple[str, KnowledgeFSUpgradeModelPreferences]:
|
||||
"""Read the legacy settings and saved defaults in a short, tenant-scoped session."""
|
||||
with self._sessions() as session:
|
||||
dataset = session.scalar(select(Dataset).where(Dataset.id == dataset_id, Dataset.tenant_id == tenant_id))
|
||||
if dataset is None:
|
||||
raise KnowledgeFSUpgradeNotFoundError("Dataset was not found")
|
||||
defaults: dict[ModelType, KnowledgeFSUpgradeModelChoice] = {}
|
||||
for row in session.scalars(
|
||||
select(TenantDefaultModel).where(
|
||||
TenantDefaultModel.tenant_id == tenant_id,
|
||||
TenantDefaultModel.model_type.in_((ModelType.TEXT_EMBEDDING, ModelType.RERANK, ModelType.LLM)),
|
||||
)
|
||||
):
|
||||
choice = self._choice(row.provider_name, row.model_name)
|
||||
if choice is not None:
|
||||
defaults[row.model_type] = choice
|
||||
retrieval = dataset.retrieval_model if isinstance(dataset.retrieval_model, dict) else {}
|
||||
reranking = retrieval.get("reranking_model") or {}
|
||||
if not isinstance(reranking, dict):
|
||||
reranking = {}
|
||||
preferences = KnowledgeFSUpgradeModelPreferences(
|
||||
embedding=self._choice(dataset.embedding_model_provider, dataset.embedding_model),
|
||||
rerank=self._choice(reranking.get("reranking_provider_name"), reranking.get("reranking_model_name")),
|
||||
defaults=defaults,
|
||||
)
|
||||
return str(dataset.provider or ""), preferences
|
||||
|
||||
def list_recoverable_jobs(self, context: RequestContext) -> list[KnowledgeFSUpgradeJob]:
|
||||
workspace_id = context.active_workspace_id
|
||||
jobs = self._snapshots().list_by_statuses(tenant_id=workspace_id, statuses=_RECOVERABLE_STATUSES)
|
||||
dataset_ids = list(dict.fromkeys(job.old_dataset_id for job in jobs))
|
||||
if not dataset_ids:
|
||||
return []
|
||||
accessible_ids = self._datasets.visibility(context).list_scope(rbac_enabled=self._rbac_enabled)
|
||||
with self._sessions() as session:
|
||||
datasets, _ = DatasetService.get_datasets_by_ids(
|
||||
dataset_ids,
|
||||
workspace_id,
|
||||
user=load_actor(session, context),
|
||||
accessible_dataset_ids=accessible_ids,
|
||||
session=session,
|
||||
)
|
||||
visible_ids = [str(dataset.id) for dataset in datasets]
|
||||
allowed_ids: set[str] = set()
|
||||
for dataset_id in visible_ids:
|
||||
try:
|
||||
self._dataset_access.require_accessible(context, dataset_id)
|
||||
except DatasetAccessError:
|
||||
continue
|
||||
allowed_ids.add(dataset_id)
|
||||
return [job for job in jobs if job.old_dataset_id in allowed_ids]
|
||||
|
||||
def discovery(self, context: RequestContext, *, dataset_id: str) -> KnowledgeFSUpgradePreparationResponse:
|
||||
dataset = self._dataset_access.require_accessible(context, dataset_id)
|
||||
with self._sessions() as session:
|
||||
provider = session.scalar(
|
||||
select(Dataset.provider).where(Dataset.id == dataset.id, Dataset.tenant_id == dataset.workspace_id)
|
||||
)
|
||||
job = self._snapshots().get_latest(tenant_id=dataset.workspace_id, dataset_id=dataset.id)
|
||||
eligibility = upgrade_discovery_response(
|
||||
job, feature_enabled=self._feature_enabled(), dataset_provider=str(provider or "")
|
||||
)
|
||||
recommendations = None
|
||||
if eligibility.can_upgrade:
|
||||
_, preferences = self._load_model_preferences(tenant_id=dataset.workspace_id, dataset_id=dataset.id)
|
||||
recommendations = KnowledgeFSUpgradeModelResolver(tenant_id=dataset.workspace_id).recommend(preferences)
|
||||
return KnowledgeFSUpgradePreparationResponse.model_validate(
|
||||
{**eligibility.model_dump(mode="json"), "models": recommendations}
|
||||
)
|
||||
|
||||
def start(
|
||||
self,
|
||||
context: RequestContext,
|
||||
*,
|
||||
dataset_id: str,
|
||||
idempotency_key: str,
|
||||
models: KnowledgeFSUpgradeModelsPayload | None = None,
|
||||
) -> KnowledgeFSUpgradeJob:
|
||||
if not self._feature_enabled():
|
||||
raise KnowledgeFSUpgradeUnavailableError()
|
||||
dataset = self._dataset_access.require_accessible(context, dataset_id)
|
||||
snapshots = self._snapshots()
|
||||
idempotent = snapshots.get_by_idempotency_key(tenant_id=dataset.workspace_id, idempotency_key=idempotency_key)
|
||||
if idempotent is not None:
|
||||
if idempotent.old_dataset_id != dataset.id:
|
||||
raise KnowledgeFSUpgradeConflictError("Idempotency key belongs to another Dataset upgrade")
|
||||
if models is not None and not upgrade_job_matches_model_choices(idempotent, models):
|
||||
raise KnowledgeFSUpgradeConflictError("Idempotency key belongs to another model selection")
|
||||
self._enqueue(snapshots, dataset.workspace_id, idempotent.id)
|
||||
return idempotent
|
||||
provider, preferences = self._load_model_preferences(tenant_id=dataset.workspace_id, dataset_id=dataset.id)
|
||||
if provider == "external":
|
||||
raise KnowledgeFSUpgradeConflictError("External-provider Datasets cannot be upgraded")
|
||||
resolver = KnowledgeFSUpgradeModelResolver(tenant_id=dataset.workspace_id)
|
||||
resolved_models = (
|
||||
resolver.validate(models) if models is not None else resolver.resolve_recommendations(preferences)
|
||||
)
|
||||
job = snapshots.create(
|
||||
tenant_id=dataset.workspace_id,
|
||||
dataset_id=dataset.id,
|
||||
requested_by_account_id=context.account_id,
|
||||
idempotency_key=idempotency_key,
|
||||
models=resolved_models,
|
||||
)
|
||||
self._enqueue(snapshots, dataset.workspace_id, job.id)
|
||||
return job
|
||||
|
||||
def get_job(self, context: RequestContext, *, dataset_id: str, job_id: str) -> KnowledgeFSUpgradeJob:
|
||||
dataset = self._dataset_access.require_accessible(context, dataset_id)
|
||||
job = self._snapshots().get(tenant_id=dataset.workspace_id, job_id=job_id)
|
||||
if job.old_dataset_id != dataset.id:
|
||||
raise KnowledgeFSUpgradeNotFoundError("Upgrade job was not found")
|
||||
return job
|
||||
|
||||
def retry(self, context: RequestContext, *, dataset_id: str, job_id: str) -> KnowledgeFSUpgradeJob:
|
||||
if not self._feature_enabled():
|
||||
raise KnowledgeFSUpgradeUnavailableError()
|
||||
dataset = self._dataset_access.require_accessible(context, dataset_id)
|
||||
snapshots = self._snapshots()
|
||||
job = snapshots.retry(tenant_id=dataset.workspace_id, dataset_id=dataset.id, job_id=job_id)
|
||||
self._enqueue(snapshots, dataset.workspace_id, job.id)
|
||||
return job
|
||||
|
||||
|
||||
def enqueue_upgrade_job(snapshots: KnowledgeFSUpgradeSnapshotService, tenant_id: str, job_id: str) -> None:
|
||||
from tasks.knowledge_fs_upgrade_tasks import run_knowledge_fs_upgrade
|
||||
|
||||
task_id = str(uuid4())
|
||||
if not snapshots.claim_enqueue(tenant_id=tenant_id, job_id=job_id, task_id=task_id):
|
||||
return
|
||||
try:
|
||||
run_knowledge_fs_upgrade.apply_async(kwargs={"job_id": job_id}, task_id=task_id)
|
||||
except Exception:
|
||||
snapshots.release_enqueue_claim(tenant_id=tenant_id, job_id=job_id, task_id=task_id)
|
||||
raise
|
||||
Reference in new issue
Block a user