chore: 纳入本地 Dify 1.19.0 源码及品牌与免登改动

演示目录只保留可重建的源码。运行密钥、数据卷、依赖和带口令的本地配置不进入仓库。

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
wuzilongandCursor committed 2026-11-18 13:59:25 +08:00
1 parent 60158935f4
commit 6519682885
15831 files changed
+3063428

No files matched your search

@@ -0,0 +1 @@
"""Dataset administration use cases and infrastructure adapters."""
@@ -0,0 +1,344 @@
"""Session-owning dataset administration adapters."""
from collections.abc import Generator, Mapping
from contextlib import contextmanager
from typing import Any
from sqlalchemy import select
from sqlalchemy.orm import Session, sessionmaker
from configs import dify_config
from core.plugin.impl.model_runtime_factory import create_plugin_provider_manager
from graphon.model_runtime.entities.model_entities import ModelType
from libs.pagination import clamp_pagination
from machinery.context import RequestContext
from models import Account, Dataset, Document
from models.dataset import DatasetPermission, DatasetPermissionEnum
from models.provider_ids import ModelProviderID
from repositories.knowledge.dataset_repository import _get_dataset
from services.app.query_service import RelatedApps
from services.dataset_knowledge_fs_upgrade_service import KnowledgeFSUpgradeSnapshotService, upgrade_discovery_response
from services.enterprise import rbac_service
from services.errors.base import NoPermissionError
from services.knowledge.dataset_access import DatasetAccessDeniedError, DatasetNotFoundError
from services.knowledge.dataset_read_service import get_dataset_queries, load_dataset_detail, load_dataset_details
from services.knowledge.dataset_service import DatasetPermissionService, DatasetService, DocumentService
from services.knowledge.datasets.application import (
DatasetListFilter,
DatasetVisibility,
)
from services.knowledge.entities.datasets import DatasetDetailRecord, DatasetPage
from services.knowledge.resource_scope import DatasetRef
from services.knowledge_fs.feature_service import get_knowledge_fs_features
from services.rbac import contracts as rbac_contracts
from services.rbac.members import MemberService
from services.tag_application_service import TagTargetQuery
@contextmanager
def _translate_permissions() -> Generator[None]:
try:
yield
except NoPermissionError as error:
raise DatasetAccessDeniedError() from error
def require_dataset(session: Session, ref: DatasetRef) -> Dataset:
dataset = _get_dataset(session, ref)
if dataset is None:
raise DatasetNotFoundError()
return dataset
def load_actor(session: Session, context: RequestContext) -> Account:
account = session.get(Account, context.account_id)
if account is None:
raise DatasetAccessDeniedError()
account.set_tenant_id_with_session(context.active_workspace_id, session=session)
return account
def _normalize_provider(item: DatasetDetailRecord) -> DatasetDetailRecord:
if item["indexing_technique"] == "high_quality" and item["embedding_model_provider"]:
item["embedding_model_provider"] = str(ModelProviderID(item["embedding_model_provider"]))
return item
def _status(document: Document, counts: tuple[int, int] | None = None) -> dict[str, Any]:
result: dict[str, Any] = {
"id": document.id,
"indexing_status": document.indexing_status,
"processing_started_at": document.processing_started_at,
"parsing_completed_at": document.parsing_completed_at,
"cleaning_completed_at": document.cleaning_completed_at,
"splitting_completed_at": document.splitting_completed_at,
"completed_at": document.completed_at,
"paused_at": document.paused_at,
"error": document.error,
"stopped_at": document.stopped_at,
}
if counts is not None:
result.update(completed_segments=counts[0], total_segments=counts[1])
return result
class SQLAlchemyDatasetOperations:
def __init__(
self,
*,
session_factory: sessionmaker[Session],
tags: TagTargetQuery,
app_queries: RelatedApps,
members: MemberService,
) -> None:
self._sessions = session_factory
self._tags = tags
self._app_queries = app_queries
self._members_service = members
def visibility(self, context: RequestContext) -> DatasetVisibility:
permissions = self._members_service.permissions(context.active_workspace_id, context.account_id)
scope = (
rbac_service.RBACService.DatasetAccess.whitelist_resources(context.active_workspace_id, context.account_id)
if dify_config.RBAC_ENABLED
else None
)
return DatasetVisibility(
default_permissions=permissions.dataset.default_permission_keys,
workspace_permissions=permissions.workspace.permission_keys,
overrides={item.resource_id: item.permission_keys for item in permissions.dataset.overrides},
unrestricted=scope is None or scope.unrestricted,
whitelist_ids=scope.resource_ids if scope is not None else (),
)
def embedding_models(self, workspace_id: str) -> set[str]:
manager = create_plugin_provider_manager(tenant_id=workspace_id)
models = manager.get_configurations(tenant_id=workspace_id).get_models(
model_type=ModelType.TEXT_EMBEDDING, only_active=True
)
return {f"{model.model}:{model.provider.provider}" for model in models}
def list_datasets(
self, context: RequestContext, query: DatasetListFilter, accessible_ids: list[str] | None
) -> DatasetPage:
page, limit = clamp_pagination(query.page, query.limit, 100)
with self._sessions() as session:
account = load_actor(session, context)
if query.ids:
datasets, total = DatasetService.get_datasets_by_ids(
list(query.ids),
context.active_workspace_id,
user=account,
accessible_dataset_ids=accessible_ids,
session=session,
)
else:
datasets, total = DatasetService.get_datasets(
page,
limit,
session,
context.active_workspace_id,
account,
query.keyword,
list(query.tag_ids),
query.include_all,
creator_ids=list(query.creator_ids),
tags=self._tags,
accessible_dataset_ids=accessible_ids,
)
data = load_dataset_details(datasets, session=session)
partial_ids = [item["id"] for item in data if item["permission"] == "partial_members"]
members: dict[str, list[str]] = {}
if partial_ids:
for dataset_id, account_id in session.execute(
select(DatasetPermission.dataset_id, DatasetPermission.account_id).where(
DatasetPermission.dataset_id.in_(partial_ids),
DatasetPermission.tenant_id == context.active_workspace_id,
)
):
members.setdefault(dataset_id, []).append(account_id)
upgrade_jobs = KnowledgeFSUpgradeSnapshotService(self._sessions).get_latest_by_dataset_ids(
tenant_id=context.active_workspace_id,
dataset_ids=[item["id"] for item in data],
session=session,
)
knowledge_fs_enabled = get_knowledge_fs_features().enabled
for item in data:
_normalize_provider(item)
item["partial_member_list"] = members.get(item["id"], [])
item["knowledge_fs_upgrade"] = upgrade_discovery_response(
upgrade_jobs.get(item["id"]),
feature_enabled=knowledge_fs_enabled,
dataset_provider=str(item["provider"]),
).model_dump(mode="json")
return {
"data": data,
"has_more": not query.ids and page * limit < total,
"total": total,
"page": page,
"limit": limit,
}
def _detail(self, session: Session, context: RequestContext, dataset: Dataset) -> DatasetDetailRecord:
result = load_dataset_detail(dataset, session=session)
permissions = self._members_service.resource_permissions(
context.active_workspace_id, context.account_id, rbac_contracts.RBACResourceType.DATASET, [dataset.id]
)
result["permission_keys"] = permissions.get(dataset.id, [])
return result
def get_dataset(self, context: RequestContext, ref: DatasetRef) -> DatasetDetailRecord:
with self._sessions() as session:
dataset = require_dataset(session, ref)
result = load_dataset_detail(dataset, session=session)
permissions = self._members_service.permissions(
context.active_workspace_id, context.account_id, dataset_id=ref.dataset_id
)
result["permission_keys"] = permissions.dataset.permission_keys_by_resource_ids([dataset.id]).get(
dataset.id, []
)
if dataset.permission == DatasetPermissionEnum.PARTIAL_TEAM:
result["partial_member_list"] = self._members(session, ref)
return _normalize_provider(result)
def create_dataset(self, context: RequestContext, values: Mapping[str, Any]) -> DatasetDetailRecord:
with _translate_permissions(), self._sessions() as session:
dataset = DatasetService.create_empty_dataset(
session=session,
tenant_id=context.active_workspace_id,
account=load_actor(session, context),
**dict(values),
)
result = self._detail(session, context, dataset)
result["partial_member_list"] = None
dataset_id = dataset.id
session.commit()
if dify_config.RBAC_ENABLED:
rbac_service.RBACService.DatasetAccess.replace_whitelist(
context.active_workspace_id,
context.account_id,
dataset_id,
rbac_contracts.ReplaceMemberBindings(automatic_include_workspace_members=False),
)
return result
def update_dataset(
self, context: RequestContext, ref: DatasetRef, values: Mapping[str, Any]
) -> DatasetDetailRecord:
data = dict(values)
if (
data.get("indexing_technique") == "high_quality"
and data.get("embedding_model_provider") is not None
and data.get("embedding_model") is not None
):
data["is_multimodal"] = DatasetService.check_is_multimodal_model(
ref.tenant_id, data["embedding_model_provider"], data["embedding_model"]
)
with _translate_permissions(), self._sessions() as session:
dataset = require_dataset(session, ref)
account = load_actor(session, context)
if not dify_config.RBAC_ENABLED:
DatasetPermissionService.check_permission(
account, dataset, data.get("permission"), data.get("partial_member_list"), session=session
)
# The legacy updater consumes fields from its input; retain the
# member selection for the permission update in this transaction.
dataset = DatasetService.update_dataset(ref.dataset_id, dict(data), account, session=session)
if dataset is None:
raise DatasetNotFoundError()
result = self._detail(session, context, dataset)
if (
data.get("partial_member_list") is not None
and data.get("permission") == DatasetPermissionEnum.PARTIAL_TEAM
):
DatasetPermissionService.update_partial_member_list(
ref.tenant_id, ref.dataset_id, data["partial_member_list"], session
)
elif data.get("permission") in {DatasetPermissionEnum.ONLY_ME, DatasetPermissionEnum.ALL_TEAM}:
DatasetPermissionService.clear_partial_member_list(ref.dataset_id, session)
result["partial_member_list"] = self._members(session, ref)
session.commit()
return result
def delete_dataset(self, context: RequestContext, ref: DatasetRef) -> None:
with _translate_permissions(), self._sessions() as session:
require_dataset(session, ref)
if not DatasetService.delete_dataset(ref.dataset_id, load_actor(session, context), session):
raise DatasetNotFoundError()
DatasetPermissionService.clear_partial_member_list(ref.dataset_id, session)
session.commit()
def is_in_use(self, ref: DatasetRef) -> bool:
with self._sessions() as session:
require_dataset(session, ref)
return DatasetService.dataset_use_check(ref, session)
def queries(self, ref: DatasetRef, *, page: int, limit: int) -> dict[str, Any]:
page, limit = clamp_pagination(page, limit, 100)
with self._sessions() as session:
require_dataset(session, ref)
queries, total = DatasetService.get_dataset_queries(
dataset_id=ref.dataset_id, page=page, per_page=limit, session=session
)
data = [
{
"id": query.id,
"queries": get_dataset_queries(query, session=session),
"source": query.source,
"source_app_id": query.source_app_id,
"created_by_role": query.created_by_role,
"created_by": query.created_by,
"created_at": query.created_at,
}
for query in queries
]
return {"data": data, "has_more": page * limit < total, "total": total, "page": page, "limit": limit}
def related_apps(self, ref: DatasetRef) -> dict[str, Any]:
with self._sessions() as session:
require_dataset(session, ref)
joins = DatasetService.get_related_apps(ref.dataset_id, session)
app_ids = [join.app_id for join in joins]
apps = self._app_queries.related_apps(ref.tenant_id, app_ids)
return {"data": apps, "total": len(apps)}
def indexing_status(self, ref: DatasetRef) -> dict[str, Any]:
with self._sessions() as session:
require_dataset(session, ref)
documents = session.scalars(
select(Document).where(Document.dataset_id == ref.dataset_id, Document.tenant_id == ref.tenant_id)
).all()
counts = DocumentService.get_document_segment_counts(documents, session=session)
return {"data": [_status(document, counts.get(document.id, (0, 0))) for document in documents]}
def error_documents(self, ref: DatasetRef) -> dict[str, Any]:
with self._sessions() as session:
require_dataset(session, ref)
documents = DocumentService.get_error_documents_by_dataset_ref(ref, session)
return {"data": [_status(document) for document in documents], "total": len(documents)}
@staticmethod
def _members(session: Session, ref: DatasetRef) -> list[str]:
return list(
session.scalars(
select(DatasetPermission.account_id).where(
DatasetPermission.dataset_id == ref.dataset_id, DatasetPermission.tenant_id == ref.tenant_id
)
).all()
)
def partial_members(self, ref: DatasetRef) -> list[str]:
with self._sessions() as session:
require_dataset(session, ref)
return self._members(session, ref)
def auto_disable_logs(self, ref: DatasetRef) -> dict[str, Any]:
with self._sessions() as session:
return dict(DatasetService.get_dataset_auto_disable_logs(ref, session))
def set_api_enabled(self, context: RequestContext, ref: DatasetRef, enabled: bool) -> None:
with self._sessions() as session:
DatasetService.update_dataset_api_status(
require_dataset(session, ref), enabled, load_actor(session, context), session
)
session.commit()
@@ -0,0 +1,168 @@
"""Dataset administration policies independent of HTTP and persistence."""
from collections.abc import Mapping, Sequence
from dataclasses import dataclass, field
from typing import Any, Protocol
from libs.url_utils import normalize_api_base_url
from machinery.context import RequestContext
from services.knowledge.dataset_access import DatasetAccess
from services.knowledge.datasets.retrieval import retrieval_methods
from services.knowledge.entities.datasets import DatasetDetailRecord, DatasetPage
from services.knowledge.resource_scope import DatasetRef
@dataclass(frozen=True)
class DatasetListFilter:
page: int = 1
limit: int = 20
keyword: str | None = None
include_all: bool = False
ids: Sequence[str] = ()
tag_ids: Sequence[str] = ()
creator_ids: Sequence[str] = ()
@dataclass(frozen=True)
class DatasetVisibility:
default_permissions: Sequence[str] = ()
workspace_permissions: Sequence[str] = ()
overrides: Mapping[str, Sequence[str]] = field(default_factory=dict)
unrestricted: bool = True
whitelist_ids: Sequence[str] = ()
def list_scope(self, *, rbac_enabled: bool) -> list[str] | None:
if not rbac_enabled:
return None
# Maintainers always retain visibility of their own datasets. The
# whitelist takes precedence over grants only for other datasets.
if not self.unrestricted:
return sorted(set(self.whitelist_ids))
read_keys = {"dataset.preview", "dataset.acl.preview", "dataset.full_access"}
default_read = bool(
read_keys.intersection(self.default_permissions) or read_keys.intersection(self.workspace_permissions)
)
ids = (
None
if default_read
else sorted(key for key, grants in self.overrides.items() if read_keys.intersection(grants))
)
return ids
class DatasetOperations(Protocol):
"""Materialize owned values and complete writes before returning."""
def visibility(self, context: RequestContext) -> DatasetVisibility: ...
def list_datasets(
self, context: RequestContext, query: DatasetListFilter, accessible_ids: list[str] | None
) -> DatasetPage: ...
def embedding_models(self, workspace_id: str) -> set[str]: ...
def get_dataset(self, context: RequestContext, ref: DatasetRef) -> DatasetDetailRecord: ...
def create_dataset(self, context: RequestContext, values: Mapping[str, Any]) -> DatasetDetailRecord: ...
def update_dataset(
self, context: RequestContext, ref: DatasetRef, values: Mapping[str, Any]
) -> DatasetDetailRecord: ...
def delete_dataset(self, context: RequestContext, ref: DatasetRef) -> None: ...
def is_in_use(self, ref: DatasetRef) -> bool: ...
def queries(self, ref: DatasetRef, *, page: int, limit: int) -> dict[str, Any]: ...
def related_apps(self, ref: DatasetRef) -> dict[str, Any]: ...
def indexing_status(self, ref: DatasetRef) -> dict[str, Any]: ...
def error_documents(self, ref: DatasetRef) -> dict[str, Any]: ...
def partial_members(self, ref: DatasetRef) -> list[str]: ...
def auto_disable_logs(self, ref: DatasetRef) -> dict[str, Any]: ...
def set_api_enabled(self, context: RequestContext, ref: DatasetRef, enabled: bool) -> None: ...
class DatasetApplicationService:
def __init__(
self,
*,
dataset_access: DatasetAccess,
operations: DatasetOperations,
rbac_enabled: bool,
service_api_url: str,
vector_store: str | None,
tidb_fulltext: bool,
) -> None:
self._dataset_access = dataset_access
self._operations = operations
self._rbac_enabled = rbac_enabled
self._service_api_url = service_api_url
self._vector_store = vector_store
self._tidb_fulltext = tidb_fulltext
def _dataset(self, context: RequestContext, dataset_id: str) -> DatasetRef:
dataset = self._dataset_access.require_accessible(context, dataset_id)
return DatasetRef(dataset.workspace_id, dataset.id)
@staticmethod
def _embedding_available(item: DatasetDetailRecord, models: set[str], *, listing: bool) -> None:
high_quality = item["indexing_technique"] == "high_quality"
item["embedding_available"] = (
not high_quality
or (listing and not item["embedding_model_provider"])
or f"{item['embedding_model']}:{item['embedding_model_provider']}" in models
)
def list_datasets(self, context: RequestContext, query: DatasetListFilter) -> DatasetPage:
visibility = self._operations.visibility(context)
ids = visibility.list_scope(rbac_enabled=self._rbac_enabled)
result = self._operations.list_datasets(context, query, ids)
models = self._operations.embedding_models(context.active_workspace_id)
for item in result["data"]:
item["permission_keys"] = list(visibility.overrides.get(item["id"], visibility.default_permissions))
self._embedding_available(item, models, listing=True)
return result
def get_dataset(self, context: RequestContext, *, dataset_id: str) -> DatasetDetailRecord:
result = self._operations.get_dataset(context, self._dataset(context, dataset_id))
self._embedding_available(result, self._operations.embedding_models(context.active_workspace_id), listing=False)
return result
def create_dataset(self, context: RequestContext, *, values: Mapping[str, Any]) -> DatasetDetailRecord:
settings = dict(values)
settings["permission"] = "all_team_members" if self._rbac_enabled else settings.get("permission") or "only_me"
return self._operations.create_dataset(context, settings)
def update_dataset(
self, context: RequestContext, *, dataset_id: str, values: Mapping[str, Any]
) -> DatasetDetailRecord:
return self._operations.update_dataset(context, self._dataset(context, dataset_id), values)
def delete_dataset(self, context: RequestContext, *, dataset_id: str) -> None:
self._operations.delete_dataset(context, self._dataset(context, dataset_id))
def is_in_use(self, context: RequestContext, *, dataset_id: str) -> bool:
return self._operations.is_in_use(self._dataset(context, dataset_id))
def queries(self, context: RequestContext, *, dataset_id: str, page: int, limit: int) -> dict[str, Any]:
return self._operations.queries(self._dataset(context, dataset_id), page=page, limit=limit)
def related_apps(self, context: RequestContext, *, dataset_id: str) -> dict[str, Any]:
return self._operations.related_apps(self._dataset(context, dataset_id))
def indexing_status(self, context: RequestContext, *, dataset_id: str) -> dict[str, Any]:
return self._operations.indexing_status(self._dataset(context, dataset_id))
def error_documents(self, context: RequestContext, *, dataset_id: str) -> dict[str, Any]:
return self._operations.error_documents(self._dataset(context, dataset_id))
def partial_members(self, context: RequestContext, *, dataset_id: str) -> list[str]:
return self._operations.partial_members(self._dataset(context, dataset_id))
def auto_disable_logs(self, context: RequestContext, *, dataset_id: str) -> dict[str, Any]:
return self._operations.auto_disable_logs(self._dataset(context, dataset_id))
def set_api_enabled(self, context: RequestContext, *, dataset_id: str, status: str) -> None:
self._operations.set_api_enabled(context, self._dataset(context, dataset_id), status == "enable")
def api_base_url(self, context: RequestContext, *, request_base_url: str) -> str:
return normalize_api_base_url(self._service_api_url or request_base_url)
def retrieval_settings(
self, context: RequestContext, *, vector_type: str | None = None, is_mock: bool = False
) -> dict[str, list[str]]:
return retrieval_methods(
vector_type if is_mock else self._vector_store, is_mock=is_mock, tidb_fulltext=self._tidb_fulltext
)
@@ -0,0 +1,85 @@
"""Supported dataset retrieval methods for each vector store."""
from core.rag.datasource.vdb.vector_type import VectorType
from core.rag.retrieval.retrieval_methods import RetrievalMethod
def retrieval_methods(
vector_type: str | None, *, is_mock: bool = False, tidb_fulltext: bool = False
) -> dict[str, list[str]]:
"""
Get supported retrieval methods based on vector database type.
Args:
vector_type: Vector database type, can be None
is_mock: Whether this is a Mock API, affects MILVUS handling
tidb_fulltext: Whether TiDB full-text retrieval is enabled
Returns:
Dictionary containing supported retrieval methods
Raises:
ValueError: If vector_type is None or unsupported
"""
if vector_type is None:
raise ValueError("Vector store type is not configured.")
# Define vector database types that only support semantic search
semantic_only_types = {
VectorType.RELYT,
VectorType.CHROMA,
VectorType.PGVECTO_RS,
VectorType.VIKINGDB,
VectorType.UPSTASH,
}
# Define vector database types that support all retrieval methods
full_search_types = {
VectorType.QDRANT,
VectorType.WEAVIATE,
VectorType.OPENSEARCH,
VectorType.ANALYTICDB,
VectorType.MYSCALE,
VectorType.ORACLE,
VectorType.ELASTICSEARCH,
VectorType.ELASTICSEARCH_JA,
VectorType.PGVECTOR,
VectorType.VASTBASE,
VectorType.TIDB_ON_QDRANT,
VectorType.LINDORM,
VectorType.COUCHBASE,
VectorType.OPENGAUSS,
VectorType.OCEANBASE,
VectorType.SEEKDB,
VectorType.TABLESTORE,
VectorType.HUAWEI_CLOUD,
VectorType.TENCENT,
VectorType.MATRIXONE,
VectorType.CLICKZETTA,
VectorType.BAIDU,
VectorType.ALIBABACLOUD_MYSQL,
VectorType.IRIS,
VectorType.HOLOGRES,
}
semantic_methods = {"retrieval_method": [RetrievalMethod.SEMANTIC_SEARCH.value]}
full_methods = {
"retrieval_method": [
RetrievalMethod.SEMANTIC_SEARCH.value,
RetrievalMethod.FULL_TEXT_SEARCH.value,
RetrievalMethod.HYBRID_SEARCH.value,
]
}
if vector_type == VectorType.MILVUS:
return semantic_methods if is_mock else full_methods
if vector_type == VectorType.TIDB_VECTOR:
return full_methods if tidb_fulltext else semantic_methods
if vector_type in semantic_only_types:
return semantic_methods
elif vector_type in full_search_types:
return full_methods
else:
raise ValueError(f"Unsupported vector db type {vector_type}.")
@@ -0,0 +1,206 @@
"""Classic-to-Agent knowledge base upgrade use cases behind the dataset console endpoints."""
from collections.abc import Callable
from uuid import uuid4
from sqlalchemy import select
from sqlalchemy.orm import Session, sessionmaker
from graphon.model_runtime.entities.model_entities import ModelType
from machinery.context import RequestContext
from models import Dataset
from models.knowledge_fs import KnowledgeFSUpgradeJob, KnowledgeFSUpgradeJobStatus
from models.provider import TenantDefaultModel
from services.dataset_knowledge_fs_upgrade_service import (
KnowledgeFSUpgradeConflictError,
KnowledgeFSUpgradeModelPreferences,
KnowledgeFSUpgradeModelResolver,
KnowledgeFSUpgradeNotFoundError,
KnowledgeFSUpgradeSnapshotService,
upgrade_discovery_response,
upgrade_job_matches_model_choices,
)
from services.knowledge.dataset_access import DatasetAccess, DatasetAccessError
from services.knowledge.dataset_service import DatasetService
from services.knowledge.datasets.adapters import SQLAlchemyDatasetOperations, load_actor
from services.knowledge_fs.product_dto import (
KnowledgeFSUpgradeModelChoice,
KnowledgeFSUpgradeModelsPayload,
KnowledgeFSUpgradePreparationResponse,
)
_RECOVERABLE_STATUSES = (
KnowledgeFSUpgradeJobStatus.QUEUED,
KnowledgeFSUpgradeJobStatus.RUNNING,
KnowledgeFSUpgradeJobStatus.FAILED,
)
class KnowledgeFSUpgradeUnavailableError(Exception):
"""The KnowledgeFS feature is disabled, so upgrades cannot start or retry."""
class KnowledgeUpgradeApplicationService:
def __init__(
self,
*,
session_factory: sessionmaker[Session],
dataset_access: DatasetAccess,
datasets: SQLAlchemyDatasetOperations,
rbac_enabled: bool,
feature_enabled: Callable[[], bool],
enqueue: Callable[[KnowledgeFSUpgradeSnapshotService, str, str], None] | None = None,
) -> None:
self._sessions = session_factory
self._dataset_access = dataset_access
self._datasets = datasets
self._rbac_enabled = rbac_enabled
self._feature_enabled = feature_enabled
self._enqueue = enqueue or enqueue_upgrade_job
def _snapshots(self) -> KnowledgeFSUpgradeSnapshotService:
return KnowledgeFSUpgradeSnapshotService(self._sessions)
@staticmethod
def _choice(provider: str | None, model: str | None) -> KnowledgeFSUpgradeModelChoice | None:
if not provider or not model:
return None
return KnowledgeFSUpgradeModelChoice(provider=provider, model=model)
def _load_model_preferences(
self, *, tenant_id: str, dataset_id: str
) -> tuple[str, KnowledgeFSUpgradeModelPreferences]:
"""Read the legacy settings and saved defaults in a short, tenant-scoped session."""
with self._sessions() as session:
dataset = session.scalar(select(Dataset).where(Dataset.id == dataset_id, Dataset.tenant_id == tenant_id))
if dataset is None:
raise KnowledgeFSUpgradeNotFoundError("Dataset was not found")
defaults: dict[ModelType, KnowledgeFSUpgradeModelChoice] = {}
for row in session.scalars(
select(TenantDefaultModel).where(
TenantDefaultModel.tenant_id == tenant_id,
TenantDefaultModel.model_type.in_((ModelType.TEXT_EMBEDDING, ModelType.RERANK, ModelType.LLM)),
)
):
choice = self._choice(row.provider_name, row.model_name)
if choice is not None:
defaults[row.model_type] = choice
retrieval = dataset.retrieval_model if isinstance(dataset.retrieval_model, dict) else {}
reranking = retrieval.get("reranking_model") or {}
if not isinstance(reranking, dict):
reranking = {}
preferences = KnowledgeFSUpgradeModelPreferences(
embedding=self._choice(dataset.embedding_model_provider, dataset.embedding_model),
rerank=self._choice(reranking.get("reranking_provider_name"), reranking.get("reranking_model_name")),
defaults=defaults,
)
return str(dataset.provider or ""), preferences
def list_recoverable_jobs(self, context: RequestContext) -> list[KnowledgeFSUpgradeJob]:
workspace_id = context.active_workspace_id
jobs = self._snapshots().list_by_statuses(tenant_id=workspace_id, statuses=_RECOVERABLE_STATUSES)
dataset_ids = list(dict.fromkeys(job.old_dataset_id for job in jobs))
if not dataset_ids:
return []
accessible_ids = self._datasets.visibility(context).list_scope(rbac_enabled=self._rbac_enabled)
with self._sessions() as session:
datasets, _ = DatasetService.get_datasets_by_ids(
dataset_ids,
workspace_id,
user=load_actor(session, context),
accessible_dataset_ids=accessible_ids,
session=session,
)
visible_ids = [str(dataset.id) for dataset in datasets]
allowed_ids: set[str] = set()
for dataset_id in visible_ids:
try:
self._dataset_access.require_accessible(context, dataset_id)
except DatasetAccessError:
continue
allowed_ids.add(dataset_id)
return [job for job in jobs if job.old_dataset_id in allowed_ids]
def discovery(self, context: RequestContext, *, dataset_id: str) -> KnowledgeFSUpgradePreparationResponse:
dataset = self._dataset_access.require_accessible(context, dataset_id)
with self._sessions() as session:
provider = session.scalar(
select(Dataset.provider).where(Dataset.id == dataset.id, Dataset.tenant_id == dataset.workspace_id)
)
job = self._snapshots().get_latest(tenant_id=dataset.workspace_id, dataset_id=dataset.id)
eligibility = upgrade_discovery_response(
job, feature_enabled=self._feature_enabled(), dataset_provider=str(provider or "")
)
recommendations = None
if eligibility.can_upgrade:
_, preferences = self._load_model_preferences(tenant_id=dataset.workspace_id, dataset_id=dataset.id)
recommendations = KnowledgeFSUpgradeModelResolver(tenant_id=dataset.workspace_id).recommend(preferences)
return KnowledgeFSUpgradePreparationResponse.model_validate(
{**eligibility.model_dump(mode="json"), "models": recommendations}
)
def start(
self,
context: RequestContext,
*,
dataset_id: str,
idempotency_key: str,
models: KnowledgeFSUpgradeModelsPayload | None = None,
) -> KnowledgeFSUpgradeJob:
if not self._feature_enabled():
raise KnowledgeFSUpgradeUnavailableError()
dataset = self._dataset_access.require_accessible(context, dataset_id)
snapshots = self._snapshots()
idempotent = snapshots.get_by_idempotency_key(tenant_id=dataset.workspace_id, idempotency_key=idempotency_key)
if idempotent is not None:
if idempotent.old_dataset_id != dataset.id:
raise KnowledgeFSUpgradeConflictError("Idempotency key belongs to another Dataset upgrade")
if models is not None and not upgrade_job_matches_model_choices(idempotent, models):
raise KnowledgeFSUpgradeConflictError("Idempotency key belongs to another model selection")
self._enqueue(snapshots, dataset.workspace_id, idempotent.id)
return idempotent
provider, preferences = self._load_model_preferences(tenant_id=dataset.workspace_id, dataset_id=dataset.id)
if provider == "external":
raise KnowledgeFSUpgradeConflictError("External-provider Datasets cannot be upgraded")
resolver = KnowledgeFSUpgradeModelResolver(tenant_id=dataset.workspace_id)
resolved_models = (
resolver.validate(models) if models is not None else resolver.resolve_recommendations(preferences)
)
job = snapshots.create(
tenant_id=dataset.workspace_id,
dataset_id=dataset.id,
requested_by_account_id=context.account_id,
idempotency_key=idempotency_key,
models=resolved_models,
)
self._enqueue(snapshots, dataset.workspace_id, job.id)
return job
def get_job(self, context: RequestContext, *, dataset_id: str, job_id: str) -> KnowledgeFSUpgradeJob:
dataset = self._dataset_access.require_accessible(context, dataset_id)
job = self._snapshots().get(tenant_id=dataset.workspace_id, job_id=job_id)
if job.old_dataset_id != dataset.id:
raise KnowledgeFSUpgradeNotFoundError("Upgrade job was not found")
return job
def retry(self, context: RequestContext, *, dataset_id: str, job_id: str) -> KnowledgeFSUpgradeJob:
if not self._feature_enabled():
raise KnowledgeFSUpgradeUnavailableError()
dataset = self._dataset_access.require_accessible(context, dataset_id)
snapshots = self._snapshots()
job = snapshots.retry(tenant_id=dataset.workspace_id, dataset_id=dataset.id, job_id=job_id)
self._enqueue(snapshots, dataset.workspace_id, job.id)
return job
def enqueue_upgrade_job(snapshots: KnowledgeFSUpgradeSnapshotService, tenant_id: str, job_id: str) -> None:
from tasks.knowledge_fs_upgrade_tasks import run_knowledge_fs_upgrade
task_id = str(uuid4())
if not snapshots.claim_enqueue(tenant_id=tenant_id, job_id=job_id, task_id=task_id):
return
try:
run_knowledge_fs_upgrade.apply_async(kwargs={"job_id": job_id}, task_id=task_id)
except Exception:
snapshots.release_enqueue_claim(tenant_id=tenant_id, job_id=job_id, task_id=task_id)
raise