Files
gsfdc/demo-dify/dify/api/services/dataset_knowledge_fs_upgrade_service.py
T
wuzilongandCursor 6519682885 chore: 纳入本地 Dify 1.19.0 源码及品牌与免登改动
演示目录只保留可重建的源码。运行密钥、数据卷、依赖和带口令的本地配置不进入仓库。

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-11-18 13:59:25 +08:00

1848 lines
85 KiB
Python

"""Snapshot and resumable orchestration for legacy Dataset upgrades."""
from __future__ import annotations
import json
import re
import uuid
from collections import defaultdict
from collections.abc import Mapping
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from enum import Enum
from hashlib import sha256
from typing import Any, cast
import sqlalchemy as sa
from pydantic import TypeAdapter
from sqlalchemy.orm import Session, sessionmaker
from core.entities.model_entities import ModelStatus, ModelWithProviderEntity
from core.plugin.impl.model_runtime_factory import create_plugin_provider_manager
from core.rag.index_processor.constant.built_in_field import BuiltInField
from extensions.ext_storage import storage
from graphon.model_runtime.entities.model_entities import ModelType
from libs.datetime_utils import naive_utc_now
from models import Account, AccountStatus, TenantAccountJoin
from models.dataset import AppDatasetJoin, Dataset, DatasetMetadata, DatasetPermission, Document
from models.knowledge_fs import (
KnowledgeFSAppSpaceJoinType,
KnowledgeFSControlSpace,
KnowledgeFSControlSpacePermissionRole,
KnowledgeFSControlSpaceState,
KnowledgeFSControlSpaceVisibility,
KnowledgeFSUpgradeDocument,
KnowledgeFSUpgradeFileLease,
KnowledgeFSUpgradeFileLeaseStatus,
KnowledgeFSUpgradeItemStatus,
KnowledgeFSUpgradeJob,
KnowledgeFSUpgradeJobStatus,
KnowledgeFSUpgradeSource,
KnowledgeFSUpgradeStage,
)
from models.model import App, AppMode, Tag, TagBinding, UploadFile
from models.oauth import DatasourceProvider
from models.provider_ids import ModelProviderID
from services.dataset_knowledge_fs_upgrade_file_lease import release_upgrade_file_lease
from services.feature_service import FeatureService
from services.knowledge_fs.data_facade import KnowledgeFSDataFacade
from services.knowledge_fs.initial_source_import import (
KnowledgeFSInitialSourceNotReadyError,
submit_initial_source_for_upgrade,
)
from services.knowledge_fs.product_dto import (
KnowledgeFSDocumentAvailabilityPayload,
KnowledgeFSDocumentMetadataPayload,
KnowledgeFSDocumentStagedUploadPayload,
KnowledgeFSExternalAccessPayload,
KnowledgeFSInitialSourcePayload,
KnowledgeFSLogicalDocumentResponse,
KnowledgeFSMemberBindingPayload,
KnowledgeFSMetadataFieldCreatePayload,
KnowledgeFSModelIntent,
KnowledgeFSRerankIntent,
KnowledgeFSRetrievalProfileIntent,
KnowledgeFSScoreThresholdIntent,
KnowledgeFSSpaceCreatePayload,
KnowledgeFSUpgradeDiscoveryResponse,
KnowledgeFSUpgradeJobResponse,
KnowledgeFSUpgradeModelChoice,
KnowledgeFSUpgradeModelIssueResponse,
KnowledgeFSUpgradeModelRecommendationResponse,
KnowledgeFSUpgradeModelRecommendationsResponse,
KnowledgeFSUpgradeModelsPayload,
KnowledgeFSUpgradeModelUnavailableReason,
)
from services.knowledge_fs.product_remote import KnowledgeFSProductResourceNotFoundError
from services.knowledge_fs.runtime import get_knowledge_fs_runtime
from services.knowledge_fs.staged_upload_service import KnowledgeFSStagedUploadService
_ACTIVE_JOB_STATUSES = (KnowledgeFSUpgradeJobStatus.QUEUED, KnowledgeFSUpgradeJobStatus.RUNNING)
_SOURCE_SELECTION_LIMIT = 200
_FILE_LEASE_TTL = timedelta(days=7)
# A checkpoint runs in a prefork child with a hard limit. Only reclaim after that
# child must have stopped, with headroom for termination and broker delivery.
UPGRADE_CHECKPOINT_TIME_LIMIT_SECONDS = 600
UPGRADE_DELIVERY_TIMEOUT = timedelta(seconds=UPGRADE_CHECKPOINT_TIME_LIMIT_SECONDS + 60)
_INITIAL_SOURCE_ADAPTER: TypeAdapter[KnowledgeFSInitialSourcePayload] = TypeAdapter(KnowledgeFSInitialSourcePayload)
class KnowledgeFSUpgradeError(RuntimeError):
"""A persisted upgrade cannot safely continue."""
class KnowledgeFSUpgradeNotFoundError(KnowledgeFSUpgradeError):
pass
class KnowledgeFSUpgradeConflictError(KnowledgeFSUpgradeError):
pass
class KnowledgeFSUpgradeNotReadyError(KnowledgeFSUpgradeError):
"""Provisioning is progressing and the dedicated worker should retry."""
class KnowledgeFSUpgradeBusyError(KnowledgeFSUpgradeError):
"""Another delivery still owns this checkpoint's bounded execution window."""
class KnowledgeFSUpgradeModelsUnavailableError(KnowledgeFSUpgradeError):
"""At least one chosen model cannot be used for this upgrade right now."""
def __init__(self, issues: list[KnowledgeFSUpgradeModelIssueResponse]) -> None:
super().__init__("One or more upgrade models are unavailable")
self.issues = issues
@dataclass(frozen=True)
class KnowledgeFSUpgradeModelPreferences:
"""Legacy choices and saved workspace defaults read before model-provider I/O."""
embedding: KnowledgeFSUpgradeModelChoice | None
rerank: KnowledgeFSUpgradeModelChoice | None
defaults: Mapping[ModelType, KnowledgeFSUpgradeModelChoice]
@dataclass(frozen=True)
class KnowledgeFSUpgradeResolvedModels:
"""Validated model identities; never reselect them after a job is accepted."""
embedding: KnowledgeFSModelIntent
rerank: KnowledgeFSModelIntent
reasoning: KnowledgeFSModelIntent
class KnowledgeFSUpgradeModelResolver:
"""Use the same workspace model statuses for suggestions and start admission."""
def __init__(self, *, tenant_id: str) -> None:
provider_manager = create_plugin_provider_manager(tenant_id=tenant_id)
configurations = provider_manager.get_configurations(tenant_id)
self._models = {
model_type: configurations.get_models(model_type=model_type, only_active=False)
for model_type in (ModelType.TEXT_EMBEDDING, ModelType.RERANK, ModelType.LLM)
}
@staticmethod
def _provider_id(provider: str) -> str | None:
try:
return str(ModelProviderID(provider))
except ValueError:
return None
def _match(self, model_type: ModelType, choice: KnowledgeFSUpgradeModelChoice) -> ModelWithProviderEntity | None:
provider_id = self._provider_id(choice.provider)
if provider_id is None:
return None
return next(
(
item
for item in self._models[model_type]
if item.model == choice.model and self._provider_id(item.provider.provider) == provider_id
),
None,
)
def _unavailable_reason(
self, model_type: ModelType, choice: KnowledgeFSUpgradeModelChoice
) -> KnowledgeFSUpgradeModelUnavailableReason | None:
"""Map a configured model to its unavailable reason without changing provider state."""
model = self._match(model_type, choice)
if model is None:
return "not_found"
match model.status:
case ModelStatus.ACTIVE:
return "deprecated" if model.deprecated else None
case ModelStatus.NO_CONFIGURE:
return "no-configure"
case ModelStatus.QUOTA_EXCEEDED:
return "quota-exceeded"
case ModelStatus.NO_PERMISSION:
return "no-permission"
case ModelStatus.CREDENTIAL_REMOVED:
return "credential-removed"
case _:
return "disabled"
def _recommend(
self,
model_type: ModelType,
*,
legacy: KnowledgeFSUpgradeModelChoice | None,
default: KnowledgeFSUpgradeModelChoice | None,
) -> KnowledgeFSUpgradeModelRecommendationResponse:
if legacy is not None and self._unavailable_reason(model_type, legacy) is None:
return KnowledgeFSUpgradeModelRecommendationResponse(value=legacy, source="legacy", available=True)
if default is not None and self._unavailable_reason(model_type, default) is None:
return KnowledgeFSUpgradeModelRecommendationResponse(
value=default, source="workspace_default", available=True
)
value = default or legacy
source = "workspace_default" if default is not None else "legacy" if legacy is not None else None
reason = self._unavailable_reason(model_type, value) if value is not None else "missing"
return KnowledgeFSUpgradeModelRecommendationResponse(
value=value, source=source, available=False, unavailable_reason=reason
)
def recommend(
self, preferences: KnowledgeFSUpgradeModelPreferences
) -> KnowledgeFSUpgradeModelRecommendationsResponse:
return KnowledgeFSUpgradeModelRecommendationsResponse(
embedding=self._recommend(
ModelType.TEXT_EMBEDDING,
legacy=preferences.embedding,
default=preferences.defaults.get(ModelType.TEXT_EMBEDDING),
),
rerank=self._recommend(
ModelType.RERANK,
legacy=preferences.rerank,
default=preferences.defaults.get(ModelType.RERANK),
),
reasoning=self._recommend(ModelType.LLM, legacy=None, default=preferences.defaults.get(ModelType.LLM)),
)
def validate(self, models: KnowledgeFSUpgradeModelsPayload) -> KnowledgeFSUpgradeResolvedModels:
"""Resolve all three choices or report each unusable field without creating a job."""
selected: dict[str, KnowledgeFSModelIntent] = {}
issues: list[KnowledgeFSUpgradeModelIssueResponse] = []
for field, model_type, choice in (
("embedding", ModelType.TEXT_EMBEDDING, models.embedding),
("rerank", ModelType.RERANK, models.rerank),
("reasoning", ModelType.LLM, models.reasoning),
):
reason = self._unavailable_reason(model_type, choice)
if reason is not None:
issues.append(
KnowledgeFSUpgradeModelIssueResponse(
field=field, reason=reason, provider=choice.provider, model=choice.model
)
)
else:
matched = self._match(model_type, choice)
assert matched is not None
selected[field] = _model_intent(matched.provider.provider, matched.model)
if issues:
raise KnowledgeFSUpgradeModelsUnavailableError(issues)
return KnowledgeFSUpgradeResolvedModels(
embedding=selected["embedding"], rerank=selected["rerank"], reasoning=selected["reasoning"]
)
def resolve_recommendations(
self, preferences: KnowledgeFSUpgradeModelPreferences
) -> KnowledgeFSUpgradeResolvedModels:
recommendations = self.recommend(preferences)
issues = [
KnowledgeFSUpgradeModelIssueResponse(
field=field,
reason=recommendation.unavailable_reason or "missing",
provider=recommendation.value.provider if recommendation.value else None,
model=recommendation.value.model if recommendation.value else None,
)
for field, recommendation in (
("embedding", recommendations.embedding),
("rerank", recommendations.rerank),
("reasoning", recommendations.reasoning),
)
if not recommendation.available
]
if issues:
raise KnowledgeFSUpgradeModelsUnavailableError(issues)
assert recommendations.embedding.value is not None
assert recommendations.rerank.value is not None
assert recommendations.reasoning.value is not None
return self.validate(
KnowledgeFSUpgradeModelsPayload(
embedding=recommendations.embedding.value,
rerank=recommendations.rerank.value,
reasoning=recommendations.reasoning.value,
)
)
def upgrade_job_matches_models(job: KnowledgeFSUpgradeJob, models: KnowledgeFSUpgradeResolvedModels) -> bool:
"""A replay may reuse a job only when its persisted model identities match."""
if job.resolved_configuration is None:
return False
try:
resolved = job.resolved_configuration
embedding = KnowledgeFSModelIntent.model_validate(resolved["embedding"])
retrieval = KnowledgeFSRetrievalProfileIntent.model_validate(resolved["retrieval"])
except (KeyError, TypeError, ValueError):
return False
return (
embedding == models.embedding
and retrieval.rerank.model == models.rerank
and retrieval.reasoning_model == models.reasoning
)
def upgrade_job_matches_model_choices(job: KnowledgeFSUpgradeJob, choices: KnowledgeFSUpgradeModelsPayload) -> bool:
"""Compare an idempotent replay without querying provider status again."""
try:
models = KnowledgeFSUpgradeResolvedModels(
embedding=_model_intent(choices.embedding.provider, choices.embedding.model),
rerank=_model_intent(choices.rerank.provider, choices.rerank.model),
reasoning=_model_intent(choices.reasoning.provider, choices.reasoning.model),
)
except ValueError:
return False
return upgrade_job_matches_models(job, models)
class KnowledgeFSUpgradeSnapshotService:
"""Persist the exact click-time Dataset manifest without external I/O."""
def __init__(self, session_maker: sessionmaker[Session]) -> None:
self._session_maker = session_maker
def create(
self,
*,
tenant_id: str,
dataset_id: str,
requested_by_account_id: str,
idempotency_key: str | None = None,
models: KnowledgeFSUpgradeResolvedModels | None = None,
) -> KnowledgeFSUpgradeJob:
snapshot_at = naive_utc_now()
with self._session_maker(expire_on_commit=False) as session, session.begin():
dataset = session.scalar(
sa.select(Dataset).where(Dataset.id == dataset_id, Dataset.tenant_id == tenant_id).with_for_update()
)
if dataset is None:
raise KnowledgeFSUpgradeNotFoundError("Dataset was not found")
if dataset.provider == "external":
raise KnowledgeFSUpgradeConflictError("External-provider Datasets cannot be upgraded")
if idempotency_key:
idempotent = session.scalar(
sa.select(KnowledgeFSUpgradeJob).where(
KnowledgeFSUpgradeJob.tenant_id == tenant_id,
KnowledgeFSUpgradeJob.idempotency_key == idempotency_key,
)
)
if idempotent is not None:
if idempotent.old_dataset_id != dataset_id:
raise KnowledgeFSUpgradeConflictError("Idempotency key belongs to another Dataset upgrade")
if models is not None and not upgrade_job_matches_models(idempotent, models):
raise KnowledgeFSUpgradeConflictError("Idempotency key belongs to another model selection")
return idempotent
existing = session.scalar(
sa.select(KnowledgeFSUpgradeJob)
.where(
KnowledgeFSUpgradeJob.tenant_id == tenant_id,
KnowledgeFSUpgradeJob.old_dataset_id == dataset_id,
KnowledgeFSUpgradeJob.status.in_(_ACTIVE_JOB_STATUSES),
)
.order_by(KnowledgeFSUpgradeJob.created_at.desc())
.limit(1)
)
if existing is not None:
if models is not None and not upgrade_job_matches_models(existing, models):
raise KnowledgeFSUpgradeConflictError("Another Dataset upgrade is already in progress")
return existing
owner_account_id = str(dataset.maintainer or dataset.created_by)
if (
session.scalar(
sa.select(Account.id)
.join(TenantAccountJoin, TenantAccountJoin.account_id == Account.id)
.where(
Account.id == owner_account_id,
Account.status != AccountStatus.BANNED,
TenantAccountJoin.tenant_id == tenant_id,
)
.limit(1)
)
is None
):
raise KnowledgeFSUpgradeConflictError("Dataset maintainer is not an active account")
permission_members = list(
session.scalars(
sa.select(DatasetPermission.account_id)
.join(Account, Account.id == DatasetPermission.account_id)
.join(
TenantAccountJoin,
sa.and_(
TenantAccountJoin.account_id == DatasetPermission.account_id,
TenantAccountJoin.tenant_id == tenant_id,
),
)
.where(
DatasetPermission.tenant_id == tenant_id,
DatasetPermission.dataset_id == dataset_id,
DatasetPermission.has_permission.is_(True),
Account.status != AccountStatus.BANNED,
)
)
)
app_bindings = _app_binding_snapshot(session, tenant_id=tenant_id, dataset_id=dataset_id)
tag_ids = list(
session.scalars(
sa.select(TagBinding.tag_id)
.join(Tag, Tag.id == TagBinding.tag_id)
.where(
TagBinding.tenant_id == tenant_id,
TagBinding.target_id == dataset_id,
Tag.type == "knowledge",
)
)
)
config_snapshot = _config_snapshot(session, dataset)
try:
resolved_configuration = _build_resolved_configuration(config_snapshot, models) if models else None
except (ValueError, TypeError) as error:
raise KnowledgeFSUpgradeConflictError("Legacy retrieval settings cannot be upgraded") from error
job = KnowledgeFSUpgradeJob(
tenant_id=tenant_id,
old_dataset_id=dataset_id,
requested_by_account_id=requested_by_account_id,
owner_account_id=owner_account_id,
idempotency_key=idempotency_key or str(uuid.uuid4()),
snapshot_at=snapshot_at,
config_snapshot=config_snapshot,
permission_snapshot={
"visibility": _enum_value(dataset.permission),
"member_account_ids": permission_members,
},
app_binding_snapshot=app_bindings,
tag_ids_snapshot=[str(tag_id) for tag_id in tag_ids if tag_id is not None],
resolved_configuration=resolved_configuration,
)
session.add(job)
session.flush()
source_groups: dict[str, list[tuple[Document, dict[str, Any]]]] = defaultdict(list)
document_snapshots: dict[str, KnowledgeFSUpgradeDocument] = {}
upload_file_ids: set[str] = set()
documents = list(
session.scalars(
sa.select(Document)
.where(Document.tenant_id == tenant_id, Document.dataset_id == dataset_id)
.order_by(Document.position, Document.id)
)
)
for document in documents:
data_source_type = _enum_value(document.data_source_type)
data_source_info = document.data_source_info_dict
old_upload_file_id = None
source_key = None
if data_source_type == "upload_file":
raw_upload_file_id = data_source_info.get("upload_file_id")
if raw_upload_file_id:
old_upload_file_id = str(raw_upload_file_id)
upload_file_ids.add(old_upload_file_id)
elif data_source_type == "notion_import":
source_key = _notion_source_group_key(data_source_info)
source_groups[source_key].append((document, data_source_info))
elif data_source_type == "website_crawl":
source_key = _website_source_group_key(data_source_info)
source_groups[source_key].append((document, data_source_info))
else:
raise KnowledgeFSUpgradeConflictError(
f"Unsupported legacy document source type: {data_source_type}"
)
document_snapshot = KnowledgeFSUpgradeDocument(
job_id=job.id,
tenant_id=tenant_id,
old_document_id=str(document.id),
name=document.name,
data_source_type=data_source_type,
data_source_info=cast(dict[str, object], data_source_info),
metadata_snapshot=cast(dict[str, object], document.doc_metadata or {}),
desired_enabled=bool(document.enabled and not document.archived),
legacy_archived=bool(document.archived),
legacy_indexing_status=_enum_value(document.indexing_status),
legacy_display_status=document.display_status,
old_upload_file_id=old_upload_file_id,
source_key=source_key,
)
session.add(document_snapshot)
document_snapshots[str(document.id)] = document_snapshot
source_count = 0
for base_source_key, group in source_groups.items():
for chunk_number, chunk in enumerate(_chunks(group, _SOURCE_SELECTION_LIMIT), start=1):
source_key = f"{base_source_key}:{chunk_number}"
payload = _source_payload_snapshot(dataset.name, chunk)
session.add(
KnowledgeFSUpgradeSource(
job_id=job.id,
tenant_id=tenant_id,
source_key=source_key,
payload_snapshot=payload,
)
)
for document, _ in chunk:
document_snapshots[str(document.id)].source_key = source_key
source_count += 1
for upload_file_id in sorted(upload_file_ids):
session.add(
KnowledgeFSUpgradeFileLease(
job_id=job.id,
old_upload_file_id=upload_file_id,
expires_at=snapshot_at + _FILE_LEASE_TTL,
)
)
job.total_documents = len(documents)
job.total_sources = source_count
session.flush()
return job
def get_by_idempotency_key(self, *, tenant_id: str, idempotency_key: str) -> KnowledgeFSUpgradeJob | None:
with self._session_maker() as session:
job = session.scalar(
sa.select(KnowledgeFSUpgradeJob).where(
KnowledgeFSUpgradeJob.tenant_id == tenant_id,
KnowledgeFSUpgradeJob.idempotency_key == idempotency_key,
)
)
if job is not None:
session.expunge(job)
return job
def get(self, *, tenant_id: str, job_id: str) -> KnowledgeFSUpgradeJob:
with self._session_maker() as session:
job = session.scalar(
sa.select(KnowledgeFSUpgradeJob).where(
KnowledgeFSUpgradeJob.id == job_id,
KnowledgeFSUpgradeJob.tenant_id == tenant_id,
)
)
if job is None:
raise KnowledgeFSUpgradeNotFoundError("Upgrade job was not found")
session.expunge(job)
return job
def get_latest(self, *, tenant_id: str, dataset_id: str) -> KnowledgeFSUpgradeJob | None:
jobs = self.get_latest_by_dataset_ids(tenant_id=tenant_id, dataset_ids=[dataset_id])
return jobs.get(dataset_id)
def list_by_statuses(
self,
*,
tenant_id: str,
statuses: tuple[KnowledgeFSUpgradeJobStatus, ...],
) -> list[KnowledgeFSUpgradeJob]:
with self._session_maker() as session:
jobs = list(
session.scalars(
sa.select(KnowledgeFSUpgradeJob)
.where(
KnowledgeFSUpgradeJob.tenant_id == tenant_id,
KnowledgeFSUpgradeJob.status.in_(statuses),
)
.order_by(KnowledgeFSUpgradeJob.created_at.desc(), KnowledgeFSUpgradeJob.id.desc())
)
)
for job in jobs:
session.expunge(job)
return jobs
def get_latest_by_dataset_ids(
self,
*,
tenant_id: str,
dataset_ids: list[str],
session: Session | None = None,
) -> dict[str, KnowledgeFSUpgradeJob]:
if not dataset_ids:
return {}
if session is not None:
return self._latest_jobs_from_session(session, tenant_id=tenant_id, dataset_ids=dataset_ids, detach=False)
with self._session_maker() as owned_session:
return self._latest_jobs_from_session(
owned_session,
tenant_id=tenant_id,
dataset_ids=dataset_ids,
detach=True,
)
@staticmethod
def _latest_jobs_from_session(
session: Session,
*,
tenant_id: str,
dataset_ids: list[str],
detach: bool,
) -> dict[str, KnowledgeFSUpgradeJob]:
jobs = session.scalars(
sa.select(KnowledgeFSUpgradeJob)
.where(
KnowledgeFSUpgradeJob.tenant_id == tenant_id,
KnowledgeFSUpgradeJob.old_dataset_id.in_(dataset_ids),
)
.order_by(
KnowledgeFSUpgradeJob.old_dataset_id,
sa.case((KnowledgeFSUpgradeJob.status.in_(_ACTIVE_JOB_STATUSES), 0), else_=1),
KnowledgeFSUpgradeJob.created_at.desc(),
KnowledgeFSUpgradeJob.id.desc(),
)
)
latest: dict[str, KnowledgeFSUpgradeJob] = {}
for job in jobs:
if job.old_dataset_id not in latest:
if detach:
session.expunge(job)
latest[job.old_dataset_id] = job
return latest
def retry(self, *, tenant_id: str, dataset_id: str, job_id: str) -> KnowledgeFSUpgradeJob:
lease_expires_at = naive_utc_now() + _FILE_LEASE_TTL
with self._session_maker(expire_on_commit=False) as session, session.begin():
job = session.scalar(
sa.select(KnowledgeFSUpgradeJob)
.where(
KnowledgeFSUpgradeJob.id == job_id,
KnowledgeFSUpgradeJob.tenant_id == tenant_id,
KnowledgeFSUpgradeJob.old_dataset_id == dataset_id,
)
.with_for_update()
)
if job is None:
raise KnowledgeFSUpgradeNotFoundError("Upgrade job was not found")
if job.status is KnowledgeFSUpgradeJobStatus.SUCCEEDED:
raise KnowledgeFSUpgradeConflictError("Successful upgrades cannot be retried")
if job.status in _ACTIVE_JOB_STATUSES and job.updated_at > naive_utc_now() - UPGRADE_DELIVERY_TIMEOUT:
return job
active_job = session.scalar(
sa.select(KnowledgeFSUpgradeJob)
.where(
KnowledgeFSUpgradeJob.tenant_id == tenant_id,
KnowledgeFSUpgradeJob.old_dataset_id == job.old_dataset_id,
KnowledgeFSUpgradeJob.id != job.id,
KnowledgeFSUpgradeJob.status.in_(_ACTIVE_JOB_STATUSES),
)
.limit(1)
)
if active_job is not None:
raise KnowledgeFSUpgradeConflictError("Another Dataset upgrade is already in progress")
job.status = KnowledgeFSUpgradeJobStatus.QUEUED
job.last_error_code = None
job.last_error_message = None
job.completed_at = None
job.celery_task_id = None
job.updated_at = naive_utc_now()
session.execute(
sa.update(KnowledgeFSUpgradeDocument)
.where(
KnowledgeFSUpgradeDocument.job_id == job.id,
KnowledgeFSUpgradeDocument.status == KnowledgeFSUpgradeItemStatus.FAILED,
)
.values(status=KnowledgeFSUpgradeItemStatus.PENDING, last_error_code=None, last_error_message=None)
)
session.execute(
sa.update(KnowledgeFSUpgradeSource)
.where(
KnowledgeFSUpgradeSource.job_id == job.id,
KnowledgeFSUpgradeSource.status == KnowledgeFSUpgradeItemStatus.FAILED,
)
.values(status=KnowledgeFSUpgradeItemStatus.PENDING, last_error_code=None, last_error_message=None)
)
session.execute(
sa.update(KnowledgeFSUpgradeFileLease)
.where(
KnowledgeFSUpgradeFileLease.job_id == job.id,
KnowledgeFSUpgradeFileLease.released_at.is_(None),
)
.values(
status=KnowledgeFSUpgradeFileLeaseStatus.ACTIVE,
expires_at=lease_expires_at,
)
)
return job
def recover_stalled_jobs(self, *, limit: int = 100) -> list[tuple[str, str]]:
"""Release abandoned execution and publication claims for bounded redispatch.
updated_at is refreshed on claim and checkpoint progress. Prefork's hard
task limit is shorter than this window, so a live child cannot overlap a
recovered delivery. This also covers a publisher dying before apply_async.
"""
now = naive_utc_now()
with self._session_maker.begin() as session:
jobs = session.scalars(
sa.select(KnowledgeFSUpgradeJob)
.where(
KnowledgeFSUpgradeJob.status.in_(_ACTIVE_JOB_STATUSES),
KnowledgeFSUpgradeJob.updated_at <= now - UPGRADE_DELIVERY_TIMEOUT,
)
.order_by(KnowledgeFSUpgradeJob.updated_at, KnowledgeFSUpgradeJob.id)
.limit(limit)
.with_for_update(skip_locked=True)
).all()
recovered = []
for job in jobs:
job.status = KnowledgeFSUpgradeJobStatus.QUEUED
job.celery_task_id = None
job.updated_at = now
recovered.append((job.tenant_id, job.id))
return recovered
def claim_enqueue(self, *, tenant_id: str, job_id: str, task_id: str) -> bool:
with self._session_maker.begin() as session:
job = session.scalar(
sa.select(KnowledgeFSUpgradeJob)
.where(KnowledgeFSUpgradeJob.id == job_id, KnowledgeFSUpgradeJob.tenant_id == tenant_id)
.with_for_update()
)
if job is None:
raise KnowledgeFSUpgradeNotFoundError("Upgrade job was not found")
if job.status is not KnowledgeFSUpgradeJobStatus.QUEUED or job.celery_task_id is not None:
return False
job.celery_task_id = task_id
job.updated_at = naive_utc_now()
return True
def release_enqueue_claim(self, *, tenant_id: str, job_id: str, task_id: str) -> None:
with self._session_maker.begin() as session:
job = session.scalar(
sa.select(KnowledgeFSUpgradeJob)
.where(KnowledgeFSUpgradeJob.id == job_id, KnowledgeFSUpgradeJob.tenant_id == tenant_id)
.with_for_update()
)
if job is not None and job.status is KnowledgeFSUpgradeJobStatus.QUEUED and job.celery_task_id == task_id:
job.celery_task_id = None
class KnowledgeFSUpgradeRunner:
"""Execute one durable stage or item per dedicated-worker delivery."""
def __init__(self, session_maker: sessionmaker[Session]) -> None:
self._session_maker = session_maker
def run_next(self, *, job_id: str, celery_task_id: str) -> bool | None:
"""Return True for another checkpoint, False for reconciliation, or None for an obsolete delivery."""
job = self._mark_running(job_id=job_id, celery_task_id=celery_task_id)
if job is None:
# Superseded/terminal delivery: do not dispatch follow-up work.
return None
if job.status is KnowledgeFSUpgradeJobStatus.SUCCEEDED:
return False
if job.stage is KnowledgeFSUpgradeStage.VALIDATING:
self._create_space(job)
raise KnowledgeFSUpgradeNotReadyError("KnowledgeFS Space provisioning is pending")
if job.stage is KnowledgeFSUpgradeStage.WAITING_FOR_SPACE:
self._advance_when_space_is_active(job)
return True
if job.stage is KnowledgeFSUpgradeStage.CREATING_SOURCES:
return self._create_next_source(job)
if job.stage is KnowledgeFSUpgradeStage.SUBMITTING_DOCUMENTS:
return self._submit_next_document(job)
if job.stage is KnowledgeFSUpgradeStage.MIGRATING_ACCESS:
self._migrate_access(job)
return True
if job.stage is KnowledgeFSUpgradeStage.FINALIZING:
self._finalize(job)
return False
return False
def fail(self, *, job_id: str, celery_task_id: str, error: Exception) -> None:
with self._session_maker.begin() as session:
job = session.get(KnowledgeFSUpgradeJob, job_id, with_for_update=True)
if (
job is None
or job.status is not KnowledgeFSUpgradeJobStatus.RUNNING
or job.celery_task_id != celery_task_id
):
return
job.status = KnowledgeFSUpgradeJobStatus.FAILED
job.last_error_code = type(error).__name__[:128]
job.last_error_message = str(error)[:4000]
job.completed_at = naive_utc_now()
def release_for_dispatch(self, *, job_id: str, celery_task_id: str | None) -> str | None:
"""Return a committed checkpoint to an unclaimed QUEUED state before the next delivery.
Dispatch then goes through the same claim/release path as console enqueues, so a broker
failure leaves a job that ``retry`` and ``claim_enqueue`` can recover instead of a RUNNING
job with no delivery. Returns the tenant id, or None when the job moved on meanwhile.
"""
with self._session_maker.begin() as session:
job = session.get(KnowledgeFSUpgradeJob, job_id, with_for_update=True)
if (
job is None
or job.status is not KnowledgeFSUpgradeJobStatus.RUNNING
or job.celery_task_id != celery_task_id
):
return None
job.status = KnowledgeFSUpgradeJobStatus.QUEUED
job.celery_task_id = None
return job.tenant_id
def release_for_retry(self, *, job_id: str, celery_task_id: str) -> None:
with self._session_maker.begin() as session:
job = session.get(KnowledgeFSUpgradeJob, job_id, with_for_update=True)
if (
job is not None
and job.status is KnowledgeFSUpgradeJobStatus.RUNNING
and job.celery_task_id == celery_task_id
):
job.status = KnowledgeFSUpgradeJobStatus.QUEUED
job.updated_at = naive_utc_now()
def _mark_running(self, *, job_id: str, celery_task_id: str) -> KnowledgeFSUpgradeJob | None:
with self._session_maker(expire_on_commit=False) as session, session.begin():
job = session.get(KnowledgeFSUpgradeJob, job_id, with_for_update=True)
now = naive_utc_now()
lease_expires_at = now + _FILE_LEASE_TTL
if job is None:
raise KnowledgeFSUpgradeNotFoundError("Upgrade job was not found")
if job.celery_task_id != celery_task_id:
return None
if job.status is KnowledgeFSUpgradeJobStatus.SUCCEEDED:
return job
if job.status not in _ACTIVE_JOB_STATUSES:
return None
if job.status is KnowledgeFSUpgradeJobStatus.RUNNING and job.updated_at > now - UPGRADE_DELIVERY_TIMEOUT:
raise KnowledgeFSUpgradeBusyError("Upgrade checkpoint is still owned by another delivery")
job.status = KnowledgeFSUpgradeJobStatus.RUNNING
job.updated_at = now
job.attempt_count += 1
job.celery_task_id = celery_task_id
session.execute(
sa.update(KnowledgeFSUpgradeFileLease)
.where(
KnowledgeFSUpgradeFileLease.job_id == job_id,
KnowledgeFSUpgradeFileLease.status == KnowledgeFSUpgradeFileLeaseStatus.ACTIVE,
)
.values(expires_at=lease_expires_at)
)
return job
def _create_space(self, job: KnowledgeFSUpgradeJob) -> None:
resolved = job.resolved_configuration or _resolve_configuration(job)
runtime = get_knowledge_fs_runtime(self._session_maker)
response = runtime.application.create_space(
tenant_id=job.tenant_id,
account_id=job.owner_account_id,
payload=KnowledgeFSSpaceCreatePayload(
name=str(job.config_snapshot["name"])[:40],
slug=f"legacy-{job.old_dataset_id[:12]}-{job.id[:8]}",
description=str(job.config_snapshot.get("description") or "")[:2000] or None,
icon=cast(str | None, job.config_snapshot.get("icon")),
visibility=KnowledgeFSControlSpaceVisibility.ONLY_ME,
embedding=KnowledgeFSModelIntent.model_validate(resolved["embedding"]),
retrieval=KnowledgeFSRetrievalProfileIntent.model_validate(resolved["retrieval"]),
idempotency_key=f"upgrade:{job.id}:space",
),
)
with self._session_maker.begin() as session:
persisted = session.get(KnowledgeFSUpgradeJob, job.id)
if persisted is None:
raise KnowledgeFSUpgradeNotFoundError("Upgrade job disappeared")
persisted.new_control_space_id = response.control_space_id
if persisted.resolved_configuration is None:
persisted.resolved_configuration = resolved
persisted.stage = KnowledgeFSUpgradeStage.WAITING_FOR_SPACE
def _advance_when_space_is_active(self, job: KnowledgeFSUpgradeJob) -> None:
if job.new_control_space_id is None:
raise KnowledgeFSUpgradeError("Upgrade Space reference is missing")
with self._session_maker.begin() as session:
control_space = session.scalar(
sa.select(KnowledgeFSControlSpace).where(
KnowledgeFSControlSpace.id == job.new_control_space_id,
KnowledgeFSControlSpace.tenant_id == job.tenant_id,
)
)
if control_space is None:
raise KnowledgeFSUpgradeError("Upgrade Space was not found")
if control_space.state is KnowledgeFSControlSpaceState.PROVISIONING:
raise KnowledgeFSUpgradeNotReadyError("KnowledgeFS Space provisioning is pending")
if control_space.state is not KnowledgeFSControlSpaceState.ACTIVE:
raise KnowledgeFSUpgradeError(f"KnowledgeFS Space provisioning failed in {control_space.state.value}")
persisted = session.get(KnowledgeFSUpgradeJob, job.id)
assert persisted is not None
persisted.stage = (
KnowledgeFSUpgradeStage.CREATING_SOURCES
if persisted.total_sources
else KnowledgeFSUpgradeStage.SUBMITTING_DOCUMENTS
)
def _create_next_source(self, job: KnowledgeFSUpgradeJob) -> bool:
with self._session_maker.begin() as session:
source = session.scalar(
sa.select(KnowledgeFSUpgradeSource)
.where(
KnowledgeFSUpgradeSource.job_id == job.id,
KnowledgeFSUpgradeSource.status.in_(
(KnowledgeFSUpgradeItemStatus.PENDING, KnowledgeFSUpgradeItemStatus.PROCESSING)
),
)
.order_by(KnowledgeFSUpgradeSource.id)
.with_for_update(skip_locked=True)
.limit(1)
)
if source is None:
persisted = session.get(KnowledgeFSUpgradeJob, job.id)
assert persisted is not None
persisted.stage = KnowledgeFSUpgradeStage.SUBMITTING_DOCUMENTS
return True
source.status = KnowledgeFSUpgradeItemStatus.PROCESSING
source_id = source.id
source_payload = dict(source.payload_snapshot)
payload = self._resolve_initial_source_payload(job, source_payload)
try:
result = submit_initial_source_for_upgrade(
tenant_id=job.tenant_id,
account_id=job.owner_account_id,
control_space_id=_required_space_id(job),
operation_id=f"upgrade:{job.id}:{source_id}",
payload=payload,
)
except KnowledgeFSUpgradeNotReadyError:
raise
except Exception as error:
if isinstance(error, KnowledgeFSInitialSourceNotReadyError):
with self._session_maker.begin() as session:
persisted_source = session.get(KnowledgeFSUpgradeSource, source_id)
if persisted_source is not None:
persisted_source.status = KnowledgeFSUpgradeItemStatus.PENDING
raise KnowledgeFSUpgradeNotReadyError(str(error)) from error
self._fail_source(source_id=source_id, error=error)
raise
with self._session_maker.begin() as session:
persisted_source = session.get(KnowledgeFSUpgradeSource, source_id)
persisted_job = session.get(KnowledgeFSUpgradeJob, job.id)
assert persisted_source is not None
assert persisted_job is not None
persisted_source.status = KnowledgeFSUpgradeItemStatus.SUCCEEDED
persisted_source.new_connection_id = result.connection_id
persisted_source.new_source_id = result.source_id
persisted_source.initial_sync_task_id = result.workflow_id
if result.workflow_error:
persisted_source.last_error_code = result.workflow_error
persisted_source.last_error_message = "The Source exists; retry its first import in KnowledgeFS"
source_documents = list(
session.scalars(
sa.select(KnowledgeFSUpgradeDocument).where(
KnowledgeFSUpgradeDocument.job_id == job.id,
KnowledgeFSUpgradeDocument.source_key == persisted_source.source_key,
)
)
)
for document in source_documents:
if document.status is not KnowledgeFSUpgradeItemStatus.SUCCEEDED:
document.status = KnowledgeFSUpgradeItemStatus.SUCCEEDED
persisted_job.completed_documents += 1
persisted_job.completed_sources += 1
return True
def _resolve_initial_source_payload(
self, job: KnowledgeFSUpgradeJob, payload_snapshot: dict[str, object]
) -> KnowledgeFSInitialSourcePayload:
payload = dict(payload_snapshot)
legacy_credential_id = payload.pop("legacy_credential_id", None)
workspace_id = payload.pop("legacy_workspace_id", None)
if payload.get("kind") == "online_document":
payload["credential_id"] = self._resolve_notion_credential(
tenant_id=job.tenant_id,
legacy_credential_id=str(legacy_credential_id or ""),
workspace_id=str(workspace_id or ""),
)
return _INITIAL_SOURCE_ADAPTER.validate_python(payload)
def _resolve_notion_credential(self, *, tenant_id: str, legacy_credential_id: str, workspace_id: str) -> str:
with self._session_maker() as session:
direct = session.scalar(
sa.select(DatasourceProvider).where(
DatasourceProvider.id == legacy_credential_id,
DatasourceProvider.tenant_id == tenant_id,
DatasourceProvider.plugin_id == "langgenius/notion_datasource",
DatasourceProvider.provider == "notion_datasource",
)
)
if direct is not None:
return direct.id
candidates = list(
session.scalars(
sa.select(DatasourceProvider)
.where(
DatasourceProvider.tenant_id == tenant_id,
DatasourceProvider.plugin_id == "langgenius/notion_datasource",
DatasourceProvider.provider == "notion_datasource",
)
.order_by(DatasourceProvider.is_default.desc(), DatasourceProvider.created_at)
)
)
for candidate in candidates:
if str(candidate.encrypted_credentials.get("workspace_id") or "") == workspace_id:
return candidate.id
if len(candidates) == 1:
return candidates[0].id
raise KnowledgeFSUpgradeError("A matching Notion datasource credential is unavailable")
def _fail_source(self, *, source_id: str, error: Exception) -> None:
with self._session_maker.begin() as session:
source = session.get(KnowledgeFSUpgradeSource, source_id)
if source is not None:
source.status = KnowledgeFSUpgradeItemStatus.FAILED
source.last_error_code = type(error).__name__[:128]
source.last_error_message = str(error)[:4000]
def _submit_next_document(self, job: KnowledgeFSUpgradeJob) -> bool:
with self._session_maker.begin() as session:
document = session.scalar(
sa.select(KnowledgeFSUpgradeDocument)
.where(
KnowledgeFSUpgradeDocument.job_id == job.id,
KnowledgeFSUpgradeDocument.data_source_type == "upload_file",
KnowledgeFSUpgradeDocument.status.in_(
(KnowledgeFSUpgradeItemStatus.PENDING, KnowledgeFSUpgradeItemStatus.PROCESSING)
),
)
.order_by(KnowledgeFSUpgradeDocument.id)
.with_for_update(skip_locked=True)
.limit(1)
)
if document is None:
persisted = session.get(KnowledgeFSUpgradeJob, job.id)
assert persisted is not None
persisted.stage = KnowledgeFSUpgradeStage.MIGRATING_ACCESS
return True
document.status = KnowledgeFSUpgradeItemStatus.PROCESSING
document_id = document.id
upload_file_id = document.old_upload_file_id
staged_upload_id = document.staged_upload_id
if not upload_file_id:
error = KnowledgeFSUpgradeError("Legacy upload document has no source file reference")
self._fail_document(document_id=document_id, error=error)
raise error
try:
staged = KnowledgeFSStagedUploadService(
self._session_maker,
facade=get_knowledge_fs_runtime(self._session_maker).facade,
)
if staged_upload_id is None:
with self._session_maker() as session:
upload_file = session.scalar(
sa.select(UploadFile).where(
UploadFile.id == upload_file_id,
UploadFile.tenant_id == job.tenant_id,
)
)
account = session.get(Account, job.owner_account_id)
if upload_file is None or account is None:
raise KnowledgeFSUpgradeError("Legacy source file is unavailable")
storage_key = upload_file.key
file_name = upload_file.name
content_type = upload_file.mime_type or "application/octet-stream"
session.expunge(account)
body = storage.load(storage_key)
if not isinstance(body, bytes):
raise KnowledgeFSUpgradeError("Legacy source file returned an invalid body")
staged_response = staged.stage(
tenant_id=job.tenant_id,
account=account,
file_name=file_name,
content_type=content_type,
body=body,
file_size_limit_mb=FeatureService.get_knowledge_file_size_limit(job.tenant_id),
)
staged_upload_id = staged_response.id
with self._session_maker.begin() as session:
persisted_document = session.get(KnowledgeFSUpgradeDocument, document_id)
assert persisted_document is not None
persisted_document.staged_upload_id = staged_upload_id
claimed = staged.claim(
tenant_id=job.tenant_id,
account_id=job.owner_account_id,
control_space_id=_required_space_id(job),
payload=KnowledgeFSDocumentStagedUploadPayload(upload_id=staged_upload_id),
)
# Completion creates a candidate revision before compilation starts. Persist its
# stable logical identity now, including for documents that never become active.
logical_document_id = _resolve_uploaded_document_id(
job, get_knowledge_fs_runtime(self._session_maker).facade, claimed.document_asset_id
)
except Exception as error:
self._fail_document(document_id=document_id, error=error)
raise
with self._session_maker.begin() as session:
persisted_document = session.get(KnowledgeFSUpgradeDocument, document_id)
persisted_job = session.get(KnowledgeFSUpgradeJob, job.id)
assert persisted_document is not None
assert persisted_job is not None
persisted_document.status = KnowledgeFSUpgradeItemStatus.SUCCEEDED
persisted_document.new_document_asset_id = claimed.document_asset_id
persisted_document.new_logical_document_id = logical_document_id
persisted_document.compilation_job_id = claimed.compilation_job_id
persisted_job.completed_documents += 1
remaining_references = session.scalar(
sa.select(sa.func.count(KnowledgeFSUpgradeDocument.id)).where(
KnowledgeFSUpgradeDocument.job_id == job.id,
KnowledgeFSUpgradeDocument.old_upload_file_id == upload_file_id,
KnowledgeFSUpgradeDocument.status != KnowledgeFSUpgradeItemStatus.SUCCEEDED,
)
)
if remaining_references == 0:
release_upgrade_file_lease(
session,
job_id=job.id,
upload_file_id=upload_file_id,
)
return True
def _fail_document(self, *, document_id: str, error: Exception) -> None:
with self._session_maker.begin() as session:
document = session.get(KnowledgeFSUpgradeDocument, document_id)
if document is not None:
document.status = KnowledgeFSUpgradeItemStatus.FAILED
document.last_error_code = type(error).__name__[:128]
document.last_error_message = str(error)[:4000]
def _migrate_access(self, job: KnowledgeFSUpgradeJob) -> None:
"""Copy access policy from a snapshotted job into its existing new space.
The job must have a persisted control space and valid access snapshots.
On success, members, visibility, channel switches, and tags are copied
and the job advances to finalizing. App bindings remain unchanged.
If a write fails, the stage does not advance; earlier writes may remain.
"""
control_space_id = _required_space_id(job)
runtime = get_knowledge_fs_runtime(self._session_maker)
_migrate_metadata_fields(job, runtime.facade)
members = [
KnowledgeFSMemberBindingPayload(
account_id=str(account_id),
role=KnowledgeFSControlSpacePermissionRole.VIEWER,
)
for account_id in cast(list[str], job.permission_snapshot.get("member_account_ids", []))
if str(account_id) != job.owner_account_id
]
runtime.control_plane.replace_members(
tenant_id=job.tenant_id,
actor_account_id=job.owner_account_id,
control_space_id=control_space_id,
members=members,
)
runtime.control_plane.update_visibility(
tenant_id=job.tenant_id,
actor_account_id=job.owner_account_id,
control_space_id=control_space_id,
visibility=KnowledgeFSControlSpaceVisibility(str(job.permission_snapshot["visibility"])),
)
has_agent = any(binding["caller_kind"] == "agent" for binding in job.app_binding_snapshot)
has_workflow = any(binding["caller_kind"] == "workflow" for binding in job.app_binding_snapshot)
runtime.control_plane.update_external_access(
tenant_id=job.tenant_id,
actor_account_id=job.owner_account_id,
control_space_id=control_space_id,
payload=KnowledgeFSExternalAccessPayload(
service_api_enabled=bool(job.config_snapshot.get("enable_api")),
agent_enabled=has_agent,
workflow_enabled=has_workflow,
mcp_enabled=False,
),
)
runtime.space_tags.replace_tags(
tenant_id=job.tenant_id,
account_id=job.owner_account_id,
control_space_id=control_space_id,
tag_ids=list(job.tag_ids_snapshot),
)
with self._session_maker.begin() as session:
persisted = session.get(KnowledgeFSUpgradeJob, job.id)
assert persisted is not None
persisted.stage = KnowledgeFSUpgradeStage.FINALIZING
def _finalize(self, job: KnowledgeFSUpgradeJob) -> None:
with self._session_maker.begin() as session:
persisted = session.scalar(
sa.select(KnowledgeFSUpgradeJob).where(KnowledgeFSUpgradeJob.id == job.id).with_for_update()
)
if persisted is None:
raise KnowledgeFSUpgradeNotFoundError("Upgrade job disappeared")
if persisted.completed_documents != persisted.total_documents:
raise KnowledgeFSUpgradeError("Not all Dataset documents were handed off")
if persisted.completed_sources != persisted.total_sources:
raise KnowledgeFSUpgradeError("Not all Dataset Sources were created")
persisted.status = KnowledgeFSUpgradeJobStatus.SUCCEEDED
persisted.stage = KnowledgeFSUpgradeStage.COMPLETED
persisted.completed_at = naive_utc_now()
persisted.last_error_code = None
persisted.last_error_message = None
class KnowledgeFSUpgradeDocumentReconciler:
"""Apply snapshot metadata and availability after new logical documents appear.
This is deliberately separate from parent-job success: KnowledgeFS owns parsing and
indexing task outcomes. Capability issuance blocks retrieval until disabled documents
have their availability restored; metadata failures cannot delay that checkpoint.
"""
def __init__(self, session_maker: sessionmaker[Session]) -> None:
self._session_maker = session_maker
def claim_deliveries(self, *, job_id: str | None = None, limit: int = 100) -> list[tuple[str, str, str]]:
"""Reserve bounded deliveries from durable unfinished document checkpoints.
A missing/expired lease also covers a publisher crash, lost broker message,
exhausted retry, or killed worker. Running workers have a hard time limit
shorter than the lease. NULL job_id is the periodic cross-workspace sweep;
a concrete id schedules the just-completed parent immediately.
"""
now = naive_utc_now()
pending_document = sa.exists().where(
KnowledgeFSUpgradeDocument.job_id == KnowledgeFSUpgradeJob.id,
KnowledgeFSUpgradeDocument.tenant_id == KnowledgeFSUpgradeJob.tenant_id,
KnowledgeFSUpgradeDocument.status == KnowledgeFSUpgradeItemStatus.SUCCEEDED,
KnowledgeFSUpgradeDocument.state_reconciled_at.is_(None),
)
statement = sa.select(KnowledgeFSUpgradeJob).where(
KnowledgeFSUpgradeJob.status == KnowledgeFSUpgradeJobStatus.SUCCEEDED,
pending_document,
sa.or_(
KnowledgeFSUpgradeJob.reconciliation_lease_expires_at.is_(None),
KnowledgeFSUpgradeJob.reconciliation_lease_expires_at <= now,
),
)
if job_id is not None:
statement = statement.where(KnowledgeFSUpgradeJob.id == job_id)
with self._session_maker.begin() as session:
jobs = session.scalars(
statement.order_by(KnowledgeFSUpgradeJob.updated_at, KnowledgeFSUpgradeJob.id)
.limit(limit)
.with_for_update(skip_locked=True)
).all()
deliveries = []
for job in jobs:
task_id = str(uuid.uuid4())
job.reconciliation_task_id = task_id
job.reconciliation_started_at = None
job.reconciliation_lease_expires_at = now + UPGRADE_DELIVERY_TIMEOUT
deliveries.append((job.tenant_id, job.id, task_id))
return deliveries
def start_delivery(self, *, job_id: str, task_id: str) -> bool:
"""Fence obsolete/duplicate deliveries and extend the lease from actual execution."""
now = naive_utc_now()
with self._session_maker.begin() as session:
job = session.get(KnowledgeFSUpgradeJob, job_id, with_for_update=True)
if (
job is None
or job.status is not KnowledgeFSUpgradeJobStatus.SUCCEEDED
or job.reconciliation_task_id != task_id
or job.reconciliation_started_at is not None
or job.reconciliation_lease_expires_at is None
or job.reconciliation_lease_expires_at <= now
):
return False
job.reconciliation_started_at = now
job.reconciliation_lease_expires_at = now + UPGRADE_DELIVERY_TIMEOUT
return True
def finish_delivery(self, *, job_id: str, task_id: str) -> None:
"""Release this attempt; unfinished checkpoints remain eligible after a short backoff."""
with self._session_maker.begin() as session:
job = session.get(KnowledgeFSUpgradeJob, job_id, with_for_update=True)
if job is None or job.reconciliation_task_id != task_id:
return
job.reconciliation_task_id = None
job.reconciliation_started_at = None
job.reconciliation_lease_expires_at = naive_utc_now() + timedelta(seconds=60)
def reconcile(self, *, job_id: str) -> int:
with self._session_maker() as session:
job = session.get(KnowledgeFSUpgradeJob, job_id)
if job is None or job.status is not KnowledgeFSUpgradeJobStatus.SUCCEEDED:
return 0
documents = list(
session.scalars(
sa.select(KnowledgeFSUpgradeDocument)
.where(
KnowledgeFSUpgradeDocument.job_id == job_id,
KnowledgeFSUpgradeDocument.status == KnowledgeFSUpgradeItemStatus.SUCCEEDED,
KnowledgeFSUpgradeDocument.state_reconciled_at.is_(None),
)
.order_by(KnowledgeFSUpgradeDocument.id)
)
)
sources = {
source.source_key: source.new_source_id
for source in session.scalars(
sa.select(KnowledgeFSUpgradeSource).where(KnowledgeFSUpgradeSource.job_id == job_id)
)
}
session.expunge(job)
for document in documents:
session.expunge(document)
if not documents:
return 0
runtime = get_knowledge_fs_runtime(self._session_maker)
logical_documents = _list_all_logical_documents(job, runtime.facade)
by_document_id = {document.id: document for document in logical_documents}
by_asset_id = {
logical_document.active.document_asset_id: logical_document
for logical_document in logical_documents
if logical_document.active is not None
}
by_source_item = {
(logical_document.source_id, logical_document.provider_item_id): logical_document
for logical_document in logical_documents
if logical_document.source_id and logical_document.provider_item_id
}
remaining = 0
for document in documents:
try:
logical_document = by_document_id.get(document.new_logical_document_id or "")
if logical_document is None and document.new_document_asset_id:
logical_document = by_asset_id.get(document.new_document_asset_id)
elif logical_document is None and document.source_key:
logical_document = by_source_item.get(
(sources.get(document.source_key), _expected_provider_item_id(document))
)
if logical_document is None:
logical_document = self._resolve_document(job, document, runtime.facade)
if logical_document is None:
if document.new_logical_document_id and _upgrade_document_deletion_completed(
job, runtime.facade, document.new_logical_document_id
):
self._finish_document(job, document.id, logical_document_id=document.new_logical_document_id)
continue
raise KnowledgeFSUpgradeNotReadyError("The new logical document is not visible yet")
# Keep the identity even if availability/metadata fails or the document is
# deleted before the next delivery. A missing active revision is not absence.
self._record_document_identity(job, document.id, logical_document.id)
current = logical_document
if document.availability_reconciled_at is None:
if current.enabled != document.desired_enabled:
current = runtime.facade.update_logical_document_availability(
tenant_id=job.tenant_id,
account_id=job.owner_account_id,
control_space_id=_required_space_id(job),
document_id=current.id,
payload=KnowledgeFSDocumentAvailabilityPayload(
enabled=document.desired_enabled,
expectedRowVersion=current.row_version,
),
)
with self._session_maker.begin() as session:
persisted = session.get(KnowledgeFSUpgradeDocument, document.id)
if persisted is not None and persisted.availability_reconciled_at is None:
persisted.new_logical_document_id = current.id
persisted.availability_reconciled_at = naive_utc_now()
metadata_patch = _upgrade_metadata_patch(job, document)
if metadata_patch:
current = runtime.facade.update_document_metadata(
tenant_id=job.tenant_id,
account_id=job.owner_account_id,
control_space_id=_required_space_id(job),
document_id=current.id,
payload=KnowledgeFSDocumentMetadataPayload(
expectedRowVersion=current.row_version,
patch=metadata_patch,
),
)
except Exception as error:
self._record_reconcile_wait(document.id, str(error))
remaining += 1
continue
self._finish_document(job, document.id, logical_document_id=current.id)
return remaining
def _resolve_document(
self, job: KnowledgeFSUpgradeJob, document: KnowledgeFSUpgradeDocument, facade: KnowledgeFSDataFacade
) -> KnowledgeFSLogicalDocumentResponse | None:
try:
if document.new_logical_document_id is None and document.new_document_asset_id:
logical_document_id = _resolve_uploaded_document_id(job, facade, document.new_document_asset_id)
document.new_logical_document_id = logical_document_id
self._record_document_identity(job, document.id, logical_document_id)
if document.new_logical_document_id:
return facade.get_logical_document(
tenant_id=job.tenant_id,
account_id=job.owner_account_id,
control_space_id=_required_space_id(job),
document_id=document.new_logical_document_id,
)
except KnowledgeFSProductResourceNotFoundError:
# 404 alone is not proof of deletion. Only a completed durable deletion
# receipt for the known logical identity may release the retrieval gate.
return None
return None
def _record_document_identity(self, job: KnowledgeFSUpgradeJob, document_id: str, logical_document_id: str) -> None:
with self._session_maker.begin() as session:
document = session.scalar(
sa.select(KnowledgeFSUpgradeDocument).where(
KnowledgeFSUpgradeDocument.id == document_id,
KnowledgeFSUpgradeDocument.job_id == job.id,
KnowledgeFSUpgradeDocument.tenant_id == job.tenant_id,
)
)
if document is not None:
document.new_logical_document_id = logical_document_id
def _finish_document(self, job: KnowledgeFSUpgradeJob, document_id: str, *, logical_document_id: str) -> None:
with self._session_maker.begin() as session:
document = session.scalar(
sa.select(KnowledgeFSUpgradeDocument).where(
KnowledgeFSUpgradeDocument.id == document_id,
KnowledgeFSUpgradeDocument.job_id == job.id,
KnowledgeFSUpgradeDocument.tenant_id == job.tenant_id,
)
)
if document is not None and document.state_reconciled_at is None:
document.new_logical_document_id = logical_document_id
document.availability_reconciled_at = document.availability_reconciled_at or naive_utc_now()
document.state_reconciled_at = naive_utc_now()
document.state_reconcile_error = None
document.state_reconcile_attempt_count += 1
def _record_reconcile_wait(self, document_id: str, message: str) -> None:
with self._session_maker.begin() as session:
document = session.get(KnowledgeFSUpgradeDocument, document_id)
if document is not None and document.state_reconciled_at is None:
document.state_reconcile_attempt_count += 1
document.state_reconcile_error = message[:4000]
def upgrade_job_response(job: KnowledgeFSUpgradeJob) -> KnowledgeFSUpgradeJobResponse:
return KnowledgeFSUpgradeJobResponse(
id=job.id,
old_dataset_id=job.old_dataset_id,
new_control_space_id=job.new_control_space_id,
status=job.status,
stage=job.stage,
snapshot_at=job.snapshot_at,
total_documents=job.total_documents,
completed_documents=job.completed_documents,
total_sources=job.total_sources,
completed_sources=job.completed_sources,
last_error_code=job.last_error_code,
last_error_message=job.last_error_message,
completed_at=job.completed_at,
)
def upgrade_discovery_response(
job: KnowledgeFSUpgradeJob | None,
*,
feature_enabled: bool,
dataset_provider: str,
) -> KnowledgeFSUpgradeDiscoveryResponse:
if not feature_enabled:
return KnowledgeFSUpgradeDiscoveryResponse(
job=upgrade_job_response(job) if job else None,
can_upgrade=False,
can_retry=False,
block_reason="feature_disabled",
)
if dataset_provider == "external":
return KnowledgeFSUpgradeDiscoveryResponse(
job=upgrade_job_response(job) if job else None,
can_upgrade=False,
can_retry=False,
block_reason="unsupported_dataset_provider",
)
if job is None:
return KnowledgeFSUpgradeDiscoveryResponse(can_upgrade=True, can_retry=False)
if job.status in _ACTIVE_JOB_STATUSES:
return KnowledgeFSUpgradeDiscoveryResponse(
job=upgrade_job_response(job),
can_upgrade=False,
can_retry=False,
block_reason="upgrade_in_progress",
)
if job.status is KnowledgeFSUpgradeJobStatus.FAILED:
return KnowledgeFSUpgradeDiscoveryResponse(
job=upgrade_job_response(job),
can_upgrade=True,
can_retry=True,
block_reason="retry_required",
)
return KnowledgeFSUpgradeDiscoveryResponse(
job=upgrade_job_response(job),
can_upgrade=True,
can_retry=False,
)
def _resolve_uploaded_document_id(job: KnowledgeFSUpgradeJob, facade: KnowledgeFSDataFacade, asset_id: str) -> str:
asset = facade.get_document(
tenant_id=job.tenant_id,
account_id=job.owner_account_id,
control_space_id=_required_space_id(job),
document_id=asset_id,
)
reference = facade.resolve_document_reference(
tenant_id=job.tenant_id,
account_id=job.owner_account_id,
control_space_id=_required_space_id(job),
document_asset_id=asset_id,
document_asset_version=asset.version,
)
return reference.document_id
def _upgrade_document_deletion_completed(
job: KnowledgeFSUpgradeJob, facade: KnowledgeFSDataFacade, logical_document_id: str
) -> bool:
"""Require durable deletion evidence, never infer deletion from a filtered list or 404."""
cursor = None
while True:
page = facade.list_background_tasks(
tenant_id=job.tenant_id,
account_id=job.owner_account_id,
control_space_id=_required_space_id(job),
cursor=cursor,
limit=100,
)
for task in page.data:
if task.operation != "document_delete" or task.document_id != logical_document_id:
continue
receipt = facade.get_deletion_job(
tenant_id=job.tenant_id,
account_id=job.owner_account_id,
control_space_id=_required_space_id(job),
job_id=task.id,
)
if (
receipt.knowledge_space_id == task.knowledge_space_id
and receipt.target_type == "logical_document"
and receipt.target_id == logical_document_id
and receipt.run_state == "completed"
and receipt.checkpoint == "completed"
):
return True
cursor = page.next_cursor
if cursor is None:
return False
def _upgrade_metadata_patch(job: KnowledgeFSUpgradeJob, document: KnowledgeFSUpgradeDocument) -> dict[str, object]:
"""Only migrate declared custom fields; KnowledgeFS owns its own system metadata.
Classic time fields store UTC epoch seconds, whereas KnowledgeFS custom time
fields require ISO timestamps. Convert on delivery so old snapshots recover too.
"""
fields = cast(list[dict[str, str]], job.config_snapshot.get("metadata_fields") or [])
patch: dict[str, object] = {}
for field in fields:
name = field["name"]
if name in BuiltInField or name not in document.metadata_snapshot:
continue
value = document.metadata_snapshot[name]
if field["type"] == "time" and value is not None:
try:
if isinstance(value, bool) or not isinstance(value, (int, float)):
raise ValueError("Classic time metadata must contain UTC epoch seconds")
value = datetime.fromtimestamp(value, UTC).isoformat().replace("+00:00", "Z")
except (ValueError, OverflowError, OSError) as error:
raise KnowledgeFSUpgradeError(f"Metadata field {name!r} has an invalid time value") from error
patch[name] = value
return patch
def _migrate_metadata_fields(job: KnowledgeFSUpgradeJob, facade: Any) -> None:
expected_fields = cast(list[dict[str, str]], job.config_snapshot.get("metadata_fields") or [])
if not expected_fields:
return
existing_by_name: dict[str, str] = {}
cursor = None
while True:
page = facade.list_metadata_fields(
tenant_id=job.tenant_id,
account_id=job.owner_account_id,
control_space_id=_required_space_id(job),
cursor=cursor,
limit=100,
)
existing_by_name.update({field.name: field.type for field in page.data})
cursor = page.next_cursor
if cursor is None:
break
for field in expected_fields:
name = str(field["name"])
field_type = str(field["type"])
existing_type = existing_by_name.get(name)
if existing_type is not None:
if existing_type != field_type:
raise KnowledgeFSUpgradeError(f"Metadata field {name!r} has an incompatible type")
continue
facade.create_metadata_field(
tenant_id=job.tenant_id,
account_id=job.owner_account_id,
control_space_id=_required_space_id(job),
payload=KnowledgeFSMetadataFieldCreatePayload.model_validate({"name": name, "type": field_type}),
)
existing_by_name[name] = field_type
def _list_all_logical_documents(job: KnowledgeFSUpgradeJob, facade: Any) -> list[Any]:
documents: list[Any] = []
cursor = None
while True:
page = facade.list_logical_documents(
tenant_id=job.tenant_id,
account_id=job.owner_account_id,
control_space_id=_required_space_id(job),
cursor=cursor,
)
documents.extend(page.data)
cursor = page.next_cursor
if cursor is None:
return documents
def _expected_provider_item_id(document: KnowledgeFSUpgradeDocument) -> str:
info = document.data_source_info
if document.data_source_type == "notion_import":
workspace_id = str(info.get("notion_workspace_id") or info.get("workspace_id") or "")
page_id = str(info.get("notion_page_id") or "")
return json.dumps([workspace_id, page_id], separators=(",", ":"))
if document.data_source_type == "website_crawl":
return sha256(str(info.get("url") or "").encode()).hexdigest()
raise KnowledgeFSUpgradeError(f"Document {document.old_document_id} has no provider item identity")
def _resolve_configuration(job: KnowledgeFSUpgradeJob) -> dict[str, object]:
provider_manager = create_plugin_provider_manager(tenant_id=job.tenant_id)
embedding = _resolve_model(
provider_manager,
tenant_id=job.tenant_id,
model_type=ModelType.TEXT_EMBEDDING,
preferred_provider=cast(str | None, job.config_snapshot.get("embedding_model_provider")),
preferred_model=cast(str | None, job.config_snapshot.get("embedding_model")),
)
retrieval = cast(dict[str, Any], job.config_snapshot.get("retrieval_model") or {})
reranking_model = cast(dict[str, Any], retrieval.get("reranking_model") or {})
rerank = _resolve_model(
provider_manager,
tenant_id=job.tenant_id,
model_type=ModelType.RERANK,
preferred_provider=cast(str | None, reranking_model.get("reranking_provider_name")),
preferred_model=cast(str | None, reranking_model.get("reranking_model_name")),
)
reasoning = _resolve_model(
provider_manager,
tenant_id=job.tenant_id,
model_type=ModelType.LLM,
preferred_provider=None,
preferred_model=None,
)
return _build_resolved_configuration(
job.config_snapshot,
KnowledgeFSUpgradeResolvedModels(embedding=embedding, rerank=rerank, reasoning=reasoning),
)
def _build_resolved_configuration(
config_snapshot: Mapping[str, object], models: KnowledgeFSUpgradeResolvedModels
) -> dict[str, object]:
"""Build a new-space profile from validated models without changing the click-time snapshot."""
retrieval_value = config_snapshot.get("retrieval_model")
retrieval = retrieval_value if isinstance(retrieval_value, dict) else {}
threshold_enabled = bool(retrieval.get("score_threshold_enabled"))
threshold_value = retrieval.get("score_threshold")
profile = KnowledgeFSRetrievalProfileIntent(
default_mode="fast",
reasoning_model=models.reasoning,
rerank=KnowledgeFSRerankIntent(enabled=True, model=models.rerank),
score_threshold=KnowledgeFSScoreThresholdIntent(
enabled=threshold_enabled,
stage="mode-final",
value=float(threshold_value) if threshold_enabled and threshold_value is not None else None,
),
top_k=max(1, min(100, int(retrieval.get("top_k") or 4))),
)
return {
"embedding": models.embedding.model_dump(mode="json", by_alias=True),
"retrieval": profile.model_dump(mode="json", by_alias=True),
}
def _resolve_model(
provider_manager,
*,
tenant_id: str,
model_type: ModelType,
preferred_provider: str | None,
preferred_model: str | None,
) -> KnowledgeFSModelIntent:
active_models = provider_manager.get_configurations(tenant_id).get_models(model_type=model_type, only_active=True)
if preferred_provider and preferred_model:
preferred_provider_id = str(ModelProviderID(preferred_provider))
for active_model in active_models:
active_provider_id = str(ModelProviderID(active_model.provider.provider))
if active_model.model == preferred_model and active_provider_id == preferred_provider_id:
return _model_intent(active_model.provider.provider, active_model.model)
default = provider_manager.get_default_model(tenant_id=tenant_id, model_type=model_type)
if default is None:
raise KnowledgeFSUpgradeError(f"Workspace default {model_type.value} model is unavailable")
default_provider_id = str(ModelProviderID(default.provider.provider))
for active_model in active_models:
if (
active_model.model == default.model
and str(ModelProviderID(active_model.provider.provider)) == default_provider_id
):
return _model_intent(active_model.provider.provider, active_model.model)
raise KnowledgeFSUpgradeError(f"Workspace default {model_type.value} model is not active")
def _model_intent(provider: str, model: str) -> KnowledgeFSModelIntent:
provider_id = ModelProviderID(provider)
return KnowledgeFSModelIntent(plugin_id=provider_id.plugin_id, provider=provider_id.provider_name, model=model)
def _config_snapshot(session: Session, dataset: Dataset) -> dict[str, object]:
icon = None
if isinstance(dataset.icon_info, dict):
candidate = dataset.icon_info.get("icon")
if isinstance(candidate, str) and re.fullmatch(r"(?:builtin:)?[+a-z0-9_-]{1,64}", candidate):
icon = candidate
return {
"name": dataset.name,
"description": dataset.description,
"icon": icon,
"indexing_technique": _enum_value(dataset.indexing_technique) if dataset.indexing_technique else None,
"embedding_model": dataset.embedding_model,
"embedding_model_provider": dataset.embedding_model_provider,
"retrieval_model": dataset.retrieval_model or {},
"summary_index_setting": dataset.summary_index_setting or {},
"built_in_field_enabled": dataset.built_in_field_enabled,
"enable_api": dataset.enable_api,
"metadata_fields": [
{"name": metadata.name, "type": _enum_value(metadata.type)}
for metadata in session.scalars(
sa.select(DatasetMetadata)
.where(
DatasetMetadata.tenant_id == dataset.tenant_id,
DatasetMetadata.dataset_id == dataset.id,
)
.order_by(DatasetMetadata.created_at, DatasetMetadata.id)
)
],
}
def _app_binding_snapshot(session: Session, *, tenant_id: str, dataset_id: str) -> list[dict[str, object]]:
result: list[dict[str, object]] = []
rows = session.execute(
sa.select(AppDatasetJoin.app_id, App.mode)
.join(App, App.id == AppDatasetJoin.app_id)
.where(AppDatasetJoin.dataset_id == dataset_id, App.tenant_id == tenant_id)
.order_by(AppDatasetJoin.app_id)
)
for app_id, mode in rows:
if mode in {AppMode.AGENT, AppMode.AGENT_CHAT}:
caller_kind = KnowledgeFSAppSpaceJoinType.AGENT
elif mode in {AppMode.ADVANCED_CHAT, AppMode.WORKFLOW}:
caller_kind = KnowledgeFSAppSpaceJoinType.WORKFLOW
else:
continue
result.append({"app_id": str(app_id), "caller_kind": caller_kind.value})
return result
def _notion_source_group_key(info: dict[str, Any]) -> str:
workspace_id = str(info.get("notion_workspace_id") or info.get("workspace_id") or "")
credential_id = str(info.get("credential_id") or "")
if not workspace_id:
raise KnowledgeFSUpgradeConflictError("Notion document has no workspace identity")
return f"notion:{workspace_id}:{credential_id}"
def _website_source_group_key(info: dict[str, Any]) -> str:
provider = str(info.get("provider") or "").strip().lower()
job_id = str(info.get("job_id") or "")
if not provider or not info.get("url"):
raise KnowledgeFSUpgradeConflictError("Website document has incomplete source identity")
return f"website:{provider}:{job_id}"
def _source_payload_snapshot(dataset_name: str, group: list[tuple[Document, dict[str, Any]]]) -> dict[str, object]:
first_document, first_info = group[0]
if _enum_value(first_document.data_source_type) == "notion_import":
workspace_id = str(first_info.get("notion_workspace_id") or first_info.get("workspace_id") or "")
return {
"kind": "online_document",
"name": f"{dataset_name} Notion"[:200],
"plugin_id": "langgenius/notion_datasource",
"provider": "notion_datasource",
"datasource": "notion_datasource",
"parameters": {},
"sync_policy": "manual",
"legacy_credential_id": str(first_info.get("credential_id") or ""),
"legacy_workspace_id": workspace_id,
"selection": [
{
"name": document.name,
"page_id": str(info.get("notion_page_id") or ""),
"provider_item_id": json.dumps(
[workspace_id, str(info.get("notion_page_id") or "")], separators=(",", ":")
),
"type": str(info.get("type") or "page"),
"workspace_id": workspace_id,
}
for document, info in group
],
}
provider = str(first_info.get("provider") or "")
return {
"kind": "website_crawl",
"name": f"{dataset_name} Web"[:200],
"provider": provider,
"datasource": "crawl",
"parameters": {"only_main_content": bool(first_info.get("only_main_content", True))},
"root_url": str(first_info.get("url")),
"crawl_options": {"include_subpages": False, "limit": min(200, len(group))},
"selection": [{"source_url": str(info.get("url")), "title": document.name} for document, info in group],
"sync_policy": "manual",
}
def _chunks[T](items: list[T], size: int) -> list[list[T]]:
return [items[index : index + size] for index in range(0, len(items), size)]
def _enum_value(value: object) -> str:
return str(value.value if isinstance(value, Enum) else value)
def _required_space_id(job: KnowledgeFSUpgradeJob) -> str:
if job.new_control_space_id is None:
raise KnowledgeFSUpgradeError("Upgrade Space reference is missing")
return job.new_control_space_id
__all__ = [
"KnowledgeFSUpgradeConflictError",
"KnowledgeFSUpgradeDocumentReconciler",
"KnowledgeFSUpgradeError",
"KnowledgeFSUpgradeNotFoundError",
"KnowledgeFSUpgradeNotReadyError",
"KnowledgeFSUpgradeRunner",
"KnowledgeFSUpgradeSnapshotService",
"upgrade_discovery_response",
"upgrade_job_response",
]