chore: 纳入本地 Dify 1.19.0 源码及品牌与免登改动

演示目录只保留可重建的源码。运行密钥、数据卷、依赖和带口令的本地配置不进入仓库。

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
wuzilongandCursor committed 2026-11-18 13:59:25 +08:00
1 parent 60158935f4
commit 6519682885
15831 files changed
+3063428

No files matched your search

Whitespace-only changes.
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,55 @@
from libs.exception import BaseHTTPException
class DatasetNotInitializedError(BaseHTTPException):
error_code = "dataset_not_initialized"
description = "The dataset is still being initialized or indexing. Please wait a moment."
code = 400
class ArchivedDocumentImmutableError(BaseHTTPException):
error_code = "archived_document_immutable"
description = "The archived document is not editable."
code = 403
class DatasetNameDuplicateError(BaseHTTPException):
error_code = "dataset_name_duplicate"
description = "The dataset name already exists. Please modify your dataset name."
code = 409
class InvalidActionError(BaseHTTPException):
error_code = "invalid_action"
description = "Invalid action."
code = 400
class DocumentAlreadyFinishedError(BaseHTTPException):
error_code = "document_already_finished"
description = "The document has been processed. Please refresh the page or go to the document details."
code = 400
class DocumentIndexingError(BaseHTTPException):
error_code = "document_indexing"
description = "The document is being processed and cannot be edited."
code = 400
class InvalidMetadataError(BaseHTTPException):
error_code = "invalid_metadata"
description = "The metadata content is incorrect. Please check and verify."
code = 400
class DatasetInUseError(BaseHTTPException):
error_code = "dataset_in_use"
description = "The dataset is being used by some apps. Please remove the dataset from the apps before deleting it."
code = 409
class PipelineRunError(BaseHTTPException):
error_code = "pipeline_run_error"
description = "An error occurred while running the pipeline."
code = 500
@@ -0,0 +1,85 @@
from uuid import UUID
from sqlalchemy.orm import Session
from controllers.common.schema import register_response_schema_models, register_schema_models
from controllers.console.app.error import (
CompletionRequestError,
ProviderModelCurrentlyNotSupportError,
ProviderNotInitializeError,
ProviderQuotaExceededError,
)
from controllers.console.app.wraps import with_session
from controllers.console.datasets.hit_testing_base import DatasetsHitTestingBase, HitTestingPayload
from controllers.service_api import service_api_ns
from controllers.service_api.wraps import DatasetApiResource, cloud_edition_billing_rate_limit_check
from fields.hit_testing_fields import HitTestingResponse
from libs.external_api import error_responses
from libs.helper import dump_response
register_schema_models(service_api_ns, HitTestingPayload)
register_response_schema_models(service_api_ns, HitTestingResponse)
@service_api_ns.route(
"/datasets/<uuid:dataset_id>/hit-testing",
doc={"post": {"deprecated": True}},
)
@service_api_ns.route("/datasets/<uuid:dataset_id>/retrieve")
class HitTestingApi(DatasetApiResource, DatasetsHitTestingBase):
@error_responses(
CompletionRequestError,
ProviderModelCurrentlyNotSupportError,
ProviderNotInitializeError,
ProviderQuotaExceededError,
)
@service_api_ns.doc(
summary="Retrieve Chunks from a Knowledge Base / Test Retrieval",
description=(
"Performs a search query against a knowledge base to retrieve the most relevant chunks. This "
"endpoint can be used for both production retrieval and test retrieval."
),
tags=["Knowledge Bases"],
responses={
200: "Retrieval results.",
400: (
"- `dataset_not_initialized` : The dataset is still being initialized or indexing. Please "
"wait a moment.\n"
"- `provider_not_initialize` : No valid model provider credentials found. Please go to "
"Settings -> Model Provider to complete your provider credentials.\n"
"- `provider_quota_exceeded` : Your quota for Dify Hosted OpenAI has been exhausted. Please "
"go to Settings -> Model Provider to complete your own provider credentials.\n"
"- `model_currently_not_support` : Dify Hosted OpenAI trial currently not support the GPT-4 "
"model.\n"
"- `completion_request_error` : Completion request failed.\n"
"- `invalid_param` : Invalid parameter value."
),
403: "`forbidden` : Insufficient permissions.",
404: "`not_found` : Knowledge base not found.",
500: "`internal_server_error` : An internal error occurred during retrieval.",
},
)
@service_api_ns.doc("dataset_hit_testing")
@service_api_ns.doc(description="Perform hit testing on a dataset")
@service_api_ns.doc(params={"dataset_id": "Knowledge base ID."})
@service_api_ns.response(
200,
"Hit testing results",
model=service_api_ns.models[HitTestingResponse.__name__],
)
@service_api_ns.response(401, "Unauthorized - invalid API token")
@service_api_ns.response(404, "Dataset not found")
@service_api_ns.expect(service_api_ns.models[HitTestingPayload.__name__])
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
@with_session
def post(self, session: Session, tenant_id: str, dataset_id: UUID) -> dict[str, object]:
"""Perform hit testing on a dataset.
Tests retrieval performance for the specified dataset.
"""
dataset_id_str = str(dataset_id)
dataset = self.get_and_validate_dataset(session, dataset_id_str)
args = self.parse_args(service_api_ns.payload)
self.hit_testing_args_check(args)
return dump_response(HitTestingResponse, self.perform_hit_testing(session, dataset, args))
@@ -0,0 +1,365 @@
from typing import Literal
from uuid import UUID
from flask_login import current_user
from sqlalchemy.orm import Session
from werkzeug.exceptions import NotFound
from controllers.common.controller_schemas import MetadataUpdatePayload
from controllers.common.schema import register_response_schema_models, register_schema_model, register_schema_models
from controllers.common.session import with_session
from controllers.console.wraps import model_validate
from controllers.service_api import service_api_ns
from controllers.service_api.wraps import DatasetApiResource, cloud_edition_billing_rate_limit_check
from extensions.ext_application_services import application_services
from fields.dataset_fields import (
DatasetMetadataActionResponse,
DatasetMetadataBuiltInFieldsResponse,
DatasetMetadataListResponse,
DatasetMetadataResponse,
)
from libs.helper import dump_response
from services.errors.metadata import MetadataResourceNotFoundError
from services.knowledge.dataset_service import DatasetService
from services.knowledge.entities.knowledge_entities import (
DocumentMetadataOperation,
MetadataArgs,
MetadataDetail,
MetadataOperationData,
)
from services.knowledge.resource_scope import DatasetRef
BUILT_IN_METADATA_ACTION_PARAM = {
"description": "`enable` to activate built-in metadata fields, `disable` to deactivate them.",
"enum": ["enable", "disable"],
"type": "string",
}
register_schema_model(service_api_ns, MetadataUpdatePayload)
register_schema_models(
service_api_ns,
MetadataArgs,
MetadataDetail,
DocumentMetadataOperation,
MetadataOperationData,
)
register_response_schema_models(
service_api_ns,
DatasetMetadataActionResponse,
DatasetMetadataBuiltInFieldsResponse,
DatasetMetadataListResponse,
DatasetMetadataResponse,
)
@service_api_ns.route("/datasets/<uuid:dataset_id>/metadata")
class DatasetMetadataCreateServiceApi(DatasetApiResource):
@service_api_ns.doc(
summary="Create Metadata Field",
description=(
"Create a custom metadata field for the knowledge base. Metadata fields can be used to "
"annotate documents with structured information."
),
tags=["Metadata"],
responses={
201: "Metadata field created successfully.",
},
)
@service_api_ns.expect(service_api_ns.models[MetadataArgs.__name__])
@service_api_ns.doc("create_dataset_metadata")
@service_api_ns.doc(description="Create metadata for a dataset")
@service_api_ns.doc(params={"dataset_id": "Knowledge base ID."})
@service_api_ns.doc(
responses={
201: "Metadata created successfully",
400: "Bad request - invalid or duplicate metadata name",
401: "Unauthorized - invalid API token",
404: "Dataset not found",
}
)
@service_api_ns.response(
201, "Metadata created successfully", service_api_ns.models[DatasetMetadataResponse.__name__]
)
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
@with_session
@model_validate(MetadataArgs)
def post(self, metadata_args: MetadataArgs, session: Session, tenant_id, dataset_id: UUID):
"""Create metadata for a dataset."""
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset_for_tenant(dataset_id_str, str(tenant_id), session=session)
if dataset is None:
raise NotFound("Dataset not found.")
DatasetService.check_dataset_permission(dataset, current_user, session)
session.commit()
try:
metadata = application_services().knowledge.metadata.create_metadata(
DatasetRef(str(tenant_id), dataset_id_str), metadata_args, actor_id=current_user.id
)
except MetadataResourceNotFoundError as exc:
raise NotFound(str(exc)) from exc
return dump_response(DatasetMetadataResponse, metadata), 201
@service_api_ns.doc(
summary="List Metadata Fields",
description=(
"Returns the list of all metadata fields (both custom and built-in) for the knowledge base, "
"along with the count of documents using each field."
),
tags=["Metadata"],
responses={
200: "Metadata fields for the knowledge base.",
},
)
@service_api_ns.doc("get_dataset_metadata")
@service_api_ns.doc(description="Get all metadata for a dataset")
@service_api_ns.doc(params={"dataset_id": "Knowledge base ID."})
@service_api_ns.doc(
responses={
200: "Metadata retrieved successfully",
401: "Unauthorized - invalid API token",
404: "Dataset not found",
}
)
@service_api_ns.response(
200, "Metadata retrieved successfully", service_api_ns.models[DatasetMetadataListResponse.__name__]
)
@with_session(write=False)
def get(self, session: Session, tenant_id, dataset_id: UUID):
"""Get all metadata for a dataset."""
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset_for_tenant(dataset_id_str, str(tenant_id), session=session)
if dataset is None:
raise NotFound("Dataset not found.")
session.commit()
try:
metadata = application_services().knowledge.metadata.get_dataset_metadatas(
DatasetRef(str(tenant_id), dataset_id_str)
)
except MetadataResourceNotFoundError as exc:
raise NotFound(str(exc)) from exc
return dump_response(DatasetMetadataListResponse, metadata), 200
@service_api_ns.route("/datasets/<uuid:dataset_id>/metadata/<uuid:metadata_id>")
class DatasetMetadataServiceApi(DatasetApiResource):
@service_api_ns.doc(
summary="Update Metadata Field",
description="Rename a custom metadata field.",
tags=["Metadata"],
responses={
200: "Metadata field updated successfully.",
},
)
@service_api_ns.expect(service_api_ns.models[MetadataUpdatePayload.__name__])
@service_api_ns.doc("update_dataset_metadata")
@service_api_ns.doc(description="Update metadata name")
@service_api_ns.doc(params={"dataset_id": "Knowledge base ID.", "metadata_id": "Metadata field ID."})
@service_api_ns.doc(
responses={
200: "Metadata updated successfully",
400: "Bad request - invalid or duplicate metadata name",
401: "Unauthorized - invalid API token",
404: "Dataset or metadata not found",
}
)
@service_api_ns.response(
200, "Metadata updated successfully", service_api_ns.models[DatasetMetadataResponse.__name__]
)
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
@with_session
@model_validate(MetadataUpdatePayload)
def patch(self, payload: MetadataUpdatePayload, session: Session, tenant_id, dataset_id: UUID, metadata_id: UUID):
"""Update metadata name."""
dataset_id_str = str(dataset_id)
metadata_id_str = str(metadata_id)
dataset = DatasetService.get_dataset_for_tenant(dataset_id_str, tenant_id, session=session)
if dataset is None:
raise NotFound("Dataset not found.")
DatasetService.check_dataset_permission(dataset, current_user, session)
session.commit()
try:
metadata = application_services().knowledge.metadata.update_metadata_name(
DatasetRef(str(tenant_id), dataset_id_str), metadata_id_str, payload.name, actor_id=current_user.id
)
except MetadataResourceNotFoundError as exc:
raise NotFound(str(exc)) from exc
return dump_response(DatasetMetadataResponse, metadata), 200
@service_api_ns.doc(
summary="Delete Metadata Field",
description=(
"Permanently delete a custom metadata field. Documents using this field will lose their "
"metadata values for it."
),
tags=["Metadata"],
responses={
204: "Success.",
},
)
@service_api_ns.doc("delete_dataset_metadata")
@service_api_ns.doc(description="Delete metadata")
@service_api_ns.doc(params={"dataset_id": "Knowledge base ID.", "metadata_id": "Metadata field ID."})
@service_api_ns.doc(
responses={
204: "Metadata deleted successfully",
401: "Unauthorized - invalid API token",
404: "Dataset or metadata not found",
}
)
@service_api_ns.response(204, "Metadata deleted successfully")
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
@with_session
def delete(self, session: Session, tenant_id, dataset_id: UUID, metadata_id: UUID):
"""Delete metadata."""
dataset_id_str = str(dataset_id)
metadata_id_str = str(metadata_id)
dataset = DatasetService.get_dataset_for_tenant(dataset_id_str, tenant_id, session=session)
if dataset is None:
raise NotFound("Dataset not found.")
DatasetService.check_dataset_permission(dataset, current_user, session)
session.commit()
try:
application_services().knowledge.metadata.delete_metadata(
DatasetRef(str(tenant_id), dataset_id_str), metadata_id_str
)
except MetadataResourceNotFoundError as exc:
raise NotFound(str(exc)) from exc
return "", 204
@service_api_ns.route("/datasets/<uuid:dataset_id>/metadata/built-in")
class DatasetMetadataBuiltInFieldServiceApi(DatasetApiResource):
@service_api_ns.doc(
summary="Get Built-in Metadata Fields",
description=(
"Returns the list of built-in metadata fields provided by the system (e.g., document type, source URL)."
),
tags=["Metadata"],
responses={
200: "Built-in metadata fields.",
404: "`not_found` : Knowledge base not found.",
},
)
@service_api_ns.doc("get_built_in_fields")
@service_api_ns.doc(description="Get all built-in metadata fields")
@service_api_ns.doc(params={"dataset_id": "Knowledge base ID."})
@service_api_ns.doc(
responses={
200: "Built-in fields retrieved successfully",
401: "Unauthorized - invalid API token",
}
)
@service_api_ns.response(
200,
"Built-in fields retrieved successfully",
service_api_ns.models[DatasetMetadataBuiltInFieldsResponse.__name__],
)
def get(self, tenant_id, dataset_id: UUID):
"""Get all built-in metadata fields."""
built_in_fields = application_services().knowledge.metadata.get_built_in_fields()
return dump_response(DatasetMetadataBuiltInFieldsResponse, {"fields": built_in_fields}), 200
@service_api_ns.route("/datasets/<uuid:dataset_id>/metadata/built-in/<string:action>")
class DatasetMetadataBuiltInFieldActionServiceApi(DatasetApiResource):
@service_api_ns.doc(
summary="Update Built-in Metadata Field",
description="Enable or disable built-in metadata fields for the knowledge base.",
tags=["Metadata"],
responses={
200: "Built-in metadata field toggled successfully.",
},
)
@service_api_ns.doc("toggle_built_in_field")
@service_api_ns.doc(description="Enable or disable built-in metadata field")
@service_api_ns.doc(params={"dataset_id": "Knowledge base ID.", "action": BUILT_IN_METADATA_ACTION_PARAM})
@service_api_ns.doc(
responses={
200: "Action completed successfully",
401: "Unauthorized - invalid API token",
404: "Dataset not found",
}
)
@service_api_ns.response(
200, "Action completed successfully", service_api_ns.models[DatasetMetadataActionResponse.__name__]
)
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
@with_session
def post(self, session: Session, tenant_id, dataset_id: UUID, action: Literal["enable", "disable"]):
"""Enable or disable built-in metadata field."""
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset_for_tenant(dataset_id_str, str(tenant_id), session=session)
if dataset is None:
raise NotFound("Dataset not found.")
DatasetService.check_dataset_permission(dataset, current_user, session)
session.commit()
try:
match action:
case "enable":
application_services().knowledge.metadata.enable_built_in_field(
DatasetRef(str(tenant_id), dataset_id_str)
)
case "disable":
application_services().knowledge.metadata.disable_built_in_field(
DatasetRef(str(tenant_id), dataset_id_str)
)
except MetadataResourceNotFoundError as exc:
raise NotFound(str(exc)) from exc
return dump_response(DatasetMetadataActionResponse, {"result": "success"}), 200
@service_api_ns.route("/datasets/<uuid:dataset_id>/documents/metadata")
class DocumentMetadataEditServiceApi(DatasetApiResource):
@service_api_ns.doc(
summary="Update Document Metadata in Batch",
description=(
"Update metadata values for multiple documents at once. Each document in the request "
"receives the specified metadata key-value pairs."
),
tags=["Metadata"],
responses={
200: "Document metadata updated successfully.",
},
)
@service_api_ns.expect(service_api_ns.models[MetadataOperationData.__name__])
@service_api_ns.doc("update_documents_metadata")
@service_api_ns.doc(description="Update metadata for multiple documents")
@service_api_ns.doc(params={"dataset_id": "Knowledge base ID."})
@service_api_ns.doc(
responses={
200: "Documents metadata updated successfully",
400: "Bad request - invalid or conflicting document metadata operation",
401: "Unauthorized - invalid API token",
404: "Dataset, document, or metadata not found",
}
)
@service_api_ns.response(
200,
"Documents metadata updated successfully",
service_api_ns.models[DatasetMetadataActionResponse.__name__],
)
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
@with_session
@model_validate(MetadataOperationData)
def post(self, metadata_args: MetadataOperationData, session: Session, tenant_id, dataset_id: UUID):
"""Update metadata for multiple documents."""
dataset = DatasetService.get_dataset_for_tenant(str(dataset_id), str(tenant_id), session=session)
if dataset is None:
raise NotFound("Dataset not found.")
DatasetService.check_dataset_permission(dataset, current_user, session)
session.commit()
try:
application_services().knowledge.metadata.update_documents_metadata(
DatasetRef(str(tenant_id), str(dataset_id)), metadata_args, actor_id=current_user.id
)
except MetadataResourceNotFoundError as exc:
raise NotFound(str(exc)) from exc
return dump_response(DatasetMetadataActionResponse, {"result": "success"}), 200
@@ -0,0 +1,410 @@
from collections.abc import Generator
from datetime import datetime
from typing import Any
from uuid import UUID
from flask import request
from pydantic import BaseModel, Field, RootModel, field_validator
from sqlalchemy import select
from sqlalchemy.orm import Session
from werkzeug.exceptions import Forbidden, NotFound
import services
from controllers.common.errors import (
FilenameNotExistsError,
FileTooLargeError,
NoFileUploadedError,
TooManyFilesError,
UnsupportedFileTypeError,
)
from controllers.common.fields import WorkflowBlockingResponse
from controllers.common.schema import (
query_params_from_model,
query_params_from_request,
register_response_schema_models,
register_schema_model,
)
from controllers.console.app.wraps import with_session
from controllers.console.wraps import model_validate
from controllers.service_api import service_api_ns
from controllers.service_api.dataset.error import PipelineRunError
from controllers.service_api.schema import event_stream_response, json_or_event_stream_response, multipart_file_params
from controllers.service_api.wraps import DatasetApiResource
from core.app.apps.pipeline.pipeline_generator import PipelineGenerator
from core.app.entities.app_invoke_entities import InvokeFrom
from core.entities.knowledge_entities import PipelineDataset, PipelineDocument
from extensions.ext_application_services import application_services
from fields.base import ResponseModel
from libs import helper
from libs.external_api import error_responses
from libs.helper import dump_response
from libs.login import current_user
from models import Account
from models.dataset import Dataset, Pipeline
from models.engine import db
from services.feature_service import FeatureService
from services.file_service import FileService
from services.rag_pipeline.entity.pipeline_service_api_entities import (
DatasourceNodeRunApiEntity,
DatasourceType,
PipelineRunApiEntity,
)
from services.rag_pipeline.pipeline_generate_service import PipelineGenerateService
from services.rag_pipeline.rag_pipeline import RagPipelineService
class DatasourceNodeRunPayload(BaseModel):
inputs: dict[str, Any] = Field(description="Input variables for the datasource node.")
datasource_type: DatasourceType = Field(description="Type of the datasource.")
credential_id: str | None = Field(
default=None, description="Datasource credential ID. Uses the default if omitted."
)
is_published: bool = Field(
description=(
"Whether to run the published or draft version of the node. `true` runs the published version, "
"`false` runs the draft."
)
)
class DatasourcePluginsQuery(BaseModel):
is_published: bool = Field(
default=True,
description=(
"Whether to retrieve nodes from the published or draft pipeline. `true` returns nodes from the published "
"version, `false` returns nodes from the draft."
),
)
class DatasourceCredentialInfoResponse(ResponseModel):
id: str | None = None
name: str | None = None
type: str | None = None
is_default: bool | None = None
class DatasourcePluginResponse(ResponseModel):
node_id: str | None = None
plugin_id: str | None = None
provider_name: str | None = None
datasource_type: str | None = None
title: str | None = None
user_input_variables: list[dict[str, Any]] = Field(default_factory=list)
credentials: list[DatasourceCredentialInfoResponse] = Field(default_factory=list)
class DatasourcePluginListResponse(RootModel[list[DatasourcePluginResponse]]):
pass
class PipelineUploadFileResponse(ResponseModel):
id: str
name: str
size: int
extension: str
mime_type: str | None = None
created_by: str
created_at: str | None = None
@field_validator("created_at", mode="before")
@classmethod
def _normalize_created_at(cls, value: datetime | str | None) -> str | None:
if isinstance(value, datetime):
return value.isoformat()
return value
class PublishedPipelineRunResponse(ResponseModel):
batch: str
dataset: PipelineDataset
documents: list[PipelineDocument]
class PipelineRunJsonResponse(RootModel[PublishedPipelineRunResponse | WorkflowBlockingResponse]):
"""JSON result for published runs and draft runs using `response_mode: blocking`."""
register_schema_model(service_api_ns, DatasourceNodeRunPayload)
register_schema_model(service_api_ns, DatasourcePluginsQuery)
register_schema_model(service_api_ns, PipelineRunApiEntity)
register_response_schema_models(
service_api_ns,
DatasourceCredentialInfoResponse,
DatasourcePluginResponse,
DatasourcePluginListResponse,
PipelineUploadFileResponse,
WorkflowBlockingResponse,
PublishedPipelineRunResponse,
PipelineRunJsonResponse,
)
@service_api_ns.route("/datasets/<uuid:dataset_id>/pipeline/datasource-plugins")
class DatasourcePluginsApi(DatasetApiResource):
"""Resource for datasource plugins."""
@service_api_ns.doc(
summary="List Datasource Plugins",
description=(
"List the datasource nodes configured in the knowledge pipeline. Each node includes the plugin it uses "
"plus the metadata needed to run it."
),
tags=["Knowledge Pipeline"],
responses={
200: "List of datasource nodes configured in the pipeline.",
404: "`not_found` : Dataset not found.",
},
)
@service_api_ns.doc(shortcut="list_rag_pipeline_datasource_plugins")
@service_api_ns.doc(description="List all datasource plugins for a rag pipeline")
@service_api_ns.doc(params={"dataset_id": "Knowledge base ID."})
@service_api_ns.doc(params=query_params_from_model(DatasourcePluginsQuery))
@service_api_ns.doc(
responses={
200: "Datasource plugins retrieved successfully",
400: "Bad request - pipeline is not configured",
401: "Unauthorized - invalid API token",
}
)
@service_api_ns.response(
200,
"Datasource plugins retrieved successfully",
service_api_ns.models[DatasourcePluginListResponse.__name__],
)
def get(self, tenant_id: str, dataset_id: UUID):
"""Resource for getting datasource plugins."""
dataset_id_str = str(dataset_id)
# Verify dataset ownership
stmt = select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id_str)
dataset = db.session.scalar(stmt)
if not dataset:
raise NotFound("Dataset not found.")
query = query_params_from_request(DatasourcePluginsQuery)
rag_pipeline_service = RagPipelineService(db.session())
datasource_plugins: list[dict[Any, Any]] = rag_pipeline_service.get_datasource_plugins(
tenant_id=tenant_id,
dataset_id=dataset_id_str,
is_published=query.is_published,
credential_query=application_services().credential_queries,
datasource_providers=application_services().data_sources.providers,
)
return dump_response(DatasourcePluginListResponse, datasource_plugins), 200
@service_api_ns.route("/datasets/<uuid:dataset_id>/pipeline/datasource/nodes/<string:node_id>/run")
class DatasourceNodeRunApi(DatasetApiResource):
"""Resource for datasource node run."""
@service_api_ns.doc(
summary="Run Datasource Node",
description=(
"Execute a single datasource node within the knowledge pipeline. Returns a streaming response with the "
"node execution results."
),
tags=["Knowledge Pipeline"],
responses={
200: "Streaming response with node execution events.",
404: "`not_found` : Dataset not found.",
},
)
@event_stream_response(service_api_ns)
@service_api_ns.doc(shortcut="run_datasource_node")
@service_api_ns.doc(description="Run a datasource node for a rag pipeline")
@service_api_ns.doc(params={"dataset_id": "Knowledge base ID.", "node_id": "ID of the datasource node to execute."})
@service_api_ns.doc(
responses={
200: "Datasource node run successfully",
400: "Bad request - invalid payload or pipeline is not configured",
401: "Unauthorized - invalid API token",
}
)
@service_api_ns.expect(service_api_ns.models[DatasourceNodeRunPayload.__name__])
@model_validate(DatasourceNodeRunPayload)
def post(self, payload: DatasourceNodeRunPayload, tenant_id: str, dataset_id: UUID, node_id: str):
"""Resource for getting datasource plugins."""
dataset_id_str = str(dataset_id)
# Verify dataset ownership
stmt = select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id_str)
dataset = db.session.scalar(stmt)
if not dataset:
raise NotFound("Dataset not found.")
assert isinstance(current_user, Account)
rag_pipeline_service: RagPipelineService = RagPipelineService(db.session())
pipeline: Pipeline = rag_pipeline_service.get_pipeline(tenant_id=tenant_id, dataset_id=dataset_id_str)
datasource_node_run_api_entity = DatasourceNodeRunApiEntity.model_validate(
{
**payload.model_dump(exclude_none=True),
"pipeline_id": pipeline.id,
"node_id": node_id,
}
)
# response-contract:ignore compact_generate_response
return helper.compact_generate_response(
PipelineGenerator.convert_to_event_stream(
rag_pipeline_service.run_datasource_workflow_node(
pipeline=pipeline,
node_id=node_id,
user_inputs=datasource_node_run_api_entity.inputs,
account=current_user,
datasource_type=datasource_node_run_api_entity.datasource_type,
is_published=datasource_node_run_api_entity.is_published,
credential_id=datasource_node_run_api_entity.credential_id,
datasource_providers=application_services().data_sources.providers,
)
)
)
@service_api_ns.route("/datasets/<uuid:dataset_id>/pipeline/run")
class PipelineRunApi(DatasetApiResource):
"""Resource for datasource node run."""
@error_responses(PipelineRunError)
@service_api_ns.doc(
summary="Run Pipeline",
description=(
"Execute the full knowledge pipeline for a knowledge base. Published runs are queued and return batch "
"metadata as JSON. Draft runs support blocking JSON and streaming Server-Sent Events."
),
tags=["Knowledge Pipeline"],
responses={
200: (
"Pipeline execution result. Published runs return a JSON object containing `batch`, `dataset`, and "
"`documents`. Draft runs return `text/event-stream` for streaming mode or a workflow result JSON "
"object for blocking mode."
),
403: "`forbidden` : Forbidden.",
404: "`not_found` : Dataset not found.",
500: "`pipeline_run_error` : Pipeline execution failed.",
},
)
@json_or_event_stream_response(service_api_ns)
@service_api_ns.doc(shortcut="run_pipeline")
@service_api_ns.doc(description="Run a datasource node for a rag pipeline")
@service_api_ns.doc(params={"dataset_id": "Knowledge base ID."})
@service_api_ns.doc(
responses={
200: "Pipeline run successfully",
400: "Bad request - invalid payload or pipeline is not configured",
401: "Unauthorized - invalid API token",
}
)
@service_api_ns.expect(service_api_ns.models[PipelineRunApiEntity.__name__])
@service_api_ns.response(
200,
"Pipeline run successfully",
service_api_ns.models[PipelineRunJsonResponse.__name__],
)
@with_session
def post(self, session: Session, tenant_id: str, dataset_id: UUID):
"""Resource for running a rag pipeline."""
dataset_id_str = str(dataset_id)
# Verify dataset ownership
stmt = select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id_str)
dataset = session.scalar(stmt)
if not dataset:
raise NotFound("Dataset not found.")
payload = PipelineRunApiEntity.model_validate(service_api_ns.payload or {})
if not isinstance(current_user, Account):
raise Forbidden()
rag_pipeline_service = RagPipelineService(session)
pipeline = rag_pipeline_service.get_pipeline(tenant_id=tenant_id, dataset_id=dataset_id_str)
try:
response: dict[Any, Any] | Generator[str, Any] = PipelineGenerateService.generate(
generator=application_services().knowledge.pipeline_generator,
session=session,
pipeline=pipeline,
user=current_user,
args=payload.model_dump(),
invoke_from=InvokeFrom.PUBLISHED_PIPELINE if payload.is_published else InvokeFrom.DEBUGGER,
streaming=payload.response_mode == "streaming",
)
# response-contract:ignore compact_generate_response
return helper.compact_generate_response(response)
except Exception as ex:
raise PipelineRunError(description=str(ex)) from ex
@service_api_ns.route("/datasets/pipeline/file-upload")
class KnowledgebasePipelineFileUploadApi(DatasetApiResource):
"""Resource for uploading a file to a knowledgebase pipeline."""
@error_responses(
FilenameNotExistsError, FileTooLargeError, NoFileUploadedError, TooManyFilesError, UnsupportedFileTypeError
)
@service_api_ns.doc(
summary="Upload Pipeline File",
description="Upload a file for use in a knowledge pipeline. Accepts a single file via `multipart/form-data`.",
tags=["Knowledge Pipeline"],
responses={
201: "File uploaded successfully.",
400: (
"- `no_file_uploaded` : Please upload your file.\n"
"- `filename_not_exists_error` : The specified filename does not exist.\n"
"- `too_many_files` : Only one file is allowed."
),
413: "`file_too_large` : File size exceeded.",
415: "`unsupported_file_type` : File type not allowed.",
},
)
@service_api_ns.doc(shortcut="knowledgebase_pipeline_file_upload")
@service_api_ns.doc(description="Upload a file to a knowledgebase pipeline")
@service_api_ns.doc(consumes=["multipart/form-data"], params=multipart_file_params(include_user=False))
@service_api_ns.doc(
responses={
201: "File uploaded successfully",
400: "Bad request - no file or invalid file",
401: "Unauthorized - invalid API token",
413: "File too large",
415: "Unsupported file type",
}
)
@service_api_ns.response(
201,
"File uploaded successfully",
service_api_ns.models[PipelineUploadFileResponse.__name__],
)
def post(self, tenant_id: str):
"""Upload a file for use in conversations.
Accepts a single file upload via multipart/form-data.
"""
# check file
if "file" not in request.files:
raise NoFileUploadedError()
if len(request.files) > 1:
raise TooManyFilesError()
file = request.files["file"]
if not file.mimetype:
raise UnsupportedFileTypeError()
if not file.filename:
raise FilenameNotExistsError
if not current_user:
raise ValueError("Invalid user account")
try:
upload_file = FileService(db.engine).upload_file(
filename=file.filename,
content=file.stream.read(),
mimetype=file.mimetype,
user=current_user,
default_file_size_limit=FeatureService.get_knowledge_file_size_limit(tenant_id),
)
except services.errors.file.FileTooLargeError as file_too_large_error:
raise FileTooLargeError(file_too_large_error.description) from file_too_large_error
except services.errors.file.UnsupportedFileTypeError as caught_error:
raise UnsupportedFileTypeError() from caught_error
return dump_response(PipelineUploadFileResponse, upload_file), 201
@@ -0,0 +1,866 @@
from typing import cast
from uuid import UUID
from pydantic import BaseModel, Field, ValidationError, field_validator
from sqlalchemy import select
from sqlalchemy.orm import Session
from werkzeug.exceptions import NotFound
from configs import dify_config
from controllers.common.controller_schemas import ChildChunkCreatePayload, ChildChunkUpdatePayload
from controllers.common.fields import ErrorMessageResponse
from controllers.common.schema import (
query_params_from_model,
query_params_from_request,
register_response_schema_models,
register_schema_models,
)
from controllers.common.session import with_session
from controllers.console.wraps import model_validate
from controllers.service_api import service_api_ns
from controllers.service_api.app.error import ProviderNotInitializeError
from controllers.service_api.wraps import (
DatasetApiResource,
cloud_edition_billing_knowledge_limit_check,
cloud_edition_billing_rate_limit_check,
cloud_edition_billing_resource_check,
)
from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
from core.model_manager import ModelManager
from core.rag.index_processor.constant.index_type import IndexTechniqueType
from extensions.ext_application_services import application_services
from fields.base import ResponseModel
from fields.segment_fields import (
ChildChunkDetailResponse,
ChildChunkListResponse,
SegmentDetailResponse,
SegmentResponse,
)
from graphon.model_runtime.entities.model_entities import ModelType
from libs.external_api import error_responses
from libs.helper import dump_response
from libs.login import current_account_with_tenant
from models.dataset import Dataset, Document, DocumentSegment
from services.errors.chunk import ChildChunkDeleteIndexError, ChildChunkIndexingError
from services.errors.chunk import ChildChunkDeleteIndexError as ChildChunkDeleteIndexServiceError
from services.errors.chunk import ChildChunkIndexingError as ChildChunkIndexingServiceError
from services.knowledge.dataset_read_service import load_segment_detail, load_segment_details
from services.knowledge.dataset_service import DatasetService, DocumentService, SegmentService
from services.knowledge.entities.segments import SegmentUpdateArgs
from services.knowledge.resource_scope import DatasetRef, SegmentRef
from services.knowledge.summaries.adapters import SummaryIndexAdapter
class SegmentCreateItemPayload(BaseModel):
content: str = Field(min_length=1, description="Chunk text content.")
answer: str | None = Field(default=None, description="Answer content for QA mode.")
keywords: list[str] | None = Field(default=None, description="Keywords for the chunk.")
attachment_ids: list[str] | None = Field(default=None, description="Attachment file IDs.")
@field_validator("content")
@classmethod
def validate_content(cls, value: str) -> str:
if not value.strip():
raise ValueError("Content is empty")
return value
class SegmentCreatePayload(BaseModel):
segments: list[SegmentCreateItemPayload] = Field(min_length=1, description="Array of chunk objects to create.")
class SegmentListQuery(BaseModel):
limit: int = Field(default=20, ge=1, description="Number of items per page. Server caps at `100`.")
page: int = Field(default=1, ge=1, description="Page number to retrieve.")
status: list[str] = Field(
default_factory=list,
description="Filter chunks by indexing status, such as `completed`, `indexing`, or `error`.",
)
keyword: str | None = Field(default=None, description="Search keyword.")
class SegmentUpdatePayload(BaseModel):
segment: SegmentUpdateArgs = Field(description="Chunk data to update.")
class ChildChunkListQuery(BaseModel):
limit: int = Field(default=20, ge=1, description="Number of items per page. Server caps at `100`.")
keyword: str | None = Field(default=None, description="Search keyword.")
page: int = Field(default=1, ge=1, description="Page number to retrieve.")
class SegmentDocParams:
DATASET_DOCUMENT = {"dataset_id": "Knowledge base ID.", "document_id": "Document ID."}
DATASET_DOCUMENT_SEGMENT = {**DATASET_DOCUMENT, "segment_id": "Chunk ID."}
DATASET_DOCUMENT_PARENT_SEGMENT = {**DATASET_DOCUMENT, "segment_id": "Chunk ID."}
DATASET_DOCUMENT_CHILD_CHUNK = {**DATASET_DOCUMENT_PARENT_SEGMENT, "child_chunk_id": "Child chunk ID."}
class SegmentCreateListResponse(ResponseModel):
data: list[SegmentResponse]
doc_form: str
class SegmentListResponse(ResponseModel):
data: list[SegmentResponse]
doc_form: str
total: int
has_more: bool
limit: int
page: int
register_schema_models(
service_api_ns,
SegmentCreatePayload,
SegmentCreateItemPayload,
SegmentListQuery,
SegmentUpdateArgs,
SegmentUpdatePayload,
ChildChunkCreatePayload,
ChildChunkListQuery,
ChildChunkUpdatePayload,
)
register_response_schema_models(
service_api_ns,
ErrorMessageResponse,
SegmentResponse,
SegmentCreateListResponse,
SegmentListResponse,
SegmentDetailResponse,
ChildChunkDetailResponse,
ChildChunkListResponse,
)
def _get_segment_for_document(
session: Session, dataset: Dataset, document: Document, segment_id: str
) -> tuple[SegmentRef, DocumentSegment]:
if document.tenant_id != dataset.tenant_id or document.dataset_id != dataset.id:
raise NotFound("Document not found.")
dataset_ref = DatasetRef(tenant_id=dataset.tenant_id, dataset_id=dataset.id)
segment_ref = dataset_ref.document(document.id).segment(segment_id)
segment = SegmentService.get_segment_by_ref(segment_ref, session=session)
if not segment:
raise NotFound("Segment not found.")
return segment_ref, segment
@service_api_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments")
class SegmentApi(DatasetApiResource):
"""Resource for segments."""
@error_responses(ProviderNotInitializeError)
@service_api_ns.doc(
summary="Create Chunks",
description=(
"Create one or more chunks within a document. Each chunk can include optional keywords and an "
"answer field (for QA-mode documents)."
),
tags=["Chunks"],
responses={
200: "Chunks created successfully.",
404: "`not_found` : Document is not completed or is disabled.",
},
)
@service_api_ns.expect(service_api_ns.models[SegmentCreatePayload.__name__])
@service_api_ns.doc("create_segments")
@service_api_ns.doc(description="Create segments in a document")
@service_api_ns.doc(params=SegmentDocParams.DATASET_DOCUMENT)
@service_api_ns.doc(
responses={
200: "Segments created successfully",
400: "Bad request - segments data is missing",
401: "Unauthorized - invalid API token",
404: "Dataset or document not found",
}
)
@service_api_ns.response(
200,
"Segments created successfully",
service_api_ns.models[SegmentCreateListResponse.__name__],
)
@service_api_ns.response(
400,
"Bad request - segments data is missing",
service_api_ns.models[ErrorMessageResponse.__name__],
)
@cloud_edition_billing_resource_check("vector_space", "dataset")
@cloud_edition_billing_knowledge_limit_check("add_segment", "dataset")
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
@with_session
def post(self, session: Session, tenant_id: str, dataset_id: UUID, document_id: UUID):
_, current_tenant_id = current_account_with_tenant()
"""Create single segment."""
dataset_id_str = str(dataset_id)
# check dataset
dataset = session.scalar(
select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id_str).limit(1)
)
if not dataset:
raise NotFound("Dataset not found.")
document_id_str = str(document_id)
# check document
document = DocumentService.get_document(dataset.id, document_id_str, session=session)
if not document:
raise NotFound("Document not found.")
if document.indexing_status != "completed":
raise NotFound("Document is not completed.")
if not document.enabled:
raise NotFound("Document is disabled.")
# check embedding model setting
if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
try:
model_manager = ModelManager.for_tenant(tenant_id=current_tenant_id)
model_manager.get_model_instance(
tenant_id=current_tenant_id,
provider=dataset.embedding_model_provider,
model_type=ModelType.TEXT_EMBEDDING,
model=dataset.embedding_model,
)
except LLMBadRequestError as caught_error:
raise ProviderNotInitializeError(
"No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
) from caught_error
except ProviderTokenNotInitError as ex:
raise ProviderNotInitializeError(ex.description) from ex
# validate args
try:
payload = SegmentCreatePayload.model_validate(service_api_ns.payload or {})
except ValidationError as e:
return {"error": str(e)}, 400
segments_limit = dify_config.DATASET_MAX_SEGMENTS_PER_REQUEST
if segments_limit > 0 and len(payload.segments) > segments_limit:
raise ValueError(f"Exceeded maximum segments limit of {segments_limit}.")
segment_items = [segment.model_dump(exclude_none=True) for segment in payload.segments]
for args_item in segment_items:
SegmentService.segment_create_args_validate(args_item, document)
segments = cast(
list[DocumentSegment],
SegmentService.multi_create_segment(
segment_items,
document,
dataset,
session,
mutations=application_services().knowledge.segments.mutations,
),
)
segment_ids = [segment.id for segment in segments]
summaries: dict[str, str | None] = {}
if segment_ids:
summary_records = SummaryIndexAdapter.get_segments_summaries(
segment_ids=segment_ids, dataset_id=dataset_id_str, session=session
)
summaries = {chunk_id: record.summary_content for chunk_id, record in summary_records.items()}
response = {
"data": load_segment_details(segments, summaries, session=session),
"doc_form": document.doc_form,
}
return dump_response(SegmentCreateListResponse, response), 200
@error_responses(ProviderNotInitializeError)
@service_api_ns.doc(
summary="List Chunks",
description="Returns a paginated list of chunks within a document. Supports filtering by keyword and status.",
tags=["Chunks"],
responses={
200: "List of chunks.",
},
)
@service_api_ns.doc("list_segments")
@service_api_ns.doc(description="List segments in a document")
@service_api_ns.doc(params=SegmentDocParams.DATASET_DOCUMENT)
@service_api_ns.doc(params=query_params_from_model(SegmentListQuery))
@service_api_ns.doc(
responses={
200: "Segments retrieved successfully",
400: "Bad request - embedding model is not configured",
401: "Unauthorized - invalid API token",
404: "Dataset or document not found",
}
)
@service_api_ns.response(
200,
"Segments retrieved successfully",
service_api_ns.models[SegmentListResponse.__name__],
)
@with_session
def get(self, session: Session, tenant_id: str, dataset_id: UUID, document_id: UUID):
_, current_tenant_id = current_account_with_tenant()
"""Get segments."""
# check dataset
args = query_params_from_request(
SegmentListQuery,
list_fields=("status",),
use_defaults_for_malformed_ints=True,
)
page = args.page
limit = min(args.limit, 100)
dataset_id_str = str(dataset_id)
dataset = session.scalar(
select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id_str).limit(1)
)
if not dataset:
raise NotFound("Dataset not found.")
document_id_str = str(document_id)
# check document
document = DocumentService.get_document(dataset.id, document_id_str, session=session)
if not document:
raise NotFound("Document not found.")
# check embedding model setting
if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
try:
model_manager = ModelManager.for_tenant(tenant_id=current_tenant_id)
model_manager.get_model_instance(
tenant_id=current_tenant_id,
provider=dataset.embedding_model_provider,
model_type=ModelType.TEXT_EMBEDDING,
model=dataset.embedding_model,
)
except LLMBadRequestError as caught_error:
raise ProviderNotInitializeError(
"No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
) from caught_error
except ProviderTokenNotInitError as ex:
raise ProviderNotInitializeError(ex.description) from ex
segments, total = SegmentService.get_segments(
session=session,
document_id=document_id_str,
tenant_id=current_tenant_id,
status_list=args.status,
keyword=args.keyword,
page=page,
limit=limit,
)
segment_ids = [segment.id for segment in segments]
summaries: dict[str, str | None] = {}
if segment_ids:
summary_records = SummaryIndexAdapter.get_segments_summaries(
segment_ids=segment_ids, dataset_id=dataset_id_str, session=session
)
summaries = {chunk_id: record.summary_content for chunk_id, record in summary_records.items()}
response = {
"data": load_segment_details(segments, summaries, session=session),
"doc_form": document.doc_form,
"total": total,
"has_more": page * limit < total,
"limit": limit,
"page": page,
}
return dump_response(SegmentListResponse, response), 200
@service_api_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments/<uuid:segment_id>")
class DatasetSegmentApi(DatasetApiResource):
@service_api_ns.doc(
summary="Delete Chunk",
description="Permanently delete a chunk from the document.",
tags=["Chunks"],
responses={
204: "Success.",
},
)
@service_api_ns.doc("delete_segment")
@service_api_ns.doc(description="Delete a specific segment")
@service_api_ns.doc(params=SegmentDocParams.DATASET_DOCUMENT_SEGMENT)
@service_api_ns.doc(
responses={
204: "Segment deleted successfully",
400: "Bad request - invalid dataset model state or concurrent deletion",
401: "Unauthorized - invalid API token",
404: "Dataset, document, or segment not found",
}
)
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
@with_session
def delete(self, session: Session, tenant_id: str, dataset_id: UUID, document_id: UUID, segment_id: UUID):
current_account_with_tenant()
dataset_id_str = str(dataset_id)
# check dataset
dataset = session.scalar(
select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id_str).limit(1)
)
if not dataset:
raise NotFound("Dataset not found.")
# check user's model setting
DatasetService.check_dataset_model_setting(dataset)
document_id_str = str(document_id)
# check document
document = DocumentService.get_document(dataset_id_str, document_id_str, session=session)
if not document:
raise NotFound("Document not found.")
segment_id_str = str(segment_id)
_, segment = _get_segment_for_document(session, dataset, document, segment_id_str)
SegmentService.delete_segment(
segment, document, dataset, session, mutations=application_services().knowledge.segments.mutations
)
return "", 204
@error_responses(ProviderNotInitializeError)
@service_api_ns.doc(
summary="Update Chunk",
description="Update a chunk's content, keywords, or answer. Re-triggers indexing for the modified chunk.",
tags=["Chunks"],
responses={
200: "Chunk updated successfully.",
},
)
@service_api_ns.expect(service_api_ns.models[SegmentUpdatePayload.__name__])
@service_api_ns.doc("update_segment")
@service_api_ns.doc(description="Update a specific segment")
@service_api_ns.doc(params=SegmentDocParams.DATASET_DOCUMENT_SEGMENT)
@service_api_ns.doc(
responses={
200: "Segment updated successfully",
400: "Bad request - invalid segment or embedding model configuration",
401: "Unauthorized - invalid API token",
404: "Dataset, document, or segment not found",
}
)
@service_api_ns.response(200, "Segment updated successfully", service_api_ns.models[SegmentDetailResponse.__name__])
@cloud_edition_billing_resource_check("vector_space", "dataset")
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
@with_session
@model_validate(SegmentUpdatePayload)
def post(
self,
payload: SegmentUpdatePayload,
session: Session,
tenant_id: str,
dataset_id: UUID,
document_id: UUID,
segment_id: UUID,
):
current_account, current_tenant_id = current_account_with_tenant()
dataset_id_str = str(dataset_id)
# check dataset
dataset = session.scalar(
select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id_str).limit(1)
)
if not dataset:
raise NotFound("Dataset not found.")
# check user's model setting
DatasetService.check_dataset_model_setting(dataset)
document_id_str = str(document_id)
# check document
document = DocumentService.get_document(dataset_id_str, document_id_str, session=session)
if not document:
raise NotFound("Document not found.")
if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
# check embedding model setting
try:
model_manager = ModelManager.for_tenant(tenant_id=current_tenant_id)
model_manager.get_model_instance(
tenant_id=current_tenant_id,
provider=dataset.embedding_model_provider,
model_type=ModelType.TEXT_EMBEDDING,
model=dataset.embedding_model,
)
except LLMBadRequestError as caught_error:
raise ProviderNotInitializeError(
"No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
) from caught_error
except ProviderTokenNotInitError as ex:
raise ProviderNotInitializeError(ex.description) from ex
segment_id_str = str(segment_id)
_, segment = _get_segment_for_document(session, dataset, document, segment_id_str)
updated_segment = SegmentService.update_segment(
payload.segment,
segment,
document,
dataset,
session,
actor_id=current_account.id,
mutations=application_services().knowledge.segments.mutations,
)
summary = SummaryIndexAdapter.get_segment_summary(
segment_id=updated_segment.id, dataset_id=dataset_id_str, session=session
)
response = {
"data": load_segment_detail(updated_segment, summary.summary_content if summary else None, session=session),
"doc_form": document.doc_form,
}
return dump_response(SegmentDetailResponse, response), 200
@service_api_ns.doc(
summary="Get Chunk",
description=(
"Retrieve detailed information about a specific chunk, including its content, keywords, and "
"indexing status."
),
tags=["Chunks"],
responses={
200: "Chunk details.",
},
)
@service_api_ns.doc("get_segment")
@service_api_ns.doc(description="Get a specific segment by ID")
@service_api_ns.doc(params=SegmentDocParams.DATASET_DOCUMENT_SEGMENT)
@service_api_ns.doc(
responses={
200: "Segment retrieved successfully",
400: "Bad request - invalid dataset model configuration",
401: "Unauthorized - invalid API token",
404: "Dataset, document, or segment not found",
}
)
@service_api_ns.response(
200,
"Segment retrieved successfully",
service_api_ns.models[SegmentDetailResponse.__name__],
)
@with_session
def get(self, session: Session, tenant_id: str, dataset_id: UUID, document_id: UUID, segment_id: UUID):
current_account_with_tenant()
dataset_id_str = str(dataset_id)
# check dataset
dataset = session.scalar(
select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id_str).limit(1)
)
if not dataset:
raise NotFound("Dataset not found.")
# check user's model setting
DatasetService.check_dataset_model_setting(dataset)
document_id_str = str(document_id)
# check document
document = DocumentService.get_document(dataset_id_str, document_id_str, session=session)
if not document:
raise NotFound("Document not found.")
segment_id_str = str(segment_id)
_, segment = _get_segment_for_document(session, dataset, document, segment_id_str)
summary = SummaryIndexAdapter.get_segment_summary(
segment_id=segment.id, dataset_id=dataset_id_str, session=session
)
response = {
"data": load_segment_detail(segment, summary.summary_content if summary else None, session=session),
"doc_form": document.doc_form,
}
return dump_response(SegmentDetailResponse, response), 200
@service_api_ns.route(
"/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments/<uuid:segment_id>/child_chunks"
)
class ChildChunkApi(DatasetApiResource):
"""Resource for child chunks."""
@error_responses(ProviderNotInitializeError)
@service_api_ns.doc(
summary="Create Child Chunk",
description="Create a child chunk under the specified segment.",
tags=["Chunks"],
responses={
200: "Child chunk created successfully.",
400: "`invalid_param` : Create child chunk index failed.",
},
)
@service_api_ns.expect(service_api_ns.models[ChildChunkCreatePayload.__name__])
@service_api_ns.doc("create_child_chunk")
@service_api_ns.doc(description="Create a new child chunk for a segment")
@service_api_ns.doc(params=SegmentDocParams.DATASET_DOCUMENT_PARENT_SEGMENT)
@service_api_ns.doc(
responses={
200: "Child chunk created successfully",
401: "Unauthorized - invalid API token",
404: "Dataset, document, or segment not found",
}
)
@service_api_ns.response(
200,
"Child chunk created successfully",
service_api_ns.models[ChildChunkDetailResponse.__name__],
)
@cloud_edition_billing_resource_check("vector_space", "dataset")
@cloud_edition_billing_knowledge_limit_check("add_segment", "dataset")
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
@with_session
@model_validate(ChildChunkCreatePayload)
def post(
self,
payload: ChildChunkCreatePayload,
session: Session,
tenant_id: str,
dataset_id: UUID,
document_id: UUID,
segment_id: UUID,
):
current_account, current_tenant_id = current_account_with_tenant()
"""Create child chunk."""
dataset_id_str = str(dataset_id)
# check dataset
dataset = session.scalar(
select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id_str).limit(1)
)
if not dataset:
raise NotFound("Dataset not found.")
document_id_str = str(document_id)
# check document
document = DocumentService.get_document(dataset.id, document_id_str, session=session)
if not document:
raise NotFound("Document not found.")
segment_id_str = str(segment_id)
_, segment = _get_segment_for_document(session, dataset, document, segment_id_str)
# check embedding model setting
if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
try:
model_manager = ModelManager.for_tenant(tenant_id=current_tenant_id)
model_manager.get_model_instance(
tenant_id=current_tenant_id,
provider=dataset.embedding_model_provider,
model_type=ModelType.TEXT_EMBEDDING,
model=dataset.embedding_model,
)
except LLMBadRequestError as caught_error:
raise ProviderNotInitializeError(
"No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
) from caught_error
except ProviderTokenNotInitError as ex:
raise ProviderNotInitializeError(ex.description) from ex
try:
child_chunk = SegmentService.create_child_chunk(
payload.content,
segment,
document,
dataset,
session,
actor_id=current_account.id,
mutations=application_services().knowledge.segments.mutations,
)
except ChildChunkIndexingServiceError as e:
raise ChildChunkIndexingError(str(e)) from e
return dump_response(ChildChunkDetailResponse, {"data": child_chunk}), 200
@service_api_ns.doc(
summary="List Child Chunks",
description="Returns a paginated list of child chunks under a specific parent chunk.",
tags=["Chunks"],
responses={
200: "List of child chunks.",
},
)
@service_api_ns.doc("list_child_chunks")
@service_api_ns.doc(description="List child chunks for a segment")
@service_api_ns.doc(params=SegmentDocParams.DATASET_DOCUMENT_PARENT_SEGMENT)
@service_api_ns.doc(params=query_params_from_model(ChildChunkListQuery))
@service_api_ns.doc(
responses={
200: "Child chunks retrieved successfully",
401: "Unauthorized - invalid API token",
404: "Dataset, document, or segment not found",
}
)
@service_api_ns.response(
200,
"Child chunks retrieved successfully",
service_api_ns.models[ChildChunkListResponse.__name__],
)
@with_session
def get(self, session: Session, tenant_id: str, dataset_id: UUID, document_id: UUID, segment_id: UUID):
current_account_with_tenant()
"""Get child chunks."""
dataset_id_str = str(dataset_id)
# check dataset
dataset = session.scalar(
select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id_str).limit(1)
)
if not dataset:
raise NotFound("Dataset not found.")
document_id_str = str(document_id)
# check document
document = DocumentService.get_document(dataset.id, document_id_str, session=session)
if not document:
raise NotFound("Document not found.")
segment_id_str = str(segment_id)
_get_segment_for_document(session, dataset, document, segment_id_str)
args = query_params_from_request(ChildChunkListQuery, use_defaults_for_malformed_ints=True)
page = args.page
limit = min(args.limit, 100)
keyword = args.keyword
child_chunks = SegmentService.get_child_chunks(
segment_id_str,
document_id_str,
dataset_id_str,
page,
limit,
keyword,
session=session,
)
response = {
"data": child_chunks.items,
"total": child_chunks.total,
"total_pages": child_chunks.pages,
"page": page,
"limit": limit,
}
return dump_response(ChildChunkListResponse, response), 200
@service_api_ns.route(
"/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments/<uuid:segment_id>/child_chunks/<uuid:child_chunk_id>"
)
class DatasetChildChunkApi(DatasetApiResource):
"""Resource for updating child chunks."""
@service_api_ns.doc(
summary="Delete Child Chunk",
description="Permanently delete a child chunk from its parent chunk.",
tags=["Chunks"],
responses={
204: "Success.",
400: "`invalid_param` : Delete child chunk index failed.",
},
)
@service_api_ns.doc("delete_child_chunk")
@service_api_ns.doc(description="Delete a specific child chunk")
@service_api_ns.doc(params=SegmentDocParams.DATASET_DOCUMENT_CHILD_CHUNK)
@service_api_ns.doc(
responses={
204: "Child chunk deleted successfully",
401: "Unauthorized - invalid API token",
404: "Dataset, document, segment, or child chunk not found",
}
)
@cloud_edition_billing_knowledge_limit_check("add_segment", "dataset")
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
@with_session
def delete(
self,
session: Session,
tenant_id: str,
dataset_id: UUID,
document_id: UUID,
segment_id: UUID,
child_chunk_id: UUID,
):
current_account_with_tenant()
"""Delete child chunk."""
dataset_id_str = str(dataset_id)
# check dataset
dataset = session.scalar(
select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id_str).limit(1)
)
if not dataset:
raise NotFound("Dataset not found.")
document_id_str = str(document_id)
# check document
document = DocumentService.get_document(dataset.id, document_id_str, session=session)
if not document:
raise NotFound("Document not found.")
segment_id_str = str(segment_id)
segment_ref, _ = _get_segment_for_document(session, dataset, document, segment_id_str)
child_chunk_id_str = str(child_chunk_id)
# check child chunk
child_chunk = SegmentService.get_child_chunk_by_segment_ref(child_chunk_id_str, segment_ref, session=session)
if not child_chunk:
raise NotFound("Child chunk not found.")
try:
SegmentService.delete_child_chunk(
child_chunk, dataset, session, mutations=application_services().knowledge.segments.mutations
)
except ChildChunkDeleteIndexServiceError as e:
raise ChildChunkDeleteIndexError(str(e)) from e
return "", 204
@service_api_ns.doc(
summary="Update Child Chunk",
description="Update the content of an existing child chunk.",
tags=["Chunks"],
responses={
200: "Child chunk updated successfully.",
400: "`invalid_param` : Update child chunk index failed.",
},
)
@service_api_ns.expect(service_api_ns.models[ChildChunkUpdatePayload.__name__])
@service_api_ns.doc("update_child_chunk")
@service_api_ns.doc(description="Update a specific child chunk")
@service_api_ns.doc(params=SegmentDocParams.DATASET_DOCUMENT_CHILD_CHUNK)
@service_api_ns.doc(
responses={
200: "Child chunk updated successfully",
401: "Unauthorized - invalid API token",
404: "Dataset, document, segment, or child chunk not found",
}
)
@service_api_ns.response(
200,
"Child chunk updated successfully",
service_api_ns.models[ChildChunkDetailResponse.__name__],
)
@cloud_edition_billing_resource_check("vector_space", "dataset")
@cloud_edition_billing_knowledge_limit_check("add_segment", "dataset")
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
@with_session
@model_validate(ChildChunkUpdatePayload)
def patch(
self,
payload: ChildChunkUpdatePayload,
session: Session,
tenant_id: str,
dataset_id: UUID,
document_id: UUID,
segment_id: UUID,
child_chunk_id: UUID,
):
current_account, _ = current_account_with_tenant()
"""Update child chunk."""
dataset_id_str = str(dataset_id)
# check dataset
dataset = session.scalar(
select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id_str).limit(1)
)
if not dataset:
raise NotFound("Dataset not found.")
document_id_str = str(document_id)
# get document
document = DocumentService.get_document(dataset_id_str, document_id_str, session=session)
if not document:
raise NotFound("Document not found.")
segment_id_str = str(segment_id)
segment_ref, segment = _get_segment_for_document(session, dataset, document, segment_id_str)
child_chunk_id_str = str(child_chunk_id)
# get child chunk
child_chunk = SegmentService.get_child_chunk_by_segment_ref(child_chunk_id_str, segment_ref, session=session)
if not child_chunk:
raise NotFound("Child chunk not found.")
try:
child_chunk = SegmentService.update_child_chunk(
payload.content,
child_chunk,
segment,
document,
dataset,
session,
actor_id=current_account.id,
mutations=application_services().knowledge.segments.mutations,
)
except ChildChunkIndexingServiceError as e:
raise ChildChunkIndexingError(str(e)) from e
return dump_response(ChildChunkDetailResponse, {"data": child_chunk}), 200