chore: 纳入本地 Dify 1.19.0 源码及品牌与免登改动
演示目录只保留可重建的源码。运行密钥、数据卷、依赖和带口令的本地配置不进入仓库。 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
60158935f4
commit
6519682885
15831 files changed
+3063428
No files matched your search
Whitespace-only changes.
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,55 @@
|
||||
from libs.exception import BaseHTTPException
|
||||
|
||||
|
||||
class DatasetNotInitializedError(BaseHTTPException):
|
||||
error_code = "dataset_not_initialized"
|
||||
description = "The dataset is still being initialized or indexing. Please wait a moment."
|
||||
code = 400
|
||||
|
||||
|
||||
class ArchivedDocumentImmutableError(BaseHTTPException):
|
||||
error_code = "archived_document_immutable"
|
||||
description = "The archived document is not editable."
|
||||
code = 403
|
||||
|
||||
|
||||
class DatasetNameDuplicateError(BaseHTTPException):
|
||||
error_code = "dataset_name_duplicate"
|
||||
description = "The dataset name already exists. Please modify your dataset name."
|
||||
code = 409
|
||||
|
||||
|
||||
class InvalidActionError(BaseHTTPException):
|
||||
error_code = "invalid_action"
|
||||
description = "Invalid action."
|
||||
code = 400
|
||||
|
||||
|
||||
class DocumentAlreadyFinishedError(BaseHTTPException):
|
||||
error_code = "document_already_finished"
|
||||
description = "The document has been processed. Please refresh the page or go to the document details."
|
||||
code = 400
|
||||
|
||||
|
||||
class DocumentIndexingError(BaseHTTPException):
|
||||
error_code = "document_indexing"
|
||||
description = "The document is being processed and cannot be edited."
|
||||
code = 400
|
||||
|
||||
|
||||
class InvalidMetadataError(BaseHTTPException):
|
||||
error_code = "invalid_metadata"
|
||||
description = "The metadata content is incorrect. Please check and verify."
|
||||
code = 400
|
||||
|
||||
|
||||
class DatasetInUseError(BaseHTTPException):
|
||||
error_code = "dataset_in_use"
|
||||
description = "The dataset is being used by some apps. Please remove the dataset from the apps before deleting it."
|
||||
code = 409
|
||||
|
||||
|
||||
class PipelineRunError(BaseHTTPException):
|
||||
error_code = "pipeline_run_error"
|
||||
description = "An error occurred while running the pipeline."
|
||||
code = 500
|
||||
@@ -0,0 +1,85 @@
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from controllers.common.schema import register_response_schema_models, register_schema_models
|
||||
from controllers.console.app.error import (
|
||||
CompletionRequestError,
|
||||
ProviderModelCurrentlyNotSupportError,
|
||||
ProviderNotInitializeError,
|
||||
ProviderQuotaExceededError,
|
||||
)
|
||||
from controllers.console.app.wraps import with_session
|
||||
from controllers.console.datasets.hit_testing_base import DatasetsHitTestingBase, HitTestingPayload
|
||||
from controllers.service_api import service_api_ns
|
||||
from controllers.service_api.wraps import DatasetApiResource, cloud_edition_billing_rate_limit_check
|
||||
from fields.hit_testing_fields import HitTestingResponse
|
||||
from libs.external_api import error_responses
|
||||
from libs.helper import dump_response
|
||||
|
||||
register_schema_models(service_api_ns, HitTestingPayload)
|
||||
register_response_schema_models(service_api_ns, HitTestingResponse)
|
||||
|
||||
|
||||
@service_api_ns.route(
|
||||
"/datasets/<uuid:dataset_id>/hit-testing",
|
||||
doc={"post": {"deprecated": True}},
|
||||
)
|
||||
@service_api_ns.route("/datasets/<uuid:dataset_id>/retrieve")
|
||||
class HitTestingApi(DatasetApiResource, DatasetsHitTestingBase):
|
||||
@error_responses(
|
||||
CompletionRequestError,
|
||||
ProviderModelCurrentlyNotSupportError,
|
||||
ProviderNotInitializeError,
|
||||
ProviderQuotaExceededError,
|
||||
)
|
||||
@service_api_ns.doc(
|
||||
summary="Retrieve Chunks from a Knowledge Base / Test Retrieval",
|
||||
description=(
|
||||
"Performs a search query against a knowledge base to retrieve the most relevant chunks. This "
|
||||
"endpoint can be used for both production retrieval and test retrieval."
|
||||
),
|
||||
tags=["Knowledge Bases"],
|
||||
responses={
|
||||
200: "Retrieval results.",
|
||||
400: (
|
||||
"- `dataset_not_initialized` : The dataset is still being initialized or indexing. Please "
|
||||
"wait a moment.\n"
|
||||
"- `provider_not_initialize` : No valid model provider credentials found. Please go to "
|
||||
"Settings -> Model Provider to complete your provider credentials.\n"
|
||||
"- `provider_quota_exceeded` : Your quota for Dify Hosted OpenAI has been exhausted. Please "
|
||||
"go to Settings -> Model Provider to complete your own provider credentials.\n"
|
||||
"- `model_currently_not_support` : Dify Hosted OpenAI trial currently not support the GPT-4 "
|
||||
"model.\n"
|
||||
"- `completion_request_error` : Completion request failed.\n"
|
||||
"- `invalid_param` : Invalid parameter value."
|
||||
),
|
||||
403: "`forbidden` : Insufficient permissions.",
|
||||
404: "`not_found` : Knowledge base not found.",
|
||||
500: "`internal_server_error` : An internal error occurred during retrieval.",
|
||||
},
|
||||
)
|
||||
@service_api_ns.doc("dataset_hit_testing")
|
||||
@service_api_ns.doc(description="Perform hit testing on a dataset")
|
||||
@service_api_ns.doc(params={"dataset_id": "Knowledge base ID."})
|
||||
@service_api_ns.response(
|
||||
200,
|
||||
"Hit testing results",
|
||||
model=service_api_ns.models[HitTestingResponse.__name__],
|
||||
)
|
||||
@service_api_ns.response(401, "Unauthorized - invalid API token")
|
||||
@service_api_ns.response(404, "Dataset not found")
|
||||
@service_api_ns.expect(service_api_ns.models[HitTestingPayload.__name__])
|
||||
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
|
||||
@with_session
|
||||
def post(self, session: Session, tenant_id: str, dataset_id: UUID) -> dict[str, object]:
|
||||
"""Perform hit testing on a dataset.
|
||||
|
||||
Tests retrieval performance for the specified dataset.
|
||||
"""
|
||||
dataset_id_str = str(dataset_id)
|
||||
dataset = self.get_and_validate_dataset(session, dataset_id_str)
|
||||
args = self.parse_args(service_api_ns.payload)
|
||||
self.hit_testing_args_check(args)
|
||||
|
||||
return dump_response(HitTestingResponse, self.perform_hit_testing(session, dataset, args))
|
||||
@@ -0,0 +1,365 @@
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from flask_login import current_user
|
||||
from sqlalchemy.orm import Session
|
||||
from werkzeug.exceptions import NotFound
|
||||
|
||||
from controllers.common.controller_schemas import MetadataUpdatePayload
|
||||
from controllers.common.schema import register_response_schema_models, register_schema_model, register_schema_models
|
||||
from controllers.common.session import with_session
|
||||
from controllers.console.wraps import model_validate
|
||||
from controllers.service_api import service_api_ns
|
||||
from controllers.service_api.wraps import DatasetApiResource, cloud_edition_billing_rate_limit_check
|
||||
from extensions.ext_application_services import application_services
|
||||
from fields.dataset_fields import (
|
||||
DatasetMetadataActionResponse,
|
||||
DatasetMetadataBuiltInFieldsResponse,
|
||||
DatasetMetadataListResponse,
|
||||
DatasetMetadataResponse,
|
||||
)
|
||||
from libs.helper import dump_response
|
||||
from services.errors.metadata import MetadataResourceNotFoundError
|
||||
from services.knowledge.dataset_service import DatasetService
|
||||
from services.knowledge.entities.knowledge_entities import (
|
||||
DocumentMetadataOperation,
|
||||
MetadataArgs,
|
||||
MetadataDetail,
|
||||
MetadataOperationData,
|
||||
)
|
||||
from services.knowledge.resource_scope import DatasetRef
|
||||
|
||||
BUILT_IN_METADATA_ACTION_PARAM = {
|
||||
"description": "`enable` to activate built-in metadata fields, `disable` to deactivate them.",
|
||||
"enum": ["enable", "disable"],
|
||||
"type": "string",
|
||||
}
|
||||
|
||||
register_schema_model(service_api_ns, MetadataUpdatePayload)
|
||||
register_schema_models(
|
||||
service_api_ns,
|
||||
MetadataArgs,
|
||||
MetadataDetail,
|
||||
DocumentMetadataOperation,
|
||||
MetadataOperationData,
|
||||
)
|
||||
register_response_schema_models(
|
||||
service_api_ns,
|
||||
DatasetMetadataActionResponse,
|
||||
DatasetMetadataBuiltInFieldsResponse,
|
||||
DatasetMetadataListResponse,
|
||||
DatasetMetadataResponse,
|
||||
)
|
||||
|
||||
|
||||
@service_api_ns.route("/datasets/<uuid:dataset_id>/metadata")
|
||||
class DatasetMetadataCreateServiceApi(DatasetApiResource):
|
||||
@service_api_ns.doc(
|
||||
summary="Create Metadata Field",
|
||||
description=(
|
||||
"Create a custom metadata field for the knowledge base. Metadata fields can be used to "
|
||||
"annotate documents with structured information."
|
||||
),
|
||||
tags=["Metadata"],
|
||||
responses={
|
||||
201: "Metadata field created successfully.",
|
||||
},
|
||||
)
|
||||
@service_api_ns.expect(service_api_ns.models[MetadataArgs.__name__])
|
||||
@service_api_ns.doc("create_dataset_metadata")
|
||||
@service_api_ns.doc(description="Create metadata for a dataset")
|
||||
@service_api_ns.doc(params={"dataset_id": "Knowledge base ID."})
|
||||
@service_api_ns.doc(
|
||||
responses={
|
||||
201: "Metadata created successfully",
|
||||
400: "Bad request - invalid or duplicate metadata name",
|
||||
401: "Unauthorized - invalid API token",
|
||||
404: "Dataset not found",
|
||||
}
|
||||
)
|
||||
@service_api_ns.response(
|
||||
201, "Metadata created successfully", service_api_ns.models[DatasetMetadataResponse.__name__]
|
||||
)
|
||||
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
|
||||
@with_session
|
||||
@model_validate(MetadataArgs)
|
||||
def post(self, metadata_args: MetadataArgs, session: Session, tenant_id, dataset_id: UUID):
|
||||
"""Create metadata for a dataset."""
|
||||
|
||||
dataset_id_str = str(dataset_id)
|
||||
dataset = DatasetService.get_dataset_for_tenant(dataset_id_str, str(tenant_id), session=session)
|
||||
if dataset is None:
|
||||
raise NotFound("Dataset not found.")
|
||||
DatasetService.check_dataset_permission(dataset, current_user, session)
|
||||
|
||||
session.commit()
|
||||
try:
|
||||
metadata = application_services().knowledge.metadata.create_metadata(
|
||||
DatasetRef(str(tenant_id), dataset_id_str), metadata_args, actor_id=current_user.id
|
||||
)
|
||||
except MetadataResourceNotFoundError as exc:
|
||||
raise NotFound(str(exc)) from exc
|
||||
return dump_response(DatasetMetadataResponse, metadata), 201
|
||||
|
||||
@service_api_ns.doc(
|
||||
summary="List Metadata Fields",
|
||||
description=(
|
||||
"Returns the list of all metadata fields (both custom and built-in) for the knowledge base, "
|
||||
"along with the count of documents using each field."
|
||||
),
|
||||
tags=["Metadata"],
|
||||
responses={
|
||||
200: "Metadata fields for the knowledge base.",
|
||||
},
|
||||
)
|
||||
@service_api_ns.doc("get_dataset_metadata")
|
||||
@service_api_ns.doc(description="Get all metadata for a dataset")
|
||||
@service_api_ns.doc(params={"dataset_id": "Knowledge base ID."})
|
||||
@service_api_ns.doc(
|
||||
responses={
|
||||
200: "Metadata retrieved successfully",
|
||||
401: "Unauthorized - invalid API token",
|
||||
404: "Dataset not found",
|
||||
}
|
||||
)
|
||||
@service_api_ns.response(
|
||||
200, "Metadata retrieved successfully", service_api_ns.models[DatasetMetadataListResponse.__name__]
|
||||
)
|
||||
@with_session(write=False)
|
||||
def get(self, session: Session, tenant_id, dataset_id: UUID):
|
||||
"""Get all metadata for a dataset."""
|
||||
dataset_id_str = str(dataset_id)
|
||||
dataset = DatasetService.get_dataset_for_tenant(dataset_id_str, str(tenant_id), session=session)
|
||||
if dataset is None:
|
||||
raise NotFound("Dataset not found.")
|
||||
session.commit()
|
||||
try:
|
||||
metadata = application_services().knowledge.metadata.get_dataset_metadatas(
|
||||
DatasetRef(str(tenant_id), dataset_id_str)
|
||||
)
|
||||
except MetadataResourceNotFoundError as exc:
|
||||
raise NotFound(str(exc)) from exc
|
||||
return dump_response(DatasetMetadataListResponse, metadata), 200
|
||||
|
||||
|
||||
@service_api_ns.route("/datasets/<uuid:dataset_id>/metadata/<uuid:metadata_id>")
|
||||
class DatasetMetadataServiceApi(DatasetApiResource):
|
||||
@service_api_ns.doc(
|
||||
summary="Update Metadata Field",
|
||||
description="Rename a custom metadata field.",
|
||||
tags=["Metadata"],
|
||||
responses={
|
||||
200: "Metadata field updated successfully.",
|
||||
},
|
||||
)
|
||||
@service_api_ns.expect(service_api_ns.models[MetadataUpdatePayload.__name__])
|
||||
@service_api_ns.doc("update_dataset_metadata")
|
||||
@service_api_ns.doc(description="Update metadata name")
|
||||
@service_api_ns.doc(params={"dataset_id": "Knowledge base ID.", "metadata_id": "Metadata field ID."})
|
||||
@service_api_ns.doc(
|
||||
responses={
|
||||
200: "Metadata updated successfully",
|
||||
400: "Bad request - invalid or duplicate metadata name",
|
||||
401: "Unauthorized - invalid API token",
|
||||
404: "Dataset or metadata not found",
|
||||
}
|
||||
)
|
||||
@service_api_ns.response(
|
||||
200, "Metadata updated successfully", service_api_ns.models[DatasetMetadataResponse.__name__]
|
||||
)
|
||||
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
|
||||
@with_session
|
||||
@model_validate(MetadataUpdatePayload)
|
||||
def patch(self, payload: MetadataUpdatePayload, session: Session, tenant_id, dataset_id: UUID, metadata_id: UUID):
|
||||
"""Update metadata name."""
|
||||
|
||||
dataset_id_str = str(dataset_id)
|
||||
metadata_id_str = str(metadata_id)
|
||||
dataset = DatasetService.get_dataset_for_tenant(dataset_id_str, tenant_id, session=session)
|
||||
if dataset is None:
|
||||
raise NotFound("Dataset not found.")
|
||||
DatasetService.check_dataset_permission(dataset, current_user, session)
|
||||
|
||||
session.commit()
|
||||
try:
|
||||
metadata = application_services().knowledge.metadata.update_metadata_name(
|
||||
DatasetRef(str(tenant_id), dataset_id_str), metadata_id_str, payload.name, actor_id=current_user.id
|
||||
)
|
||||
except MetadataResourceNotFoundError as exc:
|
||||
raise NotFound(str(exc)) from exc
|
||||
return dump_response(DatasetMetadataResponse, metadata), 200
|
||||
|
||||
@service_api_ns.doc(
|
||||
summary="Delete Metadata Field",
|
||||
description=(
|
||||
"Permanently delete a custom metadata field. Documents using this field will lose their "
|
||||
"metadata values for it."
|
||||
),
|
||||
tags=["Metadata"],
|
||||
responses={
|
||||
204: "Success.",
|
||||
},
|
||||
)
|
||||
@service_api_ns.doc("delete_dataset_metadata")
|
||||
@service_api_ns.doc(description="Delete metadata")
|
||||
@service_api_ns.doc(params={"dataset_id": "Knowledge base ID.", "metadata_id": "Metadata field ID."})
|
||||
@service_api_ns.doc(
|
||||
responses={
|
||||
204: "Metadata deleted successfully",
|
||||
401: "Unauthorized - invalid API token",
|
||||
404: "Dataset or metadata not found",
|
||||
}
|
||||
)
|
||||
@service_api_ns.response(204, "Metadata deleted successfully")
|
||||
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
|
||||
@with_session
|
||||
def delete(self, session: Session, tenant_id, dataset_id: UUID, metadata_id: UUID):
|
||||
"""Delete metadata."""
|
||||
dataset_id_str = str(dataset_id)
|
||||
metadata_id_str = str(metadata_id)
|
||||
dataset = DatasetService.get_dataset_for_tenant(dataset_id_str, tenant_id, session=session)
|
||||
if dataset is None:
|
||||
raise NotFound("Dataset not found.")
|
||||
DatasetService.check_dataset_permission(dataset, current_user, session)
|
||||
|
||||
session.commit()
|
||||
try:
|
||||
application_services().knowledge.metadata.delete_metadata(
|
||||
DatasetRef(str(tenant_id), dataset_id_str), metadata_id_str
|
||||
)
|
||||
except MetadataResourceNotFoundError as exc:
|
||||
raise NotFound(str(exc)) from exc
|
||||
return "", 204
|
||||
|
||||
|
||||
@service_api_ns.route("/datasets/<uuid:dataset_id>/metadata/built-in")
|
||||
class DatasetMetadataBuiltInFieldServiceApi(DatasetApiResource):
|
||||
@service_api_ns.doc(
|
||||
summary="Get Built-in Metadata Fields",
|
||||
description=(
|
||||
"Returns the list of built-in metadata fields provided by the system (e.g., document type, source URL)."
|
||||
),
|
||||
tags=["Metadata"],
|
||||
responses={
|
||||
200: "Built-in metadata fields.",
|
||||
404: "`not_found` : Knowledge base not found.",
|
||||
},
|
||||
)
|
||||
@service_api_ns.doc("get_built_in_fields")
|
||||
@service_api_ns.doc(description="Get all built-in metadata fields")
|
||||
@service_api_ns.doc(params={"dataset_id": "Knowledge base ID."})
|
||||
@service_api_ns.doc(
|
||||
responses={
|
||||
200: "Built-in fields retrieved successfully",
|
||||
401: "Unauthorized - invalid API token",
|
||||
}
|
||||
)
|
||||
@service_api_ns.response(
|
||||
200,
|
||||
"Built-in fields retrieved successfully",
|
||||
service_api_ns.models[DatasetMetadataBuiltInFieldsResponse.__name__],
|
||||
)
|
||||
def get(self, tenant_id, dataset_id: UUID):
|
||||
"""Get all built-in metadata fields."""
|
||||
built_in_fields = application_services().knowledge.metadata.get_built_in_fields()
|
||||
return dump_response(DatasetMetadataBuiltInFieldsResponse, {"fields": built_in_fields}), 200
|
||||
|
||||
|
||||
@service_api_ns.route("/datasets/<uuid:dataset_id>/metadata/built-in/<string:action>")
|
||||
class DatasetMetadataBuiltInFieldActionServiceApi(DatasetApiResource):
|
||||
@service_api_ns.doc(
|
||||
summary="Update Built-in Metadata Field",
|
||||
description="Enable or disable built-in metadata fields for the knowledge base.",
|
||||
tags=["Metadata"],
|
||||
responses={
|
||||
200: "Built-in metadata field toggled successfully.",
|
||||
},
|
||||
)
|
||||
@service_api_ns.doc("toggle_built_in_field")
|
||||
@service_api_ns.doc(description="Enable or disable built-in metadata field")
|
||||
@service_api_ns.doc(params={"dataset_id": "Knowledge base ID.", "action": BUILT_IN_METADATA_ACTION_PARAM})
|
||||
@service_api_ns.doc(
|
||||
responses={
|
||||
200: "Action completed successfully",
|
||||
401: "Unauthorized - invalid API token",
|
||||
404: "Dataset not found",
|
||||
}
|
||||
)
|
||||
@service_api_ns.response(
|
||||
200, "Action completed successfully", service_api_ns.models[DatasetMetadataActionResponse.__name__]
|
||||
)
|
||||
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
|
||||
@with_session
|
||||
def post(self, session: Session, tenant_id, dataset_id: UUID, action: Literal["enable", "disable"]):
|
||||
"""Enable or disable built-in metadata field."""
|
||||
dataset_id_str = str(dataset_id)
|
||||
dataset = DatasetService.get_dataset_for_tenant(dataset_id_str, str(tenant_id), session=session)
|
||||
if dataset is None:
|
||||
raise NotFound("Dataset not found.")
|
||||
DatasetService.check_dataset_permission(dataset, current_user, session)
|
||||
|
||||
session.commit()
|
||||
try:
|
||||
match action:
|
||||
case "enable":
|
||||
application_services().knowledge.metadata.enable_built_in_field(
|
||||
DatasetRef(str(tenant_id), dataset_id_str)
|
||||
)
|
||||
case "disable":
|
||||
application_services().knowledge.metadata.disable_built_in_field(
|
||||
DatasetRef(str(tenant_id), dataset_id_str)
|
||||
)
|
||||
except MetadataResourceNotFoundError as exc:
|
||||
raise NotFound(str(exc)) from exc
|
||||
return dump_response(DatasetMetadataActionResponse, {"result": "success"}), 200
|
||||
|
||||
|
||||
@service_api_ns.route("/datasets/<uuid:dataset_id>/documents/metadata")
|
||||
class DocumentMetadataEditServiceApi(DatasetApiResource):
|
||||
@service_api_ns.doc(
|
||||
summary="Update Document Metadata in Batch",
|
||||
description=(
|
||||
"Update metadata values for multiple documents at once. Each document in the request "
|
||||
"receives the specified metadata key-value pairs."
|
||||
),
|
||||
tags=["Metadata"],
|
||||
responses={
|
||||
200: "Document metadata updated successfully.",
|
||||
},
|
||||
)
|
||||
@service_api_ns.expect(service_api_ns.models[MetadataOperationData.__name__])
|
||||
@service_api_ns.doc("update_documents_metadata")
|
||||
@service_api_ns.doc(description="Update metadata for multiple documents")
|
||||
@service_api_ns.doc(params={"dataset_id": "Knowledge base ID."})
|
||||
@service_api_ns.doc(
|
||||
responses={
|
||||
200: "Documents metadata updated successfully",
|
||||
400: "Bad request - invalid or conflicting document metadata operation",
|
||||
401: "Unauthorized - invalid API token",
|
||||
404: "Dataset, document, or metadata not found",
|
||||
}
|
||||
)
|
||||
@service_api_ns.response(
|
||||
200,
|
||||
"Documents metadata updated successfully",
|
||||
service_api_ns.models[DatasetMetadataActionResponse.__name__],
|
||||
)
|
||||
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
|
||||
@with_session
|
||||
@model_validate(MetadataOperationData)
|
||||
def post(self, metadata_args: MetadataOperationData, session: Session, tenant_id, dataset_id: UUID):
|
||||
"""Update metadata for multiple documents."""
|
||||
dataset = DatasetService.get_dataset_for_tenant(str(dataset_id), str(tenant_id), session=session)
|
||||
if dataset is None:
|
||||
raise NotFound("Dataset not found.")
|
||||
DatasetService.check_dataset_permission(dataset, current_user, session)
|
||||
|
||||
session.commit()
|
||||
try:
|
||||
application_services().knowledge.metadata.update_documents_metadata(
|
||||
DatasetRef(str(tenant_id), str(dataset_id)), metadata_args, actor_id=current_user.id
|
||||
)
|
||||
except MetadataResourceNotFoundError as exc:
|
||||
raise NotFound(str(exc)) from exc
|
||||
|
||||
return dump_response(DatasetMetadataActionResponse, {"result": "success"}), 200
|
||||
Whitespace-only changes.
+410
@@ -0,0 +1,410 @@
|
||||
from collections.abc import Generator
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from flask import request
|
||||
from pydantic import BaseModel, Field, RootModel, field_validator
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
from werkzeug.exceptions import Forbidden, NotFound
|
||||
|
||||
import services
|
||||
from controllers.common.errors import (
|
||||
FilenameNotExistsError,
|
||||
FileTooLargeError,
|
||||
NoFileUploadedError,
|
||||
TooManyFilesError,
|
||||
UnsupportedFileTypeError,
|
||||
)
|
||||
from controllers.common.fields import WorkflowBlockingResponse
|
||||
from controllers.common.schema import (
|
||||
query_params_from_model,
|
||||
query_params_from_request,
|
||||
register_response_schema_models,
|
||||
register_schema_model,
|
||||
)
|
||||
from controllers.console.app.wraps import with_session
|
||||
from controllers.console.wraps import model_validate
|
||||
from controllers.service_api import service_api_ns
|
||||
from controllers.service_api.dataset.error import PipelineRunError
|
||||
from controllers.service_api.schema import event_stream_response, json_or_event_stream_response, multipart_file_params
|
||||
from controllers.service_api.wraps import DatasetApiResource
|
||||
from core.app.apps.pipeline.pipeline_generator import PipelineGenerator
|
||||
from core.app.entities.app_invoke_entities import InvokeFrom
|
||||
from core.entities.knowledge_entities import PipelineDataset, PipelineDocument
|
||||
from extensions.ext_application_services import application_services
|
||||
from fields.base import ResponseModel
|
||||
from libs import helper
|
||||
from libs.external_api import error_responses
|
||||
from libs.helper import dump_response
|
||||
from libs.login import current_user
|
||||
from models import Account
|
||||
from models.dataset import Dataset, Pipeline
|
||||
from models.engine import db
|
||||
from services.feature_service import FeatureService
|
||||
from services.file_service import FileService
|
||||
from services.rag_pipeline.entity.pipeline_service_api_entities import (
|
||||
DatasourceNodeRunApiEntity,
|
||||
DatasourceType,
|
||||
PipelineRunApiEntity,
|
||||
)
|
||||
from services.rag_pipeline.pipeline_generate_service import PipelineGenerateService
|
||||
from services.rag_pipeline.rag_pipeline import RagPipelineService
|
||||
|
||||
|
||||
class DatasourceNodeRunPayload(BaseModel):
|
||||
inputs: dict[str, Any] = Field(description="Input variables for the datasource node.")
|
||||
datasource_type: DatasourceType = Field(description="Type of the datasource.")
|
||||
credential_id: str | None = Field(
|
||||
default=None, description="Datasource credential ID. Uses the default if omitted."
|
||||
)
|
||||
is_published: bool = Field(
|
||||
description=(
|
||||
"Whether to run the published or draft version of the node. `true` runs the published version, "
|
||||
"`false` runs the draft."
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class DatasourcePluginsQuery(BaseModel):
|
||||
is_published: bool = Field(
|
||||
default=True,
|
||||
description=(
|
||||
"Whether to retrieve nodes from the published or draft pipeline. `true` returns nodes from the published "
|
||||
"version, `false` returns nodes from the draft."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class DatasourceCredentialInfoResponse(ResponseModel):
|
||||
id: str | None = None
|
||||
name: str | None = None
|
||||
type: str | None = None
|
||||
is_default: bool | None = None
|
||||
|
||||
|
||||
class DatasourcePluginResponse(ResponseModel):
|
||||
node_id: str | None = None
|
||||
plugin_id: str | None = None
|
||||
provider_name: str | None = None
|
||||
datasource_type: str | None = None
|
||||
title: str | None = None
|
||||
user_input_variables: list[dict[str, Any]] = Field(default_factory=list)
|
||||
credentials: list[DatasourceCredentialInfoResponse] = Field(default_factory=list)
|
||||
|
||||
|
||||
class DatasourcePluginListResponse(RootModel[list[DatasourcePluginResponse]]):
|
||||
pass
|
||||
|
||||
|
||||
class PipelineUploadFileResponse(ResponseModel):
|
||||
id: str
|
||||
name: str
|
||||
size: int
|
||||
extension: str
|
||||
mime_type: str | None = None
|
||||
created_by: str
|
||||
created_at: str | None = None
|
||||
|
||||
@field_validator("created_at", mode="before")
|
||||
@classmethod
|
||||
def _normalize_created_at(cls, value: datetime | str | None) -> str | None:
|
||||
if isinstance(value, datetime):
|
||||
return value.isoformat()
|
||||
return value
|
||||
|
||||
|
||||
class PublishedPipelineRunResponse(ResponseModel):
|
||||
batch: str
|
||||
dataset: PipelineDataset
|
||||
documents: list[PipelineDocument]
|
||||
|
||||
|
||||
class PipelineRunJsonResponse(RootModel[PublishedPipelineRunResponse | WorkflowBlockingResponse]):
|
||||
"""JSON result for published runs and draft runs using `response_mode: blocking`."""
|
||||
|
||||
|
||||
register_schema_model(service_api_ns, DatasourceNodeRunPayload)
|
||||
register_schema_model(service_api_ns, DatasourcePluginsQuery)
|
||||
register_schema_model(service_api_ns, PipelineRunApiEntity)
|
||||
register_response_schema_models(
|
||||
service_api_ns,
|
||||
DatasourceCredentialInfoResponse,
|
||||
DatasourcePluginResponse,
|
||||
DatasourcePluginListResponse,
|
||||
PipelineUploadFileResponse,
|
||||
WorkflowBlockingResponse,
|
||||
PublishedPipelineRunResponse,
|
||||
PipelineRunJsonResponse,
|
||||
)
|
||||
|
||||
|
||||
@service_api_ns.route("/datasets/<uuid:dataset_id>/pipeline/datasource-plugins")
|
||||
class DatasourcePluginsApi(DatasetApiResource):
|
||||
"""Resource for datasource plugins."""
|
||||
|
||||
@service_api_ns.doc(
|
||||
summary="List Datasource Plugins",
|
||||
description=(
|
||||
"List the datasource nodes configured in the knowledge pipeline. Each node includes the plugin it uses "
|
||||
"plus the metadata needed to run it."
|
||||
),
|
||||
tags=["Knowledge Pipeline"],
|
||||
responses={
|
||||
200: "List of datasource nodes configured in the pipeline.",
|
||||
404: "`not_found` : Dataset not found.",
|
||||
},
|
||||
)
|
||||
@service_api_ns.doc(shortcut="list_rag_pipeline_datasource_plugins")
|
||||
@service_api_ns.doc(description="List all datasource plugins for a rag pipeline")
|
||||
@service_api_ns.doc(params={"dataset_id": "Knowledge base ID."})
|
||||
@service_api_ns.doc(params=query_params_from_model(DatasourcePluginsQuery))
|
||||
@service_api_ns.doc(
|
||||
responses={
|
||||
200: "Datasource plugins retrieved successfully",
|
||||
400: "Bad request - pipeline is not configured",
|
||||
401: "Unauthorized - invalid API token",
|
||||
}
|
||||
)
|
||||
@service_api_ns.response(
|
||||
200,
|
||||
"Datasource plugins retrieved successfully",
|
||||
service_api_ns.models[DatasourcePluginListResponse.__name__],
|
||||
)
|
||||
def get(self, tenant_id: str, dataset_id: UUID):
|
||||
"""Resource for getting datasource plugins."""
|
||||
dataset_id_str = str(dataset_id)
|
||||
# Verify dataset ownership
|
||||
stmt = select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id_str)
|
||||
dataset = db.session.scalar(stmt)
|
||||
if not dataset:
|
||||
raise NotFound("Dataset not found.")
|
||||
|
||||
query = query_params_from_request(DatasourcePluginsQuery)
|
||||
|
||||
rag_pipeline_service = RagPipelineService(db.session())
|
||||
datasource_plugins: list[dict[Any, Any]] = rag_pipeline_service.get_datasource_plugins(
|
||||
tenant_id=tenant_id,
|
||||
dataset_id=dataset_id_str,
|
||||
is_published=query.is_published,
|
||||
credential_query=application_services().credential_queries,
|
||||
datasource_providers=application_services().data_sources.providers,
|
||||
)
|
||||
return dump_response(DatasourcePluginListResponse, datasource_plugins), 200
|
||||
|
||||
|
||||
@service_api_ns.route("/datasets/<uuid:dataset_id>/pipeline/datasource/nodes/<string:node_id>/run")
|
||||
class DatasourceNodeRunApi(DatasetApiResource):
|
||||
"""Resource for datasource node run."""
|
||||
|
||||
@service_api_ns.doc(
|
||||
summary="Run Datasource Node",
|
||||
description=(
|
||||
"Execute a single datasource node within the knowledge pipeline. Returns a streaming response with the "
|
||||
"node execution results."
|
||||
),
|
||||
tags=["Knowledge Pipeline"],
|
||||
responses={
|
||||
200: "Streaming response with node execution events.",
|
||||
404: "`not_found` : Dataset not found.",
|
||||
},
|
||||
)
|
||||
@event_stream_response(service_api_ns)
|
||||
@service_api_ns.doc(shortcut="run_datasource_node")
|
||||
@service_api_ns.doc(description="Run a datasource node for a rag pipeline")
|
||||
@service_api_ns.doc(params={"dataset_id": "Knowledge base ID.", "node_id": "ID of the datasource node to execute."})
|
||||
@service_api_ns.doc(
|
||||
responses={
|
||||
200: "Datasource node run successfully",
|
||||
400: "Bad request - invalid payload or pipeline is not configured",
|
||||
401: "Unauthorized - invalid API token",
|
||||
}
|
||||
)
|
||||
@service_api_ns.expect(service_api_ns.models[DatasourceNodeRunPayload.__name__])
|
||||
@model_validate(DatasourceNodeRunPayload)
|
||||
def post(self, payload: DatasourceNodeRunPayload, tenant_id: str, dataset_id: UUID, node_id: str):
|
||||
"""Resource for getting datasource plugins."""
|
||||
dataset_id_str = str(dataset_id)
|
||||
# Verify dataset ownership
|
||||
stmt = select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id_str)
|
||||
dataset = db.session.scalar(stmt)
|
||||
if not dataset:
|
||||
raise NotFound("Dataset not found.")
|
||||
|
||||
assert isinstance(current_user, Account)
|
||||
rag_pipeline_service: RagPipelineService = RagPipelineService(db.session())
|
||||
pipeline: Pipeline = rag_pipeline_service.get_pipeline(tenant_id=tenant_id, dataset_id=dataset_id_str)
|
||||
datasource_node_run_api_entity = DatasourceNodeRunApiEntity.model_validate(
|
||||
{
|
||||
**payload.model_dump(exclude_none=True),
|
||||
"pipeline_id": pipeline.id,
|
||||
"node_id": node_id,
|
||||
}
|
||||
)
|
||||
# response-contract:ignore compact_generate_response
|
||||
return helper.compact_generate_response(
|
||||
PipelineGenerator.convert_to_event_stream(
|
||||
rag_pipeline_service.run_datasource_workflow_node(
|
||||
pipeline=pipeline,
|
||||
node_id=node_id,
|
||||
user_inputs=datasource_node_run_api_entity.inputs,
|
||||
account=current_user,
|
||||
datasource_type=datasource_node_run_api_entity.datasource_type,
|
||||
is_published=datasource_node_run_api_entity.is_published,
|
||||
credential_id=datasource_node_run_api_entity.credential_id,
|
||||
datasource_providers=application_services().data_sources.providers,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@service_api_ns.route("/datasets/<uuid:dataset_id>/pipeline/run")
|
||||
class PipelineRunApi(DatasetApiResource):
|
||||
"""Resource for datasource node run."""
|
||||
|
||||
@error_responses(PipelineRunError)
|
||||
@service_api_ns.doc(
|
||||
summary="Run Pipeline",
|
||||
description=(
|
||||
"Execute the full knowledge pipeline for a knowledge base. Published runs are queued and return batch "
|
||||
"metadata as JSON. Draft runs support blocking JSON and streaming Server-Sent Events."
|
||||
),
|
||||
tags=["Knowledge Pipeline"],
|
||||
responses={
|
||||
200: (
|
||||
"Pipeline execution result. Published runs return a JSON object containing `batch`, `dataset`, and "
|
||||
"`documents`. Draft runs return `text/event-stream` for streaming mode or a workflow result JSON "
|
||||
"object for blocking mode."
|
||||
),
|
||||
403: "`forbidden` : Forbidden.",
|
||||
404: "`not_found` : Dataset not found.",
|
||||
500: "`pipeline_run_error` : Pipeline execution failed.",
|
||||
},
|
||||
)
|
||||
@json_or_event_stream_response(service_api_ns)
|
||||
@service_api_ns.doc(shortcut="run_pipeline")
|
||||
@service_api_ns.doc(description="Run a datasource node for a rag pipeline")
|
||||
@service_api_ns.doc(params={"dataset_id": "Knowledge base ID."})
|
||||
@service_api_ns.doc(
|
||||
responses={
|
||||
200: "Pipeline run successfully",
|
||||
400: "Bad request - invalid payload or pipeline is not configured",
|
||||
401: "Unauthorized - invalid API token",
|
||||
}
|
||||
)
|
||||
@service_api_ns.expect(service_api_ns.models[PipelineRunApiEntity.__name__])
|
||||
@service_api_ns.response(
|
||||
200,
|
||||
"Pipeline run successfully",
|
||||
service_api_ns.models[PipelineRunJsonResponse.__name__],
|
||||
)
|
||||
@with_session
|
||||
def post(self, session: Session, tenant_id: str, dataset_id: UUID):
|
||||
"""Resource for running a rag pipeline."""
|
||||
dataset_id_str = str(dataset_id)
|
||||
# Verify dataset ownership
|
||||
stmt = select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id_str)
|
||||
dataset = session.scalar(stmt)
|
||||
if not dataset:
|
||||
raise NotFound("Dataset not found.")
|
||||
|
||||
payload = PipelineRunApiEntity.model_validate(service_api_ns.payload or {})
|
||||
|
||||
if not isinstance(current_user, Account):
|
||||
raise Forbidden()
|
||||
|
||||
rag_pipeline_service = RagPipelineService(session)
|
||||
pipeline = rag_pipeline_service.get_pipeline(tenant_id=tenant_id, dataset_id=dataset_id_str)
|
||||
try:
|
||||
response: dict[Any, Any] | Generator[str, Any] = PipelineGenerateService.generate(
|
||||
generator=application_services().knowledge.pipeline_generator,
|
||||
session=session,
|
||||
pipeline=pipeline,
|
||||
user=current_user,
|
||||
args=payload.model_dump(),
|
||||
invoke_from=InvokeFrom.PUBLISHED_PIPELINE if payload.is_published else InvokeFrom.DEBUGGER,
|
||||
streaming=payload.response_mode == "streaming",
|
||||
)
|
||||
|
||||
# response-contract:ignore compact_generate_response
|
||||
return helper.compact_generate_response(response)
|
||||
except Exception as ex:
|
||||
raise PipelineRunError(description=str(ex)) from ex
|
||||
|
||||
|
||||
@service_api_ns.route("/datasets/pipeline/file-upload")
|
||||
class KnowledgebasePipelineFileUploadApi(DatasetApiResource):
|
||||
"""Resource for uploading a file to a knowledgebase pipeline."""
|
||||
|
||||
@error_responses(
|
||||
FilenameNotExistsError, FileTooLargeError, NoFileUploadedError, TooManyFilesError, UnsupportedFileTypeError
|
||||
)
|
||||
@service_api_ns.doc(
|
||||
summary="Upload Pipeline File",
|
||||
description="Upload a file for use in a knowledge pipeline. Accepts a single file via `multipart/form-data`.",
|
||||
tags=["Knowledge Pipeline"],
|
||||
responses={
|
||||
201: "File uploaded successfully.",
|
||||
400: (
|
||||
"- `no_file_uploaded` : Please upload your file.\n"
|
||||
"- `filename_not_exists_error` : The specified filename does not exist.\n"
|
||||
"- `too_many_files` : Only one file is allowed."
|
||||
),
|
||||
413: "`file_too_large` : File size exceeded.",
|
||||
415: "`unsupported_file_type` : File type not allowed.",
|
||||
},
|
||||
)
|
||||
@service_api_ns.doc(shortcut="knowledgebase_pipeline_file_upload")
|
||||
@service_api_ns.doc(description="Upload a file to a knowledgebase pipeline")
|
||||
@service_api_ns.doc(consumes=["multipart/form-data"], params=multipart_file_params(include_user=False))
|
||||
@service_api_ns.doc(
|
||||
responses={
|
||||
201: "File uploaded successfully",
|
||||
400: "Bad request - no file or invalid file",
|
||||
401: "Unauthorized - invalid API token",
|
||||
413: "File too large",
|
||||
415: "Unsupported file type",
|
||||
}
|
||||
)
|
||||
@service_api_ns.response(
|
||||
201,
|
||||
"File uploaded successfully",
|
||||
service_api_ns.models[PipelineUploadFileResponse.__name__],
|
||||
)
|
||||
def post(self, tenant_id: str):
|
||||
"""Upload a file for use in conversations.
|
||||
|
||||
Accepts a single file upload via multipart/form-data.
|
||||
"""
|
||||
# check file
|
||||
if "file" not in request.files:
|
||||
raise NoFileUploadedError()
|
||||
|
||||
if len(request.files) > 1:
|
||||
raise TooManyFilesError()
|
||||
|
||||
file = request.files["file"]
|
||||
if not file.mimetype:
|
||||
raise UnsupportedFileTypeError()
|
||||
|
||||
if not file.filename:
|
||||
raise FilenameNotExistsError
|
||||
|
||||
if not current_user:
|
||||
raise ValueError("Invalid user account")
|
||||
|
||||
try:
|
||||
upload_file = FileService(db.engine).upload_file(
|
||||
filename=file.filename,
|
||||
content=file.stream.read(),
|
||||
mimetype=file.mimetype,
|
||||
user=current_user,
|
||||
default_file_size_limit=FeatureService.get_knowledge_file_size_limit(tenant_id),
|
||||
)
|
||||
except services.errors.file.FileTooLargeError as file_too_large_error:
|
||||
raise FileTooLargeError(file_too_large_error.description) from file_too_large_error
|
||||
except services.errors.file.UnsupportedFileTypeError as caught_error:
|
||||
raise UnsupportedFileTypeError() from caught_error
|
||||
|
||||
return dump_response(PipelineUploadFileResponse, upload_file), 201
|
||||
@@ -0,0 +1,866 @@
|
||||
from typing import cast
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, Field, ValidationError, field_validator
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
from werkzeug.exceptions import NotFound
|
||||
|
||||
from configs import dify_config
|
||||
from controllers.common.controller_schemas import ChildChunkCreatePayload, ChildChunkUpdatePayload
|
||||
from controllers.common.fields import ErrorMessageResponse
|
||||
from controllers.common.schema import (
|
||||
query_params_from_model,
|
||||
query_params_from_request,
|
||||
register_response_schema_models,
|
||||
register_schema_models,
|
||||
)
|
||||
from controllers.common.session import with_session
|
||||
from controllers.console.wraps import model_validate
|
||||
from controllers.service_api import service_api_ns
|
||||
from controllers.service_api.app.error import ProviderNotInitializeError
|
||||
from controllers.service_api.wraps import (
|
||||
DatasetApiResource,
|
||||
cloud_edition_billing_knowledge_limit_check,
|
||||
cloud_edition_billing_rate_limit_check,
|
||||
cloud_edition_billing_resource_check,
|
||||
)
|
||||
from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
|
||||
from core.model_manager import ModelManager
|
||||
from core.rag.index_processor.constant.index_type import IndexTechniqueType
|
||||
from extensions.ext_application_services import application_services
|
||||
from fields.base import ResponseModel
|
||||
from fields.segment_fields import (
|
||||
ChildChunkDetailResponse,
|
||||
ChildChunkListResponse,
|
||||
SegmentDetailResponse,
|
||||
SegmentResponse,
|
||||
)
|
||||
from graphon.model_runtime.entities.model_entities import ModelType
|
||||
from libs.external_api import error_responses
|
||||
from libs.helper import dump_response
|
||||
from libs.login import current_account_with_tenant
|
||||
from models.dataset import Dataset, Document, DocumentSegment
|
||||
from services.errors.chunk import ChildChunkDeleteIndexError, ChildChunkIndexingError
|
||||
from services.errors.chunk import ChildChunkDeleteIndexError as ChildChunkDeleteIndexServiceError
|
||||
from services.errors.chunk import ChildChunkIndexingError as ChildChunkIndexingServiceError
|
||||
from services.knowledge.dataset_read_service import load_segment_detail, load_segment_details
|
||||
from services.knowledge.dataset_service import DatasetService, DocumentService, SegmentService
|
||||
from services.knowledge.entities.segments import SegmentUpdateArgs
|
||||
from services.knowledge.resource_scope import DatasetRef, SegmentRef
|
||||
from services.knowledge.summaries.adapters import SummaryIndexAdapter
|
||||
|
||||
|
||||
class SegmentCreateItemPayload(BaseModel):
|
||||
content: str = Field(min_length=1, description="Chunk text content.")
|
||||
answer: str | None = Field(default=None, description="Answer content for QA mode.")
|
||||
keywords: list[str] | None = Field(default=None, description="Keywords for the chunk.")
|
||||
attachment_ids: list[str] | None = Field(default=None, description="Attachment file IDs.")
|
||||
|
||||
@field_validator("content")
|
||||
@classmethod
|
||||
def validate_content(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError("Content is empty")
|
||||
return value
|
||||
|
||||
|
||||
class SegmentCreatePayload(BaseModel):
|
||||
segments: list[SegmentCreateItemPayload] = Field(min_length=1, description="Array of chunk objects to create.")
|
||||
|
||||
|
||||
class SegmentListQuery(BaseModel):
|
||||
limit: int = Field(default=20, ge=1, description="Number of items per page. Server caps at `100`.")
|
||||
page: int = Field(default=1, ge=1, description="Page number to retrieve.")
|
||||
status: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="Filter chunks by indexing status, such as `completed`, `indexing`, or `error`.",
|
||||
)
|
||||
keyword: str | None = Field(default=None, description="Search keyword.")
|
||||
|
||||
|
||||
class SegmentUpdatePayload(BaseModel):
|
||||
segment: SegmentUpdateArgs = Field(description="Chunk data to update.")
|
||||
|
||||
|
||||
class ChildChunkListQuery(BaseModel):
|
||||
limit: int = Field(default=20, ge=1, description="Number of items per page. Server caps at `100`.")
|
||||
keyword: str | None = Field(default=None, description="Search keyword.")
|
||||
page: int = Field(default=1, ge=1, description="Page number to retrieve.")
|
||||
|
||||
|
||||
class SegmentDocParams:
|
||||
DATASET_DOCUMENT = {"dataset_id": "Knowledge base ID.", "document_id": "Document ID."}
|
||||
DATASET_DOCUMENT_SEGMENT = {**DATASET_DOCUMENT, "segment_id": "Chunk ID."}
|
||||
DATASET_DOCUMENT_PARENT_SEGMENT = {**DATASET_DOCUMENT, "segment_id": "Chunk ID."}
|
||||
DATASET_DOCUMENT_CHILD_CHUNK = {**DATASET_DOCUMENT_PARENT_SEGMENT, "child_chunk_id": "Child chunk ID."}
|
||||
|
||||
|
||||
class SegmentCreateListResponse(ResponseModel):
|
||||
data: list[SegmentResponse]
|
||||
doc_form: str
|
||||
|
||||
|
||||
class SegmentListResponse(ResponseModel):
|
||||
data: list[SegmentResponse]
|
||||
doc_form: str
|
||||
total: int
|
||||
has_more: bool
|
||||
limit: int
|
||||
page: int
|
||||
|
||||
|
||||
register_schema_models(
|
||||
service_api_ns,
|
||||
SegmentCreatePayload,
|
||||
SegmentCreateItemPayload,
|
||||
SegmentListQuery,
|
||||
SegmentUpdateArgs,
|
||||
SegmentUpdatePayload,
|
||||
ChildChunkCreatePayload,
|
||||
ChildChunkListQuery,
|
||||
ChildChunkUpdatePayload,
|
||||
)
|
||||
register_response_schema_models(
|
||||
service_api_ns,
|
||||
ErrorMessageResponse,
|
||||
SegmentResponse,
|
||||
SegmentCreateListResponse,
|
||||
SegmentListResponse,
|
||||
SegmentDetailResponse,
|
||||
ChildChunkDetailResponse,
|
||||
ChildChunkListResponse,
|
||||
)
|
||||
|
||||
|
||||
def _get_segment_for_document(
|
||||
session: Session, dataset: Dataset, document: Document, segment_id: str
|
||||
) -> tuple[SegmentRef, DocumentSegment]:
|
||||
if document.tenant_id != dataset.tenant_id or document.dataset_id != dataset.id:
|
||||
raise NotFound("Document not found.")
|
||||
|
||||
dataset_ref = DatasetRef(tenant_id=dataset.tenant_id, dataset_id=dataset.id)
|
||||
segment_ref = dataset_ref.document(document.id).segment(segment_id)
|
||||
segment = SegmentService.get_segment_by_ref(segment_ref, session=session)
|
||||
if not segment:
|
||||
raise NotFound("Segment not found.")
|
||||
return segment_ref, segment
|
||||
|
||||
|
||||
@service_api_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments")
|
||||
class SegmentApi(DatasetApiResource):
|
||||
"""Resource for segments."""
|
||||
|
||||
@error_responses(ProviderNotInitializeError)
|
||||
@service_api_ns.doc(
|
||||
summary="Create Chunks",
|
||||
description=(
|
||||
"Create one or more chunks within a document. Each chunk can include optional keywords and an "
|
||||
"answer field (for QA-mode documents)."
|
||||
),
|
||||
tags=["Chunks"],
|
||||
responses={
|
||||
200: "Chunks created successfully.",
|
||||
404: "`not_found` : Document is not completed or is disabled.",
|
||||
},
|
||||
)
|
||||
@service_api_ns.expect(service_api_ns.models[SegmentCreatePayload.__name__])
|
||||
@service_api_ns.doc("create_segments")
|
||||
@service_api_ns.doc(description="Create segments in a document")
|
||||
@service_api_ns.doc(params=SegmentDocParams.DATASET_DOCUMENT)
|
||||
@service_api_ns.doc(
|
||||
responses={
|
||||
200: "Segments created successfully",
|
||||
400: "Bad request - segments data is missing",
|
||||
401: "Unauthorized - invalid API token",
|
||||
404: "Dataset or document not found",
|
||||
}
|
||||
)
|
||||
@service_api_ns.response(
|
||||
200,
|
||||
"Segments created successfully",
|
||||
service_api_ns.models[SegmentCreateListResponse.__name__],
|
||||
)
|
||||
@service_api_ns.response(
|
||||
400,
|
||||
"Bad request - segments data is missing",
|
||||
service_api_ns.models[ErrorMessageResponse.__name__],
|
||||
)
|
||||
@cloud_edition_billing_resource_check("vector_space", "dataset")
|
||||
@cloud_edition_billing_knowledge_limit_check("add_segment", "dataset")
|
||||
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
|
||||
@with_session
|
||||
def post(self, session: Session, tenant_id: str, dataset_id: UUID, document_id: UUID):
|
||||
_, current_tenant_id = current_account_with_tenant()
|
||||
"""Create single segment."""
|
||||
dataset_id_str = str(dataset_id)
|
||||
# check dataset
|
||||
dataset = session.scalar(
|
||||
select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id_str).limit(1)
|
||||
)
|
||||
if not dataset:
|
||||
raise NotFound("Dataset not found.")
|
||||
document_id_str = str(document_id)
|
||||
# check document
|
||||
document = DocumentService.get_document(dataset.id, document_id_str, session=session)
|
||||
if not document:
|
||||
raise NotFound("Document not found.")
|
||||
if document.indexing_status != "completed":
|
||||
raise NotFound("Document is not completed.")
|
||||
if not document.enabled:
|
||||
raise NotFound("Document is disabled.")
|
||||
# check embedding model setting
|
||||
if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
|
||||
try:
|
||||
model_manager = ModelManager.for_tenant(tenant_id=current_tenant_id)
|
||||
model_manager.get_model_instance(
|
||||
tenant_id=current_tenant_id,
|
||||
provider=dataset.embedding_model_provider,
|
||||
model_type=ModelType.TEXT_EMBEDDING,
|
||||
model=dataset.embedding_model,
|
||||
)
|
||||
except LLMBadRequestError as caught_error:
|
||||
raise ProviderNotInitializeError(
|
||||
"No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
|
||||
) from caught_error
|
||||
except ProviderTokenNotInitError as ex:
|
||||
raise ProviderNotInitializeError(ex.description) from ex
|
||||
# validate args
|
||||
try:
|
||||
payload = SegmentCreatePayload.model_validate(service_api_ns.payload or {})
|
||||
except ValidationError as e:
|
||||
return {"error": str(e)}, 400
|
||||
segments_limit = dify_config.DATASET_MAX_SEGMENTS_PER_REQUEST
|
||||
if segments_limit > 0 and len(payload.segments) > segments_limit:
|
||||
raise ValueError(f"Exceeded maximum segments limit of {segments_limit}.")
|
||||
segment_items = [segment.model_dump(exclude_none=True) for segment in payload.segments]
|
||||
|
||||
for args_item in segment_items:
|
||||
SegmentService.segment_create_args_validate(args_item, document)
|
||||
segments = cast(
|
||||
list[DocumentSegment],
|
||||
SegmentService.multi_create_segment(
|
||||
segment_items,
|
||||
document,
|
||||
dataset,
|
||||
session,
|
||||
mutations=application_services().knowledge.segments.mutations,
|
||||
),
|
||||
)
|
||||
segment_ids = [segment.id for segment in segments]
|
||||
summaries: dict[str, str | None] = {}
|
||||
if segment_ids:
|
||||
summary_records = SummaryIndexAdapter.get_segments_summaries(
|
||||
segment_ids=segment_ids, dataset_id=dataset_id_str, session=session
|
||||
)
|
||||
summaries = {chunk_id: record.summary_content for chunk_id, record in summary_records.items()}
|
||||
response = {
|
||||
"data": load_segment_details(segments, summaries, session=session),
|
||||
"doc_form": document.doc_form,
|
||||
}
|
||||
return dump_response(SegmentCreateListResponse, response), 200
|
||||
|
||||
@error_responses(ProviderNotInitializeError)
|
||||
@service_api_ns.doc(
|
||||
summary="List Chunks",
|
||||
description="Returns a paginated list of chunks within a document. Supports filtering by keyword and status.",
|
||||
tags=["Chunks"],
|
||||
responses={
|
||||
200: "List of chunks.",
|
||||
},
|
||||
)
|
||||
@service_api_ns.doc("list_segments")
|
||||
@service_api_ns.doc(description="List segments in a document")
|
||||
@service_api_ns.doc(params=SegmentDocParams.DATASET_DOCUMENT)
|
||||
@service_api_ns.doc(params=query_params_from_model(SegmentListQuery))
|
||||
@service_api_ns.doc(
|
||||
responses={
|
||||
200: "Segments retrieved successfully",
|
||||
400: "Bad request - embedding model is not configured",
|
||||
401: "Unauthorized - invalid API token",
|
||||
404: "Dataset or document not found",
|
||||
}
|
||||
)
|
||||
@service_api_ns.response(
|
||||
200,
|
||||
"Segments retrieved successfully",
|
||||
service_api_ns.models[SegmentListResponse.__name__],
|
||||
)
|
||||
@with_session
|
||||
def get(self, session: Session, tenant_id: str, dataset_id: UUID, document_id: UUID):
|
||||
_, current_tenant_id = current_account_with_tenant()
|
||||
"""Get segments."""
|
||||
# check dataset
|
||||
args = query_params_from_request(
|
||||
SegmentListQuery,
|
||||
list_fields=("status",),
|
||||
use_defaults_for_malformed_ints=True,
|
||||
)
|
||||
page = args.page
|
||||
limit = min(args.limit, 100)
|
||||
dataset_id_str = str(dataset_id)
|
||||
dataset = session.scalar(
|
||||
select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id_str).limit(1)
|
||||
)
|
||||
if not dataset:
|
||||
raise NotFound("Dataset not found.")
|
||||
document_id_str = str(document_id)
|
||||
# check document
|
||||
document = DocumentService.get_document(dataset.id, document_id_str, session=session)
|
||||
if not document:
|
||||
raise NotFound("Document not found.")
|
||||
# check embedding model setting
|
||||
if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
|
||||
try:
|
||||
model_manager = ModelManager.for_tenant(tenant_id=current_tenant_id)
|
||||
model_manager.get_model_instance(
|
||||
tenant_id=current_tenant_id,
|
||||
provider=dataset.embedding_model_provider,
|
||||
model_type=ModelType.TEXT_EMBEDDING,
|
||||
model=dataset.embedding_model,
|
||||
)
|
||||
except LLMBadRequestError as caught_error:
|
||||
raise ProviderNotInitializeError(
|
||||
"No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
|
||||
) from caught_error
|
||||
except ProviderTokenNotInitError as ex:
|
||||
raise ProviderNotInitializeError(ex.description) from ex
|
||||
|
||||
segments, total = SegmentService.get_segments(
|
||||
session=session,
|
||||
document_id=document_id_str,
|
||||
tenant_id=current_tenant_id,
|
||||
status_list=args.status,
|
||||
keyword=args.keyword,
|
||||
page=page,
|
||||
limit=limit,
|
||||
)
|
||||
segment_ids = [segment.id for segment in segments]
|
||||
summaries: dict[str, str | None] = {}
|
||||
if segment_ids:
|
||||
summary_records = SummaryIndexAdapter.get_segments_summaries(
|
||||
segment_ids=segment_ids, dataset_id=dataset_id_str, session=session
|
||||
)
|
||||
summaries = {chunk_id: record.summary_content for chunk_id, record in summary_records.items()}
|
||||
|
||||
response = {
|
||||
"data": load_segment_details(segments, summaries, session=session),
|
||||
"doc_form": document.doc_form,
|
||||
"total": total,
|
||||
"has_more": page * limit < total,
|
||||
"limit": limit,
|
||||
"page": page,
|
||||
}
|
||||
|
||||
return dump_response(SegmentListResponse, response), 200
|
||||
|
||||
|
||||
@service_api_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments/<uuid:segment_id>")
|
||||
class DatasetSegmentApi(DatasetApiResource):
|
||||
@service_api_ns.doc(
|
||||
summary="Delete Chunk",
|
||||
description="Permanently delete a chunk from the document.",
|
||||
tags=["Chunks"],
|
||||
responses={
|
||||
204: "Success.",
|
||||
},
|
||||
)
|
||||
@service_api_ns.doc("delete_segment")
|
||||
@service_api_ns.doc(description="Delete a specific segment")
|
||||
@service_api_ns.doc(params=SegmentDocParams.DATASET_DOCUMENT_SEGMENT)
|
||||
@service_api_ns.doc(
|
||||
responses={
|
||||
204: "Segment deleted successfully",
|
||||
400: "Bad request - invalid dataset model state or concurrent deletion",
|
||||
401: "Unauthorized - invalid API token",
|
||||
404: "Dataset, document, or segment not found",
|
||||
}
|
||||
)
|
||||
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
|
||||
@with_session
|
||||
def delete(self, session: Session, tenant_id: str, dataset_id: UUID, document_id: UUID, segment_id: UUID):
|
||||
current_account_with_tenant()
|
||||
dataset_id_str = str(dataset_id)
|
||||
# check dataset
|
||||
dataset = session.scalar(
|
||||
select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id_str).limit(1)
|
||||
)
|
||||
if not dataset:
|
||||
raise NotFound("Dataset not found.")
|
||||
# check user's model setting
|
||||
DatasetService.check_dataset_model_setting(dataset)
|
||||
document_id_str = str(document_id)
|
||||
# check document
|
||||
document = DocumentService.get_document(dataset_id_str, document_id_str, session=session)
|
||||
if not document:
|
||||
raise NotFound("Document not found.")
|
||||
segment_id_str = str(segment_id)
|
||||
_, segment = _get_segment_for_document(session, dataset, document, segment_id_str)
|
||||
SegmentService.delete_segment(
|
||||
segment, document, dataset, session, mutations=application_services().knowledge.segments.mutations
|
||||
)
|
||||
return "", 204
|
||||
|
||||
@error_responses(ProviderNotInitializeError)
|
||||
@service_api_ns.doc(
|
||||
summary="Update Chunk",
|
||||
description="Update a chunk's content, keywords, or answer. Re-triggers indexing for the modified chunk.",
|
||||
tags=["Chunks"],
|
||||
responses={
|
||||
200: "Chunk updated successfully.",
|
||||
},
|
||||
)
|
||||
@service_api_ns.expect(service_api_ns.models[SegmentUpdatePayload.__name__])
|
||||
@service_api_ns.doc("update_segment")
|
||||
@service_api_ns.doc(description="Update a specific segment")
|
||||
@service_api_ns.doc(params=SegmentDocParams.DATASET_DOCUMENT_SEGMENT)
|
||||
@service_api_ns.doc(
|
||||
responses={
|
||||
200: "Segment updated successfully",
|
||||
400: "Bad request - invalid segment or embedding model configuration",
|
||||
401: "Unauthorized - invalid API token",
|
||||
404: "Dataset, document, or segment not found",
|
||||
}
|
||||
)
|
||||
@service_api_ns.response(200, "Segment updated successfully", service_api_ns.models[SegmentDetailResponse.__name__])
|
||||
@cloud_edition_billing_resource_check("vector_space", "dataset")
|
||||
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
|
||||
@with_session
|
||||
@model_validate(SegmentUpdatePayload)
|
||||
def post(
|
||||
self,
|
||||
payload: SegmentUpdatePayload,
|
||||
session: Session,
|
||||
tenant_id: str,
|
||||
dataset_id: UUID,
|
||||
document_id: UUID,
|
||||
segment_id: UUID,
|
||||
):
|
||||
current_account, current_tenant_id = current_account_with_tenant()
|
||||
dataset_id_str = str(dataset_id)
|
||||
# check dataset
|
||||
dataset = session.scalar(
|
||||
select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id_str).limit(1)
|
||||
)
|
||||
if not dataset:
|
||||
raise NotFound("Dataset not found.")
|
||||
# check user's model setting
|
||||
DatasetService.check_dataset_model_setting(dataset)
|
||||
document_id_str = str(document_id)
|
||||
# check document
|
||||
document = DocumentService.get_document(dataset_id_str, document_id_str, session=session)
|
||||
if not document:
|
||||
raise NotFound("Document not found.")
|
||||
if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
|
||||
# check embedding model setting
|
||||
try:
|
||||
model_manager = ModelManager.for_tenant(tenant_id=current_tenant_id)
|
||||
model_manager.get_model_instance(
|
||||
tenant_id=current_tenant_id,
|
||||
provider=dataset.embedding_model_provider,
|
||||
model_type=ModelType.TEXT_EMBEDDING,
|
||||
model=dataset.embedding_model,
|
||||
)
|
||||
except LLMBadRequestError as caught_error:
|
||||
raise ProviderNotInitializeError(
|
||||
"No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
|
||||
) from caught_error
|
||||
except ProviderTokenNotInitError as ex:
|
||||
raise ProviderNotInitializeError(ex.description) from ex
|
||||
segment_id_str = str(segment_id)
|
||||
_, segment = _get_segment_for_document(session, dataset, document, segment_id_str)
|
||||
|
||||
updated_segment = SegmentService.update_segment(
|
||||
payload.segment,
|
||||
segment,
|
||||
document,
|
||||
dataset,
|
||||
session,
|
||||
actor_id=current_account.id,
|
||||
mutations=application_services().knowledge.segments.mutations,
|
||||
)
|
||||
summary = SummaryIndexAdapter.get_segment_summary(
|
||||
segment_id=updated_segment.id, dataset_id=dataset_id_str, session=session
|
||||
)
|
||||
response = {
|
||||
"data": load_segment_detail(updated_segment, summary.summary_content if summary else None, session=session),
|
||||
"doc_form": document.doc_form,
|
||||
}
|
||||
return dump_response(SegmentDetailResponse, response), 200
|
||||
|
||||
@service_api_ns.doc(
|
||||
summary="Get Chunk",
|
||||
description=(
|
||||
"Retrieve detailed information about a specific chunk, including its content, keywords, and "
|
||||
"indexing status."
|
||||
),
|
||||
tags=["Chunks"],
|
||||
responses={
|
||||
200: "Chunk details.",
|
||||
},
|
||||
)
|
||||
@service_api_ns.doc("get_segment")
|
||||
@service_api_ns.doc(description="Get a specific segment by ID")
|
||||
@service_api_ns.doc(params=SegmentDocParams.DATASET_DOCUMENT_SEGMENT)
|
||||
@service_api_ns.doc(
|
||||
responses={
|
||||
200: "Segment retrieved successfully",
|
||||
400: "Bad request - invalid dataset model configuration",
|
||||
401: "Unauthorized - invalid API token",
|
||||
404: "Dataset, document, or segment not found",
|
||||
}
|
||||
)
|
||||
@service_api_ns.response(
|
||||
200,
|
||||
"Segment retrieved successfully",
|
||||
service_api_ns.models[SegmentDetailResponse.__name__],
|
||||
)
|
||||
@with_session
|
||||
def get(self, session: Session, tenant_id: str, dataset_id: UUID, document_id: UUID, segment_id: UUID):
|
||||
current_account_with_tenant()
|
||||
dataset_id_str = str(dataset_id)
|
||||
# check dataset
|
||||
dataset = session.scalar(
|
||||
select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id_str).limit(1)
|
||||
)
|
||||
if not dataset:
|
||||
raise NotFound("Dataset not found.")
|
||||
# check user's model setting
|
||||
DatasetService.check_dataset_model_setting(dataset)
|
||||
document_id_str = str(document_id)
|
||||
# check document
|
||||
document = DocumentService.get_document(dataset_id_str, document_id_str, session=session)
|
||||
if not document:
|
||||
raise NotFound("Document not found.")
|
||||
segment_id_str = str(segment_id)
|
||||
_, segment = _get_segment_for_document(session, dataset, document, segment_id_str)
|
||||
|
||||
summary = SummaryIndexAdapter.get_segment_summary(
|
||||
segment_id=segment.id, dataset_id=dataset_id_str, session=session
|
||||
)
|
||||
response = {
|
||||
"data": load_segment_detail(segment, summary.summary_content if summary else None, session=session),
|
||||
"doc_form": document.doc_form,
|
||||
}
|
||||
return dump_response(SegmentDetailResponse, response), 200
|
||||
|
||||
|
||||
@service_api_ns.route(
|
||||
"/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments/<uuid:segment_id>/child_chunks"
|
||||
)
|
||||
class ChildChunkApi(DatasetApiResource):
|
||||
"""Resource for child chunks."""
|
||||
|
||||
@error_responses(ProviderNotInitializeError)
|
||||
@service_api_ns.doc(
|
||||
summary="Create Child Chunk",
|
||||
description="Create a child chunk under the specified segment.",
|
||||
tags=["Chunks"],
|
||||
responses={
|
||||
200: "Child chunk created successfully.",
|
||||
400: "`invalid_param` : Create child chunk index failed.",
|
||||
},
|
||||
)
|
||||
@service_api_ns.expect(service_api_ns.models[ChildChunkCreatePayload.__name__])
|
||||
@service_api_ns.doc("create_child_chunk")
|
||||
@service_api_ns.doc(description="Create a new child chunk for a segment")
|
||||
@service_api_ns.doc(params=SegmentDocParams.DATASET_DOCUMENT_PARENT_SEGMENT)
|
||||
@service_api_ns.doc(
|
||||
responses={
|
||||
200: "Child chunk created successfully",
|
||||
401: "Unauthorized - invalid API token",
|
||||
404: "Dataset, document, or segment not found",
|
||||
}
|
||||
)
|
||||
@service_api_ns.response(
|
||||
200,
|
||||
"Child chunk created successfully",
|
||||
service_api_ns.models[ChildChunkDetailResponse.__name__],
|
||||
)
|
||||
@cloud_edition_billing_resource_check("vector_space", "dataset")
|
||||
@cloud_edition_billing_knowledge_limit_check("add_segment", "dataset")
|
||||
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
|
||||
@with_session
|
||||
@model_validate(ChildChunkCreatePayload)
|
||||
def post(
|
||||
self,
|
||||
payload: ChildChunkCreatePayload,
|
||||
session: Session,
|
||||
tenant_id: str,
|
||||
dataset_id: UUID,
|
||||
document_id: UUID,
|
||||
segment_id: UUID,
|
||||
):
|
||||
current_account, current_tenant_id = current_account_with_tenant()
|
||||
"""Create child chunk."""
|
||||
dataset_id_str = str(dataset_id)
|
||||
# check dataset
|
||||
dataset = session.scalar(
|
||||
select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id_str).limit(1)
|
||||
)
|
||||
if not dataset:
|
||||
raise NotFound("Dataset not found.")
|
||||
|
||||
document_id_str = str(document_id)
|
||||
# check document
|
||||
document = DocumentService.get_document(dataset.id, document_id_str, session=session)
|
||||
if not document:
|
||||
raise NotFound("Document not found.")
|
||||
|
||||
segment_id_str = str(segment_id)
|
||||
_, segment = _get_segment_for_document(session, dataset, document, segment_id_str)
|
||||
|
||||
# check embedding model setting
|
||||
if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
|
||||
try:
|
||||
model_manager = ModelManager.for_tenant(tenant_id=current_tenant_id)
|
||||
model_manager.get_model_instance(
|
||||
tenant_id=current_tenant_id,
|
||||
provider=dataset.embedding_model_provider,
|
||||
model_type=ModelType.TEXT_EMBEDDING,
|
||||
model=dataset.embedding_model,
|
||||
)
|
||||
except LLMBadRequestError as caught_error:
|
||||
raise ProviderNotInitializeError(
|
||||
"No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
|
||||
) from caught_error
|
||||
except ProviderTokenNotInitError as ex:
|
||||
raise ProviderNotInitializeError(ex.description) from ex
|
||||
|
||||
try:
|
||||
child_chunk = SegmentService.create_child_chunk(
|
||||
payload.content,
|
||||
segment,
|
||||
document,
|
||||
dataset,
|
||||
session,
|
||||
actor_id=current_account.id,
|
||||
mutations=application_services().knowledge.segments.mutations,
|
||||
)
|
||||
except ChildChunkIndexingServiceError as e:
|
||||
raise ChildChunkIndexingError(str(e)) from e
|
||||
|
||||
return dump_response(ChildChunkDetailResponse, {"data": child_chunk}), 200
|
||||
|
||||
@service_api_ns.doc(
|
||||
summary="List Child Chunks",
|
||||
description="Returns a paginated list of child chunks under a specific parent chunk.",
|
||||
tags=["Chunks"],
|
||||
responses={
|
||||
200: "List of child chunks.",
|
||||
},
|
||||
)
|
||||
@service_api_ns.doc("list_child_chunks")
|
||||
@service_api_ns.doc(description="List child chunks for a segment")
|
||||
@service_api_ns.doc(params=SegmentDocParams.DATASET_DOCUMENT_PARENT_SEGMENT)
|
||||
@service_api_ns.doc(params=query_params_from_model(ChildChunkListQuery))
|
||||
@service_api_ns.doc(
|
||||
responses={
|
||||
200: "Child chunks retrieved successfully",
|
||||
401: "Unauthorized - invalid API token",
|
||||
404: "Dataset, document, or segment not found",
|
||||
}
|
||||
)
|
||||
@service_api_ns.response(
|
||||
200,
|
||||
"Child chunks retrieved successfully",
|
||||
service_api_ns.models[ChildChunkListResponse.__name__],
|
||||
)
|
||||
@with_session
|
||||
def get(self, session: Session, tenant_id: str, dataset_id: UUID, document_id: UUID, segment_id: UUID):
|
||||
current_account_with_tenant()
|
||||
"""Get child chunks."""
|
||||
dataset_id_str = str(dataset_id)
|
||||
# check dataset
|
||||
dataset = session.scalar(
|
||||
select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id_str).limit(1)
|
||||
)
|
||||
if not dataset:
|
||||
raise NotFound("Dataset not found.")
|
||||
|
||||
document_id_str = str(document_id)
|
||||
# check document
|
||||
document = DocumentService.get_document(dataset.id, document_id_str, session=session)
|
||||
if not document:
|
||||
raise NotFound("Document not found.")
|
||||
|
||||
segment_id_str = str(segment_id)
|
||||
_get_segment_for_document(session, dataset, document, segment_id_str)
|
||||
|
||||
args = query_params_from_request(ChildChunkListQuery, use_defaults_for_malformed_ints=True)
|
||||
|
||||
page = args.page
|
||||
limit = min(args.limit, 100)
|
||||
keyword = args.keyword
|
||||
|
||||
child_chunks = SegmentService.get_child_chunks(
|
||||
segment_id_str,
|
||||
document_id_str,
|
||||
dataset_id_str,
|
||||
page,
|
||||
limit,
|
||||
keyword,
|
||||
session=session,
|
||||
)
|
||||
|
||||
response = {
|
||||
"data": child_chunks.items,
|
||||
"total": child_chunks.total,
|
||||
"total_pages": child_chunks.pages,
|
||||
"page": page,
|
||||
"limit": limit,
|
||||
}
|
||||
return dump_response(ChildChunkListResponse, response), 200
|
||||
|
||||
|
||||
@service_api_ns.route(
|
||||
"/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments/<uuid:segment_id>/child_chunks/<uuid:child_chunk_id>"
|
||||
)
|
||||
class DatasetChildChunkApi(DatasetApiResource):
|
||||
"""Resource for updating child chunks."""
|
||||
|
||||
@service_api_ns.doc(
|
||||
summary="Delete Child Chunk",
|
||||
description="Permanently delete a child chunk from its parent chunk.",
|
||||
tags=["Chunks"],
|
||||
responses={
|
||||
204: "Success.",
|
||||
400: "`invalid_param` : Delete child chunk index failed.",
|
||||
},
|
||||
)
|
||||
@service_api_ns.doc("delete_child_chunk")
|
||||
@service_api_ns.doc(description="Delete a specific child chunk")
|
||||
@service_api_ns.doc(params=SegmentDocParams.DATASET_DOCUMENT_CHILD_CHUNK)
|
||||
@service_api_ns.doc(
|
||||
responses={
|
||||
204: "Child chunk deleted successfully",
|
||||
401: "Unauthorized - invalid API token",
|
||||
404: "Dataset, document, segment, or child chunk not found",
|
||||
}
|
||||
)
|
||||
@cloud_edition_billing_knowledge_limit_check("add_segment", "dataset")
|
||||
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
|
||||
@with_session
|
||||
def delete(
|
||||
self,
|
||||
session: Session,
|
||||
tenant_id: str,
|
||||
dataset_id: UUID,
|
||||
document_id: UUID,
|
||||
segment_id: UUID,
|
||||
child_chunk_id: UUID,
|
||||
):
|
||||
current_account_with_tenant()
|
||||
"""Delete child chunk."""
|
||||
dataset_id_str = str(dataset_id)
|
||||
# check dataset
|
||||
dataset = session.scalar(
|
||||
select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id_str).limit(1)
|
||||
)
|
||||
if not dataset:
|
||||
raise NotFound("Dataset not found.")
|
||||
|
||||
document_id_str = str(document_id)
|
||||
# check document
|
||||
document = DocumentService.get_document(dataset.id, document_id_str, session=session)
|
||||
if not document:
|
||||
raise NotFound("Document not found.")
|
||||
|
||||
segment_id_str = str(segment_id)
|
||||
segment_ref, _ = _get_segment_for_document(session, dataset, document, segment_id_str)
|
||||
|
||||
child_chunk_id_str = str(child_chunk_id)
|
||||
# check child chunk
|
||||
child_chunk = SegmentService.get_child_chunk_by_segment_ref(child_chunk_id_str, segment_ref, session=session)
|
||||
if not child_chunk:
|
||||
raise NotFound("Child chunk not found.")
|
||||
|
||||
try:
|
||||
SegmentService.delete_child_chunk(
|
||||
child_chunk, dataset, session, mutations=application_services().knowledge.segments.mutations
|
||||
)
|
||||
except ChildChunkDeleteIndexServiceError as e:
|
||||
raise ChildChunkDeleteIndexError(str(e)) from e
|
||||
|
||||
return "", 204
|
||||
|
||||
@service_api_ns.doc(
|
||||
summary="Update Child Chunk",
|
||||
description="Update the content of an existing child chunk.",
|
||||
tags=["Chunks"],
|
||||
responses={
|
||||
200: "Child chunk updated successfully.",
|
||||
400: "`invalid_param` : Update child chunk index failed.",
|
||||
},
|
||||
)
|
||||
@service_api_ns.expect(service_api_ns.models[ChildChunkUpdatePayload.__name__])
|
||||
@service_api_ns.doc("update_child_chunk")
|
||||
@service_api_ns.doc(description="Update a specific child chunk")
|
||||
@service_api_ns.doc(params=SegmentDocParams.DATASET_DOCUMENT_CHILD_CHUNK)
|
||||
@service_api_ns.doc(
|
||||
responses={
|
||||
200: "Child chunk updated successfully",
|
||||
401: "Unauthorized - invalid API token",
|
||||
404: "Dataset, document, segment, or child chunk not found",
|
||||
}
|
||||
)
|
||||
@service_api_ns.response(
|
||||
200,
|
||||
"Child chunk updated successfully",
|
||||
service_api_ns.models[ChildChunkDetailResponse.__name__],
|
||||
)
|
||||
@cloud_edition_billing_resource_check("vector_space", "dataset")
|
||||
@cloud_edition_billing_knowledge_limit_check("add_segment", "dataset")
|
||||
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
|
||||
@with_session
|
||||
@model_validate(ChildChunkUpdatePayload)
|
||||
def patch(
|
||||
self,
|
||||
payload: ChildChunkUpdatePayload,
|
||||
session: Session,
|
||||
tenant_id: str,
|
||||
dataset_id: UUID,
|
||||
document_id: UUID,
|
||||
segment_id: UUID,
|
||||
child_chunk_id: UUID,
|
||||
):
|
||||
current_account, _ = current_account_with_tenant()
|
||||
"""Update child chunk."""
|
||||
dataset_id_str = str(dataset_id)
|
||||
# check dataset
|
||||
dataset = session.scalar(
|
||||
select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id_str).limit(1)
|
||||
)
|
||||
if not dataset:
|
||||
raise NotFound("Dataset not found.")
|
||||
|
||||
document_id_str = str(document_id)
|
||||
# get document
|
||||
document = DocumentService.get_document(dataset_id_str, document_id_str, session=session)
|
||||
if not document:
|
||||
raise NotFound("Document not found.")
|
||||
|
||||
segment_id_str = str(segment_id)
|
||||
segment_ref, segment = _get_segment_for_document(session, dataset, document, segment_id_str)
|
||||
|
||||
child_chunk_id_str = str(child_chunk_id)
|
||||
# get child chunk
|
||||
child_chunk = SegmentService.get_child_chunk_by_segment_ref(child_chunk_id_str, segment_ref, session=session)
|
||||
if not child_chunk:
|
||||
raise NotFound("Child chunk not found.")
|
||||
|
||||
try:
|
||||
child_chunk = SegmentService.update_child_chunk(
|
||||
payload.content,
|
||||
child_chunk,
|
||||
segment,
|
||||
document,
|
||||
dataset,
|
||||
session,
|
||||
actor_id=current_account.id,
|
||||
mutations=application_services().knowledge.segments.mutations,
|
||||
)
|
||||
except ChildChunkIndexingServiceError as e:
|
||||
raise ChildChunkIndexingError(str(e)) from e
|
||||
|
||||
return dump_response(ChildChunkDetailResponse, {"data": child_chunk}), 200
|
||||
Reference in new issue
Block a user