chore: 纳入本地 Dify 1.19.0 源码及品牌与免登改动
演示目录只保留可重建的源码。运行密钥、数据卷、依赖和带口令的本地配置不进入仓库。 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
60158935f4
commit
6519682885
15831 files changed
+3063428
No files matched your search
@@ -0,0 +1,17 @@
|
||||
from services.data_migration.entities import (
|
||||
ConflictStrategy,
|
||||
ExportSelection,
|
||||
IdStrategy,
|
||||
ImportOptions,
|
||||
MigrationDataError,
|
||||
MigrationPackage,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"ConflictStrategy",
|
||||
"ExportSelection",
|
||||
"IdStrategy",
|
||||
"ImportOptions",
|
||||
"MigrationDataError",
|
||||
"MigrationPackage",
|
||||
]
|
||||
@@ -0,0 +1,92 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from services.data_migration.entities import DependencyKind
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DiscoveredDependency:
|
||||
kind: DependencyKind
|
||||
provider_id: str
|
||||
provider_name: str | None = None
|
||||
source: str | None = None
|
||||
|
||||
|
||||
class DependencyDiscoveryService:
|
||||
def discover_from_dsl(self, dsl: dict[str, Any]) -> list[DiscoveredDependency]:
|
||||
seen: set[tuple[DependencyKind, str]] = set()
|
||||
result: list[DiscoveredDependency] = []
|
||||
for node in self._nodes_from_dsl(dsl):
|
||||
data = node.get("data", {}) if isinstance(node, dict) else {}
|
||||
for dependency in self._dependencies_from_node(data):
|
||||
key = (dependency.kind, dependency.provider_id)
|
||||
if dependency.provider_id and key not in seen:
|
||||
seen.add(key)
|
||||
result.append(dependency)
|
||||
return result
|
||||
|
||||
def _nodes_from_dsl(self, dsl: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
nodes: list[dict[str, Any]] = []
|
||||
graph = dsl.get("graph") if isinstance(dsl, dict) else None
|
||||
if isinstance(graph, dict) and isinstance(graph.get("nodes"), list):
|
||||
nodes.extend(node for node in graph["nodes"] if isinstance(node, dict))
|
||||
workflow = dsl.get("workflow") if isinstance(dsl, dict) else None
|
||||
workflow_graph = workflow.get("graph") if isinstance(workflow, dict) else None
|
||||
if isinstance(workflow_graph, dict) and isinstance(workflow_graph.get("nodes"), list):
|
||||
nodes.extend(node for node in workflow_graph["nodes"] if isinstance(node, dict))
|
||||
return nodes
|
||||
|
||||
def _dependencies_from_node(self, data: dict[str, Any]) -> list[DiscoveredDependency]:
|
||||
dependencies: list[DiscoveredDependency] = []
|
||||
node_type = data.get("type")
|
||||
if node_type == "tool":
|
||||
dependency = self._from_tool_config(data, source="tool_node")
|
||||
if dependency:
|
||||
dependencies.append(dependency)
|
||||
if node_type == "agent":
|
||||
for tool_config in self._agent_tool_configs(data):
|
||||
if isinstance(tool_config, dict):
|
||||
dependency = self._from_tool_config(tool_config, source="agent_node")
|
||||
if dependency:
|
||||
dependencies.append(dependency)
|
||||
return dependencies
|
||||
|
||||
def _agent_tool_configs(self, data: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
configs = data.get("tools")
|
||||
if isinstance(configs, list):
|
||||
return [config for config in configs if isinstance(config, dict)]
|
||||
agent_parameters = data.get("agent_parameters")
|
||||
if not isinstance(agent_parameters, dict):
|
||||
return []
|
||||
tools_parameter = agent_parameters.get("tools")
|
||||
if not isinstance(tools_parameter, dict):
|
||||
return []
|
||||
value = tools_parameter.get("value", [])
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
return [config for config in value if isinstance(config, dict)]
|
||||
|
||||
def _from_tool_config(self, config: dict[str, Any], *, source: str) -> DiscoveredDependency | None:
|
||||
provider_id = config.get("provider_id") or config.get("provider_name") or config.get("provider")
|
||||
if not provider_id:
|
||||
return None
|
||||
provider_type = str(config.get("provider_type") or config.get("type") or "")
|
||||
kind = self._kind_from_provider_type(provider_type)
|
||||
return DiscoveredDependency(
|
||||
kind=kind,
|
||||
provider_id=str(provider_id),
|
||||
provider_name=config.get("provider_name"),
|
||||
source=source,
|
||||
)
|
||||
|
||||
def _kind_from_provider_type(self, provider_type: str) -> DependencyKind:
|
||||
normalized = provider_type.lower()
|
||||
if normalized in {"api", "custom", "api_tool"}:
|
||||
return DependencyKind.API_TOOL
|
||||
if normalized in {"workflow", "workflow_tool"}:
|
||||
return DependencyKind.WORKFLOW_TOOL
|
||||
if normalized == "mcp":
|
||||
return DependencyKind.MCP_TOOL
|
||||
return DependencyKind.BUILTIN_OR_PLUGIN_TOOL
|
||||
@@ -0,0 +1,241 @@
|
||||
"""Typed entities for versioned cross-environment migration packages.
|
||||
|
||||
This module is intentionally side-effect free. It owns only value objects and
|
||||
validation for migration package/config shapes; command output and database I/O
|
||||
belong in adapter and service modules built on top of these entities.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from enum import StrEnum
|
||||
from typing import Any, Literal, TypedDict
|
||||
|
||||
|
||||
class MigrationDataError(ValueError):
|
||||
"""Raised when migration config or package data is invalid."""
|
||||
|
||||
|
||||
class IdStrategy(StrEnum):
|
||||
PRESERVE_ID = "preserve-id"
|
||||
GENERATE_NEW_ID = "generate-new-id"
|
||||
|
||||
|
||||
class ConflictStrategy(StrEnum):
|
||||
FAIL = "fail"
|
||||
SKIP = "skip"
|
||||
UPDATE = "update"
|
||||
|
||||
|
||||
class ResourceType(StrEnum):
|
||||
WORKFLOW = "workflow"
|
||||
API_TOOL = "api_tool"
|
||||
WORKFLOW_TOOL = "workflow_tool"
|
||||
MCP_TOOL = "mcp_tool"
|
||||
DEPENDENCY = "dependency"
|
||||
|
||||
|
||||
class DependencyKind(StrEnum):
|
||||
API_TOOL = "api_tool"
|
||||
WORKFLOW_TOOL = "workflow_tool"
|
||||
MCP_TOOL = "mcp_tool"
|
||||
BUILTIN_OR_PLUGIN_TOOL = "builtin_or_plugin_tool"
|
||||
UNRESOLVED = "unresolved"
|
||||
|
||||
|
||||
class TargetTenantSelector(TypedDict, total=False):
|
||||
id: str
|
||||
name: str
|
||||
|
||||
|
||||
def _parse_target_tenant(value: Any) -> TargetTenantSelector | None:
|
||||
if value is None:
|
||||
return None
|
||||
if not isinstance(value, dict):
|
||||
raise MigrationDataError("metadata.target_tenant must be an object when provided.")
|
||||
target: TargetTenantSelector = {}
|
||||
target_id = value.get("id")
|
||||
if target_id is not None:
|
||||
if not isinstance(target_id, str):
|
||||
raise MigrationDataError("metadata.target_tenant.id must be a string.")
|
||||
target["id"] = target_id
|
||||
target_name = value.get("name")
|
||||
if target_name is not None:
|
||||
if not isinstance(target_name, str):
|
||||
raise MigrationDataError("metadata.target_tenant.name must be a string.")
|
||||
target["name"] = target_name
|
||||
unsupported_keys = sorted(set(value.keys()) - {"id", "name"})
|
||||
if unsupported_keys:
|
||||
raise MigrationDataError(f"metadata.target_tenant contains unsupported fields: {unsupported_keys}")
|
||||
return target
|
||||
|
||||
|
||||
def _parse_package_section(value: Any, section: str) -> list[dict[str, Any]]:
|
||||
if value is None:
|
||||
return []
|
||||
if not isinstance(value, list):
|
||||
raise MigrationDataError(f"Migration package field '{section}' must be a list.")
|
||||
for item in value:
|
||||
if not isinstance(item, dict):
|
||||
raise MigrationDataError(f"Migration package field '{section}' must contain only objects.")
|
||||
return value
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SourceTenant:
|
||||
id: str
|
||||
name: str
|
||||
|
||||
@classmethod
|
||||
def from_mapping(cls, value: dict[str, Any]) -> SourceTenant:
|
||||
return cls(id=str(value.get("id", "")), name=str(value.get("name", "")))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImportOptions:
|
||||
create_app_api_token_on_import: bool = False
|
||||
id_strategy: IdStrategy = IdStrategy.PRESERVE_ID
|
||||
conflict_strategy: ConflictStrategy = ConflictStrategy.FAIL
|
||||
|
||||
@classmethod
|
||||
def from_mapping(cls, value: dict[str, Any] | None) -> ImportOptions:
|
||||
value = value or {}
|
||||
try:
|
||||
id_strategy = IdStrategy(value.get("id_strategy", IdStrategy.PRESERVE_ID))
|
||||
except ValueError as exc:
|
||||
raise MigrationDataError(f"Unsupported import_options.id_strategy: {value.get('id_strategy')}") from exc
|
||||
try:
|
||||
conflict_strategy = ConflictStrategy(value.get("conflict_strategy", ConflictStrategy.FAIL))
|
||||
except ValueError as exc:
|
||||
raise MigrationDataError(
|
||||
f"Unsupported import_options.conflict_strategy: {value.get('conflict_strategy')}"
|
||||
) from exc
|
||||
return cls(
|
||||
create_app_api_token_on_import=bool(value.get("create_app_api_token_on_import", False)),
|
||||
id_strategy=id_strategy,
|
||||
conflict_strategy=conflict_strategy,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MigrationMetadata:
|
||||
version: str
|
||||
source_scope: Literal["single"]
|
||||
source_tenants: list[SourceTenant]
|
||||
target_tenant: TargetTenantSelector | None = None
|
||||
created_at: str | None = None
|
||||
include_secrets: bool = False
|
||||
import_options: ImportOptions = field(default_factory=ImportOptions)
|
||||
|
||||
@classmethod
|
||||
def from_mapping(cls, value: dict[str, Any]) -> MigrationMetadata:
|
||||
version = value.get("version")
|
||||
if not version:
|
||||
raise MigrationDataError("Migration package must include metadata.version.")
|
||||
source_scope = value.get("source_scope", "single")
|
||||
if source_scope != "single":
|
||||
raise MigrationDataError(f"Unsupported source_scope: {source_scope}")
|
||||
source_tenants = [
|
||||
SourceTenant.from_mapping(item) for item in value.get("source_tenants", []) if isinstance(item, dict)
|
||||
]
|
||||
return cls(
|
||||
version=str(version),
|
||||
source_scope="single",
|
||||
source_tenants=source_tenants,
|
||||
target_tenant=_parse_target_tenant(value.get("target_tenant")),
|
||||
created_at=value.get("created_at"),
|
||||
include_secrets=bool(value.get("include_secrets", False)),
|
||||
import_options=ImportOptions.from_mapping(value.get("import_options")),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MigrationPackage:
|
||||
metadata: MigrationMetadata
|
||||
workflows: list[dict[str, Any]] = field(default_factory=list)
|
||||
tools: list[dict[str, Any]] = field(default_factory=list)
|
||||
workflow_tools: list[dict[str, Any]] = field(default_factory=list)
|
||||
mcp_tools: list[dict[str, Any]] = field(default_factory=list)
|
||||
dependencies: list[dict[str, Any]] = field(default_factory=list)
|
||||
|
||||
@classmethod
|
||||
def from_mapping(cls, value: dict[str, Any]) -> MigrationPackage:
|
||||
metadata_value = value.get("metadata")
|
||||
if not isinstance(metadata_value, dict):
|
||||
raise MigrationDataError("Migration package must include metadata.version.")
|
||||
return cls(
|
||||
metadata=MigrationMetadata.from_mapping(metadata_value),
|
||||
workflows=_parse_package_section(value.get("workflows"), "workflows"),
|
||||
tools=_parse_package_section(value.get("tools"), "tools"),
|
||||
workflow_tools=_parse_package_section(value.get("workflow_tools"), "workflow_tools"),
|
||||
mcp_tools=_parse_package_section(value.get("mcp_tools"), "mcp_tools"),
|
||||
dependencies=_parse_package_section(value.get("dependencies"), "dependencies"),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ExportSelection:
|
||||
source_tenant_name: str
|
||||
app_ids: list[str]
|
||||
source_tenant_id: str | None = None
|
||||
export_all_apps: bool = False
|
||||
include_referenced_tools: bool = True
|
||||
additional_api_tools: list[str] = field(default_factory=list)
|
||||
additional_workflow_tools: list[str] = field(default_factory=list)
|
||||
additional_mcp_tools: list[str] = field(default_factory=list)
|
||||
include_secrets: bool = False
|
||||
import_options: ImportOptions = field(default_factory=ImportOptions)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResourceReportItem:
|
||||
resource_type: ResourceType
|
||||
identifier: str
|
||||
name: str | None
|
||||
status: str
|
||||
message: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResourceIdMapping:
|
||||
resource_type: ResourceType
|
||||
name: str | None
|
||||
source_id: str
|
||||
target_id: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ExportResult:
|
||||
package: MigrationPackage
|
||||
report_items: list[ResourceReportItem]
|
||||
report_context: ReportContext | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImportTarget:
|
||||
tenant_id: str
|
||||
tenant_name: str
|
||||
operator_id: str
|
||||
operator_email: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImportResult:
|
||||
report_items: list[ResourceReportItem]
|
||||
id_mapping: dict[str, str] = field(default_factory=dict)
|
||||
report_context: ReportContext | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReportContext:
|
||||
output_path: str | None = None
|
||||
source_scope: str | None = None
|
||||
selected_app_count: int | None = None
|
||||
include_secrets: bool | None = None
|
||||
target_tenant: str | None = None
|
||||
operator_email: str | None = None
|
||||
app_api_tokens_created: int = 0
|
||||
app_api_tokens_reused: int = 0
|
||||
id_mapping_count: int = 0
|
||||
id_mappings: dict[str, str] = field(default_factory=dict)
|
||||
id_mapping_details: list[ResourceIdMapping] = field(default_factory=list)
|
||||
@@ -0,0 +1,488 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
import sqlalchemy as sa
|
||||
import yaml
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from core.tools.tool_manager import ToolManager
|
||||
from graphon.model_runtime.utils.encoders import jsonable_encoder
|
||||
from models import Account, Tenant
|
||||
from models.account import TenantAccountJoin
|
||||
from models.model import App
|
||||
from models.tools import MCPToolProvider
|
||||
from services.app_dsl_service import AppDslService
|
||||
from services.data_migration.dependency_discovery_service import DependencyDiscoveryService, DiscoveredDependency
|
||||
from services.data_migration.entities import (
|
||||
DependencyKind,
|
||||
ExportResult,
|
||||
ExportSelection,
|
||||
ImportOptions,
|
||||
MigrationDataError,
|
||||
ReportContext,
|
||||
ResourceReportItem,
|
||||
ResourceType,
|
||||
)
|
||||
from services.data_migration.package_service import MigrationPackageService
|
||||
from services.tools.mcp_tools_manage_service import MCPToolManageService
|
||||
from services.tools.workflow_tools_manage_service import WorkflowToolManageService
|
||||
|
||||
SUPPORTED_APP_MODES = {"workflow", "advanced-chat"}
|
||||
|
||||
|
||||
class ExportConfigParser:
|
||||
def parse(self, data: dict[str, Any]) -> ExportSelection:
|
||||
if not isinstance(data, dict):
|
||||
raise MigrationDataError("Export config JSON must be an object.")
|
||||
|
||||
source_tenant = self._source_tenant(data)
|
||||
source_tenant_name = self._source_tenant_name(source_tenant, data)
|
||||
apps = self._mapping(data.get("apps"), field_name="apps")
|
||||
self._validate_source_scope(data)
|
||||
self._validate_app_modes(apps.get("modes", []))
|
||||
|
||||
additional_tools = self._mapping(data.get("additional_tools"), field_name="additional_tools")
|
||||
return ExportSelection(
|
||||
source_tenant_name=source_tenant_name,
|
||||
app_ids=self._string_list(apps.get("ids", data.get("workflows", [])), field_name="apps.ids"),
|
||||
source_tenant_id=source_tenant.get("id"),
|
||||
export_all_apps=bool(apps.get("all", data.get("export_all_workflows", False))),
|
||||
include_referenced_tools=bool(data.get("include_referenced_tools", True)),
|
||||
additional_api_tools=self._string_list(
|
||||
additional_tools.get("api_tools", data.get("tools", [])), field_name="additional_tools.api_tools"
|
||||
),
|
||||
additional_workflow_tools=self._string_list(
|
||||
additional_tools.get("workflow_tools", data.get("workflow_tools", [])),
|
||||
field_name="additional_tools.workflow_tools",
|
||||
),
|
||||
additional_mcp_tools=self._string_list(
|
||||
additional_tools.get("mcp_tools", data.get("mcp_tools", [])),
|
||||
field_name="additional_tools.mcp_tools",
|
||||
),
|
||||
include_secrets=bool(data.get("include_secrets", False)),
|
||||
import_options=ImportOptions.from_mapping(data.get("import_options")),
|
||||
)
|
||||
|
||||
def _source_tenant(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
if "source_tenant" in data:
|
||||
return self._mapping(data.get("source_tenant"), field_name="source_tenant")
|
||||
return {}
|
||||
|
||||
def _source_tenant_name(self, source_tenant: dict[str, Any], data: dict[str, Any]) -> str:
|
||||
if source_tenant:
|
||||
source_tenant_name = source_tenant.get("name")
|
||||
if not source_tenant_name:
|
||||
raise MigrationDataError("Export config must include source_tenant.name.")
|
||||
return str(source_tenant_name)
|
||||
source_tenant_name = data.get("tenant_name")
|
||||
if not source_tenant_name:
|
||||
raise MigrationDataError("Export config must include source_tenant.name.")
|
||||
return str(source_tenant_name)
|
||||
|
||||
def _validate_source_scope(self, data: dict[str, Any]) -> None:
|
||||
source_tenant = data.get("source_tenant")
|
||||
if not isinstance(source_tenant, dict):
|
||||
return
|
||||
mode = source_tenant.get("mode", "single")
|
||||
if mode != "single":
|
||||
raise MigrationDataError(f"Unsupported source_tenant.mode: {mode}")
|
||||
|
||||
def _validate_app_modes(self, modes: Any) -> None:
|
||||
app_modes = self._string_list(modes, field_name="apps.modes") if modes else []
|
||||
unsupported_modes = sorted(set(app_modes) - SUPPORTED_APP_MODES)
|
||||
if unsupported_modes:
|
||||
raise MigrationDataError(f"Unsupported app modes for export: {unsupported_modes}")
|
||||
|
||||
def _mapping(self, value: Any, *, field_name: str) -> dict[str, Any]:
|
||||
if value is None:
|
||||
return {}
|
||||
if not isinstance(value, dict):
|
||||
raise MigrationDataError(f"Export config field '{field_name}' must be an object.")
|
||||
return value
|
||||
|
||||
def _string_list(self, value: Any, *, field_name: str) -> list[str]:
|
||||
if value is None:
|
||||
return []
|
||||
if not isinstance(value, list):
|
||||
raise MigrationDataError(f"Export config field '{field_name}' must be a list.")
|
||||
return [str(item) for item in value]
|
||||
|
||||
|
||||
class MigrationExportService:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
package_service: MigrationPackageService | None = None,
|
||||
dependency_discovery_service: DependencyDiscoveryService | None = None,
|
||||
) -> None:
|
||||
self.package_service = package_service or MigrationPackageService()
|
||||
self.dependency_discovery_service = dependency_discovery_service or DependencyDiscoveryService()
|
||||
|
||||
def export(self, selection: ExportSelection, *, session: Session) -> ExportResult:
|
||||
tenant = self._get_tenant(selection, session=session)
|
||||
package = self.package_service.build_empty_package(
|
||||
source_tenant_id=tenant.id,
|
||||
source_tenant_name=tenant.name,
|
||||
include_secrets=selection.include_secrets,
|
||||
import_options=selection.import_options,
|
||||
)
|
||||
report_items: list[ResourceReportItem] = []
|
||||
discovered_dependencies: list[DiscoveredDependency] = []
|
||||
|
||||
apps = self._selected_apps(tenant.id, selection, session=session)
|
||||
exported_app_ids = {app.id for app in apps}
|
||||
for app in apps:
|
||||
dsl_content = AppDslService.export_dsl(
|
||||
app_model=app, session=session, include_secret=selection.include_secrets
|
||||
)
|
||||
package.workflows.append(
|
||||
{
|
||||
"id": app.id,
|
||||
"name": app.name,
|
||||
"mode": app.mode.value if hasattr(app.mode, "value") else app.mode,
|
||||
"dsl": dsl_content,
|
||||
"source_tenant_id": tenant.id,
|
||||
"create_app_api_token_on_import": selection.import_options.create_app_api_token_on_import,
|
||||
}
|
||||
)
|
||||
report_items.append(ResourceReportItem(ResourceType.WORKFLOW, app.id, app.name, "exported"))
|
||||
if selection.include_referenced_tools:
|
||||
discovered_dependencies.extend(self._discover_dependencies(dsl_content))
|
||||
|
||||
self._export_api_tools(
|
||||
tenant.id,
|
||||
self._provider_ids(selection.additional_api_tools, discovered_dependencies, DependencyKind.API_TOOL),
|
||||
include_secrets=selection.include_secrets,
|
||||
exported_tools=package.tools,
|
||||
report_items=report_items,
|
||||
)
|
||||
self._export_workflow_tools(
|
||||
tenant,
|
||||
self._provider_ids(
|
||||
selection.additional_workflow_tools, discovered_dependencies, DependencyKind.WORKFLOW_TOOL
|
||||
),
|
||||
exported_app_ids=exported_app_ids,
|
||||
exported_workflow_tools=package.workflow_tools,
|
||||
dependencies=package.dependencies,
|
||||
report_items=report_items,
|
||||
session=session,
|
||||
)
|
||||
self._export_mcp_tools(
|
||||
tenant_id=tenant.id,
|
||||
provider_ids=self._provider_ids(
|
||||
selection.additional_mcp_tools,
|
||||
discovered_dependencies,
|
||||
DependencyKind.MCP_TOOL,
|
||||
),
|
||||
include_secrets=selection.include_secrets,
|
||||
exported_mcp_tools=package.mcp_tools,
|
||||
dependencies=package.dependencies,
|
||||
report_items=report_items,
|
||||
session=session,
|
||||
)
|
||||
self._record_dependency_metadata(
|
||||
self._dependencies_by_kind(discovered_dependencies, DependencyKind.BUILTIN_OR_PLUGIN_TOOL),
|
||||
package.dependencies,
|
||||
report_items,
|
||||
)
|
||||
return ExportResult(
|
||||
package=package,
|
||||
report_items=report_items,
|
||||
report_context=ReportContext(
|
||||
source_scope=package.metadata.source_scope,
|
||||
selected_app_count=len(apps),
|
||||
include_secrets=selection.include_secrets,
|
||||
),
|
||||
)
|
||||
|
||||
def _get_tenant(self, selection: ExportSelection, *, session: Session) -> Tenant:
|
||||
if selection.source_tenant_id:
|
||||
tenant = session.get(Tenant, selection.source_tenant_id)
|
||||
if tenant is None:
|
||||
raise MigrationDataError(f"Source tenant not found: {selection.source_tenant_id}")
|
||||
if tenant.name != selection.source_tenant_name:
|
||||
raise MigrationDataError(
|
||||
f"Source tenant id/name mismatch: {selection.source_tenant_id} / {selection.source_tenant_name}"
|
||||
)
|
||||
return tenant
|
||||
tenants = list(session.scalars(sa.select(Tenant).where(Tenant.name == selection.source_tenant_name)).all())
|
||||
if not tenants:
|
||||
raise MigrationDataError(f"Source tenant not found: {selection.source_tenant_name}")
|
||||
if len(tenants) > 1:
|
||||
raise MigrationDataError(
|
||||
f"Source tenant name is ambiguous; use source_tenant.id: {selection.source_tenant_name}"
|
||||
)
|
||||
return tenants[0]
|
||||
|
||||
def _selected_apps(self, tenant_id: str, selection: ExportSelection, *, session: Session) -> list[App]:
|
||||
query = sa.select(App).where(App.tenant_id == tenant_id, App.mode.in_(SUPPORTED_APP_MODES))
|
||||
if not selection.export_all_apps:
|
||||
if not selection.app_ids:
|
||||
return []
|
||||
query = query.where(App.id.in_(selection.app_ids))
|
||||
apps = list(session.scalars(query).all())
|
||||
if not selection.export_all_apps and len(apps) != len(set(selection.app_ids)):
|
||||
found_ids = {app.id for app in apps}
|
||||
missing_ids = [app_id for app_id in selection.app_ids if app_id not in found_ids]
|
||||
raise MigrationDataError(
|
||||
f"Selected app IDs not found in source tenant or unsupported app mode: {missing_ids}"
|
||||
)
|
||||
return apps
|
||||
|
||||
def _discover_dependencies(self, dsl_content: str | dict[str, Any]) -> list[DiscoveredDependency]:
|
||||
if isinstance(dsl_content, dict):
|
||||
dsl = dsl_content
|
||||
else:
|
||||
raw_dsl = yaml.safe_load(dsl_content) if dsl_content else {}
|
||||
dsl = raw_dsl if isinstance(raw_dsl, dict) else {}
|
||||
return self.dependency_discovery_service.discover_from_dsl(dsl)
|
||||
|
||||
def _export_api_tools(
|
||||
self,
|
||||
tenant_id: str,
|
||||
provider_ids: Iterable[str],
|
||||
*,
|
||||
include_secrets: bool,
|
||||
exported_tools: list[dict[str, Any]],
|
||||
report_items: list[ResourceReportItem],
|
||||
) -> None:
|
||||
for provider_id in self._dedupe(provider_ids):
|
||||
try:
|
||||
tool_data = ToolManager.user_get_api_provider(
|
||||
provider=provider_id,
|
||||
tenant_id=tenant_id,
|
||||
mask=not include_secrets,
|
||||
)
|
||||
if not include_secrets:
|
||||
tool_data.pop("credentials", None)
|
||||
tool_data.pop("tools", None)
|
||||
tool_data["provider_name"] = provider_id
|
||||
tool_data["source_tenant_id"] = tenant_id
|
||||
exported_tools.append(tool_data)
|
||||
report_items.append(ResourceReportItem(ResourceType.API_TOOL, provider_id, provider_id, "exported"))
|
||||
except Exception as exc:
|
||||
report_items.append(
|
||||
ResourceReportItem(ResourceType.API_TOOL, provider_id, provider_id, "unresolved", str(exc))
|
||||
)
|
||||
|
||||
def _export_workflow_tools(
|
||||
self,
|
||||
tenant: Tenant,
|
||||
provider_ids: Iterable[str],
|
||||
*,
|
||||
exported_app_ids: set[str],
|
||||
exported_workflow_tools: list[dict[str, Any]],
|
||||
dependencies: list[dict[str, Any]],
|
||||
report_items: list[ResourceReportItem],
|
||||
session: Session,
|
||||
) -> None:
|
||||
provider_ids = self._dedupe(provider_ids)
|
||||
if not provider_ids:
|
||||
return
|
||||
owner = self._get_tenant_owner(tenant.id, session=session)
|
||||
if owner is None:
|
||||
for provider_id in provider_ids:
|
||||
report_items.append(
|
||||
ResourceReportItem(
|
||||
ResourceType.WORKFLOW_TOOL,
|
||||
provider_id,
|
||||
provider_id,
|
||||
"unresolved",
|
||||
f"No owner account found for source tenant: {tenant.name}",
|
||||
)
|
||||
)
|
||||
return
|
||||
|
||||
for provider_id in provider_ids:
|
||||
try:
|
||||
tool_data = WorkflowToolManageService.get_workflow_tool_by_tool_id(
|
||||
user_id=owner.id,
|
||||
tenant_id=tenant.id,
|
||||
workflow_tool_id=provider_id,
|
||||
)
|
||||
tool_info = jsonable_encoder(tool_data)
|
||||
tool_info["id"] = provider_id
|
||||
tool_info["app_id"] = tool_info.get("workflow_app_id")
|
||||
tool_info["source_tenant_id"] = tenant.id
|
||||
for field_name in ("workflow_tool_id", "workflow_app_id", "tool"):
|
||||
tool_info.pop(field_name, None)
|
||||
exported_workflow_tools.append(tool_info)
|
||||
if tool_info.get("app_id") not in exported_app_ids:
|
||||
workflow_app_id = str(tool_info.get("app_id") or "")
|
||||
workflow_app = session.get(App, workflow_app_id) if workflow_app_id else None
|
||||
self._record_dependency_metadata(
|
||||
[
|
||||
DiscoveredDependency(
|
||||
DependencyKind.WORKFLOW_TOOL,
|
||||
workflow_app_id,
|
||||
provider_name=workflow_app.name if workflow_app else tool_info.get("name"),
|
||||
source="workflow_tool_app",
|
||||
)
|
||||
],
|
||||
dependencies,
|
||||
report_items,
|
||||
)
|
||||
report_items.append(
|
||||
ResourceReportItem(ResourceType.WORKFLOW_TOOL, provider_id, tool_info.get("name"), "exported")
|
||||
)
|
||||
except Exception as exc:
|
||||
report_items.append(
|
||||
ResourceReportItem(ResourceType.WORKFLOW_TOOL, provider_id, provider_id, "unresolved", str(exc))
|
||||
)
|
||||
|
||||
def _get_tenant_owner(self, tenant_id: str, *, session: Session) -> Account | None:
|
||||
return session.scalar(
|
||||
sa.select(Account)
|
||||
.join(TenantAccountJoin, Account.id == TenantAccountJoin.account_id)
|
||||
.where(TenantAccountJoin.tenant_id == tenant_id, TenantAccountJoin.role == "owner")
|
||||
.order_by(TenantAccountJoin.created_at.asc())
|
||||
.limit(1)
|
||||
)
|
||||
|
||||
def _export_mcp_tools(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
provider_ids: Iterable[str],
|
||||
include_secrets: bool,
|
||||
exported_mcp_tools: list[dict[str, Any]],
|
||||
dependencies: list[dict[str, Any]],
|
||||
report_items: list[ResourceReportItem],
|
||||
session: Session,
|
||||
) -> None:
|
||||
for provider_id in self._dedupe(provider_ids):
|
||||
if not include_secrets:
|
||||
self._record_dependency_metadata(
|
||||
[DiscoveredDependency(DependencyKind.MCP_TOOL, provider_id, source="mcp_provider")],
|
||||
dependencies,
|
||||
report_items,
|
||||
)
|
||||
continue
|
||||
try:
|
||||
provider = self._get_mcp_provider(tenant_id, provider_id, session=session)
|
||||
exported_mcp_tools.append(self._serialize_mcp_provider(provider))
|
||||
report_items.append(ResourceReportItem(ResourceType.MCP_TOOL, provider_id, provider.name, "exported"))
|
||||
except Exception as exc:
|
||||
report_items.append(
|
||||
ResourceReportItem(ResourceType.MCP_TOOL, provider_id, provider_id, "unresolved", str(exc))
|
||||
)
|
||||
|
||||
def _get_mcp_provider(self, tenant_id: str, provider_id: str, *, session: Session) -> MCPToolProvider:
|
||||
try:
|
||||
return MCPToolManageService(session=session).get_provider_by_persisted_reference(
|
||||
id_or_server_identifier=provider_id, tenant_id=tenant_id
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise MigrationDataError(f"MCP provider not found: {provider_id}") from exc
|
||||
|
||||
def _serialize_mcp_provider(self, provider: MCPToolProvider) -> dict[str, Any]:
|
||||
provider_entity = provider.to_entity()
|
||||
provider_icon = provider_entity.provider_icon
|
||||
if isinstance(provider_icon, dict):
|
||||
icon = provider_icon.get("content")
|
||||
icon_background = provider_icon.get("background")
|
||||
icon_type = "emoji"
|
||||
else:
|
||||
icon = provider_icon
|
||||
icon_background = None
|
||||
icon_type = "url"
|
||||
return {
|
||||
"id": provider.id,
|
||||
"name": provider.name,
|
||||
"server_url": provider_entity.decrypt_server_url(),
|
||||
"server_identifier": provider.server_identifier,
|
||||
"icon": icon,
|
||||
"icon_background": icon_background,
|
||||
"icon_type": icon_type,
|
||||
"configuration": {"timeout": provider.timeout, "sse_read_timeout": provider.sse_read_timeout},
|
||||
"headers": provider_entity.decrypt_headers(),
|
||||
"authentication": self._serialize_mcp_authentication(provider_entity.decrypt_authentication()),
|
||||
"tools": provider.tool_dict,
|
||||
"source_tenant_id": provider.tenant_id,
|
||||
}
|
||||
|
||||
def _serialize_mcp_authentication(self, authentication: dict[str, Any] | None) -> dict[str, Any] | None:
|
||||
if not authentication or not authentication.get("client_id"):
|
||||
return None
|
||||
return {
|
||||
"client_id": authentication["client_id"],
|
||||
"client_secret": authentication.get("client_secret"),
|
||||
}
|
||||
|
||||
def _record_dependency_metadata(
|
||||
self,
|
||||
dependencies_to_record: Iterable[DiscoveredDependency],
|
||||
dependencies: list[dict[str, Any]],
|
||||
report_items: list[ResourceReportItem],
|
||||
) -> None:
|
||||
existing = {(item.get("kind"), item.get("provider_id")) for item in dependencies}
|
||||
for dependency in dependencies_to_record:
|
||||
key = (dependency.kind.value, dependency.provider_id)
|
||||
if key in existing:
|
||||
continue
|
||||
existing.add(key)
|
||||
dependencies.append(
|
||||
{
|
||||
"kind": dependency.kind.value,
|
||||
"provider_id": dependency.provider_id,
|
||||
"provider_name": dependency.provider_name,
|
||||
"source": dependency.source,
|
||||
}
|
||||
)
|
||||
report_items.append(
|
||||
ResourceReportItem(
|
||||
ResourceType.DEPENDENCY,
|
||||
dependency.provider_id,
|
||||
self._dependency_report_name(dependency),
|
||||
"dependency-only",
|
||||
self._dependency_message(dependency.kind),
|
||||
)
|
||||
)
|
||||
|
||||
def _provider_ids(
|
||||
self,
|
||||
manual_provider_ids: Iterable[str],
|
||||
discovered_dependencies: Iterable[DiscoveredDependency],
|
||||
kind: DependencyKind,
|
||||
) -> list[str]:
|
||||
provider_ids = list(manual_provider_ids)
|
||||
provider_ids.extend(
|
||||
self._provider_export_identifier(dependency)
|
||||
for dependency in discovered_dependencies
|
||||
if dependency.kind == kind
|
||||
)
|
||||
return self._dedupe(provider_ids)
|
||||
|
||||
def _provider_export_identifier(self, dependency: DiscoveredDependency) -> str:
|
||||
if dependency.kind == DependencyKind.API_TOOL and dependency.provider_name:
|
||||
return dependency.provider_name
|
||||
return dependency.provider_id
|
||||
|
||||
def _dependencies_by_kind(
|
||||
self, discovered_dependencies: Iterable[DiscoveredDependency], kind: DependencyKind
|
||||
) -> list[DiscoveredDependency]:
|
||||
return [dependency for dependency in discovered_dependencies if dependency.kind == kind]
|
||||
|
||||
def _dedupe(self, values: Iterable[str]) -> list[str]:
|
||||
seen: set[str] = set()
|
||||
result: list[str] = []
|
||||
for value in values:
|
||||
if value and value not in seen:
|
||||
seen.add(value)
|
||||
result.append(value)
|
||||
return result
|
||||
|
||||
def _dependency_message(self, kind: DependencyKind) -> str:
|
||||
if kind == DependencyKind.MCP_TOOL:
|
||||
return "Configure MCP provider manually in the target tenant unless exporting with secrets enabled."
|
||||
if kind == DependencyKind.BUILTIN_OR_PLUGIN_TOOL:
|
||||
return "Ensure the built-in or plugin tool exists in the target environment."
|
||||
return "Dependency metadata only; ensure the resource exists in the target environment."
|
||||
|
||||
def _dependency_report_name(self, dependency: DiscoveredDependency) -> str:
|
||||
name = dependency.provider_name or dependency.provider_id
|
||||
if dependency.kind == DependencyKind.WORKFLOW_TOOL:
|
||||
return f"workflow {name}"
|
||||
return f"{dependency.kind.value} {name}"
|
||||
@@ -0,0 +1,996 @@
|
||||
"""Apply versioned migration packages to an explicitly resolved target tenant.
|
||||
|
||||
Import target resolution is deliberately performed before any resource import
|
||||
work. The service does not write Click output; callers receive structured
|
||||
report items and can decide how to render them.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, cast
|
||||
from uuid import UUID
|
||||
|
||||
import sqlalchemy as sa
|
||||
import yaml
|
||||
from sqlalchemy import or_
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from configs import dify_config
|
||||
from core.entities.mcp_provider import IdentityMode, MCPAuthentication, MCPConfiguration
|
||||
from core.tools.entities.tool_entities import ApiProviderSchemaType, WorkflowToolParameterConfiguration
|
||||
from extensions.ext_database import db
|
||||
from libs.datetime_utils import naive_utc_now
|
||||
from models import Account, ApiToken, Tenant, TenantAccountJoin, TenantAccountRole
|
||||
from models.enums import ApiTokenType
|
||||
from models.model import App
|
||||
from models.tools import ApiToolProvider, MCPToolProvider, WorkflowToolProvider
|
||||
from services.app_dsl_service import AppDslService
|
||||
from services.data_migration.dependency_discovery_service import DependencyDiscoveryService
|
||||
from services.data_migration.entities import (
|
||||
ConflictStrategy,
|
||||
DependencyKind,
|
||||
IdStrategy,
|
||||
ImportOptions,
|
||||
ImportResult,
|
||||
ImportTarget,
|
||||
MigrationDataError,
|
||||
MigrationPackage,
|
||||
ReportContext,
|
||||
ResourceIdMapping,
|
||||
ResourceReportItem,
|
||||
ResourceType,
|
||||
)
|
||||
from services.entities.dsl_entities import ImportStatus
|
||||
from services.tools.api_tools_manage_service import ApiToolManageService
|
||||
from services.tools.mcp_tools_manage_service import MCPToolManageService
|
||||
from services.tools.workflow_tools_manage_service import WorkflowToolManageService
|
||||
from services.workflow_service import WorkflowService
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImportRequest:
|
||||
"""Structured input for package import.
|
||||
|
||||
`cli_target_tenant` and `config_target_tenant` are target tenant names from
|
||||
outer adapters. They intentionally override package metadata, because a
|
||||
migration package may be reused across environments.
|
||||
"""
|
||||
|
||||
package: MigrationPackage
|
||||
cli_target_tenant: str | None = None
|
||||
config_target_tenant: str | None = None
|
||||
operator_email: str | None = None
|
||||
options_override: ImportOptions | None = None
|
||||
|
||||
|
||||
class ImportTargetResolver:
|
||||
"""Resolve the target tenant and operator before import side effects begin."""
|
||||
|
||||
def select_target_tenant_name(self, request: ImportRequest) -> str:
|
||||
if request.cli_target_tenant:
|
||||
return request.cli_target_tenant
|
||||
if request.config_target_tenant:
|
||||
return request.config_target_tenant
|
||||
package_target = request.package.metadata.target_tenant or {}
|
||||
if package_target.get("name"):
|
||||
return package_target["name"]
|
||||
if package_target.get("id"):
|
||||
return package_target["id"]
|
||||
raise MigrationDataError(
|
||||
"Target tenant must be provided by --target-tenant, import config, or package metadata."
|
||||
)
|
||||
|
||||
def resolve(self, request: ImportRequest, *, session: Session) -> ImportTarget:
|
||||
target_tenant_name = self.select_target_tenant_name(request)
|
||||
package_target = request.package.metadata.target_tenant or {}
|
||||
if request.cli_target_tenant or request.config_target_tenant:
|
||||
tenant = self._resolve_tenant_by_id_or_name(target_tenant_name, session=session)
|
||||
elif package_target.get("id") and self._is_uuid(package_target["id"]):
|
||||
tenant = session.get(Tenant, package_target["id"])
|
||||
if tenant is not None and package_target.get("name") and tenant.name != package_target.get("name"):
|
||||
raise MigrationDataError(
|
||||
f"Target tenant id/name mismatch: {package_target['id']} / {package_target['name']}"
|
||||
)
|
||||
else:
|
||||
tenant = self._resolve_tenant_by_id_or_name(target_tenant_name, session=session)
|
||||
if tenant is None:
|
||||
raise MigrationDataError(f"Target tenant not found: {target_tenant_name}")
|
||||
|
||||
account_query = (
|
||||
session.query(Account)
|
||||
.join(TenantAccountJoin, Account.id == TenantAccountJoin.account_id)
|
||||
.filter(TenantAccountJoin.tenant_id == tenant.id)
|
||||
)
|
||||
if request.operator_email:
|
||||
account_query = account_query.filter(Account.email == request.operator_email)
|
||||
identity = request.operator_email
|
||||
else:
|
||||
account_query = account_query.filter(TenantAccountJoin.role == TenantAccountRole.OWNER).order_by(
|
||||
TenantAccountJoin.created_at.asc()
|
||||
)
|
||||
identity = "earliest owner"
|
||||
|
||||
account = account_query.first()
|
||||
if account is None:
|
||||
raise MigrationDataError(f"No operator account found for target tenant {target_tenant_name}: {identity}")
|
||||
|
||||
return ImportTarget(
|
||||
tenant_id=tenant.id,
|
||||
tenant_name=tenant.name,
|
||||
operator_id=account.id,
|
||||
operator_email=account.email,
|
||||
)
|
||||
|
||||
def _resolve_tenant_by_id_or_name(self, value: str, *, session: Session) -> Tenant | None:
|
||||
if self._is_uuid(value):
|
||||
tenant = session.get(Tenant, value)
|
||||
if tenant is not None:
|
||||
return tenant
|
||||
tenants = list(session.scalars(sa.select(Tenant).where(Tenant.name == value)).all())
|
||||
if len(tenants) > 1:
|
||||
raise MigrationDataError(f"Target tenant name is ambiguous; use target_tenant.id: {value}")
|
||||
return tenants[0] if tenants else None
|
||||
|
||||
def _is_uuid(self, value: str) -> bool:
|
||||
try:
|
||||
UUID(value)
|
||||
except ValueError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class MigrationImportService:
|
||||
"""Apply package resources using Dify service APIs and structured reporting."""
|
||||
|
||||
target_resolver: ImportTargetResolver
|
||||
|
||||
def __init__(self, *, target_resolver: ImportTargetResolver | None = None) -> None:
|
||||
self.target_resolver = target_resolver or ImportTargetResolver()
|
||||
|
||||
def import_package(self, request: ImportRequest, *, session: Session) -> ImportResult:
|
||||
target = self.target_resolver.resolve(request, session=session)
|
||||
options = request.options_override or request.package.metadata.import_options
|
||||
report_items = [
|
||||
ResourceReportItem(
|
||||
resource_type=ResourceType.DEPENDENCY,
|
||||
identifier=target.tenant_id,
|
||||
name=target.tenant_name,
|
||||
status="resolved",
|
||||
message=f"operator: {target.operator_email or target.operator_id}",
|
||||
)
|
||||
]
|
||||
id_mapping: dict[str, str] = {}
|
||||
id_mapping_details: list[ResourceIdMapping] = []
|
||||
|
||||
self._import_api_tools(
|
||||
request.package,
|
||||
target,
|
||||
options,
|
||||
report_items,
|
||||
id_mapping,
|
||||
id_mapping_details,
|
||||
self._source_api_provider_ids_by_name(request.package),
|
||||
session=session,
|
||||
)
|
||||
self._import_mcp_tools(
|
||||
request.package, target, options, report_items, id_mapping, id_mapping_details, session=session
|
||||
)
|
||||
self._preflight_dependency_only_mcp(request.package, target, report_items, session=session)
|
||||
workflow_tool_app_ids = self._workflow_tool_source_app_ids(request.package)
|
||||
imported_workflow_ids: set[str] = set()
|
||||
if workflow_tool_app_ids:
|
||||
self._import_workflows(
|
||||
request.package,
|
||||
target,
|
||||
options,
|
||||
report_items,
|
||||
id_mapping,
|
||||
id_mapping_details=id_mapping_details,
|
||||
imported_workflow_ids=imported_workflow_ids,
|
||||
only_app_ids=workflow_tool_app_ids,
|
||||
session=session,
|
||||
)
|
||||
self._import_workflow_tools(
|
||||
request.package, target, options, id_mapping, id_mapping_details, report_items, session=session
|
||||
)
|
||||
self._import_workflows(
|
||||
request.package,
|
||||
target,
|
||||
options,
|
||||
report_items,
|
||||
id_mapping,
|
||||
id_mapping_details=id_mapping_details,
|
||||
imported_workflow_ids=imported_workflow_ids,
|
||||
skip_app_ids=imported_workflow_ids,
|
||||
session=session,
|
||||
)
|
||||
return ImportResult(
|
||||
report_items=report_items,
|
||||
id_mapping=id_mapping,
|
||||
report_context=ReportContext(
|
||||
target_tenant=target.tenant_name,
|
||||
operator_email=target.operator_email,
|
||||
id_mapping_count=len(id_mapping),
|
||||
id_mappings=dict(id_mapping),
|
||||
id_mapping_details=id_mapping_details,
|
||||
),
|
||||
)
|
||||
|
||||
def _import_workflows(
|
||||
self,
|
||||
package: MigrationPackage,
|
||||
target: ImportTarget,
|
||||
options: ImportOptions,
|
||||
report_items: list[ResourceReportItem],
|
||||
id_mapping: dict[str, str],
|
||||
id_mapping_details: list[ResourceIdMapping],
|
||||
imported_workflow_ids: set[str] | None = None,
|
||||
only_app_ids: set[str] | None = None,
|
||||
skip_app_ids: set[str] | None = None,
|
||||
*,
|
||||
session: Session,
|
||||
) -> None:
|
||||
account = session.get(Account, target.operator_id)
|
||||
tenant = session.get(Tenant, target.tenant_id)
|
||||
if account is None:
|
||||
raise MigrationDataError(f"Operator account not found: {target.operator_id}")
|
||||
if tenant is None:
|
||||
raise MigrationDataError(f"Target tenant not found: {target.tenant_id}")
|
||||
account.set_current_tenant_with_session(tenant, session=session)
|
||||
|
||||
for workflow_data in package.workflows:
|
||||
app_id = self._optional_string(workflow_data.get("id"))
|
||||
if only_app_ids and app_id not in only_app_ids:
|
||||
continue
|
||||
if skip_app_ids and app_id in skip_app_ids:
|
||||
continue
|
||||
dsl_content = self._rewrite_workflow_dsl_provider_ids(
|
||||
self._required_string(workflow_data, "dsl", "workflow"),
|
||||
id_mapping,
|
||||
)
|
||||
existing_app = (
|
||||
self._find_existing_app(app_id, target.tenant_id, session=session)
|
||||
if options.id_strategy == IdStrategy.PRESERVE_ID
|
||||
else None
|
||||
)
|
||||
if existing_app is not None and options.conflict_strategy == ConflictStrategy.FAIL:
|
||||
raise MigrationDataError(f"App already exists and conflict_strategy=fail: {app_id}")
|
||||
if existing_app is not None and options.conflict_strategy == ConflictStrategy.SKIP:
|
||||
if app_id:
|
||||
self._record_id_mappings(
|
||||
id_mapping,
|
||||
id_mapping_details,
|
||||
ResourceType.WORKFLOW,
|
||||
workflow_data.get("name") if isinstance(workflow_data.get("name"), str) else None,
|
||||
{app_id},
|
||||
existing_app.id,
|
||||
)
|
||||
report_items.append(
|
||||
ResourceReportItem(ResourceType.WORKFLOW, str(app_id), workflow_data.get("name"), "skipped")
|
||||
)
|
||||
continue
|
||||
|
||||
imported_app_id = self._import_workflow_app(
|
||||
account=account,
|
||||
workflow_data=workflow_data,
|
||||
dsl_content=dsl_content,
|
||||
app_id=app_id,
|
||||
existing_app=existing_app,
|
||||
options=options,
|
||||
session=session,
|
||||
)
|
||||
if app_id:
|
||||
self._record_id_mappings(
|
||||
id_mapping,
|
||||
id_mapping_details,
|
||||
ResourceType.WORKFLOW,
|
||||
workflow_data.get("name") if isinstance(workflow_data.get("name"), str) else None,
|
||||
{app_id},
|
||||
imported_app_id,
|
||||
)
|
||||
if imported_workflow_ids is not None:
|
||||
imported_workflow_ids.add(app_id)
|
||||
if options.create_app_api_token_on_import:
|
||||
self._create_or_reuse_app_api_token(imported_app_id, target.tenant_id, session=session)
|
||||
report_items.append(
|
||||
ResourceReportItem(
|
||||
ResourceType.WORKFLOW,
|
||||
imported_app_id,
|
||||
workflow_data.get("name"),
|
||||
"updated" if existing_app is not None else "created",
|
||||
)
|
||||
)
|
||||
|
||||
def _workflow_tool_source_app_ids(self, package: MigrationPackage) -> set[str]:
|
||||
app_ids: set[str] = set()
|
||||
for workflow_tool_data in package.workflow_tools:
|
||||
app_id = self._optional_string(workflow_tool_data.get("app_id"))
|
||||
if app_id:
|
||||
app_ids.add(app_id)
|
||||
return app_ids
|
||||
|
||||
def _import_workflow_app(
|
||||
self,
|
||||
*,
|
||||
account: Account,
|
||||
workflow_data: dict[str, object],
|
||||
dsl_content: str,
|
||||
app_id: str | None,
|
||||
existing_app: App | None,
|
||||
options: ImportOptions,
|
||||
session: Session,
|
||||
) -> str:
|
||||
import_service = AppDslService(session)
|
||||
if existing_app is not None:
|
||||
existing_app_id = existing_app.id
|
||||
if dify_config.RBAC_ENABLED:
|
||||
session.commit()
|
||||
import_result = import_service.import_app(
|
||||
account=account,
|
||||
import_mode="yaml-content",
|
||||
yaml_content=dsl_content,
|
||||
app_id=existing_app_id,
|
||||
)
|
||||
else:
|
||||
import_app_id = app_id if self._should_preserve_source_app_id(options) else None
|
||||
import_result = import_service.import_app(
|
||||
account=account,
|
||||
import_mode="yaml-content",
|
||||
yaml_content=dsl_content,
|
||||
import_app_id=import_app_id,
|
||||
)
|
||||
if import_result.status not in {ImportStatus.COMPLETED, ImportStatus.COMPLETED_WITH_WARNINGS}:
|
||||
error = import_result.error or f"unexpected import status {import_result.status}"
|
||||
raise MigrationDataError(f"Workflow import failed: {error}")
|
||||
if import_result.app_id is None:
|
||||
raise MigrationDataError(f"Workflow import did not return an app id: {workflow_data.get('name')}")
|
||||
session.commit()
|
||||
return import_result.app_id
|
||||
|
||||
def _rewrite_workflow_dsl_provider_ids(self, dsl_content: str, id_mapping: dict[str, str]) -> str:
|
||||
if not id_mapping:
|
||||
return dsl_content
|
||||
parsed = yaml.safe_load(dsl_content) if dsl_content else {}
|
||||
if not isinstance(parsed, dict):
|
||||
return dsl_content
|
||||
for node in self._workflow_nodes(parsed):
|
||||
data = node.get("data") if isinstance(node, dict) else None
|
||||
if not isinstance(data, dict):
|
||||
continue
|
||||
self._rewrite_tool_config_provider_id(data, id_mapping)
|
||||
for tool_config in self._agent_tool_configs(data):
|
||||
self._rewrite_tool_config_provider_id(tool_config, id_mapping)
|
||||
return yaml.safe_dump(parsed, sort_keys=False, allow_unicode=True)
|
||||
|
||||
def _rewrite_tool_config_provider_id(self, tool_config: dict[str, Any], id_mapping: dict[str, str]) -> None:
|
||||
provider_id = self._optional_string(tool_config.get("provider_id"))
|
||||
if provider_id and provider_id in id_mapping:
|
||||
tool_config["provider_id"] = id_mapping[provider_id]
|
||||
|
||||
def _source_api_provider_ids_by_name(self, package: MigrationPackage) -> dict[str, set[str]]:
|
||||
provider_ids_by_name: dict[str, set[str]] = {}
|
||||
discovery_service = DependencyDiscoveryService()
|
||||
for workflow_data in package.workflows:
|
||||
dsl_content = self._optional_string(workflow_data.get("dsl"))
|
||||
if not dsl_content:
|
||||
continue
|
||||
parsed = yaml.safe_load(dsl_content) if dsl_content else {}
|
||||
if not isinstance(parsed, dict):
|
||||
continue
|
||||
for dependency in discovery_service.discover_from_dsl(parsed):
|
||||
if dependency.kind != DependencyKind.API_TOOL or not dependency.provider_name:
|
||||
continue
|
||||
provider_ids_by_name.setdefault(dependency.provider_name, set()).add(dependency.provider_id)
|
||||
return provider_ids_by_name
|
||||
|
||||
def _workflow_nodes(self, dsl: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
nodes: list[dict[str, Any]] = []
|
||||
graph = dsl.get("graph")
|
||||
if isinstance(graph, dict) and isinstance(graph.get("nodes"), list):
|
||||
nodes.extend(node for node in graph["nodes"] if isinstance(node, dict))
|
||||
workflow = dsl.get("workflow")
|
||||
workflow_graph = workflow.get("graph") if isinstance(workflow, dict) else None
|
||||
if isinstance(workflow_graph, dict) and isinstance(workflow_graph.get("nodes"), list):
|
||||
nodes.extend(node for node in workflow_graph["nodes"] if isinstance(node, dict))
|
||||
return nodes
|
||||
|
||||
def _agent_tool_configs(self, data: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
configs = data.get("tools")
|
||||
if isinstance(configs, list):
|
||||
return [config for config in configs if isinstance(config, dict)]
|
||||
agent_parameters = data.get("agent_parameters")
|
||||
if not isinstance(agent_parameters, dict):
|
||||
return []
|
||||
tools_parameter = agent_parameters.get("tools")
|
||||
if not isinstance(tools_parameter, dict):
|
||||
return []
|
||||
value = tools_parameter.get("value", [])
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
return [config for config in value if isinstance(config, dict)]
|
||||
|
||||
def _should_preserve_source_app_id(self, options: ImportOptions) -> bool:
|
||||
return options.id_strategy == IdStrategy.PRESERVE_ID
|
||||
|
||||
def _find_existing_app(self, app_id: str | None, tenant_id: str, *, session: Session) -> App | None:
|
||||
if not self._is_uuid_string(app_id):
|
||||
return None
|
||||
return session.scalar(sa.select(App).where(App.id == app_id, App.tenant_id == tenant_id))
|
||||
|
||||
def _create_or_reuse_app_api_token(self, app_id: str, tenant_id: str, *, session: Session) -> None:
|
||||
existing = session.scalar(
|
||||
sa.select(ApiToken).where(
|
||||
ApiToken.type == ApiTokenType.APP,
|
||||
ApiToken.app_id == app_id,
|
||||
ApiToken.tenant_id == tenant_id,
|
||||
)
|
||||
)
|
||||
if existing is not None:
|
||||
return
|
||||
api_token = ApiToken()
|
||||
api_token.app_id = app_id
|
||||
api_token.tenant_id = tenant_id
|
||||
api_token.token = ApiToken.generate_api_key("app", 24, session=session)
|
||||
api_token.type = ApiTokenType.APP
|
||||
session.add(api_token)
|
||||
session.commit()
|
||||
|
||||
def _import_api_tools(
|
||||
self,
|
||||
package: MigrationPackage,
|
||||
target: ImportTarget,
|
||||
options: ImportOptions,
|
||||
report_items: list[ResourceReportItem],
|
||||
id_mapping: dict[str, str],
|
||||
id_mapping_details: list[ResourceIdMapping],
|
||||
source_provider_ids_by_name: dict[str, set[str]],
|
||||
*,
|
||||
session: Session,
|
||||
) -> None:
|
||||
for tool_data in package.tools:
|
||||
provider_name = self._required_string(tool_data, "provider_name", "api_tool")
|
||||
schema = self._required_string(tool_data, "schema", "api_tool")
|
||||
existing = session.scalar(
|
||||
sa.select(ApiToolProvider).where(
|
||||
ApiToolProvider.tenant_id == target.tenant_id,
|
||||
ApiToolProvider.name == provider_name,
|
||||
)
|
||||
)
|
||||
if existing is not None and options.conflict_strategy == ConflictStrategy.FAIL:
|
||||
raise MigrationDataError(f"API tool already exists and conflict_strategy=fail: {provider_name}")
|
||||
if existing is not None and options.conflict_strategy == ConflictStrategy.SKIP:
|
||||
self._record_id_mappings(
|
||||
id_mapping,
|
||||
id_mapping_details,
|
||||
ResourceType.API_TOOL,
|
||||
provider_name,
|
||||
self._api_tool_source_ids(provider_name, tool_data, source_provider_ids_by_name),
|
||||
existing.id,
|
||||
)
|
||||
report_items.append(ResourceReportItem(ResourceType.API_TOOL, provider_name, provider_name, "skipped"))
|
||||
continue
|
||||
|
||||
schema_info = ApiToolManageService.parser_api_schema(schema=schema)
|
||||
schema_type = cast(ApiProviderSchemaType, schema_info["schema_type"])
|
||||
credentials = (
|
||||
cast(dict[str, Any], tool_data.get("credentials"))
|
||||
if isinstance(tool_data.get("credentials"), dict)
|
||||
else {}
|
||||
)
|
||||
credentials = credentials or {"auth_type": "none"}
|
||||
raw_icon = tool_data.get("icon")
|
||||
icon = (
|
||||
cast(dict[str, Any], raw_icon)
|
||||
if isinstance(raw_icon, dict)
|
||||
else {"content": "tool", "background": "#FEF7C3"}
|
||||
)
|
||||
raw_labels = tool_data.get("labels")
|
||||
labels = [str(label) for label in raw_labels] if isinstance(raw_labels, list) else []
|
||||
if existing is not None:
|
||||
ApiToolManageService.update_api_tool_provider(
|
||||
user_id=target.operator_id,
|
||||
tenant_id=target.tenant_id,
|
||||
provider_name=provider_name,
|
||||
original_provider=existing.name,
|
||||
_schema_type=schema_type,
|
||||
schema=schema,
|
||||
privacy_policy=self._optional_string(tool_data.get("privacy_policy")) or "",
|
||||
credentials=credentials,
|
||||
custom_disclaimer=self._optional_string(tool_data.get("custom_disclaimer")) or "",
|
||||
labels=labels,
|
||||
icon=icon,
|
||||
)
|
||||
status = "updated"
|
||||
else:
|
||||
ApiToolManageService.create_api_tool_provider(
|
||||
user_id=target.operator_id,
|
||||
tenant_id=target.tenant_id,
|
||||
provider_name=provider_name,
|
||||
schema_type=schema_type,
|
||||
schema=schema,
|
||||
privacy_policy=self._optional_string(tool_data.get("privacy_policy")) or "",
|
||||
credentials=credentials,
|
||||
custom_disclaimer=self._optional_string(tool_data.get("custom_disclaimer")) or "",
|
||||
labels=labels,
|
||||
icon=icon,
|
||||
)
|
||||
status = "created"
|
||||
target_provider = self._find_api_tool_provider(target.tenant_id, provider_name, session=session)
|
||||
if target_provider is not None:
|
||||
self._record_id_mappings(
|
||||
id_mapping,
|
||||
id_mapping_details,
|
||||
ResourceType.API_TOOL,
|
||||
provider_name,
|
||||
self._api_tool_source_ids(provider_name, tool_data, source_provider_ids_by_name),
|
||||
target_provider.id,
|
||||
)
|
||||
report_items.append(ResourceReportItem(ResourceType.API_TOOL, provider_name, provider_name, status))
|
||||
|
||||
def _find_api_tool_provider(
|
||||
self, tenant_id: str, provider_name: str, *, session: Session
|
||||
) -> ApiToolProvider | None:
|
||||
return session.scalar(
|
||||
sa.select(ApiToolProvider).where(
|
||||
ApiToolProvider.tenant_id == tenant_id,
|
||||
ApiToolProvider.name == provider_name,
|
||||
)
|
||||
)
|
||||
|
||||
def _api_tool_source_ids(
|
||||
self,
|
||||
provider_name: str,
|
||||
tool_data: dict[str, Any],
|
||||
source_provider_ids_by_name: dict[str, set[str]],
|
||||
) -> set[str]:
|
||||
source_ids = set(source_provider_ids_by_name.get(provider_name, set()))
|
||||
source_id = self._optional_string(tool_data.get("id"))
|
||||
if source_id:
|
||||
source_ids.add(source_id)
|
||||
return source_ids
|
||||
|
||||
def _record_id_mappings(
|
||||
self,
|
||||
id_mapping: dict[str, str],
|
||||
id_mapping_details: list[ResourceIdMapping],
|
||||
resource_type: ResourceType,
|
||||
name: str | None,
|
||||
source_ids: Iterable[str],
|
||||
target_id: str,
|
||||
) -> None:
|
||||
for source_id in source_ids:
|
||||
id_mapping[source_id] = target_id
|
||||
id_mapping_details[:] = [item for item in id_mapping_details if item.source_id != source_id]
|
||||
id_mapping_details.append(ResourceIdMapping(resource_type, name, source_id, target_id))
|
||||
|
||||
def _import_workflow_tools(
|
||||
self,
|
||||
package: MigrationPackage,
|
||||
target: ImportTarget,
|
||||
options: ImportOptions,
|
||||
id_mapping: dict[str, str],
|
||||
id_mapping_details: list[ResourceIdMapping],
|
||||
report_items: list[ResourceReportItem],
|
||||
*,
|
||||
session: Session,
|
||||
) -> None:
|
||||
if not package.workflow_tools:
|
||||
return
|
||||
account = session.get(Account, target.operator_id)
|
||||
if account is None:
|
||||
raise MigrationDataError(f"Operator account not found: {target.operator_id}")
|
||||
for workflow_tool_data in package.workflow_tools:
|
||||
app_id = self._optional_string(workflow_tool_data.get("app_id"))
|
||||
resolved_app_id = id_mapping.get(app_id or "", app_id)
|
||||
if (
|
||||
not resolved_app_id
|
||||
or self._find_existing_app(resolved_app_id, target.tenant_id, session=session) is None
|
||||
):
|
||||
report_items.append(
|
||||
ResourceReportItem(
|
||||
ResourceType.WORKFLOW_TOOL,
|
||||
str(workflow_tool_data.get("id", workflow_tool_data.get("name", ""))),
|
||||
self._optional_string(workflow_tool_data.get("name")),
|
||||
"unresolved",
|
||||
"Referenced workflow app was not found in the target tenant; workflow tool was skipped.",
|
||||
)
|
||||
)
|
||||
continue
|
||||
try:
|
||||
self._ensure_workflow_app_is_published(target, account, resolved_app_id, session=session)
|
||||
except Exception as exc:
|
||||
report_items.append(
|
||||
ResourceReportItem(
|
||||
ResourceType.WORKFLOW_TOOL,
|
||||
str(workflow_tool_data.get("id", workflow_tool_data.get("name", ""))),
|
||||
self._optional_string(workflow_tool_data.get("name")),
|
||||
"unresolved",
|
||||
f"Referenced workflow app could not be published: {exc}",
|
||||
)
|
||||
)
|
||||
continue
|
||||
workflow_tool_id = self._optional_string(workflow_tool_data.get("id"))
|
||||
tool_name = self._required_string(workflow_tool_data, "name", "workflow_tool")
|
||||
lookup_workflow_tool_id = workflow_tool_id if options.id_strategy == IdStrategy.PRESERVE_ID else None
|
||||
existing = self._find_existing_workflow_tool(
|
||||
target.tenant_id, lookup_workflow_tool_id, tool_name, resolved_app_id, session=session
|
||||
)
|
||||
if existing is not None and options.conflict_strategy == ConflictStrategy.FAIL:
|
||||
raise MigrationDataError(f"Workflow tool already exists and conflict_strategy=fail: {tool_name}")
|
||||
if existing is not None and options.conflict_strategy == ConflictStrategy.SKIP:
|
||||
if workflow_tool_id:
|
||||
self._record_id_mappings(
|
||||
id_mapping,
|
||||
id_mapping_details,
|
||||
ResourceType.WORKFLOW_TOOL,
|
||||
tool_name,
|
||||
{workflow_tool_id},
|
||||
existing.id,
|
||||
)
|
||||
report_items.append(ResourceReportItem(ResourceType.WORKFLOW_TOOL, existing.id, tool_name, "skipped"))
|
||||
continue
|
||||
raw_icon = workflow_tool_data.get("icon")
|
||||
icon = (
|
||||
cast(dict[str, Any], raw_icon)
|
||||
if isinstance(raw_icon, dict)
|
||||
else {"content": "🤖", "background": "#FFEAD5"}
|
||||
)
|
||||
raw_parameters = workflow_tool_data.get("parameters")
|
||||
parameters = [
|
||||
parameter
|
||||
if isinstance(parameter, WorkflowToolParameterConfiguration)
|
||||
else WorkflowToolParameterConfiguration(**parameter)
|
||||
for parameter in (raw_parameters if isinstance(raw_parameters, list) else [])
|
||||
if isinstance(parameter, dict | WorkflowToolParameterConfiguration)
|
||||
]
|
||||
raw_labels = workflow_tool_data.get("labels")
|
||||
labels = [str(label) for label in raw_labels] if isinstance(raw_labels, list) else []
|
||||
label = self._optional_string(workflow_tool_data.get("label")) or tool_name
|
||||
description = self._optional_string(workflow_tool_data.get("description")) or ""
|
||||
privacy_policy = self._optional_string(workflow_tool_data.get("privacy_policy")) or ""
|
||||
if existing is not None:
|
||||
WorkflowToolManageService.update_workflow_tool(
|
||||
user_id=account.id,
|
||||
tenant_id=target.tenant_id,
|
||||
workflow_tool_id=existing.id,
|
||||
name=tool_name,
|
||||
label=label,
|
||||
icon=icon,
|
||||
description=description,
|
||||
parameters=parameters,
|
||||
privacy_policy=privacy_policy,
|
||||
labels=labels,
|
||||
)
|
||||
status = "updated"
|
||||
identifier = existing.id
|
||||
else:
|
||||
import_id = workflow_tool_id if options.id_strategy == IdStrategy.PRESERVE_ID else ""
|
||||
WorkflowToolManageService.create_workflow_tool(
|
||||
user_id=account.id,
|
||||
tenant_id=target.tenant_id,
|
||||
workflow_app_id=resolved_app_id,
|
||||
name=tool_name,
|
||||
label=label,
|
||||
icon=icon,
|
||||
description=description,
|
||||
parameters=parameters,
|
||||
privacy_policy=privacy_policy,
|
||||
labels=labels,
|
||||
import_id=import_id or "",
|
||||
)
|
||||
status = "created"
|
||||
target_provider = self._find_existing_workflow_tool(
|
||||
target.tenant_id, import_id or None, tool_name, resolved_app_id, session=session
|
||||
)
|
||||
if target_provider is None:
|
||||
raise MigrationDataError(f"Workflow tool was not created: {tool_name}")
|
||||
identifier = target_provider.id
|
||||
if workflow_tool_id:
|
||||
self._record_id_mappings(
|
||||
id_mapping,
|
||||
id_mapping_details,
|
||||
ResourceType.WORKFLOW_TOOL,
|
||||
tool_name,
|
||||
{workflow_tool_id},
|
||||
identifier,
|
||||
)
|
||||
report_items.append(ResourceReportItem(ResourceType.WORKFLOW_TOOL, identifier, tool_name, status))
|
||||
|
||||
def _ensure_workflow_app_is_published(
|
||||
self, target: ImportTarget, account: Account, app_id: str, *, session: Session
|
||||
) -> None:
|
||||
app = self._find_existing_app(app_id, target.tenant_id, session=session)
|
||||
if app is None:
|
||||
raise MigrationDataError(f"Referenced workflow app was not found in target tenant: {app_id}")
|
||||
if app.workflow_id:
|
||||
return
|
||||
workflow_service = WorkflowService()
|
||||
with sessionmaker(db.engine).begin() as session:
|
||||
app_in_session = session.get(App, app_id)
|
||||
account_in_session = session.get(Account, account.id)
|
||||
if app_in_session is None:
|
||||
raise MigrationDataError(f"Referenced workflow app was not found in target tenant: {app_id}")
|
||||
if account_in_session is None:
|
||||
raise MigrationDataError(f"Operator account not found: {account.id}")
|
||||
workflow = workflow_service.publish_workflow(
|
||||
session=session,
|
||||
app_model=app_in_session,
|
||||
account=account_in_session,
|
||||
marked_name="Migration import",
|
||||
marked_comment="Published automatically for workflow tool import.",
|
||||
)
|
||||
app_in_session.workflow_id = workflow.id
|
||||
app_in_session.updated_by = account.id
|
||||
app_in_session.updated_at = naive_utc_now()
|
||||
|
||||
def _import_mcp_tools(
|
||||
self,
|
||||
package: MigrationPackage,
|
||||
target: ImportTarget,
|
||||
options: ImportOptions,
|
||||
report_items: list[ResourceReportItem],
|
||||
id_mapping: dict[str, str],
|
||||
id_mapping_details: list[ResourceIdMapping],
|
||||
*,
|
||||
session: Session,
|
||||
) -> None:
|
||||
for mcp_data in package.mcp_tools:
|
||||
name = self._required_string(mcp_data, "name", "mcp_tool")
|
||||
server_identifier = self._required_string(mcp_data, "server_identifier", "mcp_tool")
|
||||
provider_id = self._optional_string(mcp_data.get("id"))
|
||||
lookup_provider_id = provider_id if options.id_strategy == IdStrategy.PRESERVE_ID else None
|
||||
existing = self._find_existing_mcp_tool(
|
||||
target.tenant_id, lookup_provider_id, server_identifier, session=session
|
||||
)
|
||||
if existing is not None and options.conflict_strategy == ConflictStrategy.FAIL:
|
||||
raise MigrationDataError(f"MCP tool already exists and conflict_strategy=fail: {name}")
|
||||
if existing is not None and options.conflict_strategy == ConflictStrategy.SKIP:
|
||||
if provider_id:
|
||||
self._record_id_mappings(
|
||||
id_mapping,
|
||||
id_mapping_details,
|
||||
ResourceType.MCP_TOOL,
|
||||
name,
|
||||
{provider_id},
|
||||
existing.id,
|
||||
)
|
||||
report_items.append(ResourceReportItem(ResourceType.MCP_TOOL, existing.id, name, "skipped"))
|
||||
continue
|
||||
|
||||
service = MCPToolManageService(session=session)
|
||||
configuration = MCPConfiguration.model_validate(mcp_data.get("configuration") or {})
|
||||
authentication = (
|
||||
MCPAuthentication.model_validate(mcp_data["authentication"]) if mcp_data.get("authentication") else None
|
||||
)
|
||||
if existing is not None:
|
||||
service.update_provider(
|
||||
tenant_id=target.tenant_id,
|
||||
provider_id=existing.id,
|
||||
server_url=self._required_string(mcp_data, "server_url", "mcp_tool"),
|
||||
name=name,
|
||||
icon=self._optional_string(mcp_data.get("icon")) or "",
|
||||
icon_type=self._optional_string(mcp_data.get("icon_type")) or "emoji",
|
||||
icon_background=self._optional_string(mcp_data.get("icon_background")) or "",
|
||||
server_identifier=server_identifier,
|
||||
headers=mcp_data.get("headers") if isinstance(mcp_data.get("headers"), dict) else {},
|
||||
configuration=configuration,
|
||||
authentication=authentication,
|
||||
# Re-import must not silently reset forwarding: preserve the
|
||||
# stored mode (update_provider now defaults to OFF when omitted).
|
||||
identity_mode=IdentityMode(existing.identity_mode),
|
||||
)
|
||||
session.commit()
|
||||
status = "updated"
|
||||
identifier = existing.id
|
||||
provider = existing
|
||||
else:
|
||||
service.create_provider(
|
||||
tenant_id=target.tenant_id,
|
||||
user_id=target.operator_id,
|
||||
server_url=self._required_string(mcp_data, "server_url", "mcp_tool"),
|
||||
name=name,
|
||||
icon=self._optional_string(mcp_data.get("icon")) or "",
|
||||
icon_type=self._optional_string(mcp_data.get("icon_type")) or "emoji",
|
||||
icon_background=self._optional_string(mcp_data.get("icon_background")) or "",
|
||||
server_identifier=server_identifier,
|
||||
headers=mcp_data.get("headers") if isinstance(mcp_data.get("headers"), dict) else {},
|
||||
configuration=configuration,
|
||||
authentication=authentication,
|
||||
)
|
||||
created_provider = self._find_existing_mcp_tool(
|
||||
target.tenant_id, lookup_provider_id, server_identifier, session=session
|
||||
)
|
||||
if created_provider is None:
|
||||
raise MigrationDataError(f"MCP provider was not created: {name}")
|
||||
status = "created"
|
||||
provider = created_provider
|
||||
identifier = provider.id
|
||||
self._restore_mcp_provider_tools(provider, mcp_data)
|
||||
session.commit()
|
||||
if provider_id:
|
||||
self._record_id_mappings(
|
||||
id_mapping,
|
||||
id_mapping_details,
|
||||
ResourceType.MCP_TOOL,
|
||||
name,
|
||||
{provider_id},
|
||||
identifier,
|
||||
)
|
||||
report_items.append(ResourceReportItem(ResourceType.MCP_TOOL, identifier, name, status))
|
||||
|
||||
def _restore_mcp_provider_tools(self, provider: MCPToolProvider, mcp_data: dict[str, object]) -> None:
|
||||
tools = mcp_data.get("tools")
|
||||
if not isinstance(tools, list):
|
||||
return
|
||||
provider.tools = json.dumps(tools)
|
||||
provider.authed = True
|
||||
|
||||
def _find_existing_mcp_tool(
|
||||
self,
|
||||
tenant_id: str,
|
||||
provider_id: str | None,
|
||||
server_identifier: str,
|
||||
*,
|
||||
session: Session,
|
||||
) -> MCPToolProvider | None:
|
||||
predicates = [MCPToolProvider.server_identifier == server_identifier]
|
||||
if self._is_uuid_string(provider_id):
|
||||
predicates.append(MCPToolProvider.id == provider_id)
|
||||
return session.scalar(
|
||||
sa.select(MCPToolProvider).where(MCPToolProvider.tenant_id == tenant_id, or_(*predicates)).limit(1)
|
||||
)
|
||||
|
||||
def _is_uuid_string(self, value: str | None) -> bool:
|
||||
if not value:
|
||||
return False
|
||||
try:
|
||||
UUID(value)
|
||||
except ValueError:
|
||||
return False
|
||||
return True
|
||||
|
||||
def _find_existing_workflow_tool(
|
||||
self,
|
||||
tenant_id: str,
|
||||
workflow_tool_id: str | None,
|
||||
tool_name: str,
|
||||
app_id: str,
|
||||
*,
|
||||
session: Session,
|
||||
) -> WorkflowToolProvider | None:
|
||||
predicates = [WorkflowToolProvider.name == tool_name, WorkflowToolProvider.app_id == app_id]
|
||||
if self._is_uuid_string(workflow_tool_id):
|
||||
predicates.append(WorkflowToolProvider.id == workflow_tool_id)
|
||||
return session.scalar(
|
||||
sa.select(WorkflowToolProvider)
|
||||
.where(WorkflowToolProvider.tenant_id == tenant_id, or_(*predicates))
|
||||
.limit(1)
|
||||
)
|
||||
|
||||
def _preflight_dependency_only_mcp(
|
||||
self,
|
||||
package: MigrationPackage,
|
||||
target: ImportTarget,
|
||||
report_items: list[ResourceReportItem],
|
||||
*,
|
||||
session: Session,
|
||||
) -> None:
|
||||
for dependency in package.dependencies:
|
||||
if dependency.get("kind") != DependencyKind.MCP_TOOL.value:
|
||||
continue
|
||||
provider_id = str(dependency.get("provider_id", dependency.get("id", "")))
|
||||
provider_name = self._optional_string(dependency.get("provider_name") or dependency.get("name"))
|
||||
existing = self._find_dependency_only_mcp_provider(
|
||||
target.tenant_id, provider_id, provider_name, session=session
|
||||
)
|
||||
report_name = f"mcp_tool {provider_name or getattr(existing, 'name', None) or provider_id}"
|
||||
if existing is not None:
|
||||
report_items.append(
|
||||
ResourceReportItem(
|
||||
ResourceType.DEPENDENCY,
|
||||
provider_id,
|
||||
report_name,
|
||||
"available",
|
||||
"MCP provider exists in target tenant.",
|
||||
)
|
||||
)
|
||||
continue
|
||||
reference_summary = self._dependency_only_mcp_reference_summary(package, provider_id, provider_name)
|
||||
message = "missing in target tenant"
|
||||
if reference_summary:
|
||||
message = f"{message}; referenced by {reference_summary}"
|
||||
message = f"{message}; configure it manually before running the workflow."
|
||||
report_items.append(
|
||||
ResourceReportItem(
|
||||
ResourceType.DEPENDENCY,
|
||||
provider_id,
|
||||
report_name,
|
||||
"skipped",
|
||||
message,
|
||||
)
|
||||
)
|
||||
|
||||
def _find_dependency_only_mcp_provider(
|
||||
self,
|
||||
tenant_id: str,
|
||||
provider_id: str,
|
||||
provider_name: str | None,
|
||||
*,
|
||||
session: Session,
|
||||
) -> MCPToolProvider | None:
|
||||
predicates = [MCPToolProvider.server_identifier == provider_id]
|
||||
if self._is_uuid_string(provider_id):
|
||||
predicates.append(MCPToolProvider.id == provider_id)
|
||||
return session.scalar(
|
||||
sa.select(MCPToolProvider).where(MCPToolProvider.tenant_id == tenant_id, or_(*predicates)).limit(1)
|
||||
)
|
||||
|
||||
def _dependency_only_mcp_reference_summary(
|
||||
self, package: MigrationPackage, provider_id: str, provider_name: str | None
|
||||
) -> str:
|
||||
references = self._dependency_only_mcp_references(package, provider_id, provider_name)
|
||||
return "; ".join(references)
|
||||
|
||||
def _dependency_only_mcp_references(
|
||||
self, package: MigrationPackage, provider_id: str, provider_name: str | None
|
||||
) -> list[str]:
|
||||
references: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for workflow_data in package.workflows:
|
||||
workflow_name = self._optional_string(workflow_data.get("name"))
|
||||
workflow_id = self._optional_string(workflow_data.get("id"))
|
||||
workflow_label = workflow_name or workflow_id or "unknown workflow"
|
||||
dsl_content = self._optional_string(workflow_data.get("dsl"))
|
||||
if not dsl_content:
|
||||
continue
|
||||
parsed = yaml.safe_load(dsl_content) if dsl_content else {}
|
||||
if not isinstance(parsed, dict):
|
||||
continue
|
||||
for node in self._workflow_nodes(parsed):
|
||||
data = node.get("data") if isinstance(node, dict) else None
|
||||
if not isinstance(data, dict):
|
||||
continue
|
||||
for tool_config in [data, *self._agent_tool_configs(data)]:
|
||||
if not self._is_mcp_dependency_reference(tool_config, provider_id, provider_name):
|
||||
continue
|
||||
tool_label = self._mcp_tool_reference_label(node, tool_config)
|
||||
reference = f"{workflow_label} / {tool_label}"
|
||||
if reference not in seen:
|
||||
seen.add(reference)
|
||||
references.append(reference)
|
||||
return references
|
||||
|
||||
def _is_mcp_dependency_reference(
|
||||
self, tool_config: dict[str, Any], provider_id: str, provider_name: str | None
|
||||
) -> bool:
|
||||
provider_type = str(tool_config.get("provider_type") or tool_config.get("type") or "").lower()
|
||||
if provider_type != "mcp":
|
||||
return False
|
||||
config_provider_id = self._optional_string(
|
||||
tool_config.get("provider_id") or tool_config.get("provider_name") or tool_config.get("provider")
|
||||
)
|
||||
if config_provider_id == provider_id:
|
||||
return True
|
||||
return bool(provider_name and config_provider_id == provider_name)
|
||||
|
||||
def _mcp_tool_reference_label(self, node: dict[str, Any], tool_config: dict[str, Any]) -> str:
|
||||
for key in ("tool_name", "tool", "name"):
|
||||
value = self._optional_string(tool_config.get(key))
|
||||
if value:
|
||||
return value
|
||||
node_id = self._optional_string(node.get("id"))
|
||||
return node_id or "unknown tool"
|
||||
|
||||
def _required_string(self, value: dict[str, object], field_name: str, resource_name: str) -> str:
|
||||
field_value = value.get(field_name)
|
||||
if not isinstance(field_value, str) or not field_value:
|
||||
raise MigrationDataError(f"Missing required {resource_name} field: {field_name}")
|
||||
return field_value
|
||||
|
||||
def _optional_string(self, value: object) -> str | None:
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
return None
|
||||
@@ -0,0 +1,71 @@
|
||||
"""JSON persistence for versioned cross-environment migration packages.
|
||||
|
||||
The package service validates file shape and serializes only structured package
|
||||
entities. It does not perform CLI rendering or database access, keeping it safe
|
||||
to reuse from Click adapters, tests, and future import/export services.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import asdict
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from services.data_migration.entities import (
|
||||
ImportOptions,
|
||||
MigrationDataError,
|
||||
MigrationMetadata,
|
||||
MigrationPackage,
|
||||
SourceTenant,
|
||||
TargetTenantSelector,
|
||||
)
|
||||
|
||||
PACKAGE_VERSION = "1"
|
||||
|
||||
|
||||
class MigrationPackageService:
|
||||
def load_package(self, path: str | Path) -> MigrationPackage:
|
||||
package_path = Path(path)
|
||||
with package_path.open(encoding="utf-8") as file:
|
||||
raw = json.load(file)
|
||||
if not isinstance(raw, dict):
|
||||
raise MigrationDataError("Migration package JSON must be an object.")
|
||||
package = MigrationPackage.from_mapping(raw)
|
||||
if package.metadata.version != PACKAGE_VERSION:
|
||||
raise MigrationDataError(f"Unsupported migration package version: {package.metadata.version}")
|
||||
return package
|
||||
|
||||
def save_package(self, package: MigrationPackage, path: str | Path, *, overwrite: bool) -> None:
|
||||
package_path = Path(path)
|
||||
if package_path.exists() and not overwrite:
|
||||
raise MigrationDataError(f"Output file already exists: {package_path}")
|
||||
package_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with package_path.open("w", encoding="utf-8") as file:
|
||||
json.dump(self.to_mapping(package), file, ensure_ascii=False, indent=2)
|
||||
file.write("\n")
|
||||
|
||||
def build_empty_package(
|
||||
self,
|
||||
*,
|
||||
source_tenant_id: str,
|
||||
source_tenant_name: str,
|
||||
include_secrets: bool,
|
||||
import_options: ImportOptions | None = None,
|
||||
target_tenant: TargetTenantSelector | None = None,
|
||||
) -> MigrationPackage:
|
||||
return MigrationPackage(
|
||||
metadata=MigrationMetadata(
|
||||
version=PACKAGE_VERSION,
|
||||
source_scope="single",
|
||||
source_tenants=[SourceTenant(id=source_tenant_id, name=source_tenant_name)],
|
||||
target_tenant=target_tenant,
|
||||
created_at=datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z"),
|
||||
include_secrets=include_secrets,
|
||||
import_options=import_options or ImportOptions(),
|
||||
)
|
||||
)
|
||||
|
||||
def to_mapping(self, package: MigrationPackage) -> dict[str, Any]:
|
||||
return asdict(package)
|
||||
@@ -0,0 +1,76 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter
|
||||
|
||||
from services.data_migration.entities import ReportContext, ResourceIdMapping, ResourceReportItem
|
||||
|
||||
|
||||
class MigrationReportService:
|
||||
"""Render structured migration resource results into CLI-friendly summary lines."""
|
||||
|
||||
def render(self, items: list[ResourceReportItem], *, context: ReportContext | None = None) -> list[str]:
|
||||
counts = Counter((item.resource_type.value, item.status) for item in items)
|
||||
lines = self._render_context(context)
|
||||
lines.extend(
|
||||
[f"{resource_type} {status}: {count}" for (resource_type, status), count in sorted(counts.items())]
|
||||
)
|
||||
actionable_items = [
|
||||
item for item in items if item.status in {"dependency-only", "skipped", "unresolved"} and item.message
|
||||
]
|
||||
for item in actionable_items:
|
||||
lines.append(self._render_actionable_detail(item))
|
||||
return lines
|
||||
|
||||
def _render_context(self, context: ReportContext | None) -> list[str]:
|
||||
if context is None:
|
||||
return []
|
||||
lines: list[str] = []
|
||||
if context.output_path:
|
||||
lines.append(f"output: {context.output_path}")
|
||||
if context.source_scope:
|
||||
lines.append(f"source scope: {context.source_scope}")
|
||||
if context.selected_app_count is not None:
|
||||
lines.append(f"selected apps: {context.selected_app_count}")
|
||||
if context.include_secrets is not None:
|
||||
lines.append(f"include secrets: {str(context.include_secrets).lower()}")
|
||||
if context.target_tenant:
|
||||
lines.append(f"target tenant: {context.target_tenant}")
|
||||
if context.operator_email:
|
||||
lines.append(f"operator: {context.operator_email}")
|
||||
if context.app_api_tokens_created or context.app_api_tokens_reused:
|
||||
lines.append(
|
||||
f"app api tokens: {context.app_api_tokens_created} created, {context.app_api_tokens_reused} reused"
|
||||
)
|
||||
if context.id_mappings:
|
||||
lines.append(f"resource references resolved: {len(context.id_mappings)}")
|
||||
if context.id_mapping_details:
|
||||
lines.extend(
|
||||
self._render_id_mapping_detail(item)
|
||||
for item in sorted(
|
||||
context.id_mapping_details,
|
||||
key=lambda item: (item.resource_type.value, item.name or "", item.source_id),
|
||||
)
|
||||
)
|
||||
else:
|
||||
lines.extend(
|
||||
f"- {source_id} -> {target_id}" for source_id, target_id in sorted(context.id_mappings.items())
|
||||
)
|
||||
elif context.id_mapping_count:
|
||||
lines.append(f"resource references resolved: {context.id_mapping_count}")
|
||||
return lines
|
||||
|
||||
def _render_id_mapping_detail(self, item: ResourceIdMapping) -> str:
|
||||
label = item.resource_type.value
|
||||
if item.name:
|
||||
label = f"{label} {item.name}"
|
||||
return f"- {label}: {item.source_id} -> {item.target_id}"
|
||||
|
||||
def _render_actionable_detail(self, item: ResourceReportItem) -> str:
|
||||
if item.resource_type.value == "dependency" and item.name and self._has_dependency_type_prefix(item.name):
|
||||
if item.identifier and item.identifier not in item.name:
|
||||
return f"dependency {item.name}: {item.identifier}: {item.message}"
|
||||
return f"dependency {item.name}: {item.message}"
|
||||
return f"{item.resource_type.value} {item.identifier}: {item.message}"
|
||||
|
||||
def _has_dependency_type_prefix(self, name: str) -> bool:
|
||||
return name.startswith(("workflow ", "api_tool ", "workflow_tool ", "mcp_tool ", "builtin_or_plugin_tool "))
|
||||
Reference in new issue
Block a user