chore: 纳入本地 Dify 1.19.0 源码及品牌与免登改动

演示目录只保留可重建的源码。运行密钥、数据卷、依赖和带口令的本地配置不进入仓库。

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
wuzilongandCursor committed 2026-11-18 13:59:25 +08:00
1 parent 60158935f4
commit 6519682885
15831 files changed
+3063428

No files matched your search

@@ -0,0 +1,17 @@
from services.data_migration.entities import (
ConflictStrategy,
ExportSelection,
IdStrategy,
ImportOptions,
MigrationDataError,
MigrationPackage,
)
__all__ = [
"ConflictStrategy",
"ExportSelection",
"IdStrategy",
"ImportOptions",
"MigrationDataError",
"MigrationPackage",
]
@@ -0,0 +1,92 @@
from __future__ import annotations
from dataclasses import dataclass
from typing import Any
from services.data_migration.entities import DependencyKind
@dataclass(frozen=True)
class DiscoveredDependency:
kind: DependencyKind
provider_id: str
provider_name: str | None = None
source: str | None = None
class DependencyDiscoveryService:
def discover_from_dsl(self, dsl: dict[str, Any]) -> list[DiscoveredDependency]:
seen: set[tuple[DependencyKind, str]] = set()
result: list[DiscoveredDependency] = []
for node in self._nodes_from_dsl(dsl):
data = node.get("data", {}) if isinstance(node, dict) else {}
for dependency in self._dependencies_from_node(data):
key = (dependency.kind, dependency.provider_id)
if dependency.provider_id and key not in seen:
seen.add(key)
result.append(dependency)
return result
def _nodes_from_dsl(self, dsl: dict[str, Any]) -> list[dict[str, Any]]:
nodes: list[dict[str, Any]] = []
graph = dsl.get("graph") if isinstance(dsl, dict) else None
if isinstance(graph, dict) and isinstance(graph.get("nodes"), list):
nodes.extend(node for node in graph["nodes"] if isinstance(node, dict))
workflow = dsl.get("workflow") if isinstance(dsl, dict) else None
workflow_graph = workflow.get("graph") if isinstance(workflow, dict) else None
if isinstance(workflow_graph, dict) and isinstance(workflow_graph.get("nodes"), list):
nodes.extend(node for node in workflow_graph["nodes"] if isinstance(node, dict))
return nodes
def _dependencies_from_node(self, data: dict[str, Any]) -> list[DiscoveredDependency]:
dependencies: list[DiscoveredDependency] = []
node_type = data.get("type")
if node_type == "tool":
dependency = self._from_tool_config(data, source="tool_node")
if dependency:
dependencies.append(dependency)
if node_type == "agent":
for tool_config in self._agent_tool_configs(data):
if isinstance(tool_config, dict):
dependency = self._from_tool_config(tool_config, source="agent_node")
if dependency:
dependencies.append(dependency)
return dependencies
def _agent_tool_configs(self, data: dict[str, Any]) -> list[dict[str, Any]]:
configs = data.get("tools")
if isinstance(configs, list):
return [config for config in configs if isinstance(config, dict)]
agent_parameters = data.get("agent_parameters")
if not isinstance(agent_parameters, dict):
return []
tools_parameter = agent_parameters.get("tools")
if not isinstance(tools_parameter, dict):
return []
value = tools_parameter.get("value", [])
if not isinstance(value, list):
return []
return [config for config in value if isinstance(config, dict)]
def _from_tool_config(self, config: dict[str, Any], *, source: str) -> DiscoveredDependency | None:
provider_id = config.get("provider_id") or config.get("provider_name") or config.get("provider")
if not provider_id:
return None
provider_type = str(config.get("provider_type") or config.get("type") or "")
kind = self._kind_from_provider_type(provider_type)
return DiscoveredDependency(
kind=kind,
provider_id=str(provider_id),
provider_name=config.get("provider_name"),
source=source,
)
def _kind_from_provider_type(self, provider_type: str) -> DependencyKind:
normalized = provider_type.lower()
if normalized in {"api", "custom", "api_tool"}:
return DependencyKind.API_TOOL
if normalized in {"workflow", "workflow_tool"}:
return DependencyKind.WORKFLOW_TOOL
if normalized == "mcp":
return DependencyKind.MCP_TOOL
return DependencyKind.BUILTIN_OR_PLUGIN_TOOL
@@ -0,0 +1,241 @@
"""Typed entities for versioned cross-environment migration packages.
This module is intentionally side-effect free. It owns only value objects and
validation for migration package/config shapes; command output and database I/O
belong in adapter and service modules built on top of these entities.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from enum import StrEnum
from typing import Any, Literal, TypedDict
class MigrationDataError(ValueError):
"""Raised when migration config or package data is invalid."""
class IdStrategy(StrEnum):
PRESERVE_ID = "preserve-id"
GENERATE_NEW_ID = "generate-new-id"
class ConflictStrategy(StrEnum):
FAIL = "fail"
SKIP = "skip"
UPDATE = "update"
class ResourceType(StrEnum):
WORKFLOW = "workflow"
API_TOOL = "api_tool"
WORKFLOW_TOOL = "workflow_tool"
MCP_TOOL = "mcp_tool"
DEPENDENCY = "dependency"
class DependencyKind(StrEnum):
API_TOOL = "api_tool"
WORKFLOW_TOOL = "workflow_tool"
MCP_TOOL = "mcp_tool"
BUILTIN_OR_PLUGIN_TOOL = "builtin_or_plugin_tool"
UNRESOLVED = "unresolved"
class TargetTenantSelector(TypedDict, total=False):
id: str
name: str
def _parse_target_tenant(value: Any) -> TargetTenantSelector | None:
if value is None:
return None
if not isinstance(value, dict):
raise MigrationDataError("metadata.target_tenant must be an object when provided.")
target: TargetTenantSelector = {}
target_id = value.get("id")
if target_id is not None:
if not isinstance(target_id, str):
raise MigrationDataError("metadata.target_tenant.id must be a string.")
target["id"] = target_id
target_name = value.get("name")
if target_name is not None:
if not isinstance(target_name, str):
raise MigrationDataError("metadata.target_tenant.name must be a string.")
target["name"] = target_name
unsupported_keys = sorted(set(value.keys()) - {"id", "name"})
if unsupported_keys:
raise MigrationDataError(f"metadata.target_tenant contains unsupported fields: {unsupported_keys}")
return target
def _parse_package_section(value: Any, section: str) -> list[dict[str, Any]]:
if value is None:
return []
if not isinstance(value, list):
raise MigrationDataError(f"Migration package field '{section}' must be a list.")
for item in value:
if not isinstance(item, dict):
raise MigrationDataError(f"Migration package field '{section}' must contain only objects.")
return value
@dataclass(frozen=True)
class SourceTenant:
id: str
name: str
@classmethod
def from_mapping(cls, value: dict[str, Any]) -> SourceTenant:
return cls(id=str(value.get("id", "")), name=str(value.get("name", "")))
@dataclass(frozen=True)
class ImportOptions:
create_app_api_token_on_import: bool = False
id_strategy: IdStrategy = IdStrategy.PRESERVE_ID
conflict_strategy: ConflictStrategy = ConflictStrategy.FAIL
@classmethod
def from_mapping(cls, value: dict[str, Any] | None) -> ImportOptions:
value = value or {}
try:
id_strategy = IdStrategy(value.get("id_strategy", IdStrategy.PRESERVE_ID))
except ValueError as exc:
raise MigrationDataError(f"Unsupported import_options.id_strategy: {value.get('id_strategy')}") from exc
try:
conflict_strategy = ConflictStrategy(value.get("conflict_strategy", ConflictStrategy.FAIL))
except ValueError as exc:
raise MigrationDataError(
f"Unsupported import_options.conflict_strategy: {value.get('conflict_strategy')}"
) from exc
return cls(
create_app_api_token_on_import=bool(value.get("create_app_api_token_on_import", False)),
id_strategy=id_strategy,
conflict_strategy=conflict_strategy,
)
@dataclass(frozen=True)
class MigrationMetadata:
version: str
source_scope: Literal["single"]
source_tenants: list[SourceTenant]
target_tenant: TargetTenantSelector | None = None
created_at: str | None = None
include_secrets: bool = False
import_options: ImportOptions = field(default_factory=ImportOptions)
@classmethod
def from_mapping(cls, value: dict[str, Any]) -> MigrationMetadata:
version = value.get("version")
if not version:
raise MigrationDataError("Migration package must include metadata.version.")
source_scope = value.get("source_scope", "single")
if source_scope != "single":
raise MigrationDataError(f"Unsupported source_scope: {source_scope}")
source_tenants = [
SourceTenant.from_mapping(item) for item in value.get("source_tenants", []) if isinstance(item, dict)
]
return cls(
version=str(version),
source_scope="single",
source_tenants=source_tenants,
target_tenant=_parse_target_tenant(value.get("target_tenant")),
created_at=value.get("created_at"),
include_secrets=bool(value.get("include_secrets", False)),
import_options=ImportOptions.from_mapping(value.get("import_options")),
)
@dataclass(frozen=True)
class MigrationPackage:
metadata: MigrationMetadata
workflows: list[dict[str, Any]] = field(default_factory=list)
tools: list[dict[str, Any]] = field(default_factory=list)
workflow_tools: list[dict[str, Any]] = field(default_factory=list)
mcp_tools: list[dict[str, Any]] = field(default_factory=list)
dependencies: list[dict[str, Any]] = field(default_factory=list)
@classmethod
def from_mapping(cls, value: dict[str, Any]) -> MigrationPackage:
metadata_value = value.get("metadata")
if not isinstance(metadata_value, dict):
raise MigrationDataError("Migration package must include metadata.version.")
return cls(
metadata=MigrationMetadata.from_mapping(metadata_value),
workflows=_parse_package_section(value.get("workflows"), "workflows"),
tools=_parse_package_section(value.get("tools"), "tools"),
workflow_tools=_parse_package_section(value.get("workflow_tools"), "workflow_tools"),
mcp_tools=_parse_package_section(value.get("mcp_tools"), "mcp_tools"),
dependencies=_parse_package_section(value.get("dependencies"), "dependencies"),
)
@dataclass(frozen=True)
class ExportSelection:
source_tenant_name: str
app_ids: list[str]
source_tenant_id: str | None = None
export_all_apps: bool = False
include_referenced_tools: bool = True
additional_api_tools: list[str] = field(default_factory=list)
additional_workflow_tools: list[str] = field(default_factory=list)
additional_mcp_tools: list[str] = field(default_factory=list)
include_secrets: bool = False
import_options: ImportOptions = field(default_factory=ImportOptions)
@dataclass(frozen=True)
class ResourceReportItem:
resource_type: ResourceType
identifier: str
name: str | None
status: str
message: str | None = None
@dataclass(frozen=True)
class ResourceIdMapping:
resource_type: ResourceType
name: str | None
source_id: str
target_id: str
@dataclass(frozen=True)
class ExportResult:
package: MigrationPackage
report_items: list[ResourceReportItem]
report_context: ReportContext | None = None
@dataclass(frozen=True)
class ImportTarget:
tenant_id: str
tenant_name: str
operator_id: str
operator_email: str | None
@dataclass(frozen=True)
class ImportResult:
report_items: list[ResourceReportItem]
id_mapping: dict[str, str] = field(default_factory=dict)
report_context: ReportContext | None = None
@dataclass(frozen=True)
class ReportContext:
output_path: str | None = None
source_scope: str | None = None
selected_app_count: int | None = None
include_secrets: bool | None = None
target_tenant: str | None = None
operator_email: str | None = None
app_api_tokens_created: int = 0
app_api_tokens_reused: int = 0
id_mapping_count: int = 0
id_mappings: dict[str, str] = field(default_factory=dict)
id_mapping_details: list[ResourceIdMapping] = field(default_factory=list)
@@ -0,0 +1,488 @@
from __future__ import annotations
from collections.abc import Iterable
from typing import Any
import sqlalchemy as sa
import yaml
from sqlalchemy.orm import Session
from core.tools.tool_manager import ToolManager
from graphon.model_runtime.utils.encoders import jsonable_encoder
from models import Account, Tenant
from models.account import TenantAccountJoin
from models.model import App
from models.tools import MCPToolProvider
from services.app_dsl_service import AppDslService
from services.data_migration.dependency_discovery_service import DependencyDiscoveryService, DiscoveredDependency
from services.data_migration.entities import (
DependencyKind,
ExportResult,
ExportSelection,
ImportOptions,
MigrationDataError,
ReportContext,
ResourceReportItem,
ResourceType,
)
from services.data_migration.package_service import MigrationPackageService
from services.tools.mcp_tools_manage_service import MCPToolManageService
from services.tools.workflow_tools_manage_service import WorkflowToolManageService
SUPPORTED_APP_MODES = {"workflow", "advanced-chat"}
class ExportConfigParser:
def parse(self, data: dict[str, Any]) -> ExportSelection:
if not isinstance(data, dict):
raise MigrationDataError("Export config JSON must be an object.")
source_tenant = self._source_tenant(data)
source_tenant_name = self._source_tenant_name(source_tenant, data)
apps = self._mapping(data.get("apps"), field_name="apps")
self._validate_source_scope(data)
self._validate_app_modes(apps.get("modes", []))
additional_tools = self._mapping(data.get("additional_tools"), field_name="additional_tools")
return ExportSelection(
source_tenant_name=source_tenant_name,
app_ids=self._string_list(apps.get("ids", data.get("workflows", [])), field_name="apps.ids"),
source_tenant_id=source_tenant.get("id"),
export_all_apps=bool(apps.get("all", data.get("export_all_workflows", False))),
include_referenced_tools=bool(data.get("include_referenced_tools", True)),
additional_api_tools=self._string_list(
additional_tools.get("api_tools", data.get("tools", [])), field_name="additional_tools.api_tools"
),
additional_workflow_tools=self._string_list(
additional_tools.get("workflow_tools", data.get("workflow_tools", [])),
field_name="additional_tools.workflow_tools",
),
additional_mcp_tools=self._string_list(
additional_tools.get("mcp_tools", data.get("mcp_tools", [])),
field_name="additional_tools.mcp_tools",
),
include_secrets=bool(data.get("include_secrets", False)),
import_options=ImportOptions.from_mapping(data.get("import_options")),
)
def _source_tenant(self, data: dict[str, Any]) -> dict[str, Any]:
if "source_tenant" in data:
return self._mapping(data.get("source_tenant"), field_name="source_tenant")
return {}
def _source_tenant_name(self, source_tenant: dict[str, Any], data: dict[str, Any]) -> str:
if source_tenant:
source_tenant_name = source_tenant.get("name")
if not source_tenant_name:
raise MigrationDataError("Export config must include source_tenant.name.")
return str(source_tenant_name)
source_tenant_name = data.get("tenant_name")
if not source_tenant_name:
raise MigrationDataError("Export config must include source_tenant.name.")
return str(source_tenant_name)
def _validate_source_scope(self, data: dict[str, Any]) -> None:
source_tenant = data.get("source_tenant")
if not isinstance(source_tenant, dict):
return
mode = source_tenant.get("mode", "single")
if mode != "single":
raise MigrationDataError(f"Unsupported source_tenant.mode: {mode}")
def _validate_app_modes(self, modes: Any) -> None:
app_modes = self._string_list(modes, field_name="apps.modes") if modes else []
unsupported_modes = sorted(set(app_modes) - SUPPORTED_APP_MODES)
if unsupported_modes:
raise MigrationDataError(f"Unsupported app modes for export: {unsupported_modes}")
def _mapping(self, value: Any, *, field_name: str) -> dict[str, Any]:
if value is None:
return {}
if not isinstance(value, dict):
raise MigrationDataError(f"Export config field '{field_name}' must be an object.")
return value
def _string_list(self, value: Any, *, field_name: str) -> list[str]:
if value is None:
return []
if not isinstance(value, list):
raise MigrationDataError(f"Export config field '{field_name}' must be a list.")
return [str(item) for item in value]
class MigrationExportService:
def __init__(
self,
*,
package_service: MigrationPackageService | None = None,
dependency_discovery_service: DependencyDiscoveryService | None = None,
) -> None:
self.package_service = package_service or MigrationPackageService()
self.dependency_discovery_service = dependency_discovery_service or DependencyDiscoveryService()
def export(self, selection: ExportSelection, *, session: Session) -> ExportResult:
tenant = self._get_tenant(selection, session=session)
package = self.package_service.build_empty_package(
source_tenant_id=tenant.id,
source_tenant_name=tenant.name,
include_secrets=selection.include_secrets,
import_options=selection.import_options,
)
report_items: list[ResourceReportItem] = []
discovered_dependencies: list[DiscoveredDependency] = []
apps = self._selected_apps(tenant.id, selection, session=session)
exported_app_ids = {app.id for app in apps}
for app in apps:
dsl_content = AppDslService.export_dsl(
app_model=app, session=session, include_secret=selection.include_secrets
)
package.workflows.append(
{
"id": app.id,
"name": app.name,
"mode": app.mode.value if hasattr(app.mode, "value") else app.mode,
"dsl": dsl_content,
"source_tenant_id": tenant.id,
"create_app_api_token_on_import": selection.import_options.create_app_api_token_on_import,
}
)
report_items.append(ResourceReportItem(ResourceType.WORKFLOW, app.id, app.name, "exported"))
if selection.include_referenced_tools:
discovered_dependencies.extend(self._discover_dependencies(dsl_content))
self._export_api_tools(
tenant.id,
self._provider_ids(selection.additional_api_tools, discovered_dependencies, DependencyKind.API_TOOL),
include_secrets=selection.include_secrets,
exported_tools=package.tools,
report_items=report_items,
)
self._export_workflow_tools(
tenant,
self._provider_ids(
selection.additional_workflow_tools, discovered_dependencies, DependencyKind.WORKFLOW_TOOL
),
exported_app_ids=exported_app_ids,
exported_workflow_tools=package.workflow_tools,
dependencies=package.dependencies,
report_items=report_items,
session=session,
)
self._export_mcp_tools(
tenant_id=tenant.id,
provider_ids=self._provider_ids(
selection.additional_mcp_tools,
discovered_dependencies,
DependencyKind.MCP_TOOL,
),
include_secrets=selection.include_secrets,
exported_mcp_tools=package.mcp_tools,
dependencies=package.dependencies,
report_items=report_items,
session=session,
)
self._record_dependency_metadata(
self._dependencies_by_kind(discovered_dependencies, DependencyKind.BUILTIN_OR_PLUGIN_TOOL),
package.dependencies,
report_items,
)
return ExportResult(
package=package,
report_items=report_items,
report_context=ReportContext(
source_scope=package.metadata.source_scope,
selected_app_count=len(apps),
include_secrets=selection.include_secrets,
),
)
def _get_tenant(self, selection: ExportSelection, *, session: Session) -> Tenant:
if selection.source_tenant_id:
tenant = session.get(Tenant, selection.source_tenant_id)
if tenant is None:
raise MigrationDataError(f"Source tenant not found: {selection.source_tenant_id}")
if tenant.name != selection.source_tenant_name:
raise MigrationDataError(
f"Source tenant id/name mismatch: {selection.source_tenant_id} / {selection.source_tenant_name}"
)
return tenant
tenants = list(session.scalars(sa.select(Tenant).where(Tenant.name == selection.source_tenant_name)).all())
if not tenants:
raise MigrationDataError(f"Source tenant not found: {selection.source_tenant_name}")
if len(tenants) > 1:
raise MigrationDataError(
f"Source tenant name is ambiguous; use source_tenant.id: {selection.source_tenant_name}"
)
return tenants[0]
def _selected_apps(self, tenant_id: str, selection: ExportSelection, *, session: Session) -> list[App]:
query = sa.select(App).where(App.tenant_id == tenant_id, App.mode.in_(SUPPORTED_APP_MODES))
if not selection.export_all_apps:
if not selection.app_ids:
return []
query = query.where(App.id.in_(selection.app_ids))
apps = list(session.scalars(query).all())
if not selection.export_all_apps and len(apps) != len(set(selection.app_ids)):
found_ids = {app.id for app in apps}
missing_ids = [app_id for app_id in selection.app_ids if app_id not in found_ids]
raise MigrationDataError(
f"Selected app IDs not found in source tenant or unsupported app mode: {missing_ids}"
)
return apps
def _discover_dependencies(self, dsl_content: str | dict[str, Any]) -> list[DiscoveredDependency]:
if isinstance(dsl_content, dict):
dsl = dsl_content
else:
raw_dsl = yaml.safe_load(dsl_content) if dsl_content else {}
dsl = raw_dsl if isinstance(raw_dsl, dict) else {}
return self.dependency_discovery_service.discover_from_dsl(dsl)
def _export_api_tools(
self,
tenant_id: str,
provider_ids: Iterable[str],
*,
include_secrets: bool,
exported_tools: list[dict[str, Any]],
report_items: list[ResourceReportItem],
) -> None:
for provider_id in self._dedupe(provider_ids):
try:
tool_data = ToolManager.user_get_api_provider(
provider=provider_id,
tenant_id=tenant_id,
mask=not include_secrets,
)
if not include_secrets:
tool_data.pop("credentials", None)
tool_data.pop("tools", None)
tool_data["provider_name"] = provider_id
tool_data["source_tenant_id"] = tenant_id
exported_tools.append(tool_data)
report_items.append(ResourceReportItem(ResourceType.API_TOOL, provider_id, provider_id, "exported"))
except Exception as exc:
report_items.append(
ResourceReportItem(ResourceType.API_TOOL, provider_id, provider_id, "unresolved", str(exc))
)
def _export_workflow_tools(
self,
tenant: Tenant,
provider_ids: Iterable[str],
*,
exported_app_ids: set[str],
exported_workflow_tools: list[dict[str, Any]],
dependencies: list[dict[str, Any]],
report_items: list[ResourceReportItem],
session: Session,
) -> None:
provider_ids = self._dedupe(provider_ids)
if not provider_ids:
return
owner = self._get_tenant_owner(tenant.id, session=session)
if owner is None:
for provider_id in provider_ids:
report_items.append(
ResourceReportItem(
ResourceType.WORKFLOW_TOOL,
provider_id,
provider_id,
"unresolved",
f"No owner account found for source tenant: {tenant.name}",
)
)
return
for provider_id in provider_ids:
try:
tool_data = WorkflowToolManageService.get_workflow_tool_by_tool_id(
user_id=owner.id,
tenant_id=tenant.id,
workflow_tool_id=provider_id,
)
tool_info = jsonable_encoder(tool_data)
tool_info["id"] = provider_id
tool_info["app_id"] = tool_info.get("workflow_app_id")
tool_info["source_tenant_id"] = tenant.id
for field_name in ("workflow_tool_id", "workflow_app_id", "tool"):
tool_info.pop(field_name, None)
exported_workflow_tools.append(tool_info)
if tool_info.get("app_id") not in exported_app_ids:
workflow_app_id = str(tool_info.get("app_id") or "")
workflow_app = session.get(App, workflow_app_id) if workflow_app_id else None
self._record_dependency_metadata(
[
DiscoveredDependency(
DependencyKind.WORKFLOW_TOOL,
workflow_app_id,
provider_name=workflow_app.name if workflow_app else tool_info.get("name"),
source="workflow_tool_app",
)
],
dependencies,
report_items,
)
report_items.append(
ResourceReportItem(ResourceType.WORKFLOW_TOOL, provider_id, tool_info.get("name"), "exported")
)
except Exception as exc:
report_items.append(
ResourceReportItem(ResourceType.WORKFLOW_TOOL, provider_id, provider_id, "unresolved", str(exc))
)
def _get_tenant_owner(self, tenant_id: str, *, session: Session) -> Account | None:
return session.scalar(
sa.select(Account)
.join(TenantAccountJoin, Account.id == TenantAccountJoin.account_id)
.where(TenantAccountJoin.tenant_id == tenant_id, TenantAccountJoin.role == "owner")
.order_by(TenantAccountJoin.created_at.asc())
.limit(1)
)
def _export_mcp_tools(
self,
*,
tenant_id: str,
provider_ids: Iterable[str],
include_secrets: bool,
exported_mcp_tools: list[dict[str, Any]],
dependencies: list[dict[str, Any]],
report_items: list[ResourceReportItem],
session: Session,
) -> None:
for provider_id in self._dedupe(provider_ids):
if not include_secrets:
self._record_dependency_metadata(
[DiscoveredDependency(DependencyKind.MCP_TOOL, provider_id, source="mcp_provider")],
dependencies,
report_items,
)
continue
try:
provider = self._get_mcp_provider(tenant_id, provider_id, session=session)
exported_mcp_tools.append(self._serialize_mcp_provider(provider))
report_items.append(ResourceReportItem(ResourceType.MCP_TOOL, provider_id, provider.name, "exported"))
except Exception as exc:
report_items.append(
ResourceReportItem(ResourceType.MCP_TOOL, provider_id, provider_id, "unresolved", str(exc))
)
def _get_mcp_provider(self, tenant_id: str, provider_id: str, *, session: Session) -> MCPToolProvider:
try:
return MCPToolManageService(session=session).get_provider_by_persisted_reference(
id_or_server_identifier=provider_id, tenant_id=tenant_id
)
except ValueError as exc:
raise MigrationDataError(f"MCP provider not found: {provider_id}") from exc
def _serialize_mcp_provider(self, provider: MCPToolProvider) -> dict[str, Any]:
provider_entity = provider.to_entity()
provider_icon = provider_entity.provider_icon
if isinstance(provider_icon, dict):
icon = provider_icon.get("content")
icon_background = provider_icon.get("background")
icon_type = "emoji"
else:
icon = provider_icon
icon_background = None
icon_type = "url"
return {
"id": provider.id,
"name": provider.name,
"server_url": provider_entity.decrypt_server_url(),
"server_identifier": provider.server_identifier,
"icon": icon,
"icon_background": icon_background,
"icon_type": icon_type,
"configuration": {"timeout": provider.timeout, "sse_read_timeout": provider.sse_read_timeout},
"headers": provider_entity.decrypt_headers(),
"authentication": self._serialize_mcp_authentication(provider_entity.decrypt_authentication()),
"tools": provider.tool_dict,
"source_tenant_id": provider.tenant_id,
}
def _serialize_mcp_authentication(self, authentication: dict[str, Any] | None) -> dict[str, Any] | None:
if not authentication or not authentication.get("client_id"):
return None
return {
"client_id": authentication["client_id"],
"client_secret": authentication.get("client_secret"),
}
def _record_dependency_metadata(
self,
dependencies_to_record: Iterable[DiscoveredDependency],
dependencies: list[dict[str, Any]],
report_items: list[ResourceReportItem],
) -> None:
existing = {(item.get("kind"), item.get("provider_id")) for item in dependencies}
for dependency in dependencies_to_record:
key = (dependency.kind.value, dependency.provider_id)
if key in existing:
continue
existing.add(key)
dependencies.append(
{
"kind": dependency.kind.value,
"provider_id": dependency.provider_id,
"provider_name": dependency.provider_name,
"source": dependency.source,
}
)
report_items.append(
ResourceReportItem(
ResourceType.DEPENDENCY,
dependency.provider_id,
self._dependency_report_name(dependency),
"dependency-only",
self._dependency_message(dependency.kind),
)
)
def _provider_ids(
self,
manual_provider_ids: Iterable[str],
discovered_dependencies: Iterable[DiscoveredDependency],
kind: DependencyKind,
) -> list[str]:
provider_ids = list(manual_provider_ids)
provider_ids.extend(
self._provider_export_identifier(dependency)
for dependency in discovered_dependencies
if dependency.kind == kind
)
return self._dedupe(provider_ids)
def _provider_export_identifier(self, dependency: DiscoveredDependency) -> str:
if dependency.kind == DependencyKind.API_TOOL and dependency.provider_name:
return dependency.provider_name
return dependency.provider_id
def _dependencies_by_kind(
self, discovered_dependencies: Iterable[DiscoveredDependency], kind: DependencyKind
) -> list[DiscoveredDependency]:
return [dependency for dependency in discovered_dependencies if dependency.kind == kind]
def _dedupe(self, values: Iterable[str]) -> list[str]:
seen: set[str] = set()
result: list[str] = []
for value in values:
if value and value not in seen:
seen.add(value)
result.append(value)
return result
def _dependency_message(self, kind: DependencyKind) -> str:
if kind == DependencyKind.MCP_TOOL:
return "Configure MCP provider manually in the target tenant unless exporting with secrets enabled."
if kind == DependencyKind.BUILTIN_OR_PLUGIN_TOOL:
return "Ensure the built-in or plugin tool exists in the target environment."
return "Dependency metadata only; ensure the resource exists in the target environment."
def _dependency_report_name(self, dependency: DiscoveredDependency) -> str:
name = dependency.provider_name or dependency.provider_id
if dependency.kind == DependencyKind.WORKFLOW_TOOL:
return f"workflow {name}"
return f"{dependency.kind.value} {name}"
@@ -0,0 +1,996 @@
"""Apply versioned migration packages to an explicitly resolved target tenant.
Import target resolution is deliberately performed before any resource import
work. The service does not write Click output; callers receive structured
report items and can decide how to render them.
"""
from __future__ import annotations
import json
from collections.abc import Iterable
from dataclasses import dataclass
from typing import Any, cast
from uuid import UUID
import sqlalchemy as sa
import yaml
from sqlalchemy import or_
from sqlalchemy.orm import Session, sessionmaker
from configs import dify_config
from core.entities.mcp_provider import IdentityMode, MCPAuthentication, MCPConfiguration
from core.tools.entities.tool_entities import ApiProviderSchemaType, WorkflowToolParameterConfiguration
from extensions.ext_database import db
from libs.datetime_utils import naive_utc_now
from models import Account, ApiToken, Tenant, TenantAccountJoin, TenantAccountRole
from models.enums import ApiTokenType
from models.model import App
from models.tools import ApiToolProvider, MCPToolProvider, WorkflowToolProvider
from services.app_dsl_service import AppDslService
from services.data_migration.dependency_discovery_service import DependencyDiscoveryService
from services.data_migration.entities import (
ConflictStrategy,
DependencyKind,
IdStrategy,
ImportOptions,
ImportResult,
ImportTarget,
MigrationDataError,
MigrationPackage,
ReportContext,
ResourceIdMapping,
ResourceReportItem,
ResourceType,
)
from services.entities.dsl_entities import ImportStatus
from services.tools.api_tools_manage_service import ApiToolManageService
from services.tools.mcp_tools_manage_service import MCPToolManageService
from services.tools.workflow_tools_manage_service import WorkflowToolManageService
from services.workflow_service import WorkflowService
@dataclass(frozen=True)
class ImportRequest:
"""Structured input for package import.
`cli_target_tenant` and `config_target_tenant` are target tenant names from
outer adapters. They intentionally override package metadata, because a
migration package may be reused across environments.
"""
package: MigrationPackage
cli_target_tenant: str | None = None
config_target_tenant: str | None = None
operator_email: str | None = None
options_override: ImportOptions | None = None
class ImportTargetResolver:
"""Resolve the target tenant and operator before import side effects begin."""
def select_target_tenant_name(self, request: ImportRequest) -> str:
if request.cli_target_tenant:
return request.cli_target_tenant
if request.config_target_tenant:
return request.config_target_tenant
package_target = request.package.metadata.target_tenant or {}
if package_target.get("name"):
return package_target["name"]
if package_target.get("id"):
return package_target["id"]
raise MigrationDataError(
"Target tenant must be provided by --target-tenant, import config, or package metadata."
)
def resolve(self, request: ImportRequest, *, session: Session) -> ImportTarget:
target_tenant_name = self.select_target_tenant_name(request)
package_target = request.package.metadata.target_tenant or {}
if request.cli_target_tenant or request.config_target_tenant:
tenant = self._resolve_tenant_by_id_or_name(target_tenant_name, session=session)
elif package_target.get("id") and self._is_uuid(package_target["id"]):
tenant = session.get(Tenant, package_target["id"])
if tenant is not None and package_target.get("name") and tenant.name != package_target.get("name"):
raise MigrationDataError(
f"Target tenant id/name mismatch: {package_target['id']} / {package_target['name']}"
)
else:
tenant = self._resolve_tenant_by_id_or_name(target_tenant_name, session=session)
if tenant is None:
raise MigrationDataError(f"Target tenant not found: {target_tenant_name}")
account_query = (
session.query(Account)
.join(TenantAccountJoin, Account.id == TenantAccountJoin.account_id)
.filter(TenantAccountJoin.tenant_id == tenant.id)
)
if request.operator_email:
account_query = account_query.filter(Account.email == request.operator_email)
identity = request.operator_email
else:
account_query = account_query.filter(TenantAccountJoin.role == TenantAccountRole.OWNER).order_by(
TenantAccountJoin.created_at.asc()
)
identity = "earliest owner"
account = account_query.first()
if account is None:
raise MigrationDataError(f"No operator account found for target tenant {target_tenant_name}: {identity}")
return ImportTarget(
tenant_id=tenant.id,
tenant_name=tenant.name,
operator_id=account.id,
operator_email=account.email,
)
def _resolve_tenant_by_id_or_name(self, value: str, *, session: Session) -> Tenant | None:
if self._is_uuid(value):
tenant = session.get(Tenant, value)
if tenant is not None:
return tenant
tenants = list(session.scalars(sa.select(Tenant).where(Tenant.name == value)).all())
if len(tenants) > 1:
raise MigrationDataError(f"Target tenant name is ambiguous; use target_tenant.id: {value}")
return tenants[0] if tenants else None
def _is_uuid(self, value: str) -> bool:
try:
UUID(value)
except ValueError:
return False
return True
class MigrationImportService:
"""Apply package resources using Dify service APIs and structured reporting."""
target_resolver: ImportTargetResolver
def __init__(self, *, target_resolver: ImportTargetResolver | None = None) -> None:
self.target_resolver = target_resolver or ImportTargetResolver()
def import_package(self, request: ImportRequest, *, session: Session) -> ImportResult:
target = self.target_resolver.resolve(request, session=session)
options = request.options_override or request.package.metadata.import_options
report_items = [
ResourceReportItem(
resource_type=ResourceType.DEPENDENCY,
identifier=target.tenant_id,
name=target.tenant_name,
status="resolved",
message=f"operator: {target.operator_email or target.operator_id}",
)
]
id_mapping: dict[str, str] = {}
id_mapping_details: list[ResourceIdMapping] = []
self._import_api_tools(
request.package,
target,
options,
report_items,
id_mapping,
id_mapping_details,
self._source_api_provider_ids_by_name(request.package),
session=session,
)
self._import_mcp_tools(
request.package, target, options, report_items, id_mapping, id_mapping_details, session=session
)
self._preflight_dependency_only_mcp(request.package, target, report_items, session=session)
workflow_tool_app_ids = self._workflow_tool_source_app_ids(request.package)
imported_workflow_ids: set[str] = set()
if workflow_tool_app_ids:
self._import_workflows(
request.package,
target,
options,
report_items,
id_mapping,
id_mapping_details=id_mapping_details,
imported_workflow_ids=imported_workflow_ids,
only_app_ids=workflow_tool_app_ids,
session=session,
)
self._import_workflow_tools(
request.package, target, options, id_mapping, id_mapping_details, report_items, session=session
)
self._import_workflows(
request.package,
target,
options,
report_items,
id_mapping,
id_mapping_details=id_mapping_details,
imported_workflow_ids=imported_workflow_ids,
skip_app_ids=imported_workflow_ids,
session=session,
)
return ImportResult(
report_items=report_items,
id_mapping=id_mapping,
report_context=ReportContext(
target_tenant=target.tenant_name,
operator_email=target.operator_email,
id_mapping_count=len(id_mapping),
id_mappings=dict(id_mapping),
id_mapping_details=id_mapping_details,
),
)
def _import_workflows(
self,
package: MigrationPackage,
target: ImportTarget,
options: ImportOptions,
report_items: list[ResourceReportItem],
id_mapping: dict[str, str],
id_mapping_details: list[ResourceIdMapping],
imported_workflow_ids: set[str] | None = None,
only_app_ids: set[str] | None = None,
skip_app_ids: set[str] | None = None,
*,
session: Session,
) -> None:
account = session.get(Account, target.operator_id)
tenant = session.get(Tenant, target.tenant_id)
if account is None:
raise MigrationDataError(f"Operator account not found: {target.operator_id}")
if tenant is None:
raise MigrationDataError(f"Target tenant not found: {target.tenant_id}")
account.set_current_tenant_with_session(tenant, session=session)
for workflow_data in package.workflows:
app_id = self._optional_string(workflow_data.get("id"))
if only_app_ids and app_id not in only_app_ids:
continue
if skip_app_ids and app_id in skip_app_ids:
continue
dsl_content = self._rewrite_workflow_dsl_provider_ids(
self._required_string(workflow_data, "dsl", "workflow"),
id_mapping,
)
existing_app = (
self._find_existing_app(app_id, target.tenant_id, session=session)
if options.id_strategy == IdStrategy.PRESERVE_ID
else None
)
if existing_app is not None and options.conflict_strategy == ConflictStrategy.FAIL:
raise MigrationDataError(f"App already exists and conflict_strategy=fail: {app_id}")
if existing_app is not None and options.conflict_strategy == ConflictStrategy.SKIP:
if app_id:
self._record_id_mappings(
id_mapping,
id_mapping_details,
ResourceType.WORKFLOW,
workflow_data.get("name") if isinstance(workflow_data.get("name"), str) else None,
{app_id},
existing_app.id,
)
report_items.append(
ResourceReportItem(ResourceType.WORKFLOW, str(app_id), workflow_data.get("name"), "skipped")
)
continue
imported_app_id = self._import_workflow_app(
account=account,
workflow_data=workflow_data,
dsl_content=dsl_content,
app_id=app_id,
existing_app=existing_app,
options=options,
session=session,
)
if app_id:
self._record_id_mappings(
id_mapping,
id_mapping_details,
ResourceType.WORKFLOW,
workflow_data.get("name") if isinstance(workflow_data.get("name"), str) else None,
{app_id},
imported_app_id,
)
if imported_workflow_ids is not None:
imported_workflow_ids.add(app_id)
if options.create_app_api_token_on_import:
self._create_or_reuse_app_api_token(imported_app_id, target.tenant_id, session=session)
report_items.append(
ResourceReportItem(
ResourceType.WORKFLOW,
imported_app_id,
workflow_data.get("name"),
"updated" if existing_app is not None else "created",
)
)
def _workflow_tool_source_app_ids(self, package: MigrationPackage) -> set[str]:
app_ids: set[str] = set()
for workflow_tool_data in package.workflow_tools:
app_id = self._optional_string(workflow_tool_data.get("app_id"))
if app_id:
app_ids.add(app_id)
return app_ids
def _import_workflow_app(
self,
*,
account: Account,
workflow_data: dict[str, object],
dsl_content: str,
app_id: str | None,
existing_app: App | None,
options: ImportOptions,
session: Session,
) -> str:
import_service = AppDslService(session)
if existing_app is not None:
existing_app_id = existing_app.id
if dify_config.RBAC_ENABLED:
session.commit()
import_result = import_service.import_app(
account=account,
import_mode="yaml-content",
yaml_content=dsl_content,
app_id=existing_app_id,
)
else:
import_app_id = app_id if self._should_preserve_source_app_id(options) else None
import_result = import_service.import_app(
account=account,
import_mode="yaml-content",
yaml_content=dsl_content,
import_app_id=import_app_id,
)
if import_result.status not in {ImportStatus.COMPLETED, ImportStatus.COMPLETED_WITH_WARNINGS}:
error = import_result.error or f"unexpected import status {import_result.status}"
raise MigrationDataError(f"Workflow import failed: {error}")
if import_result.app_id is None:
raise MigrationDataError(f"Workflow import did not return an app id: {workflow_data.get('name')}")
session.commit()
return import_result.app_id
def _rewrite_workflow_dsl_provider_ids(self, dsl_content: str, id_mapping: dict[str, str]) -> str:
if not id_mapping:
return dsl_content
parsed = yaml.safe_load(dsl_content) if dsl_content else {}
if not isinstance(parsed, dict):
return dsl_content
for node in self._workflow_nodes(parsed):
data = node.get("data") if isinstance(node, dict) else None
if not isinstance(data, dict):
continue
self._rewrite_tool_config_provider_id(data, id_mapping)
for tool_config in self._agent_tool_configs(data):
self._rewrite_tool_config_provider_id(tool_config, id_mapping)
return yaml.safe_dump(parsed, sort_keys=False, allow_unicode=True)
def _rewrite_tool_config_provider_id(self, tool_config: dict[str, Any], id_mapping: dict[str, str]) -> None:
provider_id = self._optional_string(tool_config.get("provider_id"))
if provider_id and provider_id in id_mapping:
tool_config["provider_id"] = id_mapping[provider_id]
def _source_api_provider_ids_by_name(self, package: MigrationPackage) -> dict[str, set[str]]:
provider_ids_by_name: dict[str, set[str]] = {}
discovery_service = DependencyDiscoveryService()
for workflow_data in package.workflows:
dsl_content = self._optional_string(workflow_data.get("dsl"))
if not dsl_content:
continue
parsed = yaml.safe_load(dsl_content) if dsl_content else {}
if not isinstance(parsed, dict):
continue
for dependency in discovery_service.discover_from_dsl(parsed):
if dependency.kind != DependencyKind.API_TOOL or not dependency.provider_name:
continue
provider_ids_by_name.setdefault(dependency.provider_name, set()).add(dependency.provider_id)
return provider_ids_by_name
def _workflow_nodes(self, dsl: dict[str, Any]) -> list[dict[str, Any]]:
nodes: list[dict[str, Any]] = []
graph = dsl.get("graph")
if isinstance(graph, dict) and isinstance(graph.get("nodes"), list):
nodes.extend(node for node in graph["nodes"] if isinstance(node, dict))
workflow = dsl.get("workflow")
workflow_graph = workflow.get("graph") if isinstance(workflow, dict) else None
if isinstance(workflow_graph, dict) and isinstance(workflow_graph.get("nodes"), list):
nodes.extend(node for node in workflow_graph["nodes"] if isinstance(node, dict))
return nodes
def _agent_tool_configs(self, data: dict[str, Any]) -> list[dict[str, Any]]:
configs = data.get("tools")
if isinstance(configs, list):
return [config for config in configs if isinstance(config, dict)]
agent_parameters = data.get("agent_parameters")
if not isinstance(agent_parameters, dict):
return []
tools_parameter = agent_parameters.get("tools")
if not isinstance(tools_parameter, dict):
return []
value = tools_parameter.get("value", [])
if not isinstance(value, list):
return []
return [config for config in value if isinstance(config, dict)]
def _should_preserve_source_app_id(self, options: ImportOptions) -> bool:
return options.id_strategy == IdStrategy.PRESERVE_ID
def _find_existing_app(self, app_id: str | None, tenant_id: str, *, session: Session) -> App | None:
if not self._is_uuid_string(app_id):
return None
return session.scalar(sa.select(App).where(App.id == app_id, App.tenant_id == tenant_id))
def _create_or_reuse_app_api_token(self, app_id: str, tenant_id: str, *, session: Session) -> None:
existing = session.scalar(
sa.select(ApiToken).where(
ApiToken.type == ApiTokenType.APP,
ApiToken.app_id == app_id,
ApiToken.tenant_id == tenant_id,
)
)
if existing is not None:
return
api_token = ApiToken()
api_token.app_id = app_id
api_token.tenant_id = tenant_id
api_token.token = ApiToken.generate_api_key("app", 24, session=session)
api_token.type = ApiTokenType.APP
session.add(api_token)
session.commit()
def _import_api_tools(
self,
package: MigrationPackage,
target: ImportTarget,
options: ImportOptions,
report_items: list[ResourceReportItem],
id_mapping: dict[str, str],
id_mapping_details: list[ResourceIdMapping],
source_provider_ids_by_name: dict[str, set[str]],
*,
session: Session,
) -> None:
for tool_data in package.tools:
provider_name = self._required_string(tool_data, "provider_name", "api_tool")
schema = self._required_string(tool_data, "schema", "api_tool")
existing = session.scalar(
sa.select(ApiToolProvider).where(
ApiToolProvider.tenant_id == target.tenant_id,
ApiToolProvider.name == provider_name,
)
)
if existing is not None and options.conflict_strategy == ConflictStrategy.FAIL:
raise MigrationDataError(f"API tool already exists and conflict_strategy=fail: {provider_name}")
if existing is not None and options.conflict_strategy == ConflictStrategy.SKIP:
self._record_id_mappings(
id_mapping,
id_mapping_details,
ResourceType.API_TOOL,
provider_name,
self._api_tool_source_ids(provider_name, tool_data, source_provider_ids_by_name),
existing.id,
)
report_items.append(ResourceReportItem(ResourceType.API_TOOL, provider_name, provider_name, "skipped"))
continue
schema_info = ApiToolManageService.parser_api_schema(schema=schema)
schema_type = cast(ApiProviderSchemaType, schema_info["schema_type"])
credentials = (
cast(dict[str, Any], tool_data.get("credentials"))
if isinstance(tool_data.get("credentials"), dict)
else {}
)
credentials = credentials or {"auth_type": "none"}
raw_icon = tool_data.get("icon")
icon = (
cast(dict[str, Any], raw_icon)
if isinstance(raw_icon, dict)
else {"content": "tool", "background": "#FEF7C3"}
)
raw_labels = tool_data.get("labels")
labels = [str(label) for label in raw_labels] if isinstance(raw_labels, list) else []
if existing is not None:
ApiToolManageService.update_api_tool_provider(
user_id=target.operator_id,
tenant_id=target.tenant_id,
provider_name=provider_name,
original_provider=existing.name,
_schema_type=schema_type,
schema=schema,
privacy_policy=self._optional_string(tool_data.get("privacy_policy")) or "",
credentials=credentials,
custom_disclaimer=self._optional_string(tool_data.get("custom_disclaimer")) or "",
labels=labels,
icon=icon,
)
status = "updated"
else:
ApiToolManageService.create_api_tool_provider(
user_id=target.operator_id,
tenant_id=target.tenant_id,
provider_name=provider_name,
schema_type=schema_type,
schema=schema,
privacy_policy=self._optional_string(tool_data.get("privacy_policy")) or "",
credentials=credentials,
custom_disclaimer=self._optional_string(tool_data.get("custom_disclaimer")) or "",
labels=labels,
icon=icon,
)
status = "created"
target_provider = self._find_api_tool_provider(target.tenant_id, provider_name, session=session)
if target_provider is not None:
self._record_id_mappings(
id_mapping,
id_mapping_details,
ResourceType.API_TOOL,
provider_name,
self._api_tool_source_ids(provider_name, tool_data, source_provider_ids_by_name),
target_provider.id,
)
report_items.append(ResourceReportItem(ResourceType.API_TOOL, provider_name, provider_name, status))
def _find_api_tool_provider(
self, tenant_id: str, provider_name: str, *, session: Session
) -> ApiToolProvider | None:
return session.scalar(
sa.select(ApiToolProvider).where(
ApiToolProvider.tenant_id == tenant_id,
ApiToolProvider.name == provider_name,
)
)
def _api_tool_source_ids(
self,
provider_name: str,
tool_data: dict[str, Any],
source_provider_ids_by_name: dict[str, set[str]],
) -> set[str]:
source_ids = set(source_provider_ids_by_name.get(provider_name, set()))
source_id = self._optional_string(tool_data.get("id"))
if source_id:
source_ids.add(source_id)
return source_ids
def _record_id_mappings(
self,
id_mapping: dict[str, str],
id_mapping_details: list[ResourceIdMapping],
resource_type: ResourceType,
name: str | None,
source_ids: Iterable[str],
target_id: str,
) -> None:
for source_id in source_ids:
id_mapping[source_id] = target_id
id_mapping_details[:] = [item for item in id_mapping_details if item.source_id != source_id]
id_mapping_details.append(ResourceIdMapping(resource_type, name, source_id, target_id))
def _import_workflow_tools(
self,
package: MigrationPackage,
target: ImportTarget,
options: ImportOptions,
id_mapping: dict[str, str],
id_mapping_details: list[ResourceIdMapping],
report_items: list[ResourceReportItem],
*,
session: Session,
) -> None:
if not package.workflow_tools:
return
account = session.get(Account, target.operator_id)
if account is None:
raise MigrationDataError(f"Operator account not found: {target.operator_id}")
for workflow_tool_data in package.workflow_tools:
app_id = self._optional_string(workflow_tool_data.get("app_id"))
resolved_app_id = id_mapping.get(app_id or "", app_id)
if (
not resolved_app_id
or self._find_existing_app(resolved_app_id, target.tenant_id, session=session) is None
):
report_items.append(
ResourceReportItem(
ResourceType.WORKFLOW_TOOL,
str(workflow_tool_data.get("id", workflow_tool_data.get("name", ""))),
self._optional_string(workflow_tool_data.get("name")),
"unresolved",
"Referenced workflow app was not found in the target tenant; workflow tool was skipped.",
)
)
continue
try:
self._ensure_workflow_app_is_published(target, account, resolved_app_id, session=session)
except Exception as exc:
report_items.append(
ResourceReportItem(
ResourceType.WORKFLOW_TOOL,
str(workflow_tool_data.get("id", workflow_tool_data.get("name", ""))),
self._optional_string(workflow_tool_data.get("name")),
"unresolved",
f"Referenced workflow app could not be published: {exc}",
)
)
continue
workflow_tool_id = self._optional_string(workflow_tool_data.get("id"))
tool_name = self._required_string(workflow_tool_data, "name", "workflow_tool")
lookup_workflow_tool_id = workflow_tool_id if options.id_strategy == IdStrategy.PRESERVE_ID else None
existing = self._find_existing_workflow_tool(
target.tenant_id, lookup_workflow_tool_id, tool_name, resolved_app_id, session=session
)
if existing is not None and options.conflict_strategy == ConflictStrategy.FAIL:
raise MigrationDataError(f"Workflow tool already exists and conflict_strategy=fail: {tool_name}")
if existing is not None and options.conflict_strategy == ConflictStrategy.SKIP:
if workflow_tool_id:
self._record_id_mappings(
id_mapping,
id_mapping_details,
ResourceType.WORKFLOW_TOOL,
tool_name,
{workflow_tool_id},
existing.id,
)
report_items.append(ResourceReportItem(ResourceType.WORKFLOW_TOOL, existing.id, tool_name, "skipped"))
continue
raw_icon = workflow_tool_data.get("icon")
icon = (
cast(dict[str, Any], raw_icon)
if isinstance(raw_icon, dict)
else {"content": "🤖", "background": "#FFEAD5"}
)
raw_parameters = workflow_tool_data.get("parameters")
parameters = [
parameter
if isinstance(parameter, WorkflowToolParameterConfiguration)
else WorkflowToolParameterConfiguration(**parameter)
for parameter in (raw_parameters if isinstance(raw_parameters, list) else [])
if isinstance(parameter, dict | WorkflowToolParameterConfiguration)
]
raw_labels = workflow_tool_data.get("labels")
labels = [str(label) for label in raw_labels] if isinstance(raw_labels, list) else []
label = self._optional_string(workflow_tool_data.get("label")) or tool_name
description = self._optional_string(workflow_tool_data.get("description")) or ""
privacy_policy = self._optional_string(workflow_tool_data.get("privacy_policy")) or ""
if existing is not None:
WorkflowToolManageService.update_workflow_tool(
user_id=account.id,
tenant_id=target.tenant_id,
workflow_tool_id=existing.id,
name=tool_name,
label=label,
icon=icon,
description=description,
parameters=parameters,
privacy_policy=privacy_policy,
labels=labels,
)
status = "updated"
identifier = existing.id
else:
import_id = workflow_tool_id if options.id_strategy == IdStrategy.PRESERVE_ID else ""
WorkflowToolManageService.create_workflow_tool(
user_id=account.id,
tenant_id=target.tenant_id,
workflow_app_id=resolved_app_id,
name=tool_name,
label=label,
icon=icon,
description=description,
parameters=parameters,
privacy_policy=privacy_policy,
labels=labels,
import_id=import_id or "",
)
status = "created"
target_provider = self._find_existing_workflow_tool(
target.tenant_id, import_id or None, tool_name, resolved_app_id, session=session
)
if target_provider is None:
raise MigrationDataError(f"Workflow tool was not created: {tool_name}")
identifier = target_provider.id
if workflow_tool_id:
self._record_id_mappings(
id_mapping,
id_mapping_details,
ResourceType.WORKFLOW_TOOL,
tool_name,
{workflow_tool_id},
identifier,
)
report_items.append(ResourceReportItem(ResourceType.WORKFLOW_TOOL, identifier, tool_name, status))
def _ensure_workflow_app_is_published(
self, target: ImportTarget, account: Account, app_id: str, *, session: Session
) -> None:
app = self._find_existing_app(app_id, target.tenant_id, session=session)
if app is None:
raise MigrationDataError(f"Referenced workflow app was not found in target tenant: {app_id}")
if app.workflow_id:
return
workflow_service = WorkflowService()
with sessionmaker(db.engine).begin() as session:
app_in_session = session.get(App, app_id)
account_in_session = session.get(Account, account.id)
if app_in_session is None:
raise MigrationDataError(f"Referenced workflow app was not found in target tenant: {app_id}")
if account_in_session is None:
raise MigrationDataError(f"Operator account not found: {account.id}")
workflow = workflow_service.publish_workflow(
session=session,
app_model=app_in_session,
account=account_in_session,
marked_name="Migration import",
marked_comment="Published automatically for workflow tool import.",
)
app_in_session.workflow_id = workflow.id
app_in_session.updated_by = account.id
app_in_session.updated_at = naive_utc_now()
def _import_mcp_tools(
self,
package: MigrationPackage,
target: ImportTarget,
options: ImportOptions,
report_items: list[ResourceReportItem],
id_mapping: dict[str, str],
id_mapping_details: list[ResourceIdMapping],
*,
session: Session,
) -> None:
for mcp_data in package.mcp_tools:
name = self._required_string(mcp_data, "name", "mcp_tool")
server_identifier = self._required_string(mcp_data, "server_identifier", "mcp_tool")
provider_id = self._optional_string(mcp_data.get("id"))
lookup_provider_id = provider_id if options.id_strategy == IdStrategy.PRESERVE_ID else None
existing = self._find_existing_mcp_tool(
target.tenant_id, lookup_provider_id, server_identifier, session=session
)
if existing is not None and options.conflict_strategy == ConflictStrategy.FAIL:
raise MigrationDataError(f"MCP tool already exists and conflict_strategy=fail: {name}")
if existing is not None and options.conflict_strategy == ConflictStrategy.SKIP:
if provider_id:
self._record_id_mappings(
id_mapping,
id_mapping_details,
ResourceType.MCP_TOOL,
name,
{provider_id},
existing.id,
)
report_items.append(ResourceReportItem(ResourceType.MCP_TOOL, existing.id, name, "skipped"))
continue
service = MCPToolManageService(session=session)
configuration = MCPConfiguration.model_validate(mcp_data.get("configuration") or {})
authentication = (
MCPAuthentication.model_validate(mcp_data["authentication"]) if mcp_data.get("authentication") else None
)
if existing is not None:
service.update_provider(
tenant_id=target.tenant_id,
provider_id=existing.id,
server_url=self._required_string(mcp_data, "server_url", "mcp_tool"),
name=name,
icon=self._optional_string(mcp_data.get("icon")) or "",
icon_type=self._optional_string(mcp_data.get("icon_type")) or "emoji",
icon_background=self._optional_string(mcp_data.get("icon_background")) or "",
server_identifier=server_identifier,
headers=mcp_data.get("headers") if isinstance(mcp_data.get("headers"), dict) else {},
configuration=configuration,
authentication=authentication,
# Re-import must not silently reset forwarding: preserve the
# stored mode (update_provider now defaults to OFF when omitted).
identity_mode=IdentityMode(existing.identity_mode),
)
session.commit()
status = "updated"
identifier = existing.id
provider = existing
else:
service.create_provider(
tenant_id=target.tenant_id,
user_id=target.operator_id,
server_url=self._required_string(mcp_data, "server_url", "mcp_tool"),
name=name,
icon=self._optional_string(mcp_data.get("icon")) or "",
icon_type=self._optional_string(mcp_data.get("icon_type")) or "emoji",
icon_background=self._optional_string(mcp_data.get("icon_background")) or "",
server_identifier=server_identifier,
headers=mcp_data.get("headers") if isinstance(mcp_data.get("headers"), dict) else {},
configuration=configuration,
authentication=authentication,
)
created_provider = self._find_existing_mcp_tool(
target.tenant_id, lookup_provider_id, server_identifier, session=session
)
if created_provider is None:
raise MigrationDataError(f"MCP provider was not created: {name}")
status = "created"
provider = created_provider
identifier = provider.id
self._restore_mcp_provider_tools(provider, mcp_data)
session.commit()
if provider_id:
self._record_id_mappings(
id_mapping,
id_mapping_details,
ResourceType.MCP_TOOL,
name,
{provider_id},
identifier,
)
report_items.append(ResourceReportItem(ResourceType.MCP_TOOL, identifier, name, status))
def _restore_mcp_provider_tools(self, provider: MCPToolProvider, mcp_data: dict[str, object]) -> None:
tools = mcp_data.get("tools")
if not isinstance(tools, list):
return
provider.tools = json.dumps(tools)
provider.authed = True
def _find_existing_mcp_tool(
self,
tenant_id: str,
provider_id: str | None,
server_identifier: str,
*,
session: Session,
) -> MCPToolProvider | None:
predicates = [MCPToolProvider.server_identifier == server_identifier]
if self._is_uuid_string(provider_id):
predicates.append(MCPToolProvider.id == provider_id)
return session.scalar(
sa.select(MCPToolProvider).where(MCPToolProvider.tenant_id == tenant_id, or_(*predicates)).limit(1)
)
def _is_uuid_string(self, value: str | None) -> bool:
if not value:
return False
try:
UUID(value)
except ValueError:
return False
return True
def _find_existing_workflow_tool(
self,
tenant_id: str,
workflow_tool_id: str | None,
tool_name: str,
app_id: str,
*,
session: Session,
) -> WorkflowToolProvider | None:
predicates = [WorkflowToolProvider.name == tool_name, WorkflowToolProvider.app_id == app_id]
if self._is_uuid_string(workflow_tool_id):
predicates.append(WorkflowToolProvider.id == workflow_tool_id)
return session.scalar(
sa.select(WorkflowToolProvider)
.where(WorkflowToolProvider.tenant_id == tenant_id, or_(*predicates))
.limit(1)
)
def _preflight_dependency_only_mcp(
self,
package: MigrationPackage,
target: ImportTarget,
report_items: list[ResourceReportItem],
*,
session: Session,
) -> None:
for dependency in package.dependencies:
if dependency.get("kind") != DependencyKind.MCP_TOOL.value:
continue
provider_id = str(dependency.get("provider_id", dependency.get("id", "")))
provider_name = self._optional_string(dependency.get("provider_name") or dependency.get("name"))
existing = self._find_dependency_only_mcp_provider(
target.tenant_id, provider_id, provider_name, session=session
)
report_name = f"mcp_tool {provider_name or getattr(existing, 'name', None) or provider_id}"
if existing is not None:
report_items.append(
ResourceReportItem(
ResourceType.DEPENDENCY,
provider_id,
report_name,
"available",
"MCP provider exists in target tenant.",
)
)
continue
reference_summary = self._dependency_only_mcp_reference_summary(package, provider_id, provider_name)
message = "missing in target tenant"
if reference_summary:
message = f"{message}; referenced by {reference_summary}"
message = f"{message}; configure it manually before running the workflow."
report_items.append(
ResourceReportItem(
ResourceType.DEPENDENCY,
provider_id,
report_name,
"skipped",
message,
)
)
def _find_dependency_only_mcp_provider(
self,
tenant_id: str,
provider_id: str,
provider_name: str | None,
*,
session: Session,
) -> MCPToolProvider | None:
predicates = [MCPToolProvider.server_identifier == provider_id]
if self._is_uuid_string(provider_id):
predicates.append(MCPToolProvider.id == provider_id)
return session.scalar(
sa.select(MCPToolProvider).where(MCPToolProvider.tenant_id == tenant_id, or_(*predicates)).limit(1)
)
def _dependency_only_mcp_reference_summary(
self, package: MigrationPackage, provider_id: str, provider_name: str | None
) -> str:
references = self._dependency_only_mcp_references(package, provider_id, provider_name)
return "; ".join(references)
def _dependency_only_mcp_references(
self, package: MigrationPackage, provider_id: str, provider_name: str | None
) -> list[str]:
references: list[str] = []
seen: set[str] = set()
for workflow_data in package.workflows:
workflow_name = self._optional_string(workflow_data.get("name"))
workflow_id = self._optional_string(workflow_data.get("id"))
workflow_label = workflow_name or workflow_id or "unknown workflow"
dsl_content = self._optional_string(workflow_data.get("dsl"))
if not dsl_content:
continue
parsed = yaml.safe_load(dsl_content) if dsl_content else {}
if not isinstance(parsed, dict):
continue
for node in self._workflow_nodes(parsed):
data = node.get("data") if isinstance(node, dict) else None
if not isinstance(data, dict):
continue
for tool_config in [data, *self._agent_tool_configs(data)]:
if not self._is_mcp_dependency_reference(tool_config, provider_id, provider_name):
continue
tool_label = self._mcp_tool_reference_label(node, tool_config)
reference = f"{workflow_label} / {tool_label}"
if reference not in seen:
seen.add(reference)
references.append(reference)
return references
def _is_mcp_dependency_reference(
self, tool_config: dict[str, Any], provider_id: str, provider_name: str | None
) -> bool:
provider_type = str(tool_config.get("provider_type") or tool_config.get("type") or "").lower()
if provider_type != "mcp":
return False
config_provider_id = self._optional_string(
tool_config.get("provider_id") or tool_config.get("provider_name") or tool_config.get("provider")
)
if config_provider_id == provider_id:
return True
return bool(provider_name and config_provider_id == provider_name)
def _mcp_tool_reference_label(self, node: dict[str, Any], tool_config: dict[str, Any]) -> str:
for key in ("tool_name", "tool", "name"):
value = self._optional_string(tool_config.get(key))
if value:
return value
node_id = self._optional_string(node.get("id"))
return node_id or "unknown tool"
def _required_string(self, value: dict[str, object], field_name: str, resource_name: str) -> str:
field_value = value.get(field_name)
if not isinstance(field_value, str) or not field_value:
raise MigrationDataError(f"Missing required {resource_name} field: {field_name}")
return field_value
def _optional_string(self, value: object) -> str | None:
if isinstance(value, str) and value:
return value
return None
@@ -0,0 +1,71 @@
"""JSON persistence for versioned cross-environment migration packages.
The package service validates file shape and serializes only structured package
entities. It does not perform CLI rendering or database access, keeping it safe
to reuse from Click adapters, tests, and future import/export services.
"""
from __future__ import annotations
import json
from dataclasses import asdict
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from services.data_migration.entities import (
ImportOptions,
MigrationDataError,
MigrationMetadata,
MigrationPackage,
SourceTenant,
TargetTenantSelector,
)
PACKAGE_VERSION = "1"
class MigrationPackageService:
def load_package(self, path: str | Path) -> MigrationPackage:
package_path = Path(path)
with package_path.open(encoding="utf-8") as file:
raw = json.load(file)
if not isinstance(raw, dict):
raise MigrationDataError("Migration package JSON must be an object.")
package = MigrationPackage.from_mapping(raw)
if package.metadata.version != PACKAGE_VERSION:
raise MigrationDataError(f"Unsupported migration package version: {package.metadata.version}")
return package
def save_package(self, package: MigrationPackage, path: str | Path, *, overwrite: bool) -> None:
package_path = Path(path)
if package_path.exists() and not overwrite:
raise MigrationDataError(f"Output file already exists: {package_path}")
package_path.parent.mkdir(parents=True, exist_ok=True)
with package_path.open("w", encoding="utf-8") as file:
json.dump(self.to_mapping(package), file, ensure_ascii=False, indent=2)
file.write("\n")
def build_empty_package(
self,
*,
source_tenant_id: str,
source_tenant_name: str,
include_secrets: bool,
import_options: ImportOptions | None = None,
target_tenant: TargetTenantSelector | None = None,
) -> MigrationPackage:
return MigrationPackage(
metadata=MigrationMetadata(
version=PACKAGE_VERSION,
source_scope="single",
source_tenants=[SourceTenant(id=source_tenant_id, name=source_tenant_name)],
target_tenant=target_tenant,
created_at=datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z"),
include_secrets=include_secrets,
import_options=import_options or ImportOptions(),
)
)
def to_mapping(self, package: MigrationPackage) -> dict[str, Any]:
return asdict(package)
@@ -0,0 +1,76 @@
from __future__ import annotations
from collections import Counter
from services.data_migration.entities import ReportContext, ResourceIdMapping, ResourceReportItem
class MigrationReportService:
"""Render structured migration resource results into CLI-friendly summary lines."""
def render(self, items: list[ResourceReportItem], *, context: ReportContext | None = None) -> list[str]:
counts = Counter((item.resource_type.value, item.status) for item in items)
lines = self._render_context(context)
lines.extend(
[f"{resource_type} {status}: {count}" for (resource_type, status), count in sorted(counts.items())]
)
actionable_items = [
item for item in items if item.status in {"dependency-only", "skipped", "unresolved"} and item.message
]
for item in actionable_items:
lines.append(self._render_actionable_detail(item))
return lines
def _render_context(self, context: ReportContext | None) -> list[str]:
if context is None:
return []
lines: list[str] = []
if context.output_path:
lines.append(f"output: {context.output_path}")
if context.source_scope:
lines.append(f"source scope: {context.source_scope}")
if context.selected_app_count is not None:
lines.append(f"selected apps: {context.selected_app_count}")
if context.include_secrets is not None:
lines.append(f"include secrets: {str(context.include_secrets).lower()}")
if context.target_tenant:
lines.append(f"target tenant: {context.target_tenant}")
if context.operator_email:
lines.append(f"operator: {context.operator_email}")
if context.app_api_tokens_created or context.app_api_tokens_reused:
lines.append(
f"app api tokens: {context.app_api_tokens_created} created, {context.app_api_tokens_reused} reused"
)
if context.id_mappings:
lines.append(f"resource references resolved: {len(context.id_mappings)}")
if context.id_mapping_details:
lines.extend(
self._render_id_mapping_detail(item)
for item in sorted(
context.id_mapping_details,
key=lambda item: (item.resource_type.value, item.name or "", item.source_id),
)
)
else:
lines.extend(
f"- {source_id} -> {target_id}" for source_id, target_id in sorted(context.id_mappings.items())
)
elif context.id_mapping_count:
lines.append(f"resource references resolved: {context.id_mapping_count}")
return lines
def _render_id_mapping_detail(self, item: ResourceIdMapping) -> str:
label = item.resource_type.value
if item.name:
label = f"{label} {item.name}"
return f"- {label}: {item.source_id} -> {item.target_id}"
def _render_actionable_detail(self, item: ResourceReportItem) -> str:
if item.resource_type.value == "dependency" and item.name and self._has_dependency_type_prefix(item.name):
if item.identifier and item.identifier not in item.name:
return f"dependency {item.name}: {item.identifier}: {item.message}"
return f"dependency {item.name}: {item.message}"
return f"{item.resource_type.value} {item.identifier}: {item.message}"
def _has_dependency_type_prefix(self, name: str) -> bool:
return name.startswith(("workflow ", "api_tool ", "workflow_tool ", "mcp_tool ", "builtin_or_plugin_tool "))