chore: 纳入本地 Dify 1.19.0 源码及品牌与免登改动

演示目录只保留可重建的源码。运行密钥、数据卷、依赖和带口令的本地配置不进入仓库。

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
wuzilongandCursor committed 2026-11-18 13:59:25 +08:00
1 parent 60158935f4
commit 6519682885
15831 files changed
+3063428

No files matched your search

Whitespace-only changes.
@@ -0,0 +1,196 @@
"""The openapi DSL import rules; the console import uses none of them.
Before an import: refuse a stale overwrite (draft_hash), refuse a graph with errors, fill the
defaults the editor needs, and keep the secrets the draft already stores (an export blanks them).
"""
from __future__ import annotations
from collections.abc import Mapping, Sequence
from contextlib import AbstractContextManager
from dataclasses import dataclass
from typing import Any, Protocol
from urllib.parse import urlparse
import yaml
from constants import HIDDEN_VALUE
from core.file import remote_fetcher
from factories import variable_factory
from graphon.variables import SecretVariable, SegmentType, VariableBase
from graphon.variables.exc import VariableError
from libs.url_utils import github_blob_to_raw_url
from machinery.context import RequestContext
from models import AppMode, Workflow
from services.dsl_content import DSL_MAX_SIZE, dsl_content_size
from services.entities.dsl_entities import ImportMode
from services.errors.app import DraftChangedError
from services.workflow.graph_check import (
GRAPH_MODES,
GraphIssue,
IssueSeverity,
ResourceCheck,
check_graph,
)
from services.workflow.node_defaults import fill_graph
class DslNotCheckableError(ValueError):
pass
class YamlUrlFetchError(ValueError):
pass
class DslRefusedError(Exception):
def __init__(self, issues: Sequence[GraphIssue]) -> None:
super().__init__("The DSL has errors the import refuses")
self.issues = list(issues)
def stored_secret_ids(draft: Workflow) -> set[str]:
return {variable.id for variable in draft.environment_variables if isinstance(variable, SecretVariable)}
@dataclass(frozen=True)
class DslImportTarget:
"""What the import rules read from the app a DSL overwrites."""
mode: str
draft_token: str | None
stored_secret_ids: frozenset[str]
class DslImportStore(Protocol):
def load_target(self, context: RequestContext, app_id: str) -> DslImportTarget:
"""Raises ConsoleAppNotFoundError for an app outside the workspace."""
...
def credential_check(
self, workspace_id: str, environment: Mapping[str, VariableBase]
) -> AbstractContextManager[ResourceCheck]:
"""Publish's credential check, usable until the context exits."""
...
def fetch_yaml_url(yaml_url: str) -> str:
"""The YAML text at a URL, fetched as AppDslService.import_app does; a GitHub blob link reads its raw file."""
try:
parsed_url = urlparse(yaml_url)
if parsed_url.path.endswith((".yml", ".yaml")):
yaml_url = github_blob_to_raw_url(yaml_url)
response = remote_fetcher.make_request("GET", yaml_url.strip(), follow_redirects=True, timeout=(10, 10))
response.raise_for_status()
raw_content = response.content
if dsl_content_size(raw_content) > DSL_MAX_SIZE:
raise YamlUrlFetchError("File size exceeds the limit of 10MB")
content = raw_content.decode("utf-8")
except YamlUrlFetchError:
raise
except Exception as e:
raise YamlUrlFetchError(f"Error fetching YAML from URL: {str(e)}") from e
if not content:
raise YamlUrlFetchError("Empty content from url")
return content
def _parse(yaml_content: str) -> dict[str, Any]:
# Measured before parsing: re-dumping drops comments, so a padded DSL could shrink under the import's limit.
if dsl_content_size(yaml_content) > DSL_MAX_SIZE:
raise DslNotCheckableError("File size exceeds the limit of 10MB")
try:
data = yaml.safe_load(yaml_content)
except yaml.YAMLError as error:
raise DslNotCheckableError(f"Invalid YAML: {error}") from error
if not isinstance(data, dict):
raise DslNotCheckableError("The DSL must be a YAML mapping")
return data
def _section(data: Mapping[str, Any], key: str) -> dict[str, Any]:
value = data.get(key)
return value if isinstance(value, dict) else {}
def _graph(workflow: Mapping[str, Any]) -> Mapping[str, Any]:
graph = workflow.get("graph")
return graph if isinstance(graph, Mapping) else {}
def _graph_mode(data: Mapping[str, Any], target: DslImportTarget | None) -> AppMode:
"""The mode the graph must fit: the overwritten app's, else the DSL's own."""
mode = target.mode if target is not None else _section(data, "app").get("mode")
if mode not in GRAPH_MODES:
raise DslNotCheckableError("Only workflow or advanced-chat DSLs can be checked")
return AppMode(mode)
def _environment(workflow: Mapping[str, Any]) -> dict[str, VariableBase]:
items = workflow.get("environment_variables") or []
if not isinstance(items, list) or not all(isinstance(item, Mapping) for item in items):
raise DslNotCheckableError("workflow.environment_variables must be a list of mappings")
try:
variables = [variable_factory.build_environment_variable_from_mapping(item) for item in items]
except VariableError as error:
raise DslNotCheckableError(str(error)) from error
return {variable.name: variable for variable in variables}
def _keep_stored_secrets(workflow: Mapping[str, Any], stored: frozenset[str]) -> None:
for variable in workflow.get("environment_variables") or []:
if (
isinstance(variable, dict)
and variable.get("value_type") == SegmentType.SECRET
and variable.get("value") == ""
and variable.get("id") in stored
):
variable["value"] = HIDDEN_VALUE
class DslImportService:
def __init__(self, *, store: DslImportStore) -> None:
self._store = store
def _target(self, context: RequestContext, app_id: str | None) -> DslImportTarget | None:
return self._store.load_target(context, app_id) if app_id else None
def check(self, context: RequestContext, yaml_content: str, app_id: str | None) -> list[GraphIssue]:
"""Every issue, errors and credential warnings alike.
Raises DslNotCheckableError for a DSL that is not a workflow graph or has a bad environment
variable, ConsoleAppNotFoundError for an `app_id` outside the workspace.
"""
data = _parse(yaml_content)
workflow = _section(data, "workflow")
mode = _graph_mode(data, self._target(context, app_id))
environment = _environment(workflow)
with self._store.credential_check(context.active_workspace_id, environment) as resources:
return check_graph(_graph(workflow), mode=mode, resources=resources)
def prepare(
self, context: RequestContext, *, mode: ImportMode, source: str, app_id: str | None, draft_hash: str | None
) -> str:
"""The YAML to hand the import. A DSL the check can't read goes through unchanged, for the import to judge.
A URL is fetched only after the target app and draft pass. Raises DraftChangedError, DslRefusedError,
YamlUrlFetchError, or ConsoleAppNotFoundError for an `app_id` outside the workspace.
"""
target = self._target(context, app_id)
if draft_hash is not None and (target is None or target.draft_token != draft_hash):
raise DraftChangedError()
yaml_content = fetch_yaml_url(source) if mode == ImportMode.YAML_URL else source
try:
data = _parse(yaml_content)
app_mode = _graph_mode(data, target)
except DslNotCheckableError:
return yaml_content
workflow = _section(data, "workflow")
errors = [i for i in check_graph(_graph(workflow), mode=app_mode) if i.code.severity is IssueSeverity.ERROR]
if errors:
raise DslRefusedError(errors)
if "graph" in workflow:
workflow["graph"] = fill_graph(_graph(workflow))
if target is not None:
_keep_stored_secrets(workflow, target.stored_secret_ids)
return yaml.safe_dump(data, allow_unicode=True, sort_keys=False)
@@ -0,0 +1,171 @@
"""
Pydantic models for async workflow trigger system.
"""
from collections.abc import Mapping, Sequence
from enum import StrEnum
from typing import Any
from pydantic import BaseModel, ConfigDict, Field
from models.enums import AppTriggerType, WorkflowRunTriggeredFrom
class AsyncTriggerStatus(StrEnum):
"""Async trigger execution status"""
COMPLETED = "completed"
FAILED = "failed"
TIMEOUT = "timeout"
class TriggerMetadata(BaseModel):
"""Trigger metadata"""
type: AppTriggerType = Field(default=AppTriggerType.UNKNOWN)
class TriggerData(BaseModel):
"""Base trigger data model for async workflow execution"""
app_id: str
tenant_id: str
workflow_id: str | None = None
root_node_id: str
inputs: Mapping[str, Any]
files: Sequence[Mapping[str, Any]] = Field(default_factory=list)
trigger_type: AppTriggerType
trigger_from: WorkflowRunTriggeredFrom
trigger_metadata: TriggerMetadata | None = None
model_config = ConfigDict(use_enum_values=True)
class WebhookTriggerData(TriggerData):
"""Webhook-specific trigger data"""
trigger_type: AppTriggerType = AppTriggerType.TRIGGER_WEBHOOK
trigger_from: WorkflowRunTriggeredFrom = WorkflowRunTriggeredFrom.WEBHOOK
class ScheduleTriggerData(TriggerData):
"""Schedule-specific trigger data"""
trigger_type: AppTriggerType = AppTriggerType.TRIGGER_SCHEDULE
trigger_from: WorkflowRunTriggeredFrom = WorkflowRunTriggeredFrom.SCHEDULE
class PluginTriggerMetadata(TriggerMetadata):
"""Plugin trigger metadata"""
type: AppTriggerType = AppTriggerType.TRIGGER_PLUGIN
endpoint_id: str
plugin_unique_identifier: str
provider_id: str
event_name: str
icon_filename: str
icon_dark_filename: str
class PluginTriggerData(TriggerData):
"""Plugin webhook trigger data"""
trigger_type: AppTriggerType = AppTriggerType.TRIGGER_PLUGIN
trigger_from: WorkflowRunTriggeredFrom = WorkflowRunTriggeredFrom.PLUGIN
plugin_id: str
endpoint_id: str
class PluginTriggerDispatchData(BaseModel):
"""Plugin trigger dispatch data for Celery tasks"""
user_id: str
tenant_id: str
endpoint_id: str
provider_id: str
subscription_id: str
timestamp: int
events: list[str]
request_id: str
class WorkflowTaskData(BaseModel):
"""Lightweight data structure for Celery workflow tasks"""
workflow_trigger_log_id: str # Primary tracking ID - all other data can be fetched from DB
model_config = ConfigDict(arbitrary_types_allowed=True)
class WorkflowResumeTaskData(BaseModel):
"""Payload for workflow resumption tasks."""
workflow_run_id: str
class AsyncTriggerExecutionResult(BaseModel):
"""Result from async trigger-based workflow execution"""
execution_id: str
status: AsyncTriggerStatus
result: Mapping[str, Any] | None = None
error: str | None = None
elapsed_time: float | None = None
total_tokens: int | None = None
model_config = ConfigDict(use_enum_values=True)
class AsyncTriggerResponse(BaseModel):
"""Response from triggering an async workflow"""
workflow_trigger_log_id: str
task_id: str
status: str
queue: str
model_config = ConfigDict(use_enum_values=True)
class TriggerLogResponse(BaseModel):
"""Response model for trigger log data"""
id: str
tenant_id: str
app_id: str
workflow_id: str
trigger_type: WorkflowRunTriggeredFrom
status: str
queue_name: str
retry_count: int
celery_task_id: str | None = None
workflow_run_id: str | None = None
error: str | None = None
outputs: str | None = None
elapsed_time: float | None = None
total_tokens: int | None = None
created_at: str | None = None
triggered_at: str | None = None
finished_at: str | None = None
model_config = ConfigDict(use_enum_values=True)
class WorkflowScheduleCFSPlanEntity(BaseModel):
"""
CFS plan entity.
Ensure each workflow run inside Dify is associated with a CFS(Completely Fair Scheduler) plan.
"""
class Strategy(StrEnum):
"""
CFS plan strategy.
"""
TimeSlice = "time-slice" # time-slice based plan
Nop = "nop" # no plan, just run the workflow
schedule_strategy: Strategy
granularity: int = Field(default=-1) # -1 means infinite
@@ -0,0 +1,330 @@
"""Problems in a workflow graph that would fail at import, in the editor or at run time, each located."""
from __future__ import annotations
import logging
from collections.abc import Callable, Iterable, Mapping, Sequence
from dataclasses import dataclass, field
from enum import StrEnum
from typing import Any, Final
from pydantic import ValidationError
from sqlalchemy.orm import Session
from core.trigger.constants import TRIGGER_NODE_TYPES
from core.workflow.human_input_adapter import adapt_node_config_for_graph
from core.workflow.node_factory import resolve_workflow_node_class
from core.workflow.variable_prefixes import ROOT_VARIABLE_NODE_IDS
from graphon.entities.graph_config import NodeConfigDictAdapter
from graphon.enums import BuiltinNodeTypes
from graphon.nodes.base.node import Node
from graphon.variables import VariableBase
from models import AppMode
from services.workflow.node_defaults import complete_sub_models, editor_defaults, node_data_type
from services.workflow_service import WorkflowService
from services.workflow_variable_reference_validator import SOURCE_HANDLE, branch_handles
logger = logging.getLogger(__name__)
GRAPH_MODES: Final = frozenset({AppMode.WORKFLOW, AppMode.ADVANCED_CHAT})
MESSAGE_MODES: Final = frozenset({AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.COMPLETION})
CONTAINER_NODE_TYPES: Final = frozenset({BuiltinNodeTypes.ITERATION, BuiltinNodeTypes.LOOP})
_CANVAS_NOTE_TYPE: Final = "custom-note"
class IssueSeverity(StrEnum):
ERROR = "error"
WARNING = "warning"
class IssueCode(StrEnum):
GRAPH_INVALID = "graph_invalid"
MODE_INCOMPATIBLE = "mode_incompatible"
UNKNOWN_NODE_TYPE = "unknown_node_type"
NODE_DATA_INVALID = "node_data_invalid"
EDGE_ENDPOINT_MISSING = "edge_endpoint_missing"
BRANCH_HANDLE_INVALID = "branch_handle_invalid"
REFERENCE_MISSING = "reference_missing"
CONTAINER_START_MISSING = "container_start_missing"
OUTPUT_NAME_DUPLICATE = "output_name_duplicate"
BRANCH_UNCONNECTED = "branch_unconnected"
RESOURCE_UNAVAILABLE = "resource_unavailable"
@property
def severity(self) -> IssueSeverity:
return IssueSeverity.WARNING if self in _WARNING_CODES else IssueSeverity.ERROR
_WARNING_CODES: Final = frozenset({IssueCode.BRANCH_UNCONNECTED, IssueCode.RESOURCE_UNAVAILABLE})
@dataclass(frozen=True)
class GraphIssue:
code: IssueCode
message: str
node_id: str | None = None
loc: tuple[str | int, ...] = field(default_factory=tuple)
ResourceCheck = Callable[[Mapping[str, Any]], str | None]
def refused_node_types(mode: AppMode) -> frozenset[str]:
if mode == AppMode.ADVANCED_CHAT:
return frozenset({BuiltinNodeTypes.END, *TRIGGER_NODE_TYPES})
return frozenset({BuiltinNodeTypes.ANSWER})
def credential_check(workspace_id: str, environment: Mapping[str, VariableBase], session: Session) -> ResourceCheck:
"""A `ResourceCheck` reporting what publish's credential check would refuse for one node."""
service = WorkflowService()
def check(node: Mapping[str, Any]) -> str | None:
try:
service.validate_node_credentials(workspace_id, node, environment, session=session)
except ValueError as error:
return str(error)
return None
return check
@dataclass(frozen=True)
class _Node:
"""One node read once: its data with editor defaults filled, what reading it found wrong,
and the variables it references (empty when its data could not be read)."""
id: str
data: Mapping[str, Any]
issues: tuple[GraphIssue, ...] = ()
references: Mapping[str, Sequence[str]] = field(default_factory=dict)
@property
def type(self) -> str:
return str(self.data.get("type", ""))
@dataclass(frozen=True)
class _Graph:
nodes: Mapping[str, _Node]
edges: Sequence[Mapping[str, Any]]
mode: AppMode
def _data_loc(node_id: str, *path: str | int) -> tuple[str | int, ...]:
return ("nodes", node_id, "data", *path)
def _references(node_class: type[Node], config: Mapping[str, Any]) -> Mapping[str, Sequence[str]]:
"""The node's own references; a container's children are read on their own, so it gets a graph of itself."""
return node_class.extract_variable_selector_to_variable_mapping(
graph_config={"nodes": [config], "edges": []}, config=NodeConfigDictAdapter.validate_python(config)
)
def _read_known_node(node_id: str, raw_data: Mapping[str, Any], node_class: type[Node]) -> _Node:
data = editor_defaults(raw_data)
model = node_data_type(node_class.node_type, node_class)
config = adapt_node_config_for_graph({"id": node_id, "data": data})
try:
model.model_validate(config["data"])
except ValidationError as error:
problems = error.errors(include_url=False, include_input=False, include_context=False)
issues = tuple(
GraphIssue(IssueCode.NODE_DATA_INVALID, p["msg"], node_id, _data_loc(node_id, *p["loc"])) for p in problems
)
return _Node(node_id, data, issues)
return _Node(node_id, complete_sub_models(model, data), references=_references(node_class, config))
def _read_node(node_id: str, raw: Mapping[str, Any]) -> _Node:
raw_data: Mapping[str, Any] = raw["data"]
node_type = str(raw_data.get("type", ""))
try:
node_class = resolve_workflow_node_class(
node_type=node_type, node_version=str(raw_data.get("version", "1")), node_data=raw_data
)
except ValueError:
issue = GraphIssue(
IssueCode.UNKNOWN_NODE_TYPE, f"Unknown node type {node_type!r}", node_id, _data_loc(node_id, "type")
)
return _Node(node_id, raw_data, (issue,))
try:
return _read_known_node(node_id, raw_data, node_class)
except Exception as error:
# A validator or reference reader can raise other errors on bad or newer data; report this node, check the rest.
logger.warning("node %s data could not be read", node_id, exc_info=True)
message = f"The node data could not be read ({type(error).__name__}: {error})"
return _Node(
node_id, raw_data, (GraphIssue(IssueCode.NODE_DATA_INVALID, message, node_id, _data_loc(node_id)),)
)
def _mode_issues(graph: _Graph) -> Iterable[GraphIssue]:
refused = refused_node_types(graph.mode)
for node in graph.nodes.values():
if node.type in refused:
yield GraphIssue(
IssueCode.MODE_INCOMPATIBLE,
f"A {graph.mode} app can't hold a {node.type} node",
node.id,
_data_loc(node.id, "type"),
)
def _node_issues(graph: _Graph) -> Iterable[GraphIssue]:
for node in graph.nodes.values():
yield from node.issues
start = node.data.get("start_node_id")
if node.type in CONTAINER_NODE_TYPES and start and start not in graph.nodes:
yield GraphIssue(
IssueCode.CONTAINER_START_MISSING,
f"start_node_id {start!r} is not in the graph",
node.id,
_data_loc(node.id, "start_node_id"),
)
def _edge_issues(graph: _Graph) -> Iterable[GraphIssue]:
for index, edge in enumerate(graph.edges):
edge_id = str(edge.get("id", index))
for end in ("source", "target"):
if edge.get(end) not in graph.nodes:
yield GraphIssue(
IssueCode.EDGE_ENDPOINT_MISSING,
f"Edge {end} {edge.get(end)!r} is not a node",
None,
("edges", edge_id, end),
)
source = graph.nodes.get(str(edge.get("source")))
valid = branch_handles(source.data) if source else None
handle = str(edge.get("sourceHandle", SOURCE_HANDLE))
if valid is not None and handle not in valid:
yield GraphIssue(
IssueCode.BRANCH_HANDLE_INVALID,
f"sourceHandle {handle!r} is not a branch of {edge.get('source')!r}; use one of {sorted(valid)}",
str(edge.get("source")),
("edges", edge_id, "sourceHandle"),
)
def _unconnected_branch_issues(graph: _Graph) -> Iterable[GraphIssue]:
for node in graph.nodes.values():
handles = branch_handles(node.data)
if handles is None:
continue
used = {str(e.get("sourceHandle", SOURCE_HANDLE)) for e in graph.edges if e.get("source") == node.id}
for handle in sorted(handles - used):
yield GraphIssue(
IssueCode.BRANCH_UNCONNECTED,
f"Branch {handle!r} of {node.id!r} has no edge; a run that takes it stops there",
node.id,
("nodes", node.id),
)
def _output_name_issues(graph: _Graph) -> Iterable[GraphIssue]:
"""End nodes share one set of workflow outputs, so the console refuses to publish a name used twice."""
places: dict[str, list[tuple[str, int]]] = {}
for node in graph.nodes.values():
outputs = node.data.get("outputs")
if node.type != BuiltinNodeTypes.END or not isinstance(outputs, list):
continue
for index, output in enumerate(outputs):
name = str(output.get("variable") or "").strip() if isinstance(output, Mapping) else ""
if name:
places.setdefault(name, []).append((node.id, index))
for name, found in places.items():
if len(found) < 2:
continue
for node_id, index in found:
yield GraphIssue(
IssueCode.OUTPUT_NAME_DUPLICATE,
f"Output {name!r} is used by more than one End node output; output names must be unique",
node_id,
_data_loc(node_id, "outputs", index, "variable"),
)
def _reference_issues(graph: _Graph) -> Iterable[GraphIssue]:
for node in graph.nodes.values():
for key, selector in node.references.items():
if selector and selector[0] not in ROOT_VARIABLE_NODE_IDS and selector[0] not in graph.nodes:
yield GraphIssue(
IssueCode.REFERENCE_MISSING,
f"{key} reads node {selector[0]!r}, which is not in the graph",
node.id,
_data_loc(node.id),
)
_CHECKS: Final[tuple[Callable[[_Graph], Iterable[GraphIssue]], ...]] = (
_mode_issues,
_node_issues,
_edge_issues,
_unconnected_branch_issues,
_output_name_issues,
_reference_issues,
)
def _shape_issues(graph: Mapping[str, Any]) -> list[GraphIssue]:
"""The shape every later check reads: a list of nodes, each a mapping with a string id and mapping data,
and a list of edge mappings."""
nodes = graph.get("nodes")
if not isinstance(nodes, list):
return [GraphIssue(IssueCode.GRAPH_INVALID, "graph.nodes must be a list", None, ("nodes",))]
issues: list[GraphIssue] = []
seen_ids: set[str] = set()
for index, node in enumerate(nodes):
node_id = node.get("id") if isinstance(node, Mapping) else None
if not isinstance(node, Mapping):
issues.append(GraphIssue(IssueCode.GRAPH_INVALID, "A node must be a mapping", None, ("nodes", index)))
elif not isinstance(node_id, str) or not node_id:
issues.append(GraphIssue(IssueCode.GRAPH_INVALID, "A node needs a string id", None, ("nodes", index, "id")))
elif node_id in seen_ids:
issues.append(
GraphIssue(
IssueCode.GRAPH_INVALID,
f"Node id {node_id!r} is used by more than one node; node ids must be unique",
node_id,
("nodes", index, "id"),
)
)
elif not isinstance(node.get("data"), Mapping):
issues.append(
GraphIssue(IssueCode.GRAPH_INVALID, "The node data must be a mapping", node_id, _data_loc(node_id))
)
if isinstance(node_id, str) and node_id:
seen_ids.add(node_id)
edges = graph.get("edges")
if edges is None:
return issues
if not isinstance(edges, list):
return [*issues, GraphIssue(IssueCode.GRAPH_INVALID, "graph.edges must be a list", None, ("edges",))]
issues.extend(
GraphIssue(IssueCode.GRAPH_INVALID, "An edge must be a mapping", None, ("edges", index))
for index, edge in enumerate(edges)
if not isinstance(edge, Mapping)
)
return issues
def check_graph(graph: Mapping[str, Any], *, mode: AppMode, resources: ResourceCheck | None = None) -> list[GraphIssue]:
shape_issues = _shape_issues(graph)
if shape_issues:
return shape_issues
raw_nodes = {str(n["id"]): n for n in graph["nodes"] if n.get("type") != _CANVAS_NOTE_TYPE}
view = _Graph(
nodes={node_id: _read_node(node_id, raw) for node_id, raw in raw_nodes.items()},
edges=graph.get("edges") or [],
mode=mode,
)
issues = [issue for check in _CHECKS for issue in check(view)]
if resources is not None:
for node_id, raw in raw_nodes.items():
problem = resources(raw)
if problem:
issues.append(GraphIssue(IssueCode.RESOURCE_UNAVAILABLE, problem, node_id, ("nodes", node_id)))
return issues
@@ -0,0 +1,131 @@
"""What changed between two versions of a workflow, ignoring editor layout and import defaults."""
from __future__ import annotations
import json
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Any, Final
from pydantic import BaseModel, Field
from core.db.session_factory import session_factory
from graphon.variables import SecretVariable, VariableBase
from libs import helper
from models.model import App
from models.workflow import Workflow
from services.workflow.node_defaults import fill_node_data
_DATA_UI_KEYS: Final = frozenset({"selected", "_connectedSourceHandleIds", "_connectedTargetHandleIds"})
@dataclass(frozen=True)
class WorkflowSnapshot:
graph: Mapping[str, Any]
features: Mapping[str, Any]
environment_variable_names: frozenset[str]
class NodeChange(BaseModel):
id: str
type: str
title: str
fields: list[str]
class WorkflowDiff(BaseModel):
published: bool = Field(description="false when the app was never published; then everything counts as added")
nodes_added: list[NodeChange]
nodes_removed: list[NodeChange]
nodes_changed: list[NodeChange]
edges_added: list[str]
edges_removed: list[str]
features_changed: bool
env_added: list[str]
env_removed: list[str]
def _nodes(graph: Mapping[str, Any]) -> dict[str, Mapping[str, Any]]:
return {str(n.get("id")): n for n in graph.get("nodes") or [] if isinstance(n, Mapping)}
def _data(node: Mapping[str, Any]) -> dict[str, Any]:
"""Node data with import defaults filled, so a default added only by an import is not a change."""
data = node.get("data")
if not isinstance(data, Mapping):
return {}
return {k: v for k, v in fill_node_data(data).items() if k not in _DATA_UI_KEYS}
def _change(node: Mapping[str, Any], fields: list[str] | None = None) -> NodeChange:
data = _data(node)
return NodeChange(
id=str(node.get("id")), type=str(data.get("type", "")), title=str(data.get("title", "")), fields=fields or []
)
def _edges(graph: Mapping[str, Any]) -> set[str]:
return {
f"{e.get('source')} → {e.get('target')} ({e.get('sourceHandle', 'source')})"
for e in graph.get("edges") or []
if isinstance(e, Mapping)
}
def _runnable(graph: Mapping[str, Any]) -> dict[str, Any]:
"""The graph as a run sees it: the editor's layout and selection state don't count."""
return {"nodes": {i: _data(n) for i, n in _nodes(graph).items()}, "edges": sorted(_edges(graph))}
def same_graph(a: Mapping[str, Any], b: Mapping[str, Any]) -> bool:
return _runnable(a) == _runnable(b)
def _variable(variable: VariableBase) -> dict[str, Any]:
return variable.model_dump(mode="json", exclude={"value"} if isinstance(variable, SecretVariable) else None)
def draft_token(draft: Workflow) -> str:
"""Changes when anything a DSL import replaces changes; editor layout and secret values don't count."""
content = {
"graph": _runnable(draft.graph_dict),
"features": draft.features_dict,
"environment_variables": [_variable(v) for v in draft.environment_variables],
"conversation_variables": [_variable(v) for v in draft.conversation_variables],
}
return helper.generate_text_hash(json.dumps(content, sort_keys=True))
def current_draft_token(tenant_id: str, app_id: str) -> str | None:
"""The app draft's `draft_token`, read in its own session; None when the app has no draft."""
from services.workflow_service import WorkflowService
with session_factory.create_session() as session:
app = session.get(App, app_id)
if app is None or app.tenant_id != tenant_id:
return None
draft = WorkflowService.get_draft_workflow(app, session=session)
return draft_token(draft) if draft else None
def diff_workflows(published: WorkflowSnapshot | None, draft: WorkflowSnapshot) -> WorkflowDiff:
base = published or WorkflowSnapshot(graph={}, features={}, environment_variable_names=frozenset())
old, new = _nodes(base.graph), _nodes(draft.graph)
changed = []
for node_id in sorted(old.keys() & new.keys()):
before, after = _data(old[node_id]), _data(new[node_id])
fields = sorted(k for k in before.keys() | after.keys() if before.get(k) != after.get(k))
if fields:
changed.append(_change(new[node_id], fields))
old_edges, new_edges = _edges(base.graph), _edges(draft.graph)
return WorkflowDiff(
published=published is not None,
nodes_added=[_change(new[i]) for i in sorted(new.keys() - old.keys())],
nodes_removed=[_change(old[i]) for i in sorted(old.keys() - new.keys())],
nodes_changed=changed,
edges_added=sorted(new_edges - old_edges),
edges_removed=sorted(old_edges - new_edges),
features_changed=dict(base.features) != dict(draft.features) if published else False,
env_added=sorted(draft.environment_variable_names - base.environment_variable_names),
env_removed=sorted(base.environment_variable_names - draft.environment_variable_names),
)
@@ -0,0 +1,198 @@
"""Inspector pub/sub fanout for live workflow run updates (Stage 4 §8.5).
The Node Output Inspector exposes a Server-Sent Events stream alongside its
three REST endpoints so the frontend can render per-output progress without
DB polling. This module owns the redis pub/sub channel that connects the two
sides:
* :func:`publish_node_changed` / :func:`publish_workflow_completed` —
invoked by :class:`services.workflow_persistence.WorkflowPersistenceLayer`
at the very end of each handler, after the DB write has already
succeeded. Publish failures are swallowed so the engine never trips on a
flaky redis connection.
* :func:`subscribe` — async iterator the SSE endpoint consumes.
Channel layout
--------------
``dify:inspector:workflow_run:{workflow_run_id}``
One channel per workflow run; the SSE endpoint subscribes for the lifetime of
the run and unsubscribes on the terminal event. Multiple clients can attach
to the same run safely — redis pub/sub fans every message out to every
listener.
The message envelope intentionally carries only the *delta* needed to invalidate
a slice of the inspector view; the SSE handler re-reads the canonical
``WorkflowNodeExecutionModel`` row from the DB so we never serialize stale
state across the wire. This means messages stay tiny (~150 bytes) and the
inspector view stays consistent even if a publisher races persistence.
Decision D-5: the on-wire SSE envelope ``{event, data, id}`` is shared with
the babysit chat stream; this module only emits the *internal* pub/sub
message — the SSE controller turns it into the public envelope.
"""
from __future__ import annotations
import json
import logging
from collections.abc import Iterator
from dataclasses import asdict, dataclass
from typing import Final, Literal
from pydantic import BaseModel, ConfigDict, Field, ValidationError
from extensions.ext_redis import redis_client
logger = logging.getLogger(__name__)
# ──────────────────────────────────────────────────────────────────────────────
# Channel naming
# ──────────────────────────────────────────────────────────────────────────────
_CHANNEL_PREFIX: Final = "dify:inspector:workflow_run"
def channel_for(workflow_run_id: str) -> str:
"""Return the pub/sub channel name for ``workflow_run_id``.
Kept as a module-level helper so tests can pin the channel without
reaching into the publish/subscribe code paths.
"""
return f"{_CHANNEL_PREFIX}:{workflow_run_id}"
# ──────────────────────────────────────────────────────────────────────────────
# Message envelope
# ──────────────────────────────────────────────────────────────────────────────
#: Tags discriminating the wire-level message kinds. Kept narrow so the SSE
#: controller can pattern-match exhaustively.
InspectorMessageKind = Literal["node_changed", "workflow_completed"]
class _InspectorMessagePayload(BaseModel):
"""Pydantic schema for the on-wire pub/sub payload (strict, no type coercion)."""
model_config = ConfigDict(extra="ignore", strict=True)
kind: InspectorMessageKind
workflow_run_id: str = Field(min_length=1)
node_id: str | None = None
status: str | None = None
@dataclass(frozen=True, slots=True)
class InspectorMessage:
"""Minimal delta carried across the pub/sub channel.
``node_id`` is set only for ``node_changed`` messages; ``status`` is the
coarse string status straight from the persistence layer (``"running"`` /
``"succeeded"`` / ``"failed"`` for nodes, plus ``"succeeded"`` /
``"failed"`` / ``"partial_succeeded"`` / ``"stopped"`` for workflow runs).
"""
kind: InspectorMessageKind
workflow_run_id: str
node_id: str | None = None
status: str | None = None
def to_json(self) -> str:
return json.dumps(asdict(self), ensure_ascii=False)
@classmethod
def from_json(cls, blob: str) -> InspectorMessage | None:
"""Decode a payload, returning ``None`` for any shape we can't trust."""
try:
payload = _InspectorMessagePayload.model_validate_json(blob)
except ValidationError:
return None
return cls(
kind=payload.kind,
workflow_run_id=payload.workflow_run_id,
node_id=payload.node_id,
status=payload.status,
)
# ──────────────────────────────────────────────────────────────────────────────
# Publisher (called from the persistence layer)
# ──────────────────────────────────────────────────────────────────────────────
def _publish(message: InspectorMessage) -> None:
"""Best-effort fire-and-forget publish.
Persistence runs inside the workflow engine thread; we never want a redis
glitch to break the workflow. Any exception is logged at debug level so
operators still see them when they grep, but the engine keeps running.
"""
try:
redis_client.publish(channel_for(message.workflow_run_id), message.to_json())
except Exception:
logger.debug("InspectorEventPublisher: publish failed for %s", message.workflow_run_id, exc_info=True)
def publish_node_changed(*, workflow_run_id: str, node_id: str, status: str) -> None:
"""Announce that one node's execution row just changed.
The SSE handler will recompute the node slice from the DB on receipt.
"""
_publish(InspectorMessage(kind="node_changed", workflow_run_id=workflow_run_id, node_id=node_id, status=status))
def publish_workflow_completed(*, workflow_run_id: str, status: str) -> None:
"""Announce that the workflow run reached a terminal state.
The SSE handler emits one last envelope and disconnects.
"""
_publish(InspectorMessage(kind="workflow_completed", workflow_run_id=workflow_run_id, status=status))
# ──────────────────────────────────────────────────────────────────────────────
# Subscriber (consumed by the SSE controller)
# ──────────────────────────────────────────────────────────────────────────────
def subscribe(workflow_run_id: str, *, timeout_seconds: float = 1.0) -> Iterator[InspectorMessage]:
"""Yield ``InspectorMessage`` instances until the consumer abandons us.
The loop polls redis with ``timeout_seconds`` so the SSE handler can
interleave keepalive heartbeats. Yields ``None`` on timeout so the caller
can decide whether to keep blocking; malformed payloads are silently
skipped.
The pub/sub connection is closed when the iterator is garbage-collected
(the wrapping ``finally`` releases it as soon as the SSE handler exits).
"""
pubsub = redis_client.pubsub()
pubsub.subscribe(channel_for(workflow_run_id))
try:
while True:
raw = pubsub.get_message(ignore_subscribe_messages=True, timeout=timeout_seconds)
if raw is None:
# Surface a heartbeat tick — caller can keep-alive or check
# disconnection without blocking redis any longer.
yield InspectorMessage(kind="node_changed", workflow_run_id=workflow_run_id, node_id=None, status=None)
continue
data = raw.get("data") if isinstance(raw, dict) else None
if isinstance(data, bytes):
data = data.decode("utf-8", errors="replace")
if not isinstance(data, str):
continue
message = InspectorMessage.from_json(data)
if message is None:
continue
yield message
finally:
try:
pubsub.unsubscribe(channel_for(workflow_run_id))
pubsub.close()
except Exception:
logger.debug(
"InspectorEventPublisher: pubsub teardown failed for %s",
workflow_run_id,
exc_info=True,
)
@@ -0,0 +1,155 @@
"""Defaults every node needs so the workflow editor can open it; shared by DSL import and describe.node_type."""
from __future__ import annotations
import copy
from collections.abc import Callable, Iterable, Mapping, Sequence
from dataclasses import dataclass
from types import UnionType
from typing import Any, Final, Union, get_args, get_origin
from pydantic import BaseModel, Field
from core.workflow.human_input_adapter import DeliveryChannelConfig
from core.workflow.node_factory import resolve_workflow_node_class
from core.workflow.nodes.human_input.entities import HumanInputNodeData
from graphon.enums import BuiltinNodeTypes
from graphon.nodes.base.node import Node
# Only empty containers: values the editor reads without a guard and that mean "none yet".
EDITOR_EMPTY: Final[Mapping[str, Mapping[str, object]]] = {
BuiltinNodeTypes.START: {"variables": []},
BuiltinNodeTypes.END: {"outputs": []},
BuiltinNodeTypes.IF_ELSE: {"cases": []},
BuiltinNodeTypes.CODE: {"variables": []},
BuiltinNodeTypes.TEMPLATE_TRANSFORM: {"variables": []},
BuiltinNodeTypes.HTTP_REQUEST: {"body": {"type": "none", "data": []}},
BuiltinNodeTypes.TOOL: {"tool_parameters": {}, "tool_configurations": {}},
BuiltinNodeTypes.LOOP: {"break_conditions": []},
BuiltinNodeTypes.HUMAN_INPUT: {"delivery_methods": [], "user_actions": [], "inputs": []},
BuiltinNodeTypes.DATASOURCE: {"datasource_parameters": {}},
}
class HumanInputNodeSpec(HumanInputNodeData):
"""Human-input data as the editor saves it; the run parses delivery methods beside the node data."""
delivery_methods: list[DeliveryChannelConfig] = Field(default_factory=list)
# Graphon keeps these node data opaque; Dify parses them with its own model when the node runs.
_DIFY_NODE_DATA: Final[Mapping[str, type[BaseModel]]] = {
BuiltinNodeTypes.HUMAN_INPUT: HumanInputNodeSpec,
}
def node_data_type(node_type: str, node_class: type[Node]) -> type[BaseModel]:
return _DIFY_NODE_DATA.get(node_type) or node_class._get_node_data_type()
_LEGACY_IF_CASE_ID: Final = "true"
_DEFAULT_LOGICAL_OPERATOR: Final = "and"
def _cases_from_legacy_conditions(data: Mapping[str, Any]) -> Mapping[str, Any]:
"""Old if-else nodes keep one condition list and branch on "true"; the editor reads them as one case."""
if "cases" in data or "conditions" not in data:
return data
case = {
"case_id": _LEGACY_IF_CASE_ID,
"logical_operator": data.get("logical_operator") or _DEFAULT_LOGICAL_OPERATOR,
"conditions": data["conditions"],
}
return {**data, "cases": [case]}
# Rewrites of old shapes to the one the editor reads, applied before the empty defaults.
_NORMALIZERS: Final[Mapping[str, Callable[[Mapping[str, Any]], Mapping[str, Any]]]] = {
BuiltinNodeTypes.IF_ELSE: _cases_from_legacy_conditions,
}
def _single_model(candidates: Iterable[object]) -> type[BaseModel] | None:
models = [c for c in candidates if isinstance(c, type) and issubclass(c, BaseModel)]
return models[0] if len(models) == 1 else None
@dataclass(frozen=True)
class _SubModels:
"""The model of a field's mapping value (direct or optional), and of each mapping in its list value."""
value: type[BaseModel] | None
item: type[BaseModel] | None
def _sub_models(annotation: object) -> _SubModels:
arms = get_args(annotation) if get_origin(annotation) in (Union, UnionType) else (annotation,)
is_sequence = get_origin(annotation) in (list, Sequence)
return _SubModels(value=_single_model(arms), item=_single_model(get_args(annotation)) if is_sequence else None)
def _fill(model: type[BaseModel], value: Mapping[str, Any]) -> dict[str, Any]:
# graph_diff fills stored graphs that were never validated, and some graphon validators raise KeyError and the
# like on bad data instead of a ValidationError; a sub-object its model refuses is kept as it is.
try:
filled = model.model_validate(value).model_dump(mode="json")
except Exception:
return dict(value)
return {**filled, **value}
def complete_sub_models(model: type[BaseModel], data: Mapping[str, Any]) -> dict[str, Any]:
"""Add the keys each sub-object's model defaults; keys already present win."""
fields = model.model_fields
completed = dict(data)
for key, value in data.items():
field = fields.get(key)
if field is None:
continue
models = _sub_models(field.annotation)
if isinstance(value, Mapping) and models.value:
completed[key] = _fill(models.value, value)
elif isinstance(value, list) and models.item:
item_model = models.item
completed[key] = [_fill(item_model, item) if isinstance(item, Mapping) else item for item in value]
return completed
def editor_defaults(data: Mapping[str, Any]) -> dict[str, Any]:
"""Old shapes rewritten and the empty containers the editor reads added; keys already present win."""
node_type = data.get("type")
if not isinstance(node_type, str):
return dict(data)
normalize = _NORMALIZERS.get(node_type)
if normalize is not None:
data = normalize(data)
return {**copy.deepcopy(dict(EDITOR_EMPTY.get(node_type, {}))), **data}
def fill_node_data(data: Mapping[str, Any]) -> dict[str, Any]:
filled = editor_defaults(data)
node_type = filled.get("type")
if not isinstance(node_type, str):
return filled
try:
node_class = resolve_workflow_node_class(
node_type=node_type, node_version=str(filled.get("version", "1")), node_data=filled
)
except ValueError:
return filled
return complete_sub_models(node_data_type(node_type, node_class), filled)
def fill_graph(graph: Mapping[str, Any]) -> dict[str, Any]:
nodes = graph.get("nodes")
if not isinstance(nodes, list):
return dict(graph)
return {
**graph,
"nodes": [
{**node, "data": fill_node_data(node["data"])}
if isinstance(node, Mapping) and isinstance(node.get("data"), Mapping)
else node
for node in nodes
],
}
@@ -0,0 +1,802 @@
"""Node Output Inspector service (Stage 4 §8).
PRD §Node Output Inspector renames every workflow "Variable" to a "Node Output"
and re-organizes the panel by **producer node** rather than consumer node. This
service backs the new REST surface
``/apps/{app_id}/workflows/draft/runs/{run_id}/node-outputs[/...]`` with three
read-only views:
* :meth:`snapshot_workflow_run` — every node + its declared outputs + per-output
status, for one debug workflow run.
* :meth:`node_detail` — the same shape filtered down to one node.
* :meth:`output_preview` — full payload for one output, with signed download
URL when the output is a canonical Agent v2 file mapping.
Design constraints baked into this version:
1. **No new tables** (§8.1). Topology comes from ``WorkflowRun.graph`` (the
graph snapshot taken at execution time so the view stays consistent even
if the draft was edited mid-run). Execution facts come from
``WorkflowNodeExecutionModel`` rows already produced by the workflow
runtime.
2. **Draft + published runs** (decision D-1 lifted 2026-05-26). The Inspector
accepts ``WorkflowRunTriggeredFrom.DEBUGGING`` (draft test runs) as well as
``APP_RUN`` / ``WEBHOOK`` / ``SCHEDULE`` / ``PLUGIN`` / ``RAG_PIPELINE_*``
triggers; the underlying graph + executions are uniform across all of them.
Cross-tenant / cross-app rows still 404 via the standard tenant/app scope.
3. **Declared outputs by node kind**:
* Agent v2 nodes resolve their declared list via
:class:`WorkflowAgentBindingResolver` (the binding owns custom declarations;
the system ``text`` output is derived for display).
* Other node kinds don't have a declared-output schema yet; we surface the
keys present in the execution payload as a best-effort list typed
``unknown`` so the panel can still render them.
4. **Per-output status** is derived from the metadata the agent_v2 stack
already records (``output_type_check`` and ``output_check`` blobs); falling
back to the node's overall status when those signals aren't present.
5. **SSE stream** (design §8.5) lives in
:mod:`controllers.console.app.workflow_node_output_inspector` alongside the
REST endpoints. The Inspector and the babysit chat SSE share the
``{event, data, id}`` envelope per decision D-5.
"""
from __future__ import annotations
import json
import logging
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime
from enum import StrEnum
from typing import Any
from pydantic import BaseModel, ConfigDict, Field
from sqlalchemy import select
from sqlalchemy.orm import Session
from core.app.file_access import DatabaseFileAccessController
from core.workflow.nodes.agent_v2.binding_resolver import (
WorkflowAgentBindingError,
WorkflowAgentBindingResolver,
)
from domain.workflow.agent_node import is_dify_agent_node_data
from factories.file_factory.builders import build_from_mapping
from graphon.enums import WorkflowExecutionStatus, WorkflowNodeExecutionStatus
from graphon.file import helpers as file_helpers
from models import App
from models.agent_config_entities import DeclaredOutputConfig, DeclaredOutputType, effective_declared_outputs
from models.workflow import WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom, WorkflowRun
logger = logging.getLogger(__name__)
# ──────────────────────────────────────────────────────────────────────────────
# Public dataclasses / enums (Pydantic — these go straight on the wire)
# ──────────────────────────────────────────────────────────────────────────────
class NodeOutputStatus(StrEnum):
"""Lifecycle status of a single declared output within a run."""
PENDING = "pending" # node not started
RUNNING = "running" # node running, output not ready yet
READY = "ready"
TYPE_CHECK_FAILED = "type_check_failed"
OUTPUT_CHECK_FAILED = "output_check_failed"
NOT_PRODUCED = "not_produced" # node succeeded but did not produce this declared output
FAILED = "failed" # node itself failed/exception/stopped
class NodeStatus(StrEnum):
"""Coarse node-level status used by Inspector to pick a banner."""
IDLE = "idle"
RUNNING = "running"
READY = "ready"
FAILED = "failed"
class CheckResultView(BaseModel):
"""``type_check`` / ``output_check`` per-output summary block."""
model_config = ConfigDict(extra="forbid")
passed: bool
reason: str | None = None
class NodeOutputView(BaseModel):
model_config = ConfigDict(extra="forbid")
name: str
type: DeclaredOutputType | None = None
status: NodeOutputStatus
value_preview: Any = Field(default=None)
type_check: CheckResultView | None = None
output_check: CheckResultView | None = None
retried: int = 0
class NodeOutputsView(BaseModel):
model_config = ConfigDict(extra="forbid")
node_id: str
node_kind: str
node_display_name: str
node_status: NodeStatus
node_started_at: datetime | None = None
node_completed_at: datetime | None = None
outputs: list[NodeOutputView] = Field(default_factory=list)
class WorkflowRunSnapshotView(BaseModel):
model_config = ConfigDict(extra="forbid")
workflow_run_id: str
workflow_run_status: WorkflowExecutionStatus
node_outputs: list[NodeOutputsView] = Field(default_factory=list)
class OutputPreviewView(BaseModel):
model_config = ConfigDict(extra="forbid")
node_id: str
output_name: str
type: DeclaredOutputType | None = None
status: NodeOutputStatus
value: Any = Field(default=None)
class NodeOutputInspectorError(Exception):
"""Raised when a request cannot be served (404-level conditions)."""
def __init__(self, code: str, message: str) -> None:
super().__init__(message)
self.code = code
# ──────────────────────────────────────────────────────────────────────────────
# Internal helpers — declared outputs per node
# ──────────────────────────────────────────────────────────────────────────────
@dataclass(frozen=True, slots=True)
class _ResolvedDeclaration:
"""Declared output the Inspector should expose, with a normalized type.
``inferred`` is ``True`` when the node kind has no declared-output schema
(we derived the list from the execution payload). The frontend can use
this to dim the type column.
"""
name: str
declared_type: DeclaredOutputType | None
array_item_type: DeclaredOutputType | None
inferred: bool
def _is_agent_v2_node(node: Mapping[str, Any]) -> bool:
"""Return whether a graph node explicitly identifies the new Dify Agent node."""
data = node.get("data") or {}
if not isinstance(data, Mapping):
return False
return is_dify_agent_node_data(data)
def _graph_nodes(workflow_run: WorkflowRun) -> list[Mapping[str, Any]]:
"""Parse ``WorkflowRun.graph`` (LongText JSON) into a node list.
The graph blob may be missing / unparseable for very old runs; we treat
that as "no nodes" rather than failing the Inspector, so the panel still
renders an empty state.
"""
if not workflow_run.graph:
return []
try:
parsed = json.loads(workflow_run.graph)
except (json.JSONDecodeError, TypeError):
logger.warning("NodeOutputInspector: workflow_run %s has unparseable graph blob", workflow_run.id)
return []
if not isinstance(parsed, Mapping):
return []
nodes = parsed.get("nodes")
if not isinstance(nodes, list):
return []
return [n for n in nodes if isinstance(n, Mapping) and "id" in n]
# ──────────────────────────────────────────────────────────────────────────────
# Internal helpers — per-output status derivation
# ──────────────────────────────────────────────────────────────────────────────
def _decode_json_blob(blob: str | None) -> Mapping[str, Any] | None:
if not blob:
return None
try:
decoded = json.loads(blob)
except (json.JSONDecodeError, TypeError):
return None
if not isinstance(decoded, Mapping):
return None
return decoded
def _node_status_for(execution: WorkflowNodeExecutionModel | None) -> NodeStatus:
if execution is None:
return NodeStatus.IDLE
if execution.status == WorkflowNodeExecutionStatus.RUNNING:
return NodeStatus.RUNNING
if execution.status == WorkflowNodeExecutionStatus.SUCCEEDED:
return NodeStatus.READY
return NodeStatus.FAILED
def _type_check_by_name(metadata: Mapping[str, Any] | None) -> dict[str, Mapping[str, Any]]:
if not metadata:
return {}
block = metadata.get("output_type_check")
if not isinstance(block, Mapping):
return {}
results = block.get("results") or []
if not isinstance(results, list):
return {}
indexed: dict[str, Mapping[str, Any]] = {}
for r in results:
if isinstance(r, Mapping) and isinstance(r.get("name"), str):
indexed[r["name"]] = r
return indexed
def _output_check_by_name(metadata: Mapping[str, Any] | None) -> dict[str, Mapping[str, Any]]:
if not metadata:
return {}
block = metadata.get("output_check")
if not isinstance(block, Mapping):
return {}
results = block.get("results") or []
if not isinstance(results, list):
return {}
indexed: dict[str, Mapping[str, Any]] = {}
for r in results:
if isinstance(r, Mapping) and isinstance(r.get("name"), str):
indexed[r["name"]] = r
return indexed
def _retried_attempt_count(metadata: Mapping[str, Any] | None) -> int:
"""The agent_v2 runtime records the final attempt index in metadata.
``attempt`` is 0-indexed so a single execution with no retry has
``attempt == 0`` and a Inspector-friendly ``retried == 0``.
"""
if not metadata:
return 0
attempt = metadata.get("attempt")
if isinstance(attempt, int) and attempt > 0:
return attempt
return 0
# ──────────────────────────────────────────────────────────────────────────────
# Value preview (file refs get signed URLs)
# ──────────────────────────────────────────────────────────────────────────────
_PREVIEW_TEXT_LIMIT = 500
def _looks_like_file_ref(value: Any) -> bool:
"""Return whether ``value`` looks like a canonical Agent v2 file mapping."""
if not isinstance(value, Mapping):
return False
transfer_method = value.get("transfer_method")
if transfer_method == "remote_url":
return isinstance(value.get("url"), str) and bool(value.get("url"))
return isinstance(transfer_method, str) and isinstance(value.get("reference"), str) and bool(value.get("reference"))
def _resolve_preview_url(value: Mapping[str, Any], *, tenant_id: str) -> str | None:
"""Resolve one canonical file mapping into its preview/download URL.
Agent v2 output files now use the same ``transfer_method`` +
``reference``/``url`` mapping contract as the back-proxy download request,
so the Inspector rebuilds a graphon ``File`` through the standard factory
path instead of trying to special-case upload/tool file ids itself.
"""
file = build_from_mapping(
mapping=value,
tenant_id=tenant_id,
access_controller=DatabaseFileAccessController(),
)
return file_helpers.resolve_file_url(file)
def _value_preview(value: Any, *, tenant_id: str, declaration: _ResolvedDeclaration) -> Any:
"""Compact preview suitable for the snapshot endpoint.
Canonical file mappings are augmented with a signed download URL only when
the declared output is ``FILE`` or ``ARRAY[FILE]``. Canonical-looking
mappings nested inside ``OBJECT`` or non-file arrays remain plain JSON so
the Inspector does not introduce a nested file protocol the runtime itself
does not promise. Long strings are truncated; other scalar / dict / list
shapes are returned as-is (the Pydantic layer enforces JSON-safety on
serialization).
"""
if declaration.declared_type == DeclaredOutputType.FILE and _looks_like_file_ref(value):
assert isinstance(value, Mapping)
try:
preview_url = _resolve_preview_url(value, tenant_id=tenant_id)
except Exception:
logger.warning("NodeOutputInspector: signed URL failed for file mapping=%s", value, exc_info=True)
preview_url = None
return {**dict(value), "preview_url": preview_url}
if (
declaration.declared_type == DeclaredOutputType.ARRAY
and declaration.array_item_type == DeclaredOutputType.FILE
and isinstance(value, list)
and all(_looks_like_file_ref(item) for item in value)
):
resolved_items: list[Any] = []
for item in value:
assert isinstance(item, Mapping)
try:
preview_url = _resolve_preview_url(item, tenant_id=tenant_id)
except Exception:
logger.warning("NodeOutputInspector: signed URL failed for file mapping=%s", item, exc_info=True)
preview_url = None
resolved_items.append({**dict(item), "preview_url": preview_url})
return resolved_items
if isinstance(value, str) and len(value) > _PREVIEW_TEXT_LIMIT:
return value[:_PREVIEW_TEXT_LIMIT] + "…"
return value
def _full_value(value: Any, *, tenant_id: str, declaration: _ResolvedDeclaration) -> Any:
"""Same shape as :func:`_value_preview` minus the truncation.
As with preview values, only outputs declared as ``FILE`` or
``ARRAY[FILE]`` get signed URL augmentation; non-file outputs keep their raw
JSON payload unchanged even if it resembles a canonical file mapping.
"""
if declaration.declared_type == DeclaredOutputType.FILE and _looks_like_file_ref(value):
assert isinstance(value, Mapping)
try:
preview_url = _resolve_preview_url(value, tenant_id=tenant_id)
except Exception:
logger.warning("NodeOutputInspector: signed URL failed for file mapping=%s", value, exc_info=True)
preview_url = None
return {**dict(value), "preview_url": preview_url}
if (
declaration.declared_type == DeclaredOutputType.ARRAY
and declaration.array_item_type == DeclaredOutputType.FILE
and isinstance(value, list)
and all(_looks_like_file_ref(item) for item in value)
):
resolved_items: list[Any] = []
for item in value:
assert isinstance(item, Mapping)
try:
preview_url = _resolve_preview_url(item, tenant_id=tenant_id)
except Exception:
logger.warning("NodeOutputInspector: signed URL failed for file mapping=%s", item, exc_info=True)
preview_url = None
resolved_items.append({**dict(item), "preview_url": preview_url})
return resolved_items
return value
# ──────────────────────────────────────────────────────────────────────────────
# Service
# ──────────────────────────────────────────────────────────────────────────────
class NodeOutputInspectorService:
"""Read-only Inspector for draft + published workflow runs.
The service is dependency-light: it holds a single
:class:`WorkflowAgentBindingResolver` so agent v2 nodes can map to their
declared outputs without re-implementing binding lookup. All other I/O
receives an explicit SQLAlchemy session from its caller so transaction
ownership stays at the controller/task boundary.
Tenancy is enforced via ``app_model.tenant_id`` + ``app_model.id`` on
every load — the same scope guard regardless of trigger source.
"""
def __init__(self, binding_resolver: WorkflowAgentBindingResolver | None = None) -> None:
self._binding_resolver = binding_resolver or WorkflowAgentBindingResolver()
# ── public API ────────────────────────────────────────────────────────
def snapshot_workflow_run(
self, *, app_model: App, workflow_run_id: str, session: Session
) -> WorkflowRunSnapshotView:
"""Build the per-node snapshot for one debug workflow run."""
workflow_run, executions = self._load_run_and_executions(
app_model=app_model, workflow_run_id=workflow_run_id, session=session
)
executions_by_node = self._index_executions_by_node(executions)
graph_nodes = _graph_nodes(workflow_run)
node_views: list[NodeOutputsView] = []
for raw_node in graph_nodes:
node_id = str(raw_node["id"])
execution = executions_by_node.get(node_id)
view = self._build_node_view(
tenant_id=app_model.tenant_id,
app_id=app_model.id,
workflow_id=workflow_run.workflow_id,
raw_node=raw_node,
execution=execution,
)
node_views.append(view)
return WorkflowRunSnapshotView(
workflow_run_id=workflow_run.id,
workflow_run_status=workflow_run.status,
node_outputs=node_views,
)
def node_detail(self, *, app_model: App, workflow_run_id: str, node_id: str, session: Session) -> NodeOutputsView:
"""Per-node Inspector entry — returns one ``NodeOutputsView``."""
workflow_run, executions = self._load_run_and_executions(
app_model=app_model, workflow_run_id=workflow_run_id, session=session
)
graph_nodes = _graph_nodes(workflow_run)
raw_node = next((n for n in graph_nodes if str(n.get("id")) == node_id), None)
if raw_node is None:
raise NodeOutputInspectorError(
"node_not_in_workflow_run",
f"Node {node_id!r} does not appear in workflow run {workflow_run_id!r}.",
)
execution = self._index_executions_by_node(executions).get(node_id)
return self._build_node_view(
tenant_id=app_model.tenant_id,
app_id=app_model.id,
workflow_id=workflow_run.workflow_id,
raw_node=raw_node,
execution=execution,
)
def output_preview(
self,
*,
app_model: App,
workflow_run_id: str,
node_id: str,
output_name: str,
session: Session,
) -> OutputPreviewView:
"""Full payload for one declared output (with signed file URL)."""
workflow_run, executions = self._load_run_and_executions(
app_model=app_model, workflow_run_id=workflow_run_id, session=session
)
graph_nodes = _graph_nodes(workflow_run)
raw_node = next((n for n in graph_nodes if str(n.get("id")) == node_id), None)
if raw_node is None:
raise NodeOutputInspectorError(
"node_not_in_workflow_run",
f"Node {node_id!r} does not appear in workflow run {workflow_run_id!r}.",
)
execution = self._index_executions_by_node(executions).get(node_id)
detail = self._build_node_view(
tenant_id=app_model.tenant_id,
app_id=app_model.id,
workflow_id=workflow_run.workflow_id,
raw_node=raw_node,
execution=execution,
)
view = next((o for o in detail.outputs if o.name == output_name), None)
if view is None:
raise NodeOutputInspectorError(
"node_output_not_declared",
f"Output {output_name!r} is not declared on node {node_id!r}.",
)
declaration = next(
(
d
for d in self._resolve_declared_outputs(
tenant_id=app_model.tenant_id,
app_id=app_model.id,
workflow_id=workflow_run.workflow_id,
node_id=node_id,
raw_node=raw_node,
execution=execution,
)
if d.name == output_name
),
_ResolvedDeclaration(name=output_name, declared_type=view.type, array_item_type=None, inferred=True),
)
# ``node_detail`` already produced a truncated value_preview; reload
# the raw value from the execution payload so the preview endpoint can
# return the full thing (still wrapped through ``_full_value`` for
# signed file URLs).
full_value: Any = None
if execution is not None:
outputs = _decode_json_blob(execution.outputs) or {}
if output_name in outputs:
full_value = _full_value(outputs[output_name], tenant_id=app_model.tenant_id, declaration=declaration)
return OutputPreviewView(
node_id=node_id,
output_name=output_name,
type=view.type,
status=view.status,
value=full_value,
)
# ── DB loading ────────────────────────────────────────────────────────
def _load_run_and_executions(
self, *, app_model: App, workflow_run_id: str, session: Session
) -> tuple[WorkflowRun, Sequence[WorkflowNodeExecutionModel]]:
"""Fetch the run and its visible nodes, excluding internal Workflow Tool executions.
Enforces:
* row exists,
* row belongs to the app's tenant + app.
The trigger source (DEBUGGING vs. APP_RUN / WEBHOOK / SCHEDULE / ...) is
deliberately not checked here — D-1 was lifted 2026-05-26 and the
Inspector now serves both draft and published runs.
"""
workflow_run = session.scalar(
select(WorkflowRun).where(
WorkflowRun.id == workflow_run_id,
WorkflowRun.app_id == app_model.id,
WorkflowRun.tenant_id == app_model.tenant_id,
)
)
if workflow_run is None:
raise NodeOutputInspectorError("workflow_run_not_found", "Workflow run not found.")
executions = session.scalars(
select(WorkflowNodeExecutionModel).where(
WorkflowNodeExecutionModel.workflow_run_id == workflow_run_id,
WorkflowNodeExecutionModel.tenant_id == app_model.tenant_id,
WorkflowNodeExecutionModel.app_id == app_model.id,
WorkflowNodeExecutionModel.triggered_from != WorkflowNodeExecutionTriggeredFrom.WORKFLOW_TOOL,
)
).all()
return workflow_run, executions
@staticmethod
def _index_executions_by_node(
executions: Sequence[WorkflowNodeExecutionModel],
) -> dict[str, WorkflowNodeExecutionModel]:
"""Keep the latest execution per ``node_id``.
A given node may have multiple rows when retries or iterations occur;
``index`` is the per-run sequence counter, so we keep the one with
the highest index as the canonical "current" view.
"""
latest: dict[str, WorkflowNodeExecutionModel] = {}
for execution in executions:
existing = latest.get(execution.node_id)
if existing is None or execution.index > existing.index:
latest[execution.node_id] = execution
return latest
# ── Per-node view construction ────────────────────────────────────────
def _build_node_view(
self,
*,
tenant_id: str,
app_id: str,
workflow_id: str,
raw_node: Mapping[str, Any],
execution: WorkflowNodeExecutionModel | None,
) -> NodeOutputsView:
node_id = str(raw_node["id"])
data = raw_node.get("data") or {}
if not isinstance(data, Mapping):
data = {}
node_kind = str(data.get("type") or (execution.node_type if execution else "") or "unknown")
display_name = str(data.get("title") or (execution.title if execution else node_id))
node_status = _node_status_for(execution)
declarations = self._resolve_declared_outputs(
tenant_id=tenant_id,
app_id=app_id,
workflow_id=workflow_id,
node_id=node_id,
raw_node=raw_node,
execution=execution,
)
outputs_dict = _decode_json_blob(execution.outputs) if execution else None
metadata_dict = _decode_json_blob(execution.execution_metadata) if execution else None
type_check_by_name = _type_check_by_name(metadata_dict)
output_check_by_name = _output_check_by_name(metadata_dict)
retried = _retried_attempt_count(metadata_dict)
output_views: list[NodeOutputView] = []
for declaration in declarations:
output_views.append(
self._build_output_view(
tenant_id=tenant_id,
declaration=declaration,
node_status=node_status,
outputs_dict=outputs_dict,
type_check_by_name=type_check_by_name,
output_check_by_name=output_check_by_name,
retried=retried,
)
)
return NodeOutputsView(
node_id=node_id,
node_kind=node_kind,
node_display_name=display_name,
node_status=node_status,
node_started_at=execution.created_at if execution else None,
node_completed_at=execution.finished_at if execution else None,
outputs=output_views,
)
def _build_output_view(
self,
*,
tenant_id: str,
declaration: _ResolvedDeclaration,
node_status: NodeStatus,
outputs_dict: Mapping[str, Any] | None,
type_check_by_name: Mapping[str, Mapping[str, Any]],
output_check_by_name: Mapping[str, Mapping[str, Any]],
retried: int,
) -> NodeOutputView:
name = declaration.name
declared_type = declaration.declared_type
if node_status == NodeStatus.IDLE:
return NodeOutputView(
name=name,
type=declared_type,
status=NodeOutputStatus.PENDING,
retried=retried,
)
if node_status == NodeStatus.RUNNING:
return NodeOutputView(
name=name,
type=declared_type,
status=NodeOutputStatus.RUNNING,
retried=retried,
)
if node_status == NodeStatus.FAILED:
return NodeOutputView(
name=name,
type=declared_type,
status=NodeOutputStatus.FAILED,
retried=retried,
)
# ── node succeeded ────────────────────────────────────────────
type_check_result = type_check_by_name.get(name)
output_check_result = output_check_by_name.get(name)
type_check_view = self._coerce_check_view(type_check_result)
output_check_view = self._coerce_check_view(output_check_result)
# type check loses first; output check next; otherwise ready.
status: NodeOutputStatus
if type_check_result and not _is_passing(type_check_result):
status = NodeOutputStatus.TYPE_CHECK_FAILED
elif output_check_result and not _is_passing(output_check_result):
status = NodeOutputStatus.OUTPUT_CHECK_FAILED
elif outputs_dict is not None and name not in outputs_dict:
status = NodeOutputStatus.NOT_PRODUCED
else:
status = NodeOutputStatus.READY
value_preview = (
_value_preview(outputs_dict.get(name), tenant_id=tenant_id, declaration=declaration)
if outputs_dict and name in outputs_dict
else None
)
return NodeOutputView(
name=name,
type=declared_type,
status=status,
value_preview=value_preview,
type_check=type_check_view,
output_check=output_check_view,
retried=retried,
)
@staticmethod
def _coerce_check_view(result: Mapping[str, Any] | None) -> CheckResultView | None:
if not result:
return None
# type_check rows use ``status``; output_check rows use ``status`` too —
# both record per-output state. We treat ``status == "ready"``/"passed"
# as passing and everything else as failing, so the view stays
# stable regardless of which producer wrote the metadata.
return CheckResultView(passed=_is_passing(result), reason=result.get("reason"))
# ── Declared-output resolution ────────────────────────────────────────
def _resolve_declared_outputs(
self,
*,
tenant_id: str,
app_id: str,
workflow_id: str,
node_id: str,
raw_node: Mapping[str, Any],
execution: WorkflowNodeExecutionModel | None,
) -> list[_ResolvedDeclaration]:
if _is_agent_v2_node(raw_node):
agent_decl = self._declared_outputs_for_agent_v2(
tenant_id=tenant_id,
app_id=app_id,
workflow_id=workflow_id,
node_id=node_id,
)
if agent_decl is not None:
return [
_ResolvedDeclaration(
name=o.name,
declared_type=o.type,
array_item_type=o.array_item.type if o.array_item is not None else None,
inferred=False,
)
for o in agent_decl
]
# Non-agent (or agent-binding-missing) fall back to inferring from the
# produced payload so the Inspector still has something to show.
return self._infer_outputs_from_payload(execution=execution)
def _declared_outputs_for_agent_v2(
self,
*,
tenant_id: str,
app_id: str,
workflow_id: str,
node_id: str,
) -> list[DeclaredOutputConfig] | None:
try:
bundle = self._binding_resolver.resolve(
tenant_id=tenant_id,
app_id=app_id,
workflow_id=workflow_id,
node_id=node_id,
)
except WorkflowAgentBindingError:
return None
try:
from models.agent_config_entities import WorkflowNodeJobConfig
node_job = WorkflowNodeJobConfig.model_validate(bundle.binding.node_job_config_dict)
except Exception:
logger.warning(
"NodeOutputInspector: malformed node_job_config for binding %s", bundle.binding.id, exc_info=True
)
return None
return list(effective_declared_outputs(node_job.declared_outputs, node_job.output_routes))
@staticmethod
def _infer_outputs_from_payload(*, execution: WorkflowNodeExecutionModel | None) -> list[_ResolvedDeclaration]:
if execution is None:
return []
outputs = _decode_json_blob(execution.outputs)
if not outputs:
return []
return [
_ResolvedDeclaration(name=name, declared_type=None, array_item_type=None, inferred=True) for name in outputs
]
def _is_passing(result: Mapping[str, Any]) -> bool:
"""A check-result row is "passing" when its ``status`` is the ready/passed
sentinel emitted by the type-checker / output-check executor."""
status = result.get("status")
return status in {"ready", "passed", "not_produced"}
@@ -0,0 +1,111 @@
"""
Queue dispatcher system for async workflow execution.
Implements a Protocol-based pattern for handling different subscription tiers
with appropriate queue routing and priority assignment.
"""
from enum import StrEnum
from typing import Protocol, override
from configs import dify_config
from enums import DeploymentEdition
from services.billing_service import BillingService
class QueuePriority(StrEnum):
"""Queue priorities for different subscription tiers"""
PROFESSIONAL = "workflow_professional" # Highest priority
TEAM = "workflow_team"
SANDBOX = "workflow_sandbox" # Free tier
class BaseQueueDispatcher(Protocol):
"""Protocol for queue dispatchers"""
def get_queue_name(self) -> str:
"""Get the queue name for this dispatcher"""
...
def get_priority(self) -> int:
"""Get task priority level"""
...
class ProfessionalQueueDispatcher(BaseQueueDispatcher):
"""Dispatcher for professional tier"""
@override
def get_queue_name(self) -> str:
return QueuePriority.PROFESSIONAL
@override
def get_priority(self) -> int:
return 100
class TeamQueueDispatcher(BaseQueueDispatcher):
"""Dispatcher for team tier"""
@override
def get_queue_name(self) -> str:
return QueuePriority.TEAM
@override
def get_priority(self) -> int:
return 50
class SandboxQueueDispatcher(BaseQueueDispatcher):
"""Dispatcher for free/sandbox tier"""
@override
def get_queue_name(self) -> str:
return QueuePriority.SANDBOX
@override
def get_priority(self) -> int:
return 10
class QueueDispatcherManager:
"""Factory for creating appropriate dispatcher based on tenant subscription"""
# Mapping of billing plans to dispatchers
PLAN_DISPATCHER_MAP = {
"professional": ProfessionalQueueDispatcher,
"team": TeamQueueDispatcher,
"sandbox": SandboxQueueDispatcher,
# Add new tiers here as they're created
# For any unknown plan, default to sandbox
}
@classmethod
def get_dispatcher(cls, tenant_id: str) -> BaseQueueDispatcher:
"""
Get dispatcher based on tenant's subscription plan
Args:
tenant_id: The tenant identifier
Returns:
Appropriate queue dispatcher instance
"""
if dify_config.DEPLOYMENT_EDITION == DeploymentEdition.CLOUD:
try:
billing_info = BillingService.get_info(tenant_id)
plan = billing_info.subscription.plan
except Exception:
# If billing service fails, default to sandbox
plan = "sandbox"
else:
# Self-hosted editions use the team queue by default.
plan = "team"
dispatcher_class = cls.PLAN_DISPATCHER_MAP.get(
plan,
SandboxQueueDispatcher, # Default to sandbox for unknown plans
)
return dispatcher_class() # type: ignore
@@ -0,0 +1,36 @@
from abc import ABC, abstractmethod
from enum import StrEnum
from services.workflow.entities import WorkflowScheduleCFSPlanEntity
class SchedulerCommand(StrEnum):
"""
Scheduler command.
"""
RESOURCE_LIMIT_REACHED = "resource_limit_reached"
NONE = "none"
class CFSPlanScheduler[PlanT: WorkflowScheduleCFSPlanEntity](ABC):
"""
CFS plan scheduler.
"""
plan: PlanT
def __init__(self, plan: PlanT):
"""
Initialize the CFS plan scheduler.
Args:
plan: The CFS plan.
"""
self.plan = plan
@abstractmethod
def can_schedule(self) -> SchedulerCommand:
"""
Whether a workflow run can be scheduled.
"""
@@ -0,0 +1,700 @@
import json
from typing import Any, TypedDict
from sqlalchemy import select
from sqlalchemy.orm import Session
from core.app.app_config.entities import (
DatasetEntity,
DatasetRetrieveConfigEntity,
EasyUIBasedAppConfig,
ExternalDataVariableEntity,
ModelConfigEntity,
PromptTemplateEntity,
)
from core.app.apps.agent_chat.app_config_manager import AgentChatAppConfigManager
from core.app.apps.chat.app_config_manager import ChatAppConfigManager
from core.app.apps.completion.app_config_manager import CompletionAppConfigManager
from core.helper import encrypter
from core.prompt.simple_prompt_transform import SimplePromptTransform
from core.prompt.utils.prompt_template_parser import PromptTemplateParser
from events.app_event import app_was_created
from graphon.file import FileUploadConfig
from graphon.model_runtime.entities.llm_entities import LLMMode
from graphon.model_runtime.utils.encoders import jsonable_encoder
from graphon.nodes import BuiltinNodeTypes
from graphon.variables.input_entities import VariableEntity
from models import Account
from models.api_based_extension import APIBasedExtension, APIBasedExtensionPoint
from models.model import App, AppMode, AppModelConfig, IconType, load_annotation_reply_config
from models.workflow import Workflow, WorkflowType
class _NodeType(TypedDict):
id: str
position: None
data: dict[str, Any]
class _EdgeType(TypedDict):
id: str
source: str
target: str
class WorkflowGraph(TypedDict):
nodes: list[_NodeType]
edges: list[_EdgeType]
class WorkflowConverter:
"""
App Convert to Workflow Mode
"""
def convert_to_workflow(
self,
app_model: App,
account: Account,
name: str,
icon_type: str,
icon: str,
icon_background: str,
session: Session,
):
"""
Convert app to workflow
- basic mode of chatbot app
- expert mode of chatbot app
- completion app
:param app_model: App instance
:param account: Account
:param name: new app name
:param icon: new app icon
:param icon_type: new app icon type
:param icon_background: new app icon background
:return: new App instance
"""
# convert app model config
app_model_config = (
session.get(AppModelConfig, app_model.app_model_config_id) if app_model.app_model_config_id else None
)
if not app_model_config:
raise ValueError("App model config is required")
workflow = self.convert_app_model_config_to_workflow(
app_model=app_model, app_model_config=app_model_config, account_id=account.id, session=session
)
# create new app
new_app = App()
new_app.tenant_id = app_model.tenant_id
new_app.name = name or app_model.name + "(workflow)"
new_app.mode = AppMode.ADVANCED_CHAT if app_model.mode == AppMode.CHAT else AppMode.WORKFLOW
new_app.icon_type = IconType(icon_type) if icon_type else app_model.icon_type
new_app.icon = icon or app_model.icon
new_app.icon_background = icon_background or app_model.icon_background
new_app.enable_site = app_model.enable_site
new_app.enable_api = app_model.enable_api
new_app.api_rpm = app_model.api_rpm
new_app.api_rph = app_model.api_rph
new_app.is_demo = False
new_app.is_public = app_model.is_public
new_app.created_by = account.id
new_app.maintainer = account.id
new_app.updated_by = account.id
session.add(new_app)
session.flush()
workflow.app_id = new_app.id
session.commit()
app_was_created.send(new_app, account=account, session=session)
session.commit()
return new_app
def convert_app_model_config_to_workflow(
self, app_model: App, app_model_config: AppModelConfig, account_id: str, session: Session
):
"""
Convert app model config to workflow mode
:param app_model: App instance
:param app_model_config: AppModelConfig instance
:param account_id: Account ID
"""
# get new app mode
new_app_mode = self._get_new_app_mode(app_model)
# convert app model config
app_config = self._convert_to_app_config(
app_model=app_model, app_model_config=app_model_config, session=session
)
# init workflow graph
graph: WorkflowGraph = {"nodes": [], "edges": []}
# Convert list:
# - variables -> start
# - model_config -> llm
# - prompt_template -> llm
# - file_upload -> llm
# - external_data_variables -> http-request
# - dataset -> knowledge-retrieval
# - show_retrieve_source -> knowledge-retrieval
# convert to start node
start_node = self._convert_to_start_node(variables=app_config.variables)
graph["nodes"].append(start_node)
# convert to http request node
external_data_variable_node_mapping: dict[str, str] = {}
if app_config.external_data_variables:
http_request_nodes, external_data_variable_node_mapping = self._convert_to_http_request_node(
app_model=app_model,
variables=app_config.variables,
external_data_variables=app_config.external_data_variables,
session=session,
)
for http_request_node in http_request_nodes:
graph = self._append_node(graph, http_request_node)
# convert to knowledge retrieval node
if app_config.dataset:
knowledge_retrieval_node = self._convert_to_knowledge_retrieval_node(
new_app_mode=new_app_mode, dataset_config=app_config.dataset, model_config=app_config.model
)
if knowledge_retrieval_node:
graph = self._append_node(graph, knowledge_retrieval_node)
# convert to llm node
llm_node = self._convert_to_llm_node(
original_app_mode=AppMode.value_of(app_model.mode),
new_app_mode=new_app_mode,
graph=graph,
model_config=app_config.model,
prompt_template=app_config.prompt_template,
file_upload=app_config.additional_features.file_upload if app_config.additional_features else None,
external_data_variable_node_mapping=external_data_variable_node_mapping,
)
graph = self._append_node(graph, llm_node)
app_model_config_dict = app_config.app_model_config_dict
match new_app_mode:
case AppMode.WORKFLOW:
end_node = self._convert_to_end_node()
graph = self._append_node(graph, end_node)
features = {
"text_to_speech": app_model_config_dict.get("text_to_speech"),
"file_upload": app_model_config_dict.get("file_upload"),
"sensitive_word_avoidance": app_model_config_dict.get("sensitive_word_avoidance"),
}
case AppMode.ADVANCED_CHAT:
answer_node = self._convert_to_answer_node()
graph = self._append_node(graph, answer_node)
features = {
"opening_statement": app_model_config_dict.get("opening_statement"),
"suggested_questions": app_model_config_dict.get("suggested_questions"),
"suggested_questions_after_answer": app_model_config_dict.get("suggested_questions_after_answer"),
"speech_to_text": app_model_config_dict.get("speech_to_text"),
"text_to_speech": app_model_config_dict.get("text_to_speech"),
"file_upload": app_model_config_dict.get("file_upload"),
"sensitive_word_avoidance": app_model_config_dict.get("sensitive_word_avoidance"),
"retriever_resource": app_model_config_dict.get("retriever_resource"),
}
case _:
answer_node = self._convert_to_answer_node()
graph = self._append_node(graph, answer_node)
features = {
"text_to_speech": app_model_config_dict.get("text_to_speech"),
"file_upload": app_model_config_dict.get("file_upload"),
"sensitive_word_avoidance": app_model_config_dict.get("sensitive_word_avoidance"),
}
# create workflow record
workflow = Workflow(
tenant_id=app_model.tenant_id,
app_id=app_model.id,
type=WorkflowType.from_app_mode(new_app_mode),
version=Workflow.VERSION_DRAFT,
graph=json.dumps(graph),
_features=json.dumps(features),
created_by=account_id,
_environment_variables=[],
_conversation_variables=[],
)
session.add(workflow)
session.commit()
return workflow
def _convert_to_app_config(
self, app_model: App, app_model_config: AppModelConfig, *, session: Session
) -> EasyUIBasedAppConfig:
app_mode_enum = AppMode.value_of(app_model.mode)
app_config: EasyUIBasedAppConfig
effective_mode = (
AppMode.AGENT_CHAT
if app_model.is_agent_with_session(session=session) and app_mode_enum != AppMode.AGENT_CHAT
else app_mode_enum
)
match effective_mode:
case AppMode.AGENT_CHAT:
app_model.mode = AppMode.AGENT_CHAT
annotation_reply = load_annotation_reply_config(session, app_model_config.app_id)
app_config = AgentChatAppConfigManager.get_app_config(
app_model=app_model,
app_model_config=app_model_config,
annotation_reply=annotation_reply,
)
case AppMode.CHAT:
annotation_reply = load_annotation_reply_config(session, app_model_config.app_id)
app_config = ChatAppConfigManager.get_app_config(
app_model=app_model,
app_model_config=app_model_config,
annotation_reply=annotation_reply,
)
case AppMode.COMPLETION:
annotation_reply = load_annotation_reply_config(session, app_model_config.app_id)
app_config = CompletionAppConfigManager.get_app_config(
app_model=app_model,
app_model_config=app_model_config,
annotation_reply=annotation_reply,
)
case _:
raise ValueError("Invalid app mode")
return app_config
def _convert_to_start_node(self, variables: list[VariableEntity]) -> _NodeType:
"""
Convert to Start Node
:param variables: list of variables
:return:
"""
return {
"id": "start",
"position": None,
"data": {
"title": "START",
"type": BuiltinNodeTypes.START,
"variables": [jsonable_encoder(v) for v in variables],
},
}
def _convert_to_http_request_node(
self,
app_model: App,
variables: list[VariableEntity],
external_data_variables: list[ExternalDataVariableEntity],
session: Session,
) -> tuple[list[_NodeType], dict[str, str]]:
"""
Convert API Based Extension to HTTP Request Node
:param app_model: App instance
:param variables: list of variables
:param external_data_variables: list of external data variables
:return:
"""
index = 1
nodes = []
external_data_variable_node_mapping = {}
tenant_id = app_model.tenant_id
for external_data_variable in external_data_variables:
tool_type = external_data_variable.type
if tool_type != "api":
continue
tool_variable = external_data_variable.variable
tool_config = external_data_variable.config
# get params from config
api_based_extension_id = tool_config.get("api_based_extension_id")
if not api_based_extension_id:
continue
# get api_based_extension
api_based_extension = self._get_api_based_extension(
tenant_id=tenant_id, api_based_extension_id=api_based_extension_id, session=session
)
# decrypt api_key
api_key = encrypter.decrypt_token(tenant_id=tenant_id, token=api_based_extension.api_key)
inputs = {}
for v in variables:
inputs[v.variable] = "{{#start." + v.variable + "#}}"
request_body = {
"point": APIBasedExtensionPoint.APP_EXTERNAL_DATA_TOOL_QUERY,
"params": {
"app_id": app_model.id,
"tool_variable": tool_variable,
"inputs": inputs,
"query": "{{#sys.query#}}" if app_model.mode == AppMode.CHAT else "",
},
}
request_body_json = json.dumps(request_body)
request_body_json = request_body_json.replace(r"\{\{", "{{").replace(r"\}\}", "}}")
http_request_node: _NodeType = {
"id": f"http_request_{index}",
"position": None,
"data": {
"title": f"HTTP REQUEST {api_based_extension.name}",
"type": BuiltinNodeTypes.HTTP_REQUEST,
"method": "post",
"url": api_based_extension.api_endpoint,
"authorization": {"type": "api-key", "config": {"type": "bearer", "api_key": api_key}},
"headers": "",
"params": "",
"body": {"type": "json", "data": request_body_json},
},
}
nodes.append(http_request_node)
# append code node for response body parsing
code_node: _NodeType = {
"id": f"code_{index}",
"position": None,
"data": {
"title": f"Parse {api_based_extension.name} Response",
"type": BuiltinNodeTypes.CODE,
"variables": [{"variable": "response_json", "value_selector": [http_request_node["id"], "body"]}],
"code_language": "python3",
"code": "import json\n\ndef main(response_json: str) -> str:\n response_body = json.loads("
'response_json)\n return {\n "result": response_body["result"]\n }',
"outputs": {"result": {"type": "string"}},
},
}
nodes.append(code_node)
external_data_variable_node_mapping[external_data_variable.variable] = code_node["id"]
index += 1
return nodes, external_data_variable_node_mapping
def _convert_to_knowledge_retrieval_node(
self, new_app_mode: AppMode, dataset_config: DatasetEntity, model_config: ModelConfigEntity
) -> _NodeType | None:
"""
Convert datasets to Knowledge Retrieval Node
:param new_app_mode: new app mode
:param dataset_config: dataset
:param model_config: model config
:return:
"""
retrieve_config = dataset_config.retrieve_config
if new_app_mode == AppMode.ADVANCED_CHAT:
query_variable_selector = ["sys", "query"]
elif retrieve_config.query_variable:
# fetch query variable
query_variable_selector = ["start", retrieve_config.query_variable]
else:
return None
return {
"id": "knowledge_retrieval",
"position": None,
"data": {
"title": "KNOWLEDGE RETRIEVAL",
"type": BuiltinNodeTypes.KNOWLEDGE_RETRIEVAL,
"query_variable_selector": query_variable_selector,
"dataset_ids": dataset_config.dataset_ids,
"retrieval_mode": retrieve_config.retrieve_strategy.value,
"single_retrieval_config": {
"model": {
"provider": model_config.provider,
"name": model_config.model,
"mode": model_config.mode,
"completion_params": {
**model_config.parameters,
"stop": model_config.stop,
},
}
}
if retrieve_config.retrieve_strategy == DatasetRetrieveConfigEntity.RetrieveStrategy.SINGLE
else None,
"multiple_retrieval_config": {
"top_k": retrieve_config.top_k,
"score_threshold": retrieve_config.score_threshold,
"reranking_model": retrieve_config.reranking_model,
}
if retrieve_config.retrieve_strategy == DatasetRetrieveConfigEntity.RetrieveStrategy.MULTIPLE
else None,
},
}
def _convert_to_llm_node(
self,
original_app_mode: AppMode,
new_app_mode: AppMode,
graph: WorkflowGraph,
model_config: ModelConfigEntity,
prompt_template: PromptTemplateEntity,
file_upload: FileUploadConfig | None = None,
external_data_variable_node_mapping: dict[str, str] | None = None,
) -> _NodeType:
"""
Convert to LLM Node
:param original_app_mode: original app mode
:param new_app_mode: new app mode
:param graph: graph
:param model_config: model config
:param prompt_template: prompt template
:param file_upload: file upload config (optional)
:param external_data_variable_node_mapping: external data variable node mapping
"""
# fetch start and knowledge retrieval node
start_node = next(filter(lambda n: n["data"]["type"] == BuiltinNodeTypes.START, graph["nodes"]))
knowledge_retrieval_node = next(
filter(lambda n: n["data"]["type"] == BuiltinNodeTypes.KNOWLEDGE_RETRIEVAL, graph["nodes"]), None
)
role_prefix = None
prompts: Any | None = None
# Chat Model
if model_config.mode == LLMMode.CHAT:
if prompt_template.prompt_type == PromptTemplateEntity.PromptType.SIMPLE:
if not prompt_template.simple_prompt_template:
raise ValueError("Simple prompt template is required")
# get prompt template
prompt_transform = SimplePromptTransform()
prompt_template_config = prompt_transform.get_prompt_template(
app_mode=original_app_mode,
pre_prompt=prompt_template.simple_prompt_template,
has_context=knowledge_retrieval_node is not None,
query_in_prompt=False,
)
prompt_template_obj = prompt_template_config["prompt_template"]
if not isinstance(prompt_template_obj, PromptTemplateParser):
raise TypeError(f"Expected PromptTemplateParser, got {type(prompt_template_obj)}")
template = prompt_template_obj.template
if not template:
prompts = []
else:
template = self._replace_template_variables(
template, start_node["data"]["variables"], external_data_variable_node_mapping
)
prompts = [{"role": "user", "text": template}]
else:
advanced_chat_prompt_template = prompt_template.advanced_chat_prompt_template
prompts = []
if advanced_chat_prompt_template:
for m in advanced_chat_prompt_template.messages:
text = m.text
text = self._replace_template_variables(
text, start_node["data"]["variables"], external_data_variable_node_mapping
)
prompts.append({"role": m.role.value, "text": text})
# Completion Model
else:
if prompt_template.prompt_type == PromptTemplateEntity.PromptType.SIMPLE:
if not prompt_template.simple_prompt_template:
raise ValueError("Simple prompt template is required")
# get prompt template
prompt_transform = SimplePromptTransform()
prompt_template_config = prompt_transform.get_prompt_template(
app_mode=original_app_mode,
pre_prompt=prompt_template.simple_prompt_template,
has_context=knowledge_retrieval_node is not None,
query_in_prompt=False,
)
prompt_template_obj = prompt_template_config["prompt_template"]
if not isinstance(prompt_template_obj, PromptTemplateParser):
raise TypeError(f"Expected PromptTemplateParser, got {type(prompt_template_obj)}")
template = prompt_template_obj.template
template = self._replace_template_variables(
template=template,
variables=start_node["data"]["variables"],
external_data_variable_node_mapping=external_data_variable_node_mapping,
)
prompts = {"text": template}
prompt_rules = prompt_template_config["prompt_rules"]
if not isinstance(prompt_rules, dict):
raise TypeError(f"Expected dict for prompt_rules, got {type(prompt_rules)}")
role_prefix = {
"user": prompt_rules.get("human_prefix", "Human"),
"assistant": prompt_rules.get("assistant_prefix", "Assistant"),
}
else:
advanced_completion_prompt_template = prompt_template.advanced_completion_prompt_template
if advanced_completion_prompt_template:
text = advanced_completion_prompt_template.prompt
text = self._replace_template_variables(
template=text,
variables=start_node["data"]["variables"],
external_data_variable_node_mapping=external_data_variable_node_mapping,
)
else:
text = ""
text = text.replace("{{#query#}}", "{{#sys.query#}}")
prompts = {
"text": text,
}
if advanced_completion_prompt_template and advanced_completion_prompt_template.role_prefix:
role_prefix = {
"user": advanced_completion_prompt_template.role_prefix.user,
"assistant": advanced_completion_prompt_template.role_prefix.assistant,
}
memory = None
if new_app_mode == AppMode.ADVANCED_CHAT:
memory = {"role_prefix": role_prefix, "window": {"enabled": False}}
completion_params = model_config.parameters
completion_params.update({"stop": model_config.stop})
return {
"id": "llm",
"position": None,
"data": {
"title": "LLM",
"type": BuiltinNodeTypes.LLM,
"model": {
"provider": model_config.provider,
"name": model_config.model,
"mode": model_config.mode,
"completion_params": completion_params,
},
"prompt_template": prompts,
"memory": memory,
"context": {
"enabled": knowledge_retrieval_node is not None,
"variable_selector": ["knowledge_retrieval", "result"]
if knowledge_retrieval_node is not None
else None,
},
"vision": {
"enabled": file_upload is not None,
"variable_selector": ["sys", "files"] if file_upload is not None else None,
"configs": {"detail": file_upload.image_config.detail}
if file_upload is not None and file_upload.image_config is not None
else None,
},
},
}
def _replace_template_variables(
self, template: str, variables: list[dict], external_data_variable_node_mapping: dict[str, str] | None = None
) -> str:
"""
Replace Template Variables
:param template: template
:param variables: list of variables
:param external_data_variable_node_mapping: external data variable node mapping
:return:
"""
for v in variables:
template = template.replace("{{" + v["variable"] + "}}", "{{#start." + v["variable"] + "#}}")
if external_data_variable_node_mapping:
for variable, code_node_id in external_data_variable_node_mapping.items():
template = template.replace("{{" + variable + "}}", "{{#" + code_node_id + ".result#}}")
return template
def _convert_to_end_node(self) -> _NodeType:
"""
Convert to End Node
:return:
"""
# for original completion app
return {
"id": "end",
"position": None,
"data": {
"title": "END",
"type": BuiltinNodeTypes.END,
"outputs": [{"variable": "result", "value_selector": ["llm", "text"]}],
},
}
def _convert_to_answer_node(self) -> _NodeType:
"""
Convert to Answer Node
:return:
"""
# for original chat app
return {
"id": "answer",
"position": None,
"data": {"title": "ANSWER", "type": BuiltinNodeTypes.ANSWER, "answer": "{{#llm.text#}}"},
}
def _create_edge(self, source: str, target: str) -> _EdgeType:
"""
Create Edge
:param source: source node id
:param target: target node id
:return:
"""
return {"id": f"{source}-{target}", "source": source, "target": target}
def _append_node(self, graph: WorkflowGraph, node: _NodeType):
"""
Append Node to Graph
:param graph: Graph, include: nodes, edges
:param node: Node to append
:return:
"""
previous_node = graph["nodes"][-1]
graph["nodes"].append(node)
graph["edges"].append(self._create_edge(previous_node["id"], node["id"]))
return graph
def _get_new_app_mode(self, app_model: App) -> AppMode:
"""
Get new app mode
:param app_model: App instance
:return: AppMode
"""
if app_model.mode == AppMode.COMPLETION:
return AppMode.WORKFLOW
else:
return AppMode.ADVANCED_CHAT
def _get_api_based_extension(self, tenant_id: str, api_based_extension_id: str, session: Session):
"""
Get API Based Extension
:param tenant_id: tenant id
:param api_based_extension_id: api based extension id
:return:
"""
api_based_extension = session.scalar(
select(APIBasedExtension)
.where(APIBasedExtension.tenant_id == tenant_id, APIBasedExtension.id == api_based_extension_id)
.limit(1)
)
if not api_based_extension:
raise ValueError(f"API Based Extension not found, id: {api_based_extension_id}")
return api_based_extension