chore: 纳入本地 Dify 1.19.0 源码及品牌与免登改动

演示目录只保留可重建的源码。运行密钥、数据卷、依赖和带口令的本地配置不进入仓库。

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
wuzilongandCursor committed 2026-11-18 13:59:25 +08:00
1 parent 60158935f4
commit 6519682885
15831 files changed
+3063428

No files matched your search

@@ -0,0 +1 @@
@@ -0,0 +1,45 @@
"""
AppTrigger management service.
Handles AppTrigger model CRUD operations and status management.
This service centralizes all AppTrigger-related business logic.
"""
import logging
from sqlalchemy import update
from sqlalchemy.orm import sessionmaker
from extensions.ext_database import db
from models.enums import AppTriggerStatus
from models.trigger import AppTrigger
logger = logging.getLogger(__name__)
class AppTriggerService:
"""Service for managing AppTrigger lifecycle and status."""
@staticmethod
def mark_tenant_triggers_rate_limited(tenant_id: str) -> None:
"""
Mark all enabled triggers for a tenant as rate limited due to quota exceeded.
This method is called when a tenant's quota is exhausted. It updates all
enabled triggers to RATE_LIMITED status to prevent further executions until
quota is restored.
Args:
tenant_id: Tenant ID whose triggers should be marked as rate limited
"""
try:
with sessionmaker(bind=db.engine).begin() as session:
session.execute(
update(AppTrigger)
.where(AppTrigger.tenant_id == tenant_id, AppTrigger.status == AppTriggerStatus.ENABLED)
.values(status=AppTriggerStatus.RATE_LIMITED)
)
logger.info("Marked all enabled triggers as rate limited for tenant %s", tenant_id)
except Exception:
logger.exception("Failed to mark all enabled triggers as rate limited for tenant %s", tenant_id)
@@ -0,0 +1,47 @@
"""Detached records and persistence contract for Console app trigger management."""
from dataclasses import dataclass
from datetime import datetime
from typing import Protocol
from machinery.context import RequestContext
class WebhookTriggerNotFoundError(Exception):
"""The requested node has no webhook in the admitted app and workspace."""
class AppTriggerNotFoundError(Exception):
"""The trigger does not belong to the admitted app and workspace."""
@dataclass(frozen=True, slots=True)
class AppTriggerRecord:
"""Persisted values; None preserves a missing legacy node or provider name."""
id: str
trigger_type: str
title: str
node_id: str | None
provider_name: str | None
status: str
created_at: datetime
updated_at: datetime
@dataclass(frozen=True, slots=True)
class WebhookTriggerRecord:
id: str
webhook_id: str
node_id: str
created_at: datetime
class AppTriggerStore(Protocol):
def get_webhook(self, context: RequestContext, app_id: str, node_id: str) -> WebhookTriggerRecord: ...
def list_triggers(self, context: RequestContext, app_id: str) -> tuple[AppTriggerRecord, ...]: ...
def set_enabled(
self, context: RequestContext, app_id: str, trigger_id: str, *, enabled: bool
) -> AppTriggerRecord: ...
@@ -0,0 +1,305 @@
import json
import logging
from datetime import datetime
from sqlalchemy import select
from sqlalchemy.orm import Session
from core.trigger.constants import TRIGGER_SCHEDULE_NODE_TYPE
from core.workflow.nodes.trigger_schedule.entities import (
ScheduleConfig,
SchedulePlanUpdate,
TriggerScheduleNodeData,
VisualConfig,
)
from core.workflow.nodes.trigger_schedule.exc import ScheduleConfigError, ScheduleNotFoundError
from graphon.entities.graph_config import NodeConfigDict
from libs.schedule_utils import calculate_next_run_at, convert_12h_to_24h
from models.account import Account, TenantAccountJoin
from models.trigger import WorkflowSchedulePlan
from models.workflow import Workflow
from services.account_errors import AccountNotFoundError
logger = logging.getLogger(__name__)
class ScheduleService:
@staticmethod
def create_schedule(
tenant_id: str, app_id: str, config: ScheduleConfig, *, session: Session
) -> WorkflowSchedulePlan:
"""
Create a new schedule with validated configuration.
Args:
session: Database session
tenant_id: Tenant ID
app_id: Application ID
config: Validated schedule configuration
Returns:
Created WorkflowSchedulePlan instance
"""
next_run_at = calculate_next_run_at(
config.cron_expression,
config.timezone,
)
schedule = WorkflowSchedulePlan(
tenant_id=tenant_id,
app_id=app_id,
node_id=config.node_id,
cron_expression=config.cron_expression,
timezone=config.timezone,
next_run_at=next_run_at,
)
session.add(schedule)
session.flush()
return schedule
@staticmethod
def update_schedule(schedule_id: str, updates: SchedulePlanUpdate, *, session: Session) -> WorkflowSchedulePlan:
"""
Update an existing schedule with validated configuration.
Args:
session: Database session
schedule_id: Schedule ID to update
updates: Validated update configuration
Raises:
ScheduleNotFoundError: If schedule not found
Returns:
Updated WorkflowSchedulePlan instance
"""
schedule = session.get(WorkflowSchedulePlan, schedule_id)
if not schedule:
raise ScheduleNotFoundError(f"Schedule not found: {schedule_id}")
# If time-related fields are updated, synchronously update the next_run_at.
time_fields_updated = False
if updates.node_id is not None:
schedule.node_id = updates.node_id
if updates.cron_expression is not None:
schedule.cron_expression = updates.cron_expression
time_fields_updated = True
if updates.timezone is not None:
schedule.timezone = updates.timezone
time_fields_updated = True
if time_fields_updated:
schedule.next_run_at = calculate_next_run_at(
schedule.cron_expression,
schedule.timezone,
)
session.flush()
return schedule
@staticmethod
def delete_schedule(schedule_id: str, *, session: Session) -> None:
"""
Delete a schedule plan.
Args:
session: Database session
schedule_id: Schedule ID to delete
"""
schedule = session.get(WorkflowSchedulePlan, schedule_id)
if not schedule:
raise ScheduleNotFoundError(f"Schedule not found: {schedule_id}")
session.delete(schedule)
session.flush()
@staticmethod
def get_tenant_owner(tenant_id: str, *, session: Session) -> Account:
"""
Returns an account to execute scheduled workflows on behalf of the tenant.
Prioritizes owner over admin to ensure proper authorization hierarchy.
"""
result = session.execute(
select(TenantAccountJoin)
.where(TenantAccountJoin.tenant_id == tenant_id, TenantAccountJoin.role == "owner")
.limit(1)
).scalar_one_or_none()
if not result:
# Owner may not exist in some tenant configurations, fallback to admin
result = session.execute(
select(TenantAccountJoin)
.where(TenantAccountJoin.tenant_id == tenant_id, TenantAccountJoin.role == "admin")
.limit(1)
).scalar_one_or_none()
if result:
account = session.get(Account, result.account_id)
if not account:
raise AccountNotFoundError(f"Account not found: {result.account_id}")
return account
else:
raise AccountNotFoundError(f"Account not found for tenant: {tenant_id}")
@staticmethod
def update_next_run_at(schedule_id: str, *, session: Session) -> datetime:
"""
Advances the schedule to its next execution time after a successful trigger.
Uses current time as base to prevent missing executions during delays.
"""
schedule = session.get(WorkflowSchedulePlan, schedule_id)
if not schedule:
raise ScheduleNotFoundError(f"Schedule not found: {schedule_id}")
# Base on current time to handle execution delays gracefully
next_run_at = calculate_next_run_at(
schedule.cron_expression,
schedule.timezone,
)
schedule.next_run_at = next_run_at
session.flush()
return next_run_at
@staticmethod
def to_schedule_config(node_config: NodeConfigDict) -> ScheduleConfig:
"""
Converts user-friendly visual schedule settings to cron expression.
Maintains consistency with frontend UI expectations while supporting croniter's extended syntax.
"""
node_data = TriggerScheduleNodeData.model_validate(node_config["data"], from_attributes=True)
mode = node_data.mode
timezone = node_data.timezone
node_id = node_config["id"]
cron_expression = None
if mode == "cron":
cron_expression = node_data.cron_expression
if not cron_expression:
raise ScheduleConfigError("Cron expression is required for cron mode")
elif mode == "visual":
frequency = str(node_data.frequency or "")
if not frequency:
raise ScheduleConfigError("Frequency is required for visual mode")
visual_config = VisualConfig.model_validate(node_data.visual_config or {})
cron_expression = ScheduleService.visual_to_cron(frequency=frequency, visual_config=visual_config)
if not cron_expression:
raise ScheduleConfigError("Cron expression is required for visual mode")
else:
raise ScheduleConfigError(f"Invalid schedule mode: {mode}")
return ScheduleConfig(node_id=node_id, cron_expression=cron_expression, timezone=timezone)
@staticmethod
def extract_schedule_config(workflow: Workflow) -> ScheduleConfig | None:
"""
Extracts schedule configuration from workflow graph.
Searches for the first schedule trigger node in the workflow and converts
its configuration (either visual or cron mode) into a unified ScheduleConfig.
Args:
workflow: The workflow containing the graph definition
Returns:
ScheduleConfig if a valid schedule node is found, None if no schedule node exists
Raises:
ScheduleConfigError: If graph parsing fails or schedule configuration is invalid
Note:
Currently only returns the first schedule node found.
Multiple schedule nodes in the same workflow are not supported.
"""
try:
graph_data = workflow.graph_dict
except (json.JSONDecodeError, TypeError, AttributeError) as e:
raise ScheduleConfigError(f"Failed to parse workflow graph: {e}") from e
if not graph_data:
raise ScheduleConfigError("Workflow graph is empty")
nodes = graph_data.get("nodes", [])
for node in nodes:
node_data = node.get("data", {})
if node_data.get("type") != TRIGGER_SCHEDULE_NODE_TYPE:
continue
node_id = node.get("id", "start")
trigger_data = TriggerScheduleNodeData.model_validate(node_data)
mode = trigger_data.mode
timezone = trigger_data.timezone
cron_expression = None
if mode == "cron":
cron_expression = trigger_data.cron_expression
if not cron_expression:
raise ScheduleConfigError("Cron expression is required for cron mode")
elif mode == "visual":
frequency = trigger_data.frequency
if not frequency:
raise ScheduleConfigError("Frequency is required for visual mode")
visual_config = VisualConfig.model_validate(trigger_data.visual_config or {})
cron_expression = ScheduleService.visual_to_cron(frequency, visual_config)
else:
raise ScheduleConfigError(f"Invalid schedule mode: {mode}")
return ScheduleConfig(node_id=node_id, cron_expression=cron_expression, timezone=timezone)
return None
@staticmethod
def visual_to_cron(frequency: str, visual_config: VisualConfig) -> str:
"""
Converts user-friendly visual schedule settings to cron expression.
Maintains consistency with frontend UI expectations while supporting croniter's extended syntax.
"""
if frequency == "hourly":
if visual_config.on_minute is None:
raise ScheduleConfigError("on_minute is required for hourly schedules")
return f"{visual_config.on_minute} * * * *"
elif frequency == "daily":
if not visual_config.time:
raise ScheduleConfigError("time is required for daily schedules")
hour, minute = convert_12h_to_24h(visual_config.time)
return f"{minute} {hour} * * *"
elif frequency == "weekly":
if not visual_config.time:
raise ScheduleConfigError("time is required for weekly schedules")
if not visual_config.weekdays:
raise ScheduleConfigError("Weekdays are required for weekly schedules")
hour, minute = convert_12h_to_24h(visual_config.time)
weekday_map = {"sun": "0", "mon": "1", "tue": "2", "wed": "3", "thu": "4", "fri": "5", "sat": "6"}
cron_weekdays = [weekday_map[day] for day in visual_config.weekdays]
return f"{minute} {hour} * * {','.join(sorted(cron_weekdays))}"
elif frequency == "monthly":
if not visual_config.time:
raise ScheduleConfigError("time is required for monthly schedules")
if not visual_config.monthly_days:
raise ScheduleConfigError("Monthly days are required for monthly schedules")
hour, minute = convert_12h_to_24h(visual_config.time)
numeric_days: list[int] = []
has_last = False
for day in visual_config.monthly_days:
if day == "last":
has_last = True
else:
numeric_days.append(day)
result_days = [str(d) for d in sorted(set(numeric_days))]
if has_last:
result_days.append("L")
return f"{minute} {hour} {','.join(result_days)} * *"
else:
raise ScheduleConfigError(f"Unsupported frequency: {frequency}")
@@ -0,0 +1,980 @@
import json
import logging
import time as _time
import uuid
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, TypedDict
from services.credentials.query import CredentialQuery
from services.plugin.dispatch_identity import PluginOAuthClientConfig
if TYPE_CHECKING:
from models.account import Account
from sqlalchemy import delete, func, select
from sqlalchemy.orm import Session, sessionmaker
from configs import dify_config
from constants import HIDDEN_VALUE, UNKNOWN_VALUE
from core.helper.provider_cache import NoOpProviderCredentialCache
from core.helper.provider_encryption import ProviderConfigEncrypter, create_provider_encrypter
from core.plugin.entities.plugin_daemon import CredentialType
from core.plugin.impl.oauth import OAuthHandler
from core.plugin.plugin_service import PluginService
from core.tools.utils.system_encryption import decrypt_system_params
from core.trigger.entities.api_entities import (
TriggerProviderApiEntity,
TriggerProviderSubscriptionApiEntity,
)
from core.trigger.entities.entities import Subscription as TriggerSubscriptionEntity
from core.trigger.provider import PluginTriggerProviderController
from core.trigger.trigger_manager import TriggerManager
from core.trigger.utils.encryption import (
create_trigger_provider_encrypter_for_properties,
create_trigger_provider_encrypter_for_subscription,
delete_cache_for_subscription,
)
from core.trigger.utils.endpoint import generate_plugin_trigger_endpoint_url
from extensions.ext_database import db
from extensions.ext_redis import redis_client
from models.provider_ids import TriggerProviderID
from models.trigger import (
TriggerOAuthSystemClient,
TriggerOAuthTenantClient,
TriggerSubscription,
)
logger = logging.getLogger(__name__)
class VerifyCredentialsResult(TypedDict):
verified: bool
class TriggerProviderService:
"""Service for managing trigger providers and credentials"""
##########################
# Trigger provider
##########################
__MAX_TRIGGER_PROVIDER_COUNT__ = 10
@classmethod
def get_trigger_provider(cls, tenant_id: str, provider: TriggerProviderID) -> TriggerProviderApiEntity:
"""Get info for a trigger provider"""
return TriggerManager.get_trigger_provider(tenant_id, provider).to_api_entity()
@classmethod
def list_trigger_providers(cls, tenant_id: str) -> list[TriggerProviderApiEntity]:
"""List all trigger providers for the current tenant"""
return [provider.to_api_entity() for provider in TriggerManager.list_all_trigger_providers(tenant_id)]
@classmethod
def list_trigger_provider_subscriptions(
cls,
tenant_id: str,
provider_id: TriggerProviderID,
user: "Account | None" = None,
*,
credential_query: CredentialQuery,
) -> list[TriggerProviderSubscriptionApiEntity]:
"""List all trigger subscriptions for the current tenant, filtered by visibility."""
subscriptions = [
TriggerProviderSubscriptionApiEntity(
id=row.id,
name=row.name,
provider=row.provider,
credential_type=row.credential_type,
credentials=dict(row.credentials),
endpoint=generate_plugin_trigger_endpoint_url(row.endpoint_id),
parameters=dict(row.parameters),
properties=dict(row.properties),
workflows_in_use=row.workflows_in_use,
)
for row in credential_query.list_trigger_subscriptions(
workspace_id=tenant_id, provider=str(provider_id), actor_id=user.id if user else None
)
]
if not subscriptions:
return []
provider_controller = TriggerManager.get_trigger_provider(tenant_id, provider_id)
for subscription in subscriptions:
credential_encrypter, _ = create_trigger_provider_encrypter_for_subscription(
tenant_id=tenant_id,
controller=provider_controller,
subscription=subscription,
)
subscription.credentials = dict(
credential_encrypter.mask_credentials(dict(credential_encrypter.decrypt(subscription.credentials)))
)
properties_encrypter, _ = create_trigger_provider_encrypter_for_properties(
tenant_id=tenant_id,
controller=provider_controller,
subscription=subscription,
)
subscription.properties = dict(
properties_encrypter.mask_credentials(dict(properties_encrypter.decrypt(subscription.properties)))
)
subscription.parameters = dict(subscription.parameters)
return subscriptions
@classmethod
def add_trigger_subscription(
cls,
tenant_id: str,
user_id: str,
name: str,
provider_id: TriggerProviderID,
endpoint_id: str,
credential_type: CredentialType,
parameters: Mapping[str, Any],
properties: Mapping[str, Any],
credentials: Mapping[str, str],
subscription_id: str | None = None,
credential_expires_at: int = -1,
expires_at: int = -1,
) -> Mapping[str, Any]:
"""
Add a new trigger provider with credentials.
Supports multiple credential instances per provider.
:param tenant_id: Tenant ID
:param provider_id: Provider identifier (e.g., "plugin_id/provider_name")
:param credential_type: Type of credential (oauth or api_key)
:param credentials: Credential data to encrypt and store
:param name: Optional name for this credential instance
:param expires_at: OAuth token expiration timestamp
:return: Success response
"""
try:
provider_controller = TriggerManager.get_trigger_provider(tenant_id, provider_id)
with sessionmaker(bind=db.engine, expire_on_commit=False).begin() as session:
# Use distributed lock to prevent race conditions
lock_key = f"trigger_provider_create_lock:{tenant_id}_{provider_id}"
with redis_client.lock(lock_key, timeout=20):
# Check provider count limit
provider_count = (
session.scalar(
select(func.count(TriggerSubscription.id)).where(
TriggerSubscription.tenant_id == tenant_id,
TriggerSubscription.provider_id == str(provider_id),
)
)
or 0
)
if provider_count >= cls.__MAX_TRIGGER_PROVIDER_COUNT__:
raise ValueError(
f"Maximum number of providers ({cls.__MAX_TRIGGER_PROVIDER_COUNT__}) "
f"reached for {provider_id}"
)
# Check if name already exists
existing = session.scalar(
select(TriggerSubscription)
.where(
TriggerSubscription.tenant_id == tenant_id,
TriggerSubscription.provider_id == str(provider_id),
TriggerSubscription.name == name,
)
.limit(1)
)
if existing:
raise ValueError(f"Credential name '{name}' already exists for this provider")
credential_encrypter: ProviderConfigEncrypter | None = None
if credential_type != CredentialType.UNAUTHORIZED:
credential_encrypter, _ = create_provider_encrypter(
tenant_id=tenant_id,
config=provider_controller.get_credential_schema_config(credential_type),
cache=NoOpProviderCredentialCache(),
)
properties_encrypter, _ = create_provider_encrypter(
tenant_id=tenant_id,
config=provider_controller.get_properties_schema(),
cache=NoOpProviderCredentialCache(),
)
# Create provider record
subscription = TriggerSubscription(
tenant_id=tenant_id,
user_id=user_id,
name=name,
endpoint_id=endpoint_id,
provider_id=str(provider_id),
parameters=dict(parameters),
properties=dict(properties_encrypter.encrypt(dict(properties))),
credentials=dict(credential_encrypter.encrypt(dict(credentials)))
if credential_encrypter
else {},
credential_type=credential_type,
credential_expires_at=credential_expires_at,
expires_at=expires_at,
)
subscription.id = subscription_id or str(uuid.uuid4())
session.add(subscription)
return {
"result": "success",
"id": str(subscription.id),
}
except Exception as e:
logger.exception("Failed to add trigger provider")
raise ValueError(str(e)) from e
@classmethod
def update_trigger_subscription(
cls,
tenant_id: str,
subscription_id: str,
name: str | None = None,
properties: Mapping[str, Any] | None = None,
parameters: Mapping[str, Any] | None = None,
credentials: Mapping[str, Any] | None = None,
credential_expires_at: int | None = None,
expires_at: int | None = None,
) -> None:
"""
Update an existing trigger subscription.
:param tenant_id: Tenant ID
:param subscription_id: Subscription instance ID
:param name: Optional new name for this subscription
:param properties: Optional new properties
:param parameters: Optional new parameters
:param credentials: Optional new credentials
:param credential_expires_at: Optional new credential expiration timestamp
:param expires_at: Optional new expiration timestamp
:return: Success response with updated subscription info
"""
with sessionmaker(bind=db.engine, expire_on_commit=False).begin() as session:
# Use distributed lock to prevent race conditions on the same subscription
lock_key = f"trigger_subscription_update_lock:{tenant_id}_{subscription_id}"
with redis_client.lock(lock_key, timeout=20):
subscription = session.scalar(
select(TriggerSubscription)
.where(
TriggerSubscription.tenant_id == tenant_id,
TriggerSubscription.id == subscription_id,
)
.limit(1)
)
if not subscription:
raise ValueError(f"Trigger subscription {subscription_id} not found")
provider_id = TriggerProviderID(subscription.provider_id)
provider_controller = TriggerManager.get_trigger_provider(tenant_id, provider_id)
# Check for name uniqueness if name is being updated
if name is not None and name != subscription.name:
existing = session.scalar(
select(TriggerSubscription)
.where(
TriggerSubscription.tenant_id == tenant_id,
TriggerSubscription.provider_id == str(provider_id),
TriggerSubscription.name == name,
)
.limit(1)
)
if existing:
raise ValueError(f"Subscription name '{name}' already exists for this provider")
subscription.name = name
# Update properties if provided
if properties is not None:
properties_encrypter, _ = create_provider_encrypter(
tenant_id=tenant_id,
config=provider_controller.get_properties_schema(),
cache=NoOpProviderCredentialCache(),
)
# Handle hidden values - preserve original encrypted values
original_properties = properties_encrypter.decrypt(subscription.properties)
new_properties: dict[str, Any] = {
key: value if value != HIDDEN_VALUE else original_properties.get(key, UNKNOWN_VALUE)
for key, value in properties.items()
}
subscription.properties = dict(properties_encrypter.encrypt(new_properties))
# Update parameters if provided
if parameters is not None:
subscription.parameters = dict(parameters)
# Update credentials if provided
if credentials is not None:
credential_type = CredentialType.of(subscription.credential_type)
credential_encrypter, _ = create_provider_encrypter(
tenant_id=tenant_id,
config=provider_controller.get_credential_schema_config(credential_type),
cache=NoOpProviderCredentialCache(),
)
subscription.credentials = dict(credential_encrypter.encrypt(dict(credentials)))
# Update credential expiration timestamp if provided
if credential_expires_at is not None:
subscription.credential_expires_at = credential_expires_at
# Update expiration timestamp if provided
if expires_at is not None:
subscription.expires_at = expires_at
# Invalidate after commit so a cache miss observes committed credentials.
delete_cache_for_subscription(
tenant_id=tenant_id, provider_id=subscription.provider_id, subscription_id=subscription.id
)
@classmethod
def get_subscription_by_id(
cls, tenant_id: str, subscription_id: str | None = None, *, provider_id: TriggerProviderID | None = None
) -> TriggerSubscription | None:
"""
Get a trigger subscription by the ID.
"""
with Session(db.engine, expire_on_commit=False) as session:
query = select(TriggerSubscription).where(TriggerSubscription.tenant_id == tenant_id)
if subscription_id:
query = query.where(TriggerSubscription.id == subscription_id)
if provider_id is not None:
query = query.where(TriggerSubscription.provider_id == str(provider_id))
subscription = session.scalar(query.limit(1))
if subscription:
provider_controller = TriggerManager.get_trigger_provider(
tenant_id, TriggerProviderID(subscription.provider_id)
)
encrypter, _ = create_trigger_provider_encrypter_for_subscription(
tenant_id=tenant_id,
controller=provider_controller,
subscription=subscription,
)
subscription.credentials = dict(encrypter.decrypt(subscription.credentials))
properties_encrypter, _ = create_trigger_provider_encrypter_for_properties(
tenant_id=subscription.tenant_id,
controller=provider_controller,
subscription=subscription,
)
subscription.properties = dict(properties_encrypter.decrypt(subscription.properties))
return subscription
@classmethod
def delete_trigger_provider(cls, tenant_id: str, subscription_id: str, *, session: Session):
"""
Delete a trigger provider subscription within an existing session.
:param session: Database session
:param tenant_id: Tenant ID
:param subscription_id: Subscription instance ID
:return: Success response
"""
subscription = session.scalar(
select(TriggerSubscription)
.where(
TriggerSubscription.tenant_id == tenant_id,
TriggerSubscription.id == subscription_id,
)
.limit(1)
)
if not subscription:
raise ValueError(f"Trigger provider subscription {subscription_id} not found")
credential_type: CredentialType = CredentialType.of(subscription.credential_type)
provider_id = TriggerProviderID(subscription.provider_id)
provider_controller: PluginTriggerProviderController = TriggerManager.get_trigger_provider(
tenant_id=tenant_id, provider_id=provider_id
)
encrypter, _ = create_trigger_provider_encrypter_for_subscription(
tenant_id=tenant_id,
controller=provider_controller,
subscription=subscription,
)
is_auto_created: bool = credential_type in [CredentialType.OAUTH2, CredentialType.API_KEY]
if is_auto_created:
try:
TriggerManager.unsubscribe_trigger(
tenant_id=tenant_id,
user_id=subscription.user_id,
provider_id=provider_id,
subscription=subscription.to_entity(),
credentials=encrypter.decrypt(subscription.credentials),
credential_type=credential_type,
)
except Exception as e:
logger.exception("Error unsubscribing trigger", exc_info=e)
session.delete(subscription)
# Clear cache
delete_cache_for_subscription(
tenant_id=tenant_id,
provider_id=subscription.provider_id,
subscription_id=subscription.id,
)
@classmethod
def refresh_oauth_token(
cls,
tenant_id: str,
subscription_id: str,
) -> Mapping[str, Any]:
"""
Refresh OAuth token for a trigger provider.
:param tenant_id: Tenant ID
:param subscription_id: Subscription instance ID
:return: New token info
"""
with sessionmaker(bind=db.engine).begin() as session:
subscription = session.scalar(
select(TriggerSubscription)
.where(
TriggerSubscription.tenant_id == tenant_id,
TriggerSubscription.id == subscription_id,
)
.limit(1)
)
if not subscription:
raise ValueError(f"Trigger provider subscription {subscription_id} not found")
if subscription.credential_type != CredentialType.OAUTH2:
raise ValueError("Only OAuth credentials can be refreshed")
provider_id = TriggerProviderID(subscription.provider_id)
provider_controller: PluginTriggerProviderController = TriggerManager.get_trigger_provider(
tenant_id=tenant_id, provider_id=provider_id
)
# Create encrypter
encrypter, _ = create_provider_encrypter(
tenant_id=tenant_id,
config=[x.to_basic_provider_config() for x in provider_controller.get_oauth_client_schema()],
cache=NoOpProviderCredentialCache(),
)
# Decrypt current credentials
current_credentials = encrypter.decrypt(subscription.credentials)
# Get OAuth client configuration
redirect_uri = (
f"{dify_config.CONSOLE_API_URL}/console/api/oauth/plugin/{subscription.provider_id}/trigger/callback"
)
plugin_unique_identifier = provider_controller.plugin_unique_identifier
system_credentials = cls.get_oauth_client(tenant_id, provider_id, provider_controller=provider_controller)
# Refresh token
oauth_handler = OAuthHandler()
refreshed_credentials = oauth_handler.refresh_credentials(
tenant_id=tenant_id,
user_id=subscription.user_id,
plugin_id=provider_id.plugin_id,
provider=provider_id.provider_name,
redirect_uri=redirect_uri,
system_credentials=dict(system_credentials.credentials) if system_credentials is not None else {},
plugin_unique_identifier=(
system_credentials.plugin_unique_identifier
if system_credentials is not None
else plugin_unique_identifier
),
credentials=current_credentials,
)
# Update credentials
subscription.credentials = dict(encrypter.encrypt(dict(refreshed_credentials.credentials)))
subscription.credential_expires_at = refreshed_credentials.expires_at
provider_id_value = subscription.provider_id
result = {
"result": "success",
"expires_at": refreshed_credentials.expires_at,
}
# Clear the trigger runtime credential cache after the DB commit so dispatch uses the refreshed token.
delete_cache_for_subscription(
tenant_id=tenant_id,
provider_id=provider_id_value,
subscription_id=subscription_id,
)
return result
@classmethod
def refresh_subscription(
cls,
tenant_id: str,
subscription_id: str,
now: int | None = None,
) -> Mapping[str, Any]:
"""
Refresh trigger subscription if expired.
Args:
tenant_id: Tenant ID
subscription_id: Subscription instance ID
now: Current timestamp, defaults to `int(time.time())`
Returns:
Mapping with keys: `result` ("success"|"skipped") and `expires_at` (new or existing value)
"""
now_ts: int = int(now if now is not None else _time.time())
with sessionmaker(bind=db.engine).begin() as session:
subscription = session.scalar(
select(TriggerSubscription)
.where(
TriggerSubscription.tenant_id == tenant_id,
TriggerSubscription.id == subscription_id,
)
.limit(1)
)
if subscription is None:
raise ValueError(f"Trigger provider subscription {subscription_id} not found")
if subscription.expires_at == -1 or int(subscription.expires_at) > now_ts:
logger.debug(
"Subscription not due for refresh: tenant=%s id=%s expires_at=%s now=%s",
tenant_id,
subscription_id,
subscription.expires_at,
now_ts,
)
return {"result": "skipped", "expires_at": int(subscription.expires_at)}
provider_id = TriggerProviderID(subscription.provider_id)
controller: PluginTriggerProviderController = TriggerManager.get_trigger_provider(
tenant_id=tenant_id, provider_id=provider_id
)
# Decrypt credentials and properties for runtime
credential_encrypter, _ = create_trigger_provider_encrypter_for_subscription(
tenant_id=tenant_id,
controller=controller,
subscription=subscription,
)
properties_encrypter, properties_cache = create_trigger_provider_encrypter_for_properties(
tenant_id=tenant_id,
controller=controller,
subscription=subscription,
)
decrypted_credentials = credential_encrypter.decrypt(subscription.credentials)
decrypted_properties = properties_encrypter.decrypt(subscription.properties)
sub_entity: TriggerSubscriptionEntity = TriggerSubscriptionEntity(
expires_at=int(subscription.expires_at),
endpoint=generate_plugin_trigger_endpoint_url(subscription.endpoint_id),
parameters=subscription.parameters,
properties=decrypted_properties,
)
refreshed: TriggerSubscriptionEntity = controller.refresh_trigger(
subscription=sub_entity,
credentials=decrypted_credentials,
credential_type=CredentialType.of(subscription.credential_type),
)
# Persist refreshed properties and expires_at
subscription.properties = dict(properties_encrypter.encrypt(dict(refreshed.properties)))
subscription.expires_at = int(refreshed.expires_at)
logger.info(
"Subscription refreshed (service): tenant=%s id=%s new_expires_at=%s",
tenant_id,
subscription_id,
subscription.expires_at,
)
result = {"result": "success", "expires_at": int(refreshed.expires_at)}
properties_cache.delete()
return result
@classmethod
def get_oauth_client(
cls,
tenant_id: str,
provider_id: TriggerProviderID,
*,
provider_controller: PluginTriggerProviderController | None = None,
) -> PluginOAuthClientConfig | None:
"""
Get OAuth client configuration for a provider.
First tries tenant-level OAuth, then falls back to system OAuth.
:param tenant_id: Tenant ID
:param provider_id: Provider identifier
:return: OAuth client configuration or None
"""
if provider_controller is None:
provider_controller = TriggerManager.get_trigger_provider(tenant_id=tenant_id, provider_id=provider_id)
plugin_unique_identifier = provider_controller.plugin_unique_identifier
with Session(db.engine, expire_on_commit=False) as session:
tenant_client = session.scalar(
select(TriggerOAuthTenantClient)
.where(
TriggerOAuthTenantClient.tenant_id == tenant_id,
TriggerOAuthTenantClient.provider == provider_id.provider_name,
TriggerOAuthTenantClient.plugin_id == provider_id.plugin_id,
TriggerOAuthTenantClient.enabled.is_(True),
)
.limit(1)
)
oauth_params: Mapping[str, Any] | None = None
if tenant_client:
encrypter, _ = create_provider_encrypter(
tenant_id=tenant_id,
config=[x.to_basic_provider_config() for x in provider_controller.get_oauth_client_schema()],
cache=NoOpProviderCredentialCache(),
)
oauth_params = encrypter.decrypt(dict(tenant_client.oauth_params))
return PluginOAuthClientConfig(plugin_unique_identifier, dict(oauth_params))
is_verified = PluginService.is_plugin_verified(tenant_id, plugin_unique_identifier)
if not is_verified:
return None
# Check for system-level OAuth client
system_client = session.scalar(
select(TriggerOAuthSystemClient)
.where(
TriggerOAuthSystemClient.plugin_id == provider_id.plugin_id,
TriggerOAuthSystemClient.provider == provider_id.provider_name,
)
.limit(1)
)
if system_client:
try:
oauth_params = decrypt_system_params(system_client.encrypted_oauth_params)
except Exception as e:
raise ValueError(f"Error decrypting system oauth params: {e}") from e
return (
PluginOAuthClientConfig(plugin_unique_identifier, dict(oauth_params))
if oauth_params is not None
else None
)
@classmethod
def is_oauth_system_client_exists(cls, tenant_id: str, provider_id: TriggerProviderID) -> bool:
"""
Check if system OAuth client exists for a trigger provider.
"""
provider_controller = TriggerManager.get_trigger_provider(tenant_id=tenant_id, provider_id=provider_id)
is_verified = PluginService.is_plugin_verified(tenant_id, provider_controller.plugin_unique_identifier)
if not is_verified:
return False
with Session(db.engine, expire_on_commit=False) as session:
system_client = session.scalar(
select(TriggerOAuthSystemClient)
.where(
TriggerOAuthSystemClient.plugin_id == provider_id.plugin_id,
TriggerOAuthSystemClient.provider == provider_id.provider_name,
)
.limit(1)
)
return system_client is not None
@classmethod
def save_custom_oauth_client_params(
cls,
tenant_id: str,
provider_id: TriggerProviderID,
client_params: Mapping[str, Any] | None = None,
enabled: bool | None = None,
) -> Mapping[str, Any]:
"""
Save or update custom OAuth client parameters for a trigger provider.
:param tenant_id: Tenant ID
:param provider_id: Provider identifier
:param client_params: OAuth client parameters (client_id, client_secret, etc.)
:param enabled: Enable/disable the custom OAuth client
:return: Success response
"""
if client_params is None and enabled is None:
return {"result": "success"}
# Get provider controller to access schema
provider_controller: PluginTriggerProviderController = TriggerManager.get_trigger_provider(
tenant_id=tenant_id, provider_id=provider_id
)
with sessionmaker(bind=db.engine).begin() as session:
# Find existing custom client params
custom_client = session.scalar(
select(TriggerOAuthTenantClient)
.where(
TriggerOAuthTenantClient.tenant_id == tenant_id,
TriggerOAuthTenantClient.plugin_id == provider_id.plugin_id,
TriggerOAuthTenantClient.provider == provider_id.provider_name,
)
.limit(1)
)
# Create new record if doesn't exist
if custom_client is None:
custom_client = TriggerOAuthTenantClient(
tenant_id=tenant_id,
plugin_id=provider_id.plugin_id,
provider=provider_id.provider_name,
)
session.add(custom_client)
# Update client params if provided
if client_params is None:
custom_client.encrypted_oauth_params = json.dumps({})
else:
encrypter, cache = create_provider_encrypter(
tenant_id=tenant_id,
config=[x.to_basic_provider_config() for x in provider_controller.get_oauth_client_schema()],
cache=NoOpProviderCredentialCache(),
)
# Handle hidden values
original_params = encrypter.decrypt(dict(custom_client.oauth_params))
new_params: dict[str, Any] = {
key: value if value != HIDDEN_VALUE else original_params.get(key, UNKNOWN_VALUE)
for key, value in client_params.items()
}
custom_client.encrypted_oauth_params = json.dumps(encrypter.encrypt(new_params))
cache.delete()
# Update enabled status if provided
if enabled is not None:
custom_client.enabled = enabled
return {"result": "success"}
@classmethod
def get_custom_oauth_client_params(cls, tenant_id: str, provider_id: TriggerProviderID) -> Mapping[str, Any]:
"""
Get custom OAuth client parameters for a trigger provider.
:param tenant_id: Tenant ID
:param provider_id: Provider identifier
:return: Masked OAuth client parameters
"""
with Session(db.engine) as session:
custom_client = session.scalar(
select(TriggerOAuthTenantClient)
.where(
TriggerOAuthTenantClient.tenant_id == tenant_id,
TriggerOAuthTenantClient.plugin_id == provider_id.plugin_id,
TriggerOAuthTenantClient.provider == provider_id.provider_name,
)
.limit(1)
)
if custom_client is None:
return {}
# Get provider controller to access schema
provider_controller: PluginTriggerProviderController = TriggerManager.get_trigger_provider(
tenant_id=tenant_id, provider_id=provider_id
)
# Create encrypter to decrypt and mask values
encrypter, _ = create_provider_encrypter(
tenant_id=tenant_id,
config=[x.to_basic_provider_config() for x in provider_controller.get_oauth_client_schema()],
cache=NoOpProviderCredentialCache(),
)
return encrypter.mask_plugin_credentials(encrypter.decrypt(dict(custom_client.oauth_params)))
@classmethod
def delete_custom_oauth_client_params(cls, tenant_id: str, provider_id: TriggerProviderID) -> Mapping[str, Any]:
"""
Delete custom OAuth client parameters for a trigger provider.
:param tenant_id: Tenant ID
:param provider_id: Provider identifier
:return: Success response
"""
with sessionmaker(bind=db.engine).begin() as session:
session.execute(
delete(TriggerOAuthTenantClient)
.where(
TriggerOAuthTenantClient.tenant_id == tenant_id,
TriggerOAuthTenantClient.provider == provider_id.provider_name,
TriggerOAuthTenantClient.plugin_id == provider_id.plugin_id,
)
.execution_options(synchronize_session=False)
)
return {"result": "success"}
@classmethod
def is_oauth_custom_client_enabled(cls, tenant_id: str, provider_id: TriggerProviderID) -> bool:
"""
Check if custom OAuth client is enabled for a trigger provider.
:param tenant_id: Tenant ID
:param provider_id: Provider identifier
:return: True if enabled, False otherwise
"""
with Session(db.engine, expire_on_commit=False) as session:
custom_client = session.scalar(
select(TriggerOAuthTenantClient)
.where(
TriggerOAuthTenantClient.tenant_id == tenant_id,
TriggerOAuthTenantClient.plugin_id == provider_id.plugin_id,
TriggerOAuthTenantClient.provider == provider_id.provider_name,
TriggerOAuthTenantClient.enabled.is_(True),
)
.limit(1)
)
return custom_client is not None
@classmethod
def get_subscription_by_endpoint(cls, endpoint_id: str) -> TriggerSubscription | None:
"""
Get a trigger subscription by the endpoint ID.
"""
with Session(db.engine, expire_on_commit=False) as session:
subscription = session.scalar(
select(TriggerSubscription).where(TriggerSubscription.endpoint_id == endpoint_id).limit(1)
)
if not subscription:
return None
provider_controller: PluginTriggerProviderController = TriggerManager.get_trigger_provider(
tenant_id=subscription.tenant_id, provider_id=TriggerProviderID(subscription.provider_id)
)
credential_encrypter, _ = create_trigger_provider_encrypter_for_subscription(
tenant_id=subscription.tenant_id,
controller=provider_controller,
subscription=subscription,
)
subscription.credentials = dict(credential_encrypter.decrypt(subscription.credentials))
properties_encrypter, _ = create_trigger_provider_encrypter_for_properties(
tenant_id=subscription.tenant_id,
controller=provider_controller,
subscription=subscription,
)
subscription.properties = dict(properties_encrypter.decrypt(subscription.properties))
return subscription
@classmethod
def verify_subscription_credentials(
cls,
tenant_id: str,
user_id: str,
provider_id: TriggerProviderID,
subscription_id: str,
credentials: dict[str, Any],
) -> VerifyCredentialsResult:
"""
Verify credentials for an existing subscription without updating it.
This is used in edit mode to validate new credentials before rebuild.
:param tenant_id: Tenant ID
:param user_id: User ID
:param provider_id: Provider identifier
:param subscription_id: Subscription ID
:param credentials: New credentials to verify
:return: dict with 'verified' boolean
"""
provider_controller = TriggerManager.get_trigger_provider(tenant_id, provider_id)
if not provider_controller:
raise ValueError(f"Provider {provider_id} not found")
subscription = cls.get_subscription_by_id(
tenant_id=tenant_id,
subscription_id=subscription_id,
)
if not subscription:
raise ValueError(f"Subscription {subscription_id} not found")
credential_type = CredentialType.of(subscription.credential_type)
# For API Key, validate the new credentials
if credential_type == CredentialType.API_KEY:
new_credentials: dict[str, Any] = {
key: value if value != HIDDEN_VALUE else subscription.credentials.get(key, UNKNOWN_VALUE)
for key, value in credentials.items()
}
try:
provider_controller.validate_credentials(user_id, credentials=new_credentials)
return {"verified": True}
except Exception as e:
raise ValueError(f"Invalid credentials: {e}") from e
return {"verified": True}
@classmethod
def rebuild_trigger_subscription(
cls,
tenant_id: str,
provider_id: TriggerProviderID,
subscription_id: str,
credentials: Mapping[str, Any],
parameters: Mapping[str, Any],
name: str | None = None,
) -> None:
"""
Create a subscription builder for rebuilding an existing subscription.
This method rebuild the subscription by call DELETE and CREATE API of the third party provider(e.g. GitHub)
keeping the same subscription_id and endpoint_id so the webhook URL remains unchanged.
:param tenant_id: Tenant ID
:param name: Name for the subscription
:param subscription_id: Subscription ID
:param provider_id: Provider identifier
:param credentials: Credentials for the subscription
:param parameters: Parameters for the subscription
:return: SubscriptionBuilderApiEntity
"""
provider_controller = TriggerManager.get_trigger_provider(tenant_id, provider_id)
if not provider_controller:
raise ValueError(f"Provider {provider_id} not found")
subscription = TriggerProviderService.get_subscription_by_id(
tenant_id=tenant_id,
subscription_id=subscription_id,
)
if not subscription:
raise ValueError(f"Subscription {subscription_id} not found")
credential_type = CredentialType.of(subscription.credential_type)
if credential_type not in {CredentialType.OAUTH2, CredentialType.API_KEY}:
raise ValueError(f"Credential type {credential_type} not supported for auto creation")
# Delete the previous subscription
user_id = subscription.user_id
unsubscribe_result = TriggerManager.unsubscribe_trigger(
tenant_id=tenant_id,
user_id=user_id,
provider_id=provider_id,
subscription=subscription.to_entity(),
credentials=subscription.credentials,
credential_type=credential_type,
)
if not unsubscribe_result.success:
raise ValueError(f"Failed to delete previous subscription: {unsubscribe_result.message}")
# Create a new subscription with the same subscription_id and endpoint_id
new_credentials: dict[str, Any] = {
key: value if value != HIDDEN_VALUE else subscription.credentials.get(key, UNKNOWN_VALUE)
for key, value in credentials.items()
}
new_subscription: TriggerSubscriptionEntity = TriggerManager.subscribe_trigger(
tenant_id=tenant_id,
user_id=user_id,
provider_id=provider_id,
endpoint=generate_plugin_trigger_endpoint_url(subscription.endpoint_id),
parameters=parameters,
credentials=new_credentials,
credential_type=credential_type,
)
TriggerProviderService.update_trigger_subscription(
tenant_id=tenant_id,
subscription_id=subscription.id,
name=name,
parameters=parameters,
credentials=new_credentials,
properties=new_subscription.properties,
expires_at=new_subscription.expires_at,
)
@@ -0,0 +1,65 @@
from collections.abc import Mapping
from typing import Any
from flask import Request
from pydantic import TypeAdapter
from core.plugin.utils.http_parser import deserialize_request, serialize_request
from extensions.ext_storage import storage
class TriggerHttpRequestCachingService:
"""
Service for caching trigger requests.
"""
_TRIGGER_STORAGE_PATH = "triggers"
@classmethod
def get_request(cls, request_id: str) -> Request:
"""
Get the request object from the storage.
Args:
request_id: The ID of the request.
Returns:
The request object.
"""
return deserialize_request(storage.load_once(f"{cls._TRIGGER_STORAGE_PATH}/{request_id}.raw"))
@classmethod
def get_payload(cls, request_id: str) -> Mapping[str, Any]:
"""
Get the payload from the storage.
Args:
request_id: The ID of the request.
Returns:
The payload.
"""
return TypeAdapter(Mapping[str, Any]).validate_json(
storage.load_once(f"{cls._TRIGGER_STORAGE_PATH}/{request_id}.payload")
)
@classmethod
def persist_request(cls, request_id: str, request: Request) -> None:
"""
Persist the request in the storage.
Args:
request_id: The ID of the request.
request: The request object.
"""
storage.save(f"{cls._TRIGGER_STORAGE_PATH}/{request_id}.raw", serialize_request(request))
@classmethod
def persist_payload(cls, request_id: str, payload: Mapping[str, Any]) -> None:
"""
Persist the payload in the storage.
"""
storage.save(
f"{cls._TRIGGER_STORAGE_PATH}/{request_id}.payload",
TypeAdapter(Mapping[str, Any]).dump_json(payload), # type: ignore
)
@@ -0,0 +1,302 @@
import logging
import secrets
import time
from collections.abc import Mapping
from typing import Any
from flask import Request, Response
from pydantic import BaseModel
from sqlalchemy import select
from sqlalchemy.orm import sessionmaker
from core.plugin.entities.plugin_daemon import CredentialType
from core.plugin.entities.request import TriggerDispatchResponse, TriggerInvokeEventResponse
from core.plugin.impl.exc import PluginNotFoundError
from core.trigger.constants import TRIGGER_PLUGIN_NODE_TYPE
from core.trigger.debug.events import PluginTriggerDebugEvent
from core.trigger.provider import PluginTriggerProviderController
from core.trigger.trigger_manager import TriggerManager
from core.trigger.utils.encryption import create_trigger_provider_encrypter_for_subscription
from core.workflow.nodes.trigger_plugin.entities import TriggerEventNodeData
from extensions.ext_database import db
from extensions.ext_redis import redis_client
from graphon.entities.graph_config import NodeConfigDict
from models.model import App
from models.provider_ids import TriggerProviderID
from models.trigger import TriggerSubscription, WorkflowPluginTrigger
from models.workflow import Workflow
from services.trigger.trigger_provider_service import TriggerProviderService
from services.trigger.trigger_request_service import TriggerHttpRequestCachingService
from services.workflow.entities import PluginTriggerDispatchData
from tasks.trigger_processing_tasks import dispatch_triggered_workflows_async
logger = logging.getLogger(__name__)
class TriggerService:
__TEMPORARY_ENDPOINT_EXPIRE_MS__ = 5 * 60 * 1000
__ENDPOINT_REQUEST_CACHE_COUNT__ = 10
__ENDPOINT_REQUEST_CACHE_EXPIRE_MS__ = 5 * 60 * 1000
__PLUGIN_TRIGGER_NODE_CACHE_KEY__ = "plugin_trigger_nodes"
MAX_PLUGIN_TRIGGER_NODES_PER_WORKFLOW = 5 # Maximum allowed plugin trigger nodes per workflow
@classmethod
def invoke_trigger_event(
cls, tenant_id: str, user_id: str, node_config: NodeConfigDict, event: PluginTriggerDebugEvent
) -> TriggerInvokeEventResponse:
"""Invoke a trigger event."""
subscription: TriggerSubscription | None = TriggerProviderService.get_subscription_by_id(
tenant_id=tenant_id,
subscription_id=event.subscription_id,
)
if not subscription:
raise ValueError("Subscription not found")
node_data = TriggerEventNodeData.model_validate(node_config["data"], from_attributes=True)
request = TriggerHttpRequestCachingService.get_request(event.request_id)
payload = TriggerHttpRequestCachingService.get_payload(event.request_id)
# invoke triger
provider_controller: PluginTriggerProviderController = TriggerManager.get_trigger_provider(
tenant_id, TriggerProviderID(subscription.provider_id)
)
return TriggerManager.invoke_trigger_event(
tenant_id=tenant_id,
user_id=user_id,
provider_id=TriggerProviderID(event.provider_id),
event_name=event.name,
parameters=node_data.resolve_parameters(
parameter_schemas=provider_controller.get_event_parameters(event_name=event.name)
),
credentials=subscription.credentials,
credential_type=CredentialType.of(subscription.credential_type),
subscription=subscription.to_entity(),
request=request,
payload=payload,
)
@classmethod
def process_endpoint(cls, endpoint_id: str, request: Request) -> Response | None:
"""
Extract and process data from incoming endpoint request.
Args:
endpoint_id: Endpoint ID
request: Request
"""
timestamp = int(time.time())
subscription: TriggerSubscription | None = None
try:
subscription = TriggerProviderService.get_subscription_by_endpoint(endpoint_id)
except PluginNotFoundError:
return Response(status=404, response="Trigger provider not found")
except Exception:
return Response(status=500, response="Failed to get subscription by endpoint")
if not subscription:
return None
provider_id = TriggerProviderID(subscription.provider_id)
controller: PluginTriggerProviderController = TriggerManager.get_trigger_provider(
tenant_id=subscription.tenant_id, provider_id=provider_id
)
encrypter, _ = create_trigger_provider_encrypter_for_subscription(
tenant_id=subscription.tenant_id,
controller=controller,
subscription=subscription,
)
dispatch_response: TriggerDispatchResponse = controller.dispatch(
request=request,
subscription=subscription.to_entity(),
credentials=encrypter.decrypt(subscription.credentials),
credential_type=CredentialType.of(subscription.credential_type),
)
if dispatch_response.events:
request_id = f"trigger_request_{timestamp}_{secrets.token_hex(6)}"
# save the request and payload to storage as persistent data
TriggerHttpRequestCachingService.persist_request(request_id, request)
TriggerHttpRequestCachingService.persist_payload(request_id, dispatch_response.payload)
# Validate event names
for event_name in dispatch_response.events:
if controller.get_event(event_name) is None:
logger.error(
"Event name %s not found in provider %s for endpoint %s",
event_name,
subscription.provider_id,
endpoint_id,
)
raise ValueError(f"Event name {event_name} not found in provider {subscription.provider_id}")
plugin_trigger_dispatch_data = PluginTriggerDispatchData(
user_id=dispatch_response.user_id,
tenant_id=subscription.tenant_id,
endpoint_id=endpoint_id,
provider_id=subscription.provider_id,
subscription_id=subscription.id,
timestamp=timestamp,
events=list(dispatch_response.events),
request_id=request_id,
)
dispatch_data = plugin_trigger_dispatch_data.model_dump(mode="json")
dispatch_triggered_workflows_async.delay(dispatch_data)
logger.info(
"Queued async dispatching for %d triggers on endpoint %s with request_id %s",
len(dispatch_response.events),
endpoint_id,
request_id,
)
return dispatch_response.response
@classmethod
def sync_plugin_trigger_relationships(cls, app: App, workflow: Workflow):
"""
Sync plugin trigger relationships in DB.
1. Check if the workflow has any plugin trigger nodes
2. Fetch the nodes from DB, see if there were any plugin trigger records already
3. Diff the nodes and the plugin trigger records, create/update/delete the records as needed
Approach:
Frequent DB operations may cause performance issues, using Redis to cache it instead.
If any record exists, cache it.
Limits:
- Maximum 5 plugin trigger nodes per workflow
"""
class Cache(BaseModel):
"""
Cache model for plugin trigger nodes
"""
record_id: str
node_id: str
provider_id: str
event_name: str
subscription_id: str
# Walk nodes to find plugin triggers
nodes_in_graph: list[Mapping[str, Any]] = []
for node_id, node_config in workflow.walk_nodes(TRIGGER_PLUGIN_NODE_TYPE):
# Extract plugin trigger configuration from node
plugin_id = node_config.get("plugin_id", "")
provider_id = node_config.get("provider_id", "")
event_name = node_config.get("event_name", "")
subscription_id = node_config.get("subscription_id", "")
if not subscription_id:
continue
nodes_in_graph.append(
{
"node_id": node_id,
"plugin_id": plugin_id,
"provider_id": provider_id,
"event_name": event_name,
"subscription_id": subscription_id,
}
)
# Check plugin trigger node limit
if len(nodes_in_graph) > cls.MAX_PLUGIN_TRIGGER_NODES_PER_WORKFLOW:
raise ValueError(
f"Workflow exceeds maximum plugin trigger node limit. "
f"Found {len(nodes_in_graph)} plugin trigger nodes, "
f"maximum allowed is {cls.MAX_PLUGIN_TRIGGER_NODES_PER_WORKFLOW}"
)
not_found_in_cache: list[Mapping[str, Any]] = []
for node_info in nodes_in_graph:
node_id = node_info["node_id"]
# firstly check if the node exists in cache
if not redis_client.get(f"{cls.__PLUGIN_TRIGGER_NODE_CACHE_KEY__}:{app.id}:{node_id}"):
not_found_in_cache.append(node_info)
continue
with sessionmaker(bind=db.engine, expire_on_commit=False).begin() as session:
try:
# lock the concurrent plugin trigger creation
redis_client.lock(f"{cls.__PLUGIN_TRIGGER_NODE_CACHE_KEY__}:apps:{app.id}:lock", timeout=10)
# fetch the non-cached nodes from DB
all_records = session.scalars(
select(WorkflowPluginTrigger).where(
WorkflowPluginTrigger.app_id == app.id,
WorkflowPluginTrigger.tenant_id == app.tenant_id,
)
).all()
nodes_id_in_db = {node.node_id: node for node in all_records}
nodes_id_in_graph = {node["node_id"] for node in nodes_in_graph}
# get the nodes not found both in cache and DB
nodes_not_found = [
node_info for node_info in not_found_in_cache if node_info["node_id"] not in nodes_id_in_db
]
# create new plugin trigger records
for node_info in nodes_not_found:
plugin_trigger = WorkflowPluginTrigger(
app_id=app.id,
tenant_id=app.tenant_id,
node_id=node_info["node_id"],
provider_id=node_info["provider_id"],
event_name=node_info["event_name"],
subscription_id=node_info["subscription_id"],
)
session.add(plugin_trigger)
session.flush() # Get the ID for caching
cache = Cache(
record_id=plugin_trigger.id,
node_id=node_info["node_id"],
provider_id=node_info["provider_id"],
event_name=node_info["event_name"],
subscription_id=node_info["subscription_id"],
)
redis_client.set(
f"{cls.__PLUGIN_TRIGGER_NODE_CACHE_KEY__}:{app.id}:{node_info['node_id']}",
cache.model_dump_json(),
ex=60 * 60,
)
# Update existing records if subscription_id changed
for node_info in nodes_in_graph:
node_id = node_info["node_id"]
if node_id in nodes_id_in_db:
existing_record = nodes_id_in_db[node_id]
if (
existing_record.subscription_id != node_info["subscription_id"]
or existing_record.provider_id != node_info["provider_id"]
or existing_record.event_name != node_info["event_name"]
):
existing_record.subscription_id = node_info["subscription_id"]
existing_record.provider_id = node_info["provider_id"]
existing_record.event_name = node_info["event_name"]
session.add(existing_record)
# Update cache
cache = Cache(
record_id=existing_record.id,
node_id=node_id,
provider_id=node_info["provider_id"],
event_name=node_info["event_name"],
subscription_id=node_info["subscription_id"],
)
redis_client.set(
f"{cls.__PLUGIN_TRIGGER_NODE_CACHE_KEY__}:{app.id}:{node_id}",
cache.model_dump_json(),
ex=60 * 60,
)
# delete the nodes not found in the graph
for node_id in nodes_id_in_db:
if node_id not in nodes_id_in_graph:
session.delete(nodes_id_in_db[node_id])
redis_client.delete(f"{cls.__PLUGIN_TRIGGER_NODE_CACHE_KEY__}:{app.id}:{node_id}")
except Exception:
logger.exception("Failed to sync plugin trigger relationships for app %s", app.id)
raise
finally:
redis_client.delete(f"{cls.__PLUGIN_TRIGGER_NODE_CACHE_KEY__}:apps:{app.id}:lock")
@@ -0,0 +1,465 @@
import logging
import uuid
from collections.abc import Mapping
from contextlib import contextmanager
from datetime import datetime
from typing import Any
from flask import Request, Response
from pydantic import TypeAdapter
from core.plugin.entities.plugin_daemon import CredentialType
from core.plugin.entities.request import TriggerDispatchResponse
from core.tools.errors import ToolProviderCredentialValidationError
from core.trigger.entities.api_entities import SubscriptionBuilderApiEntity
from core.trigger.entities.entities import (
RequestLog,
Subscription,
SubscriptionBuilder,
SubscriptionBuilderUpdater,
SubscriptionConstructor,
)
from core.trigger.provider import PluginTriggerProviderController
from core.trigger.trigger_manager import TriggerManager
from core.trigger.utils.encryption import masked_credentials
from core.trigger.utils.endpoint import generate_plugin_trigger_endpoint_url
from extensions.ext_redis import redis_client
from models.provider_ids import TriggerProviderID
from services.trigger.trigger_provider_service import TriggerProviderService
logger = logging.getLogger(__name__)
_request_logs_adapter: TypeAdapter[list[RequestLog]] = TypeAdapter(list[RequestLog])
class TriggerSubscriptionBuilderService:
"""Service for managing trigger providers and credentials"""
##########################
# Trigger provider
##########################
__MAX_TRIGGER_PROVIDER_COUNT__ = 10
##########################
# Builder endpoint
##########################
__BUILDER_CACHE_EXPIRE_SECONDS__ = 30 * 60
__VALIDATION_REQUEST_CACHE_COUNT__ = 10
__VALIDATION_REQUEST_CACHE_EXPIRE_SECONDS__ = 30 * 60
##########################
# Distributed lock
##########################
__LOCK_EXPIRE_SECONDS__ = 30
@classmethod
def encode_cache_key(cls, subscription_id: str) -> str:
return f"trigger:subscription:builder:{subscription_id}"
@classmethod
def encode_lock_key(cls, subscription_id: str) -> str:
return f"trigger:subscription:builder:lock:{subscription_id}"
@classmethod
@contextmanager
def acquire_builder_lock(cls, subscription_id: str):
"""
Acquire a distributed lock for a subscription builder.
:param subscription_id: The subscription builder ID
"""
lock_key = cls.encode_lock_key(subscription_id)
with redis_client.lock(lock_key, timeout=cls.__LOCK_EXPIRE_SECONDS__):
yield
@classmethod
def create_trigger_subscription_builder(
cls,
tenant_id: str,
user_id: str,
provider_id: TriggerProviderID,
credential_type: CredentialType,
) -> SubscriptionBuilderApiEntity:
"""
Add a new trigger subscription validation.
"""
provider_controller = TriggerManager.get_trigger_provider(tenant_id, provider_id)
if not provider_controller:
raise ValueError(f"Provider {provider_id} not found")
subscription_constructor: SubscriptionConstructor | None = provider_controller.get_subscription_constructor()
subscription_id = str(uuid.uuid4())
subscription_builder = SubscriptionBuilder(
id=subscription_id,
name=None,
endpoint_id=subscription_id,
tenant_id=tenant_id,
user_id=user_id,
provider_id=str(provider_id),
parameters=subscription_constructor.get_default_parameters() if subscription_constructor else {},
properties=provider_controller.get_subscription_default_properties(),
credentials={},
credential_type=credential_type,
credential_expires_at=-1,
expires_at=-1,
)
cache_key = cls.encode_cache_key(subscription_id)
redis_client.setex(cache_key, cls.__BUILDER_CACHE_EXPIRE_SECONDS__, subscription_builder.model_dump_json())
return cls.builder_to_api_entity(controller=provider_controller, entity=subscription_builder)
@classmethod
def update_trigger_subscription_builder(
cls,
tenant_id: str,
user_id: str,
provider_id: TriggerProviderID,
subscription_builder_id: str,
subscription_builder_updater: SubscriptionBuilderUpdater,
) -> SubscriptionBuilderApiEntity:
"""
Update a trigger subscription validation.
"""
subscription_id = subscription_builder_id
provider_controller = TriggerManager.get_trigger_provider(tenant_id, provider_id)
if not provider_controller:
raise ValueError(f"Provider {provider_id} not found")
# Acquire lock to prevent concurrent updates
with cls.acquire_builder_lock(subscription_id):
cache_key = cls.encode_cache_key(subscription_id)
subscription_builder_cache = cls._require_owned_subscription_builder(
tenant_id=tenant_id,
user_id=user_id,
provider_id=provider_id,
subscription_builder_id=subscription_builder_id,
)
subscription_builder_updater.update(subscription_builder_cache)
redis_client.setex(
cache_key, cls.__BUILDER_CACHE_EXPIRE_SECONDS__, subscription_builder_cache.model_dump_json()
)
return cls.builder_to_api_entity(controller=provider_controller, entity=subscription_builder_cache)
@classmethod
def update_and_verify_builder(
cls,
tenant_id: str,
user_id: str,
provider_id: TriggerProviderID,
subscription_builder_id: str,
subscription_builder_updater: SubscriptionBuilderUpdater,
) -> Mapping[str, Any]:
"""
Atomically update and verify a subscription builder.
This ensures the verification is done on the exact data that was just updated.
"""
subscription_id = subscription_builder_id
provider_controller = TriggerManager.get_trigger_provider(tenant_id, provider_id)
if not provider_controller:
raise ValueError(f"Provider {provider_id} not found")
# Acquire lock for the entire update + verify operation
with cls.acquire_builder_lock(subscription_id):
cache_key = cls.encode_cache_key(subscription_id)
subscription_builder_cache = cls._require_owned_subscription_builder(
tenant_id=tenant_id,
user_id=user_id,
provider_id=provider_id,
subscription_builder_id=subscription_builder_id,
)
# Update
subscription_builder_updater.update(subscription_builder_cache)
redis_client.setex(
cache_key, cls.__BUILDER_CACHE_EXPIRE_SECONDS__, subscription_builder_cache.model_dump_json()
)
# Verify (using the just-updated data)
if subscription_builder_cache.credential_type == CredentialType.OAUTH2:
return {"verified": bool(subscription_builder_cache.credentials)}
if subscription_builder_cache.credential_type == CredentialType.API_KEY:
credentials_to_validate = subscription_builder_cache.credentials
try:
provider_controller.validate_credentials(user_id, credentials_to_validate)
except ToolProviderCredentialValidationError as e:
raise ValueError(f"Invalid credentials: {e}") from e
return {"verified": True}
return {"verified": True}
@classmethod
def update_and_build_builder(
cls,
tenant_id: str,
user_id: str,
provider_id: TriggerProviderID,
subscription_builder_id: str,
subscription_builder_updater: SubscriptionBuilderUpdater,
) -> None:
"""
Atomically update and build a subscription builder.
This ensures the build uses the exact data that was just updated.
"""
subscription_id = subscription_builder_id
provider_controller = TriggerManager.get_trigger_provider(tenant_id, provider_id)
if not provider_controller:
raise ValueError(f"Provider {provider_id} not found")
# Acquire lock for the entire update + build operation
with cls.acquire_builder_lock(subscription_id):
cache_key = cls.encode_cache_key(subscription_id)
subscription_builder = cls._require_owned_subscription_builder(
tenant_id=tenant_id,
user_id=user_id,
provider_id=provider_id,
subscription_builder_id=subscription_builder_id,
)
# Update
subscription_builder_updater.update(subscription_builder)
redis_client.setex(cache_key, cls.__BUILDER_CACHE_EXPIRE_SECONDS__, subscription_builder.model_dump_json())
if not subscription_builder.name:
raise ValueError("Subscription builder name is required")
# Build
credential_type = CredentialType.of(subscription_builder.credential_type or CredentialType.UNAUTHORIZED)
if credential_type == CredentialType.UNAUTHORIZED:
# manually create
TriggerProviderService.add_trigger_subscription(
subscription_id=subscription_builder.id,
tenant_id=tenant_id,
user_id=user_id,
name=subscription_builder.name,
provider_id=provider_id,
endpoint_id=subscription_builder.endpoint_id,
parameters=subscription_builder.parameters,
properties=subscription_builder.properties,
credential_expires_at=subscription_builder.credential_expires_at or -1,
expires_at=subscription_builder.expires_at,
credentials=subscription_builder.credentials,
credential_type=credential_type,
)
else:
# automatically create
subscription: Subscription = TriggerManager.subscribe_trigger(
tenant_id=tenant_id,
user_id=user_id,
provider_id=provider_id,
endpoint=generate_plugin_trigger_endpoint_url(subscription_builder.endpoint_id),
parameters=subscription_builder.parameters,
credentials=subscription_builder.credentials,
credential_type=credential_type,
)
TriggerProviderService.add_trigger_subscription(
subscription_id=subscription_builder.id,
tenant_id=tenant_id,
user_id=user_id,
name=subscription_builder.name,
provider_id=provider_id,
endpoint_id=subscription_builder.endpoint_id,
parameters=subscription_builder.parameters,
properties=subscription.properties,
credentials=subscription_builder.credentials,
credential_type=credential_type,
credential_expires_at=subscription_builder.credential_expires_at or -1,
expires_at=(
subscription.expires_at
if subscription.expires_at is not None
else (subscription_builder.expires_at or -1)
),
)
# Delete the builder after successful subscription creation
redis_client.delete(cache_key)
@classmethod
def builder_to_api_entity(
cls, controller: PluginTriggerProviderController, entity: SubscriptionBuilder
) -> SubscriptionBuilderApiEntity:
credential_type = CredentialType.of(entity.credential_type or CredentialType.UNAUTHORIZED.value)
return SubscriptionBuilderApiEntity(
id=entity.id,
name=entity.name or "",
provider=entity.provider_id,
endpoint=generate_plugin_trigger_endpoint_url(entity.endpoint_id),
parameters=entity.parameters,
properties=entity.properties,
credential_type=credential_type,
credentials=masked_credentials(
schemas=controller.get_credentials_schema(credential_type),
credentials=entity.credentials,
)
if controller.get_subscription_constructor()
else {},
)
@classmethod
def _get_subscription_builder_by_endpoint_id(cls, endpoint_id: str) -> SubscriptionBuilder | None:
"""Resolve the public validation capability without authenticated owner context."""
cache_key = cls.encode_cache_key(endpoint_id)
subscription_cache = redis_client.get(cache_key)
if subscription_cache:
subscription_builder = SubscriptionBuilder.model_validate_json(subscription_cache)
if subscription_builder.endpoint_id == endpoint_id:
return subscription_builder
return None
@classmethod
def get_subscription_builder(
cls,
tenant_id: str,
user_id: str,
provider_id: TriggerProviderID,
subscription_builder_id: str,
) -> SubscriptionBuilder | None:
"""Return an owned temporary builder, or None when no temporary builder exists."""
subscription_builder = cls._get_subscription_builder_by_endpoint_id(subscription_builder_id)
if subscription_builder is None:
return None
if (
subscription_builder.id != subscription_builder_id
or subscription_builder.tenant_id != tenant_id
or subscription_builder.user_id != user_id
or subscription_builder.provider_id != str(provider_id)
):
raise ValueError(f"Subscription builder {subscription_builder_id} not found")
return subscription_builder
@classmethod
def _require_owned_subscription_builder(
cls,
tenant_id: str,
user_id: str,
provider_id: TriggerProviderID,
subscription_builder_id: str,
) -> SubscriptionBuilder:
"""Return an owned temporary builder or reject an absent capability."""
subscription_builder = cls.get_subscription_builder(
tenant_id=tenant_id,
user_id=user_id,
provider_id=provider_id,
subscription_builder_id=subscription_builder_id,
)
if subscription_builder is None:
raise ValueError(f"Subscription builder {subscription_builder_id} not found")
return subscription_builder
@classmethod
def append_log(cls, endpoint_id: str, request: Request, response: Response) -> None:
"""Append validation request log to Redis."""
log = RequestLog(
id=str(uuid.uuid4()),
endpoint=endpoint_id,
request={
"method": request.method,
"url": request.url,
"headers": dict(request.headers),
"data": request.get_data(as_text=True),
},
response={
"status_code": response.status_code,
"headers": dict(response.headers),
"data": response.get_data(as_text=True),
},
created_at=datetime.now(),
)
key = f"trigger:subscription:builder:logs:{endpoint_id}"
logs = _request_logs_adapter.validate_json(redis_client.get(key) or b"[]")
logs.append(log)
# Keep last N logs
logs = logs[-cls.__VALIDATION_REQUEST_CACHE_COUNT__ :]
redis_client.setex(
key,
cls.__VALIDATION_REQUEST_CACHE_EXPIRE_SECONDS__,
_request_logs_adapter.dump_json(logs),
)
@classmethod
def list_logs(
cls,
tenant_id: str,
user_id: str,
provider_id: TriggerProviderID,
subscription_builder_id: str,
) -> list[RequestLog]:
"""List request logs for validation endpoint."""
subscription_builder = cls._require_owned_subscription_builder(
tenant_id=tenant_id,
user_id=user_id,
provider_id=provider_id,
subscription_builder_id=subscription_builder_id,
)
key = f"trigger:subscription:builder:logs:{subscription_builder.endpoint_id}"
logs_json = redis_client.get(key)
if not logs_json:
return []
return _request_logs_adapter.validate_json(logs_json)
@classmethod
def process_builder_validation_endpoint(cls, endpoint_id: str, request: Request) -> Response | None:
"""
Process a temporary endpoint request.
:param endpoint_id: The endpoint identifier
:param request: The Flask request object
:return: The Flask response object
"""
# check if validation endpoint exists
subscription_builder: SubscriptionBuilder | None = cls._get_subscription_builder_by_endpoint_id(endpoint_id)
if not subscription_builder:
return None
try:
# response to validation endpoint
controller: PluginTriggerProviderController = TriggerManager.get_trigger_provider(
tenant_id=subscription_builder.tenant_id,
provider_id=TriggerProviderID(subscription_builder.provider_id),
)
dispatch_response: TriggerDispatchResponse = controller.dispatch(
request=request,
subscription=subscription_builder.to_subscription(),
credentials={},
credential_type=CredentialType.UNAUTHORIZED,
)
response: Response = dispatch_response.response
# append the request log
cls.append_log(
endpoint_id=endpoint_id,
request=request,
response=response,
)
return response
except Exception:
logger.exception("Error during validation endpoint dispatch for endpoint_id=%s", endpoint_id)
error_response = Response(status=500, response="An internal error has occurred.")
cls.append_log(endpoint_id=endpoint_id, request=request, response=error_response)
return error_response
@classmethod
def get_subscription_builder_by_id(
cls,
tenant_id: str,
user_id: str,
provider_id: TriggerProviderID,
subscription_builder_id: str,
) -> SubscriptionBuilderApiEntity:
"""Get a trigger subscription builder API entity."""
subscription_builder = cls._require_owned_subscription_builder(
tenant_id=tenant_id,
user_id=user_id,
provider_id=provider_id,
subscription_builder_id=subscription_builder_id,
)
return cls.builder_to_api_entity(
controller=TriggerManager.get_trigger_provider(tenant_id, provider_id),
entity=subscription_builder,
)
@@ -0,0 +1,65 @@
from sqlalchemy import and_, select
from sqlalchemy.orm import Session
from extensions.ext_database import db
from models.enums import AppTriggerStatus
from models.trigger import AppTrigger, WorkflowPluginTrigger
class TriggerSubscriptionOperatorService:
@classmethod
def get_subscriber_triggers(
cls, tenant_id: str, subscription_id: str, event_name: str
) -> list[WorkflowPluginTrigger]:
"""
Get WorkflowPluginTriggers for a subscription and trigger.
Args:
tenant_id: Tenant ID
subscription_id: Subscription ID
event_name: Event name
"""
with Session(db.engine, expire_on_commit=False) as session:
subscribers = session.scalars(
select(WorkflowPluginTrigger)
.join(
AppTrigger,
and_(
AppTrigger.tenant_id == WorkflowPluginTrigger.tenant_id,
AppTrigger.app_id == WorkflowPluginTrigger.app_id,
AppTrigger.node_id == WorkflowPluginTrigger.node_id,
),
)
.where(
WorkflowPluginTrigger.tenant_id == tenant_id,
WorkflowPluginTrigger.subscription_id == subscription_id,
WorkflowPluginTrigger.event_name == event_name,
AppTrigger.status == AppTriggerStatus.ENABLED,
)
).all()
return list(subscribers)
@classmethod
def delete_plugin_trigger_by_subscription(cls, tenant_id: str, subscription_id: str, *, session: Session) -> None:
"""Delete a plugin trigger by tenant_id and subscription_id within an existing session
Args:
session: Database session
tenant_id: The tenant ID
subscription_id: The subscription ID
Raises:
NotFound: If plugin trigger not found
"""
# Find plugin trigger using indexed columns
plugin_trigger = session.scalar(
select(WorkflowPluginTrigger).where(
WorkflowPluginTrigger.tenant_id == tenant_id,
WorkflowPluginTrigger.subscription_id == subscription_id,
)
)
if not plugin_trigger:
return
session.delete(plugin_trigger)
File diff suppressed because it is too large. Load diff