chore: 纳入本地 Dify 1.19.0 源码及品牌与免登改动
演示目录只保留可重建的源码。运行密钥、数据卷、依赖和带口令的本地配置不进入仓库。 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
60158935f4
commit
6519682885
15831 files changed
+3063428
No files matched your search
@@ -0,0 +1 @@
|
||||
"""Persistence adapters for the data-source domain."""
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Persistence adapter for data-source API-key authentication bindings."""
|
||||
|
||||
import json
|
||||
from copy import deepcopy
|
||||
from typing import override
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from models.source import DataSourceApiKeyAuthBinding
|
||||
from services.data_source.auth.api_key_service import DataSourceApiKeyAuthBindingRepository
|
||||
from services.data_source.entities.api_key_auth import (
|
||||
DataSourceApiKeyAuthBindingRecord,
|
||||
DataSourceApiKeyAuthCredentials,
|
||||
)
|
||||
|
||||
|
||||
class SQLAlchemyDataSourceApiKeyAuthBindingRepository(DataSourceApiKeyAuthBindingRepository):
|
||||
"""Own a short-lived Session for every binding operation."""
|
||||
|
||||
def __init__(self, session_factory: sessionmaker[Session]) -> None:
|
||||
self._session_factory = session_factory
|
||||
|
||||
@override
|
||||
def list_enabled(self, workspace_id: str) -> tuple[DataSourceApiKeyAuthBindingRecord, ...]:
|
||||
stmt = (
|
||||
select(
|
||||
DataSourceApiKeyAuthBinding.id,
|
||||
DataSourceApiKeyAuthBinding.category,
|
||||
DataSourceApiKeyAuthBinding.provider,
|
||||
DataSourceApiKeyAuthBinding.disabled,
|
||||
DataSourceApiKeyAuthBinding.created_at,
|
||||
DataSourceApiKeyAuthBinding.updated_at,
|
||||
)
|
||||
.where(
|
||||
DataSourceApiKeyAuthBinding.tenant_id == workspace_id,
|
||||
DataSourceApiKeyAuthBinding.disabled.is_(False),
|
||||
)
|
||||
.order_by(DataSourceApiKeyAuthBinding.created_at.asc())
|
||||
)
|
||||
with self._session_factory() as session:
|
||||
rows = session.execute(stmt).all()
|
||||
|
||||
return tuple(
|
||||
DataSourceApiKeyAuthBindingRecord(
|
||||
id=binding_id,
|
||||
category=category,
|
||||
provider=provider,
|
||||
disabled=bool(disabled),
|
||||
created_at=created_at,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
for binding_id, category, provider, disabled, created_at, updated_at in rows
|
||||
)
|
||||
|
||||
@override
|
||||
def create(
|
||||
self,
|
||||
workspace_id: str,
|
||||
category: str,
|
||||
provider: str,
|
||||
credentials: DataSourceApiKeyAuthCredentials,
|
||||
) -> None:
|
||||
config = deepcopy(dict(credentials.options))
|
||||
config["api_key"] = credentials.api_key
|
||||
binding = DataSourceApiKeyAuthBinding(
|
||||
tenant_id=workspace_id,
|
||||
category=category,
|
||||
provider=provider,
|
||||
credentials=json.dumps(
|
||||
{"auth_type": credentials.auth_type, "config": config},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
)
|
||||
with self._session_factory.begin() as session:
|
||||
session.add(binding)
|
||||
|
||||
@override
|
||||
def delete(self, workspace_id: str, binding_id: str) -> None:
|
||||
stmt = select(DataSourceApiKeyAuthBinding).where(
|
||||
DataSourceApiKeyAuthBinding.tenant_id == workspace_id,
|
||||
DataSourceApiKeyAuthBinding.id == binding_id,
|
||||
)
|
||||
with self._session_factory.begin() as session:
|
||||
binding = session.scalar(stmt)
|
||||
if binding is not None:
|
||||
session.delete(binding)
|
||||
@@ -0,0 +1,294 @@
|
||||
"""Actor-aware and trusted-source SQLAlchemy repository for datasource credentials."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import cast
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from sqlalchemy import cast as sql_cast
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.engine import CursorResult
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from libs.datetime_utils import naive_utc_now
|
||||
from models.credential_permission import CredentialType
|
||||
from models.dataset import Dataset, Document
|
||||
from models.oauth import DatasourceOauthParamConfig, DatasourceOauthTenantParamConfig, DatasourceProvider
|
||||
from repositories.credentials.query_repository import apply_credential_visibility_filter_for_actor
|
||||
from services.data_source.entities.credential import DatasourceCredentialRecord, DatasourceOAuthClientConfigRecord
|
||||
|
||||
_CREDENTIALS_ADAPTER = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
def _source_mapping(value: object) -> Mapping[str, object]:
|
||||
try:
|
||||
if isinstance(value, str):
|
||||
return _CREDENTIALS_ADAPTER.validate_json(value)
|
||||
return _CREDENTIALS_ADAPTER.validate_python(value)
|
||||
except (TypeError, ValueError):
|
||||
return {}
|
||||
|
||||
|
||||
def _datasource_credential_record(provider: DatasourceProvider) -> DatasourceCredentialRecord:
|
||||
return DatasourceCredentialRecord(
|
||||
id=provider.id,
|
||||
workspace_id=provider.tenant_id,
|
||||
owner_id=provider.user_id,
|
||||
name=provider.name,
|
||||
provider=provider.provider,
|
||||
plugin_id=provider.plugin_id,
|
||||
auth_type=provider.auth_type,
|
||||
encrypted_credentials=_CREDENTIALS_ADAPTER.validate_python(provider.encrypted_credentials),
|
||||
expires_at=provider.expires_at,
|
||||
updated_at=provider.updated_at,
|
||||
)
|
||||
|
||||
|
||||
class SQLAlchemyDatasourceCredentialRepository:
|
||||
def __init__(self, *, session_factory: sessionmaker[Session]) -> None:
|
||||
self._session_factory = session_factory
|
||||
|
||||
def get_provider_credential(
|
||||
self, *, workspace_id: str, provider: str, plugin_id: str, credential_id: str | None
|
||||
) -> DatasourceCredentialRecord | None:
|
||||
with self._session_factory() as session:
|
||||
rows = _read_provider_credentials(
|
||||
session,
|
||||
workspace_id=workspace_id,
|
||||
provider=provider,
|
||||
plugin_id=plugin_id,
|
||||
credential_id=credential_id,
|
||||
first_only=True,
|
||||
)
|
||||
return _datasource_credential_record(rows[0]) if rows else None
|
||||
|
||||
def list_provider_credentials(
|
||||
self, *, workspace_id: str, provider: str, plugin_id: str
|
||||
) -> list[DatasourceCredentialRecord]:
|
||||
with self._session_factory() as session:
|
||||
return [
|
||||
_datasource_credential_record(row)
|
||||
for row in _read_provider_credentials(
|
||||
session,
|
||||
workspace_id=workspace_id,
|
||||
provider=provider,
|
||||
plugin_id=plugin_id,
|
||||
)
|
||||
]
|
||||
|
||||
def get_visible(
|
||||
self,
|
||||
*,
|
||||
workspace_id: str,
|
||||
actor_id: str,
|
||||
credential_id: str,
|
||||
provider: str,
|
||||
plugin_id: str,
|
||||
) -> DatasourceCredentialRecord | None:
|
||||
query = select(DatasourceProvider).where(
|
||||
DatasourceProvider.tenant_id == workspace_id,
|
||||
DatasourceProvider.id == credential_id,
|
||||
DatasourceProvider.provider == provider,
|
||||
DatasourceProvider.plugin_id == plugin_id,
|
||||
)
|
||||
query = apply_credential_visibility_filter_for_actor(
|
||||
query,
|
||||
tenant_id=workspace_id,
|
||||
model_id_column=DatasourceProvider.id,
|
||||
model_user_id_column=DatasourceProvider.user_id,
|
||||
model_visibility_column=DatasourceProvider.visibility,
|
||||
credential_type=CredentialType.DATASOURCE_PROVIDER,
|
||||
actor_id=actor_id,
|
||||
)
|
||||
with self._session_factory() as session:
|
||||
provider_record = session.scalar(query.limit(1))
|
||||
return _datasource_credential_record(provider_record) if provider_record is not None else None
|
||||
|
||||
def get_visible_preferred(
|
||||
self,
|
||||
*,
|
||||
workspace_id: str,
|
||||
actor_id: str,
|
||||
provider: str,
|
||||
plugin_id: str,
|
||||
credential_id: str | None,
|
||||
) -> DatasourceCredentialRecord | None:
|
||||
"""Return the requested visible credential, or the actor's preferred one when none is requested.
|
||||
|
||||
Without ``credential_id`` the workspace default wins, then the oldest visible credential.
|
||||
"""
|
||||
query = select(DatasourceProvider).where(
|
||||
DatasourceProvider.tenant_id == workspace_id,
|
||||
DatasourceProvider.provider == provider,
|
||||
DatasourceProvider.plugin_id == plugin_id,
|
||||
)
|
||||
if credential_id is not None:
|
||||
query = query.where(DatasourceProvider.id == credential_id)
|
||||
query = apply_credential_visibility_filter_for_actor(
|
||||
query,
|
||||
tenant_id=workspace_id,
|
||||
model_id_column=DatasourceProvider.id,
|
||||
model_user_id_column=DatasourceProvider.user_id,
|
||||
model_visibility_column=DatasourceProvider.visibility,
|
||||
credential_type=CredentialType.DATASOURCE_PROVIDER,
|
||||
actor_id=actor_id,
|
||||
)
|
||||
with self._session_factory() as session:
|
||||
provider_record = session.scalar(
|
||||
query.order_by(DatasourceProvider.is_default.desc(), DatasourceProvider.created_at.asc()).limit(1)
|
||||
)
|
||||
return _datasource_credential_record(provider_record) if provider_record is not None else None
|
||||
|
||||
def get_oauth_client_config(
|
||||
self,
|
||||
*,
|
||||
workspace_id: str,
|
||||
provider: str,
|
||||
plugin_id: str,
|
||||
) -> DatasourceOAuthClientConfigRecord:
|
||||
"""Load OAuth client rows in one bounded read transaction."""
|
||||
|
||||
with self._session_factory() as session:
|
||||
return _read_oauth_client_config(session, workspace_id=workspace_id, provider=provider, plugin_id=plugin_id)
|
||||
|
||||
def update_if_unchanged(
|
||||
self,
|
||||
*,
|
||||
record: DatasourceCredentialRecord,
|
||||
encrypted_credentials: Mapping[str, object],
|
||||
expires_at: int,
|
||||
) -> bool:
|
||||
with self._session_factory.begin() as session:
|
||||
return _update_datasource_credentials_if_unchanged(
|
||||
session, record=record, encrypted_credentials=encrypted_credentials, expires_at=expires_at
|
||||
)
|
||||
|
||||
def get_for_stored_document(
|
||||
self,
|
||||
*,
|
||||
workspace_id: str,
|
||||
dataset_id: str,
|
||||
document_id: str,
|
||||
credential_id: str | None,
|
||||
provider: str,
|
||||
plugin_id: str,
|
||||
) -> DatasourceCredentialRecord | None:
|
||||
with self._session_factory() as session:
|
||||
query = (
|
||||
select(DatasourceProvider, Document.data_source_info)
|
||||
.select_from(Document)
|
||||
.join(Dataset, Dataset.id == Document.dataset_id)
|
||||
.join(DatasourceProvider, DatasourceProvider.tenant_id == Document.tenant_id)
|
||||
.where(
|
||||
Dataset.id == dataset_id,
|
||||
Dataset.tenant_id == workspace_id,
|
||||
Document.id == document_id,
|
||||
Document.dataset_id == dataset_id,
|
||||
Document.tenant_id == workspace_id,
|
||||
Document.data_source_type == "notion_import",
|
||||
DatasourceProvider.tenant_id == workspace_id,
|
||||
DatasourceProvider.provider == provider,
|
||||
DatasourceProvider.plugin_id == plugin_id,
|
||||
)
|
||||
)
|
||||
if credential_id is not None:
|
||||
query = query.where(DatasourceProvider.id == credential_id)
|
||||
else:
|
||||
query = query.where(DatasourceProvider.is_default.is_(True)).order_by(
|
||||
DatasourceProvider.created_at.asc()
|
||||
)
|
||||
row = session.execute(query.limit(1)).one_or_none()
|
||||
if row is None:
|
||||
return None
|
||||
datasource_provider, raw_source_info = row
|
||||
source_info = _source_mapping(raw_source_info)
|
||||
if credential_id is not None and source_info.get("credential_id") != credential_id:
|
||||
return None
|
||||
return _datasource_credential_record(datasource_provider)
|
||||
|
||||
|
||||
def _read_oauth_client_config(
|
||||
session: Session, *, workspace_id: str, provider: str, plugin_id: str
|
||||
) -> DatasourceOAuthClientConfigRecord:
|
||||
tenant_config = session.scalar(
|
||||
select(DatasourceOauthTenantParamConfig)
|
||||
.where(
|
||||
DatasourceOauthTenantParamConfig.tenant_id == workspace_id,
|
||||
DatasourceOauthTenantParamConfig.provider == provider,
|
||||
DatasourceOauthTenantParamConfig.plugin_id == plugin_id,
|
||||
DatasourceOauthTenantParamConfig.enabled.is_(True),
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
system_config = session.scalar(
|
||||
select(DatasourceOauthParamConfig)
|
||||
.where(
|
||||
DatasourceOauthParamConfig.provider == provider,
|
||||
DatasourceOauthParamConfig.plugin_id == plugin_id,
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
return DatasourceOAuthClientConfigRecord(
|
||||
encrypted_tenant_params=(dict(tenant_config.client_params) if tenant_config is not None else None),
|
||||
system_credentials=(dict(system_config.system_credentials) if system_config is not None else None),
|
||||
)
|
||||
|
||||
|
||||
def _update_datasource_credentials_if_unchanged(
|
||||
session: Session,
|
||||
*,
|
||||
record: DatasourceCredentialRecord,
|
||||
encrypted_credentials: Mapping[str, object],
|
||||
expires_at: int,
|
||||
) -> bool:
|
||||
credentials = _CREDENTIALS_ADAPTER.validate_python(encrypted_credentials)
|
||||
expected_credentials = _CREDENTIALS_ADAPTER.validate_python(record.encrypted_credentials)
|
||||
credentials_match = DatasourceProvider.encrypted_credentials == expected_credentials
|
||||
if session.get_bind().dialect.name == "mysql":
|
||||
# MySQL otherwise compares the JSON column with the bound value as a string scalar.
|
||||
credentials_match = DatasourceProvider.encrypted_credentials == sql_cast(
|
||||
expected_credentials, DatasourceProvider.encrypted_credentials.type
|
||||
)
|
||||
result = session.execute(
|
||||
update(DatasourceProvider)
|
||||
.where(
|
||||
DatasourceProvider.id == record.id,
|
||||
DatasourceProvider.tenant_id == record.workspace_id,
|
||||
DatasourceProvider.provider == record.provider,
|
||||
DatasourceProvider.plugin_id == record.plugin_id,
|
||||
DatasourceProvider.user_id == record.owner_id,
|
||||
DatasourceProvider.auth_type == record.auth_type,
|
||||
credentials_match,
|
||||
DatasourceProvider.expires_at == record.expires_at,
|
||||
)
|
||||
.values(
|
||||
encrypted_credentials=credentials,
|
||||
expires_at=expires_at,
|
||||
updated_at=naive_utc_now(),
|
||||
)
|
||||
)
|
||||
return cast(CursorResult[object], result).rowcount == 1
|
||||
|
||||
|
||||
def _read_provider_credentials(
|
||||
session: Session,
|
||||
*,
|
||||
workspace_id: str,
|
||||
provider: str,
|
||||
plugin_id: str,
|
||||
credential_id: str | None = None,
|
||||
first_only: bool = False,
|
||||
) -> list[DatasourceProvider]:
|
||||
statement = (
|
||||
select(DatasourceProvider)
|
||||
.where(
|
||||
DatasourceProvider.tenant_id == workspace_id,
|
||||
DatasourceProvider.provider == provider,
|
||||
DatasourceProvider.plugin_id == plugin_id,
|
||||
)
|
||||
.order_by(DatasourceProvider.is_default.desc(), DatasourceProvider.created_at.asc())
|
||||
)
|
||||
if credential_id is not None:
|
||||
statement = statement.where(DatasourceProvider.id == credential_id)
|
||||
if first_only:
|
||||
statement = statement.limit(1)
|
||||
return list(session.scalars(statement))
|
||||
@@ -0,0 +1,166 @@
|
||||
"""SQLAlchemy repository for OAuth data-source bindings."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import cast, override
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.engine import CursorResult
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from libs.datetime_utils import naive_utc_now
|
||||
from models.source import DataSourceOauthBinding
|
||||
from services.data_source.binding_application_service import BindingMutationResult, DataSourceBindingStore
|
||||
from services.data_source.entities.oauth import (
|
||||
DataSourceBindingSummary,
|
||||
DataSourceOAuthAuthorization,
|
||||
DataSourceOAuthBindingRecord,
|
||||
)
|
||||
from services.data_source.oauth_service import DataSourceOAuthBindingRepository
|
||||
|
||||
_SOURCE_INFO_ADAPTER = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
class SQLAlchemyDataSourceOAuthBindingRepository(DataSourceOAuthBindingRepository, DataSourceBindingStore):
|
||||
def __init__(self, session_factory: sessionmaker[Session]) -> None:
|
||||
self._session_factory = session_factory
|
||||
|
||||
@override
|
||||
def upsert_authorization(
|
||||
self,
|
||||
*,
|
||||
workspace_id: str,
|
||||
provider: str,
|
||||
authorization: DataSourceOAuthAuthorization,
|
||||
) -> None:
|
||||
source_info = _SOURCE_INFO_ADAPTER.validate_python(authorization.source_info)
|
||||
with self._session_factory.begin() as session:
|
||||
binding = session.scalar(
|
||||
select(DataSourceOauthBinding).where(
|
||||
DataSourceOauthBinding.tenant_id == workspace_id,
|
||||
DataSourceOauthBinding.provider == provider,
|
||||
DataSourceOauthBinding.access_token == authorization.access_token,
|
||||
)
|
||||
)
|
||||
if binding is None:
|
||||
session.add(
|
||||
DataSourceOauthBinding(
|
||||
tenant_id=workspace_id,
|
||||
provider=provider,
|
||||
access_token=authorization.access_token,
|
||||
source_info=source_info,
|
||||
)
|
||||
)
|
||||
return
|
||||
|
||||
binding.source_info = source_info
|
||||
binding.disabled = False
|
||||
binding.updated_at = naive_utc_now()
|
||||
|
||||
@override
|
||||
def get_enabled(
|
||||
self,
|
||||
*,
|
||||
workspace_id: str,
|
||||
provider: str,
|
||||
binding_id: str,
|
||||
) -> DataSourceOAuthBindingRecord | None:
|
||||
with self._session_factory() as session:
|
||||
binding = session.scalar(
|
||||
select(DataSourceOauthBinding).where(
|
||||
DataSourceOauthBinding.tenant_id == workspace_id,
|
||||
DataSourceOauthBinding.provider == provider,
|
||||
DataSourceOauthBinding.id == binding_id,
|
||||
DataSourceOauthBinding.disabled.is_not(True),
|
||||
)
|
||||
)
|
||||
if binding is None:
|
||||
return None
|
||||
|
||||
return DataSourceOAuthBindingRecord(
|
||||
id=binding.id,
|
||||
access_token=binding.access_token,
|
||||
source_info=_SOURCE_INFO_ADAPTER.validate_python(binding.source_info),
|
||||
)
|
||||
|
||||
@override
|
||||
def update_source_info(
|
||||
self,
|
||||
*,
|
||||
workspace_id: str,
|
||||
provider: str,
|
||||
binding_id: str,
|
||||
source_info: Mapping[str, object],
|
||||
) -> bool:
|
||||
validated_source_info = _SOURCE_INFO_ADAPTER.validate_python(source_info)
|
||||
with self._session_factory.begin() as session:
|
||||
binding = session.scalar(
|
||||
select(DataSourceOauthBinding).where(
|
||||
DataSourceOauthBinding.tenant_id == workspace_id,
|
||||
DataSourceOauthBinding.provider == provider,
|
||||
DataSourceOauthBinding.id == binding_id,
|
||||
DataSourceOauthBinding.disabled.is_not(True),
|
||||
)
|
||||
)
|
||||
if binding is None:
|
||||
return False
|
||||
|
||||
binding.source_info = validated_source_info
|
||||
binding.updated_at = naive_utc_now()
|
||||
return True
|
||||
|
||||
@override
|
||||
def list_enabled_bindings(self, *, workspace_id: str) -> tuple[DataSourceBindingSummary, ...]:
|
||||
with self._session_factory() as session:
|
||||
bindings = session.scalars(
|
||||
select(DataSourceOauthBinding).where(
|
||||
DataSourceOauthBinding.tenant_id == workspace_id,
|
||||
DataSourceOauthBinding.disabled.is_not(True),
|
||||
)
|
||||
).all()
|
||||
return tuple(
|
||||
DataSourceBindingSummary(
|
||||
id=binding.id,
|
||||
provider=binding.provider,
|
||||
created_at=binding.created_at,
|
||||
disabled=False,
|
||||
source_info=_SOURCE_INFO_ADAPTER.validate_python(binding.source_info),
|
||||
)
|
||||
for binding in bindings
|
||||
)
|
||||
|
||||
@override
|
||||
def change_disabled_state(
|
||||
self,
|
||||
*,
|
||||
workspace_id: str,
|
||||
binding_id: str,
|
||||
disabled: bool,
|
||||
) -> BindingMutationResult:
|
||||
state_predicate = (
|
||||
DataSourceOauthBinding.disabled.is_not(True) if disabled else DataSourceOauthBinding.disabled.is_(True)
|
||||
)
|
||||
with self._session_factory.begin() as session:
|
||||
result = session.execute(
|
||||
update(DataSourceOauthBinding)
|
||||
.where(
|
||||
DataSourceOauthBinding.tenant_id == workspace_id,
|
||||
DataSourceOauthBinding.id == binding_id,
|
||||
state_predicate,
|
||||
)
|
||||
.values(disabled=disabled, updated_at=naive_utc_now())
|
||||
)
|
||||
if cast(CursorResult[object], result).rowcount == 1:
|
||||
return "updated"
|
||||
|
||||
binding_exists = session.scalar(
|
||||
select(DataSourceOauthBinding.id)
|
||||
.where(
|
||||
DataSourceOauthBinding.tenant_id == workspace_id,
|
||||
DataSourceOauthBinding.id == binding_id,
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
if binding_exists is None:
|
||||
return "not_found"
|
||||
return "already_disabled" if disabled else "already_enabled"
|
||||
Reference in new issue
Block a user