chore: 纳入本地 Dify 1.19.0 源码及品牌与免登改动

演示目录只保留可重建的源码。运行密钥、数据卷、依赖和带口令的本地配置不进入仓库。

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
wuzilongandCursor committed 2026-11-18 13:59:25 +08:00
1 parent 60158935f4
commit 6519682885
15831 files changed
+3063428

No files matched your search

@@ -0,0 +1 @@
"""Unit tests for `controllers.console.datasets` controllers."""
@@ -0,0 +1,988 @@
import inspect
from datetime import UTC, datetime
from importlib import import_module
from types import SimpleNamespace
from unittest.mock import ANY, MagicMock, patch
import pytest
from flask import Flask
from sqlalchemy.orm import Session, sessionmaker
from werkzeug.exceptions import Forbidden, NotFound
from controllers.console import console_ns
from controllers.console.datasets.rag_pipeline.datasource_auth import (
DatasourceAuth,
DatasourceAuthDefaultApi,
DatasourceAuthDeleteApi,
DatasourceAuthListApi,
DatasourceAuthOauthCustomClient,
DatasourceAuthUpdateApi,
DatasourceCredentialDeletePayload,
DatasourceCredentialPayload,
DatasourceCredentialUpdatePayload,
DatasourceCustomClientPayload,
DatasourceDefaultPayload,
DatasourceHardCodeAuthListApi,
DatasourceOAuthCallback,
DatasourcePluginOAuthAuthorizationUrl,
DatasourceUpdateNamePayload,
DatasourceUpdateProviderNameApi,
)
from core.plugin.impl.oauth import OAuthHandler
from extensions.application_services.data_sources import build_data_source_credentials
from graphon.model_runtime.errors.validate import CredentialsValidateFailedError
from models.account import Account
from repositories.credentials.query_repository import CredentialQueryRepository
from services.data_source.provider_service import DatasourceProviderService
from services.plugin.dispatch_identity import PluginOAuthClientConfig
from services.plugin.oauth_service import OAuthProxyService
from tests.helpers.unit.model_factories import make_account
_PROVIDER_ID = "langgenius/notion_datasource/notion"
@pytest.fixture(autouse=True)
def credential_query_dependency(monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session]) -> None:
query = CredentialQueryRepository(session_factory=sqlite_session_factory)
providers = build_data_source_credentials(database_client=sqlite_session_factory).providers
monkeypatch.setattr(
import_module("controllers.console.datasets.rag_pipeline.datasource_auth"),
"application_services",
lambda: SimpleNamespace(credential_queries=query, data_sources=SimpleNamespace(providers=providers)),
)
def _account() -> Account:
return make_account(account_id="user-1", name="Datasource Auth Tester", email="datasource-auth@example.com")
def _i18n(text: str) -> dict[str, str]:
return {"en_US": text, "zh_Hans": text, "pt_BR": text, "ja_JP": text}
def _provider_config(name: str, type_: str, label: str, *, required: bool = True) -> dict:
return {
"type": type_,
"name": name,
"scope": None,
"required": required,
"default": None,
"options": None,
"multiple": False,
"label": _i18n(label),
"help": None,
"url": None,
"placeholder": None,
}
def _datasource_credential(credential_id: str = "cred-1", *, is_default: bool = True) -> dict:
return {
"credential": {
"api_key": "******",
"workspace": "engineering",
"database_id": "db-123",
},
"type": "api-key",
"name": "API Key",
"avatar_url": "https://cdn.example.com/notion.png",
"id": credential_id,
"is_default": is_default,
}
def _datasource_auth() -> dict:
return {
"author": "Dify",
"provider": "notion",
"plugin_id": "langgenius/notion_datasource",
"plugin_unique_identifier": "langgenius/notion_datasource:0.0.1",
"icon": "icon.svg",
"name": "notion",
"label": _i18n("Notion"),
"description": _i18n("Notion datasource"),
"credential_schema": [
_provider_config("api_key", "secret-input", "API key"),
],
"oauth_schema": {
"client_schema": [
_provider_config("client_id", "text-input", "Client ID"),
],
"credentials_schema": [
_provider_config("access_token", "secret-input", "Access token"),
],
"oauth_custom_client_params": {"client_id": "masked-client", "client_secret": "********"},
"is_oauth_custom_client_enabled": True,
"is_system_oauth_params_exists": True,
"redirect_uri": "https://api.example.com/oauth/callback",
},
"credentials_list": [_datasource_credential(), _datasource_credential("cred-2", is_default=False)],
}
def _success_response() -> dict[str, str]:
return {"result": "success"}
class TestDatasourcePluginOAuthAuthorizationUrl:
def test_get_success(self, app: Flask):
api = DatasourcePluginOAuthAuthorizationUrl()
method = inspect.unwrap(api.get)
user = _account()
oauth_client = {"client_id": "abc", "client_secret": "shh", "scopes": ["read", "write"]}
auth_url_payload = {
"authorization_url": "https://auth.example.com/oauth?client_id=abc&state=xyz",
}
with (
app.test_request_context("/?credential_id=cred-1"),
patch.object(
DatasourceProviderService,
"get_oauth_client",
return_value=PluginOAuthClientConfig(
"langgenius/notion_datasource:1.0.0@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", oauth_client
),
) as get_oauth_client,
patch.object(
OAuthProxyService,
"create_proxy_context",
return_value="ctx-1",
) as create_proxy_context,
patch.object(
OAuthHandler,
"get_authorization_url",
return_value=auth_url_payload,
) as get_authorization_url,
):
response = method(api, "tenant-1", user, _PROVIDER_ID)
assert response.status_code == 200
assert response.get_json() == auth_url_payload
assert "context_id=ctx-1" in response.headers.get("Set-Cookie")
provider_id = get_oauth_client.call_args.kwargs["datasource_provider_id"]
assert str(provider_id) == _PROVIDER_ID
get_oauth_client.assert_called_once()
create_proxy_context.assert_called_once_with(
user_id="user-1",
tenant_id="tenant-1",
plugin_id="langgenius/notion_datasource",
provider="notion",
credential_id="cred-1",
extra_data={
"visibility": "only_me",
"plugin_unique_identifier": "langgenius/notion_datasource:1.0.0@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
},
)
get_authorization_url.assert_called_once()
assert get_authorization_url.call_args.kwargs["tenant_id"] == "tenant-1"
assert get_authorization_url.call_args.kwargs["user_id"] == "user-1"
assert get_authorization_url.call_args.kwargs["plugin_id"] == "langgenius/notion_datasource"
assert get_authorization_url.call_args.kwargs["provider"] == "notion"
assert get_authorization_url.call_args.kwargs["system_credentials"] == oauth_client
def test_get_no_oauth_config(self, app: Flask):
api = DatasourcePluginOAuthAuthorizationUrl()
method = inspect.unwrap(api.get)
user = _account()
with (
app.test_request_context("/"),
patch.object(
DatasourceProviderService,
"get_oauth_client",
return_value=None,
),
):
with pytest.raises(ValueError, match="No OAuth Client Config for notion"):
method(api, "tenant-1", user, "notion")
def test_get_without_credential_id_sets_cookie(self, app: Flask):
api = DatasourcePluginOAuthAuthorizationUrl()
method = inspect.unwrap(api.get)
user = _account()
with (
app.test_request_context("/"),
patch.object(
DatasourceProviderService,
"get_oauth_client",
return_value=PluginOAuthClientConfig(
"langgenius/notion_datasource:1.0.0@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", {"client_id": "abc"}
),
),
patch.object(
OAuthProxyService,
"create_proxy_context",
return_value="ctx-123",
),
patch.object(
OAuthHandler,
"get_authorization_url",
return_value={"authorization_url": "http://auth"},
),
):
response = method(api, "tenant-1", user, _PROVIDER_ID)
assert response.status_code == 200
assert "context_id" in response.headers.get("Set-Cookie")
class TestDatasourceOAuthCallback:
def test_callback_rejects_replaced_package_before_dispatch(
self, app: Flask, monkeypatch: pytest.MonkeyPatch
) -> None:
plugin_id = "langgenius/notion_datasource"
context = {
"user_id": "user-1",
"tenant_id": "tenant-1",
"plugin_id": plugin_id,
"provider": "notion",
"plugin_unique_identifier": f"{plugin_id}:1.0.0@{'a' * 64}",
}
config = PluginOAuthClientConfig(f"{plugin_id}:2.0.0@{'b' * 64}", {"client_secret": "test-secret"})
monkeypatch.setattr(OAuthProxyService, "use_proxy_context", lambda _context_id: context)
monkeypatch.setattr(DatasourceProviderService, "get_oauth_client", lambda *_args, **_kwargs: config)
def unexpected_dispatch(*_args: object, **_kwargs: object) -> None:
raise AssertionError("Changed packages must not receive OAuth credentials")
monkeypatch.setattr(OAuthHandler, "get_credentials", unexpected_dispatch)
api = DatasourceOAuthCallback()
with app.test_request_context("/?context_id=ctx"):
with pytest.raises(Forbidden, match="Plugin changed"):
inspect.unwrap(api.get)(api, _PROVIDER_ID)
def test_callback_success_new_credential(self, app: Flask):
api = DatasourceOAuthCallback()
method = inspect.unwrap(api.get)
oauth_response = MagicMock()
oauth_response.credentials = {"token": "abc"}
expires_at = datetime(2024, 1, 2, 3, 4, 5, tzinfo=UTC)
oauth_response.expires_at = expires_at
oauth_response.metadata = {"name": "Workspace Bot", "avatar_url": "https://avatar.example.com/bot.png"}
context = {
"user_id": "user-1",
"tenant_id": "tenant-1",
"plugin_id": "langgenius/notion_datasource",
"provider": "notion",
"credential_id": None,
}
with (
app.test_request_context("/?context_id=ctx"),
patch.object(
OAuthProxyService,
"use_proxy_context",
return_value=context,
),
patch.object(
DatasourceProviderService,
"get_oauth_client",
return_value=PluginOAuthClientConfig(
"langgenius/notion_datasource:1.0.0@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
{"client_id": "abc", "client_secret": "secret"},
),
),
patch.object(
OAuthHandler,
"get_credentials",
return_value=oauth_response,
),
patch.object(
DatasourceProviderService,
"add_datasource_oauth_provider",
return_value=None,
) as add_oauth_provider,
):
response = method(api, _PROVIDER_ID)
assert response.status_code == 302
assert "/oauth-callback" in response.location
add_oauth_provider.assert_called_once()
# Legacy context without a visibility key falls back to ONLY_ME, and
# the callback now also propagates the creator's user_id.
from models.enums import PermissionEnum
assert add_oauth_provider.call_args.kwargs == {
"tenant_id": "tenant-1",
"provider_id": add_oauth_provider.call_args.kwargs["provider_id"],
"avatar_url": "https://avatar.example.com/bot.png",
"name": "Workspace Bot",
"expire_at": expires_at,
"credentials": {"token": "abc"},
"user_id": "user-1",
"visibility": PermissionEnum.ONLY_ME,
}
assert str(add_oauth_provider.call_args.kwargs["provider_id"]) == _PROVIDER_ID
def test_callback_missing_context(self, app: Flask):
api = DatasourceOAuthCallback()
method = inspect.unwrap(api.get)
with app.test_request_context("/"):
with pytest.raises(Forbidden):
method(api, "notion")
def test_callback_invalid_context(self, app: Flask):
api = DatasourceOAuthCallback()
method = inspect.unwrap(api.get)
with (
app.test_request_context("/?context_id=bad"),
patch.object(
OAuthProxyService,
"use_proxy_context",
return_value=None,
),
):
with pytest.raises(Forbidden):
method(api, "notion")
def test_callback_oauth_config_not_found(self, app: Flask):
api = DatasourceOAuthCallback()
method = inspect.unwrap(api.get)
context = {"user_id": "u", "tenant_id": "t"}
with (
app.test_request_context("/?context_id=ctx"),
patch.object(
OAuthProxyService,
"use_proxy_context",
return_value=context,
),
patch.object(
DatasourceProviderService,
"get_oauth_client",
return_value=None,
),
):
with pytest.raises(NotFound):
method(api, "notion")
def test_callback_reauthorize_existing_credential(self, app: Flask):
api = DatasourceOAuthCallback()
method = inspect.unwrap(api.get)
oauth_response = MagicMock()
oauth_response.credentials = {"token": "abc"}
oauth_response.expires_at = None
oauth_response.metadata = {} # avatar + name missing
context = {
"user_id": "user-1",
"tenant_id": "tenant-1",
"plugin_id": "langgenius/notion_datasource",
"provider": "notion",
"credential_id": "cred-1",
}
with (
app.test_request_context("/?context_id=ctx"),
patch.object(
OAuthProxyService,
"use_proxy_context",
return_value=context,
),
patch.object(
DatasourceProviderService,
"get_oauth_client",
return_value=PluginOAuthClientConfig(
"langgenius/notion_datasource:1.0.0@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", {"client_id": "abc"}
),
),
patch.object(
OAuthHandler,
"get_credentials",
return_value=oauth_response,
),
patch.object(
DatasourceProviderService,
"reauthorize_datasource_oauth_provider",
return_value=None,
) as reauthorize_provider,
):
response = method(api, _PROVIDER_ID)
assert response.status_code == 302
assert "/oauth-callback" in response.location
reauthorize_provider.assert_called_once()
assert str(reauthorize_provider.call_args.kwargs["provider_id"]) == _PROVIDER_ID
assert reauthorize_provider.call_args.kwargs["credential_id"] == "cred-1"
assert reauthorize_provider.call_args.kwargs["credentials"] == {"token": "abc"}
def test_callback_context_id_from_cookie(self, app: Flask):
api = DatasourceOAuthCallback()
method = inspect.unwrap(api.get)
oauth_response = MagicMock()
oauth_response.credentials = {"token": "abc"}
oauth_response.expires_at = None
oauth_response.metadata = {}
context = {
"user_id": "user-1",
"tenant_id": "tenant-1",
"plugin_id": "langgenius/notion_datasource",
"provider": "notion",
"credential_id": None,
}
with (
app.test_request_context("/", headers={"Cookie": "context_id=ctx"}),
patch.object(
OAuthProxyService,
"use_proxy_context",
return_value=context,
),
patch.object(
DatasourceProviderService,
"get_oauth_client",
return_value=PluginOAuthClientConfig(
"langgenius/notion_datasource:1.0.0@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", {"client_id": "abc"}
),
),
patch.object(
OAuthHandler,
"get_credentials",
return_value=oauth_response,
),
patch.object(
DatasourceProviderService,
"add_datasource_oauth_provider",
return_value=None,
),
):
response = method(api, _PROVIDER_ID)
assert response.status_code == 302
class TestDatasourceAuth:
def test_post_success(self, app: Flask):
api = DatasourceAuth()
method = inspect.unwrap(api.post)
payload = {
"name": "Engineering Notion",
"credentials": {
"api_key": "secret-token",
"workspace": "engineering",
"database_id": "db-123",
},
}
with (
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", payload),
patch.object(
DatasourceProviderService,
"add_datasource_api_key_provider",
return_value=None,
) as add_api_key_provider,
):
req_data = DatasourceCredentialPayload.model_validate(payload)
response, status = method(api, req_data, "tenant-1", _PROVIDER_ID)
assert response == _success_response()
assert status == 200
add_api_key_provider.assert_called_once()
assert add_api_key_provider.call_args.kwargs["tenant_id"] == "tenant-1"
assert str(add_api_key_provider.call_args.kwargs["provider_id"]) == _PROVIDER_ID
assert add_api_key_provider.call_args.kwargs["credentials"] == payload["credentials"]
assert add_api_key_provider.call_args.kwargs["name"] == "Engineering Notion"
def test_post_invalid_credentials(self, app: Flask):
api = DatasourceAuth()
method = inspect.unwrap(api.post)
payload = {"credentials": {"key": "bad"}}
with (
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", payload),
patch.object(
DatasourceProviderService,
"add_datasource_api_key_provider",
side_effect=CredentialsValidateFailedError("invalid"),
),
):
with pytest.raises(ValueError, match="invalid"):
method(api, DatasourceCredentialPayload.model_validate(payload), "tenant-1", "notion")
def test_get_success(self, app: Flask):
api = DatasourceAuth()
method = inspect.unwrap(api.get)
user = _account()
with (
app.test_request_context("/"),
patch.object(
DatasourceProviderService,
"list_datasource_credentials",
return_value=[_datasource_credential()],
),
):
response, status = method(api, "tenant-1", user, _PROVIDER_ID)
assert status == 200
assert response == {"result": [_datasource_credential()]}
def test_post_missing_credentials(self, app: Flask):
api = DatasourceAuth()
method = inspect.unwrap(api.post)
payload: dict[str, object] = {}
with (
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", payload),
):
with pytest.raises(ValueError, match="1 validation error for DatasourceCredentialPayload\\\ncredentials"):
method(api, DatasourceCredentialPayload.model_validate(payload), "tenant-1", "notion")
def test_get_empty_list(self, app: Flask):
api = DatasourceAuth()
method = inspect.unwrap(api.get)
user = _account()
with (
app.test_request_context("/"),
patch.object(
DatasourceProviderService,
"list_datasource_credentials",
return_value=[],
),
):
response, status = method(api, "tenant-1", user, "notion")
assert status == 200
assert response["result"] == []
class TestDatasourceAuthDeleteApi:
def test_delete_success(self, app: Flask):
api = DatasourceAuthDeleteApi()
method = inspect.unwrap(api.post)
payload = {"credential_id": "cred-1"}
with (
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", payload),
patch.object(
DatasourceProviderService,
"remove_datasource_credentials",
return_value=None,
) as remove_datasource_credentials,
):
req_data = DatasourceCredentialDeletePayload.model_validate(payload)
response, status = method(api, req_data, "tenant-1", _PROVIDER_ID)
assert response == _success_response()
assert status == 200
remove_datasource_credentials.assert_called_once_with(
tenant_id="tenant-1",
auth_id="cred-1",
provider="notion",
plugin_id="langgenius/notion_datasource",
session=ANY,
)
def test_delete_missing_credential_id(self, app: Flask):
api = DatasourceAuthDeleteApi()
method = inspect.unwrap(api.post)
payload: dict[str, object] = {}
with (
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", payload),
):
with pytest.raises(
ValueError, match="1 validation error for DatasourceCredentialDeletePayload\\\ncredential_id"
):
method(api, DatasourceCredentialDeletePayload.model_validate(payload), "tenant-1", "notion")
class TestDatasourceAuthUpdateApi:
def test_update_success(self, app: Flask):
api = DatasourceAuthUpdateApi()
method = inspect.unwrap(api.post)
payload = {
"credential_id": "cred-1",
"name": "Updated Notion",
"credentials": {"api_key": "new-secret", "database_id": "db-456"},
}
with (
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", payload),
patch.object(
DatasourceProviderService,
"update_datasource_credentials",
return_value=None,
) as update_datasource_credentials,
):
req_data = DatasourceCredentialUpdatePayload.model_validate(payload)
response, status = method(api, req_data, "tenant-1", _PROVIDER_ID)
assert response == _success_response()
assert status == 201
update_datasource_credentials.assert_called_once_with(
tenant_id="tenant-1",
auth_id="cred-1",
provider="notion",
plugin_id="langgenius/notion_datasource",
credentials=payload["credentials"],
name="Updated Notion",
)
def test_update_with_credentials_none(self, app: Flask):
api = DatasourceAuthUpdateApi()
method = inspect.unwrap(api.post)
payload = {"credential_id": "id", "credentials": None}
with (
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", payload),
patch.object(
DatasourceProviderService,
"update_datasource_credentials",
return_value=None,
) as update_mock,
):
req_data = DatasourceCredentialUpdatePayload.model_validate(payload)
response, status = method(api, req_data, "tenant-1", "notion")
assert response == _success_response()
update_mock.assert_called_once()
assert update_mock.call_args.kwargs["credentials"] == {}
assert status == 201
def test_update_name_only(self, app: Flask):
api = DatasourceAuthUpdateApi()
method = inspect.unwrap(api.post)
payload = {"credential_id": "id", "name": "New Name"}
with (
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", payload),
patch.object(
DatasourceProviderService,
"update_datasource_credentials",
return_value=None,
),
):
req_data = DatasourceCredentialUpdatePayload.model_validate(payload)
response, status = method(api, req_data, "tenant-1", "notion")
assert response == _success_response()
assert status == 201
def test_update_with_empty_credentials_dict(self, app: Flask):
api = DatasourceAuthUpdateApi()
method = inspect.unwrap(api.post)
payload = {"credential_id": "id", "credentials": {}}
with (
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", payload),
patch.object(
DatasourceProviderService,
"update_datasource_credentials",
return_value=None,
) as update_mock,
):
req_data = DatasourceCredentialUpdatePayload.model_validate(payload)
response, status = method(api, req_data, "tenant-1", "notion")
assert response == _success_response()
update_mock.assert_called_once()
assert status == 201
class TestDatasourceAuthListApi:
def test_list_success(self, app: Flask):
api = DatasourceAuthListApi()
method = inspect.unwrap(api.get)
user = MagicMock(id="user-1")
with (
app.test_request_context("/"),
patch.object(
DatasourceProviderService,
"get_all_datasource_credentials",
return_value=[_datasource_auth()],
) as get_all,
):
response, status = method(api, "tenant-1", user)
assert status == 200
assert response == {"result": [_datasource_auth()]}
# user is threaded through so list_datasource_credentials applies the
# visibility filter for the current viewer.
assert get_all.call_args.kwargs["user"] is user
def test_auth_list_empty(self, app: Flask):
api = DatasourceAuthListApi()
method = inspect.unwrap(api.get)
user = MagicMock(id="user-1")
with (
app.test_request_context("/"),
patch.object(
DatasourceProviderService,
"get_all_datasource_credentials",
return_value=[],
),
):
response, status = method(api, "tenant-1", user)
assert status == 200
assert response["result"] == []
def test_hardcode_list_empty(self, app: Flask):
api = DatasourceHardCodeAuthListApi()
method = inspect.unwrap(api.get)
user = MagicMock(id="user-1")
with (
app.test_request_context("/"),
patch.object(
DatasourceProviderService,
"get_hard_code_datasource_credentials",
return_value=[],
),
):
response, status = method(api, "tenant-1", user)
assert status == 200
assert response["result"] == []
class TestDatasourceHardCodeAuthListApi:
def test_list_success(self, app: Flask):
api = DatasourceHardCodeAuthListApi()
method = inspect.unwrap(api.get)
user = MagicMock(id="user-1")
with (
app.test_request_context("/"),
patch.object(
DatasourceProviderService,
"get_hard_code_datasource_credentials",
return_value=[_datasource_auth()],
) as get_hardcode,
):
response, status = method(api, "tenant-1", user)
assert status == 200
assert get_hardcode.call_args.kwargs["user"] is user
class TestDatasourceAuthOauthCustomClient:
def test_post_success(self, app: Flask):
api = DatasourceAuthOauthCustomClient()
method = inspect.unwrap(api.post)
payload = {
"client_params": {
"client_id": "custom-client",
"client_secret": "custom-secret",
"authorize_url": "https://auth.example.com/authorize",
},
"enable_oauth_custom_client": True,
}
with (
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", payload),
patch.object(
DatasourceProviderService,
"setup_oauth_custom_client_params",
return_value=None,
) as setup_custom_client,
):
req_data = DatasourceCustomClientPayload.model_validate(payload)
response, status = method(api, req_data, "tenant-1", _PROVIDER_ID)
assert response == _success_response()
assert status == 200
setup_custom_client.assert_called_once()
assert setup_custom_client.call_args.kwargs["tenant_id"] == "tenant-1"
assert str(setup_custom_client.call_args.kwargs["datasource_provider_id"]) == _PROVIDER_ID
assert setup_custom_client.call_args.kwargs["client_params"] == payload["client_params"]
assert setup_custom_client.call_args.kwargs["enabled"] is True
def test_delete_success(self, app: Flask):
api = DatasourceAuthOauthCustomClient()
method = inspect.unwrap(api.delete)
with (
app.test_request_context("/"),
patch.object(
DatasourceProviderService,
"remove_oauth_custom_client_params",
return_value=None,
) as remove_custom_client,
):
response, status = method(api, "tenant-1", _PROVIDER_ID)
assert response == _success_response()
assert status == 200
remove_custom_client.assert_called_once()
assert str(remove_custom_client.call_args.kwargs["datasource_provider_id"]) == _PROVIDER_ID
def test_post_empty_payload(self, app: Flask):
api = DatasourceAuthOauthCustomClient()
method = inspect.unwrap(api.post)
payload: dict[str, object] = {}
with (
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", payload),
patch.object(
DatasourceProviderService,
"setup_oauth_custom_client_params",
return_value=None,
),
):
req_data = DatasourceCustomClientPayload.model_validate(payload)
response, status = method(api, req_data, "tenant-1", "notion")
assert response == _success_response()
assert status == 200
def test_post_disabled_flag(self, app: Flask):
api = DatasourceAuthOauthCustomClient()
method = inspect.unwrap(api.post)
payload = {
"client_params": {"a": 1},
"enable_oauth_custom_client": False,
}
with (
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", payload),
patch.object(
DatasourceProviderService,
"setup_oauth_custom_client_params",
return_value=None,
) as setup_mock,
):
req_data = DatasourceCustomClientPayload.model_validate(payload)
response, status = method(api, req_data, "tenant-1", "notion")
assert response == _success_response()
setup_mock.assert_called_once()
assert setup_mock.call_args.kwargs["client_params"] == {"a": 1}
assert setup_mock.call_args.kwargs["enabled"] is False
assert status == 200
class TestDatasourceAuthDefaultApi:
def test_set_default_success(self, app: Flask):
api = DatasourceAuthDefaultApi()
method = inspect.unwrap(api.post)
payload = {"id": "cred-1"}
with (
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", payload),
patch.object(
DatasourceProviderService,
"set_default_datasource_provider",
return_value=None,
) as set_default_datasource_provider,
):
req_data = DatasourceDefaultPayload.model_validate(payload)
response, status = method(api, req_data, "tenant-1", _PROVIDER_ID)
assert response == _success_response()
assert status == 200
set_default_datasource_provider.assert_called_once()
assert set_default_datasource_provider.call_args.kwargs["tenant_id"] == "tenant-1"
assert str(set_default_datasource_provider.call_args.kwargs["datasource_provider_id"]) == _PROVIDER_ID
assert set_default_datasource_provider.call_args.kwargs["credential_id"] == "cred-1"
def test_default_missing_id(self, app: Flask):
api = DatasourceAuthDefaultApi()
method = inspect.unwrap(api.post)
payload: dict[str, object] = {}
with (
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", payload),
):
with pytest.raises(ValueError, match="1 validation error for DatasourceDefaultPayload\\\nid"):
method(api, DatasourceDefaultPayload.model_validate(payload), "tenant-1", "notion")
class TestDatasourceUpdateProviderNameApi:
def test_update_name_success(self, app: Flask):
api = DatasourceUpdateProviderNameApi()
method = inspect.unwrap(api.post)
payload = {"credential_id": "cred-1", "name": "New Name"}
with (
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", payload),
patch.object(
DatasourceProviderService,
"update_datasource_provider_name",
return_value=None,
) as update_datasource_provider_name,
):
req_data = DatasourceUpdateNamePayload.model_validate(payload)
response, status = method(api, req_data, "tenant-1", _PROVIDER_ID)
assert response == _success_response()
assert status == 200
update_datasource_provider_name.assert_called_once()
assert update_datasource_provider_name.call_args.kwargs["tenant_id"] == "tenant-1"
assert str(update_datasource_provider_name.call_args.kwargs["datasource_provider_id"]) == _PROVIDER_ID
assert update_datasource_provider_name.call_args.kwargs["name"] == "New Name"
assert update_datasource_provider_name.call_args.kwargs["credential_id"] == "cred-1"
def test_update_name_too_long(self, app: Flask):
api = DatasourceUpdateProviderNameApi()
method = inspect.unwrap(api.post)
payload = {
"credential_id": "id",
"name": "x" * 101,
}
with (
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", payload),
):
with pytest.raises(ValueError, match="1 validation error for DatasourceUpdateNamePayload\\\nname"):
method(api, DatasourceUpdateNamePayload.model_validate(payload), "tenant-1", "notion")
def test_update_name_missing_credential_id(self, app: Flask):
api = DatasourceUpdateProviderNameApi()
method = inspect.unwrap(api.post)
payload = {"name": "Valid"}
with (
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", payload),
):
with pytest.raises(ValueError, match="1 validation error for DatasourceUpdateNamePayload\\\ncredential_id"):
method(api, DatasourceUpdateNamePayload.model_validate(payload), "tenant-1", "notion")
@@ -0,0 +1,117 @@
from inspect import unwrap
from unittest.mock import ANY, MagicMock, patch
import pytest
from flask import Flask
from controllers.console import console_ns
from controllers.console.datasets.rag_pipeline.datasource_content_preview import (
DataSourceContentPreviewApi,
DatasourceContentPreviewPayload,
)
from models import Account
from models.dataset import Pipeline
def make_account() -> Account:
account = Account(name="Test User", email="user@example.com")
account.id = "account-1"
return account
class TestDataSourceContentPreviewApi:
def _valid_payload(self) -> dict[str, object]:
return {
"inputs": {"query": "hello"},
"datasource_type": "notion",
"credential_id": "cred-1",
}
def test_post_success(self, app: Flask) -> None:
api = DataSourceContentPreviewApi()
method = unwrap(api.post)
payload = self._valid_payload()
pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")
node_id = "node-1"
account = make_account()
preview_result = {"content": "preview data"}
service_instance = MagicMock()
service_instance.run_datasource_node_preview.return_value = preview_result
with (
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", payload),
patch(
"controllers.console.datasets.rag_pipeline.datasource_content_preview.RagPipelineService",
return_value=service_instance,
),
):
req_data = DatasourceContentPreviewPayload.model_validate(payload)
response, status = method(api, req_data, account, pipeline, node_id)
service_instance.run_datasource_node_preview.assert_called_once_with(
pipeline=pipeline,
node_id=node_id,
user_inputs=payload["inputs"],
account=account,
datasource_type=payload["datasource_type"],
is_published=True,
credential_id=payload["credential_id"],
datasource_providers=ANY,
)
assert status == 200
assert response == preview_result
def test_post_invalid_payload(self, app: Flask) -> None:
api = DataSourceContentPreviewApi()
method = unwrap(api.post)
payload = {
"inputs": {"query": "hello"},
# datasource_type missing
}
pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")
account = make_account()
with (
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", payload),
):
with pytest.raises(ValueError, match=r"datasource_type\n\s+Field required"):
method(api, DatasourceContentPreviewPayload.model_validate(payload), account, pipeline, "node-1")
def test_post_without_credential_id(self, app: Flask) -> None:
api = DataSourceContentPreviewApi()
method = unwrap(api.post)
payload = {
"inputs": {"query": "hello"},
"datasource_type": "notion",
"credential_id": None,
}
pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")
account = make_account()
service_instance = MagicMock()
service_instance.run_datasource_node_preview.return_value = {"ok": True}
with (
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", payload),
patch(
"controllers.console.datasets.rag_pipeline.datasource_content_preview.RagPipelineService",
return_value=service_instance,
),
):
req_data = DatasourceContentPreviewPayload.model_validate(payload)
response, status = method(api, req_data, account, pipeline, "node-1")
service_instance.run_datasource_node_preview.assert_called_once()
assert status == 200
assert response == {"ok": True}
@@ -0,0 +1,513 @@
from __future__ import annotations
from collections.abc import Iterator
from inspect import unwrap
from unittest.mock import ANY, PropertyMock, patch
import pytest
from flask import Flask
from sqlalchemy import Engine
from sqlalchemy.orm import Session
from werkzeug.exceptions import Forbidden, NotFound
from controllers.console import console_ns
from controllers.console.datasets.rag_pipeline import rag_pipeline as module
from controllers.console.datasets.rag_pipeline.rag_pipeline import (
CustomizedPipelineTemplateApi,
CustomizedPipelineTemplatePayload,
PipelineTemplateDetailApi,
PipelineTemplateDetailQuery,
PipelineTemplateListApi,
PipelineTemplateListQuery,
PublishCustomizedPipelineTemplateApi,
)
from models.account import Account, TenantAccountRole
from models.dataset import Pipeline, PipelineCustomizedTemplate
from models.engine import db
from services.entities.knowledge_entities.rag_pipeline_entities import PipelineTemplateInfoEntity
from services.errors.base import NoPermissionError
from services.errors.rag_pipeline import RagPipelineResourceNotFoundError
from tests.helpers.unit.config_override import config_overrides_context
from tests.helpers.unit.controllers.rbac_introspection import rbac_checks
from tests.helpers.unit.model_factories import make_account
def _template_item() -> dict[str, object]:
return {
"id": "template-1",
"name": "Template",
"icon": {"icon": "book", "icon_type": "emoji", "icon_background": "#fff"},
"description": "Description",
"position": 1,
"chunk_structure": "general",
}
def _template_detail() -> dict[str, object]:
return {
"id": "template-1",
"name": "Template",
"icon_info": {"icon": "book", "icon_type": "emoji", "icon_background": "#fff"},
"description": "Description",
"chunk_structure": "general",
"export_data": "dsl: value",
"graph": {"nodes": [], "edges": [], "viewport": {"x": 0, "y": 0, "zoom": 1}},
}
def _payload() -> dict[str, object]:
return {
"name": "Updated template",
"description": "Updated description",
"icon_info": {"icon": "book", "icon_type": "emoji", "icon_background": "#fff"},
}
def _account() -> Account:
return make_account(name="Test User", email="test@example.com")
def _pipeline() -> Pipeline:
pipeline = Pipeline(tenant_id="tenant-1", name="Pipeline")
pipeline.id = "pipeline-1"
return pipeline
@pytest.fixture
def database_app() -> Iterator[Flask]:
app = Flask(__name__)
app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///:memory:"
db.init_app(app)
with app.app_context():
PipelineCustomizedTemplate.__table__.create(db.engine)
yield app
class TestPipelineTemplateListApi:
def test_get_uses_query_defaults_and_serializes_nullable_fields(self, app: Flask, sqlite_engine: Engine) -> None:
api = PipelineTemplateListApi()
method = unwrap(api.get)
tenant_id = "tenant-1"
service_calls: list[tuple[str, str, str]] = []
def get_pipeline_templates(*, type: str, language: str, current_tenant_id: str, session) -> dict[str, object]:
del session
service_calls.append((type, language, current_tenant_id))
return {"pipeline_templates": [_template_item()]}
with (
Session(sqlite_engine) as session,
app.test_request_context("/rag/pipeline/templates"),
patch.object(module.RagPipelineService, "get_pipeline_templates", side_effect=get_pipeline_templates),
):
response, status = method(api, PipelineTemplateListQuery(), session, tenant_id)
assert status == 200
assert service_calls == [("built-in", "en-US", tenant_id)]
assert response == {
"pipeline_templates": [
{
**_template_item(),
"copyright": None,
"privacy_policy": None,
}
]
}
def test_get_passes_explicit_query_to_service(self, app: Flask, sqlite_engine: Engine) -> None:
api = PipelineTemplateListApi()
method = unwrap(api.get)
tenant_id = "tenant-1"
service_calls: list[tuple[str, str, str]] = []
def get_pipeline_templates(*, type: str, language: str, current_tenant_id: str, session) -> dict[str, object]:
del session
service_calls.append((type, language, current_tenant_id))
return {"pipeline_templates": []}
with (
Session(sqlite_engine) as session,
app.test_request_context("/rag/pipeline/templates?type=customized&language=ja-JP"),
patch.object(module.RagPipelineService, "get_pipeline_templates", side_effect=get_pipeline_templates),
):
response, status = method(
api, PipelineTemplateListQuery(type="customized", language="ja-JP"), session, tenant_id
)
assert status == 200
assert response == {"pipeline_templates": []}
assert service_calls == [("customized", "ja-JP", tenant_id)]
class TestPipelineTemplateDetailApi:
def test_get_serializes_template_detail(self, app: Flask, sqlite_engine: Engine) -> None:
api = PipelineTemplateDetailApi()
method = unwrap(api.get)
service_calls: list[tuple[str, str, str]] = []
def get_pipeline_template_detail(
template_id: str, current_tenant_id: str, type: str, *, session
) -> dict[str, object]:
del session
service_calls.append((template_id, current_tenant_id, type))
return _template_detail()
with (
Session(sqlite_engine) as session,
app.test_request_context("/rag/pipeline/templates/template-1?type=customized"),
patch.object(
module.RagPipelineService,
"get_pipeline_template_detail",
side_effect=get_pipeline_template_detail,
),
):
response, status = method(
api, PipelineTemplateDetailQuery(type="customized"), session, "tenant-1", "template-1"
)
assert status == 200
assert response == {**_template_detail(), "created_by": None}
assert service_calls == [("template-1", "tenant-1", "customized")]
def test_get_raises_not_found_without_custom_response_body(self, app: Flask, sqlite_engine: Engine) -> None:
api = PipelineTemplateDetailApi()
method = unwrap(api.get)
def get_pipeline_template_detail(template_id: str, current_tenant_id: str, type: str, *, session) -> None:
del template_id, current_tenant_id, type, session
with (
Session(sqlite_engine) as session,
app.test_request_context("/rag/pipeline/templates/missing"),
patch.object(
module.RagPipelineService,
"get_pipeline_template_detail",
side_effect=get_pipeline_template_detail,
),
pytest.raises(NotFound),
):
method(api, PipelineTemplateDetailQuery(), session, "tenant-1", "missing")
class TestCustomizedPipelineTemplateApi:
def test_patch_validates_payload_and_returns_empty_204(self, app: Flask) -> None:
api = CustomizedPipelineTemplateApi()
method = unwrap(api.patch)
payload = _payload()
account = _account()
tenant_id = "tenant-1"
service_calls: list[tuple[str, PipelineTemplateInfoEntity, Account, str]] = []
def update_template(
template_id: str,
template_info: PipelineTemplateInfoEntity,
current_user: Account,
current_tenant_id: str,
*,
session,
) -> None:
del session
service_calls.append((template_id, template_info, current_user, current_tenant_id))
with (
app.test_request_context("/rag/pipeline/customized/templates/template-1", method="PATCH", json=payload),
patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),
patch.object(module.RagPipelineService, "update_customized_pipeline_template", side_effect=update_template),
):
response, status = method(
api, CustomizedPipelineTemplatePayload.model_validate(payload), tenant_id, account, "template-1"
)
assert (response, status) == ("", 204)
assert len(service_calls) == 1
template_id, template_info, current_user, current_tenant_id = service_calls[0]
assert template_id == "template-1"
assert current_user is account
assert current_tenant_id == tenant_id
assert template_info.name == "Updated template"
assert template_info.description == "Updated description"
assert template_info.icon_info.model_dump() == {
"icon": "book",
"icon_background": "#fff",
"icon_type": "emoji",
"icon_url": None,
}
def test_patch_defaults_missing_icon_info_before_service_call(self, app: Flask) -> None:
api = CustomizedPipelineTemplateApi()
method = unwrap(api.patch)
payload: dict[str, object] = {
"name": "Updated template",
"description": "Updated description",
}
account = _account()
tenant_id = "tenant-1"
service_calls: list[tuple[str, PipelineTemplateInfoEntity, Account, str]] = []
def update_template(
template_id: str,
template_info: PipelineTemplateInfoEntity,
current_user: Account,
current_tenant_id: str,
*,
session,
) -> None:
del session
service_calls.append((template_id, template_info, current_user, current_tenant_id))
with (
app.test_request_context("/rag/pipeline/customized/templates/template-1", method="PATCH", json=payload),
patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),
patch.object(module.RagPipelineService, "update_customized_pipeline_template", side_effect=update_template),
):
response, status = method(
api, CustomizedPipelineTemplatePayload.model_validate(payload), tenant_id, account, "template-1"
)
assert (response, status) == ("", 204)
assert len(service_calls) == 1
template_id, template_info, current_user, current_tenant_id = service_calls[0]
assert template_id == "template-1"
assert current_user is account
assert current_tenant_id == tenant_id
assert template_info.icon_info.model_dump() == {
"icon": "",
"icon_background": None,
"icon_type": None,
"icon_url": None,
}
def test_delete_returns_empty_204(self, app: Flask) -> None:
api = CustomizedPipelineTemplateApi()
method = unwrap(api.delete)
tenant_id = "tenant-1"
deleted_templates: list[tuple[str, str]] = []
def delete_template(template_id: str, current_tenant_id: str, *, session) -> None:
del session
deleted_templates.append((template_id, current_tenant_id))
with (
app.test_request_context("/rag/pipeline/customized/templates/template-1", method="DELETE"),
patch.object(module.RagPipelineService, "delete_customized_pipeline_template", side_effect=delete_template),
):
response, status = method(api, tenant_id, "template-1")
assert (response, status) == ("", 204)
assert deleted_templates == [("template-1", tenant_id)]
@pytest.mark.parametrize("sqlite_session", [(PipelineCustomizedTemplate,)], indirect=True)
def test_post_exports_yaml_from_orm_template(self, app: Flask, sqlite_session: Session) -> None:
api = CustomizedPipelineTemplateApi()
method = unwrap(api.post)
template = PipelineCustomizedTemplate(
tenant_id="00000000-0000-0000-0000-000000000001",
name="Template",
description="Description",
chunk_structure="general",
icon={"icon": "book", "icon_type": "emoji", "icon_background": "#fff"},
position=1,
yaml_content="dsl: value",
install_count=0,
language="en-US",
created_by="00000000-0000-0000-0000-000000000002",
)
template.id = "template-1"
sqlite_session.add(template)
sqlite_session.commit()
with app.test_request_context("/rag/pipeline/customized/templates/template-1", method="POST"):
response, status = method(
api,
sqlite_session,
"00000000-0000-0000-0000-000000000001",
"template-1",
)
assert status == 200
assert response == {"data": "dsl: value"}
@pytest.mark.parametrize("sqlite_session", [(PipelineCustomizedTemplate,)], indirect=True)
def test_post_returns_not_found_for_other_tenant(self, app: Flask, sqlite_session: Session) -> None:
api = CustomizedPipelineTemplateApi()
method = unwrap(api.post)
template = PipelineCustomizedTemplate(
tenant_id="00000000-0000-0000-0000-000000000002",
name="Other tenant template",
description="Description",
chunk_structure="general",
icon={},
position=1,
yaml_content="secret: value",
install_count=0,
language="en-US",
created_by="00000000-0000-0000-0000-000000000003",
)
template.id = "template-1"
sqlite_session.add(template)
sqlite_session.commit()
with (
app.test_request_context("/rag/pipeline/customized/templates/template-1", method="POST"),
pytest.raises(NotFound, match="Customized pipeline template not found"),
):
method(
api,
sqlite_session,
"00000000-0000-0000-0000-000000000001",
"template-1",
)
class TestPublishCustomizedPipelineTemplateApi:
def test_post_uses_pipeline_release_rbac_scene(self) -> None:
[check] = rbac_checks(PublishCustomizedPipelineTemplateApi.post)
assert check.scene == module.RBACPermission.DATASET_PIPELINE_RELEASE
def test_post_validates_payload_and_returns_empty_204(self) -> None:
api = PublishCustomizedPipelineTemplateApi()
method = unwrap(api.post)
payload = _payload()
account = _account()
pipeline = _pipeline()
dataset = object()
with (
config_overrides_context(RBAC_ENABLED=True),
patch.object(module, "get_pipeline_dataset", return_value=dataset),
patch.object(module.DatasetService, "check_dataset_permission") as legacy_acl,
patch.object(module.RagPipelineService, "publish_customized_pipeline_template") as publish,
):
response, status = method(api, CustomizedPipelineTemplatePayload.model_validate(payload), account, pipeline)
assert (response, status) == ("", 204)
publish.assert_called_once_with(pipeline, dataset, payload, account, session=ANY)
legacy_acl.assert_not_called()
@pytest.mark.parametrize(
("payload", "expected_icon_info"),
[
(
{"name": "Published template", "description": "Description"},
{"icon": "", "icon_background": None, "icon_type": None, "icon_url": None},
),
({"name": "Published template", "description": "Description", "icon_info": {}}, {}),
],
)
def test_post_preserves_valid_icon_info(
self,
payload: dict[str, object],
expected_icon_info: dict[str, object | None],
) -> None:
api = PublishCustomizedPipelineTemplateApi()
method = unwrap(api.post)
account = _account()
pipeline = _pipeline()
dataset = object()
with (
config_overrides_context(RBAC_ENABLED=True),
patch.object(module, "get_pipeline_dataset", return_value=dataset),
patch.object(module.RagPipelineService, "publish_customized_pipeline_template") as publish,
):
response, status = method(api, CustomizedPipelineTemplatePayload.model_validate(payload), account, pipeline)
assert (response, status) == ("", 204)
publish.assert_called_once_with(pipeline, dataset, ANY, account, session=ANY)
assert publish.call_args.args[2]["icon_info"] == expected_icon_info
def test_post_translates_missing_owned_resource_to_not_found(self) -> None:
api = PublishCustomizedPipelineTemplateApi()
method = unwrap(api.post)
payload = _payload()
with (
config_overrides_context(RBAC_ENABLED=True),
patch.object(module, "get_pipeline_dataset", return_value=object()),
patch.object(
module.RagPipelineService,
"publish_customized_pipeline_template",
side_effect=RagPipelineResourceNotFoundError("Workflow not found"),
),
pytest.raises(NotFound, match="Workflow not found"),
):
method(api, CustomizedPipelineTemplatePayload.model_validate(payload), _account(), _pipeline())
def test_post_allows_legacy_dataset_operator_after_dataset_acl(self) -> None:
api = PublishCustomizedPipelineTemplateApi()
method = unwrap(api.post)
account = _account()
account.role = TenantAccountRole.DATASET_OPERATOR
pipeline = _pipeline()
dataset = object()
payload = _payload()
with (
config_overrides_context(RBAC_ENABLED=False),
patch.object(module, "get_pipeline_dataset", return_value=dataset),
patch.object(module.DatasetService, "check_dataset_permission") as check_permission,
patch.object(module.RagPipelineService, "publish_customized_pipeline_template") as publish,
):
response = method(api, CustomizedPipelineTemplatePayload.model_validate(payload), account, pipeline)
assert response == ("", 204)
assert check_permission.call_args.args[:2] == (dataset, account)
publish.assert_called_once_with(pipeline, dataset, payload, account, session=ANY)
def test_post_rejects_legacy_non_editor_before_dataset_acl(self) -> None:
api = PublishCustomizedPipelineTemplateApi()
method = unwrap(api.post)
account = _account()
account.role = TenantAccountRole.NORMAL
with (
config_overrides_context(RBAC_ENABLED=False),
patch.object(module, "get_pipeline_dataset", return_value=object()),
patch.object(module.DatasetService, "check_dataset_permission") as check_permission,
patch.object(module.RagPipelineService, "publish_customized_pipeline_template") as publish,
pytest.raises(Forbidden),
):
method(api, CustomizedPipelineTemplatePayload.model_validate(_payload()), account, _pipeline())
check_permission.assert_not_called()
publish.assert_not_called()
def test_post_rejects_legacy_dataset_acl_before_publish(self) -> None:
api = PublishCustomizedPipelineTemplateApi()
method = unwrap(api.post)
account = _account()
account.role = TenantAccountRole.EDITOR
with (
config_overrides_context(RBAC_ENABLED=False),
patch.object(module, "get_pipeline_dataset", return_value=object()),
patch.object(
module.DatasetService,
"check_dataset_permission",
side_effect=NoPermissionError("Dataset is private"),
),
patch.object(module.RagPipelineService, "publish_customized_pipeline_template") as publish,
pytest.raises(Forbidden, match="Dataset is private"),
):
method(api, CustomizedPipelineTemplatePayload.model_validate(_payload()), account, _pipeline())
publish.assert_not_called()
def test_post_rejects_missing_legacy_dataset_before_publish(self) -> None:
api = PublishCustomizedPipelineTemplateApi()
method = unwrap(api.post)
account = _account()
account.role = TenantAccountRole.EDITOR
with (
config_overrides_context(RBAC_ENABLED=False),
patch.object(module, "get_pipeline_dataset", return_value=None),
patch.object(module.DatasetService, "check_dataset_permission") as check_permission,
patch.object(module.RagPipelineService, "publish_customized_pipeline_template") as publish,
pytest.raises(NotFound, match="Dataset not found"),
):
method(api, CustomizedPipelineTemplatePayload.model_validate(_payload()), account, _pipeline())
check_permission.assert_not_called()
publish.assert_not_called()
@@ -0,0 +1,189 @@
"""Unit tests for rag_pipeline_datasets controller endpoints."""
from __future__ import annotations
from inspect import unwrap
from unittest.mock import MagicMock, patch
import pytest
from flask import Flask
from werkzeug.exceptions import Forbidden
import services
from controllers.console import console_ns
from controllers.console.datasets.error import DatasetNameDuplicateError
from controllers.console.datasets.rag_pipeline.rag_pipeline_datasets import (
CreateEmptyRagPipelineDatasetApi,
CreateRagPipelineDatasetApi,
RagPipelineDatasetImportPayload,
)
from models.account import Account, TenantAccountRole
from services.entities.dsl_entities import ImportStatus
from services.rbac import contracts as rbac_contracts
from tests.helpers.unit.config_override import config_overrides_context
from tests.helpers.unit.model_factories import make_account
def _account(*, editor: bool) -> Account:
return make_account(
account_id=None,
name="RAG Pipeline Tester",
email="rag-pipeline@example.com",
role=TenantAccountRole.EDITOR if editor else TenantAccountRole.NORMAL,
)
class TestCreateRagPipelineDatasetApi:
def _valid_payload(self) -> dict[str, str]:
return {"yaml_content": "name: test"}
def test_post_success(self, app: Flask) -> None:
api = CreateRagPipelineDatasetApi()
method = unwrap(api.post)
payload = self._valid_payload()
user = _account(editor=True)
import_info = {
"id": "import-1",
"status": ImportStatus.COMPLETED,
"dataset_id": "ds-1",
"pipeline_id": "pipeline-1",
"current_dsl_version": "0.1.0",
"imported_dsl_version": "0.1.0",
"error": "",
}
mock_service = MagicMock()
mock_service.create_rag_pipeline_dataset.return_value = import_info
with (
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", payload),
patch(
"controllers.console.datasets.rag_pipeline.rag_pipeline_datasets.RagPipelineDslService",
return_value=mock_service,
),
):
response, status = method(api, RagPipelineDatasetImportPayload.model_validate(payload), "tenant-1", user)
assert status == 201
assert response == {
"id": "import-1",
"status": "completed",
"dataset_id": "ds-1",
"pipeline_id": "pipeline-1",
"current_dsl_version": "0.1.0",
"imported_dsl_version": "0.1.0",
"error": "",
}
def test_post_success_syncs_creator_under_rbac(self, app: Flask) -> None:
"""RBAC: RAG-pipeline dataset create must keep the creator bound to their own dataset.
Regression guard for the #42430 flip to automatic_include_workspace_members=False:
this path bypasses DatasetService.create_empty_dataset (which syncs the creator
internally), so the controller has to sync the creator explicitly or the creator
gets 403 on their own dataset.
"""
api = CreateRagPipelineDatasetApi()
method = unwrap(api.post)
payload = self._valid_payload()
user = _account(editor=True)
import_info = {
"id": "import-1",
"status": ImportStatus.COMPLETED,
"dataset_id": "ds-1",
"pipeline_id": "pipeline-1",
"current_dsl_version": "0.1.0",
"imported_dsl_version": "0.1.0",
"error": "",
}
mock_service = MagicMock()
mock_service.create_rag_pipeline_dataset.return_value = import_info
with (
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", payload),
config_overrides_context(RBAC_ENABLED=True),
patch(
"controllers.console.datasets.rag_pipeline.rag_pipeline_datasets.RagPipelineDslService",
return_value=mock_service,
),
patch(
"controllers.console.datasets.rag_pipeline.rag_pipeline_datasets.enterprise_rbac_service.RBACService.DatasetAccess.replace_whitelist"
),
patch(
"controllers.console.datasets.rag_pipeline.rag_pipeline_datasets.enterprise_rbac_service.try_sync_creator_access_policy_member_bindings"
) as sync_creator,
):
response, status = method(api, RagPipelineDatasetImportPayload.model_validate(payload), "tenant-1", user)
assert status == 201
assert sync_creator.called, "RAG pipeline create must sync the creator's own access"
sync_creator.assert_called_once_with("tenant-1", user.id, rbac_contracts.RBACResourceType.DATASET, "ds-1")
assert response["dataset_id"] == "ds-1"
def test_post_forbidden_non_editor(self, app: Flask) -> None:
api = CreateRagPipelineDatasetApi()
method = unwrap(api.post)
payload = self._valid_payload()
user = _account(editor=False)
with (
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", payload),
):
with pytest.raises(Forbidden):
method(api, RagPipelineDatasetImportPayload.model_validate(payload), "tenant-1", user)
def test_post_dataset_name_duplicate(self, app: Flask) -> None:
api = CreateRagPipelineDatasetApi()
method = unwrap(api.post)
payload = self._valid_payload()
user = _account(editor=True)
mock_service = MagicMock()
mock_service.create_rag_pipeline_dataset.side_effect = services.errors.dataset.DatasetNameDuplicateError()
with (
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", payload),
patch(
"controllers.console.datasets.rag_pipeline.rag_pipeline_datasets.RagPipelineDslService",
return_value=mock_service,
),
):
with pytest.raises(DatasetNameDuplicateError):
method(api, RagPipelineDatasetImportPayload.model_validate(payload), "tenant-1", user)
def test_post_invalid_payload(self, app: Flask) -> None:
api = CreateRagPipelineDatasetApi()
method = unwrap(api.post)
payload: dict[str, str] = {}
user = _account(editor=True)
with (
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", payload),
):
with pytest.raises(
ValueError, match="1 validation error for RagPipelineDatasetImportPayload\\\nyaml_content"
):
method(api, RagPipelineDatasetImportPayload.model_validate(payload), "tenant-1", user)
class TestCreateEmptyRagPipelineDatasetApi:
def test_post_forbidden_non_editor(self, app: Flask) -> None:
api = CreateEmptyRagPipelineDatasetApi()
method = unwrap(api.post)
user = _account(editor=False)
with app.test_request_context("/"):
with pytest.raises(Forbidden):
method(api, "tenant-1", user)
@@ -0,0 +1,470 @@
from collections.abc import Callable, Iterator
from inspect import unwrap
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
from flask import Flask, Response
from sqlalchemy import Engine
from sqlalchemy.orm import Session, scoped_session, sessionmaker
from controllers.common.errors import InvalidArgumentError, NotFoundError
from controllers.common.rbac import RBACResourceScope
from controllers.console import console_ns
from controllers.console.app.error import DraftWorkflowNotExist
from controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable import (
PaginationQuery,
RagPipelineEnvironmentVariableCollectionApi,
RagPipelineNodeVariableCollectionApi,
RagPipelineSystemVariableCollectionApi,
RagPipelineVariableApi,
RagPipelineVariableCollectionApi,
RagPipelineVariableResetApi,
WorkflowDraftVariablePatchPayload,
)
from controllers.console.wraps import RBACPermission
from core.workflow.llm_environment_variable import LLMEnvironmentVariable
from core.workflow.variable_prefixes import SYSTEM_VARIABLE_NODE_ID
from factories.variable_factory import build_segment
from graphon.file import File, FileTransferMethod, FileType
from graphon.variables import IntegerVariable, StringVariable
from models.account import Account, TenantAccountRole
from models.dataset import Pipeline
from models.workflow import Workflow, WorkflowDraftVariable, WorkflowType
from tests.helpers.unit.controllers.rbac_introspection import rbac_checks
from tests.helpers.unit.model_factories import make_workflow
@pytest.fixture
def sqlite_db(sqlite_engine: Engine, sqlite_session_factory: sessionmaker[Session]) -> Iterator[SimpleNamespace]:
session = scoped_session(sqlite_session_factory)
yield SimpleNamespace(engine=sqlite_engine, session=session)
session.remove()
def _pipeline(*, pipeline_id: str = "p1", tenant_id: str = "t1") -> Pipeline:
pipeline = Pipeline(tenant_id=tenant_id, name="Pipeline")
pipeline.id = pipeline_id
return pipeline
def _workflow(pipeline: Pipeline, environment_variables=()) -> Workflow:
return make_workflow(
tenant_id=pipeline.tenant_id,
app_id=pipeline.id,
workflow_type=WorkflowType.RAG_PIPELINE,
graph="{}",
environment_variables=environment_variables,
)
def _variable(*, app_id: str = "p1", user_id: str = "account-1", file_value: bool = False) -> WorkflowDraftVariable:
if file_value:
value = build_segment(
File(
file_id="file-1",
file_type=FileType.IMAGE,
transfer_method=FileTransferMethod.REMOTE_URL,
remote_url="https://example.com/image.png",
filename="image.png",
extension=".png",
mime_type="image/png",
size=1,
)
)
else:
value = build_segment("value")
return WorkflowDraftVariable.new_node_variable(
app_id=app_id,
user_id=user_id,
node_id="node-1",
name="variable",
value=value,
node_execution_id="execution-1",
)
@pytest.fixture
def editor_user() -> Account:
user = Account(name="Test User", email="user@example.com")
user.id = "account-1"
user.role = TenantAccountRole.EDITOR
return user
@pytest.fixture
def restx_config(app):
return patch.dict(app.config, {"RESTX_MASK_HEADER": "X-Fields"})
def test_rag_draft_variable_routes_require_dataset_edit_permission() -> None:
route = RagPipelineVariableApi.get
legacy_gate = unwrap(route, stop=lambda decorator: "edit_permission_required" in decorator.__code__.co_qualname)
assert "edit_permission_required" in legacy_gate.__code__.co_qualname
[check] = rbac_checks(route)
assert check.locator.scope == RBACResourceScope.DATASET
assert check.scene == RBACPermission.DATASET_EDIT
class TestRagPipelineVariableCollectionApi:
def test_get_variables_success(self, app: Flask, sqlite_db, editor_user, restx_config):
api = RagPipelineVariableCollectionApi()
method = unwrap(api.get)
pipeline = _pipeline()
rag_srv = MagicMock()
rag_srv.is_workflow_exist.return_value = True
var_list = MagicMock()
var_list.variables = []
var_list.total = None
draft_srv = MagicMock()
draft_srv.list_variables_without_values.return_value = var_list
with (
app.test_request_context("/?page=1&limit=10"),
restx_config,
patch("controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.db", sqlite_db),
patch(
"controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.RagPipelineService",
return_value=rag_srv,
),
patch(
"controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.WorkflowDraftVariableService",
return_value=draft_srv,
),
):
result = method(api, PaginationQuery(page=1, limit=10), editor_user, pipeline)
assert result == {"items": [], "total": None}
draft_srv.list_variables_without_values.assert_called_once_with(
app_id="p1",
page=1,
limit=10,
user_id="account-1",
)
def test_get_variables_workflow_not_exist(self, app: Flask, sqlite_db, editor_user):
api = RagPipelineVariableCollectionApi()
method = unwrap(api.get)
pipeline = _pipeline()
rag_srv = MagicMock()
rag_srv.is_workflow_exist.return_value = False
with (
app.test_request_context("/"),
patch("controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.db", sqlite_db),
patch(
"controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.RagPipelineService",
return_value=rag_srv,
),
):
with pytest.raises(DraftWorkflowNotExist):
method(api, PaginationQuery(), editor_user, pipeline)
def test_delete_variables_success(self, app: Flask, sqlite_db, editor_user):
api = RagPipelineVariableCollectionApi()
method = unwrap(api.delete)
pipeline = _pipeline()
with (
app.test_request_context("/"),
patch("controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.db", sqlite_db),
patch("controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.WorkflowDraftVariableService"),
):
result = method(api, editor_user, pipeline)
assert isinstance(result, Response)
assert result.status_code == 204
class TestRagPipelineNodeVariableCollectionApi:
def test_get_node_variables_success(self, app: Flask, sqlite_db, editor_user, restx_config):
api = RagPipelineNodeVariableCollectionApi()
method = unwrap(api.get)
pipeline = _pipeline()
var_list = MagicMock()
var_list.variables = []
srv = MagicMock()
srv.list_node_variables.return_value = var_list
with (
app.test_request_context("/"),
restx_config,
patch("controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.db", sqlite_db),
patch(
"controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.WorkflowDraftVariableService",
return_value=srv,
),
):
result = method(api, editor_user, pipeline, "node1")
assert result == {"items": []}
srv.list_node_variables.assert_called_once_with("p1", "node1", user_id="account-1")
def test_get_node_variables_invalid_node(self, app: Flask, editor_user):
api = RagPipelineNodeVariableCollectionApi()
method = unwrap(api.get)
with (
app.test_request_context("/"),
):
with pytest.raises(InvalidArgumentError):
method(api, editor_user, _pipeline(), SYSTEM_VARIABLE_NODE_ID)
class TestRagPipelineVariableApi:
def test_get_variable_not_found(self, app: Flask, sqlite_db, editor_user):
api = RagPipelineVariableApi()
method = unwrap(api.get)
srv = MagicMock()
srv.get_variable.return_value = None
with (
app.test_request_context("/"),
patch("controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.db", sqlite_db),
patch(
"controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.WorkflowDraftVariableService",
return_value=srv,
),
):
with pytest.raises(NotFoundError):
method(api, editor_user, _pipeline(), "v1")
def test_patch_variable_invalid_file_payload(self, app: Flask, sqlite_db, editor_user):
api = RagPipelineVariableApi()
method = unwrap(api.patch)
pipeline = _pipeline()
variable = _variable(file_value=True)
srv = MagicMock()
srv.get_variable.return_value = variable
payload = {"value": "invalid"}
with (
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", payload),
patch("controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.db", sqlite_db),
patch(
"controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.WorkflowDraftVariableService",
return_value=srv,
),
):
with pytest.raises(InvalidArgumentError):
method(api, WorkflowDraftVariablePatchPayload.model_validate(payload), editor_user, pipeline, "v1")
def test_delete_variable_success(self, app: Flask, sqlite_db, editor_user):
api = RagPipelineVariableApi()
method = unwrap(api.delete)
pipeline = _pipeline()
variable = _variable()
srv = MagicMock()
srv.get_variable.return_value = variable
with (
app.test_request_context("/"),
patch("controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.db", sqlite_db),
patch(
"controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.WorkflowDraftVariableService",
return_value=srv,
),
):
result = method(api, editor_user, pipeline, "v1")
assert result.status_code == 204
@pytest.mark.parametrize(
("api_type", "method", "payload"),
[
(RagPipelineVariableApi, RagPipelineVariableApi.get, None),
(
RagPipelineVariableApi,
RagPipelineVariableApi.patch,
WorkflowDraftVariablePatchPayload(name="new name"),
),
(RagPipelineVariableApi, RagPipelineVariableApi.delete, None),
(RagPipelineVariableResetApi, RagPipelineVariableResetApi.put, None),
],
)
def test_direct_variable_access_rejects_different_user(
app: Flask,
sqlite_db: SimpleNamespace,
editor_user: Account,
api_type: type[RagPipelineVariableApi] | type[RagPipelineVariableResetApi],
method: Callable[..., object],
payload: WorkflowDraftVariablePatchPayload | None,
) -> None:
api = api_type()
method = unwrap(method)
pipeline = _pipeline()
variable = _variable(user_id="account-2")
draft_service = MagicMock()
draft_service.get_variable.return_value = variable
rag_service = MagicMock()
rag_service.get_draft_workflow.return_value = _workflow(pipeline)
if payload is not None:
call_args = (api, payload, editor_user, pipeline, "v1")
else:
call_args = (api, editor_user, pipeline, "v1")
with (
app.test_request_context("/"),
patch("controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.db", sqlite_db),
patch.object(sqlite_db.session, "commit") as commit,
patch(
"controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.RagPipelineService",
return_value=rag_service,
),
patch(
"controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.WorkflowDraftVariableService",
return_value=draft_service,
),
pytest.raises(NotFoundError),
):
method(*call_args)
draft_service.update_variable.assert_not_called()
draft_service.delete_variable.assert_not_called()
draft_service.reset_variable.assert_not_called()
commit.assert_not_called()
class TestRagPipelineVariableResetApi:
def test_reset_variable_success(self, app: Flask, sqlite_db, editor_user):
api = RagPipelineVariableResetApi()
method = unwrap(api.put)
pipeline = _pipeline()
workflow = _workflow(pipeline)
variable = _variable()
srv = MagicMock()
srv.get_variable.return_value = variable
srv.reset_variable.return_value = variable
rag_srv = MagicMock()
rag_srv.get_draft_workflow.return_value = workflow
with (
app.test_request_context("/"),
patch("controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.db", sqlite_db),
patch(
"controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.RagPipelineService",
return_value=rag_srv,
),
patch(
"controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.WorkflowDraftVariableService",
return_value=srv,
),
patch(
"controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.dump_response",
return_value={"id": "v1"},
),
):
result = method(api, editor_user, pipeline, "v1")
assert result == {"id": "v1"}
class TestSystemAndEnvironmentVariablesApi:
def test_system_variables_success(self, app: Flask, sqlite_db, editor_user, restx_config):
api = RagPipelineSystemVariableCollectionApi()
method = unwrap(api.get)
pipeline = _pipeline()
var_list = MagicMock()
var_list.variables = []
srv = MagicMock()
srv.list_system_variables.return_value = var_list
with (
app.test_request_context("/"),
restx_config,
patch("controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.db", sqlite_db),
patch(
"controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.WorkflowDraftVariableService",
return_value=srv,
),
):
result = method(api, editor_user, pipeline)
assert result == {"items": []}
srv.list_system_variables.assert_called_once_with("p1", user_id="account-1")
def test_environment_variables_success(self, app: Flask, editor_user):
api = RagPipelineEnvironmentVariableCollectionApi()
method = unwrap(api.get)
env_var = StringVariable(
id="e1",
name="ENV",
description="d",
selector=["env", "ENV"],
value="x",
)
pipeline = _pipeline()
workflow = _workflow(pipeline, [env_var])
rag_srv = MagicMock()
rag_srv.get_draft_workflow.return_value = workflow
with (
app.test_request_context("/"),
patch(
"controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.RagPipelineService",
return_value=rag_srv,
),
):
result = method(api, editor_user, pipeline)
assert len(result["items"]) == 1
assert result["items"][0]["value_type"] == "string"
def test_environment_variables_preserve_number_subtype_and_llm_type(self, app: Flask, editor_user):
api = RagPipelineEnvironmentVariableCollectionApi()
method = unwrap(api.get)
number_var = IntegerVariable(
id="number",
name="NUMBER",
description="",
selector=["env", "NUMBER"],
value=1,
)
llm_var = LLMEnvironmentVariable(
id="llm",
name="MODEL",
value={"provider": "provider", "name": "model", "mode": "chat"},
selector=["env", "MODEL"],
)
pipeline = _pipeline()
rag_srv = MagicMock()
rag_srv.get_draft_workflow.return_value = _workflow(pipeline, [number_var, llm_var])
with (
app.test_request_context("/"),
patch(
"controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.RagPipelineService",
return_value=rag_srv,
),
):
result = method(api, editor_user, pipeline)
assert [item["value_type"] for item in result["items"]] == ["integer", "llm"]
@@ -0,0 +1,402 @@
"""Unit tests for rag_pipeline_import controller endpoints."""
from __future__ import annotations
from collections.abc import Callable, Iterator
from inspect import unwrap
from unittest.mock import MagicMock, patch
import pytest
from flask import Flask
from sqlalchemy.orm import Session
from controllers.common.schema import JsonResponseWithStatus
from controllers.console import console_ns
from controllers.console.datasets.rag_pipeline.rag_pipeline_import import (
IncludeSecretQuery,
RagPipelineExportApi,
RagPipelineImportApi,
RagPipelineImportCheckDependenciesApi,
RagPipelineImportConfirmApi,
RagPipelineImportPayload,
)
from core.plugin.entities.plugin import PluginDependency, PluginDependencyType
from models.account import Account
from models.dataset import Pipeline
from models.engine import db
from services.enterprise import rbac_service
from services.entities.dsl_entities import CheckDependenciesResult, ImportStatus
from services.rag_pipeline.rag_pipeline_dsl_service import RagPipelineImportInfo
from tests.helpers.unit.model_factories import make_account, make_tenant
def _account() -> Account:
return make_account(name="RAG Import Tester", email="rag-import@example.com")
@pytest.fixture
def app() -> Iterator[Flask]:
app = Flask(__name__)
app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///:memory:"
db.init_app(app)
with app.app_context():
yield app
class TestRagPipelineImportApi:
def _payload(self, mode: str = "create") -> dict[str, str]:
return {
"mode": mode,
"yaml_content": "content",
"name": "Test",
}
def test_post_success_200(self, app: Flask) -> None:
api = RagPipelineImportApi()
method = unwrap(api.post)
payload = self._payload()
user = _account()
result = RagPipelineImportInfo(
id="import-1",
status=ImportStatus.COMPLETED,
pipeline_id="pipeline-1",
dataset_id="dataset-1",
current_dsl_version="0.1.0",
imported_dsl_version="0.1.0",
)
service = MagicMock()
service.import_rag_pipeline.return_value = result
with (
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", payload),
patch(
"controllers.console.datasets.rag_pipeline.rag_pipeline_import.RagPipelineDslService",
return_value=service,
),
):
response, status = method(api, RagPipelineImportPayload(mode="create"), user)
assert status == 200
assert response == {
"id": "import-1",
"status": "completed",
"pipeline_id": "pipeline-1",
"dataset_id": "dataset-1",
"current_dsl_version": "0.1.0",
"imported_dsl_version": "0.1.0",
"error": "",
}
def test_post_failed_400(self, app: Flask) -> None:
api = RagPipelineImportApi()
method = unwrap(api.post)
payload = self._payload()
user = _account()
result = RagPipelineImportInfo(
id="import-1",
status=ImportStatus.FAILED,
current_dsl_version="0.1.0",
imported_dsl_version="",
error="bad dsl",
)
service = MagicMock()
service.import_rag_pipeline.return_value = result
with (
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", payload),
patch(
"controllers.console.datasets.rag_pipeline.rag_pipeline_import.RagPipelineDslService",
return_value=service,
),
):
response, status = method(api, RagPipelineImportPayload(mode="create"), user)
assert status == 400
assert response["status"] == "failed"
assert response["error"] == "bad dsl"
assert response["pipeline_id"] is None
assert response["dataset_id"] is None
def test_post_pending_202(self, app: Flask) -> None:
api = RagPipelineImportApi()
method = unwrap(api.post)
payload = self._payload()
user = _account()
result = RagPipelineImportInfo(
id="import-1",
status=ImportStatus.PENDING,
pipeline_id="pipeline-1",
dataset_id="dataset-1",
current_dsl_version="0.1.0",
imported_dsl_version="0.2.0",
)
service = MagicMock()
service.import_rag_pipeline.return_value = result
with (
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", payload),
patch(
"controllers.console.datasets.rag_pipeline.rag_pipeline_import.RagPipelineDslService",
return_value=service,
),
):
response, status = method(api, RagPipelineImportPayload(mode="create"), user)
assert status == 202
assert response["status"] == "pending"
assert response["pipeline_id"] == "pipeline-1"
assert response["dataset_id"] == "dataset-1"
class TestRagPipelineImportConfirmApi:
def test_confirm_success(self, app: Flask) -> None:
api = RagPipelineImportConfirmApi()
method = unwrap(api.post)
user = _account()
result = RagPipelineImportInfo(
id="import-1",
status=ImportStatus.COMPLETED,
pipeline_id="pipeline-1",
dataset_id="dataset-1",
current_dsl_version="0.1.0",
imported_dsl_version="0.1.0",
)
service = MagicMock()
service.confirm_import.return_value = result
with (
app.test_request_context("/"),
patch(
"controllers.console.datasets.rag_pipeline.rag_pipeline_import.RagPipelineDslService",
return_value=service,
),
):
response, status = method(api, user, "import-1")
assert status == 200
assert response["status"] == "completed"
assert response["pipeline_id"] == "pipeline-1"
def test_confirm_failed(self, app: Flask) -> None:
api = RagPipelineImportConfirmApi()
method = unwrap(api.post)
user = _account()
result = RagPipelineImportInfo(
id="import-1",
status=ImportStatus.FAILED,
current_dsl_version="0.1.0",
imported_dsl_version="",
error="missing dependency",
)
service = MagicMock()
service.confirm_import.return_value = result
with (
app.test_request_context("/"),
patch(
"controllers.console.datasets.rag_pipeline.rag_pipeline_import.RagPipelineDslService",
return_value=service,
),
):
response, status = method(api, user, "import-1")
assert status == 400
assert response["status"] == "failed"
assert response["error"] == "missing dependency"
class TestRagPipelineImportCheckDependenciesApi:
def test_get_success(self, app: Flask) -> None:
api = RagPipelineImportCheckDependenciesApi()
method = unwrap(api.get)
pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")
result = CheckDependenciesResult()
service = MagicMock()
service.check_dependencies.return_value = result
with (
app.test_request_context("/"),
patch(
"controllers.console.datasets.rag_pipeline.rag_pipeline_import.RagPipelineDslService",
return_value=service,
),
):
response, status = method(api, pipeline)
assert status == 200
assert response == {"leaked_dependencies": []}
def test_get_serializes_leaked_dependencies(self, app: Flask) -> None:
api = RagPipelineImportCheckDependenciesApi()
method = unwrap(api.get)
pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")
dependency = PluginDependency(
type=PluginDependencyType.Marketplace,
value=PluginDependency.Marketplace(
marketplace_plugin_unique_identifier="langgenius/example:0.1.0",
version="0.1.0",
),
current_identifier="langgenius/example:0.0.1",
)
service = MagicMock()
service.check_dependencies.return_value = CheckDependenciesResult(leaked_dependencies=[dependency])
with (
app.test_request_context("/"),
patch(
"controllers.console.datasets.rag_pipeline.rag_pipeline_import.RagPipelineDslService",
return_value=service,
),
):
response, status = method(api, pipeline)
assert status == 200
assert response == {
"leaked_dependencies": [
{
"type": "marketplace",
"value": {
"marketplace_plugin_unique_identifier": "langgenius/example:0.1.0",
"version": "0.1.0",
},
"current_identifier": "langgenius/example:0.0.1",
}
]
}
class TestRagPipelineExportApi:
def test_get_with_include_secret(self, app: Flask) -> None:
api = RagPipelineExportApi()
method = unwrap(api.get)
pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")
service = MagicMock()
service.export_rag_pipeline_dsl.return_value = "yaml: data"
with (
app.test_request_context("/?include_secret=true"),
patch(
"controllers.console.datasets.rag_pipeline.rag_pipeline_import.RagPipelineDslService",
return_value=service,
),
):
response, status = method(api, IncludeSecretQuery(), pipeline)
assert status == 200
assert response == {"data": "yaml: data"}
@pytest.mark.parametrize("confirm", [False, True])
@pytest.mark.parametrize(
("import_status", "created", "dataset_id", "rbac_enabled", "commit_fails", "expected_calls"),
[
(ImportStatus.COMPLETED, True, "dataset-1", True, False, 1),
(ImportStatus.COMPLETED_WITH_WARNINGS, True, "dataset-1", True, False, 1),
(ImportStatus.COMPLETED, False, "dataset-1", True, False, 0),
(ImportStatus.PENDING, True, "dataset-1", True, False, 0),
(ImportStatus.FAILED, True, "dataset-1", True, False, 0),
(ImportStatus.COMPLETED, True, None, True, False, 0),
(ImportStatus.COMPLETED, True, "dataset-1", False, False, 0),
(ImportStatus.COMPLETED, True, "dataset-1", True, True, 0),
],
)
def test_creator_binding_only_after_successful_new_dataset_commit(
app: Flask,
config_overrides: Callable[..., None],
confirm: bool,
import_status: ImportStatus,
created: bool,
dataset_id: str | None,
rbac_enabled: bool,
commit_fails: bool,
expected_calls: int,
) -> None:
config_overrides(RBAC_ENABLED=rbac_enabled)
user = make_account(account_id="creator-1", tenant=make_tenant(tenant_id="tenant-1"))
result = RagPipelineImportInfo.model_validate(
{
"id": "import-1",
"status": import_status,
"dataset_id": dataset_id,
"dataset_created": created,
}
)
service = MagicMock()
service.import_rag_pipeline.return_value = result
service.confirm_import.return_value = result
events: list[str] = []
session = Session(db.engine, expire_on_commit=False)
commit = session.commit
close = session.close
def commit_session() -> None:
if commit_fails:
raise RuntimeError("commit failed")
commit()
events.append("commit")
def close_session() -> None:
close()
events.append("close")
def sync_binding(*_args: object, **_kwargs: object) -> list[object]:
events.append("sync")
return []
with (
app.test_request_context("/"),
patch("controllers.console.datasets.rag_pipeline.rag_pipeline_import.Session", return_value=session),
patch(
"controllers.console.datasets.rag_pipeline.rag_pipeline_import.RagPipelineDslService", return_value=service
),
patch.object(session, "commit", side_effect=commit_session),
patch.object(session, "close", side_effect=close_session),
patch.object(
rbac_service.RBACService.AccessPolicies,
"sync_creator_access_policy_member_bindings",
side_effect=sync_binding,
) as sync,
patch.object(rbac_service.RBACService.DatasetAccess, "replace_whitelist") as replace,
):
def invoke() -> JsonResponseWithStatus:
if confirm:
api = RagPipelineImportConfirmApi()
return unwrap(api.post)(api, user, "import-1")
api = RagPipelineImportApi()
return unwrap(api.post)(api, RagPipelineImportPayload(mode="yaml-content"), user)
if commit_fails:
with pytest.raises(RuntimeError, match="commit failed"):
invoke()
else:
response, _ = invoke()
assert "dataset_created" not in response
assert sync.call_count == expected_calls
if expected_calls:
assert events == ["commit", "close", "sync"]
sync.assert_called_once_with(
"tenant-1",
"creator-1",
resource_type=rbac_service.RBACResourceType.DATASET,
resource_id="dataset-1",
)
replace.assert_not_called()
@@ -0,0 +1,711 @@
from __future__ import annotations
import ast
from collections.abc import Callable
from dataclasses import dataclass, field
from datetime import UTC, datetime
from inspect import unwrap
from pathlib import Path
from typing import TYPE_CHECKING, cast
import pytest
from flask import Flask
from flask.testing import FlaskClient
from controllers.common.errors import NotFoundError
from controllers.common.rbac import DatasetId
from controllers.console import api as console_api
from controllers.console import console_ns
from controllers.console.app.error import ProviderNotInitializeError
from controllers.console.datasets import data_source as data_source_controller
from controllers.console.datasets.error import IndexingEstimateError
from core.entities.knowledge_entities import IndexingEstimate
from core.rbac import RBACPermission
from enums import DeploymentEdition
from libs.external_api import ExternalApi
from libs.login import AccountWithTenant
from machinery.context import RequestContext
from services.credentials.refresh import CredentialRefreshUnavailableError
from services.data_source.binding_application_service import (
BindingMutationResult,
DataSourceBindingApplicationService,
)
from services.data_source.entities.notion_import import (
AuthorizedNotionPage,
NotionPageIcon,
NotionPageType,
NotionWorkspace,
)
from services.data_source.entities.oauth import DataSourceBindingSummary
from services.data_source.notion_import_application_service import NotionImportApplicationService
from services.knowledge.dataset_access import DatasetAccessRecord, DatasetAccessService, DatasetAccessSnapshot
from services.knowledge.document_sync import DocumentSyncApplicationService, SyncDocumentRecord
from services.knowledge.entities.indexing_estimate import (
NewSourcesEstimateCommand,
NotionEstimateSource,
)
from services.knowledge.indexing.estimate import (
DatasetEstimateRecord,
EstimateDocumentRecord,
EstimateSourceNotFoundError,
IndexingEstimateCredentialUnavailableError,
IndexingEstimateExecutionError,
IndexingEstimateProviderUnavailableError,
)
from services.knowledge.resource_scope import DatasetRef, DocumentRef
from tests.helpers.unit.controllers.rbac_introspection import rbac_checks
"""Guard all migrated dataset endpoints against transport/persistence regressions."""
class TestApplicationBoundariesCases:
@staticmethod
@pytest.mark.parametrize(
("filename", "count"),
[("data_source.py", 7), ("datasets.py", 25), ("datasets_document.py", 24), ("external.py", 8)],
)
def test_every_endpoint_declares_admission_context_and_application_service(filename: str, count: int) -> None:
root = Path(__file__).resolve().parents[5]
tree = ast.parse((root / "controllers/console/datasets" / filename).read_text())
# This namespace package is not traversed by import-linter's package graph.
forbidden = (
"sqlalchemy",
"repositories",
"controllers.common.session",
"libs.login",
"services.knowledge.dataset_service",
"services.knowledge.external.service",
"services.hit_testing_service",
"services.knowledge.dataset_read_service",
"services.knowledge.datasets.adapters",
"services.knowledge.documents.adapters",
"services.knowledge.external.adapters",
)
for node in ast.walk(tree):
modules = (
[node.module or ""]
if isinstance(node, ast.ImportFrom)
else [item.name for item in node.names]
if isinstance(node, ast.Import)
else []
)
for module in modules:
assert not any(module == name or module.startswith(name + ".") for name in forbidden), module
endpoints = [
method
for resource in tree.body
if isinstance(resource, ast.ClassDef)
for method in resource.body
if isinstance(method, ast.FunctionDef) and method.name in {"get", "post", "patch", "put", "delete"}
]
assert len(endpoints) == count
for endpoint in endpoints:
assert any(
isinstance(d, ast.Call) and ast.unparse(d.func) == "console_account_admission"
for d in endpoint.decorator_list
), endpoint.name
assert any(
arg.arg == "request_context"
and arg.annotation is not None
and ast.unparse(arg.annotation) == "RequestContext"
for arg in endpoint.args.args
), endpoint.name
assert any(
isinstance(n, ast.Call) and ast.unparse(n.func) == "application_services" for n in ast.walk(endpoint)
), endpoint.name
assert not {"session", "current_user", "current_tenant_id"}.intersection(
arg.arg for arg in endpoint.args.args
)
# Importing the module registers only the resources under test on console_ns.
if TYPE_CHECKING:
from models import Account
TENANT_ID = "11111111-1111-1111-1111-111111111111"
ACCOUNT_ID = "22222222-2222-2222-2222-222222222222"
BINDING_ID = "33333333-3333-3333-3333-333333333333"
DATASET_ID = "44444444-4444-4444-4444-444444444444"
DOCUMENT_ID = "55555555-5555-5555-5555-555555555555"
PAGE_ID = "66666666-6666-6666-6666-666666666666"
FOREIGN_DATASET_ID = "77777777-7777-7777-7777-777777777777"
@dataclass
class RecordingBindingStore:
bindings: tuple[DataSourceBindingSummary, ...] = ()
mutations: list[tuple[str, str, bool]] = field(default_factory=list)
mutation_result: BindingMutationResult = "updated"
def list_enabled_bindings(self, *, workspace_id: str) -> tuple[DataSourceBindingSummary, ...]:
assert workspace_id == TENANT_ID
return self.bindings
def change_disabled_state(
self,
*,
workspace_id: str,
binding_id: str,
disabled: bool,
) -> BindingMutationResult:
self.mutations.append((workspace_id, binding_id, disabled))
return self.mutation_result
@dataclass
class DatasetCatalog:
record: DatasetAccessRecord
def get_access_snapshot(
self,
*,
workspace_id: str,
dataset_id: str,
actor_id: str,
) -> DatasetAccessSnapshot | None:
assert actor_id == ACCOUNT_ID
if workspace_id != self.record.workspace_id or dataset_id != self.record.id:
return None
return DatasetAccessSnapshot(dataset=self.record, actor_has_partial_access=False)
def get_estimate_record(self, dataset_ref: DatasetRef) -> DatasetEstimateRecord | None:
if dataset_ref != DatasetRef(self.record.workspace_id, self.record.id):
return None
return DatasetEstimateRecord(
id=self.record.id,
workspace_id=self.record.workspace_id,
indexing_technique="economy",
)
def is_notion_dataset(self, dataset_ref: DatasetRef) -> bool:
return dataset_ref == DatasetRef(self.record.workspace_id, self.record.id)
class OwnerWorkspaceRoles:
def get_legacy_role(self, *, workspace_id: str, account_id: str) -> str | None:
assert workspace_id == TENANT_ID
assert account_id == ACCOUNT_ID
return "owner"
@dataclass
class KnowledgeDocuments:
bound_page_ids: frozenset[str] = frozenset()
active_refs: tuple[DocumentRef, ...] = ()
def list_bound_notion_page_ids(self, dataset_ref: DatasetRef) -> frozenset[str]:
assert dataset_ref == DatasetRef(TENANT_ID, DATASET_ID)
return self.bound_page_ids
def list_active_notion_refs(self, dataset_ref: DatasetRef) -> tuple[DocumentRef, ...]:
assert dataset_ref == DatasetRef(TENANT_ID, DATASET_ID)
return self.active_refs
def get_estimate_document(self, document_ref: DocumentRef) -> EstimateDocumentRecord | None:
if document_ref.document_id != DOCUMENT_ID:
return None
return EstimateDocumentRecord(
id=DOCUMENT_ID,
workspace_id=TENANT_ID,
dataset_id=DATASET_ID,
data_source_type="notion_import",
data_source_info=None,
indexing_status="waiting",
doc_form="text_model",
doc_language="English",
dataset_process_rule_id=None,
)
def list_estimate_documents_by_batch(
self, dataset_ref: DatasetRef, batch: str
) -> tuple[EstimateDocumentRecord, ...]:
document = self.get_estimate_document(dataset_ref.document(DOCUMENT_ID))
if document is None or batch != "batch":
return ()
return (document,)
def get_sync_document(self, document_ref: DocumentRef) -> SyncDocumentRecord | None:
if document_ref.document_id != DOCUMENT_ID:
return None
return SyncDocumentRecord(id=DOCUMENT_ID, data_source_type="notion_import")
@dataclass
class RecordingNotionSource:
workspaces: tuple[NotionWorkspace, ...]
list_calls: list[tuple[str, str, str]] = field(default_factory=list)
preview_calls: list[tuple[str, str, str, str, str]] = field(default_factory=list)
error: Exception | None = None
def list_authorized_pages(
self,
*,
workspace_id: str,
actor_id: str,
credential_id: str,
) -> tuple[NotionWorkspace, ...]:
self.list_calls.append((workspace_id, actor_id, credential_id))
if self.error is not None:
raise self.error
return self.workspaces
def preview_page(
self,
*,
workspace_id: str,
actor_id: str,
credential_id: str,
page_id: str,
page_type: str,
) -> str:
self.preview_calls.append((workspace_id, actor_id, credential_id, page_id, page_type))
if self.error is not None:
raise self.error
return "preview text"
@dataclass
class RecordingEstimateService:
calls: list[tuple[RequestContext, NewSourcesEstimateCommand]] = field(default_factory=list)
error: Exception | None = None
def estimate_new_sources(self, context: RequestContext, command: NewSourcesEstimateCommand) -> IndexingEstimate:
self.calls.append((context, command))
if self.error is not None:
raise self.error
return IndexingEstimate(total_segments=3, preview=[])
@dataclass
class RecordingDispatcher:
calls: list[tuple[str, str]] = field(default_factory=list)
def dispatch(self, *, dataset_id: str, document_id: str) -> None:
self.calls.append((dataset_id, document_id))
@dataclass(frozen=True)
class DataSourceApplicationServices:
bindings: DataSourceBindingApplicationService
notion_imports: NotionImportApplicationService
@dataclass(frozen=True)
class KnowledgeApplicationServices:
document_sync: DocumentSyncApplicationService
indexing_estimates: RecordingEstimateService
@dataclass(frozen=True)
class ApplicationServiceRegistry:
data_sources: DataSourceApplicationServices
knowledge: KnowledgeApplicationServices
@dataclass(frozen=True)
class ControllerCollaborators:
client: FlaskClient
account: AdmittedAccount
bindings: RecordingBindingStore
notion_source: RecordingNotionSource
estimates: RecordingEstimateService
dispatcher: RecordingDispatcher
@dataclass
class AdmittedAccount:
id: str
role: str
status: str
@pytest.fixture(scope="module")
def console_app() -> Flask:
"""One app with the console namespace, which takes a few hundred milliseconds to mount.
Each test registers its own ``extensions["application_services"]`` for the duration of that test.
"""
app = Flask(__name__)
app.config.update(TESTING=False, PROPAGATE_EXCEPTIONS=False)
api = ExternalApi(app)
api.error_handlers = console_api.error_handlers.copy()
api.add_namespace(console_ns, path="/console/api")
return app
class TestDataSourceCases:
@staticmethod
@pytest.fixture
def collaborators(
console_app: Flask,
config_overrides: Callable[..., None],
monkeypatch: pytest.MonkeyPatch,
) -> ControllerCollaborators:
config_overrides(
DEPLOYMENT_EDITION=DeploymentEdition.CLOUD,
LOGIN_DISABLED=True,
RBAC_ENABLED=False,
CONSOLE_API_URL="https://console-api.example.com",
)
account = AdmittedAccount(id=ACCOUNT_ID, role="owner", status="active")
account_with_tenant = AccountWithTenant(account=cast("Account", account), tenant_id=TENANT_ID)
monkeypatch.setattr(
"controllers.console.flask_admission.current_account_with_tenant",
lambda: account_with_tenant,
)
monkeypatch.setattr(
"controllers.console.wraps.current_account_with_tenant",
lambda: account_with_tenant,
)
dataset = DatasetAccessRecord(
id=DATASET_ID,
workspace_id=TENANT_ID,
maintainer_id=ACCOUNT_ID,
permission="only_me",
)
dataset_catalog = DatasetCatalog(dataset)
dataset_access = DatasetAccessService(
datasets=dataset_catalog,
workspace_roles=OwnerWorkspaceRoles(),
legacy_permissions_enabled=True,
)
documents = KnowledgeDocuments(
bound_page_ids=frozenset({PAGE_ID}),
active_refs=(DatasetRef(TENANT_ID, DATASET_ID).document(DOCUMENT_ID),),
)
bindings = RecordingBindingStore()
notion_source = RecordingNotionSource(
workspaces=(
NotionWorkspace(
workspace_id="notion-workspace-1",
workspace_name="One",
workspace_icon=None,
pages=(
AuthorizedNotionPage(
page_id=PAGE_ID,
page_name="Page one",
page_icon=NotionPageIcon(type="emoji", emoji="📄"),
parent_id=None,
page_type=NotionPageType.PAGE,
),
),
),
NotionWorkspace(
workspace_id="notion-workspace-2",
workspace_name="Two",
workspace_icon=None,
pages=(),
),
)
)
estimates = RecordingEstimateService()
dispatcher = RecordingDispatcher()
services = ApplicationServiceRegistry(
data_sources=DataSourceApplicationServices(
bindings=DataSourceBindingApplicationService(bindings=bindings),
notion_imports=NotionImportApplicationService(
dataset_access=dataset_access,
datasets=dataset_catalog,
documents=documents,
source=notion_source,
),
),
knowledge=KnowledgeApplicationServices(
document_sync=DocumentSyncApplicationService(
dataset_access=dataset_access,
documents=documents,
dispatcher=dispatcher,
),
indexing_estimates=estimates,
),
)
monkeypatch.setitem(console_app.extensions, "application_services", services)
return ControllerCollaborators(
client=console_app.test_client(),
account=account,
bindings=bindings,
notion_source=notion_source,
estimates=estimates,
dispatcher=dispatcher,
)
@staticmethod
def test_integration_collection_only_exposes_get(collaborators: ControllerCollaborators) -> None:
client = collaborators.client
assert client.get("/console/api/data-source/integrates").status_code == 200
assert client.patch("/console/api/data-source/integrates").status_code == 405
@staticmethod
@pytest.mark.parametrize(
("method", "path"),
[
("get", "/console/api/data-source/integrates"),
("patch", f"/console/api/data-source/integrates/{BINDING_ID}/enable"),
],
)
def test_integration_management_rejects_non_admin_role(
collaborators: ControllerCollaborators,
method: str,
path: str,
) -> None:
collaborators.account.role = "editor"
assert collaborators.client.open(path, method=method.upper()).status_code == 403
@staticmethod
def test_integration_management_enforces_workspace_credential_permission(
collaborators: ControllerCollaborators,
config_overrides: Callable[..., None],
monkeypatch: pytest.MonkeyPatch,
) -> None:
config_overrides(RBAC_ENABLED=True)
calls: list[dict[str, object]] = []
def record_access(tenant_id: str, account_id: str, **requirement: object) -> bool:
calls.append({"tenant_id": tenant_id, "account_id": account_id, **requirement})
return True
monkeypatch.setattr("controllers.common.rbac.checks.RBACService.CheckAccess.check", record_access)
list_response = collaborators.client.get("/console/api/data-source/integrates")
mutation_response = collaborators.client.patch(f"/console/api/data-source/integrates/{BINDING_ID}/enable")
assert list_response.status_code == 200
assert mutation_response.status_code == 200
assert calls == [
{
"tenant_id": TENANT_ID,
"account_id": ACCOUNT_ID,
"scene": RBACPermission.CREDENTIAL_MANAGE,
"resource_type": None,
"resource_id": None,
},
{
"tenant_id": TENANT_ID,
"account_id": ACCOUNT_ID,
"scene": RBACPermission.CREDENTIAL_MANAGE,
"resource_type": None,
"resource_id": None,
},
]
@staticmethod
@pytest.mark.parametrize(("action", "disabled"), [("enable", False), ("disable", True)])
def test_integration_item_only_exposes_patch_and_rejects_unknown_action(
collaborators: ControllerCollaborators,
action: str,
disabled: bool,
) -> None:
client = collaborators.client
item_path = f"/console/api/data-source/integrates/{BINDING_ID}/{action}"
assert client.get(item_path).status_code == 405
response = client.patch(item_path)
assert response.status_code == 200
assert response.get_json() == {"result": "success"}
assert collaborators.bindings.mutations == [(TENANT_ID, BINDING_ID, disabled)]
invalid = client.patch(f"/console/api/data-source/integrates/{BINDING_ID}/remove")
assert invalid.status_code == 400
assert invalid.get_json()["code"] == "invalid_action"
assert len(collaborators.bindings.mutations) == 1
@staticmethod
def test_integration_list_serializes_application_result_and_http_link(
collaborators: ControllerCollaborators,
) -> None:
collaborators.bindings.bindings = (
DataSourceBindingSummary(
id=BINDING_ID,
provider="notion",
created_at=datetime(2026, 5, 25, 1, 2, 3, tzinfo=UTC),
disabled=False,
source_info={
"workspace_name": "Workspace",
"workspace_id": "workspace-1",
"workspace_icon": None,
"pages": [],
"total": 0,
},
),
)
response = collaborators.client.get("/console/api/data-source/integrates")
assert response.status_code == 200
assert response.get_json()["data"][0] == {
"id": BINDING_ID,
"provider": "notion",
"created_at": 1779670923,
"is_bound": True,
"disabled": False,
"source_info": {
"workspace_name": "Workspace",
"workspace_id": "workspace-1",
"workspace_icon": None,
"pages": [],
"total": 0,
},
"link": "https://console-api.example.com/console/api/oauth/data-source/notion",
}
@staticmethod
def test_notion_list_preserves_workspace_groups_and_bound_state(collaborators: ControllerCollaborators) -> None:
response = collaborators.client.get(
"/console/api/notion/pre-import/pages",
query_string={"credential_id": "credential-1", "dataset_id": DATASET_ID},
)
assert response.status_code == 200
payload = response.get_json()["notion_info"]
assert [workspace["workspace_id"] for workspace in payload] == [
"notion-workspace-1",
"notion-workspace-2",
]
assert payload[0]["pages"][0]["is_bound"] is True
assert payload[0]["pages"][0]["type"] == "page"
assert "page_type" not in payload[0]["pages"][0]
assert collaborators.notion_source.list_calls == [(TENANT_ID, ACCOUNT_ID, "credential-1")]
@staticmethod
def test_notion_list_accepts_empty_dataset_id_for_new_dataset(collaborators: ControllerCollaborators) -> None:
response = collaborators.client.get(
"/console/api/notion/pre-import/pages",
query_string={"credential_id": "credential-1", "dataset_id": ""},
)
assert response.status_code == 200
assert response.get_json()["notion_info"][0]["pages"][0]["is_bound"] is False
assert collaborators.notion_source.list_calls == [(TENANT_ID, ACCOUNT_ID, "credential-1")]
@staticmethod
@pytest.mark.parametrize("page_type", ["page", "database"])
def test_notion_preview_validates_page_type_and_delegates(
collaborators: ControllerCollaborators, page_type: str
) -> None:
response = collaborators.client.get(
f"/console/api/notion/pages/{PAGE_ID}/{page_type}/preview",
query_string={"credential_id": "credential-1"},
)
invalid = collaborators.client.get(
f"/console/api/notion/pages/{PAGE_ID}/collection/preview",
query_string={"credential_id": "credential-1"},
)
assert response.status_code == 200
assert response.get_json() == {"content": "preview text"}
assert invalid.status_code == 400
assert invalid.get_json()["code"] == "invalid_param"
assert collaborators.notion_source.preview_calls == [
(TENANT_ID, ACCOUNT_ID, "credential-1", PAGE_ID, page_type)
]
@staticmethod
@pytest.mark.parametrize("path", ["/notion/pre-import/pages", f"/notion/pages/{PAGE_ID}/page/preview"])
def test_notion_refresh_unavailable_returns_retryable_response(
collaborators: ControllerCollaborators, path: str
) -> None:
error = CredentialRefreshUnavailableError()
collaborators.notion_source.error = error
response = collaborators.client.get(f"/console/api{path}", query_string={"credential_id": "credential-1"})
assert response.status_code == 503
assert response.get_json() == {
"code": "service_unavailable",
"message": str(error),
"status": 503,
}
@staticmethod
def test_notion_estimate_maps_nested_payload_to_shared_command(collaborators: ControllerCollaborators) -> None:
response = collaborators.client.post(
"/console/api/datasets/notion-indexing-estimate",
json={
"notion_info_list": [
{
"workspace_id": "notion-workspace-1",
"credential_id": "credential-1",
"pages": [{"page_id": PAGE_ID, "type": "page"}],
}
],
"process_rule": {"mode": "automatic"},
"doc_form": "text_model",
"doc_language": "English",
},
)
assert response.status_code == 200
assert response.get_json()["total_segments"] == 3
context, command = collaborators.estimates.calls[0]
assert context.active_workspace_id == TENANT_ID
assert context.account_id == ACCOUNT_ID
assert command.process_rule == {"mode": "automatic"}
source = command.sources[0]
assert isinstance(source, NotionEstimateSource)
assert source.credential_id == "credential-1"
assert source.page_id == PAGE_ID
@staticmethod
@pytest.mark.parametrize(
("error", "expected_http_error"),
[
(IndexingEstimateCredentialUnavailableError(), NotFoundError),
(EstimateSourceNotFoundError("page-1"), NotFoundError),
(IndexingEstimateProviderUnavailableError(), ProviderNotInitializeError),
(IndexingEstimateExecutionError(), IndexingEstimateError),
],
)
def test_notion_estimate_maps_application_errors(
error: Exception,
expected_http_error: type[Exception],
collaborators: ControllerCollaborators,
) -> None:
collaborators.estimates.error = error
api = data_source_controller.DataSourceNotionIndexingEstimateApi()
method = unwrap(api.post)
payload = data_source_controller.NotionEstimatePayload.model_validate(
{
"notion_info_list": [
{
"workspace_id": "notion-workspace",
"credential_id": "credential-1",
"pages": [{"page_id": "page-1", "type": "page"}],
}
],
"process_rule": {"mode": "automatic"},
}
)
context = RequestContext("request-1", None, "account-1", "workspace-1")
with collaborators.client.application.app_context(), pytest.raises(expected_http_error):
method(api, payload, context)
@staticmethod
def test_sync_routes_dispatch_only_application_selected_documents(collaborators: ControllerCollaborators) -> None:
dataset_response = collaborators.client.get(f"/console/api/datasets/{DATASET_ID}/notion/sync")
document_response = collaborators.client.get(
f"/console/api/datasets/{DATASET_ID}/documents/{DOCUMENT_ID}/notion/sync"
)
assert dataset_response.status_code == 200
assert dataset_response.get_json() == {"result": "success"}
assert document_response.status_code == 200
assert document_response.get_json() == {"result": "success"}
assert collaborators.dispatcher.calls == [(DATASET_ID, DOCUMENT_ID), (DATASET_ID, DOCUMENT_ID)]
@staticmethod
def test_sync_route_does_not_dispatch_cross_workspace_dataset(collaborators: ControllerCollaborators) -> None:
response = collaborators.client.get(f"/console/api/datasets/{FOREIGN_DATASET_ID}/notion/sync")
assert response.status_code == 404
assert response.get_json()["code"] == "not_found"
assert collaborators.dispatcher.calls == []
@staticmethod
@pytest.mark.parametrize(
"resource",
[data_source_controller.DataSourceNotionDatasetSyncApi, data_source_controller.DataSourceNotionDocumentSyncApi],
)
def test_notion_sync_requires_dataset_edit_permission(resource):
[check] = rbac_checks(resource.get)
assert check.scene is RBACPermission.DATASET_EDIT
assert isinstance(check.locator, DatasetId)
@@ -0,0 +1,597 @@
import datetime
from dataclasses import dataclass
from inspect import unwrap
from types import SimpleNamespace
from unittest.mock import MagicMock, create_autospec, patch
from uuid import UUID
import pytest
from werkzeug.exceptions import BadRequest, Conflict, Forbidden, NotFound
import services
from controllers.common.errors import InvalidArgumentError, NotFoundError
from controllers.common.rbac import DatasetId, RBACPermission, Workspace
from controllers.console.app.error import ProviderNotInitializeError
from controllers.console.datasets import datasets as controller
from controllers.console.datasets.datasets import (
DatasetApi,
DatasetApiBaseUrlApi,
DatasetApiKeyApi,
DatasetAutoDisableLogApi,
DatasetCreatePayload,
DatasetEnableApiApi,
DatasetErrorDocs,
DatasetIndexingEstimateApi,
DatasetIndexingStatusApi,
DatasetKnowledgeFSUpgradeApi,
DatasetKnowledgeFSUpgradeJobApi,
DatasetKnowledgeFSUpgradeJobsApi,
DatasetListApi,
DatasetPermissionUserListApi,
DatasetQueryApi,
DatasetRelatedAppListApi,
DatasetRetrievalSettingApi,
DatasetRetrievalSettingMockApi,
DatasetUpdatePayload,
DatasetUseCheckApi,
IndexingEstimatePayload,
_new_estimate_sources,
)
from controllers.console.datasets.error import (
DatasetAccessDeniedRequestError,
DatasetNameDuplicateError,
IndexingEstimateError,
KnowledgeFSUpgradeModelsUnavailableHTTPError,
)
from machinery.context import RequestContext
from models.knowledge_fs import KnowledgeFSUpgradeJob, KnowledgeFSUpgradeJobStatus, KnowledgeFSUpgradeStage
from services.data_source.entities.notion_import import NotionPageType
from services.dataset_knowledge_fs_upgrade_service import (
KnowledgeFSUpgradeConflictError,
KnowledgeFSUpgradeModelsUnavailableError,
KnowledgeFSUpgradeNotFoundError,
)
from services.knowledge.dataset_access import DatasetAccessDeniedError, DatasetNotFoundError
from services.knowledge.datasets.application import DatasetApplicationService, DatasetListFilter
from services.knowledge.datasets.upgrades import (
KnowledgeFSUpgradeUnavailableError,
KnowledgeUpgradeApplicationService,
)
from services.knowledge.entities.indexing_estimate import (
NotionEstimateSource,
UploadFileEstimateSource,
WebsiteEstimateSource,
)
from services.knowledge.indexing.estimate import (
EstimateSourceNotFoundError,
IndexingEstimateCredentialUnavailableError,
IndexingEstimateExecutionError,
IndexingEstimateProviderUnavailableError,
UnsupportedEstimateSourceError,
)
from services.knowledge_fs.product_dto import (
KnowledgeFSUpgradeModelIssueResponse,
KnowledgeFSUpgradeModelsPayload,
KnowledgeFSUpgradePreparationResponse,
)
from tests.helpers.unit.controllers.rbac_introspection import rbac_checks
CONTEXT = RequestContext("request-1", None, "account-1", "tenant-1")
DATASET_ID = UUID(int=1)
def test_dataset_delete_requires_dataset_delete_permission() -> None:
[check] = rbac_checks(DatasetApi.delete)
assert check.scene is RBACPermission.DATASET_DELETE
assert isinstance(check.locator, DatasetId)
@pytest.fixture
def datasets(monkeypatch):
service = create_autospec(DatasetApplicationService, instance=True, spec_set=True)
monkeypatch.setattr(
controller, "application_services", lambda: SimpleNamespace(knowledge=SimpleNamespace(datasets=service))
)
return service
@pytest.mark.parametrize(
("resource", "operation"),
[
(DatasetApi, lambda service: service.get_dataset),
(DatasetUseCheckApi, lambda service: service.is_in_use),
(DatasetQueryApi, lambda service: service.queries),
(DatasetRelatedAppListApi, lambda service: service.related_apps),
(DatasetIndexingStatusApi, lambda service: service.indexing_status),
(DatasetErrorDocs, lambda service: service.error_documents),
(DatasetPermissionUserListApi, lambda service: service.partial_members),
(DatasetAutoDisableLogApi, lambda service: service.auto_disable_logs),
],
)
@pytest.mark.parametrize(
("error", "http_error"), [(DatasetNotFoundError(), NotFound), (DatasetAccessDeniedError(), Forbidden)]
)
def test_scoped_reads_pass_context_and_map_access_errors(app, datasets, resource, operation, error, http_error):
method = operation(datasets)
method.side_effect = error
with app.test_request_context("/"), pytest.raises(http_error):
unwrap(resource.get)(resource(), CONTEXT, DATASET_ID)
assert method.call_args.args == (CONTEXT,)
assert method.call_args.kwargs["dataset_id"] == str(DATASET_ID)
def test_list_parses_repeated_filters_and_serializes_page(app, datasets):
datasets.list_datasets.return_value = {"data": [], "page": 2, "limit": 3, "total": 4, "has_more": False}
with app.test_request_context(
"/?page=2&limit=3&ids=a&ids=b&tag_ids=x&tag_ids=y&creator_ids=c&include_all=true&keyword=term"
):
result, status = unwrap(DatasetListApi.get)(DatasetListApi(), CONTEXT)
datasets.list_datasets.assert_called_once_with(
CONTEXT,
DatasetListFilter(
page=2,
limit=3,
ids=["a", "b"],
tag_ids=["x", "y"],
creator_ids=["c"],
include_all=True,
keyword="term",
),
)
assert status == 200
assert result == datasets.list_datasets.return_value
def test_patch_does_not_turn_omitted_fields_into_updates(datasets):
datasets.update_dataset.side_effect = DatasetNotFoundError()
with pytest.raises(NotFound):
unwrap(DatasetApi.patch)(DatasetApi(), DatasetUpdatePayload(name="Changed"), CONTEXT, DATASET_ID)
datasets.update_dataset.assert_called_once_with(CONTEXT, dataset_id=str(DATASET_ID), values={"name": "Changed"})
def test_duplicate_name_is_mapped_at_transport_on_create(datasets):
datasets.create_dataset.side_effect = services.errors.dataset.DatasetNameDuplicateError()
with pytest.raises(DatasetNameDuplicateError):
unwrap(DatasetListApi.post)(DatasetListApi(), DatasetCreatePayload(name="Duplicate"), CONTEXT)
def test_delete_success_returns_empty_204(datasets):
assert unwrap(DatasetApi.delete)(DatasetApi(), CONTEXT, DATASET_ID) == ("", 204)
datasets.delete_dataset.assert_called_once_with(CONTEXT, dataset_id=str(DATASET_ID))
def test_request_base_url_and_explicit_status_are_forwarded(app, datasets):
datasets.api_base_url.return_value = "https://api.example/v1"
with app.test_request_context("/", base_url="https://console.example/"):
assert unwrap(DatasetApiBaseUrlApi.get)(DatasetApiBaseUrlApi(), CONTEXT) == {
"api_base_url": "https://api.example/v1"
}
datasets.api_base_url.assert_called_once_with(CONTEXT, request_base_url="https://console.example")
assert unwrap(DatasetEnableApiApi.post)(DatasetEnableApiApi(), CONTEXT, DATASET_ID, "disable") == (
{"result": "success"},
200,
)
datasets.set_api_enabled.assert_called_once_with(CONTEXT, dataset_id=str(DATASET_ID), status="disable")
def test_retrieval_settings_forward_mock_flag(datasets):
datasets.retrieval_settings.return_value = {"retrieval_method": ["semantic_search"]}
assert (
unwrap(DatasetRetrievalSettingApi.get)(DatasetRetrievalSettingApi(), CONTEXT)
== datasets.retrieval_settings.return_value
)
unwrap(DatasetRetrievalSettingMockApi.get)(DatasetRetrievalSettingMockApi(), CONTEXT, "milvus")
datasets.retrieval_settings.assert_called_with(CONTEXT, vector_type="milvus", is_mock=True)
def test_new_estimate_sources_maps_each_supported_transport_shape() -> None:
upload_sources = _new_estimate_sources(
{"data_source_type": "upload_file", "file_info_list": {"file_ids": ["file-1", "file-2"]}}
)
notion_sources = _new_estimate_sources(
{
"data_source_type": "notion_import",
"notion_info_list": [
{
"workspace_id": "notion-workspace",
"credential_id": "credential-1",
"pages": [{"page_id": "page-1", "type": "page"}],
}
],
}
)
website_sources = _new_estimate_sources(
{
"data_source_type": "website_crawl",
"website_info_list": {
"provider": "firecrawl",
"job_id": "job-1",
"urls": ["https://example.com/a", "https://example.com/b"],
"only_main_content": True,
},
}
)
assert upload_sources == (UploadFileEstimateSource("file-1"), UploadFileEstimateSource("file-2"))
assert notion_sources == (NotionEstimateSource("notion-workspace", "page-1", NotionPageType.PAGE, "credential-1"),)
assert website_sources == (
WebsiteEstimateSource("firecrawl", "job-1", "https://example.com/a", only_main_content=True),
WebsiteEstimateSource("firecrawl", "job-1", "https://example.com/b", only_main_content=True),
)
@pytest.mark.parametrize(("value", "expected"), [("false", False), ("true", True), (False, False), (True, True)])
def test_website_estimate_parses_boolean_values(value: str | bool, expected: bool) -> None:
sources = _new_estimate_sources(
{
"data_source_type": "website_crawl",
"website_info_list": {
"provider": "firecrawl",
"job_id": "job-1",
"urls": ["https://example.com"],
"only_main_content": value,
},
}
)
assert sources == (WebsiteEstimateSource("firecrawl", "job-1", "https://example.com", only_main_content=expected),)
@pytest.mark.parametrize("values", [{"only_main_content": "invalid"}, {"urls": [42]}])
def test_website_estimate_rejects_invalid_field_types(values: dict[str, object]) -> None:
with pytest.raises(
ValueError,
match=(
"(?:1 validation error for WebsiteInfo\\\nonly_main_content|1 validation error for WebsiteInfo\\\nurls\\.0)"
),
):
_new_estimate_sources(
{
"data_source_type": "website_crawl",
"website_info_list": {
"provider": "firecrawl",
"job_id": "job-1",
"urls": ["https://example.com"],
**values,
},
}
)
def test_new_estimate_sources_deduplicates_upload_ids_without_reordering() -> None:
sources = _new_estimate_sources(
{"data_source_type": "upload_file", "file_info_list": {"file_ids": ["file-2", "file-1", "file-2"]}}
)
assert sources == (UploadFileEstimateSource("file-2"), UploadFileEstimateSource("file-1"))
@pytest.mark.parametrize(
"info_list",
[
{"data_source_type": "upload_file", "file_info_list": {}},
{"data_source_type": "notion_import", "notion_info_list": [{"workspace_id": "workspace"}]},
{
"data_source_type": "notion_import",
"notion_info_list": [
{
"workspace_id": "workspace",
"credential_id": "credential",
"pages": [{"page_id": "page", "type": "unknown"}],
}
],
},
{"data_source_type": "unsupported"},
],
)
def test_new_estimate_sources_rejects_malformed_transport_shapes(info_list: dict[str, object]) -> None:
with pytest.raises(
ValueError,
match=(
"(?:1 validation error for FileInfo\\\nfile_ids|1 validation error for "
"list\\[NotionEstimateWorkspacePayload\\]\\\n0\\.pages\\.0\\.type|2 "
"validation errors for list\\[NotionEstimateWorkspacePayload\\]\\\n0\\."
"credential_id|Data source type not support)"
),
):
_new_estimate_sources(info_list)
@pytest.mark.parametrize(
("error", "expected_http_error"),
[
(IndexingEstimateCredentialUnavailableError(), NotFoundError),
(EstimateSourceNotFoundError("source-1"), NotFoundError),
(DatasetNotFoundError(), NotFoundError),
(DatasetAccessDeniedError(), DatasetAccessDeniedRequestError),
(UnsupportedEstimateSourceError("unsupported"), InvalidArgumentError),
(IndexingEstimateProviderUnavailableError(), ProviderNotInitializeError),
(IndexingEstimateExecutionError(), IndexingEstimateError),
],
)
def test_new_source_estimate_maps_application_errors(
error: Exception,
expected_http_error: type[Exception],
) -> None:
estimates = MagicMock()
estimates.estimate_new_sources.side_effect = error
registry = SimpleNamespace(knowledge=SimpleNamespace(indexing_estimates=estimates))
api = DatasetIndexingEstimateApi()
method = unwrap(api.post)
payload = IndexingEstimatePayload(
info_list={"data_source_type": "upload_file", "file_info_list": {"file_ids": ["file-1"]}},
process_rule={"mode": "automatic"},
indexing_technique="economy",
)
context = RequestContext("request-1", None, "account-1", "workspace-1")
with patch("controllers.console.datasets.datasets.application_services", return_value=registry):
with pytest.raises(expected_http_error):
method(api, payload, context)
@pytest.mark.parametrize(
("dataset_id", "scene", "locator_type"),
[
("dataset-1", RBACPermission.DATASET_USE, DatasetId),
(None, RBACPermission.DATASET_CREATE_AND_MANAGEMENT, Workspace),
],
)
def test_new_source_estimate_authorizes_before_execution(dataset_id, scene, locator_type):
estimates = MagicMock()
registry = SimpleNamespace(knowledge=SimpleNamespace(indexing_estimates=estimates))
payload = IndexingEstimatePayload(
info_list={"data_source_type": "upload_file", "file_info_list": {"file_ids": ["file-1"]}},
process_rule={"mode": "automatic"},
indexing_technique="economy",
dataset_id=dataset_id,
)
with (
patch.object(controller, "application_services", return_value=registry),
patch.object(controller, "enforce_rbac_checks", side_effect=Forbidden) as enforce_checks,
pytest.raises(Forbidden),
):
unwrap(DatasetIndexingEstimateApi.post)(DatasetIndexingEstimateApi(), payload, CONTEXT)
enforce_checks.assert_called_once()
kwargs = enforce_checks.call_args.kwargs
assert kwargs["tenant_id"] == CONTEXT.active_workspace_id
assert kwargs["account_id"] == CONTEXT.account_id
assert kwargs["path_args"] == ({"dataset_id": dataset_id} if dataset_id else None)
[check] = kwargs["checks"]
assert check.scene is scene
assert isinstance(check.locator, locator_type)
estimates.estimate_new_sources.assert_not_called()
def _upgrade_job(job_id: str = "upgrade-job-1") -> KnowledgeFSUpgradeJob:
job = KnowledgeFSUpgradeJob(
tenant_id="tenant-1",
old_dataset_id=str(DATASET_ID),
requested_by_account_id="account-1",
owner_account_id="account-1",
idempotency_key="upgrade-request-1",
snapshot_at=datetime.datetime(2026, 8, 17, tzinfo=datetime.UTC),
config_snapshot={},
permission_snapshot={},
app_binding_snapshot=[],
tag_ids_snapshot=[],
status=KnowledgeFSUpgradeJobStatus.QUEUED,
stage=KnowledgeFSUpgradeStage.VALIDATING,
total_documents=1,
)
job.id = job_id
return job
@dataclass(frozen=True)
class _UpgradeKnowledgeServices:
upgrades: KnowledgeUpgradeApplicationService
@dataclass(frozen=True)
class _UpgradeApplicationServices:
knowledge: _UpgradeKnowledgeServices
@pytest.fixture
def upgrades(monkeypatch):
service = create_autospec(KnowledgeUpgradeApplicationService, instance=True, spec_set=True)
services = _UpgradeApplicationServices(knowledge=_UpgradeKnowledgeServices(upgrades=service))
monkeypatch.setattr(controller, "application_services", lambda: services)
return service
def test_upgrade_writes_require_dataset_edit_permission() -> None:
for method in (DatasetKnowledgeFSUpgradeApi.post, DatasetKnowledgeFSUpgradeJobApi.post):
[check] = rbac_checks(method)
assert check.scene is RBACPermission.DATASET_EDIT
assert isinstance(check.locator, DatasetId)
[check] = rbac_checks(DatasetKnowledgeFSUpgradeJobsApi.get)
assert check.scene is RBACPermission.DATASET_READONLY
assert isinstance(check.locator, Workspace)
def test_upgrade_jobs_serializes_recoverable_jobs(app, upgrades):
upgrades.list_recoverable_jobs.return_value = [_upgrade_job(job_id="job-1")]
with app.test_request_context("/datasets/knowledge-fs-upgrade-jobs"):
response = unwrap(DatasetKnowledgeFSUpgradeJobsApi.get)(DatasetKnowledgeFSUpgradeJobsApi(), CONTEXT)
upgrades.list_recoverable_jobs.assert_called_once_with(CONTEXT)
assert [job["id"] for job in response["data"]] == ["job-1"]
def test_upgrade_start_forwards_idempotency_key(app, upgrades):
upgrades.start.return_value = _upgrade_job()
with app.test_request_context(
f"/datasets/{DATASET_ID}/knowledge-fs-upgrades", headers={"Idempotency-Key": "upgrade-request-1"}
):
response, status = unwrap(DatasetKnowledgeFSUpgradeApi.post)(
DatasetKnowledgeFSUpgradeApi(), CONTEXT, DATASET_ID
)
assert status == 202
assert response["id"] == "upgrade-job-1"
upgrades.start.assert_called_once_with(
CONTEXT, dataset_id=str(DATASET_ID), idempotency_key="upgrade-request-1", models=None
)
def test_upgrade_start_forwards_all_model_choices(app, upgrades):
upgrades.start.return_value = _upgrade_job()
choices = {
"embedding": {"provider": "langgenius/openai/openai", "model": "embedding"},
"rerank": {"provider": "langgenius/cohere/cohere", "model": "rerank"},
"reasoning": {"provider": "langgenius/anthropic/anthropic", "model": "reasoning"},
}
with app.test_request_context(
f"/datasets/{DATASET_ID}/knowledge-fs-upgrades",
method="POST",
headers={"Idempotency-Key": "upgrade-request-1"},
json={"models": choices},
):
response, status = unwrap(DatasetKnowledgeFSUpgradeApi.post)(
DatasetKnowledgeFSUpgradeApi(), CONTEXT, DATASET_ID
)
assert status == 202
assert response["id"] == "upgrade-job-1"
upgrades.start.assert_called_once_with(
CONTEXT,
dataset_id=str(DATASET_ID),
idempotency_key="upgrade-request-1",
models=KnowledgeFSUpgradeModelsPayload.model_validate(choices),
)
def test_upgrade_start_rejects_incomplete_model_selection(app, upgrades):
with (
app.test_request_context(
f"/datasets/{DATASET_ID}/knowledge-fs-upgrades",
method="POST",
headers={"Idempotency-Key": "upgrade-request-1"},
json={"models": {"embedding": {"provider": "openai", "model": "embedding"}}},
),
pytest.raises(BadRequest, match="Invalid upgrade model selection"),
):
unwrap(DatasetKnowledgeFSUpgradeApi.post)(DatasetKnowledgeFSUpgradeApi(), CONTEXT, DATASET_ID)
upgrades.start.assert_not_called()
def test_upgrade_start_returns_field_specific_model_errors(app, upgrades):
upgrades.start.side_effect = KnowledgeFSUpgradeModelsUnavailableError(
[
KnowledgeFSUpgradeModelIssueResponse(
field="embedding", reason="credential-removed", provider="openai", model="embedding"
)
]
)
with (
app.test_request_context(
f"/datasets/{DATASET_ID}/knowledge-fs-upgrades",
method="POST",
headers={"Idempotency-Key": "upgrade-request-1"},
),
pytest.raises(KnowledgeFSUpgradeModelsUnavailableHTTPError) as error,
):
unwrap(DatasetKnowledgeFSUpgradeApi.post)(DatasetKnowledgeFSUpgradeApi(), CONTEXT, DATASET_ID)
assert error.value.data is not None
assert error.value.data["details"]["issues"] == [
{"field": "embedding", "reason": "credential-removed", "provider": "openai", "model": "embedding"}
]
def test_upgrade_start_requires_idempotency_key(app, upgrades):
with (
app.test_request_context(f"/datasets/{DATASET_ID}/knowledge-fs-upgrades"),
pytest.raises(BadRequest, match="Idempotency-Key"),
):
unwrap(DatasetKnowledgeFSUpgradeApi.post)(DatasetKnowledgeFSUpgradeApi(), CONTEXT, DATASET_ID)
upgrades.start.assert_not_called()
@pytest.mark.parametrize(
("error", "http_error"),
[
(KnowledgeFSUpgradeUnavailableError(), NotFound),
(DatasetNotFoundError(), NotFound),
(DatasetAccessDeniedError(), Forbidden),
(KnowledgeFSUpgradeConflictError("already running"), Conflict),
],
)
def test_upgrade_start_maps_domain_errors(app, upgrades, error, http_error):
upgrades.start.side_effect = error
with (
app.test_request_context(
f"/datasets/{DATASET_ID}/knowledge-fs-upgrades", headers={"Idempotency-Key": "upgrade-request-1"}
),
pytest.raises(http_error),
):
unwrap(DatasetKnowledgeFSUpgradeApi.post)(DatasetKnowledgeFSUpgradeApi(), CONTEXT, DATASET_ID)
def test_upgrade_discovery_serializes_service_result(app, upgrades):
upgrades.discovery.return_value = KnowledgeFSUpgradePreparationResponse.model_validate(
{
"job": None,
"can_upgrade": True,
"can_retry": False,
"models": {
"embedding": {
"value": {"provider": "openai", "model": "embedding"},
"source": "legacy",
"available": True,
},
"rerank": {
"value": {"provider": "cohere", "model": "rerank"},
"source": "workspace_default",
"available": True,
},
"reasoning": {
"value": {"provider": "anthropic", "model": "reasoning"},
"source": "workspace_default",
"available": False,
"unavailable_reason": "credential-removed",
},
},
}
)
with app.test_request_context(f"/datasets/{DATASET_ID}/knowledge-fs-upgrades"):
response = unwrap(DatasetKnowledgeFSUpgradeApi.get)(DatasetKnowledgeFSUpgradeApi(), CONTEXT, DATASET_ID)
upgrades.discovery.assert_called_once_with(CONTEXT, dataset_id=str(DATASET_ID))
assert response["can_upgrade"] is True
assert response["models"]["embedding"]["value"] == {"provider": "openai", "model": "embedding"}
assert response["models"]["reasoning"]["unavailable_reason"] == "credential-removed"
def test_upgrade_job_status_maps_missing_job_to_not_found(app, upgrades):
upgrades.get_job.side_effect = KnowledgeFSUpgradeNotFoundError("Upgrade job was not found")
with (
app.test_request_context(f"/datasets/{DATASET_ID}/knowledge-fs-upgrades/job-1"),
pytest.raises(NotFound, match="Upgrade job was not found"),
):
unwrap(DatasetKnowledgeFSUpgradeJobApi.get)(DatasetKnowledgeFSUpgradeJobApi(), CONTEXT, DATASET_ID, "job-1")
def test_upgrade_retry_returns_queued(app, upgrades):
upgrades.retry.return_value = _upgrade_job(job_id="job-1")
with app.test_request_context(f"/datasets/{DATASET_ID}/knowledge-fs-upgrades/job-1"):
response, status = unwrap(DatasetKnowledgeFSUpgradeJobApi.post)(
DatasetKnowledgeFSUpgradeJobApi(), CONTEXT, DATASET_ID, "job-1"
)
assert status == 202
assert response == {"id": "job-1", "status": "queued"}
def test_upgrade_retry_maps_disabled_feature_to_not_found(app, upgrades):
upgrades.retry.side_effect = KnowledgeFSUpgradeUnavailableError()
with (
app.test_request_context(f"/datasets/{DATASET_ID}/knowledge-fs-upgrades/job-1"),
pytest.raises(NotFound),
):
unwrap(DatasetKnowledgeFSUpgradeJobApi.post)(DatasetKnowledgeFSUpgradeJobApi(), CONTEXT, DATASET_ID, "job-1")
def test_api_key_create_rejects_non_string_dataset_ids_with_the_standard_error(app):
# A raised BadRequest gets the standard error body; ``Namespace.abort(message=...)`` replaced it.
with app.test_request_context("/", method="POST", json={"dataset_ids": [123]}), pytest.raises(BadRequest) as error:
unwrap(DatasetApiKeyApi.post)(DatasetApiKeyApi(), CONTEXT)
assert error.value.description == "dataset_ids must be a list of strings."
@@ -0,0 +1,784 @@
import datetime as _datasets_document_datetime
import json
from datetime import datetime as _datasets_document_download_datetime
from inspect import unwrap
from io import BytesIO
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import MagicMock, create_autospec, patch
from uuid import UUID
from zipfile import ZipFile
import pytest
from flask import send_file
from sqlalchemy import event
from werkzeug.exceptions import Forbidden, NotFound
from controllers.common.errors import InvalidArgumentError, NotFoundError
from controllers.common.rbac import DatasetByDocument, DatasetId, RBACPermission
from controllers.console.app.error import ProviderNotInitializeError
from controllers.console.datasets import datasets_document as controller
from controllers.console.datasets.datasets_document import (
DatasetDocumentListApi,
DatasetInitApi,
DocumentApi,
DocumentBatchDownloadZipApi,
DocumentBatchIndexingEstimateApi,
DocumentBatchIndexingStatusApi,
DocumentDownloadApi,
DocumentGenerateSummaryApi,
DocumentIndexingEstimateApi,
DocumentIndexingStatusApi,
DocumentMetadataApi,
DocumentMetadataUpdatePayload,
DocumentPauseApi,
DocumentPipelineExecutionLogApi,
DocumentProcessingApi,
DocumentRecoverApi,
DocumentRenameApi,
DocumentRenamePayload,
DocumentRetryApi,
DocumentRetryPayload,
DocumentStatusApi,
DocumentSummaryStatusApi,
GenerateSummaryPayload,
GetProcessRuleApi,
WebsiteDocumentSyncApi,
)
from controllers.console.datasets.error import (
ArchivedDocumentImmutableError,
DatasetAccessDeniedRequestError,
DocumentAlreadyFinishedError,
DocumentIndexingError,
IndexingEstimateError,
InvalidActionError,
InvalidMetadataError,
)
from machinery.context import RequestContext
from models.dataset import Dataset, Document
from models.model import UploadFile
from services.knowledge.dataset_access import (
AccessibleDataset,
DatasetAccess,
DatasetAccessDeniedError,
DatasetNotFoundError,
)
from services.knowledge.documents.adapters import SQLAlchemyDocumentOperations
from services.knowledge.documents.application import (
DatasetDocumentApplicationService,
DocumentArchivedError,
DocumentIndexingStateError,
DocumentInvalidActionError,
DocumentListFilter,
DocumentNotFoundError,
DocumentProviderError,
)
from services.knowledge.indexing.estimate import (
EstimateDocumentAlreadyFinishedError,
EstimateDocumentNotFoundError,
EstimateSourceNotFoundError,
IndexingEstimateCredentialUnavailableError,
IndexingEstimateExecutionError,
IndexingEstimateProviderUnavailableError,
UnsupportedEstimateSourceError,
)
from tests.helpers.unit.controllers.rbac_introspection import rbac_checks
_DATASETS_DOCUMENT_CONTEXT = RequestContext("request-1", None, "account-1", "tenant-1")
DS = UUID(int=1)
DOC = UUID(int=2)
class TestDatasetsDocumentCases:
@staticmethod
@pytest.fixture
def documents(monkeypatch):
documents = create_autospec(DatasetDocumentApplicationService, instance=True, spec_set=True)
monkeypatch.setattr(
controller, "application_services", lambda: SimpleNamespace(knowledge=SimpleNamespace(documents=documents))
)
monkeypatch.setattr(controller, "check_knowledge_rate_limit", lambda: None)
return documents
@staticmethod
def test_process_rule_authorizes_document_dataset_before_read(app, documents):
with (
app.test_request_context(f"/?document_id={DOC}"),
patch.object(controller, "enforce_rbac_checks", side_effect=Forbidden) as enforce_checks,
pytest.raises(Forbidden),
):
unwrap(GetProcessRuleApi.get)(GetProcessRuleApi(), _DATASETS_DOCUMENT_CONTEXT)
enforce_checks.assert_called_once()
kwargs = enforce_checks.call_args.kwargs
assert kwargs["tenant_id"] == _DATASETS_DOCUMENT_CONTEXT.active_workspace_id
assert kwargs["account_id"] == _DATASETS_DOCUMENT_CONTEXT.account_id
assert kwargs["path_args"] == {"document_id": str(DOC)}
[check] = kwargs["checks"]
assert check.scene is RBACPermission.DATASET_READONLY
assert isinstance(check.locator, DatasetByDocument)
documents.get_process_rule.assert_not_called()
@staticmethod
def test_default_process_rule_does_not_require_dataset_permission(app, documents):
documents.get_process_rule.return_value = {
"mode": "automatic",
"rules": {},
"limits": {"indexing_max_segmentation_tokens_length": 4000},
}
with app.test_request_context("/"), patch.object(controller, "enforce_rbac_checks") as enforce_checks:
unwrap(GetProcessRuleApi.get)(GetProcessRuleApi(), _DATASETS_DOCUMENT_CONTEXT)
enforce_checks.assert_not_called()
documents.get_process_rule.assert_called_once_with(_DATASETS_DOCUMENT_CONTEXT, document_id=None)
@staticmethod
@pytest.mark.parametrize(
("resource", "verb", "operation", "kwargs"),
[
(GetProcessRuleApi, GetProcessRuleApi.get, lambda service: service.get_process_rule, {}),
(
DatasetDocumentListApi,
DatasetDocumentListApi.get,
lambda service: service.list_documents,
{"dataset_id": DS},
),
(
DatasetDocumentListApi,
DatasetDocumentListApi.post,
lambda service: service.create_documents,
{"dataset_id": DS},
),
(
DatasetDocumentListApi,
DatasetDocumentListApi.delete,
lambda service: service.delete_documents,
{"dataset_id": DS},
),
(DatasetInitApi, DatasetInitApi.post, lambda service: service.initialize_dataset, {}),
(
DocumentBatchIndexingStatusApi,
DocumentBatchIndexingStatusApi.get,
lambda service: service.get_batch_indexing_status,
{"dataset_id": DS, "batch": "batch-1"},
),
(
DocumentIndexingStatusApi,
DocumentIndexingStatusApi.get,
lambda service: service.get_indexing_status,
{"dataset_id": DS, "document_id": DOC},
),
(
DocumentApi,
DocumentApi.get,
lambda service: service.get_document,
{"dataset_id": DS, "document_id": DOC},
),
(
DocumentApi,
DocumentApi.delete,
lambda service: service.delete_document,
{"dataset_id": DS, "document_id": DOC},
),
(
DocumentDownloadApi,
DocumentDownloadApi.get,
lambda service: service.get_download_url,
{"dataset_id": DS, "document_id": DOC},
),
(
DocumentBatchDownloadZipApi,
DocumentBatchDownloadZipApi.post,
lambda service: service.build_download_zip,
{"dataset_id": DS},
),
(
DocumentProcessingApi,
DocumentProcessingApi.patch,
lambda service: service.update_processing,
{"dataset_id": DS, "document_id": DOC, "action": "pause"},
),
(
DocumentMetadataApi,
DocumentMetadataApi.put,
lambda service: service.update_metadata,
{
"dataset_id": DS,
"document_id": DOC,
"req_data": DocumentMetadataUpdatePayload(doc_type="book", doc_metadata={"title": "Book"}),
},
),
(
DocumentStatusApi,
DocumentStatusApi.patch,
lambda service: service.change_status,
{"dataset_id": DS, "action": "enable"},
),
(
DocumentPauseApi,
DocumentPauseApi.patch,
lambda service: service.pause_document,
{"dataset_id": DS, "document_id": DOC},
),
(
DocumentRecoverApi,
DocumentRecoverApi.patch,
lambda service: service.recover_document,
{"dataset_id": DS, "document_id": DOC},
),
(
DocumentRetryApi,
DocumentRetryApi.post,
lambda service: service.retry_documents,
{"dataset_id": DS, "req_data": DocumentRetryPayload(document_ids=[str(DOC)])},
),
(
DocumentRenameApi,
DocumentRenameApi.post,
lambda service: service.rename_document,
{"dataset_id": DS, "document_id": DOC, "req_data": DocumentRenamePayload(name="New name")},
),
(
WebsiteDocumentSyncApi,
WebsiteDocumentSyncApi.get,
lambda service: service.sync_website,
{"dataset_id": DS, "document_id": DOC},
),
(
DocumentPipelineExecutionLogApi,
DocumentPipelineExecutionLogApi.get,
lambda service: service.get_execution_log,
{"dataset_id": DS, "document_id": DOC},
),
(
DocumentGenerateSummaryApi,
DocumentGenerateSummaryApi.post,
lambda service: service.generate_summary,
{"dataset_id": DS, "req_data": GenerateSummaryPayload(document_list=[str(DOC)])},
),
(
DocumentSummaryStatusApi,
DocumentSummaryStatusApi.get,
lambda service: service.get_summary_status,
{"dataset_id": DS, "document_id": DOC},
),
],
)
def test_endpoints_pass_request_context_and_map_application_errors(
app, documents, resource, verb, operation, kwargs
):
operation(documents).side_effect = DocumentNotFoundError("Document not found.")
with app.test_request_context(
"/?document_id=" + str(DOC), json={"indexing_technique": "economy", "document_ids": [str(DOC)]}
):
with pytest.raises(NotFound):
unwrap(verb)(resource(), request_context=_DATASETS_DOCUMENT_CONTEXT, **kwargs)
called = operation(documents)
assert called.call_count == 1
assert called.call_args.args == (_DATASETS_DOCUMENT_CONTEXT,)
if "dataset_id" in kwargs:
assert called.call_args.kwargs["dataset_id"] == str(DS)
if "document_id" in kwargs:
assert called.call_args.kwargs["document_id"] == str(DOC)
@staticmethod
@pytest.mark.parametrize(
("fetch", "expected"), [("true", True), ("YES", True), ("false", False), ("invalid", False)]
)
def test_list_preserves_filters_pagination_and_fetch_parsing(app, documents, fetch, expected):
documents.list_documents.return_value = {"data": [], "has_more": False, "page": 2, "limit": 3, "total": 3}
with app.test_request_context(f"/?page=2&limit=3&keyword=needle&status=available&sort=hit_count&fetch={fetch}"):
response = unwrap(DatasetDocumentListApi.get)(DatasetDocumentListApi(), _DATASETS_DOCUMENT_CONTEXT, DS)
documents.list_documents.assert_called_once_with(
_DATASETS_DOCUMENT_CONTEXT,
dataset_id=str(DS),
query=DocumentListFilter(
page=2, limit=3, search="needle", status="available", sort="hit_count", fetch=expected
),
)
assert response == {"data": [], "has_more": False, "page": 2, "limit": 3, "total": 3}
@staticmethod
@pytest.mark.parametrize("metadata", ["only", "without", "all"])
def test_document_detail_metadata_projection_and_raw_source(app, documents, metadata):
documents.get_document.return_value = {
"id": str(DOC),
"doc_type": "others",
"doc_metadata": [],
"name": "Name",
"data_source_info": {"file_path": "/path/to/local/file"},
"data_source_detail_dict": {},
}
with app.test_request_context(f"/?metadata={metadata}"):
response, status = unwrap(DocumentApi.get)(DocumentApi(), _DATASETS_DOCUMENT_CONTEXT, DS, DOC)
assert status == 200
documents.get_document.assert_called_once_with(
_DATASETS_DOCUMENT_CONTEXT, dataset_id=str(DS), document_id=str(DOC), metadata_only=metadata == "only"
)
if metadata == "only":
assert response == {"id": str(DOC), "doc_type": "others", "doc_metadata": []}
else:
assert response["data_source_info"] == {"file_path": "/path/to/local/file"}
assert ("doc_metadata" in response) is (metadata == "all")
@staticmethod
def test_invalid_metadata_fails_before_application_call(app, documents):
with app.test_request_context("/?metadata=invalid"), pytest.raises(InvalidMetadataError):
unwrap(DocumentApi.get)(DocumentApi(), _DATASETS_DOCUMENT_CONTEXT, DS, DOC)
documents.get_document.assert_not_called()
@staticmethod
@pytest.mark.parametrize(
("resource", "verb", "kwargs"),
[
(DocumentApi, DocumentApi.delete, {"document_id": DOC}),
(DatasetDocumentListApi, DatasetDocumentListApi.delete, {}),
(DocumentPauseApi, DocumentPauseApi.patch, {"document_id": DOC}),
(DocumentRecoverApi, DocumentRecoverApi.patch, {"document_id": DOC}),
(DocumentRetryApi, DocumentRetryApi.post, {"req_data": DocumentRetryPayload(document_ids=[str(DOC)])}),
],
)
@pytest.mark.usefixtures("documents")
def test_no_content_mutations_return_empty_204(app, resource, verb, kwargs):
with app.test_request_context("/?document_id=" + str(DOC)):
result = unwrap(verb)(resource(), request_context=_DATASETS_DOCUMENT_CONTEXT, dataset_id=DS, **kwargs)
assert result == ("", 204)
@staticmethod
@pytest.mark.parametrize(
("resource", "verb", "operation", "kwargs"),
[
(
DocumentProcessingApi,
DocumentProcessingApi.patch,
lambda service: service.update_processing,
{"document_id": DOC, "action": "resume"},
),
(DocumentStatusApi, DocumentStatusApi.patch, lambda service: service.change_status, {"action": "disable"}),
(
WebsiteDocumentSyncApi,
WebsiteDocumentSyncApi.get,
lambda service: service.sync_website,
{"document_id": DOC},
),
(
DocumentGenerateSummaryApi,
DocumentGenerateSummaryApi.post,
lambda service: service.generate_summary,
{"req_data": GenerateSummaryPayload(document_list=[str(DOC)])},
),
],
)
def test_success_mutations_return_success_response(app, documents, resource, verb, operation, kwargs):
with app.test_request_context("/?document_id=" + str(DOC)):
result = unwrap(verb)(resource(), request_context=_DATASETS_DOCUMENT_CONTEXT, dataset_id=DS, **kwargs)
assert result == ({"result": "success"}, 200)
assert operation(documents).call_count == 1
@staticmethod
@pytest.mark.parametrize("resource", [DatasetInitApi, DatasetDocumentListApi])
def test_creation_serializes_materialized_dataset_and_documents(app, documents, resource):
operation = documents.initialize_dataset if resource is DatasetInitApi else documents.create_documents
operation.return_value = {
"dataset": {
"id": str(DS),
"name": "Dataset",
"created_at": _datasets_document_datetime.datetime(2024, 1, 1, tzinfo=_datasets_document_datetime.UTC),
},
"documents": [
{
"id": str(DOC),
"name": "Document",
"data_source_detail_dict": {},
"data_source_info_dict": {},
"doc_metadata_details": [],
}
],
"batch": "batch-1",
}
with app.test_request_context("/", json={"indexing_technique": "economy"}):
response = unwrap(resource.post)(
resource(),
request_context=_DATASETS_DOCUMENT_CONTEXT,
**({} if resource is DatasetInitApi else {"dataset_id": DS}),
)
assert response["dataset"]["created_at"] == 1704067200
assert response["documents"][0]["data_source_info"] == {}
assert response["documents"][0]["doc_metadata"] == []
assert response["batch"] == "batch-1"
@staticmethod
@pytest.mark.parametrize("resource", [DocumentIndexingStatusApi, DocumentBatchIndexingStatusApi])
def test_indexing_status_keeps_timestamps_counts_and_error_fields(app, documents, resource):
value = {
"id": str(DOC),
"indexing_status": "paused",
"completed_segments": 2,
"total_segments": 3,
**dict.fromkeys(
(
"processing_started_at",
"parsing_completed_at",
"cleaning_completed_at",
"splitting_completed_at",
"completed_at",
"paused_at",
"error",
"stopped_at",
)
),
}
value["processing_started_at"] = _datasets_document_datetime.datetime(
2024, 1, 1, tzinfo=_datasets_document_datetime.UTC
)
batch = resource is DocumentBatchIndexingStatusApi
operation = documents.get_batch_indexing_status if batch else documents.get_indexing_status
operation.return_value = {"data": [value]} if batch else value
with app.test_request_context("/"):
response = unwrap(resource.get)(
resource(),
request_context=_DATASETS_DOCUMENT_CONTEXT,
dataset_id=DS,
**({"batch": "batch-1"} if batch else {"document_id": DOC}),
)
row = response["data"][0] if batch else response
assert row["indexing_status"] == "paused"
assert row["processing_started_at"] == 1704067200
assert row["completed_segments"] == 2
assert row["total_segments"] == 3
@staticmethod
@pytest.mark.parametrize(
("translate", "error", "http_error"),
[
(controller._raise_document_error, DatasetNotFoundError(), NotFound),
(controller._raise_document_error, DocumentNotFoundError("missing"), NotFound),
(controller._raise_document_error, DatasetAccessDeniedError(), Forbidden),
(controller._raise_document_indexing_error, DocumentIndexingStateError("indexing"), DocumentIndexingError),
(
controller._raise_document_indexing_error,
DocumentProviderError("provider", kind="uninitialized"),
ProviderNotInitializeError,
),
(controller._raise_document_action_error, DocumentInvalidActionError("invalid"), InvalidActionError),
(controller._raise_document_status_error, DocumentInvalidActionError("invalid"), InvalidActionError),
(controller._raise_document_status_error, DocumentIndexingStateError("indexing"), DocumentIndexingError),
(controller._raise_document_archive_error, DocumentArchivedError(), ArchivedDocumentImmutableError),
(controller._raise_document_archive_error, DocumentIndexingStateError("indexing"), DocumentIndexingError),
# Every narrow translator falls back to the shared scope mapping.
(controller._raise_document_archive_error, DatasetNotFoundError(), NotFound),
],
)
def test_document_errors_are_mapped_at_transport_boundary(translate, error, http_error):
with pytest.raises(http_error):
translate(error)
@staticmethod
def test_scope_translator_does_not_map_other_operations_errors():
error = DocumentInvalidActionError("invalid")
with pytest.raises(DocumentInvalidActionError) as raised:
controller._raise_document_error(error)
assert raised.value is error
@staticmethod
def test_unexpected_error_is_not_hidden():
error = RuntimeError("unexpected")
with pytest.raises(RuntimeError) as raised:
controller._raise_document_error(error)
assert raised.value is error
class TestIndexingEstimateExceptionMapping:
@staticmethod
def _registry(method_name: str, error: Exception) -> SimpleNamespace:
estimates = MagicMock()
method = estimates.estimate_document if method_name == "estimate_document" else estimates.estimate_batch
method.side_effect = error
return SimpleNamespace(knowledge=SimpleNamespace(indexing_estimates=estimates))
@pytest.mark.parametrize(
("error", "expected_http_error"),
[
(DatasetNotFoundError(), NotFoundError),
(EstimateDocumentNotFoundError(), NotFoundError),
(EstimateSourceNotFoundError("source-1"), NotFoundError),
(IndexingEstimateCredentialUnavailableError(), NotFoundError),
(DatasetAccessDeniedError(), DatasetAccessDeniedRequestError),
(EstimateDocumentAlreadyFinishedError(), DocumentAlreadyFinishedError),
(UnsupportedEstimateSourceError("unsupported"), InvalidArgumentError),
(IndexingEstimateProviderUnavailableError(), ProviderNotInitializeError),
(IndexingEstimateExecutionError(), IndexingEstimateError),
],
)
def test_document_estimate_maps_application_errors(
self,
error: Exception,
expected_http_error: type[Exception],
) -> None:
api = DocumentIndexingEstimateApi()
method = unwrap(api.get)
context = RequestContext("request-1", None, "account-1", "workspace-1")
registry = self._registry("estimate_document", error)
with patch(
"controllers.console.datasets.datasets_document.application_services",
return_value=registry,
):
with pytest.raises(expected_http_error):
method(api, context, UUID(int=1), UUID(int=2))
@pytest.mark.parametrize(
("error", "expected_http_error"),
[
(DatasetNotFoundError(), NotFoundError),
(EstimateDocumentNotFoundError(), NotFoundError),
(EstimateSourceNotFoundError("source-1"), NotFoundError),
(IndexingEstimateCredentialUnavailableError(), NotFoundError),
(DatasetAccessDeniedError(), DatasetAccessDeniedRequestError),
(EstimateDocumentAlreadyFinishedError(), DocumentAlreadyFinishedError),
(UnsupportedEstimateSourceError("unsupported"), InvalidArgumentError),
(IndexingEstimateProviderUnavailableError(), ProviderNotInitializeError),
(IndexingEstimateExecutionError(), IndexingEstimateError),
],
)
def test_batch_estimate_maps_application_errors(
self,
error: Exception,
expected_http_error: type[Exception],
) -> None:
api = DocumentBatchIndexingEstimateApi()
method = unwrap(api.get)
context = RequestContext("request-1", None, "account-1", "workspace-1")
registry = self._registry("estimate_batch", error)
with patch(
"controllers.console.datasets.datasets_document.application_services",
return_value=registry,
):
with pytest.raises(expected_http_error):
method(api, context, UUID(int=1), "unknown-batch")
@staticmethod
@pytest.mark.parametrize(
"method",
[
DatasetDocumentListApi.get,
DocumentBatchIndexingStatusApi.get,
DocumentIndexingStatusApi.get,
DocumentApi.get,
DocumentPipelineExecutionLogApi.get,
DocumentSummaryStatusApi.get,
],
)
def test_document_read_routes_require_dataset_readonly_permission(method) -> None:
[check] = rbac_checks(method)
assert check.scene is RBACPermission.DATASET_READONLY
assert isinstance(check.locator, DatasetId)
@staticmethod
@pytest.mark.parametrize(
"method",
[DocumentIndexingEstimateApi.get, DocumentBatchIndexingEstimateApi.get],
)
def test_document_indexing_estimates_require_dataset_use_permission(method) -> None:
[check] = rbac_checks(method)
assert check.scene is RBACPermission.DATASET_USE
assert isinstance(check.locator, DatasetId)
@staticmethod
@pytest.mark.parametrize(
("method", "expected_scene"),
[
(DatasetDocumentListApi.post, RBACPermission.DATASET_USE),
(DatasetDocumentListApi.delete, RBACPermission.DATASET_DELETE_FILE),
(DocumentApi.delete, RBACPermission.DATASET_DELETE_FILE),
(DocumentBatchDownloadZipApi.post, RBACPermission.DATASET_DOCUMENT_DOWNLOAD),
],
)
def test_document_mutation_routes_require_operation_specific_permissions(method, expected_scene) -> None:
[check] = rbac_checks(method)
assert check.scene is expected_scene
assert isinstance(check.locator, DatasetId)
"""Document downloads through the controller, application service, and SQLite adapter."""
_DATASETS_DOCUMENT_DOWNLOAD_CONTEXT = RequestContext("request-1", None, "actor-1", "tenant-1")
DOC_ID = str(UUID(int=1))
FILE_ID = str(UUID(int=2))
def _document(**values):
return Document(
**{
"id": DOC_ID,
"tenant_id": "tenant-1",
"dataset_id": "dataset-1",
"position": 1,
"data_source_type": "upload_file",
"data_source_info": json.dumps({"upload_file_id": FILE_ID}),
"batch": "batch-1",
"name": "document.txt",
"created_from": "web",
"created_by": "actor-1",
**values,
}
)
class TestDatasetsDocumentDownloadCases:
@staticmethod
@pytest.fixture(autouse=True)
def downloads(monkeypatch, sqlite_session_factory):
with sqlite_session_factory.begin() as session:
session.add(Dataset(id="dataset-1", tenant_id="tenant-1", name="Dataset", created_by="actor-1"))
session.add(_document())
upload = UploadFile(
tenant_id="tenant-1",
storage_type="local",
key="key-1",
name="document.txt",
size=5,
extension="txt",
mime_type="text/plain",
created_by_role="account",
created_by="actor-1",
created_at=_datasets_document_download_datetime(2024, 1, 1),
used=False,
)
upload.id = FILE_ID
session.add(upload)
access = create_autospec(DatasetAccess, instance=True, spec_set=True)
access.require_accessible.return_value = AccessibleDataset("dataset-1", "tenant-1")
documents = DatasetDocumentApplicationService(
dataset_access=access,
operations=SQLAlchemyDocumentOperations(session_factory=sqlite_session_factory),
metadata_schema={},
)
monkeypatch.setattr(
controller, "application_services", lambda: SimpleNamespace(knowledge=SimpleNamespace(documents=documents))
)
return documents
@staticmethod
def test_document_download_returns_signed_url(app, monkeypatch):
from services.knowledge import dataset_service
signed = []
def sign(**kwargs):
signed.append(kwargs)
return "https://files.example/download"
monkeypatch.setattr(dataset_service.file_helpers, "get_signed_file_url", sign)
with app.test_request_context("/"):
result = unwrap(controller.DocumentDownloadApi.get)(
controller.DocumentDownloadApi(), _DATASETS_DOCUMENT_DOWNLOAD_CONTEXT, "dataset-1", DOC_ID
)
assert result == {"url": "https://files.example/download"}
assert signed == [{"upload_file_id": FILE_ID, "as_attachment": True}]
@staticmethod
@pytest.mark.parametrize(
"values",
[
{"data_source_type": "website_crawl"},
{"data_source_info": "{}"},
{"data_source_info": json.dumps({"upload_file_id": "missing"})},
{"tenant_id": "foreign"},
{"dataset_id": "foreign"},
],
)
def test_invalid_or_foreign_document_is_not_downloadable(app, sqlite_session_factory, values):
with sqlite_session_factory.begin() as session:
document = session.get(Document, DOC_ID)
for key, value in values.items():
setattr(document, key, value)
with app.test_request_context("/"), pytest.raises(NotFound):
unwrap(controller.DocumentDownloadApi.get)(
controller.DocumentDownloadApi(), _DATASETS_DOCUMENT_DOWNLOAD_CONTEXT, "dataset-1", DOC_ID
)
@staticmethod
def test_cross_tenant_upload_file_is_not_signed(app, sqlite_session_factory):
with sqlite_session_factory.begin() as session:
session.get(UploadFile, FILE_ID).tenant_id = "foreign"
with app.test_request_context("/"), pytest.raises(NotFound):
unwrap(controller.DocumentDownloadApi.get)(
controller.DocumentDownloadApi(), _DATASETS_DOCUMENT_DOWNLOAD_CONTEXT, "dataset-1", DOC_ID
)
@staticmethod
def test_zip_is_readable_and_cleanup_waits_for_response_close(app, monkeypatch, sqlite_engine):
from services import file_service
paths = []
checked_out = set()
def checkout(_connection, record, _proxy):
checked_out.add(record)
def checkin(_connection, record):
checked_out.discard(record)
def load(key, *, stream):
assert not checked_out, "storage I/O must run after the read transaction closes"
assert key == "key-1"
assert stream is True
return iter([b"hello"])
def send(path, **kwargs):
paths.append(Path(path))
return send_file(path, **kwargs)
monkeypatch.setattr(file_service.storage, "load", load)
monkeypatch.setattr(controller, "send_file", send)
event.listen(sqlite_engine, "checkout", checkout)
event.listen(sqlite_engine, "checkin", checkin)
try:
with app.test_request_context("/", json={"document_ids": [DOC_ID]}):
response = unwrap(controller.DocumentBatchDownloadZipApi.post)(
controller.DocumentBatchDownloadZipApi(), _DATASETS_DOCUMENT_DOWNLOAD_CONTEXT, "dataset-1"
)
assert paths[0].exists()
response.direct_passthrough = False
with ZipFile(BytesIO(response.get_data())) as archive:
assert archive.read("document.txt") == b"hello"
response.close()
assert not paths[0].exists()
finally:
event.remove(sqlite_engine, "checkout", checkout)
event.remove(sqlite_engine, "checkin", checkin)
@staticmethod
def test_zip_cleanup_when_response_creation_fails(app, monkeypatch):
from services import file_service
paths = []
monkeypatch.setattr(file_service.storage, "load", lambda *_args, **_kwargs: iter([b"hello"]))
def fail_send(path, **_kwargs):
paths.append(Path(path))
raise RuntimeError("response failed")
monkeypatch.setattr(controller, "send_file", fail_send)
with (
app.test_request_context("/", json={"document_ids": [DOC_ID]}),
pytest.raises(RuntimeError, match="response failed"),
):
unwrap(controller.DocumentBatchDownloadZipApi.post)(
controller.DocumentBatchDownloadZipApi(), _DATASETS_DOCUMENT_DOWNLOAD_CONTEXT, "dataset-1"
)
assert paths
assert not paths[0].exists()
@staticmethod
def test_zip_rejects_non_upload_document(app, sqlite_session_factory):
with sqlite_session_factory.begin() as session:
session.get(Document, DOC_ID).data_source_type = "website_crawl"
with app.test_request_context("/", json={"document_ids": [DOC_ID]}), pytest.raises(NotFound):
unwrap(controller.DocumentBatchDownloadZipApi.post)(
controller.DocumentBatchDownloadZipApi(), _DATASETS_DOCUMENT_DOWNLOAD_CONTEXT, "dataset-1"
)
@@ -0,0 +1,402 @@
from collections.abc import Callable
from contextlib import AbstractContextManager
from dataclasses import dataclass
from datetime import UTC, datetime
from inspect import unwrap
from typing import Never
from unittest.mock import create_autospec, patch
import pytest
from flask import Flask
from werkzeug.exceptions import Forbidden, NotFound
from controllers.common.controller_schemas import ChildChunkCreatePayload, ChildChunkUpdatePayload
from controllers.console.app.error import ProviderNotInitializeError
from controllers.console.datasets.datasets_segments import (
BatchImportPayload,
ChildChunkAddApi,
ChildChunkBatchUpdatePayload,
ChildChunkUpdateApi,
DatasetDocumentSegmentAddApi,
DatasetDocumentSegmentApi,
DatasetDocumentSegmentBatchImportApi,
DatasetDocumentSegmentBatchImportStatusApi,
DatasetDocumentSegmentListApi,
DatasetDocumentSegmentUpdateApi,
SegmentCreatePayload,
SegmentUpdatePayload,
_raise_child_chunk_create_error,
_raise_child_chunk_delete_error,
_raise_child_chunk_write_error,
_raise_segment_embedding_error,
_raise_segment_error,
_raise_segment_status_error,
)
from controllers.console.datasets.error import ChildChunkDeleteIndexError, ChildChunkIndexingError, InvalidActionError
from machinery.context import RequestContext
from services.knowledge.entities.segments import ChildChunkRecord, ChildChunkUpdateArgs, SegmentRecord
from services.knowledge.segments.application import (
ChildChunkDeleteIndexApplicationError,
ChildChunkIndexingApplicationError,
ChildChunkNotFoundError,
ChildChunkPage,
DatasetSegmentApplicationService,
SegmentBatchImport,
SegmentBatchImportDispatchError,
SegmentBatchImportNotFoundError,
SegmentDatasetModelUnavailableError,
SegmentDatasetNotFoundError,
SegmentDetail,
SegmentDocumentIndexingError,
SegmentDocumentNotFoundError,
SegmentEmbeddingModelUnavailableError,
SegmentInvalidFileTypeError,
SegmentListFilter,
SegmentNotFoundError,
SegmentPage,
SegmentPermissionDeniedError,
SegmentStatusUpdateError,
SegmentUploadFileNotFoundError,
)
def _context() -> RequestContext:
return RequestContext(
request_id="request-1",
trace_id=None,
account_id="account-1",
active_workspace_id="tenant-1",
)
def _child_chunk_data() -> ChildChunkRecord:
now = datetime(2026, 1, 1, tzinfo=UTC)
return ChildChunkRecord.model_validate(
{
"id": "child-1",
"segment_id": "segment-1",
"content": "child",
"position": 1,
"word_count": 5,
"type": "customized",
"created_at": now,
"updated_at": now,
}
)
def _segment_data() -> SegmentRecord:
now = datetime(2026, 1, 1, tzinfo=UTC)
return SegmentRecord.model_validate(
{
"id": "segment-1",
"position": 1,
"document_id": "document-1",
"content": "content",
"sign_content": "content",
"answer": None,
"word_count": 7,
"tokens": 2,
"keywords": [],
"index_node_id": "node-1",
"index_node_hash": "hash",
"hit_count": 0,
"enabled": True,
"disabled_at": None,
"disabled_by": None,
"status": "completed",
"created_by": "account-1",
"created_at": now,
"updated_at": now,
"updated_by": None,
"indexing_at": None,
"completed_at": now,
"error": None,
"stopped_at": None,
"child_chunks": [],
"attachments": [],
"summary": None,
}
)
@dataclass(frozen=True)
class KnowledgeServiceStub:
segments: DatasetSegmentApplicationService
@dataclass(frozen=True)
class ApplicationServiceStub:
knowledge: KnowledgeServiceStub
def _patch_services(segments: DatasetSegmentApplicationService) -> AbstractContextManager[object]:
return patch(
"controllers.console.datasets.datasets_segments.application_services",
return_value=ApplicationServiceStub(knowledge=KnowledgeServiceStub(segments=segments)),
)
def test_list_segments_delegates_typed_query_and_serializes_page(app: Flask) -> None:
segments = create_autospec(DatasetSegmentApplicationService, instance=True, spec_set=True)
segments.list_segments.return_value = SegmentPage(
items=(_segment_data(),), total=1, total_pages=1, page=2, limit=10
)
method = unwrap(DatasetDocumentSegmentListApi.get)
with (
app.test_request_context("/?status=completed&limit=10&page=2&enabled=true&keyword=needle"),
_patch_services(segments),
):
response, status = method(DatasetDocumentSegmentListApi(), _context(), "dataset-1", "document-1")
assert status == 200
assert response["data"][0]["id"] == "segment-1"
assert response["data"][0]["created_at"] == 1767225600
assert response["data"][0]["keywords"] == []
assert response["data"][0]["disabled_at"] is None
query = segments.list_segments.call_args.kwargs["query"]
assert query == SegmentListFilter(
page=2,
limit=10,
statuses=("completed",),
enabled="true",
keyword="needle",
)
def test_delete_segments_delegates_all_selected_ids(app: Flask) -> None:
segments = create_autospec(DatasetSegmentApplicationService, instance=True, spec_set=True)
method = unwrap(DatasetDocumentSegmentListApi.delete)
with app.test_request_context("/?segment_id=one&segment_id=two"), _patch_services(segments):
response, status = method(DatasetDocumentSegmentListApi(), _context(), "dataset-1", "document-1")
assert (response, status) == ("", 204)
assert segments.delete_segments.call_args.kwargs["segment_ids"] == ["one", "two"]
@pytest.mark.parametrize("action", ["enable", "disable"])
def test_change_segment_status_delegates_action(app: Flask, action: str) -> None:
segments = create_autospec(DatasetSegmentApplicationService, instance=True, spec_set=True)
method = unwrap(DatasetDocumentSegmentApi.patch)
with app.test_request_context("/?segment_id=one"), _patch_services(segments):
response, status = method(DatasetDocumentSegmentApi(), _context(), "dataset-1", "document-1", action)
assert status == 200
assert response == {"result": "success"}
assert segments.change_segment_status.call_args.kwargs["action"] == action
@pytest.mark.parametrize("action", ["typo", "ENABLE", "enable "])
def test_change_segment_status_rejects_unknown_action_before_service_call(app: Flask, action: str) -> None:
segments = create_autospec(DatasetSegmentApplicationService, instance=True, spec_set=True)
method = unwrap(DatasetDocumentSegmentApi.patch)
with (
app.test_request_context("/?segment_id=one"),
_patch_services(segments),
pytest.raises(InvalidActionError) as error,
):
method(DatasetDocumentSegmentApi(), _context(), "dataset-1", "document-1", action)
assert error.value.code == 400
segments.change_segment_status.assert_not_called()
def test_create_segment_delegates_payload_and_serializes_detail(app: Flask) -> None:
segments = create_autospec(DatasetSegmentApplicationService, instance=True, spec_set=True)
segments.create_segment.return_value = SegmentDetail(data=_segment_data(), doc_form="text_model")
method = unwrap(DatasetDocumentSegmentAddApi.post)
with app.test_request_context("/"), _patch_services(segments):
response, status = method(
DatasetDocumentSegmentAddApi(),
SegmentCreatePayload(content="content"),
_context(),
"dataset-1",
"document-1",
)
assert status == 200
assert response["data"]["id"] == "segment-1"
assert segments.create_segment.call_args.kwargs["values"] == {"content": "content"}
def test_update_and_delete_segment_delegate_owned_ids(app: Flask) -> None:
segments = create_autospec(DatasetSegmentApplicationService, instance=True, spec_set=True)
segments.update_segment.return_value = SegmentDetail(data=_segment_data(), doc_form="text_model")
with app.test_request_context("/"), _patch_services(segments):
update_response, update_status = unwrap(DatasetDocumentSegmentUpdateApi.patch)(
DatasetDocumentSegmentUpdateApi(),
SegmentUpdatePayload(content="updated"),
_context(),
"dataset-1",
"document-1",
"segment-1",
)
delete_response, delete_status = unwrap(DatasetDocumentSegmentUpdateApi.delete)(
DatasetDocumentSegmentUpdateApi(),
_context(),
"dataset-1",
"document-1",
"segment-1",
)
assert update_status == 200
assert update_response["data"]["content"] == "content"
assert (delete_response, delete_status) == ("", 204)
assert segments.update_segment.call_args.kwargs["segment_id"] == "segment-1"
assert segments.delete_segment.call_args.kwargs["segment_id"] == "segment-1"
def test_batch_import_delegates_and_preserves_dispatch_failure_response(app: Flask) -> None:
segments = create_autospec(DatasetSegmentApplicationService, instance=True, spec_set=True)
segments.start_batch_import.return_value = SegmentBatchImport(job_id="job-1", job_status="waiting")
method = unwrap(DatasetDocumentSegmentBatchImportApi.post)
with app.test_request_context("/"), _patch_services(segments):
response, status = method(
DatasetDocumentSegmentBatchImportApi(),
BatchImportPayload(upload_file_id="file-1"),
_context(),
"dataset-1",
"document-1",
)
assert status == 200
assert response == {"job_id": "job-1", "job_status": "waiting"}
segments.start_batch_import.side_effect = SegmentBatchImportDispatchError("redis down")
with app.test_request_context("/"), _patch_services(segments):
response, status = method(
DatasetDocumentSegmentBatchImportApi(),
BatchImportPayload(upload_file_id="file-1"),
_context(),
"dataset-1",
"document-1",
)
assert (response, status) == ({"error": "redis down"}, 500)
def test_batch_import_status_preserves_missing_job_contract(app: Flask) -> None:
segments = create_autospec(DatasetSegmentApplicationService, instance=True, spec_set=True)
method = unwrap(DatasetDocumentSegmentBatchImportStatusApi.get)
segments.get_batch_import_status.side_effect = SegmentBatchImportNotFoundError("The job does not exist.")
with (
app.test_request_context("/"),
_patch_services(segments),
pytest.raises(ValueError, match="The job does not exist\\."),
):
method(DatasetDocumentSegmentBatchImportStatusApi(), _context(), "job-1")
segments.get_batch_import_status.side_effect = None
segments.get_batch_import_status.return_value = SegmentBatchImport(job_id="job-1", job_status="completed")
with app.test_request_context("/"), _patch_services(segments):
response, status = method(DatasetDocumentSegmentBatchImportStatusApi(), _context(), "job-1")
assert status == 200
assert response == {"job_id": "job-1", "job_status": "completed"}
def test_child_chunk_create_and_list_delegate(app: Flask) -> None:
segments = create_autospec(DatasetSegmentApplicationService, instance=True, spec_set=True)
segments.create_child_chunk.return_value = _child_chunk_data()
segments.list_child_chunks.return_value = ChildChunkPage(
items=(_child_chunk_data(),), total=1, total_pages=1, page=1, limit=20
)
with app.test_request_context("/"), _patch_services(segments):
create_response, create_status = unwrap(ChildChunkAddApi.post)(
ChildChunkAddApi(),
ChildChunkCreatePayload(content="child"),
_context(),
"dataset-1",
"document-1",
"segment-1",
)
list_response, list_status = unwrap(ChildChunkAddApi.get)(
ChildChunkAddApi(), _context(), "dataset-1", "document-1", "segment-1"
)
assert create_status == list_status == 200
assert create_response["data"]["id"] == "child-1"
assert create_response["data"]["created_at"] == 1767225600
assert list_response["data"][0]["id"] == "child-1"
def test_child_chunk_mutations_delegate(app: Flask) -> None:
segments = create_autospec(DatasetSegmentApplicationService, instance=True, spec_set=True)
segments.update_child_chunks.return_value = (_child_chunk_data(),)
segments.update_child_chunk.return_value = _child_chunk_data()
chunks = [ChildChunkUpdateArgs(id="child-1", content="updated")]
with app.test_request_context("/"), _patch_services(segments):
batch_response, batch_status = unwrap(ChildChunkAddApi.patch)(
ChildChunkAddApi(),
ChildChunkBatchUpdatePayload(chunks=chunks),
_context(),
"dataset-1",
"document-1",
"segment-1",
)
update_response, update_status = unwrap(ChildChunkUpdateApi.patch)(
ChildChunkUpdateApi(),
ChildChunkUpdatePayload(content="updated"),
_context(),
"dataset-1",
"document-1",
"segment-1",
"child-1",
)
delete_response, delete_status = unwrap(ChildChunkUpdateApi.delete)(
ChildChunkUpdateApi(),
_context(),
"dataset-1",
"document-1",
"segment-1",
"child-1",
)
assert batch_status == update_status == 200
assert batch_response["data"][0]["id"] == "child-1"
assert update_response["data"]["id"] == "child-1"
assert (delete_response, delete_status) == ("", 204)
@pytest.mark.parametrize(
("translate", "error", "expected"),
[
(_raise_segment_error, SegmentDatasetNotFoundError(), NotFound),
(_raise_segment_error, SegmentDocumentNotFoundError(), NotFound),
(_raise_segment_error, SegmentNotFoundError(), NotFound),
(_raise_segment_error, ChildChunkNotFoundError(), NotFound),
(_raise_segment_error, SegmentUploadFileNotFoundError(), NotFound),
(_raise_segment_error, SegmentPermissionDeniedError(), Forbidden),
(_raise_segment_error, SegmentDatasetModelUnavailableError(), ValueError),
(_raise_segment_error, SegmentInvalidFileTypeError(), ValueError),
(_raise_segment_embedding_error, SegmentEmbeddingModelUnavailableError(), ProviderNotInitializeError),
(_raise_segment_status_error, SegmentDocumentIndexingError(), InvalidActionError),
(_raise_segment_status_error, SegmentStatusUpdateError(), InvalidActionError),
(_raise_segment_status_error, SegmentEmbeddingModelUnavailableError(), ProviderNotInitializeError),
(_raise_child_chunk_write_error, ChildChunkIndexingApplicationError(), ChildChunkIndexingError),
(_raise_child_chunk_create_error, ChildChunkIndexingApplicationError(), ChildChunkIndexingError),
(_raise_child_chunk_create_error, SegmentEmbeddingModelUnavailableError(), ProviderNotInitializeError),
(_raise_child_chunk_delete_error, ChildChunkDeleteIndexApplicationError(), ChildChunkDeleteIndexError),
# Every narrow translator falls back to the shared scope mapping.
(_raise_child_chunk_delete_error, SegmentNotFoundError(), NotFound),
],
)
def test_segment_application_errors_map_to_existing_http_contract(
translate: Callable[[Exception], Never], error: Exception, expected: type[Exception]
) -> None:
with pytest.raises(expected):
translate(error)
def test_scope_translator_does_not_map_other_operations_errors() -> None:
error = ChildChunkIndexingApplicationError()
with pytest.raises(ChildChunkIndexingApplicationError) as raised:
_raise_segment_error(error)
assert raised.value is error
@@ -0,0 +1,164 @@
from inspect import unwrap
from types import SimpleNamespace
from unittest.mock import create_autospec
from uuid import UUID
import pytest
from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
from controllers.console.datasets import external as controller
from controllers.console.datasets.error import DatasetNameDuplicateError
from machinery.context import RequestContext
from services.errors.dataset import DatasetNameDuplicateError as DuplicateName
from services.knowledge.dataset_access import DatasetAccessDeniedError, DatasetNotFoundError
from services.knowledge.external.application import (
ExternalHitTestingError,
ExternalKnowledgeApplicationService,
ExternalTemplateNotFoundError,
)
CONTEXT = RequestContext("request-1", None, "actor-1", "tenant-1")
ID = UUID(int=1)
@pytest.fixture
def external(monkeypatch):
service = create_autospec(ExternalKnowledgeApplicationService, instance=True, spec_set=True)
monkeypatch.setattr(
controller, "application_services", lambda: SimpleNamespace(knowledge=SimpleNamespace(external=service))
)
return service
def test_templates_preserve_pagination(external):
page = {"data": [], "total": 0, "has_more": False, "page": 1, "limit": 100}
external.list_templates.return_value = page
result = unwrap(controller.ExternalApiTemplateListApi.get)(
controller.ExternalApiTemplateListApi(),
controller.ExternalApiTemplateListQuery(page=0, limit=1000, keyword="test"),
CONTEXT,
)
assert result == (page, 200)
external.list_templates.assert_called_once_with(CONTEXT, page=0, limit=1000, keyword="test")
@pytest.mark.parametrize(
("resource", "verb", "operation", "payload"),
[
(
controller.ExternalApiTemplateListApi,
controller.ExternalApiTemplateListApi.post,
lambda service: service.create_template,
True,
),
(
controller.ExternalApiTemplateApi,
controller.ExternalApiTemplateApi.get,
lambda service: service.get_template,
False,
),
(
controller.ExternalApiTemplateApi,
controller.ExternalApiTemplateApi.patch,
lambda service: service.update_template,
True,
),
(
controller.ExternalApiTemplateApi,
controller.ExternalApiTemplateApi.delete,
lambda service: service.delete_template,
False,
),
],
)
def test_template_missing_is_mapped_and_context_forwarded(external, resource, verb, operation, payload):
method = operation(external)
method.side_effect = ExternalTemplateNotFoundError("API template not found.")
args = (
[
controller.ExternalKnowledgeApiPayload(
name="API", settings={"endpoint": "https://example", "api_key": "secret"}
)
]
if payload
else []
)
args.append(CONTEXT)
if resource is controller.ExternalApiTemplateApi:
args.append(ID)
with pytest.raises(NotFound):
unwrap(verb)(resource(), *args)
assert method.call_args.args == (CONTEXT,)
if resource is controller.ExternalApiTemplateApi:
assert method.call_args.kwargs["template_id"] == str(ID)
def test_template_serializes_settings_and_timestamp(external):
external.get_template.return_value = {
"id": str(ID),
"name": "API",
"tenant_id": "tenant-1",
"settings": {"endpoint": "https://example", "api_key": "[__HIDDEN__]"},
"description": "",
"created_by": "actor-1",
"created_at": "2024-01-01T00:00:00",
"dataset_bindings": [],
}
result, status = unwrap(controller.ExternalApiTemplateApi.get)(controller.ExternalApiTemplateApi(), CONTEXT, ID)
assert status == 200
assert result == external.get_template.return_value
def test_delete_and_usage_response_contracts(external):
assert unwrap(controller.ExternalApiTemplateApi.delete)(controller.ExternalApiTemplateApi(), CONTEXT, ID) == (
"",
204,
)
external.template_usage.return_value = (True, 3)
assert unwrap(controller.ExternalApiUseCheckApi.get)(controller.ExternalApiUseCheckApi(), CONTEXT, ID) == (
{"is_using": True, "count": 3},
200,
)
def test_external_dataset_duplicate_name_maps_to_409(external):
payload = controller.ExternalDatasetCreatePayload(
name="Dataset", external_knowledge_api_id="api", external_knowledge_id="knowledge"
)
external.create_dataset.side_effect = DuplicateName()
with pytest.raises(DatasetNameDuplicateError):
unwrap(controller.ExternalDatasetCreateApi.post)(controller.ExternalDatasetCreateApi(), payload, CONTEXT)
external.create_dataset.assert_called_once_with(CONTEXT, payload=payload)
@pytest.mark.parametrize(
("error", "http_error"),
[
(DatasetNotFoundError(), NotFound),
(DatasetAccessDeniedError(), Forbidden),
(ExternalHitTestingError("failed"), InternalServerError),
],
)
def test_retrieval_maps_application_errors(external, error, http_error):
external.hit_testing.side_effect = error
with pytest.raises(http_error):
unwrap(controller.ExternalKnowledgeHitTestingApi.post)(
controller.ExternalKnowledgeHitTestingApi(),
controller.ExternalHitTestingPayload(query="query"),
CONTEXT,
ID,
)
external.hit_testing.assert_called_once_with(
CONTEXT, dataset_id=str(ID), query="query", retrieval_model=None, metadata_filters=None
)
def test_retrieval_response_is_serialized(external):
external.hit_testing.return_value = {
"query": {"content": "query"},
"records": [{"content": "answer", "title": "title", "score": 0.8, "metadata": {"source": "doc"}}],
}
result = unwrap(controller.ExternalKnowledgeHitTestingApi.post)(
controller.ExternalKnowledgeHitTestingApi(), controller.ExternalHitTestingPayload(query="query"), CONTEXT, ID
)
assert result == external.hit_testing.return_value
@@ -0,0 +1,237 @@
import uuid
from inspect import unwrap
from unittest.mock import MagicMock, PropertyMock, patch
import pytest
from flask import Flask
from pytest_mock import MockerFixture
from werkzeug.exceptions import NotFound
from controllers.console import console_ns
from controllers.console.datasets.hit_testing import HitTestingApi
from models.account import Account, Tenant, TenantAccountRole
from models.dataset import Dataset
@pytest.fixture
def app():
app = Flask("test_hit_testing")
app.config["TESTING"] = True
return app
@pytest.fixture
def dataset_id():
return uuid.uuid4()
@pytest.fixture
def dataset():
return Dataset(id="dataset-1", tenant_id="tenant-1", name="Dataset", created_by="account-1")
@pytest.fixture
def account() -> Account:
account = Account(name="User", email="user@example.com")
account.id = "account-1"
tenant = Tenant(name="Tenant")
tenant.id = "tenant-1"
account._current_tenant = tenant
account.role = TenantAccountRole.OWNER
return account
def hit_testing_record() -> dict[str, object]:
return {
"segment": {
"id": "segment-1",
"position": 1,
"document_id": "document-1",
"content": "Chunk text",
"sign_content": "Chunk text",
"answer": None,
"word_count": 2,
"tokens": 3,
"keywords": [],
"index_node_id": None,
"index_node_hash": None,
"hit_count": 0,
"enabled": True,
"disabled_at": None,
"disabled_by": None,
"status": "completed",
"created_by": "account-1",
"created_at": 1_700_000_000,
"indexing_at": None,
"completed_at": None,
"error": None,
"stopped_at": None,
"document": {
"id": "document-1",
"data_source_type": "upload_file",
"name": "guide.md",
"doc_type": None,
"doc_metadata": None,
},
},
"child_chunks": [],
"score": None,
"tsne_position": None,
"files": [],
"summary": None,
}
@pytest.fixture(autouse=True)
def bypass_decorators(mocker: MockerFixture):
"""Bypass all decorators on the API method."""
mocker.patch(
"controllers.console.datasets.hit_testing.setup_required",
lambda f: f,
)
mocker.patch(
"controllers.console.datasets.hit_testing.login_required",
return_value=lambda f: f,
)
mocker.patch(
"controllers.console.datasets.hit_testing.account_initialization_required",
return_value=lambda f: f,
)
mocker.patch(
"controllers.console.datasets.hit_testing.cloud_edition_billing_rate_limit_check",
return_value=lambda *_: lambda f: f,
)
class TestHitTestingApi:
def test_hit_testing_success(self, app: Flask, dataset, dataset_id, account: Account):
api = HitTestingApi()
method = unwrap(api.post)
payload = {
"query": "what is vector search",
}
with (
app.test_request_context("/"),
patch.object(
type(console_ns),
"payload",
new_callable=PropertyMock,
return_value=payload,
),
patch.object(
HitTestingApi,
"get_and_validate_dataset",
return_value=dataset,
),
patch.object(
HitTestingApi,
"hit_testing_args_check",
),
patch.object(
HitTestingApi,
"perform_hit_testing",
return_value={"query": {"content": "what is vector search"}, "records": []},
),
):
result = method(api, MagicMock(), account, "tenant-1", dataset_id)
assert "query" in result
assert "records" in result
assert result["records"] == []
def test_hit_testing_success_with_optional_record_fields(self, app: Flask, dataset, dataset_id, account: Account):
api = HitTestingApi()
method = unwrap(api.post)
payload = {
"query": "what is vector search",
}
records = [hit_testing_record()]
with (
app.test_request_context("/"),
patch.object(
type(console_ns),
"payload",
new_callable=PropertyMock,
return_value=payload,
),
patch.object(
HitTestingApi,
"get_and_validate_dataset",
return_value=dataset,
),
patch.object(
HitTestingApi,
"hit_testing_args_check",
),
patch.object(
HitTestingApi,
"perform_hit_testing",
return_value={"query": {"content": payload["query"]}, "records": records},
),
):
result = method(api, MagicMock(), account, "tenant-1", dataset_id)
assert result["query"] == {"content": payload["query"]}
assert result["records"][0]["segment"]["keywords"] == []
assert result["records"][0]["child_chunks"] == []
assert result["records"][0]["files"] == []
assert result["records"][0]["score"] is None
def test_hit_testing_dataset_not_found(self, app: Flask, dataset_id, account: Account):
api = HitTestingApi()
method = unwrap(api.post)
payload = {
"query": "test",
}
with (
app.test_request_context("/"),
patch.object(
type(console_ns),
"payload",
new_callable=PropertyMock,
return_value=payload,
),
patch.object(
HitTestingApi,
"get_and_validate_dataset",
side_effect=NotFound("Dataset not found"),
),
):
with pytest.raises(NotFound, match="Dataset not found"):
method(api, MagicMock(), account, "tenant-1", dataset_id)
def test_hit_testing_invalid_args(self, app: Flask, dataset, dataset_id, account: Account):
api = HitTestingApi()
method = unwrap(api.post)
payload = {
"query": "",
}
with (
app.test_request_context("/"),
patch.object(
type(console_ns),
"payload",
new_callable=PropertyMock,
return_value=payload,
),
patch.object(
HitTestingApi,
"get_and_validate_dataset",
return_value=dataset,
),
patch.object(
HitTestingApi,
"hit_testing_args_check",
side_effect=ValueError("Invalid parameters"),
),
):
with pytest.raises(ValueError, match="Invalid parameters"):
method(api, MagicMock(), account, "tenant-1", dataset_id)
@@ -0,0 +1,296 @@
from unittest.mock import Mock, patch
import pytest
from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
import services.errors.base
from controllers.console.app.error import (
CompletionRequestError,
ProviderModelCurrentlyNotSupportError,
ProviderNotInitializeError,
ProviderQuotaExceededError,
)
from controllers.console.datasets.error import DatasetNotInitializedError
from controllers.console.datasets.hit_testing_base import (
DatasetsHitTestingBase,
)
from core.errors.error import (
LLMBadRequestError,
ModelCurrentlyNotSupportError,
ProviderTokenNotInitError,
QuotaExceededError,
)
from graphon.model_runtime.errors.invoke import InvokeError
from models.account import Account, Tenant, TenantAccountRole
from models.dataset import Dataset
from services.hit_testing_service import HitTestingService
from services.knowledge.dataset_service import DatasetService
@pytest.fixture
def account():
acc = Account(name="User", email="user@example.com")
acc.id = "account-1"
tenant = Tenant(name="Tenant")
tenant.id = "tenant-1"
acc._current_tenant = tenant
acc.role = TenantAccountRole.OWNER
return acc
@pytest.fixture
def dataset():
return Dataset(id="dataset-1", tenant_id="tenant-1", name="Dataset", created_by="account-1")
def hit_testing_record() -> dict[str, object]:
return {
"segment": {
"id": "segment-1",
"position": 1,
"document_id": "document-1",
"content": "Chunk text",
"answer": None,
"word_count": 2,
"tokens": 3,
"keywords": None,
"index_node_id": None,
"index_node_hash": None,
"hit_count": 0,
"enabled": True,
"disabled_at": None,
"disabled_by": None,
"status": "completed",
"created_by": "account-1",
"created_at": 1_700_000_000,
"indexing_at": None,
"completed_at": None,
"error": None,
"stopped_at": None,
"document": {
"id": "document-1",
"data_source_type": "upload_file",
"name": "guide.md",
"doc_type": None,
"doc_metadata": None,
},
},
"child_chunks": None,
"files": None,
"score": 0.8,
}
class TestGetAndValidateDataset:
def test_success(self, dataset, account):
with (
patch.object(
DatasetService,
"get_dataset",
return_value=dataset,
),
patch.object(
DatasetService,
"check_dataset_permission",
),
):
result = DatasetsHitTestingBase.get_and_validate_dataset(Mock(), "dataset-1", account, "tenant-1")
assert result == dataset
def test_dataset_not_found(self, account):
with patch.object(
DatasetService,
"get_dataset",
return_value=None,
):
with pytest.raises(NotFound, match="Dataset not found"):
DatasetsHitTestingBase.get_and_validate_dataset(Mock(), "dataset-1", account, "tenant-1")
def test_permission_denied(self, dataset, account):
with (
patch.object(
DatasetService,
"get_dataset",
return_value=dataset,
),
patch.object(
DatasetService,
"check_dataset_permission",
side_effect=services.errors.base.NoPermissionError("no access"),
),
):
with pytest.raises(Forbidden, match="no access"):
DatasetsHitTestingBase.get_and_validate_dataset(Mock(), "dataset-1", account, "tenant-1")
class TestHitTestingArgsCheck:
def test_args_check_called(self):
args = {"query": "test"}
with patch.object(
HitTestingService,
"hit_testing_args_check",
) as check_mock:
DatasetsHitTestingBase.hit_testing_args_check(args)
check_mock.assert_called_once_with(args)
class TestParseArgs:
def test_parse_args_success(self):
payload = {"query": "hello"}
result = DatasetsHitTestingBase.parse_args(payload)
assert result["query"] == "hello"
def test_parse_args_invalid(self):
payload = {"query": "x" * 300}
with pytest.raises(ValueError, match="1 validation error for HitTestingPayload\\\nquery"):
DatasetsHitTestingBase.parse_args(payload)
def test_parse_args_ignores_unknown_fields_for_compatibility(self):
payload = {"query": "hello", "top_k": 3}
result = DatasetsHitTestingBase.parse_args(payload)
assert result == {"query": "hello"}
class TestPerformHitTesting:
def test_success(self, dataset, account):
response = {
"query": {"content": "hello"},
"records": [],
}
with patch.object(
HitTestingService,
"retrieve",
return_value=response,
):
result = DatasetsHitTestingBase.perform_hit_testing(
Mock(), dataset, {"query": "hello"}, account, "tenant-1"
)
assert result["query"] == {"content": "hello"}
assert result["records"] == []
def test_success_prepares_nullable_list_fields(self, dataset, account):
response = {
"query": {"content": "hello"},
"records": [hit_testing_record()],
}
with patch.object(
HitTestingService,
"retrieve",
return_value=response,
):
result = DatasetsHitTestingBase.perform_hit_testing(
Mock(), dataset, {"query": "hello"}, account, "tenant-1"
)
assert result["query"] == {"content": "hello"}
record = result["records"][0]
assert record["segment"]["keywords"] == []
assert record["segment"]["sign_content"] is None
assert record["child_chunks"] == []
assert record["files"] == []
assert record["score"] == 0.8
assert record["tsne_position"] is None
assert record["summary"] is None
def test_invalid_query_response_raises_value_error(self, dataset, account):
with (
patch.object(
HitTestingService,
"retrieve",
return_value={"query": "hello", "records": []},
),
pytest.raises(ValueError, match="Invalid hit testing query response"),
):
DatasetsHitTestingBase.perform_hit_testing(Mock(), dataset, {"query": "hello"}, account, "tenant-1")
def test_invalid_records_response_raises_value_error(self):
with pytest.raises(ValueError, match="Invalid hit testing records response"):
DatasetsHitTestingBase._prepare_hit_testing_records({"records": []})
def test_invalid_record_response_raises_value_error(self):
with pytest.raises(ValueError, match="Invalid hit testing record response"):
DatasetsHitTestingBase._prepare_hit_testing_records(["record"])
def test_index_not_initialized(self, dataset, account):
with patch.object(
HitTestingService,
"retrieve",
side_effect=services.errors.index.IndexNotInitializedError(),
):
with pytest.raises(DatasetNotInitializedError):
DatasetsHitTestingBase.perform_hit_testing(Mock(), dataset, {"query": "hello"}, account, "tenant-1")
def test_provider_token_not_init(self, dataset, account):
with patch.object(
HitTestingService,
"retrieve",
side_effect=ProviderTokenNotInitError("token missing"),
):
with pytest.raises(ProviderNotInitializeError):
DatasetsHitTestingBase.perform_hit_testing(Mock(), dataset, {"query": "hello"}, account, "tenant-1")
def test_quota_exceeded(self, dataset, account):
with patch.object(
HitTestingService,
"retrieve",
side_effect=QuotaExceededError(),
):
with pytest.raises(ProviderQuotaExceededError):
DatasetsHitTestingBase.perform_hit_testing(Mock(), dataset, {"query": "hello"}, account, "tenant-1")
def test_model_not_supported(self, dataset, account):
with patch.object(
HitTestingService,
"retrieve",
side_effect=ModelCurrentlyNotSupportError(),
):
with pytest.raises(ProviderModelCurrentlyNotSupportError):
DatasetsHitTestingBase.perform_hit_testing(Mock(), dataset, {"query": "hello"}, account, "tenant-1")
def test_llm_bad_request(self, dataset, account):
with patch.object(
HitTestingService,
"retrieve",
side_effect=LLMBadRequestError("bad request"),
):
with pytest.raises(ProviderNotInitializeError):
DatasetsHitTestingBase.perform_hit_testing(Mock(), dataset, {"query": "hello"}, account, "tenant-1")
def test_invoke_error(self, dataset, account):
with patch.object(
HitTestingService,
"retrieve",
side_effect=InvokeError("invoke failed"),
):
with pytest.raises(CompletionRequestError):
DatasetsHitTestingBase.perform_hit_testing(Mock(), dataset, {"query": "hello"}, account, "tenant-1")
def test_value_error(self, dataset, account):
with patch.object(
HitTestingService,
"retrieve",
side_effect=ValueError("bad args"),
):
with pytest.raises(ValueError, match="bad args"):
DatasetsHitTestingBase.perform_hit_testing(Mock(), dataset, {"query": "hello"}, account, "tenant-1")
def test_unexpected_error(self, dataset, account):
with patch.object(
HitTestingService,
"retrieve",
side_effect=Exception("boom"),
):
with pytest.raises(InternalServerError) as exc_info:
DatasetsHitTestingBase.perform_hit_testing(Mock(), dataset, {"query": "hello"}, account, "tenant-1")
assert "boom" not in str(exc_info.value)
@@ -0,0 +1,145 @@
"""Console metadata routes use admission, explicit context, and response models."""
from types import SimpleNamespace
from unittest.mock import Mock, create_autospec
from uuid import uuid4
import pytest
from flask import Flask
from flask.testing import FlaskClient
from flask_restx import Api
from controllers.common.rbac import DatasetId, RBACPermission
from controllers.console import api as console_api
from controllers.console import console_ns
from controllers.console.datasets.metadata import DatasetMetadataCreateApi
from enums import DeploymentEdition
from libs.login import AccountWithTenant
from models.account import Account, TenantAccountRole
from services.errors.base import NoPermissionError
from services.errors.metadata import MetadataResourceNotFoundError
from services.knowledge.dataset_access import DatasetAccessDeniedError, DatasetNotFoundError
from services.knowledge.metadata.application import MetadataService
from services.knowledge.resource_scope import DatasetRef
from tests.helpers.unit.controllers.rbac_introspection import rbac_checks
@pytest.fixture
def routes(monkeypatch: pytest.MonkeyPatch, config_overrides):
config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD, LOGIN_DISABLED=True, RBAC_ENABLED=False)
app = Flask(__name__)
app.config.update(TESTING=False, PROPAGATE_EXCEPTIONS=False)
api = Api(app)
api.error_handlers = console_api.error_handlers.copy()
api.add_namespace(console_ns, path="/console/api")
account = Account(name="Owner", email="metadata@example.com")
account.id = "account-1"
account.role = TenantAccountRole.OWNER
identity = AccountWithTenant(account=account, tenant_id="tenant-1")
monkeypatch.setattr("controllers.console.flask_admission.current_account_with_tenant", lambda: identity)
monkeypatch.setattr("controllers.console.wraps.current_account_with_tenant", lambda: identity)
service = create_autospec(MetadataService, instance=True, spec_set=True)
service.require_dataset.side_effect = lambda context, dataset_id: DatasetRef(
context.active_workspace_id, dataset_id
)
app.extensions["application_services"] = SimpleNamespace(knowledge=SimpleNamespace(metadata=service))
return app.test_client(), service
@pytest.mark.parametrize(
("error", "status", "code", "message"),
[
(NoPermissionError("Access denied"), 400, "invalid_param", "Access denied"),
(RuntimeError("backend detail"), 500, "unknown", "Internal Server Error"),
],
)
def test_metadata_permission_error_http_contract(
routes: tuple[FlaskClient, Mock], error: Exception, status: int, code: str, message: str
) -> None:
client, service = routes
service.require_dataset.side_effect = error
response = client.post(f"/console/api/datasets/{uuid4()}/metadata", json={"type": "string", "name": "author"})
assert response.status_code == status
assert response.json == {"code": code, "message": message, "status": status}
service.create_metadata.assert_not_called()
def test_create_serializes_result_and_passes_actor(routes):
client, service = routes
dataset_id = str(uuid4())
service.create_metadata.return_value = {"id": str(uuid4()), "name": "author", "type": "string"}
response = client.post(f"/console/api/datasets/{dataset_id}/metadata", json={"type": "string", "name": "author"})
assert response.status_code == 201
assert response.get_json()["name"] == "author"
assert service.create_metadata.call_args.args[0] == DatasetRef("tenant-1", dataset_id)
assert service.create_metadata.call_args.kwargs == {"actor_id": "account-1"}
@pytest.mark.parametrize("payload", [{"name": None, "type": "string"}, {"name": "author", "type": None}])
def test_create_rejects_invalid_payload_before_application(routes, payload):
client, service = routes
response = client.post(f"/console/api/datasets/{uuid4()}/metadata", json=payload)
assert response.status_code == 422
service.create_metadata.assert_not_called()
@pytest.mark.parametrize(("error", "status"), [(DatasetNotFoundError(), 404), (DatasetAccessDeniedError(), 403)])
def test_get_maps_access_errors(routes, error, status):
client, service = routes
service.require_dataset.side_effect = error
assert client.get(f"/console/api/datasets/{uuid4()}/metadata").status_code == status
service.get_dataset_metadatas.assert_not_called()
def test_get_serializes_field_counts(routes):
client, service = routes
service.get_dataset_metadatas.return_value = {"doc_metadata": [], "built_in_field_enabled": False}
response = client.get(f"/console/api/datasets/{uuid4()}/metadata")
assert response.status_code == 200
assert response.get_json() == {"doc_metadata": [], "built_in_field_enabled": False}
def test_rename_and_delete_use_explicit_metadata_id(routes):
client, service = routes
dataset_id, metadata_id = str(uuid4()), str(uuid4())
path = f"/console/api/datasets/{dataset_id}/metadata/{metadata_id}"
service.update_metadata_name.return_value = {"id": metadata_id, "name": "writer", "type": "string"}
assert client.patch(path, json={"name": "writer"}).status_code == 200
service.update_metadata_name.assert_called_once_with(
DatasetRef("tenant-1", dataset_id), metadata_id, "writer", actor_id="account-1"
)
response = client.delete(path)
assert response.status_code == 204
assert response.data == b""
service.delete_metadata.assert_called_once_with(DatasetRef("tenant-1", dataset_id), metadata_id)
@pytest.mark.parametrize("action", ["enable", "disable"])
def test_builtin_toggle_is_dispatched(routes, action):
client, service = routes
dataset_id = str(uuid4())
response = client.post(f"/console/api/datasets/{dataset_id}/metadata/built-in/{action}")
assert response.status_code == 204
operation = service.enable_built_in_field if action == "enable" else service.disable_built_in_field
operation.assert_called_once_with(DatasetRef("tenant-1", dataset_id))
def test_builtin_listing_uses_response_model(routes):
client, service = routes
service.get_built_in_fields.return_value = [{"name": "uploader", "type": "string"}]
response = client.get("/console/api/datasets/metadata/built-in")
assert response.status_code == 200
assert response.get_json() == {"fields": [{"name": "uploader", "type": "string"}]}
def test_document_metadata_missing_resource_is_not_found(routes):
client, service = routes
service.update_documents_metadata.side_effect = MetadataResourceNotFoundError("Document not found.")
response = client.post(f"/console/api/datasets/{uuid4()}/documents/metadata", json={"operation_data": []})
assert response.status_code == 404
def test_get_requires_dataset_readonly_permission():
[check] = rbac_checks(DatasetMetadataCreateApi.get)
assert check.scene is RBACPermission.DATASET_READONLY
assert isinstance(check.locator, DatasetId)
@@ -0,0 +1,174 @@
import inspect
import pytest
from flask import Flask
from pytest_mock import MockerFixture
from controllers.console.datasets.error import WebsiteCrawlError
from controllers.console.datasets.website import (
WebsiteCrawlApi,
WebsiteCrawlPayload,
WebsiteCrawlStatusApi,
WebsiteCrawlStatusQuery,
)
from services.data_source.website_service import (
WebsiteCrawlApiRequest,
WebsiteCrawlStatusApiRequest,
WebsiteService,
)
@pytest.fixture
def app() -> Flask:
app = Flask("test_website_crawl")
app.config["TESTING"] = True
return app
@pytest.fixture(autouse=True)
def bypass_auth_and_setup(mocker: MockerFixture) -> None:
"""Bypass setup/login/account decorators."""
mocker.patch(
"controllers.console.datasets.website.login_required",
lambda f: f,
)
mocker.patch(
"controllers.console.datasets.website.setup_required",
lambda f: f,
)
mocker.patch(
"controllers.console.datasets.website.account_initialization_required",
lambda f: f,
)
class TestWebsiteCrawlApi:
def test_crawl_success(self, app: Flask, mocker: MockerFixture) -> None:
api = WebsiteCrawlApi()
method = inspect.unwrap(api.post)
payload = {
"provider": "firecrawl",
"url": "https://example.com",
"options": {"depth": 1},
}
req_data = WebsiteCrawlPayload.model_validate(payload)
with app.test_request_context("/", json=payload):
crawl_url = mocker.patch.object(
WebsiteService,
"crawl_url",
return_value={"job_id": "job-1"},
)
result, status = method(api, req_data)
assert status == 200
assert result["job_id"] == "job-1"
crawl_url.assert_called_once_with(
WebsiteCrawlApiRequest(provider="firecrawl", url="https://example.com", options={"depth": 1})
)
def test_crawl_invalid_payload(self, app: Flask, mocker: MockerFixture) -> None:
api = WebsiteCrawlApi()
method = inspect.unwrap(api.post)
payload = {
"provider": "firecrawl",
"url": "bad-url",
"options": dict[str, object](),
}
req_data = WebsiteCrawlPayload.model_validate(payload)
with app.test_request_context("/", json=payload):
mocker.patch.object(
WebsiteCrawlApiRequest,
"from_args",
side_effect=ValueError("invalid payload"),
)
with pytest.raises(WebsiteCrawlError, match="invalid payload"):
method(api, req_data)
def test_crawl_service_error(self, app: Flask, mocker: MockerFixture) -> None:
api = WebsiteCrawlApi()
method = inspect.unwrap(api.post)
payload = {
"provider": "firecrawl",
"url": "https://example.com",
"options": {"limit": 1},
}
req_data = WebsiteCrawlPayload.model_validate(payload)
with app.test_request_context("/", json=payload):
crawl_url = mocker.patch.object(
WebsiteService,
"crawl_url",
side_effect=Exception("crawl failed"),
)
with pytest.raises(WebsiteCrawlError, match="crawl failed"):
method(api, req_data)
crawl_url.assert_called_once_with(
WebsiteCrawlApiRequest(provider="firecrawl", url="https://example.com", options={"limit": 1})
)
class TestWebsiteCrawlStatusApi:
def test_get_status_success(self, app: Flask, mocker: MockerFixture) -> None:
api = WebsiteCrawlStatusApi()
method = inspect.unwrap(api.get)
job_id = "job-123"
req_data = WebsiteCrawlStatusQuery.model_validate({"provider": "firecrawl"})
with app.test_request_context("/?provider=firecrawl"):
get_status = mocker.patch.object(
WebsiteService,
"get_crawl_status_typed",
return_value={"status": "completed"},
)
result, status = method(api, req_data, job_id)
assert status == 200
assert result["status"] == "completed"
get_status.assert_called_once_with(WebsiteCrawlStatusApiRequest(provider="firecrawl", job_id=job_id))
def test_get_status_invalid_provider(self, app: Flask, mocker: MockerFixture) -> None:
api = WebsiteCrawlStatusApi()
method = inspect.unwrap(api.get)
job_id = "job-123"
req_data = WebsiteCrawlStatusQuery.model_validate({"provider": "firecrawl"})
with app.test_request_context("/?provider=firecrawl"):
mocker.patch.object(
WebsiteCrawlStatusApiRequest,
"from_args",
side_effect=ValueError("invalid provider"),
)
with pytest.raises(WebsiteCrawlError, match="invalid provider"):
method(api, req_data, job_id)
def test_get_status_service_error(self, app: Flask, mocker: MockerFixture) -> None:
api = WebsiteCrawlStatusApi()
method = inspect.unwrap(api.get)
job_id = "job-123"
req_data = WebsiteCrawlStatusQuery.model_validate({"provider": "firecrawl"})
with app.test_request_context("/?provider=firecrawl"):
get_status = mocker.patch.object(
WebsiteService,
"get_crawl_status_typed",
side_effect=Exception("status lookup failed"),
)
with pytest.raises(WebsiteCrawlError, match="status lookup failed"):
method(api, req_data, job_id)
get_status.assert_called_once_with(WebsiteCrawlStatusApiRequest(provider="firecrawl", job_id=job_id))
@@ -0,0 +1,168 @@
from collections.abc import Iterator
import pytest
from pytest_mock import MockerFixture
from sqlalchemy.orm import Session, scoped_session, sessionmaker
from controllers.console.datasets.error import PipelineNotFoundError
from controllers.console.datasets.wraps import get_rag_pipeline, load_rag_pipeline
from extensions.ext_database import db
from models.account import Account
from models.dataset import Pipeline
@pytest.fixture
def database_session(
monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session]
) -> Iterator[scoped_session[Session]]:
"""Install a real scoped SQLite session for the legacy decorator entrypoint."""
session = scoped_session(sqlite_session_factory)
monkeypatch.setattr(db, "session", session)
yield session
session.remove()
@pytest.fixture
def current_account() -> Account:
account = Account(name="Test User", email="test@example.com")
account.id = "account-1"
return account
class TestGetRagPipeline:
def test_missing_pipeline_id(self) -> None:
@get_rag_pipeline
def dummy_view(**_kwargs: object) -> str:
return "ok"
with pytest.raises(ValueError, match="missing pipeline_id"):
dummy_view()
def test_pipeline_not_found(
self,
mocker: MockerFixture,
database_session: scoped_session[Session],
current_account: Account,
) -> None:
@get_rag_pipeline
def dummy_view(**_kwargs: object) -> str:
return "ok"
mocker.patch(
"controllers.console.datasets.wraps.current_account_with_tenant",
return_value=(current_account, "tenant-1"),
)
get_pipeline_by_id = mocker.patch(
"controllers.console.datasets.wraps.RagPipelineService.get_pipeline_by_id",
return_value=None,
)
with pytest.raises(PipelineNotFoundError):
dummy_view(pipeline_id="pipeline-1")
get_pipeline_by_id.assert_called_once_with("pipeline-1", "tenant-1", session=database_session())
def test_pipeline_found_and_injected(
self,
mocker: MockerFixture,
database_session: scoped_session[Session],
current_account: Account,
) -> None:
pipeline = Pipeline(
tenant_id="tenant-1",
name="Test Pipeline",
)
pipeline.id = "pipeline-1"
@get_rag_pipeline
def dummy_view(**kwargs: object) -> object:
return kwargs["pipeline"]
mocker.patch(
"controllers.console.datasets.wraps.current_account_with_tenant",
return_value=(current_account, "tenant-1"),
)
get_pipeline_by_id = mocker.patch(
"controllers.console.datasets.wraps.RagPipelineService.get_pipeline_by_id",
return_value=pipeline,
)
result = dummy_view(pipeline_id="pipeline-1")
assert result is pipeline
get_pipeline_by_id.assert_called_once_with("pipeline-1", "tenant-1", session=database_session())
def test_load_rag_pipeline_uses_provided_session(
self, mocker: MockerFixture, sqlite_session: Session, current_account: Account
) -> None:
pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")
mocker.patch(
"controllers.console.datasets.wraps.current_account_with_tenant",
return_value=(current_account, "tenant-1"),
)
get_pipeline_by_id = mocker.patch(
"controllers.console.datasets.wraps.RagPipelineService.get_pipeline_by_id",
return_value=pipeline,
)
result = load_rag_pipeline(sqlite_session, "pipeline-1")
assert result is pipeline
get_pipeline_by_id.assert_called_once_with("pipeline-1", "tenant-1", session=sqlite_session)
def test_pipeline_id_removed_from_kwargs(
self,
mocker: MockerFixture,
database_session: scoped_session[Session],
current_account: Account,
) -> None:
pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")
@get_rag_pipeline
def dummy_view(**kwargs: object) -> str:
assert "pipeline_id" not in kwargs
return "ok"
mocker.patch(
"controllers.console.datasets.wraps.current_account_with_tenant",
return_value=(current_account, "tenant-1"),
)
mocker.patch(
"controllers.console.datasets.wraps.RagPipelineService.get_pipeline_by_id",
return_value=pipeline,
)
result = dummy_view(pipeline_id="pipeline-1")
assert result == "ok"
assert db.session is database_session
def test_pipeline_id_cast_to_string(
self,
mocker: MockerFixture,
database_session: scoped_session[Session],
current_account: Account,
) -> None:
pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")
@get_rag_pipeline
def dummy_view(**kwargs: object) -> object:
return kwargs["pipeline"]
mocker.patch(
"controllers.console.datasets.wraps.current_account_with_tenant",
return_value=(current_account, "tenant-1"),
)
get_pipeline_by_id = mocker.patch(
"controllers.console.datasets.wraps.RagPipelineService.get_pipeline_by_id",
return_value=pipeline,
)
result = dummy_view(pipeline_id=123)
assert result is pipeline
get_pipeline_by_id.assert_called_once_with("123", "tenant-1", session=database_session())