chore: 纳入本地 Dify 1.19.0 源码及品牌与免登改动
演示目录只保留可重建的源码。运行密钥、数据卷、依赖和带口令的本地配置不进入仓库。 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
60158935f4
commit
6519682885
15831 files changed
+3063428
No files matched your search
@@ -0,0 +1 @@
|
||||
"""Unit tests for `controllers.console.datasets` controllers."""
|
||||
Whitespace-only changes.
+988
@@ -0,0 +1,988 @@
|
||||
import inspect
|
||||
from datetime import UTC, datetime
|
||||
from importlib import import_module
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import ANY, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
from werkzeug.exceptions import Forbidden, NotFound
|
||||
|
||||
from controllers.console import console_ns
|
||||
from controllers.console.datasets.rag_pipeline.datasource_auth import (
|
||||
DatasourceAuth,
|
||||
DatasourceAuthDefaultApi,
|
||||
DatasourceAuthDeleteApi,
|
||||
DatasourceAuthListApi,
|
||||
DatasourceAuthOauthCustomClient,
|
||||
DatasourceAuthUpdateApi,
|
||||
DatasourceCredentialDeletePayload,
|
||||
DatasourceCredentialPayload,
|
||||
DatasourceCredentialUpdatePayload,
|
||||
DatasourceCustomClientPayload,
|
||||
DatasourceDefaultPayload,
|
||||
DatasourceHardCodeAuthListApi,
|
||||
DatasourceOAuthCallback,
|
||||
DatasourcePluginOAuthAuthorizationUrl,
|
||||
DatasourceUpdateNamePayload,
|
||||
DatasourceUpdateProviderNameApi,
|
||||
)
|
||||
from core.plugin.impl.oauth import OAuthHandler
|
||||
from extensions.application_services.data_sources import build_data_source_credentials
|
||||
from graphon.model_runtime.errors.validate import CredentialsValidateFailedError
|
||||
from models.account import Account
|
||||
from repositories.credentials.query_repository import CredentialQueryRepository
|
||||
from services.data_source.provider_service import DatasourceProviderService
|
||||
from services.plugin.dispatch_identity import PluginOAuthClientConfig
|
||||
from services.plugin.oauth_service import OAuthProxyService
|
||||
from tests.helpers.unit.model_factories import make_account
|
||||
|
||||
_PROVIDER_ID = "langgenius/notion_datasource/notion"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def credential_query_dependency(monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session]) -> None:
|
||||
query = CredentialQueryRepository(session_factory=sqlite_session_factory)
|
||||
providers = build_data_source_credentials(database_client=sqlite_session_factory).providers
|
||||
monkeypatch.setattr(
|
||||
import_module("controllers.console.datasets.rag_pipeline.datasource_auth"),
|
||||
"application_services",
|
||||
lambda: SimpleNamespace(credential_queries=query, data_sources=SimpleNamespace(providers=providers)),
|
||||
)
|
||||
|
||||
|
||||
def _account() -> Account:
|
||||
return make_account(account_id="user-1", name="Datasource Auth Tester", email="datasource-auth@example.com")
|
||||
|
||||
|
||||
def _i18n(text: str) -> dict[str, str]:
|
||||
return {"en_US": text, "zh_Hans": text, "pt_BR": text, "ja_JP": text}
|
||||
|
||||
|
||||
def _provider_config(name: str, type_: str, label: str, *, required: bool = True) -> dict:
|
||||
return {
|
||||
"type": type_,
|
||||
"name": name,
|
||||
"scope": None,
|
||||
"required": required,
|
||||
"default": None,
|
||||
"options": None,
|
||||
"multiple": False,
|
||||
"label": _i18n(label),
|
||||
"help": None,
|
||||
"url": None,
|
||||
"placeholder": None,
|
||||
}
|
||||
|
||||
|
||||
def _datasource_credential(credential_id: str = "cred-1", *, is_default: bool = True) -> dict:
|
||||
return {
|
||||
"credential": {
|
||||
"api_key": "******",
|
||||
"workspace": "engineering",
|
||||
"database_id": "db-123",
|
||||
},
|
||||
"type": "api-key",
|
||||
"name": "API Key",
|
||||
"avatar_url": "https://cdn.example.com/notion.png",
|
||||
"id": credential_id,
|
||||
"is_default": is_default,
|
||||
}
|
||||
|
||||
|
||||
def _datasource_auth() -> dict:
|
||||
return {
|
||||
"author": "Dify",
|
||||
"provider": "notion",
|
||||
"plugin_id": "langgenius/notion_datasource",
|
||||
"plugin_unique_identifier": "langgenius/notion_datasource:0.0.1",
|
||||
"icon": "icon.svg",
|
||||
"name": "notion",
|
||||
"label": _i18n("Notion"),
|
||||
"description": _i18n("Notion datasource"),
|
||||
"credential_schema": [
|
||||
_provider_config("api_key", "secret-input", "API key"),
|
||||
],
|
||||
"oauth_schema": {
|
||||
"client_schema": [
|
||||
_provider_config("client_id", "text-input", "Client ID"),
|
||||
],
|
||||
"credentials_schema": [
|
||||
_provider_config("access_token", "secret-input", "Access token"),
|
||||
],
|
||||
"oauth_custom_client_params": {"client_id": "masked-client", "client_secret": "********"},
|
||||
"is_oauth_custom_client_enabled": True,
|
||||
"is_system_oauth_params_exists": True,
|
||||
"redirect_uri": "https://api.example.com/oauth/callback",
|
||||
},
|
||||
"credentials_list": [_datasource_credential(), _datasource_credential("cred-2", is_default=False)],
|
||||
}
|
||||
|
||||
|
||||
def _success_response() -> dict[str, str]:
|
||||
return {"result": "success"}
|
||||
|
||||
|
||||
class TestDatasourcePluginOAuthAuthorizationUrl:
|
||||
def test_get_success(self, app: Flask):
|
||||
api = DatasourcePluginOAuthAuthorizationUrl()
|
||||
method = inspect.unwrap(api.get)
|
||||
|
||||
user = _account()
|
||||
oauth_client = {"client_id": "abc", "client_secret": "shh", "scopes": ["read", "write"]}
|
||||
auth_url_payload = {
|
||||
"authorization_url": "https://auth.example.com/oauth?client_id=abc&state=xyz",
|
||||
}
|
||||
|
||||
with (
|
||||
app.test_request_context("/?credential_id=cred-1"),
|
||||
patch.object(
|
||||
DatasourceProviderService,
|
||||
"get_oauth_client",
|
||||
return_value=PluginOAuthClientConfig(
|
||||
"langgenius/notion_datasource:1.0.0@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", oauth_client
|
||||
),
|
||||
) as get_oauth_client,
|
||||
patch.object(
|
||||
OAuthProxyService,
|
||||
"create_proxy_context",
|
||||
return_value="ctx-1",
|
||||
) as create_proxy_context,
|
||||
patch.object(
|
||||
OAuthHandler,
|
||||
"get_authorization_url",
|
||||
return_value=auth_url_payload,
|
||||
) as get_authorization_url,
|
||||
):
|
||||
response = method(api, "tenant-1", user, _PROVIDER_ID)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.get_json() == auth_url_payload
|
||||
assert "context_id=ctx-1" in response.headers.get("Set-Cookie")
|
||||
provider_id = get_oauth_client.call_args.kwargs["datasource_provider_id"]
|
||||
assert str(provider_id) == _PROVIDER_ID
|
||||
get_oauth_client.assert_called_once()
|
||||
create_proxy_context.assert_called_once_with(
|
||||
user_id="user-1",
|
||||
tenant_id="tenant-1",
|
||||
plugin_id="langgenius/notion_datasource",
|
||||
provider="notion",
|
||||
credential_id="cred-1",
|
||||
extra_data={
|
||||
"visibility": "only_me",
|
||||
"plugin_unique_identifier": "langgenius/notion_datasource:1.0.0@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
},
|
||||
)
|
||||
get_authorization_url.assert_called_once()
|
||||
assert get_authorization_url.call_args.kwargs["tenant_id"] == "tenant-1"
|
||||
assert get_authorization_url.call_args.kwargs["user_id"] == "user-1"
|
||||
assert get_authorization_url.call_args.kwargs["plugin_id"] == "langgenius/notion_datasource"
|
||||
assert get_authorization_url.call_args.kwargs["provider"] == "notion"
|
||||
assert get_authorization_url.call_args.kwargs["system_credentials"] == oauth_client
|
||||
|
||||
def test_get_no_oauth_config(self, app: Flask):
|
||||
api = DatasourcePluginOAuthAuthorizationUrl()
|
||||
method = inspect.unwrap(api.get)
|
||||
user = _account()
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch.object(
|
||||
DatasourceProviderService,
|
||||
"get_oauth_client",
|
||||
return_value=None,
|
||||
),
|
||||
):
|
||||
with pytest.raises(ValueError, match="No OAuth Client Config for notion"):
|
||||
method(api, "tenant-1", user, "notion")
|
||||
|
||||
def test_get_without_credential_id_sets_cookie(self, app: Flask):
|
||||
api = DatasourcePluginOAuthAuthorizationUrl()
|
||||
method = inspect.unwrap(api.get)
|
||||
|
||||
user = _account()
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch.object(
|
||||
DatasourceProviderService,
|
||||
"get_oauth_client",
|
||||
return_value=PluginOAuthClientConfig(
|
||||
"langgenius/notion_datasource:1.0.0@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", {"client_id": "abc"}
|
||||
),
|
||||
),
|
||||
patch.object(
|
||||
OAuthProxyService,
|
||||
"create_proxy_context",
|
||||
return_value="ctx-123",
|
||||
),
|
||||
patch.object(
|
||||
OAuthHandler,
|
||||
"get_authorization_url",
|
||||
return_value={"authorization_url": "http://auth"},
|
||||
),
|
||||
):
|
||||
response = method(api, "tenant-1", user, _PROVIDER_ID)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert "context_id" in response.headers.get("Set-Cookie")
|
||||
|
||||
|
||||
class TestDatasourceOAuthCallback:
|
||||
def test_callback_rejects_replaced_package_before_dispatch(
|
||||
self, app: Flask, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
plugin_id = "langgenius/notion_datasource"
|
||||
context = {
|
||||
"user_id": "user-1",
|
||||
"tenant_id": "tenant-1",
|
||||
"plugin_id": plugin_id,
|
||||
"provider": "notion",
|
||||
"plugin_unique_identifier": f"{plugin_id}:1.0.0@{'a' * 64}",
|
||||
}
|
||||
config = PluginOAuthClientConfig(f"{plugin_id}:2.0.0@{'b' * 64}", {"client_secret": "test-secret"})
|
||||
monkeypatch.setattr(OAuthProxyService, "use_proxy_context", lambda _context_id: context)
|
||||
monkeypatch.setattr(DatasourceProviderService, "get_oauth_client", lambda *_args, **_kwargs: config)
|
||||
|
||||
def unexpected_dispatch(*_args: object, **_kwargs: object) -> None:
|
||||
raise AssertionError("Changed packages must not receive OAuth credentials")
|
||||
|
||||
monkeypatch.setattr(OAuthHandler, "get_credentials", unexpected_dispatch)
|
||||
api = DatasourceOAuthCallback()
|
||||
with app.test_request_context("/?context_id=ctx"):
|
||||
with pytest.raises(Forbidden, match="Plugin changed"):
|
||||
inspect.unwrap(api.get)(api, _PROVIDER_ID)
|
||||
|
||||
def test_callback_success_new_credential(self, app: Flask):
|
||||
api = DatasourceOAuthCallback()
|
||||
method = inspect.unwrap(api.get)
|
||||
|
||||
oauth_response = MagicMock()
|
||||
oauth_response.credentials = {"token": "abc"}
|
||||
expires_at = datetime(2024, 1, 2, 3, 4, 5, tzinfo=UTC)
|
||||
oauth_response.expires_at = expires_at
|
||||
oauth_response.metadata = {"name": "Workspace Bot", "avatar_url": "https://avatar.example.com/bot.png"}
|
||||
|
||||
context = {
|
||||
"user_id": "user-1",
|
||||
"tenant_id": "tenant-1",
|
||||
"plugin_id": "langgenius/notion_datasource",
|
||||
"provider": "notion",
|
||||
"credential_id": None,
|
||||
}
|
||||
|
||||
with (
|
||||
app.test_request_context("/?context_id=ctx"),
|
||||
patch.object(
|
||||
OAuthProxyService,
|
||||
"use_proxy_context",
|
||||
return_value=context,
|
||||
),
|
||||
patch.object(
|
||||
DatasourceProviderService,
|
||||
"get_oauth_client",
|
||||
return_value=PluginOAuthClientConfig(
|
||||
"langgenius/notion_datasource:1.0.0@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
{"client_id": "abc", "client_secret": "secret"},
|
||||
),
|
||||
),
|
||||
patch.object(
|
||||
OAuthHandler,
|
||||
"get_credentials",
|
||||
return_value=oauth_response,
|
||||
),
|
||||
patch.object(
|
||||
DatasourceProviderService,
|
||||
"add_datasource_oauth_provider",
|
||||
return_value=None,
|
||||
) as add_oauth_provider,
|
||||
):
|
||||
response = method(api, _PROVIDER_ID)
|
||||
|
||||
assert response.status_code == 302
|
||||
assert "/oauth-callback" in response.location
|
||||
add_oauth_provider.assert_called_once()
|
||||
# Legacy context without a visibility key falls back to ONLY_ME, and
|
||||
# the callback now also propagates the creator's user_id.
|
||||
from models.enums import PermissionEnum
|
||||
|
||||
assert add_oauth_provider.call_args.kwargs == {
|
||||
"tenant_id": "tenant-1",
|
||||
"provider_id": add_oauth_provider.call_args.kwargs["provider_id"],
|
||||
"avatar_url": "https://avatar.example.com/bot.png",
|
||||
"name": "Workspace Bot",
|
||||
"expire_at": expires_at,
|
||||
"credentials": {"token": "abc"},
|
||||
"user_id": "user-1",
|
||||
"visibility": PermissionEnum.ONLY_ME,
|
||||
}
|
||||
assert str(add_oauth_provider.call_args.kwargs["provider_id"]) == _PROVIDER_ID
|
||||
|
||||
def test_callback_missing_context(self, app: Flask):
|
||||
api = DatasourceOAuthCallback()
|
||||
method = inspect.unwrap(api.get)
|
||||
|
||||
with app.test_request_context("/"):
|
||||
with pytest.raises(Forbidden):
|
||||
method(api, "notion")
|
||||
|
||||
def test_callback_invalid_context(self, app: Flask):
|
||||
api = DatasourceOAuthCallback()
|
||||
method = inspect.unwrap(api.get)
|
||||
|
||||
with (
|
||||
app.test_request_context("/?context_id=bad"),
|
||||
patch.object(
|
||||
OAuthProxyService,
|
||||
"use_proxy_context",
|
||||
return_value=None,
|
||||
),
|
||||
):
|
||||
with pytest.raises(Forbidden):
|
||||
method(api, "notion")
|
||||
|
||||
def test_callback_oauth_config_not_found(self, app: Flask):
|
||||
api = DatasourceOAuthCallback()
|
||||
method = inspect.unwrap(api.get)
|
||||
|
||||
context = {"user_id": "u", "tenant_id": "t"}
|
||||
|
||||
with (
|
||||
app.test_request_context("/?context_id=ctx"),
|
||||
patch.object(
|
||||
OAuthProxyService,
|
||||
"use_proxy_context",
|
||||
return_value=context,
|
||||
),
|
||||
patch.object(
|
||||
DatasourceProviderService,
|
||||
"get_oauth_client",
|
||||
return_value=None,
|
||||
),
|
||||
):
|
||||
with pytest.raises(NotFound):
|
||||
method(api, "notion")
|
||||
|
||||
def test_callback_reauthorize_existing_credential(self, app: Flask):
|
||||
api = DatasourceOAuthCallback()
|
||||
method = inspect.unwrap(api.get)
|
||||
|
||||
oauth_response = MagicMock()
|
||||
oauth_response.credentials = {"token": "abc"}
|
||||
oauth_response.expires_at = None
|
||||
oauth_response.metadata = {} # avatar + name missing
|
||||
|
||||
context = {
|
||||
"user_id": "user-1",
|
||||
"tenant_id": "tenant-1",
|
||||
"plugin_id": "langgenius/notion_datasource",
|
||||
"provider": "notion",
|
||||
"credential_id": "cred-1",
|
||||
}
|
||||
|
||||
with (
|
||||
app.test_request_context("/?context_id=ctx"),
|
||||
patch.object(
|
||||
OAuthProxyService,
|
||||
"use_proxy_context",
|
||||
return_value=context,
|
||||
),
|
||||
patch.object(
|
||||
DatasourceProviderService,
|
||||
"get_oauth_client",
|
||||
return_value=PluginOAuthClientConfig(
|
||||
"langgenius/notion_datasource:1.0.0@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", {"client_id": "abc"}
|
||||
),
|
||||
),
|
||||
patch.object(
|
||||
OAuthHandler,
|
||||
"get_credentials",
|
||||
return_value=oauth_response,
|
||||
),
|
||||
patch.object(
|
||||
DatasourceProviderService,
|
||||
"reauthorize_datasource_oauth_provider",
|
||||
return_value=None,
|
||||
) as reauthorize_provider,
|
||||
):
|
||||
response = method(api, _PROVIDER_ID)
|
||||
|
||||
assert response.status_code == 302
|
||||
assert "/oauth-callback" in response.location
|
||||
reauthorize_provider.assert_called_once()
|
||||
assert str(reauthorize_provider.call_args.kwargs["provider_id"]) == _PROVIDER_ID
|
||||
assert reauthorize_provider.call_args.kwargs["credential_id"] == "cred-1"
|
||||
assert reauthorize_provider.call_args.kwargs["credentials"] == {"token": "abc"}
|
||||
|
||||
def test_callback_context_id_from_cookie(self, app: Flask):
|
||||
api = DatasourceOAuthCallback()
|
||||
method = inspect.unwrap(api.get)
|
||||
|
||||
oauth_response = MagicMock()
|
||||
oauth_response.credentials = {"token": "abc"}
|
||||
oauth_response.expires_at = None
|
||||
oauth_response.metadata = {}
|
||||
|
||||
context = {
|
||||
"user_id": "user-1",
|
||||
"tenant_id": "tenant-1",
|
||||
"plugin_id": "langgenius/notion_datasource",
|
||||
"provider": "notion",
|
||||
"credential_id": None,
|
||||
}
|
||||
|
||||
with (
|
||||
app.test_request_context("/", headers={"Cookie": "context_id=ctx"}),
|
||||
patch.object(
|
||||
OAuthProxyService,
|
||||
"use_proxy_context",
|
||||
return_value=context,
|
||||
),
|
||||
patch.object(
|
||||
DatasourceProviderService,
|
||||
"get_oauth_client",
|
||||
return_value=PluginOAuthClientConfig(
|
||||
"langgenius/notion_datasource:1.0.0@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", {"client_id": "abc"}
|
||||
),
|
||||
),
|
||||
patch.object(
|
||||
OAuthHandler,
|
||||
"get_credentials",
|
||||
return_value=oauth_response,
|
||||
),
|
||||
patch.object(
|
||||
DatasourceProviderService,
|
||||
"add_datasource_oauth_provider",
|
||||
return_value=None,
|
||||
),
|
||||
):
|
||||
response = method(api, _PROVIDER_ID)
|
||||
|
||||
assert response.status_code == 302
|
||||
|
||||
|
||||
class TestDatasourceAuth:
|
||||
def test_post_success(self, app: Flask):
|
||||
api = DatasourceAuth()
|
||||
method = inspect.unwrap(api.post)
|
||||
|
||||
payload = {
|
||||
"name": "Engineering Notion",
|
||||
"credentials": {
|
||||
"api_key": "secret-token",
|
||||
"workspace": "engineering",
|
||||
"database_id": "db-123",
|
||||
},
|
||||
}
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch.object(type(console_ns), "payload", payload),
|
||||
patch.object(
|
||||
DatasourceProviderService,
|
||||
"add_datasource_api_key_provider",
|
||||
return_value=None,
|
||||
) as add_api_key_provider,
|
||||
):
|
||||
req_data = DatasourceCredentialPayload.model_validate(payload)
|
||||
response, status = method(api, req_data, "tenant-1", _PROVIDER_ID)
|
||||
|
||||
assert response == _success_response()
|
||||
assert status == 200
|
||||
add_api_key_provider.assert_called_once()
|
||||
assert add_api_key_provider.call_args.kwargs["tenant_id"] == "tenant-1"
|
||||
assert str(add_api_key_provider.call_args.kwargs["provider_id"]) == _PROVIDER_ID
|
||||
assert add_api_key_provider.call_args.kwargs["credentials"] == payload["credentials"]
|
||||
assert add_api_key_provider.call_args.kwargs["name"] == "Engineering Notion"
|
||||
|
||||
def test_post_invalid_credentials(self, app: Flask):
|
||||
api = DatasourceAuth()
|
||||
method = inspect.unwrap(api.post)
|
||||
|
||||
payload = {"credentials": {"key": "bad"}}
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch.object(type(console_ns), "payload", payload),
|
||||
patch.object(
|
||||
DatasourceProviderService,
|
||||
"add_datasource_api_key_provider",
|
||||
side_effect=CredentialsValidateFailedError("invalid"),
|
||||
),
|
||||
):
|
||||
with pytest.raises(ValueError, match="invalid"):
|
||||
method(api, DatasourceCredentialPayload.model_validate(payload), "tenant-1", "notion")
|
||||
|
||||
def test_get_success(self, app: Flask):
|
||||
api = DatasourceAuth()
|
||||
method = inspect.unwrap(api.get)
|
||||
user = _account()
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch.object(
|
||||
DatasourceProviderService,
|
||||
"list_datasource_credentials",
|
||||
return_value=[_datasource_credential()],
|
||||
),
|
||||
):
|
||||
response, status = method(api, "tenant-1", user, _PROVIDER_ID)
|
||||
|
||||
assert status == 200
|
||||
assert response == {"result": [_datasource_credential()]}
|
||||
|
||||
def test_post_missing_credentials(self, app: Flask):
|
||||
api = DatasourceAuth()
|
||||
method = inspect.unwrap(api.post)
|
||||
|
||||
payload: dict[str, object] = {}
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch.object(type(console_ns), "payload", payload),
|
||||
):
|
||||
with pytest.raises(ValueError, match="1 validation error for DatasourceCredentialPayload\\\ncredentials"):
|
||||
method(api, DatasourceCredentialPayload.model_validate(payload), "tenant-1", "notion")
|
||||
|
||||
def test_get_empty_list(self, app: Flask):
|
||||
api = DatasourceAuth()
|
||||
method = inspect.unwrap(api.get)
|
||||
user = _account()
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch.object(
|
||||
DatasourceProviderService,
|
||||
"list_datasource_credentials",
|
||||
return_value=[],
|
||||
),
|
||||
):
|
||||
response, status = method(api, "tenant-1", user, "notion")
|
||||
|
||||
assert status == 200
|
||||
assert response["result"] == []
|
||||
|
||||
|
||||
class TestDatasourceAuthDeleteApi:
|
||||
def test_delete_success(self, app: Flask):
|
||||
api = DatasourceAuthDeleteApi()
|
||||
method = inspect.unwrap(api.post)
|
||||
|
||||
payload = {"credential_id": "cred-1"}
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch.object(type(console_ns), "payload", payload),
|
||||
patch.object(
|
||||
DatasourceProviderService,
|
||||
"remove_datasource_credentials",
|
||||
return_value=None,
|
||||
) as remove_datasource_credentials,
|
||||
):
|
||||
req_data = DatasourceCredentialDeletePayload.model_validate(payload)
|
||||
response, status = method(api, req_data, "tenant-1", _PROVIDER_ID)
|
||||
|
||||
assert response == _success_response()
|
||||
assert status == 200
|
||||
remove_datasource_credentials.assert_called_once_with(
|
||||
tenant_id="tenant-1",
|
||||
auth_id="cred-1",
|
||||
provider="notion",
|
||||
plugin_id="langgenius/notion_datasource",
|
||||
session=ANY,
|
||||
)
|
||||
|
||||
def test_delete_missing_credential_id(self, app: Flask):
|
||||
api = DatasourceAuthDeleteApi()
|
||||
method = inspect.unwrap(api.post)
|
||||
|
||||
payload: dict[str, object] = {}
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch.object(type(console_ns), "payload", payload),
|
||||
):
|
||||
with pytest.raises(
|
||||
ValueError, match="1 validation error for DatasourceCredentialDeletePayload\\\ncredential_id"
|
||||
):
|
||||
method(api, DatasourceCredentialDeletePayload.model_validate(payload), "tenant-1", "notion")
|
||||
|
||||
|
||||
class TestDatasourceAuthUpdateApi:
|
||||
def test_update_success(self, app: Flask):
|
||||
api = DatasourceAuthUpdateApi()
|
||||
method = inspect.unwrap(api.post)
|
||||
|
||||
payload = {
|
||||
"credential_id": "cred-1",
|
||||
"name": "Updated Notion",
|
||||
"credentials": {"api_key": "new-secret", "database_id": "db-456"},
|
||||
}
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch.object(type(console_ns), "payload", payload),
|
||||
patch.object(
|
||||
DatasourceProviderService,
|
||||
"update_datasource_credentials",
|
||||
return_value=None,
|
||||
) as update_datasource_credentials,
|
||||
):
|
||||
req_data = DatasourceCredentialUpdatePayload.model_validate(payload)
|
||||
response, status = method(api, req_data, "tenant-1", _PROVIDER_ID)
|
||||
|
||||
assert response == _success_response()
|
||||
assert status == 201
|
||||
update_datasource_credentials.assert_called_once_with(
|
||||
tenant_id="tenant-1",
|
||||
auth_id="cred-1",
|
||||
provider="notion",
|
||||
plugin_id="langgenius/notion_datasource",
|
||||
credentials=payload["credentials"],
|
||||
name="Updated Notion",
|
||||
)
|
||||
|
||||
def test_update_with_credentials_none(self, app: Flask):
|
||||
api = DatasourceAuthUpdateApi()
|
||||
method = inspect.unwrap(api.post)
|
||||
|
||||
payload = {"credential_id": "id", "credentials": None}
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch.object(type(console_ns), "payload", payload),
|
||||
patch.object(
|
||||
DatasourceProviderService,
|
||||
"update_datasource_credentials",
|
||||
return_value=None,
|
||||
) as update_mock,
|
||||
):
|
||||
req_data = DatasourceCredentialUpdatePayload.model_validate(payload)
|
||||
response, status = method(api, req_data, "tenant-1", "notion")
|
||||
|
||||
assert response == _success_response()
|
||||
update_mock.assert_called_once()
|
||||
assert update_mock.call_args.kwargs["credentials"] == {}
|
||||
assert status == 201
|
||||
|
||||
def test_update_name_only(self, app: Flask):
|
||||
api = DatasourceAuthUpdateApi()
|
||||
method = inspect.unwrap(api.post)
|
||||
|
||||
payload = {"credential_id": "id", "name": "New Name"}
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch.object(type(console_ns), "payload", payload),
|
||||
patch.object(
|
||||
DatasourceProviderService,
|
||||
"update_datasource_credentials",
|
||||
return_value=None,
|
||||
),
|
||||
):
|
||||
req_data = DatasourceCredentialUpdatePayload.model_validate(payload)
|
||||
response, status = method(api, req_data, "tenant-1", "notion")
|
||||
|
||||
assert response == _success_response()
|
||||
assert status == 201
|
||||
|
||||
def test_update_with_empty_credentials_dict(self, app: Flask):
|
||||
api = DatasourceAuthUpdateApi()
|
||||
method = inspect.unwrap(api.post)
|
||||
|
||||
payload = {"credential_id": "id", "credentials": {}}
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch.object(type(console_ns), "payload", payload),
|
||||
patch.object(
|
||||
DatasourceProviderService,
|
||||
"update_datasource_credentials",
|
||||
return_value=None,
|
||||
) as update_mock,
|
||||
):
|
||||
req_data = DatasourceCredentialUpdatePayload.model_validate(payload)
|
||||
response, status = method(api, req_data, "tenant-1", "notion")
|
||||
|
||||
assert response == _success_response()
|
||||
update_mock.assert_called_once()
|
||||
assert status == 201
|
||||
|
||||
|
||||
class TestDatasourceAuthListApi:
|
||||
def test_list_success(self, app: Flask):
|
||||
api = DatasourceAuthListApi()
|
||||
method = inspect.unwrap(api.get)
|
||||
user = MagicMock(id="user-1")
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch.object(
|
||||
DatasourceProviderService,
|
||||
"get_all_datasource_credentials",
|
||||
return_value=[_datasource_auth()],
|
||||
) as get_all,
|
||||
):
|
||||
response, status = method(api, "tenant-1", user)
|
||||
|
||||
assert status == 200
|
||||
assert response == {"result": [_datasource_auth()]}
|
||||
# user is threaded through so list_datasource_credentials applies the
|
||||
# visibility filter for the current viewer.
|
||||
assert get_all.call_args.kwargs["user"] is user
|
||||
|
||||
def test_auth_list_empty(self, app: Flask):
|
||||
api = DatasourceAuthListApi()
|
||||
method = inspect.unwrap(api.get)
|
||||
user = MagicMock(id="user-1")
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch.object(
|
||||
DatasourceProviderService,
|
||||
"get_all_datasource_credentials",
|
||||
return_value=[],
|
||||
),
|
||||
):
|
||||
response, status = method(api, "tenant-1", user)
|
||||
|
||||
assert status == 200
|
||||
assert response["result"] == []
|
||||
|
||||
def test_hardcode_list_empty(self, app: Flask):
|
||||
api = DatasourceHardCodeAuthListApi()
|
||||
method = inspect.unwrap(api.get)
|
||||
user = MagicMock(id="user-1")
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch.object(
|
||||
DatasourceProviderService,
|
||||
"get_hard_code_datasource_credentials",
|
||||
return_value=[],
|
||||
),
|
||||
):
|
||||
response, status = method(api, "tenant-1", user)
|
||||
|
||||
assert status == 200
|
||||
assert response["result"] == []
|
||||
|
||||
|
||||
class TestDatasourceHardCodeAuthListApi:
|
||||
def test_list_success(self, app: Flask):
|
||||
api = DatasourceHardCodeAuthListApi()
|
||||
method = inspect.unwrap(api.get)
|
||||
user = MagicMock(id="user-1")
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch.object(
|
||||
DatasourceProviderService,
|
||||
"get_hard_code_datasource_credentials",
|
||||
return_value=[_datasource_auth()],
|
||||
) as get_hardcode,
|
||||
):
|
||||
response, status = method(api, "tenant-1", user)
|
||||
|
||||
assert status == 200
|
||||
assert get_hardcode.call_args.kwargs["user"] is user
|
||||
|
||||
|
||||
class TestDatasourceAuthOauthCustomClient:
|
||||
def test_post_success(self, app: Flask):
|
||||
api = DatasourceAuthOauthCustomClient()
|
||||
method = inspect.unwrap(api.post)
|
||||
|
||||
payload = {
|
||||
"client_params": {
|
||||
"client_id": "custom-client",
|
||||
"client_secret": "custom-secret",
|
||||
"authorize_url": "https://auth.example.com/authorize",
|
||||
},
|
||||
"enable_oauth_custom_client": True,
|
||||
}
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch.object(type(console_ns), "payload", payload),
|
||||
patch.object(
|
||||
DatasourceProviderService,
|
||||
"setup_oauth_custom_client_params",
|
||||
return_value=None,
|
||||
) as setup_custom_client,
|
||||
):
|
||||
req_data = DatasourceCustomClientPayload.model_validate(payload)
|
||||
response, status = method(api, req_data, "tenant-1", _PROVIDER_ID)
|
||||
|
||||
assert response == _success_response()
|
||||
assert status == 200
|
||||
setup_custom_client.assert_called_once()
|
||||
assert setup_custom_client.call_args.kwargs["tenant_id"] == "tenant-1"
|
||||
assert str(setup_custom_client.call_args.kwargs["datasource_provider_id"]) == _PROVIDER_ID
|
||||
assert setup_custom_client.call_args.kwargs["client_params"] == payload["client_params"]
|
||||
assert setup_custom_client.call_args.kwargs["enabled"] is True
|
||||
|
||||
def test_delete_success(self, app: Flask):
|
||||
api = DatasourceAuthOauthCustomClient()
|
||||
method = inspect.unwrap(api.delete)
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch.object(
|
||||
DatasourceProviderService,
|
||||
"remove_oauth_custom_client_params",
|
||||
return_value=None,
|
||||
) as remove_custom_client,
|
||||
):
|
||||
response, status = method(api, "tenant-1", _PROVIDER_ID)
|
||||
|
||||
assert response == _success_response()
|
||||
assert status == 200
|
||||
remove_custom_client.assert_called_once()
|
||||
assert str(remove_custom_client.call_args.kwargs["datasource_provider_id"]) == _PROVIDER_ID
|
||||
|
||||
def test_post_empty_payload(self, app: Flask):
|
||||
api = DatasourceAuthOauthCustomClient()
|
||||
method = inspect.unwrap(api.post)
|
||||
|
||||
payload: dict[str, object] = {}
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch.object(type(console_ns), "payload", payload),
|
||||
patch.object(
|
||||
DatasourceProviderService,
|
||||
"setup_oauth_custom_client_params",
|
||||
return_value=None,
|
||||
),
|
||||
):
|
||||
req_data = DatasourceCustomClientPayload.model_validate(payload)
|
||||
response, status = method(api, req_data, "tenant-1", "notion")
|
||||
|
||||
assert response == _success_response()
|
||||
assert status == 200
|
||||
|
||||
def test_post_disabled_flag(self, app: Flask):
|
||||
api = DatasourceAuthOauthCustomClient()
|
||||
method = inspect.unwrap(api.post)
|
||||
|
||||
payload = {
|
||||
"client_params": {"a": 1},
|
||||
"enable_oauth_custom_client": False,
|
||||
}
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch.object(type(console_ns), "payload", payload),
|
||||
patch.object(
|
||||
DatasourceProviderService,
|
||||
"setup_oauth_custom_client_params",
|
||||
return_value=None,
|
||||
) as setup_mock,
|
||||
):
|
||||
req_data = DatasourceCustomClientPayload.model_validate(payload)
|
||||
response, status = method(api, req_data, "tenant-1", "notion")
|
||||
|
||||
assert response == _success_response()
|
||||
setup_mock.assert_called_once()
|
||||
assert setup_mock.call_args.kwargs["client_params"] == {"a": 1}
|
||||
assert setup_mock.call_args.kwargs["enabled"] is False
|
||||
assert status == 200
|
||||
|
||||
|
||||
class TestDatasourceAuthDefaultApi:
|
||||
def test_set_default_success(self, app: Flask):
|
||||
api = DatasourceAuthDefaultApi()
|
||||
method = inspect.unwrap(api.post)
|
||||
|
||||
payload = {"id": "cred-1"}
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch.object(type(console_ns), "payload", payload),
|
||||
patch.object(
|
||||
DatasourceProviderService,
|
||||
"set_default_datasource_provider",
|
||||
return_value=None,
|
||||
) as set_default_datasource_provider,
|
||||
):
|
||||
req_data = DatasourceDefaultPayload.model_validate(payload)
|
||||
response, status = method(api, req_data, "tenant-1", _PROVIDER_ID)
|
||||
|
||||
assert response == _success_response()
|
||||
assert status == 200
|
||||
set_default_datasource_provider.assert_called_once()
|
||||
assert set_default_datasource_provider.call_args.kwargs["tenant_id"] == "tenant-1"
|
||||
assert str(set_default_datasource_provider.call_args.kwargs["datasource_provider_id"]) == _PROVIDER_ID
|
||||
assert set_default_datasource_provider.call_args.kwargs["credential_id"] == "cred-1"
|
||||
|
||||
def test_default_missing_id(self, app: Flask):
|
||||
api = DatasourceAuthDefaultApi()
|
||||
method = inspect.unwrap(api.post)
|
||||
|
||||
payload: dict[str, object] = {}
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch.object(type(console_ns), "payload", payload),
|
||||
):
|
||||
with pytest.raises(ValueError, match="1 validation error for DatasourceDefaultPayload\\\nid"):
|
||||
method(api, DatasourceDefaultPayload.model_validate(payload), "tenant-1", "notion")
|
||||
|
||||
|
||||
class TestDatasourceUpdateProviderNameApi:
|
||||
def test_update_name_success(self, app: Flask):
|
||||
api = DatasourceUpdateProviderNameApi()
|
||||
method = inspect.unwrap(api.post)
|
||||
|
||||
payload = {"credential_id": "cred-1", "name": "New Name"}
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch.object(type(console_ns), "payload", payload),
|
||||
patch.object(
|
||||
DatasourceProviderService,
|
||||
"update_datasource_provider_name",
|
||||
return_value=None,
|
||||
) as update_datasource_provider_name,
|
||||
):
|
||||
req_data = DatasourceUpdateNamePayload.model_validate(payload)
|
||||
response, status = method(api, req_data, "tenant-1", _PROVIDER_ID)
|
||||
|
||||
assert response == _success_response()
|
||||
assert status == 200
|
||||
update_datasource_provider_name.assert_called_once()
|
||||
assert update_datasource_provider_name.call_args.kwargs["tenant_id"] == "tenant-1"
|
||||
assert str(update_datasource_provider_name.call_args.kwargs["datasource_provider_id"]) == _PROVIDER_ID
|
||||
assert update_datasource_provider_name.call_args.kwargs["name"] == "New Name"
|
||||
assert update_datasource_provider_name.call_args.kwargs["credential_id"] == "cred-1"
|
||||
|
||||
def test_update_name_too_long(self, app: Flask):
|
||||
api = DatasourceUpdateProviderNameApi()
|
||||
method = inspect.unwrap(api.post)
|
||||
|
||||
payload = {
|
||||
"credential_id": "id",
|
||||
"name": "x" * 101,
|
||||
}
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch.object(type(console_ns), "payload", payload),
|
||||
):
|
||||
with pytest.raises(ValueError, match="1 validation error for DatasourceUpdateNamePayload\\\nname"):
|
||||
method(api, DatasourceUpdateNamePayload.model_validate(payload), "tenant-1", "notion")
|
||||
|
||||
def test_update_name_missing_credential_id(self, app: Flask):
|
||||
api = DatasourceUpdateProviderNameApi()
|
||||
method = inspect.unwrap(api.post)
|
||||
|
||||
payload = {"name": "Valid"}
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch.object(type(console_ns), "payload", payload),
|
||||
):
|
||||
with pytest.raises(ValueError, match="1 validation error for DatasourceUpdateNamePayload\\\ncredential_id"):
|
||||
method(api, DatasourceUpdateNamePayload.model_validate(payload), "tenant-1", "notion")
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
from inspect import unwrap
|
||||
from unittest.mock import ANY, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
|
||||
from controllers.console import console_ns
|
||||
from controllers.console.datasets.rag_pipeline.datasource_content_preview import (
|
||||
DataSourceContentPreviewApi,
|
||||
DatasourceContentPreviewPayload,
|
||||
)
|
||||
from models import Account
|
||||
from models.dataset import Pipeline
|
||||
|
||||
|
||||
def make_account() -> Account:
|
||||
account = Account(name="Test User", email="user@example.com")
|
||||
account.id = "account-1"
|
||||
return account
|
||||
|
||||
|
||||
class TestDataSourceContentPreviewApi:
|
||||
def _valid_payload(self) -> dict[str, object]:
|
||||
return {
|
||||
"inputs": {"query": "hello"},
|
||||
"datasource_type": "notion",
|
||||
"credential_id": "cred-1",
|
||||
}
|
||||
|
||||
def test_post_success(self, app: Flask) -> None:
|
||||
api = DataSourceContentPreviewApi()
|
||||
method = unwrap(api.post)
|
||||
|
||||
payload = self._valid_payload()
|
||||
|
||||
pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")
|
||||
node_id = "node-1"
|
||||
account = make_account()
|
||||
|
||||
preview_result = {"content": "preview data"}
|
||||
|
||||
service_instance = MagicMock()
|
||||
service_instance.run_datasource_node_preview.return_value = preview_result
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch.object(type(console_ns), "payload", payload),
|
||||
patch(
|
||||
"controllers.console.datasets.rag_pipeline.datasource_content_preview.RagPipelineService",
|
||||
return_value=service_instance,
|
||||
),
|
||||
):
|
||||
req_data = DatasourceContentPreviewPayload.model_validate(payload)
|
||||
response, status = method(api, req_data, account, pipeline, node_id)
|
||||
|
||||
service_instance.run_datasource_node_preview.assert_called_once_with(
|
||||
pipeline=pipeline,
|
||||
node_id=node_id,
|
||||
user_inputs=payload["inputs"],
|
||||
account=account,
|
||||
datasource_type=payload["datasource_type"],
|
||||
is_published=True,
|
||||
credential_id=payload["credential_id"],
|
||||
datasource_providers=ANY,
|
||||
)
|
||||
assert status == 200
|
||||
assert response == preview_result
|
||||
|
||||
def test_post_invalid_payload(self, app: Flask) -> None:
|
||||
api = DataSourceContentPreviewApi()
|
||||
method = unwrap(api.post)
|
||||
|
||||
payload = {
|
||||
"inputs": {"query": "hello"},
|
||||
# datasource_type missing
|
||||
}
|
||||
|
||||
pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")
|
||||
account = make_account()
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch.object(type(console_ns), "payload", payload),
|
||||
):
|
||||
with pytest.raises(ValueError, match=r"datasource_type\n\s+Field required"):
|
||||
method(api, DatasourceContentPreviewPayload.model_validate(payload), account, pipeline, "node-1")
|
||||
|
||||
def test_post_without_credential_id(self, app: Flask) -> None:
|
||||
api = DataSourceContentPreviewApi()
|
||||
method = unwrap(api.post)
|
||||
|
||||
payload = {
|
||||
"inputs": {"query": "hello"},
|
||||
"datasource_type": "notion",
|
||||
"credential_id": None,
|
||||
}
|
||||
|
||||
pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")
|
||||
account = make_account()
|
||||
|
||||
service_instance = MagicMock()
|
||||
service_instance.run_datasource_node_preview.return_value = {"ok": True}
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch.object(type(console_ns), "payload", payload),
|
||||
patch(
|
||||
"controllers.console.datasets.rag_pipeline.datasource_content_preview.RagPipelineService",
|
||||
return_value=service_instance,
|
||||
),
|
||||
):
|
||||
req_data = DatasourceContentPreviewPayload.model_validate(payload)
|
||||
response, status = method(api, req_data, account, pipeline, "node-1")
|
||||
|
||||
service_instance.run_datasource_node_preview.assert_called_once()
|
||||
assert status == 200
|
||||
assert response == {"ok": True}
|
||||
+513
@@ -0,0 +1,513 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterator
|
||||
from inspect import unwrap
|
||||
from unittest.mock import ANY, PropertyMock, patch
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
from sqlalchemy import Engine
|
||||
from sqlalchemy.orm import Session
|
||||
from werkzeug.exceptions import Forbidden, NotFound
|
||||
|
||||
from controllers.console import console_ns
|
||||
from controllers.console.datasets.rag_pipeline import rag_pipeline as module
|
||||
from controllers.console.datasets.rag_pipeline.rag_pipeline import (
|
||||
CustomizedPipelineTemplateApi,
|
||||
CustomizedPipelineTemplatePayload,
|
||||
PipelineTemplateDetailApi,
|
||||
PipelineTemplateDetailQuery,
|
||||
PipelineTemplateListApi,
|
||||
PipelineTemplateListQuery,
|
||||
PublishCustomizedPipelineTemplateApi,
|
||||
)
|
||||
from models.account import Account, TenantAccountRole
|
||||
from models.dataset import Pipeline, PipelineCustomizedTemplate
|
||||
from models.engine import db
|
||||
from services.entities.knowledge_entities.rag_pipeline_entities import PipelineTemplateInfoEntity
|
||||
from services.errors.base import NoPermissionError
|
||||
from services.errors.rag_pipeline import RagPipelineResourceNotFoundError
|
||||
from tests.helpers.unit.config_override import config_overrides_context
|
||||
from tests.helpers.unit.controllers.rbac_introspection import rbac_checks
|
||||
from tests.helpers.unit.model_factories import make_account
|
||||
|
||||
|
||||
def _template_item() -> dict[str, object]:
|
||||
return {
|
||||
"id": "template-1",
|
||||
"name": "Template",
|
||||
"icon": {"icon": "book", "icon_type": "emoji", "icon_background": "#fff"},
|
||||
"description": "Description",
|
||||
"position": 1,
|
||||
"chunk_structure": "general",
|
||||
}
|
||||
|
||||
|
||||
def _template_detail() -> dict[str, object]:
|
||||
return {
|
||||
"id": "template-1",
|
||||
"name": "Template",
|
||||
"icon_info": {"icon": "book", "icon_type": "emoji", "icon_background": "#fff"},
|
||||
"description": "Description",
|
||||
"chunk_structure": "general",
|
||||
"export_data": "dsl: value",
|
||||
"graph": {"nodes": [], "edges": [], "viewport": {"x": 0, "y": 0, "zoom": 1}},
|
||||
}
|
||||
|
||||
|
||||
def _payload() -> dict[str, object]:
|
||||
return {
|
||||
"name": "Updated template",
|
||||
"description": "Updated description",
|
||||
"icon_info": {"icon": "book", "icon_type": "emoji", "icon_background": "#fff"},
|
||||
}
|
||||
|
||||
|
||||
def _account() -> Account:
|
||||
return make_account(name="Test User", email="test@example.com")
|
||||
|
||||
|
||||
def _pipeline() -> Pipeline:
|
||||
pipeline = Pipeline(tenant_id="tenant-1", name="Pipeline")
|
||||
pipeline.id = "pipeline-1"
|
||||
return pipeline
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def database_app() -> Iterator[Flask]:
|
||||
app = Flask(__name__)
|
||||
app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///:memory:"
|
||||
db.init_app(app)
|
||||
|
||||
with app.app_context():
|
||||
PipelineCustomizedTemplate.__table__.create(db.engine)
|
||||
yield app
|
||||
|
||||
|
||||
class TestPipelineTemplateListApi:
|
||||
def test_get_uses_query_defaults_and_serializes_nullable_fields(self, app: Flask, sqlite_engine: Engine) -> None:
|
||||
api = PipelineTemplateListApi()
|
||||
method = unwrap(api.get)
|
||||
tenant_id = "tenant-1"
|
||||
service_calls: list[tuple[str, str, str]] = []
|
||||
|
||||
def get_pipeline_templates(*, type: str, language: str, current_tenant_id: str, session) -> dict[str, object]:
|
||||
del session
|
||||
service_calls.append((type, language, current_tenant_id))
|
||||
return {"pipeline_templates": [_template_item()]}
|
||||
|
||||
with (
|
||||
Session(sqlite_engine) as session,
|
||||
app.test_request_context("/rag/pipeline/templates"),
|
||||
patch.object(module.RagPipelineService, "get_pipeline_templates", side_effect=get_pipeline_templates),
|
||||
):
|
||||
response, status = method(api, PipelineTemplateListQuery(), session, tenant_id)
|
||||
|
||||
assert status == 200
|
||||
assert service_calls == [("built-in", "en-US", tenant_id)]
|
||||
assert response == {
|
||||
"pipeline_templates": [
|
||||
{
|
||||
**_template_item(),
|
||||
"copyright": None,
|
||||
"privacy_policy": None,
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
def test_get_passes_explicit_query_to_service(self, app: Flask, sqlite_engine: Engine) -> None:
|
||||
api = PipelineTemplateListApi()
|
||||
method = unwrap(api.get)
|
||||
tenant_id = "tenant-1"
|
||||
service_calls: list[tuple[str, str, str]] = []
|
||||
|
||||
def get_pipeline_templates(*, type: str, language: str, current_tenant_id: str, session) -> dict[str, object]:
|
||||
del session
|
||||
service_calls.append((type, language, current_tenant_id))
|
||||
return {"pipeline_templates": []}
|
||||
|
||||
with (
|
||||
Session(sqlite_engine) as session,
|
||||
app.test_request_context("/rag/pipeline/templates?type=customized&language=ja-JP"),
|
||||
patch.object(module.RagPipelineService, "get_pipeline_templates", side_effect=get_pipeline_templates),
|
||||
):
|
||||
response, status = method(
|
||||
api, PipelineTemplateListQuery(type="customized", language="ja-JP"), session, tenant_id
|
||||
)
|
||||
|
||||
assert status == 200
|
||||
assert response == {"pipeline_templates": []}
|
||||
assert service_calls == [("customized", "ja-JP", tenant_id)]
|
||||
|
||||
|
||||
class TestPipelineTemplateDetailApi:
|
||||
def test_get_serializes_template_detail(self, app: Flask, sqlite_engine: Engine) -> None:
|
||||
api = PipelineTemplateDetailApi()
|
||||
method = unwrap(api.get)
|
||||
service_calls: list[tuple[str, str, str]] = []
|
||||
|
||||
def get_pipeline_template_detail(
|
||||
template_id: str, current_tenant_id: str, type: str, *, session
|
||||
) -> dict[str, object]:
|
||||
del session
|
||||
service_calls.append((template_id, current_tenant_id, type))
|
||||
return _template_detail()
|
||||
|
||||
with (
|
||||
Session(sqlite_engine) as session,
|
||||
app.test_request_context("/rag/pipeline/templates/template-1?type=customized"),
|
||||
patch.object(
|
||||
module.RagPipelineService,
|
||||
"get_pipeline_template_detail",
|
||||
side_effect=get_pipeline_template_detail,
|
||||
),
|
||||
):
|
||||
response, status = method(
|
||||
api, PipelineTemplateDetailQuery(type="customized"), session, "tenant-1", "template-1"
|
||||
)
|
||||
|
||||
assert status == 200
|
||||
assert response == {**_template_detail(), "created_by": None}
|
||||
assert service_calls == [("template-1", "tenant-1", "customized")]
|
||||
|
||||
def test_get_raises_not_found_without_custom_response_body(self, app: Flask, sqlite_engine: Engine) -> None:
|
||||
api = PipelineTemplateDetailApi()
|
||||
method = unwrap(api.get)
|
||||
|
||||
def get_pipeline_template_detail(template_id: str, current_tenant_id: str, type: str, *, session) -> None:
|
||||
del template_id, current_tenant_id, type, session
|
||||
|
||||
with (
|
||||
Session(sqlite_engine) as session,
|
||||
app.test_request_context("/rag/pipeline/templates/missing"),
|
||||
patch.object(
|
||||
module.RagPipelineService,
|
||||
"get_pipeline_template_detail",
|
||||
side_effect=get_pipeline_template_detail,
|
||||
),
|
||||
pytest.raises(NotFound),
|
||||
):
|
||||
method(api, PipelineTemplateDetailQuery(), session, "tenant-1", "missing")
|
||||
|
||||
|
||||
class TestCustomizedPipelineTemplateApi:
|
||||
def test_patch_validates_payload_and_returns_empty_204(self, app: Flask) -> None:
|
||||
api = CustomizedPipelineTemplateApi()
|
||||
method = unwrap(api.patch)
|
||||
payload = _payload()
|
||||
account = _account()
|
||||
tenant_id = "tenant-1"
|
||||
service_calls: list[tuple[str, PipelineTemplateInfoEntity, Account, str]] = []
|
||||
|
||||
def update_template(
|
||||
template_id: str,
|
||||
template_info: PipelineTemplateInfoEntity,
|
||||
current_user: Account,
|
||||
current_tenant_id: str,
|
||||
*,
|
||||
session,
|
||||
) -> None:
|
||||
del session
|
||||
service_calls.append((template_id, template_info, current_user, current_tenant_id))
|
||||
|
||||
with (
|
||||
app.test_request_context("/rag/pipeline/customized/templates/template-1", method="PATCH", json=payload),
|
||||
patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),
|
||||
patch.object(module.RagPipelineService, "update_customized_pipeline_template", side_effect=update_template),
|
||||
):
|
||||
response, status = method(
|
||||
api, CustomizedPipelineTemplatePayload.model_validate(payload), tenant_id, account, "template-1"
|
||||
)
|
||||
|
||||
assert (response, status) == ("", 204)
|
||||
assert len(service_calls) == 1
|
||||
template_id, template_info, current_user, current_tenant_id = service_calls[0]
|
||||
assert template_id == "template-1"
|
||||
assert current_user is account
|
||||
assert current_tenant_id == tenant_id
|
||||
assert template_info.name == "Updated template"
|
||||
assert template_info.description == "Updated description"
|
||||
assert template_info.icon_info.model_dump() == {
|
||||
"icon": "book",
|
||||
"icon_background": "#fff",
|
||||
"icon_type": "emoji",
|
||||
"icon_url": None,
|
||||
}
|
||||
|
||||
def test_patch_defaults_missing_icon_info_before_service_call(self, app: Flask) -> None:
|
||||
api = CustomizedPipelineTemplateApi()
|
||||
method = unwrap(api.patch)
|
||||
payload: dict[str, object] = {
|
||||
"name": "Updated template",
|
||||
"description": "Updated description",
|
||||
}
|
||||
account = _account()
|
||||
tenant_id = "tenant-1"
|
||||
service_calls: list[tuple[str, PipelineTemplateInfoEntity, Account, str]] = []
|
||||
|
||||
def update_template(
|
||||
template_id: str,
|
||||
template_info: PipelineTemplateInfoEntity,
|
||||
current_user: Account,
|
||||
current_tenant_id: str,
|
||||
*,
|
||||
session,
|
||||
) -> None:
|
||||
del session
|
||||
service_calls.append((template_id, template_info, current_user, current_tenant_id))
|
||||
|
||||
with (
|
||||
app.test_request_context("/rag/pipeline/customized/templates/template-1", method="PATCH", json=payload),
|
||||
patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),
|
||||
patch.object(module.RagPipelineService, "update_customized_pipeline_template", side_effect=update_template),
|
||||
):
|
||||
response, status = method(
|
||||
api, CustomizedPipelineTemplatePayload.model_validate(payload), tenant_id, account, "template-1"
|
||||
)
|
||||
|
||||
assert (response, status) == ("", 204)
|
||||
assert len(service_calls) == 1
|
||||
template_id, template_info, current_user, current_tenant_id = service_calls[0]
|
||||
assert template_id == "template-1"
|
||||
assert current_user is account
|
||||
assert current_tenant_id == tenant_id
|
||||
assert template_info.icon_info.model_dump() == {
|
||||
"icon": "",
|
||||
"icon_background": None,
|
||||
"icon_type": None,
|
||||
"icon_url": None,
|
||||
}
|
||||
|
||||
def test_delete_returns_empty_204(self, app: Flask) -> None:
|
||||
api = CustomizedPipelineTemplateApi()
|
||||
method = unwrap(api.delete)
|
||||
tenant_id = "tenant-1"
|
||||
deleted_templates: list[tuple[str, str]] = []
|
||||
|
||||
def delete_template(template_id: str, current_tenant_id: str, *, session) -> None:
|
||||
del session
|
||||
deleted_templates.append((template_id, current_tenant_id))
|
||||
|
||||
with (
|
||||
app.test_request_context("/rag/pipeline/customized/templates/template-1", method="DELETE"),
|
||||
patch.object(module.RagPipelineService, "delete_customized_pipeline_template", side_effect=delete_template),
|
||||
):
|
||||
response, status = method(api, tenant_id, "template-1")
|
||||
|
||||
assert (response, status) == ("", 204)
|
||||
assert deleted_templates == [("template-1", tenant_id)]
|
||||
|
||||
@pytest.mark.parametrize("sqlite_session", [(PipelineCustomizedTemplate,)], indirect=True)
|
||||
def test_post_exports_yaml_from_orm_template(self, app: Flask, sqlite_session: Session) -> None:
|
||||
api = CustomizedPipelineTemplateApi()
|
||||
method = unwrap(api.post)
|
||||
template = PipelineCustomizedTemplate(
|
||||
tenant_id="00000000-0000-0000-0000-000000000001",
|
||||
name="Template",
|
||||
description="Description",
|
||||
chunk_structure="general",
|
||||
icon={"icon": "book", "icon_type": "emoji", "icon_background": "#fff"},
|
||||
position=1,
|
||||
yaml_content="dsl: value",
|
||||
install_count=0,
|
||||
language="en-US",
|
||||
created_by="00000000-0000-0000-0000-000000000002",
|
||||
)
|
||||
template.id = "template-1"
|
||||
sqlite_session.add(template)
|
||||
sqlite_session.commit()
|
||||
|
||||
with app.test_request_context("/rag/pipeline/customized/templates/template-1", method="POST"):
|
||||
response, status = method(
|
||||
api,
|
||||
sqlite_session,
|
||||
"00000000-0000-0000-0000-000000000001",
|
||||
"template-1",
|
||||
)
|
||||
|
||||
assert status == 200
|
||||
assert response == {"data": "dsl: value"}
|
||||
|
||||
@pytest.mark.parametrize("sqlite_session", [(PipelineCustomizedTemplate,)], indirect=True)
|
||||
def test_post_returns_not_found_for_other_tenant(self, app: Flask, sqlite_session: Session) -> None:
|
||||
api = CustomizedPipelineTemplateApi()
|
||||
method = unwrap(api.post)
|
||||
template = PipelineCustomizedTemplate(
|
||||
tenant_id="00000000-0000-0000-0000-000000000002",
|
||||
name="Other tenant template",
|
||||
description="Description",
|
||||
chunk_structure="general",
|
||||
icon={},
|
||||
position=1,
|
||||
yaml_content="secret: value",
|
||||
install_count=0,
|
||||
language="en-US",
|
||||
created_by="00000000-0000-0000-0000-000000000003",
|
||||
)
|
||||
template.id = "template-1"
|
||||
sqlite_session.add(template)
|
||||
sqlite_session.commit()
|
||||
|
||||
with (
|
||||
app.test_request_context("/rag/pipeline/customized/templates/template-1", method="POST"),
|
||||
pytest.raises(NotFound, match="Customized pipeline template not found"),
|
||||
):
|
||||
method(
|
||||
api,
|
||||
sqlite_session,
|
||||
"00000000-0000-0000-0000-000000000001",
|
||||
"template-1",
|
||||
)
|
||||
|
||||
|
||||
class TestPublishCustomizedPipelineTemplateApi:
|
||||
def test_post_uses_pipeline_release_rbac_scene(self) -> None:
|
||||
[check] = rbac_checks(PublishCustomizedPipelineTemplateApi.post)
|
||||
assert check.scene == module.RBACPermission.DATASET_PIPELINE_RELEASE
|
||||
|
||||
def test_post_validates_payload_and_returns_empty_204(self) -> None:
|
||||
api = PublishCustomizedPipelineTemplateApi()
|
||||
method = unwrap(api.post)
|
||||
payload = _payload()
|
||||
account = _account()
|
||||
pipeline = _pipeline()
|
||||
dataset = object()
|
||||
|
||||
with (
|
||||
config_overrides_context(RBAC_ENABLED=True),
|
||||
patch.object(module, "get_pipeline_dataset", return_value=dataset),
|
||||
patch.object(module.DatasetService, "check_dataset_permission") as legacy_acl,
|
||||
patch.object(module.RagPipelineService, "publish_customized_pipeline_template") as publish,
|
||||
):
|
||||
response, status = method(api, CustomizedPipelineTemplatePayload.model_validate(payload), account, pipeline)
|
||||
|
||||
assert (response, status) == ("", 204)
|
||||
publish.assert_called_once_with(pipeline, dataset, payload, account, session=ANY)
|
||||
legacy_acl.assert_not_called()
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("payload", "expected_icon_info"),
|
||||
[
|
||||
(
|
||||
{"name": "Published template", "description": "Description"},
|
||||
{"icon": "", "icon_background": None, "icon_type": None, "icon_url": None},
|
||||
),
|
||||
({"name": "Published template", "description": "Description", "icon_info": {}}, {}),
|
||||
],
|
||||
)
|
||||
def test_post_preserves_valid_icon_info(
|
||||
self,
|
||||
payload: dict[str, object],
|
||||
expected_icon_info: dict[str, object | None],
|
||||
) -> None:
|
||||
api = PublishCustomizedPipelineTemplateApi()
|
||||
method = unwrap(api.post)
|
||||
account = _account()
|
||||
pipeline = _pipeline()
|
||||
dataset = object()
|
||||
|
||||
with (
|
||||
config_overrides_context(RBAC_ENABLED=True),
|
||||
patch.object(module, "get_pipeline_dataset", return_value=dataset),
|
||||
patch.object(module.RagPipelineService, "publish_customized_pipeline_template") as publish,
|
||||
):
|
||||
response, status = method(api, CustomizedPipelineTemplatePayload.model_validate(payload), account, pipeline)
|
||||
|
||||
assert (response, status) == ("", 204)
|
||||
publish.assert_called_once_with(pipeline, dataset, ANY, account, session=ANY)
|
||||
assert publish.call_args.args[2]["icon_info"] == expected_icon_info
|
||||
|
||||
def test_post_translates_missing_owned_resource_to_not_found(self) -> None:
|
||||
api = PublishCustomizedPipelineTemplateApi()
|
||||
method = unwrap(api.post)
|
||||
payload = _payload()
|
||||
|
||||
with (
|
||||
config_overrides_context(RBAC_ENABLED=True),
|
||||
patch.object(module, "get_pipeline_dataset", return_value=object()),
|
||||
patch.object(
|
||||
module.RagPipelineService,
|
||||
"publish_customized_pipeline_template",
|
||||
side_effect=RagPipelineResourceNotFoundError("Workflow not found"),
|
||||
),
|
||||
pytest.raises(NotFound, match="Workflow not found"),
|
||||
):
|
||||
method(api, CustomizedPipelineTemplatePayload.model_validate(payload), _account(), _pipeline())
|
||||
|
||||
def test_post_allows_legacy_dataset_operator_after_dataset_acl(self) -> None:
|
||||
api = PublishCustomizedPipelineTemplateApi()
|
||||
method = unwrap(api.post)
|
||||
account = _account()
|
||||
account.role = TenantAccountRole.DATASET_OPERATOR
|
||||
pipeline = _pipeline()
|
||||
dataset = object()
|
||||
payload = _payload()
|
||||
|
||||
with (
|
||||
config_overrides_context(RBAC_ENABLED=False),
|
||||
patch.object(module, "get_pipeline_dataset", return_value=dataset),
|
||||
patch.object(module.DatasetService, "check_dataset_permission") as check_permission,
|
||||
patch.object(module.RagPipelineService, "publish_customized_pipeline_template") as publish,
|
||||
):
|
||||
response = method(api, CustomizedPipelineTemplatePayload.model_validate(payload), account, pipeline)
|
||||
|
||||
assert response == ("", 204)
|
||||
assert check_permission.call_args.args[:2] == (dataset, account)
|
||||
publish.assert_called_once_with(pipeline, dataset, payload, account, session=ANY)
|
||||
|
||||
def test_post_rejects_legacy_non_editor_before_dataset_acl(self) -> None:
|
||||
api = PublishCustomizedPipelineTemplateApi()
|
||||
method = unwrap(api.post)
|
||||
account = _account()
|
||||
account.role = TenantAccountRole.NORMAL
|
||||
|
||||
with (
|
||||
config_overrides_context(RBAC_ENABLED=False),
|
||||
patch.object(module, "get_pipeline_dataset", return_value=object()),
|
||||
patch.object(module.DatasetService, "check_dataset_permission") as check_permission,
|
||||
patch.object(module.RagPipelineService, "publish_customized_pipeline_template") as publish,
|
||||
pytest.raises(Forbidden),
|
||||
):
|
||||
method(api, CustomizedPipelineTemplatePayload.model_validate(_payload()), account, _pipeline())
|
||||
|
||||
check_permission.assert_not_called()
|
||||
publish.assert_not_called()
|
||||
|
||||
def test_post_rejects_legacy_dataset_acl_before_publish(self) -> None:
|
||||
api = PublishCustomizedPipelineTemplateApi()
|
||||
method = unwrap(api.post)
|
||||
account = _account()
|
||||
account.role = TenantAccountRole.EDITOR
|
||||
|
||||
with (
|
||||
config_overrides_context(RBAC_ENABLED=False),
|
||||
patch.object(module, "get_pipeline_dataset", return_value=object()),
|
||||
patch.object(
|
||||
module.DatasetService,
|
||||
"check_dataset_permission",
|
||||
side_effect=NoPermissionError("Dataset is private"),
|
||||
),
|
||||
patch.object(module.RagPipelineService, "publish_customized_pipeline_template") as publish,
|
||||
pytest.raises(Forbidden, match="Dataset is private"),
|
||||
):
|
||||
method(api, CustomizedPipelineTemplatePayload.model_validate(_payload()), account, _pipeline())
|
||||
|
||||
publish.assert_not_called()
|
||||
|
||||
def test_post_rejects_missing_legacy_dataset_before_publish(self) -> None:
|
||||
api = PublishCustomizedPipelineTemplateApi()
|
||||
method = unwrap(api.post)
|
||||
account = _account()
|
||||
account.role = TenantAccountRole.EDITOR
|
||||
|
||||
with (
|
||||
config_overrides_context(RBAC_ENABLED=False),
|
||||
patch.object(module, "get_pipeline_dataset", return_value=None),
|
||||
patch.object(module.DatasetService, "check_dataset_permission") as check_permission,
|
||||
patch.object(module.RagPipelineService, "publish_customized_pipeline_template") as publish,
|
||||
pytest.raises(NotFound, match="Dataset not found"),
|
||||
):
|
||||
method(api, CustomizedPipelineTemplatePayload.model_validate(_payload()), account, _pipeline())
|
||||
|
||||
check_permission.assert_not_called()
|
||||
publish.assert_not_called()
|
||||
+189
@@ -0,0 +1,189 @@
|
||||
"""Unit tests for rag_pipeline_datasets controller endpoints."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from inspect import unwrap
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
from werkzeug.exceptions import Forbidden
|
||||
|
||||
import services
|
||||
from controllers.console import console_ns
|
||||
from controllers.console.datasets.error import DatasetNameDuplicateError
|
||||
from controllers.console.datasets.rag_pipeline.rag_pipeline_datasets import (
|
||||
CreateEmptyRagPipelineDatasetApi,
|
||||
CreateRagPipelineDatasetApi,
|
||||
RagPipelineDatasetImportPayload,
|
||||
)
|
||||
from models.account import Account, TenantAccountRole
|
||||
from services.entities.dsl_entities import ImportStatus
|
||||
from services.rbac import contracts as rbac_contracts
|
||||
from tests.helpers.unit.config_override import config_overrides_context
|
||||
from tests.helpers.unit.model_factories import make_account
|
||||
|
||||
|
||||
def _account(*, editor: bool) -> Account:
|
||||
return make_account(
|
||||
account_id=None,
|
||||
name="RAG Pipeline Tester",
|
||||
email="rag-pipeline@example.com",
|
||||
role=TenantAccountRole.EDITOR if editor else TenantAccountRole.NORMAL,
|
||||
)
|
||||
|
||||
|
||||
class TestCreateRagPipelineDatasetApi:
|
||||
def _valid_payload(self) -> dict[str, str]:
|
||||
return {"yaml_content": "name: test"}
|
||||
|
||||
def test_post_success(self, app: Flask) -> None:
|
||||
api = CreateRagPipelineDatasetApi()
|
||||
method = unwrap(api.post)
|
||||
|
||||
payload = self._valid_payload()
|
||||
user = _account(editor=True)
|
||||
import_info = {
|
||||
"id": "import-1",
|
||||
"status": ImportStatus.COMPLETED,
|
||||
"dataset_id": "ds-1",
|
||||
"pipeline_id": "pipeline-1",
|
||||
"current_dsl_version": "0.1.0",
|
||||
"imported_dsl_version": "0.1.0",
|
||||
"error": "",
|
||||
}
|
||||
|
||||
mock_service = MagicMock()
|
||||
mock_service.create_rag_pipeline_dataset.return_value = import_info
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch.object(type(console_ns), "payload", payload),
|
||||
patch(
|
||||
"controllers.console.datasets.rag_pipeline.rag_pipeline_datasets.RagPipelineDslService",
|
||||
return_value=mock_service,
|
||||
),
|
||||
):
|
||||
response, status = method(api, RagPipelineDatasetImportPayload.model_validate(payload), "tenant-1", user)
|
||||
|
||||
assert status == 201
|
||||
assert response == {
|
||||
"id": "import-1",
|
||||
"status": "completed",
|
||||
"dataset_id": "ds-1",
|
||||
"pipeline_id": "pipeline-1",
|
||||
"current_dsl_version": "0.1.0",
|
||||
"imported_dsl_version": "0.1.0",
|
||||
"error": "",
|
||||
}
|
||||
|
||||
def test_post_success_syncs_creator_under_rbac(self, app: Flask) -> None:
|
||||
"""RBAC: RAG-pipeline dataset create must keep the creator bound to their own dataset.
|
||||
|
||||
Regression guard for the #42430 flip to automatic_include_workspace_members=False:
|
||||
this path bypasses DatasetService.create_empty_dataset (which syncs the creator
|
||||
internally), so the controller has to sync the creator explicitly or the creator
|
||||
gets 403 on their own dataset.
|
||||
"""
|
||||
api = CreateRagPipelineDatasetApi()
|
||||
method = unwrap(api.post)
|
||||
|
||||
payload = self._valid_payload()
|
||||
user = _account(editor=True)
|
||||
import_info = {
|
||||
"id": "import-1",
|
||||
"status": ImportStatus.COMPLETED,
|
||||
"dataset_id": "ds-1",
|
||||
"pipeline_id": "pipeline-1",
|
||||
"current_dsl_version": "0.1.0",
|
||||
"imported_dsl_version": "0.1.0",
|
||||
"error": "",
|
||||
}
|
||||
|
||||
mock_service = MagicMock()
|
||||
mock_service.create_rag_pipeline_dataset.return_value = import_info
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch.object(type(console_ns), "payload", payload),
|
||||
config_overrides_context(RBAC_ENABLED=True),
|
||||
patch(
|
||||
"controllers.console.datasets.rag_pipeline.rag_pipeline_datasets.RagPipelineDslService",
|
||||
return_value=mock_service,
|
||||
),
|
||||
patch(
|
||||
"controllers.console.datasets.rag_pipeline.rag_pipeline_datasets.enterprise_rbac_service.RBACService.DatasetAccess.replace_whitelist"
|
||||
),
|
||||
patch(
|
||||
"controllers.console.datasets.rag_pipeline.rag_pipeline_datasets.enterprise_rbac_service.try_sync_creator_access_policy_member_bindings"
|
||||
) as sync_creator,
|
||||
):
|
||||
response, status = method(api, RagPipelineDatasetImportPayload.model_validate(payload), "tenant-1", user)
|
||||
|
||||
assert status == 201
|
||||
assert sync_creator.called, "RAG pipeline create must sync the creator's own access"
|
||||
sync_creator.assert_called_once_with("tenant-1", user.id, rbac_contracts.RBACResourceType.DATASET, "ds-1")
|
||||
assert response["dataset_id"] == "ds-1"
|
||||
|
||||
def test_post_forbidden_non_editor(self, app: Flask) -> None:
|
||||
api = CreateRagPipelineDatasetApi()
|
||||
method = unwrap(api.post)
|
||||
|
||||
payload = self._valid_payload()
|
||||
user = _account(editor=False)
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch.object(type(console_ns), "payload", payload),
|
||||
):
|
||||
with pytest.raises(Forbidden):
|
||||
method(api, RagPipelineDatasetImportPayload.model_validate(payload), "tenant-1", user)
|
||||
|
||||
def test_post_dataset_name_duplicate(self, app: Flask) -> None:
|
||||
api = CreateRagPipelineDatasetApi()
|
||||
method = unwrap(api.post)
|
||||
|
||||
payload = self._valid_payload()
|
||||
user = _account(editor=True)
|
||||
|
||||
mock_service = MagicMock()
|
||||
mock_service.create_rag_pipeline_dataset.side_effect = services.errors.dataset.DatasetNameDuplicateError()
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch.object(type(console_ns), "payload", payload),
|
||||
patch(
|
||||
"controllers.console.datasets.rag_pipeline.rag_pipeline_datasets.RagPipelineDslService",
|
||||
return_value=mock_service,
|
||||
),
|
||||
):
|
||||
with pytest.raises(DatasetNameDuplicateError):
|
||||
method(api, RagPipelineDatasetImportPayload.model_validate(payload), "tenant-1", user)
|
||||
|
||||
def test_post_invalid_payload(self, app: Flask) -> None:
|
||||
api = CreateRagPipelineDatasetApi()
|
||||
method = unwrap(api.post)
|
||||
|
||||
payload: dict[str, str] = {}
|
||||
user = _account(editor=True)
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch.object(type(console_ns), "payload", payload),
|
||||
):
|
||||
with pytest.raises(
|
||||
ValueError, match="1 validation error for RagPipelineDatasetImportPayload\\\nyaml_content"
|
||||
):
|
||||
method(api, RagPipelineDatasetImportPayload.model_validate(payload), "tenant-1", user)
|
||||
|
||||
|
||||
class TestCreateEmptyRagPipelineDatasetApi:
|
||||
def test_post_forbidden_non_editor(self, app: Flask) -> None:
|
||||
api = CreateEmptyRagPipelineDatasetApi()
|
||||
method = unwrap(api.post)
|
||||
|
||||
user = _account(editor=False)
|
||||
|
||||
with app.test_request_context("/"):
|
||||
with pytest.raises(Forbidden):
|
||||
method(api, "tenant-1", user)
|
||||
+470
@@ -0,0 +1,470 @@
|
||||
from collections.abc import Callable, Iterator
|
||||
from inspect import unwrap
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from flask import Flask, Response
|
||||
from sqlalchemy import Engine
|
||||
from sqlalchemy.orm import Session, scoped_session, sessionmaker
|
||||
|
||||
from controllers.common.errors import InvalidArgumentError, NotFoundError
|
||||
from controllers.common.rbac import RBACResourceScope
|
||||
from controllers.console import console_ns
|
||||
from controllers.console.app.error import DraftWorkflowNotExist
|
||||
from controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable import (
|
||||
PaginationQuery,
|
||||
RagPipelineEnvironmentVariableCollectionApi,
|
||||
RagPipelineNodeVariableCollectionApi,
|
||||
RagPipelineSystemVariableCollectionApi,
|
||||
RagPipelineVariableApi,
|
||||
RagPipelineVariableCollectionApi,
|
||||
RagPipelineVariableResetApi,
|
||||
WorkflowDraftVariablePatchPayload,
|
||||
)
|
||||
from controllers.console.wraps import RBACPermission
|
||||
from core.workflow.llm_environment_variable import LLMEnvironmentVariable
|
||||
from core.workflow.variable_prefixes import SYSTEM_VARIABLE_NODE_ID
|
||||
from factories.variable_factory import build_segment
|
||||
from graphon.file import File, FileTransferMethod, FileType
|
||||
from graphon.variables import IntegerVariable, StringVariable
|
||||
from models.account import Account, TenantAccountRole
|
||||
from models.dataset import Pipeline
|
||||
from models.workflow import Workflow, WorkflowDraftVariable, WorkflowType
|
||||
from tests.helpers.unit.controllers.rbac_introspection import rbac_checks
|
||||
from tests.helpers.unit.model_factories import make_workflow
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sqlite_db(sqlite_engine: Engine, sqlite_session_factory: sessionmaker[Session]) -> Iterator[SimpleNamespace]:
|
||||
session = scoped_session(sqlite_session_factory)
|
||||
yield SimpleNamespace(engine=sqlite_engine, session=session)
|
||||
session.remove()
|
||||
|
||||
|
||||
def _pipeline(*, pipeline_id: str = "p1", tenant_id: str = "t1") -> Pipeline:
|
||||
pipeline = Pipeline(tenant_id=tenant_id, name="Pipeline")
|
||||
pipeline.id = pipeline_id
|
||||
return pipeline
|
||||
|
||||
|
||||
def _workflow(pipeline: Pipeline, environment_variables=()) -> Workflow:
|
||||
return make_workflow(
|
||||
tenant_id=pipeline.tenant_id,
|
||||
app_id=pipeline.id,
|
||||
workflow_type=WorkflowType.RAG_PIPELINE,
|
||||
graph="{}",
|
||||
environment_variables=environment_variables,
|
||||
)
|
||||
|
||||
|
||||
def _variable(*, app_id: str = "p1", user_id: str = "account-1", file_value: bool = False) -> WorkflowDraftVariable:
|
||||
if file_value:
|
||||
value = build_segment(
|
||||
File(
|
||||
file_id="file-1",
|
||||
file_type=FileType.IMAGE,
|
||||
transfer_method=FileTransferMethod.REMOTE_URL,
|
||||
remote_url="https://example.com/image.png",
|
||||
filename="image.png",
|
||||
extension=".png",
|
||||
mime_type="image/png",
|
||||
size=1,
|
||||
)
|
||||
)
|
||||
else:
|
||||
value = build_segment("value")
|
||||
return WorkflowDraftVariable.new_node_variable(
|
||||
app_id=app_id,
|
||||
user_id=user_id,
|
||||
node_id="node-1",
|
||||
name="variable",
|
||||
value=value,
|
||||
node_execution_id="execution-1",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def editor_user() -> Account:
|
||||
user = Account(name="Test User", email="user@example.com")
|
||||
user.id = "account-1"
|
||||
user.role = TenantAccountRole.EDITOR
|
||||
return user
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def restx_config(app):
|
||||
return patch.dict(app.config, {"RESTX_MASK_HEADER": "X-Fields"})
|
||||
|
||||
|
||||
def test_rag_draft_variable_routes_require_dataset_edit_permission() -> None:
|
||||
route = RagPipelineVariableApi.get
|
||||
legacy_gate = unwrap(route, stop=lambda decorator: "edit_permission_required" in decorator.__code__.co_qualname)
|
||||
assert "edit_permission_required" in legacy_gate.__code__.co_qualname
|
||||
|
||||
[check] = rbac_checks(route)
|
||||
assert check.locator.scope == RBACResourceScope.DATASET
|
||||
assert check.scene == RBACPermission.DATASET_EDIT
|
||||
|
||||
|
||||
class TestRagPipelineVariableCollectionApi:
|
||||
def test_get_variables_success(self, app: Flask, sqlite_db, editor_user, restx_config):
|
||||
api = RagPipelineVariableCollectionApi()
|
||||
method = unwrap(api.get)
|
||||
|
||||
pipeline = _pipeline()
|
||||
|
||||
rag_srv = MagicMock()
|
||||
rag_srv.is_workflow_exist.return_value = True
|
||||
|
||||
var_list = MagicMock()
|
||||
var_list.variables = []
|
||||
var_list.total = None
|
||||
|
||||
draft_srv = MagicMock()
|
||||
draft_srv.list_variables_without_values.return_value = var_list
|
||||
|
||||
with (
|
||||
app.test_request_context("/?page=1&limit=10"),
|
||||
restx_config,
|
||||
patch("controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.db", sqlite_db),
|
||||
patch(
|
||||
"controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.RagPipelineService",
|
||||
return_value=rag_srv,
|
||||
),
|
||||
patch(
|
||||
"controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.WorkflowDraftVariableService",
|
||||
return_value=draft_srv,
|
||||
),
|
||||
):
|
||||
result = method(api, PaginationQuery(page=1, limit=10), editor_user, pipeline)
|
||||
|
||||
assert result == {"items": [], "total": None}
|
||||
draft_srv.list_variables_without_values.assert_called_once_with(
|
||||
app_id="p1",
|
||||
page=1,
|
||||
limit=10,
|
||||
user_id="account-1",
|
||||
)
|
||||
|
||||
def test_get_variables_workflow_not_exist(self, app: Flask, sqlite_db, editor_user):
|
||||
api = RagPipelineVariableCollectionApi()
|
||||
method = unwrap(api.get)
|
||||
|
||||
pipeline = _pipeline()
|
||||
|
||||
rag_srv = MagicMock()
|
||||
rag_srv.is_workflow_exist.return_value = False
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch("controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.db", sqlite_db),
|
||||
patch(
|
||||
"controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.RagPipelineService",
|
||||
return_value=rag_srv,
|
||||
),
|
||||
):
|
||||
with pytest.raises(DraftWorkflowNotExist):
|
||||
method(api, PaginationQuery(), editor_user, pipeline)
|
||||
|
||||
def test_delete_variables_success(self, app: Flask, sqlite_db, editor_user):
|
||||
api = RagPipelineVariableCollectionApi()
|
||||
method = unwrap(api.delete)
|
||||
|
||||
pipeline = _pipeline()
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch("controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.db", sqlite_db),
|
||||
patch("controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.WorkflowDraftVariableService"),
|
||||
):
|
||||
result = method(api, editor_user, pipeline)
|
||||
|
||||
assert isinstance(result, Response)
|
||||
assert result.status_code == 204
|
||||
|
||||
|
||||
class TestRagPipelineNodeVariableCollectionApi:
|
||||
def test_get_node_variables_success(self, app: Flask, sqlite_db, editor_user, restx_config):
|
||||
api = RagPipelineNodeVariableCollectionApi()
|
||||
method = unwrap(api.get)
|
||||
|
||||
pipeline = _pipeline()
|
||||
|
||||
var_list = MagicMock()
|
||||
var_list.variables = []
|
||||
|
||||
srv = MagicMock()
|
||||
srv.list_node_variables.return_value = var_list
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
restx_config,
|
||||
patch("controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.db", sqlite_db),
|
||||
patch(
|
||||
"controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.WorkflowDraftVariableService",
|
||||
return_value=srv,
|
||||
),
|
||||
):
|
||||
result = method(api, editor_user, pipeline, "node1")
|
||||
|
||||
assert result == {"items": []}
|
||||
srv.list_node_variables.assert_called_once_with("p1", "node1", user_id="account-1")
|
||||
|
||||
def test_get_node_variables_invalid_node(self, app: Flask, editor_user):
|
||||
api = RagPipelineNodeVariableCollectionApi()
|
||||
method = unwrap(api.get)
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
):
|
||||
with pytest.raises(InvalidArgumentError):
|
||||
method(api, editor_user, _pipeline(), SYSTEM_VARIABLE_NODE_ID)
|
||||
|
||||
|
||||
class TestRagPipelineVariableApi:
|
||||
def test_get_variable_not_found(self, app: Flask, sqlite_db, editor_user):
|
||||
api = RagPipelineVariableApi()
|
||||
method = unwrap(api.get)
|
||||
|
||||
srv = MagicMock()
|
||||
srv.get_variable.return_value = None
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch("controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.db", sqlite_db),
|
||||
patch(
|
||||
"controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.WorkflowDraftVariableService",
|
||||
return_value=srv,
|
||||
),
|
||||
):
|
||||
with pytest.raises(NotFoundError):
|
||||
method(api, editor_user, _pipeline(), "v1")
|
||||
|
||||
def test_patch_variable_invalid_file_payload(self, app: Flask, sqlite_db, editor_user):
|
||||
api = RagPipelineVariableApi()
|
||||
method = unwrap(api.patch)
|
||||
|
||||
pipeline = _pipeline()
|
||||
variable = _variable(file_value=True)
|
||||
|
||||
srv = MagicMock()
|
||||
srv.get_variable.return_value = variable
|
||||
|
||||
payload = {"value": "invalid"}
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch.object(type(console_ns), "payload", payload),
|
||||
patch("controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.db", sqlite_db),
|
||||
patch(
|
||||
"controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.WorkflowDraftVariableService",
|
||||
return_value=srv,
|
||||
),
|
||||
):
|
||||
with pytest.raises(InvalidArgumentError):
|
||||
method(api, WorkflowDraftVariablePatchPayload.model_validate(payload), editor_user, pipeline, "v1")
|
||||
|
||||
def test_delete_variable_success(self, app: Flask, sqlite_db, editor_user):
|
||||
api = RagPipelineVariableApi()
|
||||
method = unwrap(api.delete)
|
||||
|
||||
pipeline = _pipeline()
|
||||
variable = _variable()
|
||||
|
||||
srv = MagicMock()
|
||||
srv.get_variable.return_value = variable
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch("controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.db", sqlite_db),
|
||||
patch(
|
||||
"controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.WorkflowDraftVariableService",
|
||||
return_value=srv,
|
||||
),
|
||||
):
|
||||
result = method(api, editor_user, pipeline, "v1")
|
||||
|
||||
assert result.status_code == 204
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("api_type", "method", "payload"),
|
||||
[
|
||||
(RagPipelineVariableApi, RagPipelineVariableApi.get, None),
|
||||
(
|
||||
RagPipelineVariableApi,
|
||||
RagPipelineVariableApi.patch,
|
||||
WorkflowDraftVariablePatchPayload(name="new name"),
|
||||
),
|
||||
(RagPipelineVariableApi, RagPipelineVariableApi.delete, None),
|
||||
(RagPipelineVariableResetApi, RagPipelineVariableResetApi.put, None),
|
||||
],
|
||||
)
|
||||
def test_direct_variable_access_rejects_different_user(
|
||||
app: Flask,
|
||||
sqlite_db: SimpleNamespace,
|
||||
editor_user: Account,
|
||||
api_type: type[RagPipelineVariableApi] | type[RagPipelineVariableResetApi],
|
||||
method: Callable[..., object],
|
||||
payload: WorkflowDraftVariablePatchPayload | None,
|
||||
) -> None:
|
||||
api = api_type()
|
||||
method = unwrap(method)
|
||||
pipeline = _pipeline()
|
||||
variable = _variable(user_id="account-2")
|
||||
draft_service = MagicMock()
|
||||
draft_service.get_variable.return_value = variable
|
||||
rag_service = MagicMock()
|
||||
rag_service.get_draft_workflow.return_value = _workflow(pipeline)
|
||||
if payload is not None:
|
||||
call_args = (api, payload, editor_user, pipeline, "v1")
|
||||
else:
|
||||
call_args = (api, editor_user, pipeline, "v1")
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch("controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.db", sqlite_db),
|
||||
patch.object(sqlite_db.session, "commit") as commit,
|
||||
patch(
|
||||
"controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.RagPipelineService",
|
||||
return_value=rag_service,
|
||||
),
|
||||
patch(
|
||||
"controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.WorkflowDraftVariableService",
|
||||
return_value=draft_service,
|
||||
),
|
||||
pytest.raises(NotFoundError),
|
||||
):
|
||||
method(*call_args)
|
||||
|
||||
draft_service.update_variable.assert_not_called()
|
||||
draft_service.delete_variable.assert_not_called()
|
||||
draft_service.reset_variable.assert_not_called()
|
||||
commit.assert_not_called()
|
||||
|
||||
|
||||
class TestRagPipelineVariableResetApi:
|
||||
def test_reset_variable_success(self, app: Flask, sqlite_db, editor_user):
|
||||
api = RagPipelineVariableResetApi()
|
||||
method = unwrap(api.put)
|
||||
|
||||
pipeline = _pipeline()
|
||||
workflow = _workflow(pipeline)
|
||||
variable = _variable()
|
||||
|
||||
srv = MagicMock()
|
||||
srv.get_variable.return_value = variable
|
||||
srv.reset_variable.return_value = variable
|
||||
|
||||
rag_srv = MagicMock()
|
||||
rag_srv.get_draft_workflow.return_value = workflow
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch("controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.db", sqlite_db),
|
||||
patch(
|
||||
"controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.RagPipelineService",
|
||||
return_value=rag_srv,
|
||||
),
|
||||
patch(
|
||||
"controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.WorkflowDraftVariableService",
|
||||
return_value=srv,
|
||||
),
|
||||
patch(
|
||||
"controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.dump_response",
|
||||
return_value={"id": "v1"},
|
||||
),
|
||||
):
|
||||
result = method(api, editor_user, pipeline, "v1")
|
||||
|
||||
assert result == {"id": "v1"}
|
||||
|
||||
|
||||
class TestSystemAndEnvironmentVariablesApi:
|
||||
def test_system_variables_success(self, app: Flask, sqlite_db, editor_user, restx_config):
|
||||
api = RagPipelineSystemVariableCollectionApi()
|
||||
method = unwrap(api.get)
|
||||
|
||||
pipeline = _pipeline()
|
||||
|
||||
var_list = MagicMock()
|
||||
var_list.variables = []
|
||||
|
||||
srv = MagicMock()
|
||||
srv.list_system_variables.return_value = var_list
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
restx_config,
|
||||
patch("controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.db", sqlite_db),
|
||||
patch(
|
||||
"controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.WorkflowDraftVariableService",
|
||||
return_value=srv,
|
||||
),
|
||||
):
|
||||
result = method(api, editor_user, pipeline)
|
||||
|
||||
assert result == {"items": []}
|
||||
srv.list_system_variables.assert_called_once_with("p1", user_id="account-1")
|
||||
|
||||
def test_environment_variables_success(self, app: Flask, editor_user):
|
||||
api = RagPipelineEnvironmentVariableCollectionApi()
|
||||
method = unwrap(api.get)
|
||||
|
||||
env_var = StringVariable(
|
||||
id="e1",
|
||||
name="ENV",
|
||||
description="d",
|
||||
selector=["env", "ENV"],
|
||||
value="x",
|
||||
)
|
||||
|
||||
pipeline = _pipeline()
|
||||
workflow = _workflow(pipeline, [env_var])
|
||||
|
||||
rag_srv = MagicMock()
|
||||
rag_srv.get_draft_workflow.return_value = workflow
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch(
|
||||
"controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.RagPipelineService",
|
||||
return_value=rag_srv,
|
||||
),
|
||||
):
|
||||
result = method(api, editor_user, pipeline)
|
||||
|
||||
assert len(result["items"]) == 1
|
||||
assert result["items"][0]["value_type"] == "string"
|
||||
|
||||
def test_environment_variables_preserve_number_subtype_and_llm_type(self, app: Flask, editor_user):
|
||||
api = RagPipelineEnvironmentVariableCollectionApi()
|
||||
method = unwrap(api.get)
|
||||
number_var = IntegerVariable(
|
||||
id="number",
|
||||
name="NUMBER",
|
||||
description="",
|
||||
selector=["env", "NUMBER"],
|
||||
value=1,
|
||||
)
|
||||
llm_var = LLMEnvironmentVariable(
|
||||
id="llm",
|
||||
name="MODEL",
|
||||
value={"provider": "provider", "name": "model", "mode": "chat"},
|
||||
selector=["env", "MODEL"],
|
||||
)
|
||||
pipeline = _pipeline()
|
||||
rag_srv = MagicMock()
|
||||
rag_srv.get_draft_workflow.return_value = _workflow(pipeline, [number_var, llm_var])
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch(
|
||||
"controllers.console.datasets.rag_pipeline.rag_pipeline_draft_variable.RagPipelineService",
|
||||
return_value=rag_srv,
|
||||
),
|
||||
):
|
||||
result = method(api, editor_user, pipeline)
|
||||
|
||||
assert [item["value_type"] for item in result["items"]] == ["integer", "llm"]
|
||||
+402
@@ -0,0 +1,402 @@
|
||||
"""Unit tests for rag_pipeline_import controller endpoints."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Iterator
|
||||
from inspect import unwrap
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from controllers.common.schema import JsonResponseWithStatus
|
||||
from controllers.console import console_ns
|
||||
from controllers.console.datasets.rag_pipeline.rag_pipeline_import import (
|
||||
IncludeSecretQuery,
|
||||
RagPipelineExportApi,
|
||||
RagPipelineImportApi,
|
||||
RagPipelineImportCheckDependenciesApi,
|
||||
RagPipelineImportConfirmApi,
|
||||
RagPipelineImportPayload,
|
||||
)
|
||||
from core.plugin.entities.plugin import PluginDependency, PluginDependencyType
|
||||
from models.account import Account
|
||||
from models.dataset import Pipeline
|
||||
from models.engine import db
|
||||
from services.enterprise import rbac_service
|
||||
from services.entities.dsl_entities import CheckDependenciesResult, ImportStatus
|
||||
from services.rag_pipeline.rag_pipeline_dsl_service import RagPipelineImportInfo
|
||||
from tests.helpers.unit.model_factories import make_account, make_tenant
|
||||
|
||||
|
||||
def _account() -> Account:
|
||||
return make_account(name="RAG Import Tester", email="rag-import@example.com")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app() -> Iterator[Flask]:
|
||||
app = Flask(__name__)
|
||||
app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///:memory:"
|
||||
db.init_app(app)
|
||||
|
||||
with app.app_context():
|
||||
yield app
|
||||
|
||||
|
||||
class TestRagPipelineImportApi:
|
||||
def _payload(self, mode: str = "create") -> dict[str, str]:
|
||||
return {
|
||||
"mode": mode,
|
||||
"yaml_content": "content",
|
||||
"name": "Test",
|
||||
}
|
||||
|
||||
def test_post_success_200(self, app: Flask) -> None:
|
||||
api = RagPipelineImportApi()
|
||||
method = unwrap(api.post)
|
||||
|
||||
payload = self._payload()
|
||||
user = _account()
|
||||
result = RagPipelineImportInfo(
|
||||
id="import-1",
|
||||
status=ImportStatus.COMPLETED,
|
||||
pipeline_id="pipeline-1",
|
||||
dataset_id="dataset-1",
|
||||
current_dsl_version="0.1.0",
|
||||
imported_dsl_version="0.1.0",
|
||||
)
|
||||
|
||||
service = MagicMock()
|
||||
service.import_rag_pipeline.return_value = result
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch.object(type(console_ns), "payload", payload),
|
||||
patch(
|
||||
"controllers.console.datasets.rag_pipeline.rag_pipeline_import.RagPipelineDslService",
|
||||
return_value=service,
|
||||
),
|
||||
):
|
||||
response, status = method(api, RagPipelineImportPayload(mode="create"), user)
|
||||
|
||||
assert status == 200
|
||||
assert response == {
|
||||
"id": "import-1",
|
||||
"status": "completed",
|
||||
"pipeline_id": "pipeline-1",
|
||||
"dataset_id": "dataset-1",
|
||||
"current_dsl_version": "0.1.0",
|
||||
"imported_dsl_version": "0.1.0",
|
||||
"error": "",
|
||||
}
|
||||
|
||||
def test_post_failed_400(self, app: Flask) -> None:
|
||||
api = RagPipelineImportApi()
|
||||
method = unwrap(api.post)
|
||||
|
||||
payload = self._payload()
|
||||
user = _account()
|
||||
result = RagPipelineImportInfo(
|
||||
id="import-1",
|
||||
status=ImportStatus.FAILED,
|
||||
current_dsl_version="0.1.0",
|
||||
imported_dsl_version="",
|
||||
error="bad dsl",
|
||||
)
|
||||
|
||||
service = MagicMock()
|
||||
service.import_rag_pipeline.return_value = result
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch.object(type(console_ns), "payload", payload),
|
||||
patch(
|
||||
"controllers.console.datasets.rag_pipeline.rag_pipeline_import.RagPipelineDslService",
|
||||
return_value=service,
|
||||
),
|
||||
):
|
||||
response, status = method(api, RagPipelineImportPayload(mode="create"), user)
|
||||
|
||||
assert status == 400
|
||||
assert response["status"] == "failed"
|
||||
assert response["error"] == "bad dsl"
|
||||
assert response["pipeline_id"] is None
|
||||
assert response["dataset_id"] is None
|
||||
|
||||
def test_post_pending_202(self, app: Flask) -> None:
|
||||
api = RagPipelineImportApi()
|
||||
method = unwrap(api.post)
|
||||
|
||||
payload = self._payload()
|
||||
user = _account()
|
||||
result = RagPipelineImportInfo(
|
||||
id="import-1",
|
||||
status=ImportStatus.PENDING,
|
||||
pipeline_id="pipeline-1",
|
||||
dataset_id="dataset-1",
|
||||
current_dsl_version="0.1.0",
|
||||
imported_dsl_version="0.2.0",
|
||||
)
|
||||
|
||||
service = MagicMock()
|
||||
service.import_rag_pipeline.return_value = result
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch.object(type(console_ns), "payload", payload),
|
||||
patch(
|
||||
"controllers.console.datasets.rag_pipeline.rag_pipeline_import.RagPipelineDslService",
|
||||
return_value=service,
|
||||
),
|
||||
):
|
||||
response, status = method(api, RagPipelineImportPayload(mode="create"), user)
|
||||
|
||||
assert status == 202
|
||||
assert response["status"] == "pending"
|
||||
assert response["pipeline_id"] == "pipeline-1"
|
||||
assert response["dataset_id"] == "dataset-1"
|
||||
|
||||
|
||||
class TestRagPipelineImportConfirmApi:
|
||||
def test_confirm_success(self, app: Flask) -> None:
|
||||
api = RagPipelineImportConfirmApi()
|
||||
method = unwrap(api.post)
|
||||
|
||||
user = _account()
|
||||
result = RagPipelineImportInfo(
|
||||
id="import-1",
|
||||
status=ImportStatus.COMPLETED,
|
||||
pipeline_id="pipeline-1",
|
||||
dataset_id="dataset-1",
|
||||
current_dsl_version="0.1.0",
|
||||
imported_dsl_version="0.1.0",
|
||||
)
|
||||
|
||||
service = MagicMock()
|
||||
service.confirm_import.return_value = result
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch(
|
||||
"controllers.console.datasets.rag_pipeline.rag_pipeline_import.RagPipelineDslService",
|
||||
return_value=service,
|
||||
),
|
||||
):
|
||||
response, status = method(api, user, "import-1")
|
||||
|
||||
assert status == 200
|
||||
assert response["status"] == "completed"
|
||||
assert response["pipeline_id"] == "pipeline-1"
|
||||
|
||||
def test_confirm_failed(self, app: Flask) -> None:
|
||||
api = RagPipelineImportConfirmApi()
|
||||
method = unwrap(api.post)
|
||||
|
||||
user = _account()
|
||||
result = RagPipelineImportInfo(
|
||||
id="import-1",
|
||||
status=ImportStatus.FAILED,
|
||||
current_dsl_version="0.1.0",
|
||||
imported_dsl_version="",
|
||||
error="missing dependency",
|
||||
)
|
||||
|
||||
service = MagicMock()
|
||||
service.confirm_import.return_value = result
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch(
|
||||
"controllers.console.datasets.rag_pipeline.rag_pipeline_import.RagPipelineDslService",
|
||||
return_value=service,
|
||||
),
|
||||
):
|
||||
response, status = method(api, user, "import-1")
|
||||
|
||||
assert status == 400
|
||||
assert response["status"] == "failed"
|
||||
assert response["error"] == "missing dependency"
|
||||
|
||||
|
||||
class TestRagPipelineImportCheckDependenciesApi:
|
||||
def test_get_success(self, app: Flask) -> None:
|
||||
api = RagPipelineImportCheckDependenciesApi()
|
||||
method = unwrap(api.get)
|
||||
|
||||
pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")
|
||||
result = CheckDependenciesResult()
|
||||
|
||||
service = MagicMock()
|
||||
service.check_dependencies.return_value = result
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch(
|
||||
"controllers.console.datasets.rag_pipeline.rag_pipeline_import.RagPipelineDslService",
|
||||
return_value=service,
|
||||
),
|
||||
):
|
||||
response, status = method(api, pipeline)
|
||||
|
||||
assert status == 200
|
||||
assert response == {"leaked_dependencies": []}
|
||||
|
||||
def test_get_serializes_leaked_dependencies(self, app: Flask) -> None:
|
||||
api = RagPipelineImportCheckDependenciesApi()
|
||||
method = unwrap(api.get)
|
||||
|
||||
pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")
|
||||
dependency = PluginDependency(
|
||||
type=PluginDependencyType.Marketplace,
|
||||
value=PluginDependency.Marketplace(
|
||||
marketplace_plugin_unique_identifier="langgenius/example:0.1.0",
|
||||
version="0.1.0",
|
||||
),
|
||||
current_identifier="langgenius/example:0.0.1",
|
||||
)
|
||||
service = MagicMock()
|
||||
service.check_dependencies.return_value = CheckDependenciesResult(leaked_dependencies=[dependency])
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch(
|
||||
"controllers.console.datasets.rag_pipeline.rag_pipeline_import.RagPipelineDslService",
|
||||
return_value=service,
|
||||
),
|
||||
):
|
||||
response, status = method(api, pipeline)
|
||||
|
||||
assert status == 200
|
||||
assert response == {
|
||||
"leaked_dependencies": [
|
||||
{
|
||||
"type": "marketplace",
|
||||
"value": {
|
||||
"marketplace_plugin_unique_identifier": "langgenius/example:0.1.0",
|
||||
"version": "0.1.0",
|
||||
},
|
||||
"current_identifier": "langgenius/example:0.0.1",
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
class TestRagPipelineExportApi:
|
||||
def test_get_with_include_secret(self, app: Flask) -> None:
|
||||
api = RagPipelineExportApi()
|
||||
method = unwrap(api.get)
|
||||
|
||||
pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")
|
||||
service = MagicMock()
|
||||
service.export_rag_pipeline_dsl.return_value = "yaml: data"
|
||||
|
||||
with (
|
||||
app.test_request_context("/?include_secret=true"),
|
||||
patch(
|
||||
"controllers.console.datasets.rag_pipeline.rag_pipeline_import.RagPipelineDslService",
|
||||
return_value=service,
|
||||
),
|
||||
):
|
||||
response, status = method(api, IncludeSecretQuery(), pipeline)
|
||||
|
||||
assert status == 200
|
||||
assert response == {"data": "yaml: data"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("confirm", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
("import_status", "created", "dataset_id", "rbac_enabled", "commit_fails", "expected_calls"),
|
||||
[
|
||||
(ImportStatus.COMPLETED, True, "dataset-1", True, False, 1),
|
||||
(ImportStatus.COMPLETED_WITH_WARNINGS, True, "dataset-1", True, False, 1),
|
||||
(ImportStatus.COMPLETED, False, "dataset-1", True, False, 0),
|
||||
(ImportStatus.PENDING, True, "dataset-1", True, False, 0),
|
||||
(ImportStatus.FAILED, True, "dataset-1", True, False, 0),
|
||||
(ImportStatus.COMPLETED, True, None, True, False, 0),
|
||||
(ImportStatus.COMPLETED, True, "dataset-1", False, False, 0),
|
||||
(ImportStatus.COMPLETED, True, "dataset-1", True, True, 0),
|
||||
],
|
||||
)
|
||||
def test_creator_binding_only_after_successful_new_dataset_commit(
|
||||
app: Flask,
|
||||
config_overrides: Callable[..., None],
|
||||
confirm: bool,
|
||||
import_status: ImportStatus,
|
||||
created: bool,
|
||||
dataset_id: str | None,
|
||||
rbac_enabled: bool,
|
||||
commit_fails: bool,
|
||||
expected_calls: int,
|
||||
) -> None:
|
||||
config_overrides(RBAC_ENABLED=rbac_enabled)
|
||||
user = make_account(account_id="creator-1", tenant=make_tenant(tenant_id="tenant-1"))
|
||||
result = RagPipelineImportInfo.model_validate(
|
||||
{
|
||||
"id": "import-1",
|
||||
"status": import_status,
|
||||
"dataset_id": dataset_id,
|
||||
"dataset_created": created,
|
||||
}
|
||||
)
|
||||
service = MagicMock()
|
||||
service.import_rag_pipeline.return_value = result
|
||||
service.confirm_import.return_value = result
|
||||
events: list[str] = []
|
||||
session = Session(db.engine, expire_on_commit=False)
|
||||
commit = session.commit
|
||||
close = session.close
|
||||
|
||||
def commit_session() -> None:
|
||||
if commit_fails:
|
||||
raise RuntimeError("commit failed")
|
||||
commit()
|
||||
events.append("commit")
|
||||
|
||||
def close_session() -> None:
|
||||
close()
|
||||
events.append("close")
|
||||
|
||||
def sync_binding(*_args: object, **_kwargs: object) -> list[object]:
|
||||
events.append("sync")
|
||||
return []
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch("controllers.console.datasets.rag_pipeline.rag_pipeline_import.Session", return_value=session),
|
||||
patch(
|
||||
"controllers.console.datasets.rag_pipeline.rag_pipeline_import.RagPipelineDslService", return_value=service
|
||||
),
|
||||
patch.object(session, "commit", side_effect=commit_session),
|
||||
patch.object(session, "close", side_effect=close_session),
|
||||
patch.object(
|
||||
rbac_service.RBACService.AccessPolicies,
|
||||
"sync_creator_access_policy_member_bindings",
|
||||
side_effect=sync_binding,
|
||||
) as sync,
|
||||
patch.object(rbac_service.RBACService.DatasetAccess, "replace_whitelist") as replace,
|
||||
):
|
||||
|
||||
def invoke() -> JsonResponseWithStatus:
|
||||
if confirm:
|
||||
api = RagPipelineImportConfirmApi()
|
||||
return unwrap(api.post)(api, user, "import-1")
|
||||
api = RagPipelineImportApi()
|
||||
return unwrap(api.post)(api, RagPipelineImportPayload(mode="yaml-content"), user)
|
||||
|
||||
if commit_fails:
|
||||
with pytest.raises(RuntimeError, match="commit failed"):
|
||||
invoke()
|
||||
else:
|
||||
response, _ = invoke()
|
||||
assert "dataset_created" not in response
|
||||
assert sync.call_count == expected_calls
|
||||
if expected_calls:
|
||||
assert events == ["commit", "close", "sync"]
|
||||
sync.assert_called_once_with(
|
||||
"tenant-1",
|
||||
"creator-1",
|
||||
resource_type=rbac_service.RBACResourceType.DATASET,
|
||||
resource_id="dataset-1",
|
||||
)
|
||||
replace.assert_not_called()
|
||||
+1647
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,711 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from inspect import unwrap
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
from flask.testing import FlaskClient
|
||||
|
||||
from controllers.common.errors import NotFoundError
|
||||
from controllers.common.rbac import DatasetId
|
||||
from controllers.console import api as console_api
|
||||
from controllers.console import console_ns
|
||||
from controllers.console.app.error import ProviderNotInitializeError
|
||||
from controllers.console.datasets import data_source as data_source_controller
|
||||
from controllers.console.datasets.error import IndexingEstimateError
|
||||
from core.entities.knowledge_entities import IndexingEstimate
|
||||
from core.rbac import RBACPermission
|
||||
from enums import DeploymentEdition
|
||||
from libs.external_api import ExternalApi
|
||||
from libs.login import AccountWithTenant
|
||||
from machinery.context import RequestContext
|
||||
from services.credentials.refresh import CredentialRefreshUnavailableError
|
||||
from services.data_source.binding_application_service import (
|
||||
BindingMutationResult,
|
||||
DataSourceBindingApplicationService,
|
||||
)
|
||||
from services.data_source.entities.notion_import import (
|
||||
AuthorizedNotionPage,
|
||||
NotionPageIcon,
|
||||
NotionPageType,
|
||||
NotionWorkspace,
|
||||
)
|
||||
from services.data_source.entities.oauth import DataSourceBindingSummary
|
||||
from services.data_source.notion_import_application_service import NotionImportApplicationService
|
||||
from services.knowledge.dataset_access import DatasetAccessRecord, DatasetAccessService, DatasetAccessSnapshot
|
||||
from services.knowledge.document_sync import DocumentSyncApplicationService, SyncDocumentRecord
|
||||
from services.knowledge.entities.indexing_estimate import (
|
||||
NewSourcesEstimateCommand,
|
||||
NotionEstimateSource,
|
||||
)
|
||||
from services.knowledge.indexing.estimate import (
|
||||
DatasetEstimateRecord,
|
||||
EstimateDocumentRecord,
|
||||
EstimateSourceNotFoundError,
|
||||
IndexingEstimateCredentialUnavailableError,
|
||||
IndexingEstimateExecutionError,
|
||||
IndexingEstimateProviderUnavailableError,
|
||||
)
|
||||
from services.knowledge.resource_scope import DatasetRef, DocumentRef
|
||||
from tests.helpers.unit.controllers.rbac_introspection import rbac_checks
|
||||
|
||||
"""Guard all migrated dataset endpoints against transport/persistence regressions."""
|
||||
|
||||
|
||||
class TestApplicationBoundariesCases:
|
||||
@staticmethod
|
||||
@pytest.mark.parametrize(
|
||||
("filename", "count"),
|
||||
[("data_source.py", 7), ("datasets.py", 25), ("datasets_document.py", 24), ("external.py", 8)],
|
||||
)
|
||||
def test_every_endpoint_declares_admission_context_and_application_service(filename: str, count: int) -> None:
|
||||
root = Path(__file__).resolve().parents[5]
|
||||
tree = ast.parse((root / "controllers/console/datasets" / filename).read_text())
|
||||
# This namespace package is not traversed by import-linter's package graph.
|
||||
forbidden = (
|
||||
"sqlalchemy",
|
||||
"repositories",
|
||||
"controllers.common.session",
|
||||
"libs.login",
|
||||
"services.knowledge.dataset_service",
|
||||
"services.knowledge.external.service",
|
||||
"services.hit_testing_service",
|
||||
"services.knowledge.dataset_read_service",
|
||||
"services.knowledge.datasets.adapters",
|
||||
"services.knowledge.documents.adapters",
|
||||
"services.knowledge.external.adapters",
|
||||
)
|
||||
for node in ast.walk(tree):
|
||||
modules = (
|
||||
[node.module or ""]
|
||||
if isinstance(node, ast.ImportFrom)
|
||||
else [item.name for item in node.names]
|
||||
if isinstance(node, ast.Import)
|
||||
else []
|
||||
)
|
||||
for module in modules:
|
||||
assert not any(module == name or module.startswith(name + ".") for name in forbidden), module
|
||||
endpoints = [
|
||||
method
|
||||
for resource in tree.body
|
||||
if isinstance(resource, ast.ClassDef)
|
||||
for method in resource.body
|
||||
if isinstance(method, ast.FunctionDef) and method.name in {"get", "post", "patch", "put", "delete"}
|
||||
]
|
||||
assert len(endpoints) == count
|
||||
for endpoint in endpoints:
|
||||
assert any(
|
||||
isinstance(d, ast.Call) and ast.unparse(d.func) == "console_account_admission"
|
||||
for d in endpoint.decorator_list
|
||||
), endpoint.name
|
||||
assert any(
|
||||
arg.arg == "request_context"
|
||||
and arg.annotation is not None
|
||||
and ast.unparse(arg.annotation) == "RequestContext"
|
||||
for arg in endpoint.args.args
|
||||
), endpoint.name
|
||||
assert any(
|
||||
isinstance(n, ast.Call) and ast.unparse(n.func) == "application_services" for n in ast.walk(endpoint)
|
||||
), endpoint.name
|
||||
assert not {"session", "current_user", "current_tenant_id"}.intersection(
|
||||
arg.arg for arg in endpoint.args.args
|
||||
)
|
||||
|
||||
|
||||
# Importing the module registers only the resources under test on console_ns.
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from models import Account
|
||||
|
||||
TENANT_ID = "11111111-1111-1111-1111-111111111111"
|
||||
ACCOUNT_ID = "22222222-2222-2222-2222-222222222222"
|
||||
BINDING_ID = "33333333-3333-3333-3333-333333333333"
|
||||
DATASET_ID = "44444444-4444-4444-4444-444444444444"
|
||||
DOCUMENT_ID = "55555555-5555-5555-5555-555555555555"
|
||||
PAGE_ID = "66666666-6666-6666-6666-666666666666"
|
||||
FOREIGN_DATASET_ID = "77777777-7777-7777-7777-777777777777"
|
||||
|
||||
|
||||
@dataclass
|
||||
class RecordingBindingStore:
|
||||
bindings: tuple[DataSourceBindingSummary, ...] = ()
|
||||
mutations: list[tuple[str, str, bool]] = field(default_factory=list)
|
||||
mutation_result: BindingMutationResult = "updated"
|
||||
|
||||
def list_enabled_bindings(self, *, workspace_id: str) -> tuple[DataSourceBindingSummary, ...]:
|
||||
assert workspace_id == TENANT_ID
|
||||
return self.bindings
|
||||
|
||||
def change_disabled_state(
|
||||
self,
|
||||
*,
|
||||
workspace_id: str,
|
||||
binding_id: str,
|
||||
disabled: bool,
|
||||
) -> BindingMutationResult:
|
||||
self.mutations.append((workspace_id, binding_id, disabled))
|
||||
return self.mutation_result
|
||||
|
||||
|
||||
@dataclass
|
||||
class DatasetCatalog:
|
||||
record: DatasetAccessRecord
|
||||
|
||||
def get_access_snapshot(
|
||||
self,
|
||||
*,
|
||||
workspace_id: str,
|
||||
dataset_id: str,
|
||||
actor_id: str,
|
||||
) -> DatasetAccessSnapshot | None:
|
||||
assert actor_id == ACCOUNT_ID
|
||||
if workspace_id != self.record.workspace_id or dataset_id != self.record.id:
|
||||
return None
|
||||
return DatasetAccessSnapshot(dataset=self.record, actor_has_partial_access=False)
|
||||
|
||||
def get_estimate_record(self, dataset_ref: DatasetRef) -> DatasetEstimateRecord | None:
|
||||
if dataset_ref != DatasetRef(self.record.workspace_id, self.record.id):
|
||||
return None
|
||||
return DatasetEstimateRecord(
|
||||
id=self.record.id,
|
||||
workspace_id=self.record.workspace_id,
|
||||
indexing_technique="economy",
|
||||
)
|
||||
|
||||
def is_notion_dataset(self, dataset_ref: DatasetRef) -> bool:
|
||||
return dataset_ref == DatasetRef(self.record.workspace_id, self.record.id)
|
||||
|
||||
|
||||
class OwnerWorkspaceRoles:
|
||||
def get_legacy_role(self, *, workspace_id: str, account_id: str) -> str | None:
|
||||
assert workspace_id == TENANT_ID
|
||||
assert account_id == ACCOUNT_ID
|
||||
return "owner"
|
||||
|
||||
|
||||
@dataclass
|
||||
class KnowledgeDocuments:
|
||||
bound_page_ids: frozenset[str] = frozenset()
|
||||
active_refs: tuple[DocumentRef, ...] = ()
|
||||
|
||||
def list_bound_notion_page_ids(self, dataset_ref: DatasetRef) -> frozenset[str]:
|
||||
assert dataset_ref == DatasetRef(TENANT_ID, DATASET_ID)
|
||||
return self.bound_page_ids
|
||||
|
||||
def list_active_notion_refs(self, dataset_ref: DatasetRef) -> tuple[DocumentRef, ...]:
|
||||
assert dataset_ref == DatasetRef(TENANT_ID, DATASET_ID)
|
||||
return self.active_refs
|
||||
|
||||
def get_estimate_document(self, document_ref: DocumentRef) -> EstimateDocumentRecord | None:
|
||||
if document_ref.document_id != DOCUMENT_ID:
|
||||
return None
|
||||
return EstimateDocumentRecord(
|
||||
id=DOCUMENT_ID,
|
||||
workspace_id=TENANT_ID,
|
||||
dataset_id=DATASET_ID,
|
||||
data_source_type="notion_import",
|
||||
data_source_info=None,
|
||||
indexing_status="waiting",
|
||||
doc_form="text_model",
|
||||
doc_language="English",
|
||||
dataset_process_rule_id=None,
|
||||
)
|
||||
|
||||
def list_estimate_documents_by_batch(
|
||||
self, dataset_ref: DatasetRef, batch: str
|
||||
) -> tuple[EstimateDocumentRecord, ...]:
|
||||
document = self.get_estimate_document(dataset_ref.document(DOCUMENT_ID))
|
||||
if document is None or batch != "batch":
|
||||
return ()
|
||||
return (document,)
|
||||
|
||||
def get_sync_document(self, document_ref: DocumentRef) -> SyncDocumentRecord | None:
|
||||
if document_ref.document_id != DOCUMENT_ID:
|
||||
return None
|
||||
return SyncDocumentRecord(id=DOCUMENT_ID, data_source_type="notion_import")
|
||||
|
||||
|
||||
@dataclass
|
||||
class RecordingNotionSource:
|
||||
workspaces: tuple[NotionWorkspace, ...]
|
||||
list_calls: list[tuple[str, str, str]] = field(default_factory=list)
|
||||
preview_calls: list[tuple[str, str, str, str, str]] = field(default_factory=list)
|
||||
error: Exception | None = None
|
||||
|
||||
def list_authorized_pages(
|
||||
self,
|
||||
*,
|
||||
workspace_id: str,
|
||||
actor_id: str,
|
||||
credential_id: str,
|
||||
) -> tuple[NotionWorkspace, ...]:
|
||||
self.list_calls.append((workspace_id, actor_id, credential_id))
|
||||
if self.error is not None:
|
||||
raise self.error
|
||||
return self.workspaces
|
||||
|
||||
def preview_page(
|
||||
self,
|
||||
*,
|
||||
workspace_id: str,
|
||||
actor_id: str,
|
||||
credential_id: str,
|
||||
page_id: str,
|
||||
page_type: str,
|
||||
) -> str:
|
||||
self.preview_calls.append((workspace_id, actor_id, credential_id, page_id, page_type))
|
||||
if self.error is not None:
|
||||
raise self.error
|
||||
return "preview text"
|
||||
|
||||
|
||||
@dataclass
|
||||
class RecordingEstimateService:
|
||||
calls: list[tuple[RequestContext, NewSourcesEstimateCommand]] = field(default_factory=list)
|
||||
error: Exception | None = None
|
||||
|
||||
def estimate_new_sources(self, context: RequestContext, command: NewSourcesEstimateCommand) -> IndexingEstimate:
|
||||
self.calls.append((context, command))
|
||||
if self.error is not None:
|
||||
raise self.error
|
||||
return IndexingEstimate(total_segments=3, preview=[])
|
||||
|
||||
|
||||
@dataclass
|
||||
class RecordingDispatcher:
|
||||
calls: list[tuple[str, str]] = field(default_factory=list)
|
||||
|
||||
def dispatch(self, *, dataset_id: str, document_id: str) -> None:
|
||||
self.calls.append((dataset_id, document_id))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DataSourceApplicationServices:
|
||||
bindings: DataSourceBindingApplicationService
|
||||
notion_imports: NotionImportApplicationService
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class KnowledgeApplicationServices:
|
||||
document_sync: DocumentSyncApplicationService
|
||||
indexing_estimates: RecordingEstimateService
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ApplicationServiceRegistry:
|
||||
data_sources: DataSourceApplicationServices
|
||||
knowledge: KnowledgeApplicationServices
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ControllerCollaborators:
|
||||
client: FlaskClient
|
||||
account: AdmittedAccount
|
||||
bindings: RecordingBindingStore
|
||||
notion_source: RecordingNotionSource
|
||||
estimates: RecordingEstimateService
|
||||
dispatcher: RecordingDispatcher
|
||||
|
||||
|
||||
@dataclass
|
||||
class AdmittedAccount:
|
||||
id: str
|
||||
role: str
|
||||
status: str
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def console_app() -> Flask:
|
||||
"""One app with the console namespace, which takes a few hundred milliseconds to mount.
|
||||
|
||||
Each test registers its own ``extensions["application_services"]`` for the duration of that test.
|
||||
"""
|
||||
app = Flask(__name__)
|
||||
app.config.update(TESTING=False, PROPAGATE_EXCEPTIONS=False)
|
||||
api = ExternalApi(app)
|
||||
api.error_handlers = console_api.error_handlers.copy()
|
||||
api.add_namespace(console_ns, path="/console/api")
|
||||
return app
|
||||
|
||||
|
||||
class TestDataSourceCases:
|
||||
@staticmethod
|
||||
@pytest.fixture
|
||||
def collaborators(
|
||||
console_app: Flask,
|
||||
config_overrides: Callable[..., None],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> ControllerCollaborators:
|
||||
config_overrides(
|
||||
DEPLOYMENT_EDITION=DeploymentEdition.CLOUD,
|
||||
LOGIN_DISABLED=True,
|
||||
RBAC_ENABLED=False,
|
||||
CONSOLE_API_URL="https://console-api.example.com",
|
||||
)
|
||||
account = AdmittedAccount(id=ACCOUNT_ID, role="owner", status="active")
|
||||
account_with_tenant = AccountWithTenant(account=cast("Account", account), tenant_id=TENANT_ID)
|
||||
monkeypatch.setattr(
|
||||
"controllers.console.flask_admission.current_account_with_tenant",
|
||||
lambda: account_with_tenant,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"controllers.console.wraps.current_account_with_tenant",
|
||||
lambda: account_with_tenant,
|
||||
)
|
||||
|
||||
dataset = DatasetAccessRecord(
|
||||
id=DATASET_ID,
|
||||
workspace_id=TENANT_ID,
|
||||
maintainer_id=ACCOUNT_ID,
|
||||
permission="only_me",
|
||||
)
|
||||
dataset_catalog = DatasetCatalog(dataset)
|
||||
dataset_access = DatasetAccessService(
|
||||
datasets=dataset_catalog,
|
||||
workspace_roles=OwnerWorkspaceRoles(),
|
||||
legacy_permissions_enabled=True,
|
||||
)
|
||||
documents = KnowledgeDocuments(
|
||||
bound_page_ids=frozenset({PAGE_ID}),
|
||||
active_refs=(DatasetRef(TENANT_ID, DATASET_ID).document(DOCUMENT_ID),),
|
||||
)
|
||||
bindings = RecordingBindingStore()
|
||||
notion_source = RecordingNotionSource(
|
||||
workspaces=(
|
||||
NotionWorkspace(
|
||||
workspace_id="notion-workspace-1",
|
||||
workspace_name="One",
|
||||
workspace_icon=None,
|
||||
pages=(
|
||||
AuthorizedNotionPage(
|
||||
page_id=PAGE_ID,
|
||||
page_name="Page one",
|
||||
page_icon=NotionPageIcon(type="emoji", emoji="📄"),
|
||||
parent_id=None,
|
||||
page_type=NotionPageType.PAGE,
|
||||
),
|
||||
),
|
||||
),
|
||||
NotionWorkspace(
|
||||
workspace_id="notion-workspace-2",
|
||||
workspace_name="Two",
|
||||
workspace_icon=None,
|
||||
pages=(),
|
||||
),
|
||||
)
|
||||
)
|
||||
estimates = RecordingEstimateService()
|
||||
dispatcher = RecordingDispatcher()
|
||||
services = ApplicationServiceRegistry(
|
||||
data_sources=DataSourceApplicationServices(
|
||||
bindings=DataSourceBindingApplicationService(bindings=bindings),
|
||||
notion_imports=NotionImportApplicationService(
|
||||
dataset_access=dataset_access,
|
||||
datasets=dataset_catalog,
|
||||
documents=documents,
|
||||
source=notion_source,
|
||||
),
|
||||
),
|
||||
knowledge=KnowledgeApplicationServices(
|
||||
document_sync=DocumentSyncApplicationService(
|
||||
dataset_access=dataset_access,
|
||||
documents=documents,
|
||||
dispatcher=dispatcher,
|
||||
),
|
||||
indexing_estimates=estimates,
|
||||
),
|
||||
)
|
||||
monkeypatch.setitem(console_app.extensions, "application_services", services)
|
||||
return ControllerCollaborators(
|
||||
client=console_app.test_client(),
|
||||
account=account,
|
||||
bindings=bindings,
|
||||
notion_source=notion_source,
|
||||
estimates=estimates,
|
||||
dispatcher=dispatcher,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def test_integration_collection_only_exposes_get(collaborators: ControllerCollaborators) -> None:
|
||||
client = collaborators.client
|
||||
assert client.get("/console/api/data-source/integrates").status_code == 200
|
||||
assert client.patch("/console/api/data-source/integrates").status_code == 405
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.parametrize(
|
||||
("method", "path"),
|
||||
[
|
||||
("get", "/console/api/data-source/integrates"),
|
||||
("patch", f"/console/api/data-source/integrates/{BINDING_ID}/enable"),
|
||||
],
|
||||
)
|
||||
def test_integration_management_rejects_non_admin_role(
|
||||
collaborators: ControllerCollaborators,
|
||||
method: str,
|
||||
path: str,
|
||||
) -> None:
|
||||
collaborators.account.role = "editor"
|
||||
|
||||
assert collaborators.client.open(path, method=method.upper()).status_code == 403
|
||||
|
||||
@staticmethod
|
||||
def test_integration_management_enforces_workspace_credential_permission(
|
||||
collaborators: ControllerCollaborators,
|
||||
config_overrides: Callable[..., None],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
config_overrides(RBAC_ENABLED=True)
|
||||
calls: list[dict[str, object]] = []
|
||||
|
||||
def record_access(tenant_id: str, account_id: str, **requirement: object) -> bool:
|
||||
calls.append({"tenant_id": tenant_id, "account_id": account_id, **requirement})
|
||||
return True
|
||||
|
||||
monkeypatch.setattr("controllers.common.rbac.checks.RBACService.CheckAccess.check", record_access)
|
||||
|
||||
list_response = collaborators.client.get("/console/api/data-source/integrates")
|
||||
mutation_response = collaborators.client.patch(f"/console/api/data-source/integrates/{BINDING_ID}/enable")
|
||||
|
||||
assert list_response.status_code == 200
|
||||
assert mutation_response.status_code == 200
|
||||
assert calls == [
|
||||
{
|
||||
"tenant_id": TENANT_ID,
|
||||
"account_id": ACCOUNT_ID,
|
||||
"scene": RBACPermission.CREDENTIAL_MANAGE,
|
||||
"resource_type": None,
|
||||
"resource_id": None,
|
||||
},
|
||||
{
|
||||
"tenant_id": TENANT_ID,
|
||||
"account_id": ACCOUNT_ID,
|
||||
"scene": RBACPermission.CREDENTIAL_MANAGE,
|
||||
"resource_type": None,
|
||||
"resource_id": None,
|
||||
},
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.parametrize(("action", "disabled"), [("enable", False), ("disable", True)])
|
||||
def test_integration_item_only_exposes_patch_and_rejects_unknown_action(
|
||||
collaborators: ControllerCollaborators,
|
||||
action: str,
|
||||
disabled: bool,
|
||||
) -> None:
|
||||
client = collaborators.client
|
||||
item_path = f"/console/api/data-source/integrates/{BINDING_ID}/{action}"
|
||||
assert client.get(item_path).status_code == 405
|
||||
response = client.patch(item_path)
|
||||
assert response.status_code == 200
|
||||
assert response.get_json() == {"result": "success"}
|
||||
assert collaborators.bindings.mutations == [(TENANT_ID, BINDING_ID, disabled)]
|
||||
invalid = client.patch(f"/console/api/data-source/integrates/{BINDING_ID}/remove")
|
||||
assert invalid.status_code == 400
|
||||
assert invalid.get_json()["code"] == "invalid_action"
|
||||
assert len(collaborators.bindings.mutations) == 1
|
||||
|
||||
@staticmethod
|
||||
def test_integration_list_serializes_application_result_and_http_link(
|
||||
collaborators: ControllerCollaborators,
|
||||
) -> None:
|
||||
collaborators.bindings.bindings = (
|
||||
DataSourceBindingSummary(
|
||||
id=BINDING_ID,
|
||||
provider="notion",
|
||||
created_at=datetime(2026, 5, 25, 1, 2, 3, tzinfo=UTC),
|
||||
disabled=False,
|
||||
source_info={
|
||||
"workspace_name": "Workspace",
|
||||
"workspace_id": "workspace-1",
|
||||
"workspace_icon": None,
|
||||
"pages": [],
|
||||
"total": 0,
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
response = collaborators.client.get("/console/api/data-source/integrates")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.get_json()["data"][0] == {
|
||||
"id": BINDING_ID,
|
||||
"provider": "notion",
|
||||
"created_at": 1779670923,
|
||||
"is_bound": True,
|
||||
"disabled": False,
|
||||
"source_info": {
|
||||
"workspace_name": "Workspace",
|
||||
"workspace_id": "workspace-1",
|
||||
"workspace_icon": None,
|
||||
"pages": [],
|
||||
"total": 0,
|
||||
},
|
||||
"link": "https://console-api.example.com/console/api/oauth/data-source/notion",
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def test_notion_list_preserves_workspace_groups_and_bound_state(collaborators: ControllerCollaborators) -> None:
|
||||
response = collaborators.client.get(
|
||||
"/console/api/notion/pre-import/pages",
|
||||
query_string={"credential_id": "credential-1", "dataset_id": DATASET_ID},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.get_json()["notion_info"]
|
||||
assert [workspace["workspace_id"] for workspace in payload] == [
|
||||
"notion-workspace-1",
|
||||
"notion-workspace-2",
|
||||
]
|
||||
assert payload[0]["pages"][0]["is_bound"] is True
|
||||
assert payload[0]["pages"][0]["type"] == "page"
|
||||
assert "page_type" not in payload[0]["pages"][0]
|
||||
assert collaborators.notion_source.list_calls == [(TENANT_ID, ACCOUNT_ID, "credential-1")]
|
||||
|
||||
@staticmethod
|
||||
def test_notion_list_accepts_empty_dataset_id_for_new_dataset(collaborators: ControllerCollaborators) -> None:
|
||||
response = collaborators.client.get(
|
||||
"/console/api/notion/pre-import/pages",
|
||||
query_string={"credential_id": "credential-1", "dataset_id": ""},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.get_json()["notion_info"][0]["pages"][0]["is_bound"] is False
|
||||
assert collaborators.notion_source.list_calls == [(TENANT_ID, ACCOUNT_ID, "credential-1")]
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.parametrize("page_type", ["page", "database"])
|
||||
def test_notion_preview_validates_page_type_and_delegates(
|
||||
collaborators: ControllerCollaborators, page_type: str
|
||||
) -> None:
|
||||
response = collaborators.client.get(
|
||||
f"/console/api/notion/pages/{PAGE_ID}/{page_type}/preview",
|
||||
query_string={"credential_id": "credential-1"},
|
||||
)
|
||||
invalid = collaborators.client.get(
|
||||
f"/console/api/notion/pages/{PAGE_ID}/collection/preview",
|
||||
query_string={"credential_id": "credential-1"},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.get_json() == {"content": "preview text"}
|
||||
assert invalid.status_code == 400
|
||||
assert invalid.get_json()["code"] == "invalid_param"
|
||||
assert collaborators.notion_source.preview_calls == [
|
||||
(TENANT_ID, ACCOUNT_ID, "credential-1", PAGE_ID, page_type)
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.parametrize("path", ["/notion/pre-import/pages", f"/notion/pages/{PAGE_ID}/page/preview"])
|
||||
def test_notion_refresh_unavailable_returns_retryable_response(
|
||||
collaborators: ControllerCollaborators, path: str
|
||||
) -> None:
|
||||
error = CredentialRefreshUnavailableError()
|
||||
collaborators.notion_source.error = error
|
||||
response = collaborators.client.get(f"/console/api{path}", query_string={"credential_id": "credential-1"})
|
||||
assert response.status_code == 503
|
||||
assert response.get_json() == {
|
||||
"code": "service_unavailable",
|
||||
"message": str(error),
|
||||
"status": 503,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def test_notion_estimate_maps_nested_payload_to_shared_command(collaborators: ControllerCollaborators) -> None:
|
||||
response = collaborators.client.post(
|
||||
"/console/api/datasets/notion-indexing-estimate",
|
||||
json={
|
||||
"notion_info_list": [
|
||||
{
|
||||
"workspace_id": "notion-workspace-1",
|
||||
"credential_id": "credential-1",
|
||||
"pages": [{"page_id": PAGE_ID, "type": "page"}],
|
||||
}
|
||||
],
|
||||
"process_rule": {"mode": "automatic"},
|
||||
"doc_form": "text_model",
|
||||
"doc_language": "English",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.get_json()["total_segments"] == 3
|
||||
context, command = collaborators.estimates.calls[0]
|
||||
assert context.active_workspace_id == TENANT_ID
|
||||
assert context.account_id == ACCOUNT_ID
|
||||
assert command.process_rule == {"mode": "automatic"}
|
||||
source = command.sources[0]
|
||||
assert isinstance(source, NotionEstimateSource)
|
||||
assert source.credential_id == "credential-1"
|
||||
assert source.page_id == PAGE_ID
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.parametrize(
|
||||
("error", "expected_http_error"),
|
||||
[
|
||||
(IndexingEstimateCredentialUnavailableError(), NotFoundError),
|
||||
(EstimateSourceNotFoundError("page-1"), NotFoundError),
|
||||
(IndexingEstimateProviderUnavailableError(), ProviderNotInitializeError),
|
||||
(IndexingEstimateExecutionError(), IndexingEstimateError),
|
||||
],
|
||||
)
|
||||
def test_notion_estimate_maps_application_errors(
|
||||
error: Exception,
|
||||
expected_http_error: type[Exception],
|
||||
collaborators: ControllerCollaborators,
|
||||
) -> None:
|
||||
collaborators.estimates.error = error
|
||||
api = data_source_controller.DataSourceNotionIndexingEstimateApi()
|
||||
method = unwrap(api.post)
|
||||
payload = data_source_controller.NotionEstimatePayload.model_validate(
|
||||
{
|
||||
"notion_info_list": [
|
||||
{
|
||||
"workspace_id": "notion-workspace",
|
||||
"credential_id": "credential-1",
|
||||
"pages": [{"page_id": "page-1", "type": "page"}],
|
||||
}
|
||||
],
|
||||
"process_rule": {"mode": "automatic"},
|
||||
}
|
||||
)
|
||||
context = RequestContext("request-1", None, "account-1", "workspace-1")
|
||||
|
||||
with collaborators.client.application.app_context(), pytest.raises(expected_http_error):
|
||||
method(api, payload, context)
|
||||
|
||||
@staticmethod
|
||||
def test_sync_routes_dispatch_only_application_selected_documents(collaborators: ControllerCollaborators) -> None:
|
||||
dataset_response = collaborators.client.get(f"/console/api/datasets/{DATASET_ID}/notion/sync")
|
||||
document_response = collaborators.client.get(
|
||||
f"/console/api/datasets/{DATASET_ID}/documents/{DOCUMENT_ID}/notion/sync"
|
||||
)
|
||||
|
||||
assert dataset_response.status_code == 200
|
||||
assert dataset_response.get_json() == {"result": "success"}
|
||||
assert document_response.status_code == 200
|
||||
assert document_response.get_json() == {"result": "success"}
|
||||
assert collaborators.dispatcher.calls == [(DATASET_ID, DOCUMENT_ID), (DATASET_ID, DOCUMENT_ID)]
|
||||
|
||||
@staticmethod
|
||||
def test_sync_route_does_not_dispatch_cross_workspace_dataset(collaborators: ControllerCollaborators) -> None:
|
||||
response = collaborators.client.get(f"/console/api/datasets/{FOREIGN_DATASET_ID}/notion/sync")
|
||||
|
||||
assert response.status_code == 404
|
||||
assert response.get_json()["code"] == "not_found"
|
||||
assert collaborators.dispatcher.calls == []
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.parametrize(
|
||||
"resource",
|
||||
[data_source_controller.DataSourceNotionDatasetSyncApi, data_source_controller.DataSourceNotionDocumentSyncApi],
|
||||
)
|
||||
def test_notion_sync_requires_dataset_edit_permission(resource):
|
||||
[check] = rbac_checks(resource.get)
|
||||
assert check.scene is RBACPermission.DATASET_EDIT
|
||||
assert isinstance(check.locator, DatasetId)
|
||||
@@ -0,0 +1,597 @@
|
||||
import datetime
|
||||
from dataclasses import dataclass
|
||||
from inspect import unwrap
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, create_autospec, patch
|
||||
from uuid import UUID
|
||||
|
||||
import pytest
|
||||
from werkzeug.exceptions import BadRequest, Conflict, Forbidden, NotFound
|
||||
|
||||
import services
|
||||
from controllers.common.errors import InvalidArgumentError, NotFoundError
|
||||
from controllers.common.rbac import DatasetId, RBACPermission, Workspace
|
||||
from controllers.console.app.error import ProviderNotInitializeError
|
||||
from controllers.console.datasets import datasets as controller
|
||||
from controllers.console.datasets.datasets import (
|
||||
DatasetApi,
|
||||
DatasetApiBaseUrlApi,
|
||||
DatasetApiKeyApi,
|
||||
DatasetAutoDisableLogApi,
|
||||
DatasetCreatePayload,
|
||||
DatasetEnableApiApi,
|
||||
DatasetErrorDocs,
|
||||
DatasetIndexingEstimateApi,
|
||||
DatasetIndexingStatusApi,
|
||||
DatasetKnowledgeFSUpgradeApi,
|
||||
DatasetKnowledgeFSUpgradeJobApi,
|
||||
DatasetKnowledgeFSUpgradeJobsApi,
|
||||
DatasetListApi,
|
||||
DatasetPermissionUserListApi,
|
||||
DatasetQueryApi,
|
||||
DatasetRelatedAppListApi,
|
||||
DatasetRetrievalSettingApi,
|
||||
DatasetRetrievalSettingMockApi,
|
||||
DatasetUpdatePayload,
|
||||
DatasetUseCheckApi,
|
||||
IndexingEstimatePayload,
|
||||
_new_estimate_sources,
|
||||
)
|
||||
from controllers.console.datasets.error import (
|
||||
DatasetAccessDeniedRequestError,
|
||||
DatasetNameDuplicateError,
|
||||
IndexingEstimateError,
|
||||
KnowledgeFSUpgradeModelsUnavailableHTTPError,
|
||||
)
|
||||
from machinery.context import RequestContext
|
||||
from models.knowledge_fs import KnowledgeFSUpgradeJob, KnowledgeFSUpgradeJobStatus, KnowledgeFSUpgradeStage
|
||||
from services.data_source.entities.notion_import import NotionPageType
|
||||
from services.dataset_knowledge_fs_upgrade_service import (
|
||||
KnowledgeFSUpgradeConflictError,
|
||||
KnowledgeFSUpgradeModelsUnavailableError,
|
||||
KnowledgeFSUpgradeNotFoundError,
|
||||
)
|
||||
from services.knowledge.dataset_access import DatasetAccessDeniedError, DatasetNotFoundError
|
||||
from services.knowledge.datasets.application import DatasetApplicationService, DatasetListFilter
|
||||
from services.knowledge.datasets.upgrades import (
|
||||
KnowledgeFSUpgradeUnavailableError,
|
||||
KnowledgeUpgradeApplicationService,
|
||||
)
|
||||
from services.knowledge.entities.indexing_estimate import (
|
||||
NotionEstimateSource,
|
||||
UploadFileEstimateSource,
|
||||
WebsiteEstimateSource,
|
||||
)
|
||||
from services.knowledge.indexing.estimate import (
|
||||
EstimateSourceNotFoundError,
|
||||
IndexingEstimateCredentialUnavailableError,
|
||||
IndexingEstimateExecutionError,
|
||||
IndexingEstimateProviderUnavailableError,
|
||||
UnsupportedEstimateSourceError,
|
||||
)
|
||||
from services.knowledge_fs.product_dto import (
|
||||
KnowledgeFSUpgradeModelIssueResponse,
|
||||
KnowledgeFSUpgradeModelsPayload,
|
||||
KnowledgeFSUpgradePreparationResponse,
|
||||
)
|
||||
from tests.helpers.unit.controllers.rbac_introspection import rbac_checks
|
||||
|
||||
CONTEXT = RequestContext("request-1", None, "account-1", "tenant-1")
|
||||
DATASET_ID = UUID(int=1)
|
||||
|
||||
|
||||
def test_dataset_delete_requires_dataset_delete_permission() -> None:
|
||||
[check] = rbac_checks(DatasetApi.delete)
|
||||
|
||||
assert check.scene is RBACPermission.DATASET_DELETE
|
||||
assert isinstance(check.locator, DatasetId)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def datasets(monkeypatch):
|
||||
service = create_autospec(DatasetApplicationService, instance=True, spec_set=True)
|
||||
monkeypatch.setattr(
|
||||
controller, "application_services", lambda: SimpleNamespace(knowledge=SimpleNamespace(datasets=service))
|
||||
)
|
||||
return service
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("resource", "operation"),
|
||||
[
|
||||
(DatasetApi, lambda service: service.get_dataset),
|
||||
(DatasetUseCheckApi, lambda service: service.is_in_use),
|
||||
(DatasetQueryApi, lambda service: service.queries),
|
||||
(DatasetRelatedAppListApi, lambda service: service.related_apps),
|
||||
(DatasetIndexingStatusApi, lambda service: service.indexing_status),
|
||||
(DatasetErrorDocs, lambda service: service.error_documents),
|
||||
(DatasetPermissionUserListApi, lambda service: service.partial_members),
|
||||
(DatasetAutoDisableLogApi, lambda service: service.auto_disable_logs),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("error", "http_error"), [(DatasetNotFoundError(), NotFound), (DatasetAccessDeniedError(), Forbidden)]
|
||||
)
|
||||
def test_scoped_reads_pass_context_and_map_access_errors(app, datasets, resource, operation, error, http_error):
|
||||
method = operation(datasets)
|
||||
method.side_effect = error
|
||||
with app.test_request_context("/"), pytest.raises(http_error):
|
||||
unwrap(resource.get)(resource(), CONTEXT, DATASET_ID)
|
||||
assert method.call_args.args == (CONTEXT,)
|
||||
assert method.call_args.kwargs["dataset_id"] == str(DATASET_ID)
|
||||
|
||||
|
||||
def test_list_parses_repeated_filters_and_serializes_page(app, datasets):
|
||||
datasets.list_datasets.return_value = {"data": [], "page": 2, "limit": 3, "total": 4, "has_more": False}
|
||||
with app.test_request_context(
|
||||
"/?page=2&limit=3&ids=a&ids=b&tag_ids=x&tag_ids=y&creator_ids=c&include_all=true&keyword=term"
|
||||
):
|
||||
result, status = unwrap(DatasetListApi.get)(DatasetListApi(), CONTEXT)
|
||||
datasets.list_datasets.assert_called_once_with(
|
||||
CONTEXT,
|
||||
DatasetListFilter(
|
||||
page=2,
|
||||
limit=3,
|
||||
ids=["a", "b"],
|
||||
tag_ids=["x", "y"],
|
||||
creator_ids=["c"],
|
||||
include_all=True,
|
||||
keyword="term",
|
||||
),
|
||||
)
|
||||
assert status == 200
|
||||
assert result == datasets.list_datasets.return_value
|
||||
|
||||
|
||||
def test_patch_does_not_turn_omitted_fields_into_updates(datasets):
|
||||
datasets.update_dataset.side_effect = DatasetNotFoundError()
|
||||
with pytest.raises(NotFound):
|
||||
unwrap(DatasetApi.patch)(DatasetApi(), DatasetUpdatePayload(name="Changed"), CONTEXT, DATASET_ID)
|
||||
datasets.update_dataset.assert_called_once_with(CONTEXT, dataset_id=str(DATASET_ID), values={"name": "Changed"})
|
||||
|
||||
|
||||
def test_duplicate_name_is_mapped_at_transport_on_create(datasets):
|
||||
datasets.create_dataset.side_effect = services.errors.dataset.DatasetNameDuplicateError()
|
||||
with pytest.raises(DatasetNameDuplicateError):
|
||||
unwrap(DatasetListApi.post)(DatasetListApi(), DatasetCreatePayload(name="Duplicate"), CONTEXT)
|
||||
|
||||
|
||||
def test_delete_success_returns_empty_204(datasets):
|
||||
assert unwrap(DatasetApi.delete)(DatasetApi(), CONTEXT, DATASET_ID) == ("", 204)
|
||||
datasets.delete_dataset.assert_called_once_with(CONTEXT, dataset_id=str(DATASET_ID))
|
||||
|
||||
|
||||
def test_request_base_url_and_explicit_status_are_forwarded(app, datasets):
|
||||
datasets.api_base_url.return_value = "https://api.example/v1"
|
||||
with app.test_request_context("/", base_url="https://console.example/"):
|
||||
assert unwrap(DatasetApiBaseUrlApi.get)(DatasetApiBaseUrlApi(), CONTEXT) == {
|
||||
"api_base_url": "https://api.example/v1"
|
||||
}
|
||||
datasets.api_base_url.assert_called_once_with(CONTEXT, request_base_url="https://console.example")
|
||||
assert unwrap(DatasetEnableApiApi.post)(DatasetEnableApiApi(), CONTEXT, DATASET_ID, "disable") == (
|
||||
{"result": "success"},
|
||||
200,
|
||||
)
|
||||
datasets.set_api_enabled.assert_called_once_with(CONTEXT, dataset_id=str(DATASET_ID), status="disable")
|
||||
|
||||
|
||||
def test_retrieval_settings_forward_mock_flag(datasets):
|
||||
datasets.retrieval_settings.return_value = {"retrieval_method": ["semantic_search"]}
|
||||
assert (
|
||||
unwrap(DatasetRetrievalSettingApi.get)(DatasetRetrievalSettingApi(), CONTEXT)
|
||||
== datasets.retrieval_settings.return_value
|
||||
)
|
||||
unwrap(DatasetRetrievalSettingMockApi.get)(DatasetRetrievalSettingMockApi(), CONTEXT, "milvus")
|
||||
datasets.retrieval_settings.assert_called_with(CONTEXT, vector_type="milvus", is_mock=True)
|
||||
|
||||
|
||||
def test_new_estimate_sources_maps_each_supported_transport_shape() -> None:
|
||||
upload_sources = _new_estimate_sources(
|
||||
{"data_source_type": "upload_file", "file_info_list": {"file_ids": ["file-1", "file-2"]}}
|
||||
)
|
||||
notion_sources = _new_estimate_sources(
|
||||
{
|
||||
"data_source_type": "notion_import",
|
||||
"notion_info_list": [
|
||||
{
|
||||
"workspace_id": "notion-workspace",
|
||||
"credential_id": "credential-1",
|
||||
"pages": [{"page_id": "page-1", "type": "page"}],
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
website_sources = _new_estimate_sources(
|
||||
{
|
||||
"data_source_type": "website_crawl",
|
||||
"website_info_list": {
|
||||
"provider": "firecrawl",
|
||||
"job_id": "job-1",
|
||||
"urls": ["https://example.com/a", "https://example.com/b"],
|
||||
"only_main_content": True,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
assert upload_sources == (UploadFileEstimateSource("file-1"), UploadFileEstimateSource("file-2"))
|
||||
assert notion_sources == (NotionEstimateSource("notion-workspace", "page-1", NotionPageType.PAGE, "credential-1"),)
|
||||
assert website_sources == (
|
||||
WebsiteEstimateSource("firecrawl", "job-1", "https://example.com/a", only_main_content=True),
|
||||
WebsiteEstimateSource("firecrawl", "job-1", "https://example.com/b", only_main_content=True),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("value", "expected"), [("false", False), ("true", True), (False, False), (True, True)])
|
||||
def test_website_estimate_parses_boolean_values(value: str | bool, expected: bool) -> None:
|
||||
sources = _new_estimate_sources(
|
||||
{
|
||||
"data_source_type": "website_crawl",
|
||||
"website_info_list": {
|
||||
"provider": "firecrawl",
|
||||
"job_id": "job-1",
|
||||
"urls": ["https://example.com"],
|
||||
"only_main_content": value,
|
||||
},
|
||||
}
|
||||
)
|
||||
assert sources == (WebsiteEstimateSource("firecrawl", "job-1", "https://example.com", only_main_content=expected),)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("values", [{"only_main_content": "invalid"}, {"urls": [42]}])
|
||||
def test_website_estimate_rejects_invalid_field_types(values: dict[str, object]) -> None:
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match=(
|
||||
"(?:1 validation error for WebsiteInfo\\\nonly_main_content|1 validation error for WebsiteInfo\\\nurls\\.0)"
|
||||
),
|
||||
):
|
||||
_new_estimate_sources(
|
||||
{
|
||||
"data_source_type": "website_crawl",
|
||||
"website_info_list": {
|
||||
"provider": "firecrawl",
|
||||
"job_id": "job-1",
|
||||
"urls": ["https://example.com"],
|
||||
**values,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_new_estimate_sources_deduplicates_upload_ids_without_reordering() -> None:
|
||||
sources = _new_estimate_sources(
|
||||
{"data_source_type": "upload_file", "file_info_list": {"file_ids": ["file-2", "file-1", "file-2"]}}
|
||||
)
|
||||
|
||||
assert sources == (UploadFileEstimateSource("file-2"), UploadFileEstimateSource("file-1"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"info_list",
|
||||
[
|
||||
{"data_source_type": "upload_file", "file_info_list": {}},
|
||||
{"data_source_type": "notion_import", "notion_info_list": [{"workspace_id": "workspace"}]},
|
||||
{
|
||||
"data_source_type": "notion_import",
|
||||
"notion_info_list": [
|
||||
{
|
||||
"workspace_id": "workspace",
|
||||
"credential_id": "credential",
|
||||
"pages": [{"page_id": "page", "type": "unknown"}],
|
||||
}
|
||||
],
|
||||
},
|
||||
{"data_source_type": "unsupported"},
|
||||
],
|
||||
)
|
||||
def test_new_estimate_sources_rejects_malformed_transport_shapes(info_list: dict[str, object]) -> None:
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match=(
|
||||
"(?:1 validation error for FileInfo\\\nfile_ids|1 validation error for "
|
||||
"list\\[NotionEstimateWorkspacePayload\\]\\\n0\\.pages\\.0\\.type|2 "
|
||||
"validation errors for list\\[NotionEstimateWorkspacePayload\\]\\\n0\\."
|
||||
"credential_id|Data source type not support)"
|
||||
),
|
||||
):
|
||||
_new_estimate_sources(info_list)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("error", "expected_http_error"),
|
||||
[
|
||||
(IndexingEstimateCredentialUnavailableError(), NotFoundError),
|
||||
(EstimateSourceNotFoundError("source-1"), NotFoundError),
|
||||
(DatasetNotFoundError(), NotFoundError),
|
||||
(DatasetAccessDeniedError(), DatasetAccessDeniedRequestError),
|
||||
(UnsupportedEstimateSourceError("unsupported"), InvalidArgumentError),
|
||||
(IndexingEstimateProviderUnavailableError(), ProviderNotInitializeError),
|
||||
(IndexingEstimateExecutionError(), IndexingEstimateError),
|
||||
],
|
||||
)
|
||||
def test_new_source_estimate_maps_application_errors(
|
||||
error: Exception,
|
||||
expected_http_error: type[Exception],
|
||||
) -> None:
|
||||
estimates = MagicMock()
|
||||
estimates.estimate_new_sources.side_effect = error
|
||||
registry = SimpleNamespace(knowledge=SimpleNamespace(indexing_estimates=estimates))
|
||||
api = DatasetIndexingEstimateApi()
|
||||
method = unwrap(api.post)
|
||||
payload = IndexingEstimatePayload(
|
||||
info_list={"data_source_type": "upload_file", "file_info_list": {"file_ids": ["file-1"]}},
|
||||
process_rule={"mode": "automatic"},
|
||||
indexing_technique="economy",
|
||||
)
|
||||
context = RequestContext("request-1", None, "account-1", "workspace-1")
|
||||
|
||||
with patch("controllers.console.datasets.datasets.application_services", return_value=registry):
|
||||
with pytest.raises(expected_http_error):
|
||||
method(api, payload, context)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("dataset_id", "scene", "locator_type"),
|
||||
[
|
||||
("dataset-1", RBACPermission.DATASET_USE, DatasetId),
|
||||
(None, RBACPermission.DATASET_CREATE_AND_MANAGEMENT, Workspace),
|
||||
],
|
||||
)
|
||||
def test_new_source_estimate_authorizes_before_execution(dataset_id, scene, locator_type):
|
||||
estimates = MagicMock()
|
||||
registry = SimpleNamespace(knowledge=SimpleNamespace(indexing_estimates=estimates))
|
||||
payload = IndexingEstimatePayload(
|
||||
info_list={"data_source_type": "upload_file", "file_info_list": {"file_ids": ["file-1"]}},
|
||||
process_rule={"mode": "automatic"},
|
||||
indexing_technique="economy",
|
||||
dataset_id=dataset_id,
|
||||
)
|
||||
with (
|
||||
patch.object(controller, "application_services", return_value=registry),
|
||||
patch.object(controller, "enforce_rbac_checks", side_effect=Forbidden) as enforce_checks,
|
||||
pytest.raises(Forbidden),
|
||||
):
|
||||
unwrap(DatasetIndexingEstimateApi.post)(DatasetIndexingEstimateApi(), payload, CONTEXT)
|
||||
|
||||
enforce_checks.assert_called_once()
|
||||
kwargs = enforce_checks.call_args.kwargs
|
||||
assert kwargs["tenant_id"] == CONTEXT.active_workspace_id
|
||||
assert kwargs["account_id"] == CONTEXT.account_id
|
||||
assert kwargs["path_args"] == ({"dataset_id": dataset_id} if dataset_id else None)
|
||||
[check] = kwargs["checks"]
|
||||
assert check.scene is scene
|
||||
assert isinstance(check.locator, locator_type)
|
||||
estimates.estimate_new_sources.assert_not_called()
|
||||
|
||||
|
||||
def _upgrade_job(job_id: str = "upgrade-job-1") -> KnowledgeFSUpgradeJob:
|
||||
job = KnowledgeFSUpgradeJob(
|
||||
tenant_id="tenant-1",
|
||||
old_dataset_id=str(DATASET_ID),
|
||||
requested_by_account_id="account-1",
|
||||
owner_account_id="account-1",
|
||||
idempotency_key="upgrade-request-1",
|
||||
snapshot_at=datetime.datetime(2026, 8, 17, tzinfo=datetime.UTC),
|
||||
config_snapshot={},
|
||||
permission_snapshot={},
|
||||
app_binding_snapshot=[],
|
||||
tag_ids_snapshot=[],
|
||||
status=KnowledgeFSUpgradeJobStatus.QUEUED,
|
||||
stage=KnowledgeFSUpgradeStage.VALIDATING,
|
||||
total_documents=1,
|
||||
)
|
||||
job.id = job_id
|
||||
return job
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _UpgradeKnowledgeServices:
|
||||
upgrades: KnowledgeUpgradeApplicationService
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _UpgradeApplicationServices:
|
||||
knowledge: _UpgradeKnowledgeServices
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def upgrades(monkeypatch):
|
||||
service = create_autospec(KnowledgeUpgradeApplicationService, instance=True, spec_set=True)
|
||||
services = _UpgradeApplicationServices(knowledge=_UpgradeKnowledgeServices(upgrades=service))
|
||||
monkeypatch.setattr(controller, "application_services", lambda: services)
|
||||
return service
|
||||
|
||||
|
||||
def test_upgrade_writes_require_dataset_edit_permission() -> None:
|
||||
for method in (DatasetKnowledgeFSUpgradeApi.post, DatasetKnowledgeFSUpgradeJobApi.post):
|
||||
[check] = rbac_checks(method)
|
||||
assert check.scene is RBACPermission.DATASET_EDIT
|
||||
assert isinstance(check.locator, DatasetId)
|
||||
[check] = rbac_checks(DatasetKnowledgeFSUpgradeJobsApi.get)
|
||||
assert check.scene is RBACPermission.DATASET_READONLY
|
||||
assert isinstance(check.locator, Workspace)
|
||||
|
||||
|
||||
def test_upgrade_jobs_serializes_recoverable_jobs(app, upgrades):
|
||||
upgrades.list_recoverable_jobs.return_value = [_upgrade_job(job_id="job-1")]
|
||||
with app.test_request_context("/datasets/knowledge-fs-upgrade-jobs"):
|
||||
response = unwrap(DatasetKnowledgeFSUpgradeJobsApi.get)(DatasetKnowledgeFSUpgradeJobsApi(), CONTEXT)
|
||||
upgrades.list_recoverable_jobs.assert_called_once_with(CONTEXT)
|
||||
assert [job["id"] for job in response["data"]] == ["job-1"]
|
||||
|
||||
|
||||
def test_upgrade_start_forwards_idempotency_key(app, upgrades):
|
||||
upgrades.start.return_value = _upgrade_job()
|
||||
with app.test_request_context(
|
||||
f"/datasets/{DATASET_ID}/knowledge-fs-upgrades", headers={"Idempotency-Key": "upgrade-request-1"}
|
||||
):
|
||||
response, status = unwrap(DatasetKnowledgeFSUpgradeApi.post)(
|
||||
DatasetKnowledgeFSUpgradeApi(), CONTEXT, DATASET_ID
|
||||
)
|
||||
assert status == 202
|
||||
assert response["id"] == "upgrade-job-1"
|
||||
upgrades.start.assert_called_once_with(
|
||||
CONTEXT, dataset_id=str(DATASET_ID), idempotency_key="upgrade-request-1", models=None
|
||||
)
|
||||
|
||||
|
||||
def test_upgrade_start_forwards_all_model_choices(app, upgrades):
|
||||
upgrades.start.return_value = _upgrade_job()
|
||||
choices = {
|
||||
"embedding": {"provider": "langgenius/openai/openai", "model": "embedding"},
|
||||
"rerank": {"provider": "langgenius/cohere/cohere", "model": "rerank"},
|
||||
"reasoning": {"provider": "langgenius/anthropic/anthropic", "model": "reasoning"},
|
||||
}
|
||||
with app.test_request_context(
|
||||
f"/datasets/{DATASET_ID}/knowledge-fs-upgrades",
|
||||
method="POST",
|
||||
headers={"Idempotency-Key": "upgrade-request-1"},
|
||||
json={"models": choices},
|
||||
):
|
||||
response, status = unwrap(DatasetKnowledgeFSUpgradeApi.post)(
|
||||
DatasetKnowledgeFSUpgradeApi(), CONTEXT, DATASET_ID
|
||||
)
|
||||
assert status == 202
|
||||
assert response["id"] == "upgrade-job-1"
|
||||
upgrades.start.assert_called_once_with(
|
||||
CONTEXT,
|
||||
dataset_id=str(DATASET_ID),
|
||||
idempotency_key="upgrade-request-1",
|
||||
models=KnowledgeFSUpgradeModelsPayload.model_validate(choices),
|
||||
)
|
||||
|
||||
|
||||
def test_upgrade_start_rejects_incomplete_model_selection(app, upgrades):
|
||||
with (
|
||||
app.test_request_context(
|
||||
f"/datasets/{DATASET_ID}/knowledge-fs-upgrades",
|
||||
method="POST",
|
||||
headers={"Idempotency-Key": "upgrade-request-1"},
|
||||
json={"models": {"embedding": {"provider": "openai", "model": "embedding"}}},
|
||||
),
|
||||
pytest.raises(BadRequest, match="Invalid upgrade model selection"),
|
||||
):
|
||||
unwrap(DatasetKnowledgeFSUpgradeApi.post)(DatasetKnowledgeFSUpgradeApi(), CONTEXT, DATASET_ID)
|
||||
|
||||
upgrades.start.assert_not_called()
|
||||
|
||||
|
||||
def test_upgrade_start_returns_field_specific_model_errors(app, upgrades):
|
||||
upgrades.start.side_effect = KnowledgeFSUpgradeModelsUnavailableError(
|
||||
[
|
||||
KnowledgeFSUpgradeModelIssueResponse(
|
||||
field="embedding", reason="credential-removed", provider="openai", model="embedding"
|
||||
)
|
||||
]
|
||||
)
|
||||
with (
|
||||
app.test_request_context(
|
||||
f"/datasets/{DATASET_ID}/knowledge-fs-upgrades",
|
||||
method="POST",
|
||||
headers={"Idempotency-Key": "upgrade-request-1"},
|
||||
),
|
||||
pytest.raises(KnowledgeFSUpgradeModelsUnavailableHTTPError) as error,
|
||||
):
|
||||
unwrap(DatasetKnowledgeFSUpgradeApi.post)(DatasetKnowledgeFSUpgradeApi(), CONTEXT, DATASET_ID)
|
||||
assert error.value.data is not None
|
||||
assert error.value.data["details"]["issues"] == [
|
||||
{"field": "embedding", "reason": "credential-removed", "provider": "openai", "model": "embedding"}
|
||||
]
|
||||
|
||||
|
||||
def test_upgrade_start_requires_idempotency_key(app, upgrades):
|
||||
with (
|
||||
app.test_request_context(f"/datasets/{DATASET_ID}/knowledge-fs-upgrades"),
|
||||
pytest.raises(BadRequest, match="Idempotency-Key"),
|
||||
):
|
||||
unwrap(DatasetKnowledgeFSUpgradeApi.post)(DatasetKnowledgeFSUpgradeApi(), CONTEXT, DATASET_ID)
|
||||
upgrades.start.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("error", "http_error"),
|
||||
[
|
||||
(KnowledgeFSUpgradeUnavailableError(), NotFound),
|
||||
(DatasetNotFoundError(), NotFound),
|
||||
(DatasetAccessDeniedError(), Forbidden),
|
||||
(KnowledgeFSUpgradeConflictError("already running"), Conflict),
|
||||
],
|
||||
)
|
||||
def test_upgrade_start_maps_domain_errors(app, upgrades, error, http_error):
|
||||
upgrades.start.side_effect = error
|
||||
with (
|
||||
app.test_request_context(
|
||||
f"/datasets/{DATASET_ID}/knowledge-fs-upgrades", headers={"Idempotency-Key": "upgrade-request-1"}
|
||||
),
|
||||
pytest.raises(http_error),
|
||||
):
|
||||
unwrap(DatasetKnowledgeFSUpgradeApi.post)(DatasetKnowledgeFSUpgradeApi(), CONTEXT, DATASET_ID)
|
||||
|
||||
|
||||
def test_upgrade_discovery_serializes_service_result(app, upgrades):
|
||||
upgrades.discovery.return_value = KnowledgeFSUpgradePreparationResponse.model_validate(
|
||||
{
|
||||
"job": None,
|
||||
"can_upgrade": True,
|
||||
"can_retry": False,
|
||||
"models": {
|
||||
"embedding": {
|
||||
"value": {"provider": "openai", "model": "embedding"},
|
||||
"source": "legacy",
|
||||
"available": True,
|
||||
},
|
||||
"rerank": {
|
||||
"value": {"provider": "cohere", "model": "rerank"},
|
||||
"source": "workspace_default",
|
||||
"available": True,
|
||||
},
|
||||
"reasoning": {
|
||||
"value": {"provider": "anthropic", "model": "reasoning"},
|
||||
"source": "workspace_default",
|
||||
"available": False,
|
||||
"unavailable_reason": "credential-removed",
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
with app.test_request_context(f"/datasets/{DATASET_ID}/knowledge-fs-upgrades"):
|
||||
response = unwrap(DatasetKnowledgeFSUpgradeApi.get)(DatasetKnowledgeFSUpgradeApi(), CONTEXT, DATASET_ID)
|
||||
upgrades.discovery.assert_called_once_with(CONTEXT, dataset_id=str(DATASET_ID))
|
||||
assert response["can_upgrade"] is True
|
||||
assert response["models"]["embedding"]["value"] == {"provider": "openai", "model": "embedding"}
|
||||
assert response["models"]["reasoning"]["unavailable_reason"] == "credential-removed"
|
||||
|
||||
|
||||
def test_upgrade_job_status_maps_missing_job_to_not_found(app, upgrades):
|
||||
upgrades.get_job.side_effect = KnowledgeFSUpgradeNotFoundError("Upgrade job was not found")
|
||||
with (
|
||||
app.test_request_context(f"/datasets/{DATASET_ID}/knowledge-fs-upgrades/job-1"),
|
||||
pytest.raises(NotFound, match="Upgrade job was not found"),
|
||||
):
|
||||
unwrap(DatasetKnowledgeFSUpgradeJobApi.get)(DatasetKnowledgeFSUpgradeJobApi(), CONTEXT, DATASET_ID, "job-1")
|
||||
|
||||
|
||||
def test_upgrade_retry_returns_queued(app, upgrades):
|
||||
upgrades.retry.return_value = _upgrade_job(job_id="job-1")
|
||||
with app.test_request_context(f"/datasets/{DATASET_ID}/knowledge-fs-upgrades/job-1"):
|
||||
response, status = unwrap(DatasetKnowledgeFSUpgradeJobApi.post)(
|
||||
DatasetKnowledgeFSUpgradeJobApi(), CONTEXT, DATASET_ID, "job-1"
|
||||
)
|
||||
assert status == 202
|
||||
assert response == {"id": "job-1", "status": "queued"}
|
||||
|
||||
|
||||
def test_upgrade_retry_maps_disabled_feature_to_not_found(app, upgrades):
|
||||
upgrades.retry.side_effect = KnowledgeFSUpgradeUnavailableError()
|
||||
with (
|
||||
app.test_request_context(f"/datasets/{DATASET_ID}/knowledge-fs-upgrades/job-1"),
|
||||
pytest.raises(NotFound),
|
||||
):
|
||||
unwrap(DatasetKnowledgeFSUpgradeJobApi.post)(DatasetKnowledgeFSUpgradeJobApi(), CONTEXT, DATASET_ID, "job-1")
|
||||
|
||||
|
||||
def test_api_key_create_rejects_non_string_dataset_ids_with_the_standard_error(app):
|
||||
# A raised BadRequest gets the standard error body; ``Namespace.abort(message=...)`` replaced it.
|
||||
with app.test_request_context("/", method="POST", json={"dataset_ids": [123]}), pytest.raises(BadRequest) as error:
|
||||
unwrap(DatasetApiKeyApi.post)(DatasetApiKeyApi(), CONTEXT)
|
||||
assert error.value.description == "dataset_ids must be a list of strings."
|
||||
+784
@@ -0,0 +1,784 @@
|
||||
import datetime as _datasets_document_datetime
|
||||
import json
|
||||
from datetime import datetime as _datasets_document_download_datetime
|
||||
from inspect import unwrap
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, create_autospec, patch
|
||||
from uuid import UUID
|
||||
from zipfile import ZipFile
|
||||
|
||||
import pytest
|
||||
from flask import send_file
|
||||
from sqlalchemy import event
|
||||
from werkzeug.exceptions import Forbidden, NotFound
|
||||
|
||||
from controllers.common.errors import InvalidArgumentError, NotFoundError
|
||||
from controllers.common.rbac import DatasetByDocument, DatasetId, RBACPermission
|
||||
from controllers.console.app.error import ProviderNotInitializeError
|
||||
from controllers.console.datasets import datasets_document as controller
|
||||
from controllers.console.datasets.datasets_document import (
|
||||
DatasetDocumentListApi,
|
||||
DatasetInitApi,
|
||||
DocumentApi,
|
||||
DocumentBatchDownloadZipApi,
|
||||
DocumentBatchIndexingEstimateApi,
|
||||
DocumentBatchIndexingStatusApi,
|
||||
DocumentDownloadApi,
|
||||
DocumentGenerateSummaryApi,
|
||||
DocumentIndexingEstimateApi,
|
||||
DocumentIndexingStatusApi,
|
||||
DocumentMetadataApi,
|
||||
DocumentMetadataUpdatePayload,
|
||||
DocumentPauseApi,
|
||||
DocumentPipelineExecutionLogApi,
|
||||
DocumentProcessingApi,
|
||||
DocumentRecoverApi,
|
||||
DocumentRenameApi,
|
||||
DocumentRenamePayload,
|
||||
DocumentRetryApi,
|
||||
DocumentRetryPayload,
|
||||
DocumentStatusApi,
|
||||
DocumentSummaryStatusApi,
|
||||
GenerateSummaryPayload,
|
||||
GetProcessRuleApi,
|
||||
WebsiteDocumentSyncApi,
|
||||
)
|
||||
from controllers.console.datasets.error import (
|
||||
ArchivedDocumentImmutableError,
|
||||
DatasetAccessDeniedRequestError,
|
||||
DocumentAlreadyFinishedError,
|
||||
DocumentIndexingError,
|
||||
IndexingEstimateError,
|
||||
InvalidActionError,
|
||||
InvalidMetadataError,
|
||||
)
|
||||
from machinery.context import RequestContext
|
||||
from models.dataset import Dataset, Document
|
||||
from models.model import UploadFile
|
||||
from services.knowledge.dataset_access import (
|
||||
AccessibleDataset,
|
||||
DatasetAccess,
|
||||
DatasetAccessDeniedError,
|
||||
DatasetNotFoundError,
|
||||
)
|
||||
from services.knowledge.documents.adapters import SQLAlchemyDocumentOperations
|
||||
from services.knowledge.documents.application import (
|
||||
DatasetDocumentApplicationService,
|
||||
DocumentArchivedError,
|
||||
DocumentIndexingStateError,
|
||||
DocumentInvalidActionError,
|
||||
DocumentListFilter,
|
||||
DocumentNotFoundError,
|
||||
DocumentProviderError,
|
||||
)
|
||||
from services.knowledge.indexing.estimate import (
|
||||
EstimateDocumentAlreadyFinishedError,
|
||||
EstimateDocumentNotFoundError,
|
||||
EstimateSourceNotFoundError,
|
||||
IndexingEstimateCredentialUnavailableError,
|
||||
IndexingEstimateExecutionError,
|
||||
IndexingEstimateProviderUnavailableError,
|
||||
UnsupportedEstimateSourceError,
|
||||
)
|
||||
from tests.helpers.unit.controllers.rbac_introspection import rbac_checks
|
||||
|
||||
_DATASETS_DOCUMENT_CONTEXT = RequestContext("request-1", None, "account-1", "tenant-1")
|
||||
DS = UUID(int=1)
|
||||
DOC = UUID(int=2)
|
||||
|
||||
|
||||
class TestDatasetsDocumentCases:
|
||||
@staticmethod
|
||||
@pytest.fixture
|
||||
def documents(monkeypatch):
|
||||
documents = create_autospec(DatasetDocumentApplicationService, instance=True, spec_set=True)
|
||||
monkeypatch.setattr(
|
||||
controller, "application_services", lambda: SimpleNamespace(knowledge=SimpleNamespace(documents=documents))
|
||||
)
|
||||
monkeypatch.setattr(controller, "check_knowledge_rate_limit", lambda: None)
|
||||
return documents
|
||||
|
||||
@staticmethod
|
||||
def test_process_rule_authorizes_document_dataset_before_read(app, documents):
|
||||
with (
|
||||
app.test_request_context(f"/?document_id={DOC}"),
|
||||
patch.object(controller, "enforce_rbac_checks", side_effect=Forbidden) as enforce_checks,
|
||||
pytest.raises(Forbidden),
|
||||
):
|
||||
unwrap(GetProcessRuleApi.get)(GetProcessRuleApi(), _DATASETS_DOCUMENT_CONTEXT)
|
||||
|
||||
enforce_checks.assert_called_once()
|
||||
kwargs = enforce_checks.call_args.kwargs
|
||||
assert kwargs["tenant_id"] == _DATASETS_DOCUMENT_CONTEXT.active_workspace_id
|
||||
assert kwargs["account_id"] == _DATASETS_DOCUMENT_CONTEXT.account_id
|
||||
assert kwargs["path_args"] == {"document_id": str(DOC)}
|
||||
[check] = kwargs["checks"]
|
||||
assert check.scene is RBACPermission.DATASET_READONLY
|
||||
assert isinstance(check.locator, DatasetByDocument)
|
||||
documents.get_process_rule.assert_not_called()
|
||||
|
||||
@staticmethod
|
||||
def test_default_process_rule_does_not_require_dataset_permission(app, documents):
|
||||
documents.get_process_rule.return_value = {
|
||||
"mode": "automatic",
|
||||
"rules": {},
|
||||
"limits": {"indexing_max_segmentation_tokens_length": 4000},
|
||||
}
|
||||
with app.test_request_context("/"), patch.object(controller, "enforce_rbac_checks") as enforce_checks:
|
||||
unwrap(GetProcessRuleApi.get)(GetProcessRuleApi(), _DATASETS_DOCUMENT_CONTEXT)
|
||||
enforce_checks.assert_not_called()
|
||||
documents.get_process_rule.assert_called_once_with(_DATASETS_DOCUMENT_CONTEXT, document_id=None)
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.parametrize(
|
||||
("resource", "verb", "operation", "kwargs"),
|
||||
[
|
||||
(GetProcessRuleApi, GetProcessRuleApi.get, lambda service: service.get_process_rule, {}),
|
||||
(
|
||||
DatasetDocumentListApi,
|
||||
DatasetDocumentListApi.get,
|
||||
lambda service: service.list_documents,
|
||||
{"dataset_id": DS},
|
||||
),
|
||||
(
|
||||
DatasetDocumentListApi,
|
||||
DatasetDocumentListApi.post,
|
||||
lambda service: service.create_documents,
|
||||
{"dataset_id": DS},
|
||||
),
|
||||
(
|
||||
DatasetDocumentListApi,
|
||||
DatasetDocumentListApi.delete,
|
||||
lambda service: service.delete_documents,
|
||||
{"dataset_id": DS},
|
||||
),
|
||||
(DatasetInitApi, DatasetInitApi.post, lambda service: service.initialize_dataset, {}),
|
||||
(
|
||||
DocumentBatchIndexingStatusApi,
|
||||
DocumentBatchIndexingStatusApi.get,
|
||||
lambda service: service.get_batch_indexing_status,
|
||||
{"dataset_id": DS, "batch": "batch-1"},
|
||||
),
|
||||
(
|
||||
DocumentIndexingStatusApi,
|
||||
DocumentIndexingStatusApi.get,
|
||||
lambda service: service.get_indexing_status,
|
||||
{"dataset_id": DS, "document_id": DOC},
|
||||
),
|
||||
(
|
||||
DocumentApi,
|
||||
DocumentApi.get,
|
||||
lambda service: service.get_document,
|
||||
{"dataset_id": DS, "document_id": DOC},
|
||||
),
|
||||
(
|
||||
DocumentApi,
|
||||
DocumentApi.delete,
|
||||
lambda service: service.delete_document,
|
||||
{"dataset_id": DS, "document_id": DOC},
|
||||
),
|
||||
(
|
||||
DocumentDownloadApi,
|
||||
DocumentDownloadApi.get,
|
||||
lambda service: service.get_download_url,
|
||||
{"dataset_id": DS, "document_id": DOC},
|
||||
),
|
||||
(
|
||||
DocumentBatchDownloadZipApi,
|
||||
DocumentBatchDownloadZipApi.post,
|
||||
lambda service: service.build_download_zip,
|
||||
{"dataset_id": DS},
|
||||
),
|
||||
(
|
||||
DocumentProcessingApi,
|
||||
DocumentProcessingApi.patch,
|
||||
lambda service: service.update_processing,
|
||||
{"dataset_id": DS, "document_id": DOC, "action": "pause"},
|
||||
),
|
||||
(
|
||||
DocumentMetadataApi,
|
||||
DocumentMetadataApi.put,
|
||||
lambda service: service.update_metadata,
|
||||
{
|
||||
"dataset_id": DS,
|
||||
"document_id": DOC,
|
||||
"req_data": DocumentMetadataUpdatePayload(doc_type="book", doc_metadata={"title": "Book"}),
|
||||
},
|
||||
),
|
||||
(
|
||||
DocumentStatusApi,
|
||||
DocumentStatusApi.patch,
|
||||
lambda service: service.change_status,
|
||||
{"dataset_id": DS, "action": "enable"},
|
||||
),
|
||||
(
|
||||
DocumentPauseApi,
|
||||
DocumentPauseApi.patch,
|
||||
lambda service: service.pause_document,
|
||||
{"dataset_id": DS, "document_id": DOC},
|
||||
),
|
||||
(
|
||||
DocumentRecoverApi,
|
||||
DocumentRecoverApi.patch,
|
||||
lambda service: service.recover_document,
|
||||
{"dataset_id": DS, "document_id": DOC},
|
||||
),
|
||||
(
|
||||
DocumentRetryApi,
|
||||
DocumentRetryApi.post,
|
||||
lambda service: service.retry_documents,
|
||||
{"dataset_id": DS, "req_data": DocumentRetryPayload(document_ids=[str(DOC)])},
|
||||
),
|
||||
(
|
||||
DocumentRenameApi,
|
||||
DocumentRenameApi.post,
|
||||
lambda service: service.rename_document,
|
||||
{"dataset_id": DS, "document_id": DOC, "req_data": DocumentRenamePayload(name="New name")},
|
||||
),
|
||||
(
|
||||
WebsiteDocumentSyncApi,
|
||||
WebsiteDocumentSyncApi.get,
|
||||
lambda service: service.sync_website,
|
||||
{"dataset_id": DS, "document_id": DOC},
|
||||
),
|
||||
(
|
||||
DocumentPipelineExecutionLogApi,
|
||||
DocumentPipelineExecutionLogApi.get,
|
||||
lambda service: service.get_execution_log,
|
||||
{"dataset_id": DS, "document_id": DOC},
|
||||
),
|
||||
(
|
||||
DocumentGenerateSummaryApi,
|
||||
DocumentGenerateSummaryApi.post,
|
||||
lambda service: service.generate_summary,
|
||||
{"dataset_id": DS, "req_data": GenerateSummaryPayload(document_list=[str(DOC)])},
|
||||
),
|
||||
(
|
||||
DocumentSummaryStatusApi,
|
||||
DocumentSummaryStatusApi.get,
|
||||
lambda service: service.get_summary_status,
|
||||
{"dataset_id": DS, "document_id": DOC},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_endpoints_pass_request_context_and_map_application_errors(
|
||||
app, documents, resource, verb, operation, kwargs
|
||||
):
|
||||
operation(documents).side_effect = DocumentNotFoundError("Document not found.")
|
||||
with app.test_request_context(
|
||||
"/?document_id=" + str(DOC), json={"indexing_technique": "economy", "document_ids": [str(DOC)]}
|
||||
):
|
||||
with pytest.raises(NotFound):
|
||||
unwrap(verb)(resource(), request_context=_DATASETS_DOCUMENT_CONTEXT, **kwargs)
|
||||
called = operation(documents)
|
||||
assert called.call_count == 1
|
||||
assert called.call_args.args == (_DATASETS_DOCUMENT_CONTEXT,)
|
||||
if "dataset_id" in kwargs:
|
||||
assert called.call_args.kwargs["dataset_id"] == str(DS)
|
||||
if "document_id" in kwargs:
|
||||
assert called.call_args.kwargs["document_id"] == str(DOC)
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.parametrize(
|
||||
("fetch", "expected"), [("true", True), ("YES", True), ("false", False), ("invalid", False)]
|
||||
)
|
||||
def test_list_preserves_filters_pagination_and_fetch_parsing(app, documents, fetch, expected):
|
||||
documents.list_documents.return_value = {"data": [], "has_more": False, "page": 2, "limit": 3, "total": 3}
|
||||
with app.test_request_context(f"/?page=2&limit=3&keyword=needle&status=available&sort=hit_count&fetch={fetch}"):
|
||||
response = unwrap(DatasetDocumentListApi.get)(DatasetDocumentListApi(), _DATASETS_DOCUMENT_CONTEXT, DS)
|
||||
documents.list_documents.assert_called_once_with(
|
||||
_DATASETS_DOCUMENT_CONTEXT,
|
||||
dataset_id=str(DS),
|
||||
query=DocumentListFilter(
|
||||
page=2, limit=3, search="needle", status="available", sort="hit_count", fetch=expected
|
||||
),
|
||||
)
|
||||
assert response == {"data": [], "has_more": False, "page": 2, "limit": 3, "total": 3}
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.parametrize("metadata", ["only", "without", "all"])
|
||||
def test_document_detail_metadata_projection_and_raw_source(app, documents, metadata):
|
||||
documents.get_document.return_value = {
|
||||
"id": str(DOC),
|
||||
"doc_type": "others",
|
||||
"doc_metadata": [],
|
||||
"name": "Name",
|
||||
"data_source_info": {"file_path": "/path/to/local/file"},
|
||||
"data_source_detail_dict": {},
|
||||
}
|
||||
with app.test_request_context(f"/?metadata={metadata}"):
|
||||
response, status = unwrap(DocumentApi.get)(DocumentApi(), _DATASETS_DOCUMENT_CONTEXT, DS, DOC)
|
||||
assert status == 200
|
||||
documents.get_document.assert_called_once_with(
|
||||
_DATASETS_DOCUMENT_CONTEXT, dataset_id=str(DS), document_id=str(DOC), metadata_only=metadata == "only"
|
||||
)
|
||||
if metadata == "only":
|
||||
assert response == {"id": str(DOC), "doc_type": "others", "doc_metadata": []}
|
||||
else:
|
||||
assert response["data_source_info"] == {"file_path": "/path/to/local/file"}
|
||||
assert ("doc_metadata" in response) is (metadata == "all")
|
||||
|
||||
@staticmethod
|
||||
def test_invalid_metadata_fails_before_application_call(app, documents):
|
||||
with app.test_request_context("/?metadata=invalid"), pytest.raises(InvalidMetadataError):
|
||||
unwrap(DocumentApi.get)(DocumentApi(), _DATASETS_DOCUMENT_CONTEXT, DS, DOC)
|
||||
documents.get_document.assert_not_called()
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.parametrize(
|
||||
("resource", "verb", "kwargs"),
|
||||
[
|
||||
(DocumentApi, DocumentApi.delete, {"document_id": DOC}),
|
||||
(DatasetDocumentListApi, DatasetDocumentListApi.delete, {}),
|
||||
(DocumentPauseApi, DocumentPauseApi.patch, {"document_id": DOC}),
|
||||
(DocumentRecoverApi, DocumentRecoverApi.patch, {"document_id": DOC}),
|
||||
(DocumentRetryApi, DocumentRetryApi.post, {"req_data": DocumentRetryPayload(document_ids=[str(DOC)])}),
|
||||
],
|
||||
)
|
||||
@pytest.mark.usefixtures("documents")
|
||||
def test_no_content_mutations_return_empty_204(app, resource, verb, kwargs):
|
||||
with app.test_request_context("/?document_id=" + str(DOC)):
|
||||
result = unwrap(verb)(resource(), request_context=_DATASETS_DOCUMENT_CONTEXT, dataset_id=DS, **kwargs)
|
||||
assert result == ("", 204)
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.parametrize(
|
||||
("resource", "verb", "operation", "kwargs"),
|
||||
[
|
||||
(
|
||||
DocumentProcessingApi,
|
||||
DocumentProcessingApi.patch,
|
||||
lambda service: service.update_processing,
|
||||
{"document_id": DOC, "action": "resume"},
|
||||
),
|
||||
(DocumentStatusApi, DocumentStatusApi.patch, lambda service: service.change_status, {"action": "disable"}),
|
||||
(
|
||||
WebsiteDocumentSyncApi,
|
||||
WebsiteDocumentSyncApi.get,
|
||||
lambda service: service.sync_website,
|
||||
{"document_id": DOC},
|
||||
),
|
||||
(
|
||||
DocumentGenerateSummaryApi,
|
||||
DocumentGenerateSummaryApi.post,
|
||||
lambda service: service.generate_summary,
|
||||
{"req_data": GenerateSummaryPayload(document_list=[str(DOC)])},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_success_mutations_return_success_response(app, documents, resource, verb, operation, kwargs):
|
||||
with app.test_request_context("/?document_id=" + str(DOC)):
|
||||
result = unwrap(verb)(resource(), request_context=_DATASETS_DOCUMENT_CONTEXT, dataset_id=DS, **kwargs)
|
||||
assert result == ({"result": "success"}, 200)
|
||||
assert operation(documents).call_count == 1
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.parametrize("resource", [DatasetInitApi, DatasetDocumentListApi])
|
||||
def test_creation_serializes_materialized_dataset_and_documents(app, documents, resource):
|
||||
operation = documents.initialize_dataset if resource is DatasetInitApi else documents.create_documents
|
||||
operation.return_value = {
|
||||
"dataset": {
|
||||
"id": str(DS),
|
||||
"name": "Dataset",
|
||||
"created_at": _datasets_document_datetime.datetime(2024, 1, 1, tzinfo=_datasets_document_datetime.UTC),
|
||||
},
|
||||
"documents": [
|
||||
{
|
||||
"id": str(DOC),
|
||||
"name": "Document",
|
||||
"data_source_detail_dict": {},
|
||||
"data_source_info_dict": {},
|
||||
"doc_metadata_details": [],
|
||||
}
|
||||
],
|
||||
"batch": "batch-1",
|
||||
}
|
||||
with app.test_request_context("/", json={"indexing_technique": "economy"}):
|
||||
response = unwrap(resource.post)(
|
||||
resource(),
|
||||
request_context=_DATASETS_DOCUMENT_CONTEXT,
|
||||
**({} if resource is DatasetInitApi else {"dataset_id": DS}),
|
||||
)
|
||||
assert response["dataset"]["created_at"] == 1704067200
|
||||
assert response["documents"][0]["data_source_info"] == {}
|
||||
assert response["documents"][0]["doc_metadata"] == []
|
||||
assert response["batch"] == "batch-1"
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.parametrize("resource", [DocumentIndexingStatusApi, DocumentBatchIndexingStatusApi])
|
||||
def test_indexing_status_keeps_timestamps_counts_and_error_fields(app, documents, resource):
|
||||
value = {
|
||||
"id": str(DOC),
|
||||
"indexing_status": "paused",
|
||||
"completed_segments": 2,
|
||||
"total_segments": 3,
|
||||
**dict.fromkeys(
|
||||
(
|
||||
"processing_started_at",
|
||||
"parsing_completed_at",
|
||||
"cleaning_completed_at",
|
||||
"splitting_completed_at",
|
||||
"completed_at",
|
||||
"paused_at",
|
||||
"error",
|
||||
"stopped_at",
|
||||
)
|
||||
),
|
||||
}
|
||||
value["processing_started_at"] = _datasets_document_datetime.datetime(
|
||||
2024, 1, 1, tzinfo=_datasets_document_datetime.UTC
|
||||
)
|
||||
batch = resource is DocumentBatchIndexingStatusApi
|
||||
operation = documents.get_batch_indexing_status if batch else documents.get_indexing_status
|
||||
operation.return_value = {"data": [value]} if batch else value
|
||||
with app.test_request_context("/"):
|
||||
response = unwrap(resource.get)(
|
||||
resource(),
|
||||
request_context=_DATASETS_DOCUMENT_CONTEXT,
|
||||
dataset_id=DS,
|
||||
**({"batch": "batch-1"} if batch else {"document_id": DOC}),
|
||||
)
|
||||
row = response["data"][0] if batch else response
|
||||
assert row["indexing_status"] == "paused"
|
||||
assert row["processing_started_at"] == 1704067200
|
||||
assert row["completed_segments"] == 2
|
||||
assert row["total_segments"] == 3
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.parametrize(
|
||||
("translate", "error", "http_error"),
|
||||
[
|
||||
(controller._raise_document_error, DatasetNotFoundError(), NotFound),
|
||||
(controller._raise_document_error, DocumentNotFoundError("missing"), NotFound),
|
||||
(controller._raise_document_error, DatasetAccessDeniedError(), Forbidden),
|
||||
(controller._raise_document_indexing_error, DocumentIndexingStateError("indexing"), DocumentIndexingError),
|
||||
(
|
||||
controller._raise_document_indexing_error,
|
||||
DocumentProviderError("provider", kind="uninitialized"),
|
||||
ProviderNotInitializeError,
|
||||
),
|
||||
(controller._raise_document_action_error, DocumentInvalidActionError("invalid"), InvalidActionError),
|
||||
(controller._raise_document_status_error, DocumentInvalidActionError("invalid"), InvalidActionError),
|
||||
(controller._raise_document_status_error, DocumentIndexingStateError("indexing"), DocumentIndexingError),
|
||||
(controller._raise_document_archive_error, DocumentArchivedError(), ArchivedDocumentImmutableError),
|
||||
(controller._raise_document_archive_error, DocumentIndexingStateError("indexing"), DocumentIndexingError),
|
||||
# Every narrow translator falls back to the shared scope mapping.
|
||||
(controller._raise_document_archive_error, DatasetNotFoundError(), NotFound),
|
||||
],
|
||||
)
|
||||
def test_document_errors_are_mapped_at_transport_boundary(translate, error, http_error):
|
||||
with pytest.raises(http_error):
|
||||
translate(error)
|
||||
|
||||
@staticmethod
|
||||
def test_scope_translator_does_not_map_other_operations_errors():
|
||||
error = DocumentInvalidActionError("invalid")
|
||||
with pytest.raises(DocumentInvalidActionError) as raised:
|
||||
controller._raise_document_error(error)
|
||||
assert raised.value is error
|
||||
|
||||
@staticmethod
|
||||
def test_unexpected_error_is_not_hidden():
|
||||
error = RuntimeError("unexpected")
|
||||
with pytest.raises(RuntimeError) as raised:
|
||||
controller._raise_document_error(error)
|
||||
assert raised.value is error
|
||||
|
||||
class TestIndexingEstimateExceptionMapping:
|
||||
@staticmethod
|
||||
def _registry(method_name: str, error: Exception) -> SimpleNamespace:
|
||||
estimates = MagicMock()
|
||||
method = estimates.estimate_document if method_name == "estimate_document" else estimates.estimate_batch
|
||||
method.side_effect = error
|
||||
return SimpleNamespace(knowledge=SimpleNamespace(indexing_estimates=estimates))
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("error", "expected_http_error"),
|
||||
[
|
||||
(DatasetNotFoundError(), NotFoundError),
|
||||
(EstimateDocumentNotFoundError(), NotFoundError),
|
||||
(EstimateSourceNotFoundError("source-1"), NotFoundError),
|
||||
(IndexingEstimateCredentialUnavailableError(), NotFoundError),
|
||||
(DatasetAccessDeniedError(), DatasetAccessDeniedRequestError),
|
||||
(EstimateDocumentAlreadyFinishedError(), DocumentAlreadyFinishedError),
|
||||
(UnsupportedEstimateSourceError("unsupported"), InvalidArgumentError),
|
||||
(IndexingEstimateProviderUnavailableError(), ProviderNotInitializeError),
|
||||
(IndexingEstimateExecutionError(), IndexingEstimateError),
|
||||
],
|
||||
)
|
||||
def test_document_estimate_maps_application_errors(
|
||||
self,
|
||||
error: Exception,
|
||||
expected_http_error: type[Exception],
|
||||
) -> None:
|
||||
api = DocumentIndexingEstimateApi()
|
||||
method = unwrap(api.get)
|
||||
context = RequestContext("request-1", None, "account-1", "workspace-1")
|
||||
registry = self._registry("estimate_document", error)
|
||||
|
||||
with patch(
|
||||
"controllers.console.datasets.datasets_document.application_services",
|
||||
return_value=registry,
|
||||
):
|
||||
with pytest.raises(expected_http_error):
|
||||
method(api, context, UUID(int=1), UUID(int=2))
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("error", "expected_http_error"),
|
||||
[
|
||||
(DatasetNotFoundError(), NotFoundError),
|
||||
(EstimateDocumentNotFoundError(), NotFoundError),
|
||||
(EstimateSourceNotFoundError("source-1"), NotFoundError),
|
||||
(IndexingEstimateCredentialUnavailableError(), NotFoundError),
|
||||
(DatasetAccessDeniedError(), DatasetAccessDeniedRequestError),
|
||||
(EstimateDocumentAlreadyFinishedError(), DocumentAlreadyFinishedError),
|
||||
(UnsupportedEstimateSourceError("unsupported"), InvalidArgumentError),
|
||||
(IndexingEstimateProviderUnavailableError(), ProviderNotInitializeError),
|
||||
(IndexingEstimateExecutionError(), IndexingEstimateError),
|
||||
],
|
||||
)
|
||||
def test_batch_estimate_maps_application_errors(
|
||||
self,
|
||||
error: Exception,
|
||||
expected_http_error: type[Exception],
|
||||
) -> None:
|
||||
api = DocumentBatchIndexingEstimateApi()
|
||||
method = unwrap(api.get)
|
||||
context = RequestContext("request-1", None, "account-1", "workspace-1")
|
||||
registry = self._registry("estimate_batch", error)
|
||||
|
||||
with patch(
|
||||
"controllers.console.datasets.datasets_document.application_services",
|
||||
return_value=registry,
|
||||
):
|
||||
with pytest.raises(expected_http_error):
|
||||
method(api, context, UUID(int=1), "unknown-batch")
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.parametrize(
|
||||
"method",
|
||||
[
|
||||
DatasetDocumentListApi.get,
|
||||
DocumentBatchIndexingStatusApi.get,
|
||||
DocumentIndexingStatusApi.get,
|
||||
DocumentApi.get,
|
||||
DocumentPipelineExecutionLogApi.get,
|
||||
DocumentSummaryStatusApi.get,
|
||||
],
|
||||
)
|
||||
def test_document_read_routes_require_dataset_readonly_permission(method) -> None:
|
||||
[check] = rbac_checks(method)
|
||||
|
||||
assert check.scene is RBACPermission.DATASET_READONLY
|
||||
assert isinstance(check.locator, DatasetId)
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.parametrize(
|
||||
"method",
|
||||
[DocumentIndexingEstimateApi.get, DocumentBatchIndexingEstimateApi.get],
|
||||
)
|
||||
def test_document_indexing_estimates_require_dataset_use_permission(method) -> None:
|
||||
[check] = rbac_checks(method)
|
||||
|
||||
assert check.scene is RBACPermission.DATASET_USE
|
||||
assert isinstance(check.locator, DatasetId)
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.parametrize(
|
||||
("method", "expected_scene"),
|
||||
[
|
||||
(DatasetDocumentListApi.post, RBACPermission.DATASET_USE),
|
||||
(DatasetDocumentListApi.delete, RBACPermission.DATASET_DELETE_FILE),
|
||||
(DocumentApi.delete, RBACPermission.DATASET_DELETE_FILE),
|
||||
(DocumentBatchDownloadZipApi.post, RBACPermission.DATASET_DOCUMENT_DOWNLOAD),
|
||||
],
|
||||
)
|
||||
def test_document_mutation_routes_require_operation_specific_permissions(method, expected_scene) -> None:
|
||||
[check] = rbac_checks(method)
|
||||
|
||||
assert check.scene is expected_scene
|
||||
assert isinstance(check.locator, DatasetId)
|
||||
|
||||
|
||||
"""Document downloads through the controller, application service, and SQLite adapter."""
|
||||
|
||||
|
||||
_DATASETS_DOCUMENT_DOWNLOAD_CONTEXT = RequestContext("request-1", None, "actor-1", "tenant-1")
|
||||
DOC_ID = str(UUID(int=1))
|
||||
FILE_ID = str(UUID(int=2))
|
||||
|
||||
|
||||
def _document(**values):
|
||||
return Document(
|
||||
**{
|
||||
"id": DOC_ID,
|
||||
"tenant_id": "tenant-1",
|
||||
"dataset_id": "dataset-1",
|
||||
"position": 1,
|
||||
"data_source_type": "upload_file",
|
||||
"data_source_info": json.dumps({"upload_file_id": FILE_ID}),
|
||||
"batch": "batch-1",
|
||||
"name": "document.txt",
|
||||
"created_from": "web",
|
||||
"created_by": "actor-1",
|
||||
**values,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class TestDatasetsDocumentDownloadCases:
|
||||
@staticmethod
|
||||
@pytest.fixture(autouse=True)
|
||||
def downloads(monkeypatch, sqlite_session_factory):
|
||||
with sqlite_session_factory.begin() as session:
|
||||
session.add(Dataset(id="dataset-1", tenant_id="tenant-1", name="Dataset", created_by="actor-1"))
|
||||
session.add(_document())
|
||||
upload = UploadFile(
|
||||
tenant_id="tenant-1",
|
||||
storage_type="local",
|
||||
key="key-1",
|
||||
name="document.txt",
|
||||
size=5,
|
||||
extension="txt",
|
||||
mime_type="text/plain",
|
||||
created_by_role="account",
|
||||
created_by="actor-1",
|
||||
created_at=_datasets_document_download_datetime(2024, 1, 1),
|
||||
used=False,
|
||||
)
|
||||
upload.id = FILE_ID
|
||||
session.add(upload)
|
||||
access = create_autospec(DatasetAccess, instance=True, spec_set=True)
|
||||
access.require_accessible.return_value = AccessibleDataset("dataset-1", "tenant-1")
|
||||
documents = DatasetDocumentApplicationService(
|
||||
dataset_access=access,
|
||||
operations=SQLAlchemyDocumentOperations(session_factory=sqlite_session_factory),
|
||||
metadata_schema={},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
controller, "application_services", lambda: SimpleNamespace(knowledge=SimpleNamespace(documents=documents))
|
||||
)
|
||||
return documents
|
||||
|
||||
@staticmethod
|
||||
def test_document_download_returns_signed_url(app, monkeypatch):
|
||||
from services.knowledge import dataset_service
|
||||
|
||||
signed = []
|
||||
|
||||
def sign(**kwargs):
|
||||
signed.append(kwargs)
|
||||
return "https://files.example/download"
|
||||
|
||||
monkeypatch.setattr(dataset_service.file_helpers, "get_signed_file_url", sign)
|
||||
with app.test_request_context("/"):
|
||||
result = unwrap(controller.DocumentDownloadApi.get)(
|
||||
controller.DocumentDownloadApi(), _DATASETS_DOCUMENT_DOWNLOAD_CONTEXT, "dataset-1", DOC_ID
|
||||
)
|
||||
assert result == {"url": "https://files.example/download"}
|
||||
assert signed == [{"upload_file_id": FILE_ID, "as_attachment": True}]
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.parametrize(
|
||||
"values",
|
||||
[
|
||||
{"data_source_type": "website_crawl"},
|
||||
{"data_source_info": "{}"},
|
||||
{"data_source_info": json.dumps({"upload_file_id": "missing"})},
|
||||
{"tenant_id": "foreign"},
|
||||
{"dataset_id": "foreign"},
|
||||
],
|
||||
)
|
||||
def test_invalid_or_foreign_document_is_not_downloadable(app, sqlite_session_factory, values):
|
||||
with sqlite_session_factory.begin() as session:
|
||||
document = session.get(Document, DOC_ID)
|
||||
for key, value in values.items():
|
||||
setattr(document, key, value)
|
||||
with app.test_request_context("/"), pytest.raises(NotFound):
|
||||
unwrap(controller.DocumentDownloadApi.get)(
|
||||
controller.DocumentDownloadApi(), _DATASETS_DOCUMENT_DOWNLOAD_CONTEXT, "dataset-1", DOC_ID
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def test_cross_tenant_upload_file_is_not_signed(app, sqlite_session_factory):
|
||||
with sqlite_session_factory.begin() as session:
|
||||
session.get(UploadFile, FILE_ID).tenant_id = "foreign"
|
||||
with app.test_request_context("/"), pytest.raises(NotFound):
|
||||
unwrap(controller.DocumentDownloadApi.get)(
|
||||
controller.DocumentDownloadApi(), _DATASETS_DOCUMENT_DOWNLOAD_CONTEXT, "dataset-1", DOC_ID
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def test_zip_is_readable_and_cleanup_waits_for_response_close(app, monkeypatch, sqlite_engine):
|
||||
from services import file_service
|
||||
|
||||
paths = []
|
||||
checked_out = set()
|
||||
|
||||
def checkout(_connection, record, _proxy):
|
||||
checked_out.add(record)
|
||||
|
||||
def checkin(_connection, record):
|
||||
checked_out.discard(record)
|
||||
|
||||
def load(key, *, stream):
|
||||
assert not checked_out, "storage I/O must run after the read transaction closes"
|
||||
assert key == "key-1"
|
||||
assert stream is True
|
||||
return iter([b"hello"])
|
||||
|
||||
def send(path, **kwargs):
|
||||
paths.append(Path(path))
|
||||
return send_file(path, **kwargs)
|
||||
|
||||
monkeypatch.setattr(file_service.storage, "load", load)
|
||||
monkeypatch.setattr(controller, "send_file", send)
|
||||
event.listen(sqlite_engine, "checkout", checkout)
|
||||
event.listen(sqlite_engine, "checkin", checkin)
|
||||
try:
|
||||
with app.test_request_context("/", json={"document_ids": [DOC_ID]}):
|
||||
response = unwrap(controller.DocumentBatchDownloadZipApi.post)(
|
||||
controller.DocumentBatchDownloadZipApi(), _DATASETS_DOCUMENT_DOWNLOAD_CONTEXT, "dataset-1"
|
||||
)
|
||||
assert paths[0].exists()
|
||||
response.direct_passthrough = False
|
||||
with ZipFile(BytesIO(response.get_data())) as archive:
|
||||
assert archive.read("document.txt") == b"hello"
|
||||
response.close()
|
||||
assert not paths[0].exists()
|
||||
finally:
|
||||
event.remove(sqlite_engine, "checkout", checkout)
|
||||
event.remove(sqlite_engine, "checkin", checkin)
|
||||
|
||||
@staticmethod
|
||||
def test_zip_cleanup_when_response_creation_fails(app, monkeypatch):
|
||||
from services import file_service
|
||||
|
||||
paths = []
|
||||
monkeypatch.setattr(file_service.storage, "load", lambda *_args, **_kwargs: iter([b"hello"]))
|
||||
|
||||
def fail_send(path, **_kwargs):
|
||||
paths.append(Path(path))
|
||||
raise RuntimeError("response failed")
|
||||
|
||||
monkeypatch.setattr(controller, "send_file", fail_send)
|
||||
with (
|
||||
app.test_request_context("/", json={"document_ids": [DOC_ID]}),
|
||||
pytest.raises(RuntimeError, match="response failed"),
|
||||
):
|
||||
unwrap(controller.DocumentBatchDownloadZipApi.post)(
|
||||
controller.DocumentBatchDownloadZipApi(), _DATASETS_DOCUMENT_DOWNLOAD_CONTEXT, "dataset-1"
|
||||
)
|
||||
assert paths
|
||||
assert not paths[0].exists()
|
||||
|
||||
@staticmethod
|
||||
def test_zip_rejects_non_upload_document(app, sqlite_session_factory):
|
||||
with sqlite_session_factory.begin() as session:
|
||||
session.get(Document, DOC_ID).data_source_type = "website_crawl"
|
||||
with app.test_request_context("/", json={"document_ids": [DOC_ID]}), pytest.raises(NotFound):
|
||||
unwrap(controller.DocumentBatchDownloadZipApi.post)(
|
||||
controller.DocumentBatchDownloadZipApi(), _DATASETS_DOCUMENT_DOWNLOAD_CONTEXT, "dataset-1"
|
||||
)
|
||||
+402
@@ -0,0 +1,402 @@
|
||||
from collections.abc import Callable
|
||||
from contextlib import AbstractContextManager
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from inspect import unwrap
|
||||
from typing import Never
|
||||
from unittest.mock import create_autospec, patch
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
from werkzeug.exceptions import Forbidden, NotFound
|
||||
|
||||
from controllers.common.controller_schemas import ChildChunkCreatePayload, ChildChunkUpdatePayload
|
||||
from controllers.console.app.error import ProviderNotInitializeError
|
||||
from controllers.console.datasets.datasets_segments import (
|
||||
BatchImportPayload,
|
||||
ChildChunkAddApi,
|
||||
ChildChunkBatchUpdatePayload,
|
||||
ChildChunkUpdateApi,
|
||||
DatasetDocumentSegmentAddApi,
|
||||
DatasetDocumentSegmentApi,
|
||||
DatasetDocumentSegmentBatchImportApi,
|
||||
DatasetDocumentSegmentBatchImportStatusApi,
|
||||
DatasetDocumentSegmentListApi,
|
||||
DatasetDocumentSegmentUpdateApi,
|
||||
SegmentCreatePayload,
|
||||
SegmentUpdatePayload,
|
||||
_raise_child_chunk_create_error,
|
||||
_raise_child_chunk_delete_error,
|
||||
_raise_child_chunk_write_error,
|
||||
_raise_segment_embedding_error,
|
||||
_raise_segment_error,
|
||||
_raise_segment_status_error,
|
||||
)
|
||||
from controllers.console.datasets.error import ChildChunkDeleteIndexError, ChildChunkIndexingError, InvalidActionError
|
||||
from machinery.context import RequestContext
|
||||
from services.knowledge.entities.segments import ChildChunkRecord, ChildChunkUpdateArgs, SegmentRecord
|
||||
from services.knowledge.segments.application import (
|
||||
ChildChunkDeleteIndexApplicationError,
|
||||
ChildChunkIndexingApplicationError,
|
||||
ChildChunkNotFoundError,
|
||||
ChildChunkPage,
|
||||
DatasetSegmentApplicationService,
|
||||
SegmentBatchImport,
|
||||
SegmentBatchImportDispatchError,
|
||||
SegmentBatchImportNotFoundError,
|
||||
SegmentDatasetModelUnavailableError,
|
||||
SegmentDatasetNotFoundError,
|
||||
SegmentDetail,
|
||||
SegmentDocumentIndexingError,
|
||||
SegmentDocumentNotFoundError,
|
||||
SegmentEmbeddingModelUnavailableError,
|
||||
SegmentInvalidFileTypeError,
|
||||
SegmentListFilter,
|
||||
SegmentNotFoundError,
|
||||
SegmentPage,
|
||||
SegmentPermissionDeniedError,
|
||||
SegmentStatusUpdateError,
|
||||
SegmentUploadFileNotFoundError,
|
||||
)
|
||||
|
||||
|
||||
def _context() -> RequestContext:
|
||||
return RequestContext(
|
||||
request_id="request-1",
|
||||
trace_id=None,
|
||||
account_id="account-1",
|
||||
active_workspace_id="tenant-1",
|
||||
)
|
||||
|
||||
|
||||
def _child_chunk_data() -> ChildChunkRecord:
|
||||
now = datetime(2026, 1, 1, tzinfo=UTC)
|
||||
return ChildChunkRecord.model_validate(
|
||||
{
|
||||
"id": "child-1",
|
||||
"segment_id": "segment-1",
|
||||
"content": "child",
|
||||
"position": 1,
|
||||
"word_count": 5,
|
||||
"type": "customized",
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _segment_data() -> SegmentRecord:
|
||||
now = datetime(2026, 1, 1, tzinfo=UTC)
|
||||
return SegmentRecord.model_validate(
|
||||
{
|
||||
"id": "segment-1",
|
||||
"position": 1,
|
||||
"document_id": "document-1",
|
||||
"content": "content",
|
||||
"sign_content": "content",
|
||||
"answer": None,
|
||||
"word_count": 7,
|
||||
"tokens": 2,
|
||||
"keywords": [],
|
||||
"index_node_id": "node-1",
|
||||
"index_node_hash": "hash",
|
||||
"hit_count": 0,
|
||||
"enabled": True,
|
||||
"disabled_at": None,
|
||||
"disabled_by": None,
|
||||
"status": "completed",
|
||||
"created_by": "account-1",
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
"updated_by": None,
|
||||
"indexing_at": None,
|
||||
"completed_at": now,
|
||||
"error": None,
|
||||
"stopped_at": None,
|
||||
"child_chunks": [],
|
||||
"attachments": [],
|
||||
"summary": None,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class KnowledgeServiceStub:
|
||||
segments: DatasetSegmentApplicationService
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ApplicationServiceStub:
|
||||
knowledge: KnowledgeServiceStub
|
||||
|
||||
|
||||
def _patch_services(segments: DatasetSegmentApplicationService) -> AbstractContextManager[object]:
|
||||
return patch(
|
||||
"controllers.console.datasets.datasets_segments.application_services",
|
||||
return_value=ApplicationServiceStub(knowledge=KnowledgeServiceStub(segments=segments)),
|
||||
)
|
||||
|
||||
|
||||
def test_list_segments_delegates_typed_query_and_serializes_page(app: Flask) -> None:
|
||||
segments = create_autospec(DatasetSegmentApplicationService, instance=True, spec_set=True)
|
||||
segments.list_segments.return_value = SegmentPage(
|
||||
items=(_segment_data(),), total=1, total_pages=1, page=2, limit=10
|
||||
)
|
||||
method = unwrap(DatasetDocumentSegmentListApi.get)
|
||||
|
||||
with (
|
||||
app.test_request_context("/?status=completed&limit=10&page=2&enabled=true&keyword=needle"),
|
||||
_patch_services(segments),
|
||||
):
|
||||
response, status = method(DatasetDocumentSegmentListApi(), _context(), "dataset-1", "document-1")
|
||||
|
||||
assert status == 200
|
||||
assert response["data"][0]["id"] == "segment-1"
|
||||
assert response["data"][0]["created_at"] == 1767225600
|
||||
assert response["data"][0]["keywords"] == []
|
||||
assert response["data"][0]["disabled_at"] is None
|
||||
query = segments.list_segments.call_args.kwargs["query"]
|
||||
assert query == SegmentListFilter(
|
||||
page=2,
|
||||
limit=10,
|
||||
statuses=("completed",),
|
||||
enabled="true",
|
||||
keyword="needle",
|
||||
)
|
||||
|
||||
|
||||
def test_delete_segments_delegates_all_selected_ids(app: Flask) -> None:
|
||||
segments = create_autospec(DatasetSegmentApplicationService, instance=True, spec_set=True)
|
||||
method = unwrap(DatasetDocumentSegmentListApi.delete)
|
||||
|
||||
with app.test_request_context("/?segment_id=one&segment_id=two"), _patch_services(segments):
|
||||
response, status = method(DatasetDocumentSegmentListApi(), _context(), "dataset-1", "document-1")
|
||||
|
||||
assert (response, status) == ("", 204)
|
||||
assert segments.delete_segments.call_args.kwargs["segment_ids"] == ["one", "two"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("action", ["enable", "disable"])
|
||||
def test_change_segment_status_delegates_action(app: Flask, action: str) -> None:
|
||||
segments = create_autospec(DatasetSegmentApplicationService, instance=True, spec_set=True)
|
||||
method = unwrap(DatasetDocumentSegmentApi.patch)
|
||||
|
||||
with app.test_request_context("/?segment_id=one"), _patch_services(segments):
|
||||
response, status = method(DatasetDocumentSegmentApi(), _context(), "dataset-1", "document-1", action)
|
||||
|
||||
assert status == 200
|
||||
assert response == {"result": "success"}
|
||||
assert segments.change_segment_status.call_args.kwargs["action"] == action
|
||||
|
||||
|
||||
@pytest.mark.parametrize("action", ["typo", "ENABLE", "enable "])
|
||||
def test_change_segment_status_rejects_unknown_action_before_service_call(app: Flask, action: str) -> None:
|
||||
segments = create_autospec(DatasetSegmentApplicationService, instance=True, spec_set=True)
|
||||
method = unwrap(DatasetDocumentSegmentApi.patch)
|
||||
|
||||
with (
|
||||
app.test_request_context("/?segment_id=one"),
|
||||
_patch_services(segments),
|
||||
pytest.raises(InvalidActionError) as error,
|
||||
):
|
||||
method(DatasetDocumentSegmentApi(), _context(), "dataset-1", "document-1", action)
|
||||
|
||||
assert error.value.code == 400
|
||||
segments.change_segment_status.assert_not_called()
|
||||
|
||||
|
||||
def test_create_segment_delegates_payload_and_serializes_detail(app: Flask) -> None:
|
||||
segments = create_autospec(DatasetSegmentApplicationService, instance=True, spec_set=True)
|
||||
segments.create_segment.return_value = SegmentDetail(data=_segment_data(), doc_form="text_model")
|
||||
method = unwrap(DatasetDocumentSegmentAddApi.post)
|
||||
|
||||
with app.test_request_context("/"), _patch_services(segments):
|
||||
response, status = method(
|
||||
DatasetDocumentSegmentAddApi(),
|
||||
SegmentCreatePayload(content="content"),
|
||||
_context(),
|
||||
"dataset-1",
|
||||
"document-1",
|
||||
)
|
||||
|
||||
assert status == 200
|
||||
assert response["data"]["id"] == "segment-1"
|
||||
assert segments.create_segment.call_args.kwargs["values"] == {"content": "content"}
|
||||
|
||||
|
||||
def test_update_and_delete_segment_delegate_owned_ids(app: Flask) -> None:
|
||||
segments = create_autospec(DatasetSegmentApplicationService, instance=True, spec_set=True)
|
||||
segments.update_segment.return_value = SegmentDetail(data=_segment_data(), doc_form="text_model")
|
||||
|
||||
with app.test_request_context("/"), _patch_services(segments):
|
||||
update_response, update_status = unwrap(DatasetDocumentSegmentUpdateApi.patch)(
|
||||
DatasetDocumentSegmentUpdateApi(),
|
||||
SegmentUpdatePayload(content="updated"),
|
||||
_context(),
|
||||
"dataset-1",
|
||||
"document-1",
|
||||
"segment-1",
|
||||
)
|
||||
delete_response, delete_status = unwrap(DatasetDocumentSegmentUpdateApi.delete)(
|
||||
DatasetDocumentSegmentUpdateApi(),
|
||||
_context(),
|
||||
"dataset-1",
|
||||
"document-1",
|
||||
"segment-1",
|
||||
)
|
||||
|
||||
assert update_status == 200
|
||||
assert update_response["data"]["content"] == "content"
|
||||
assert (delete_response, delete_status) == ("", 204)
|
||||
assert segments.update_segment.call_args.kwargs["segment_id"] == "segment-1"
|
||||
assert segments.delete_segment.call_args.kwargs["segment_id"] == "segment-1"
|
||||
|
||||
|
||||
def test_batch_import_delegates_and_preserves_dispatch_failure_response(app: Flask) -> None:
|
||||
segments = create_autospec(DatasetSegmentApplicationService, instance=True, spec_set=True)
|
||||
segments.start_batch_import.return_value = SegmentBatchImport(job_id="job-1", job_status="waiting")
|
||||
method = unwrap(DatasetDocumentSegmentBatchImportApi.post)
|
||||
|
||||
with app.test_request_context("/"), _patch_services(segments):
|
||||
response, status = method(
|
||||
DatasetDocumentSegmentBatchImportApi(),
|
||||
BatchImportPayload(upload_file_id="file-1"),
|
||||
_context(),
|
||||
"dataset-1",
|
||||
"document-1",
|
||||
)
|
||||
assert status == 200
|
||||
assert response == {"job_id": "job-1", "job_status": "waiting"}
|
||||
|
||||
segments.start_batch_import.side_effect = SegmentBatchImportDispatchError("redis down")
|
||||
with app.test_request_context("/"), _patch_services(segments):
|
||||
response, status = method(
|
||||
DatasetDocumentSegmentBatchImportApi(),
|
||||
BatchImportPayload(upload_file_id="file-1"),
|
||||
_context(),
|
||||
"dataset-1",
|
||||
"document-1",
|
||||
)
|
||||
assert (response, status) == ({"error": "redis down"}, 500)
|
||||
|
||||
|
||||
def test_batch_import_status_preserves_missing_job_contract(app: Flask) -> None:
|
||||
segments = create_autospec(DatasetSegmentApplicationService, instance=True, spec_set=True)
|
||||
method = unwrap(DatasetDocumentSegmentBatchImportStatusApi.get)
|
||||
|
||||
segments.get_batch_import_status.side_effect = SegmentBatchImportNotFoundError("The job does not exist.")
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
_patch_services(segments),
|
||||
pytest.raises(ValueError, match="The job does not exist\\."),
|
||||
):
|
||||
method(DatasetDocumentSegmentBatchImportStatusApi(), _context(), "job-1")
|
||||
|
||||
segments.get_batch_import_status.side_effect = None
|
||||
segments.get_batch_import_status.return_value = SegmentBatchImport(job_id="job-1", job_status="completed")
|
||||
with app.test_request_context("/"), _patch_services(segments):
|
||||
response, status = method(DatasetDocumentSegmentBatchImportStatusApi(), _context(), "job-1")
|
||||
assert status == 200
|
||||
assert response == {"job_id": "job-1", "job_status": "completed"}
|
||||
|
||||
|
||||
def test_child_chunk_create_and_list_delegate(app: Flask) -> None:
|
||||
segments = create_autospec(DatasetSegmentApplicationService, instance=True, spec_set=True)
|
||||
segments.create_child_chunk.return_value = _child_chunk_data()
|
||||
segments.list_child_chunks.return_value = ChildChunkPage(
|
||||
items=(_child_chunk_data(),), total=1, total_pages=1, page=1, limit=20
|
||||
)
|
||||
|
||||
with app.test_request_context("/"), _patch_services(segments):
|
||||
create_response, create_status = unwrap(ChildChunkAddApi.post)(
|
||||
ChildChunkAddApi(),
|
||||
ChildChunkCreatePayload(content="child"),
|
||||
_context(),
|
||||
"dataset-1",
|
||||
"document-1",
|
||||
"segment-1",
|
||||
)
|
||||
list_response, list_status = unwrap(ChildChunkAddApi.get)(
|
||||
ChildChunkAddApi(), _context(), "dataset-1", "document-1", "segment-1"
|
||||
)
|
||||
|
||||
assert create_status == list_status == 200
|
||||
assert create_response["data"]["id"] == "child-1"
|
||||
assert create_response["data"]["created_at"] == 1767225600
|
||||
assert list_response["data"][0]["id"] == "child-1"
|
||||
|
||||
|
||||
def test_child_chunk_mutations_delegate(app: Flask) -> None:
|
||||
segments = create_autospec(DatasetSegmentApplicationService, instance=True, spec_set=True)
|
||||
segments.update_child_chunks.return_value = (_child_chunk_data(),)
|
||||
segments.update_child_chunk.return_value = _child_chunk_data()
|
||||
chunks = [ChildChunkUpdateArgs(id="child-1", content="updated")]
|
||||
|
||||
with app.test_request_context("/"), _patch_services(segments):
|
||||
batch_response, batch_status = unwrap(ChildChunkAddApi.patch)(
|
||||
ChildChunkAddApi(),
|
||||
ChildChunkBatchUpdatePayload(chunks=chunks),
|
||||
_context(),
|
||||
"dataset-1",
|
||||
"document-1",
|
||||
"segment-1",
|
||||
)
|
||||
update_response, update_status = unwrap(ChildChunkUpdateApi.patch)(
|
||||
ChildChunkUpdateApi(),
|
||||
ChildChunkUpdatePayload(content="updated"),
|
||||
_context(),
|
||||
"dataset-1",
|
||||
"document-1",
|
||||
"segment-1",
|
||||
"child-1",
|
||||
)
|
||||
delete_response, delete_status = unwrap(ChildChunkUpdateApi.delete)(
|
||||
ChildChunkUpdateApi(),
|
||||
_context(),
|
||||
"dataset-1",
|
||||
"document-1",
|
||||
"segment-1",
|
||||
"child-1",
|
||||
)
|
||||
|
||||
assert batch_status == update_status == 200
|
||||
assert batch_response["data"][0]["id"] == "child-1"
|
||||
assert update_response["data"]["id"] == "child-1"
|
||||
assert (delete_response, delete_status) == ("", 204)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("translate", "error", "expected"),
|
||||
[
|
||||
(_raise_segment_error, SegmentDatasetNotFoundError(), NotFound),
|
||||
(_raise_segment_error, SegmentDocumentNotFoundError(), NotFound),
|
||||
(_raise_segment_error, SegmentNotFoundError(), NotFound),
|
||||
(_raise_segment_error, ChildChunkNotFoundError(), NotFound),
|
||||
(_raise_segment_error, SegmentUploadFileNotFoundError(), NotFound),
|
||||
(_raise_segment_error, SegmentPermissionDeniedError(), Forbidden),
|
||||
(_raise_segment_error, SegmentDatasetModelUnavailableError(), ValueError),
|
||||
(_raise_segment_error, SegmentInvalidFileTypeError(), ValueError),
|
||||
(_raise_segment_embedding_error, SegmentEmbeddingModelUnavailableError(), ProviderNotInitializeError),
|
||||
(_raise_segment_status_error, SegmentDocumentIndexingError(), InvalidActionError),
|
||||
(_raise_segment_status_error, SegmentStatusUpdateError(), InvalidActionError),
|
||||
(_raise_segment_status_error, SegmentEmbeddingModelUnavailableError(), ProviderNotInitializeError),
|
||||
(_raise_child_chunk_write_error, ChildChunkIndexingApplicationError(), ChildChunkIndexingError),
|
||||
(_raise_child_chunk_create_error, ChildChunkIndexingApplicationError(), ChildChunkIndexingError),
|
||||
(_raise_child_chunk_create_error, SegmentEmbeddingModelUnavailableError(), ProviderNotInitializeError),
|
||||
(_raise_child_chunk_delete_error, ChildChunkDeleteIndexApplicationError(), ChildChunkDeleteIndexError),
|
||||
# Every narrow translator falls back to the shared scope mapping.
|
||||
(_raise_child_chunk_delete_error, SegmentNotFoundError(), NotFound),
|
||||
],
|
||||
)
|
||||
def test_segment_application_errors_map_to_existing_http_contract(
|
||||
translate: Callable[[Exception], Never], error: Exception, expected: type[Exception]
|
||||
) -> None:
|
||||
with pytest.raises(expected):
|
||||
translate(error)
|
||||
|
||||
|
||||
def test_scope_translator_does_not_map_other_operations_errors() -> None:
|
||||
error = ChildChunkIndexingApplicationError()
|
||||
with pytest.raises(ChildChunkIndexingApplicationError) as raised:
|
||||
_raise_segment_error(error)
|
||||
assert raised.value is error
|
||||
@@ -0,0 +1,164 @@
|
||||
from inspect import unwrap
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import create_autospec
|
||||
from uuid import UUID
|
||||
|
||||
import pytest
|
||||
from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
|
||||
|
||||
from controllers.console.datasets import external as controller
|
||||
from controllers.console.datasets.error import DatasetNameDuplicateError
|
||||
from machinery.context import RequestContext
|
||||
from services.errors.dataset import DatasetNameDuplicateError as DuplicateName
|
||||
from services.knowledge.dataset_access import DatasetAccessDeniedError, DatasetNotFoundError
|
||||
from services.knowledge.external.application import (
|
||||
ExternalHitTestingError,
|
||||
ExternalKnowledgeApplicationService,
|
||||
ExternalTemplateNotFoundError,
|
||||
)
|
||||
|
||||
CONTEXT = RequestContext("request-1", None, "actor-1", "tenant-1")
|
||||
ID = UUID(int=1)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def external(monkeypatch):
|
||||
service = create_autospec(ExternalKnowledgeApplicationService, instance=True, spec_set=True)
|
||||
monkeypatch.setattr(
|
||||
controller, "application_services", lambda: SimpleNamespace(knowledge=SimpleNamespace(external=service))
|
||||
)
|
||||
return service
|
||||
|
||||
|
||||
def test_templates_preserve_pagination(external):
|
||||
page = {"data": [], "total": 0, "has_more": False, "page": 1, "limit": 100}
|
||||
external.list_templates.return_value = page
|
||||
result = unwrap(controller.ExternalApiTemplateListApi.get)(
|
||||
controller.ExternalApiTemplateListApi(),
|
||||
controller.ExternalApiTemplateListQuery(page=0, limit=1000, keyword="test"),
|
||||
CONTEXT,
|
||||
)
|
||||
assert result == (page, 200)
|
||||
external.list_templates.assert_called_once_with(CONTEXT, page=0, limit=1000, keyword="test")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("resource", "verb", "operation", "payload"),
|
||||
[
|
||||
(
|
||||
controller.ExternalApiTemplateListApi,
|
||||
controller.ExternalApiTemplateListApi.post,
|
||||
lambda service: service.create_template,
|
||||
True,
|
||||
),
|
||||
(
|
||||
controller.ExternalApiTemplateApi,
|
||||
controller.ExternalApiTemplateApi.get,
|
||||
lambda service: service.get_template,
|
||||
False,
|
||||
),
|
||||
(
|
||||
controller.ExternalApiTemplateApi,
|
||||
controller.ExternalApiTemplateApi.patch,
|
||||
lambda service: service.update_template,
|
||||
True,
|
||||
),
|
||||
(
|
||||
controller.ExternalApiTemplateApi,
|
||||
controller.ExternalApiTemplateApi.delete,
|
||||
lambda service: service.delete_template,
|
||||
False,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_template_missing_is_mapped_and_context_forwarded(external, resource, verb, operation, payload):
|
||||
method = operation(external)
|
||||
method.side_effect = ExternalTemplateNotFoundError("API template not found.")
|
||||
args = (
|
||||
[
|
||||
controller.ExternalKnowledgeApiPayload(
|
||||
name="API", settings={"endpoint": "https://example", "api_key": "secret"}
|
||||
)
|
||||
]
|
||||
if payload
|
||||
else []
|
||||
)
|
||||
args.append(CONTEXT)
|
||||
if resource is controller.ExternalApiTemplateApi:
|
||||
args.append(ID)
|
||||
with pytest.raises(NotFound):
|
||||
unwrap(verb)(resource(), *args)
|
||||
assert method.call_args.args == (CONTEXT,)
|
||||
if resource is controller.ExternalApiTemplateApi:
|
||||
assert method.call_args.kwargs["template_id"] == str(ID)
|
||||
|
||||
|
||||
def test_template_serializes_settings_and_timestamp(external):
|
||||
external.get_template.return_value = {
|
||||
"id": str(ID),
|
||||
"name": "API",
|
||||
"tenant_id": "tenant-1",
|
||||
"settings": {"endpoint": "https://example", "api_key": "[__HIDDEN__]"},
|
||||
"description": "",
|
||||
"created_by": "actor-1",
|
||||
"created_at": "2024-01-01T00:00:00",
|
||||
"dataset_bindings": [],
|
||||
}
|
||||
result, status = unwrap(controller.ExternalApiTemplateApi.get)(controller.ExternalApiTemplateApi(), CONTEXT, ID)
|
||||
assert status == 200
|
||||
assert result == external.get_template.return_value
|
||||
|
||||
|
||||
def test_delete_and_usage_response_contracts(external):
|
||||
assert unwrap(controller.ExternalApiTemplateApi.delete)(controller.ExternalApiTemplateApi(), CONTEXT, ID) == (
|
||||
"",
|
||||
204,
|
||||
)
|
||||
external.template_usage.return_value = (True, 3)
|
||||
assert unwrap(controller.ExternalApiUseCheckApi.get)(controller.ExternalApiUseCheckApi(), CONTEXT, ID) == (
|
||||
{"is_using": True, "count": 3},
|
||||
200,
|
||||
)
|
||||
|
||||
|
||||
def test_external_dataset_duplicate_name_maps_to_409(external):
|
||||
payload = controller.ExternalDatasetCreatePayload(
|
||||
name="Dataset", external_knowledge_api_id="api", external_knowledge_id="knowledge"
|
||||
)
|
||||
external.create_dataset.side_effect = DuplicateName()
|
||||
with pytest.raises(DatasetNameDuplicateError):
|
||||
unwrap(controller.ExternalDatasetCreateApi.post)(controller.ExternalDatasetCreateApi(), payload, CONTEXT)
|
||||
external.create_dataset.assert_called_once_with(CONTEXT, payload=payload)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("error", "http_error"),
|
||||
[
|
||||
(DatasetNotFoundError(), NotFound),
|
||||
(DatasetAccessDeniedError(), Forbidden),
|
||||
(ExternalHitTestingError("failed"), InternalServerError),
|
||||
],
|
||||
)
|
||||
def test_retrieval_maps_application_errors(external, error, http_error):
|
||||
external.hit_testing.side_effect = error
|
||||
with pytest.raises(http_error):
|
||||
unwrap(controller.ExternalKnowledgeHitTestingApi.post)(
|
||||
controller.ExternalKnowledgeHitTestingApi(),
|
||||
controller.ExternalHitTestingPayload(query="query"),
|
||||
CONTEXT,
|
||||
ID,
|
||||
)
|
||||
external.hit_testing.assert_called_once_with(
|
||||
CONTEXT, dataset_id=str(ID), query="query", retrieval_model=None, metadata_filters=None
|
||||
)
|
||||
|
||||
|
||||
def test_retrieval_response_is_serialized(external):
|
||||
external.hit_testing.return_value = {
|
||||
"query": {"content": "query"},
|
||||
"records": [{"content": "answer", "title": "title", "score": 0.8, "metadata": {"source": "doc"}}],
|
||||
}
|
||||
result = unwrap(controller.ExternalKnowledgeHitTestingApi.post)(
|
||||
controller.ExternalKnowledgeHitTestingApi(), controller.ExternalHitTestingPayload(query="query"), CONTEXT, ID
|
||||
)
|
||||
assert result == external.hit_testing.return_value
|
||||
@@ -0,0 +1,237 @@
|
||||
import uuid
|
||||
from inspect import unwrap
|
||||
from unittest.mock import MagicMock, PropertyMock, patch
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
from pytest_mock import MockerFixture
|
||||
from werkzeug.exceptions import NotFound
|
||||
|
||||
from controllers.console import console_ns
|
||||
from controllers.console.datasets.hit_testing import HitTestingApi
|
||||
from models.account import Account, Tenant, TenantAccountRole
|
||||
from models.dataset import Dataset
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app():
|
||||
app = Flask("test_hit_testing")
|
||||
app.config["TESTING"] = True
|
||||
return app
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dataset_id():
|
||||
return uuid.uuid4()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dataset():
|
||||
return Dataset(id="dataset-1", tenant_id="tenant-1", name="Dataset", created_by="account-1")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def account() -> Account:
|
||||
account = Account(name="User", email="user@example.com")
|
||||
account.id = "account-1"
|
||||
tenant = Tenant(name="Tenant")
|
||||
tenant.id = "tenant-1"
|
||||
account._current_tenant = tenant
|
||||
account.role = TenantAccountRole.OWNER
|
||||
return account
|
||||
|
||||
|
||||
def hit_testing_record() -> dict[str, object]:
|
||||
return {
|
||||
"segment": {
|
||||
"id": "segment-1",
|
||||
"position": 1,
|
||||
"document_id": "document-1",
|
||||
"content": "Chunk text",
|
||||
"sign_content": "Chunk text",
|
||||
"answer": None,
|
||||
"word_count": 2,
|
||||
"tokens": 3,
|
||||
"keywords": [],
|
||||
"index_node_id": None,
|
||||
"index_node_hash": None,
|
||||
"hit_count": 0,
|
||||
"enabled": True,
|
||||
"disabled_at": None,
|
||||
"disabled_by": None,
|
||||
"status": "completed",
|
||||
"created_by": "account-1",
|
||||
"created_at": 1_700_000_000,
|
||||
"indexing_at": None,
|
||||
"completed_at": None,
|
||||
"error": None,
|
||||
"stopped_at": None,
|
||||
"document": {
|
||||
"id": "document-1",
|
||||
"data_source_type": "upload_file",
|
||||
"name": "guide.md",
|
||||
"doc_type": None,
|
||||
"doc_metadata": None,
|
||||
},
|
||||
},
|
||||
"child_chunks": [],
|
||||
"score": None,
|
||||
"tsne_position": None,
|
||||
"files": [],
|
||||
"summary": None,
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def bypass_decorators(mocker: MockerFixture):
|
||||
"""Bypass all decorators on the API method."""
|
||||
mocker.patch(
|
||||
"controllers.console.datasets.hit_testing.setup_required",
|
||||
lambda f: f,
|
||||
)
|
||||
mocker.patch(
|
||||
"controllers.console.datasets.hit_testing.login_required",
|
||||
return_value=lambda f: f,
|
||||
)
|
||||
mocker.patch(
|
||||
"controllers.console.datasets.hit_testing.account_initialization_required",
|
||||
return_value=lambda f: f,
|
||||
)
|
||||
mocker.patch(
|
||||
"controllers.console.datasets.hit_testing.cloud_edition_billing_rate_limit_check",
|
||||
return_value=lambda *_: lambda f: f,
|
||||
)
|
||||
|
||||
|
||||
class TestHitTestingApi:
|
||||
def test_hit_testing_success(self, app: Flask, dataset, dataset_id, account: Account):
|
||||
api = HitTestingApi()
|
||||
method = unwrap(api.post)
|
||||
|
||||
payload = {
|
||||
"query": "what is vector search",
|
||||
}
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch.object(
|
||||
type(console_ns),
|
||||
"payload",
|
||||
new_callable=PropertyMock,
|
||||
return_value=payload,
|
||||
),
|
||||
patch.object(
|
||||
HitTestingApi,
|
||||
"get_and_validate_dataset",
|
||||
return_value=dataset,
|
||||
),
|
||||
patch.object(
|
||||
HitTestingApi,
|
||||
"hit_testing_args_check",
|
||||
),
|
||||
patch.object(
|
||||
HitTestingApi,
|
||||
"perform_hit_testing",
|
||||
return_value={"query": {"content": "what is vector search"}, "records": []},
|
||||
),
|
||||
):
|
||||
result = method(api, MagicMock(), account, "tenant-1", dataset_id)
|
||||
|
||||
assert "query" in result
|
||||
assert "records" in result
|
||||
assert result["records"] == []
|
||||
|
||||
def test_hit_testing_success_with_optional_record_fields(self, app: Flask, dataset, dataset_id, account: Account):
|
||||
api = HitTestingApi()
|
||||
method = unwrap(api.post)
|
||||
|
||||
payload = {
|
||||
"query": "what is vector search",
|
||||
}
|
||||
records = [hit_testing_record()]
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch.object(
|
||||
type(console_ns),
|
||||
"payload",
|
||||
new_callable=PropertyMock,
|
||||
return_value=payload,
|
||||
),
|
||||
patch.object(
|
||||
HitTestingApi,
|
||||
"get_and_validate_dataset",
|
||||
return_value=dataset,
|
||||
),
|
||||
patch.object(
|
||||
HitTestingApi,
|
||||
"hit_testing_args_check",
|
||||
),
|
||||
patch.object(
|
||||
HitTestingApi,
|
||||
"perform_hit_testing",
|
||||
return_value={"query": {"content": payload["query"]}, "records": records},
|
||||
),
|
||||
):
|
||||
result = method(api, MagicMock(), account, "tenant-1", dataset_id)
|
||||
|
||||
assert result["query"] == {"content": payload["query"]}
|
||||
assert result["records"][0]["segment"]["keywords"] == []
|
||||
assert result["records"][0]["child_chunks"] == []
|
||||
assert result["records"][0]["files"] == []
|
||||
assert result["records"][0]["score"] is None
|
||||
|
||||
def test_hit_testing_dataset_not_found(self, app: Flask, dataset_id, account: Account):
|
||||
api = HitTestingApi()
|
||||
method = unwrap(api.post)
|
||||
|
||||
payload = {
|
||||
"query": "test",
|
||||
}
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch.object(
|
||||
type(console_ns),
|
||||
"payload",
|
||||
new_callable=PropertyMock,
|
||||
return_value=payload,
|
||||
),
|
||||
patch.object(
|
||||
HitTestingApi,
|
||||
"get_and_validate_dataset",
|
||||
side_effect=NotFound("Dataset not found"),
|
||||
),
|
||||
):
|
||||
with pytest.raises(NotFound, match="Dataset not found"):
|
||||
method(api, MagicMock(), account, "tenant-1", dataset_id)
|
||||
|
||||
def test_hit_testing_invalid_args(self, app: Flask, dataset, dataset_id, account: Account):
|
||||
api = HitTestingApi()
|
||||
method = unwrap(api.post)
|
||||
|
||||
payload = {
|
||||
"query": "",
|
||||
}
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch.object(
|
||||
type(console_ns),
|
||||
"payload",
|
||||
new_callable=PropertyMock,
|
||||
return_value=payload,
|
||||
),
|
||||
patch.object(
|
||||
HitTestingApi,
|
||||
"get_and_validate_dataset",
|
||||
return_value=dataset,
|
||||
),
|
||||
patch.object(
|
||||
HitTestingApi,
|
||||
"hit_testing_args_check",
|
||||
side_effect=ValueError("Invalid parameters"),
|
||||
),
|
||||
):
|
||||
with pytest.raises(ValueError, match="Invalid parameters"):
|
||||
method(api, MagicMock(), account, "tenant-1", dataset_id)
|
||||
+296
@@ -0,0 +1,296 @@
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
|
||||
|
||||
import services.errors.base
|
||||
from controllers.console.app.error import (
|
||||
CompletionRequestError,
|
||||
ProviderModelCurrentlyNotSupportError,
|
||||
ProviderNotInitializeError,
|
||||
ProviderQuotaExceededError,
|
||||
)
|
||||
from controllers.console.datasets.error import DatasetNotInitializedError
|
||||
from controllers.console.datasets.hit_testing_base import (
|
||||
DatasetsHitTestingBase,
|
||||
)
|
||||
from core.errors.error import (
|
||||
LLMBadRequestError,
|
||||
ModelCurrentlyNotSupportError,
|
||||
ProviderTokenNotInitError,
|
||||
QuotaExceededError,
|
||||
)
|
||||
from graphon.model_runtime.errors.invoke import InvokeError
|
||||
from models.account import Account, Tenant, TenantAccountRole
|
||||
from models.dataset import Dataset
|
||||
from services.hit_testing_service import HitTestingService
|
||||
from services.knowledge.dataset_service import DatasetService
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def account():
|
||||
acc = Account(name="User", email="user@example.com")
|
||||
acc.id = "account-1"
|
||||
tenant = Tenant(name="Tenant")
|
||||
tenant.id = "tenant-1"
|
||||
acc._current_tenant = tenant
|
||||
acc.role = TenantAccountRole.OWNER
|
||||
return acc
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dataset():
|
||||
return Dataset(id="dataset-1", tenant_id="tenant-1", name="Dataset", created_by="account-1")
|
||||
|
||||
|
||||
def hit_testing_record() -> dict[str, object]:
|
||||
return {
|
||||
"segment": {
|
||||
"id": "segment-1",
|
||||
"position": 1,
|
||||
"document_id": "document-1",
|
||||
"content": "Chunk text",
|
||||
"answer": None,
|
||||
"word_count": 2,
|
||||
"tokens": 3,
|
||||
"keywords": None,
|
||||
"index_node_id": None,
|
||||
"index_node_hash": None,
|
||||
"hit_count": 0,
|
||||
"enabled": True,
|
||||
"disabled_at": None,
|
||||
"disabled_by": None,
|
||||
"status": "completed",
|
||||
"created_by": "account-1",
|
||||
"created_at": 1_700_000_000,
|
||||
"indexing_at": None,
|
||||
"completed_at": None,
|
||||
"error": None,
|
||||
"stopped_at": None,
|
||||
"document": {
|
||||
"id": "document-1",
|
||||
"data_source_type": "upload_file",
|
||||
"name": "guide.md",
|
||||
"doc_type": None,
|
||||
"doc_metadata": None,
|
||||
},
|
||||
},
|
||||
"child_chunks": None,
|
||||
"files": None,
|
||||
"score": 0.8,
|
||||
}
|
||||
|
||||
|
||||
class TestGetAndValidateDataset:
|
||||
def test_success(self, dataset, account):
|
||||
with (
|
||||
patch.object(
|
||||
DatasetService,
|
||||
"get_dataset",
|
||||
return_value=dataset,
|
||||
),
|
||||
patch.object(
|
||||
DatasetService,
|
||||
"check_dataset_permission",
|
||||
),
|
||||
):
|
||||
result = DatasetsHitTestingBase.get_and_validate_dataset(Mock(), "dataset-1", account, "tenant-1")
|
||||
|
||||
assert result == dataset
|
||||
|
||||
def test_dataset_not_found(self, account):
|
||||
with patch.object(
|
||||
DatasetService,
|
||||
"get_dataset",
|
||||
return_value=None,
|
||||
):
|
||||
with pytest.raises(NotFound, match="Dataset not found"):
|
||||
DatasetsHitTestingBase.get_and_validate_dataset(Mock(), "dataset-1", account, "tenant-1")
|
||||
|
||||
def test_permission_denied(self, dataset, account):
|
||||
with (
|
||||
patch.object(
|
||||
DatasetService,
|
||||
"get_dataset",
|
||||
return_value=dataset,
|
||||
),
|
||||
patch.object(
|
||||
DatasetService,
|
||||
"check_dataset_permission",
|
||||
side_effect=services.errors.base.NoPermissionError("no access"),
|
||||
),
|
||||
):
|
||||
with pytest.raises(Forbidden, match="no access"):
|
||||
DatasetsHitTestingBase.get_and_validate_dataset(Mock(), "dataset-1", account, "tenant-1")
|
||||
|
||||
|
||||
class TestHitTestingArgsCheck:
|
||||
def test_args_check_called(self):
|
||||
args = {"query": "test"}
|
||||
|
||||
with patch.object(
|
||||
HitTestingService,
|
||||
"hit_testing_args_check",
|
||||
) as check_mock:
|
||||
DatasetsHitTestingBase.hit_testing_args_check(args)
|
||||
|
||||
check_mock.assert_called_once_with(args)
|
||||
|
||||
|
||||
class TestParseArgs:
|
||||
def test_parse_args_success(self):
|
||||
payload = {"query": "hello"}
|
||||
|
||||
result = DatasetsHitTestingBase.parse_args(payload)
|
||||
|
||||
assert result["query"] == "hello"
|
||||
|
||||
def test_parse_args_invalid(self):
|
||||
payload = {"query": "x" * 300}
|
||||
|
||||
with pytest.raises(ValueError, match="1 validation error for HitTestingPayload\\\nquery"):
|
||||
DatasetsHitTestingBase.parse_args(payload)
|
||||
|
||||
def test_parse_args_ignores_unknown_fields_for_compatibility(self):
|
||||
payload = {"query": "hello", "top_k": 3}
|
||||
|
||||
result = DatasetsHitTestingBase.parse_args(payload)
|
||||
|
||||
assert result == {"query": "hello"}
|
||||
|
||||
|
||||
class TestPerformHitTesting:
|
||||
def test_success(self, dataset, account):
|
||||
response = {
|
||||
"query": {"content": "hello"},
|
||||
"records": [],
|
||||
}
|
||||
|
||||
with patch.object(
|
||||
HitTestingService,
|
||||
"retrieve",
|
||||
return_value=response,
|
||||
):
|
||||
result = DatasetsHitTestingBase.perform_hit_testing(
|
||||
Mock(), dataset, {"query": "hello"}, account, "tenant-1"
|
||||
)
|
||||
|
||||
assert result["query"] == {"content": "hello"}
|
||||
assert result["records"] == []
|
||||
|
||||
def test_success_prepares_nullable_list_fields(self, dataset, account):
|
||||
response = {
|
||||
"query": {"content": "hello"},
|
||||
"records": [hit_testing_record()],
|
||||
}
|
||||
|
||||
with patch.object(
|
||||
HitTestingService,
|
||||
"retrieve",
|
||||
return_value=response,
|
||||
):
|
||||
result = DatasetsHitTestingBase.perform_hit_testing(
|
||||
Mock(), dataset, {"query": "hello"}, account, "tenant-1"
|
||||
)
|
||||
|
||||
assert result["query"] == {"content": "hello"}
|
||||
record = result["records"][0]
|
||||
assert record["segment"]["keywords"] == []
|
||||
assert record["segment"]["sign_content"] is None
|
||||
assert record["child_chunks"] == []
|
||||
assert record["files"] == []
|
||||
assert record["score"] == 0.8
|
||||
assert record["tsne_position"] is None
|
||||
assert record["summary"] is None
|
||||
|
||||
def test_invalid_query_response_raises_value_error(self, dataset, account):
|
||||
with (
|
||||
patch.object(
|
||||
HitTestingService,
|
||||
"retrieve",
|
||||
return_value={"query": "hello", "records": []},
|
||||
),
|
||||
pytest.raises(ValueError, match="Invalid hit testing query response"),
|
||||
):
|
||||
DatasetsHitTestingBase.perform_hit_testing(Mock(), dataset, {"query": "hello"}, account, "tenant-1")
|
||||
|
||||
def test_invalid_records_response_raises_value_error(self):
|
||||
with pytest.raises(ValueError, match="Invalid hit testing records response"):
|
||||
DatasetsHitTestingBase._prepare_hit_testing_records({"records": []})
|
||||
|
||||
def test_invalid_record_response_raises_value_error(self):
|
||||
with pytest.raises(ValueError, match="Invalid hit testing record response"):
|
||||
DatasetsHitTestingBase._prepare_hit_testing_records(["record"])
|
||||
|
||||
def test_index_not_initialized(self, dataset, account):
|
||||
with patch.object(
|
||||
HitTestingService,
|
||||
"retrieve",
|
||||
side_effect=services.errors.index.IndexNotInitializedError(),
|
||||
):
|
||||
with pytest.raises(DatasetNotInitializedError):
|
||||
DatasetsHitTestingBase.perform_hit_testing(Mock(), dataset, {"query": "hello"}, account, "tenant-1")
|
||||
|
||||
def test_provider_token_not_init(self, dataset, account):
|
||||
with patch.object(
|
||||
HitTestingService,
|
||||
"retrieve",
|
||||
side_effect=ProviderTokenNotInitError("token missing"),
|
||||
):
|
||||
with pytest.raises(ProviderNotInitializeError):
|
||||
DatasetsHitTestingBase.perform_hit_testing(Mock(), dataset, {"query": "hello"}, account, "tenant-1")
|
||||
|
||||
def test_quota_exceeded(self, dataset, account):
|
||||
with patch.object(
|
||||
HitTestingService,
|
||||
"retrieve",
|
||||
side_effect=QuotaExceededError(),
|
||||
):
|
||||
with pytest.raises(ProviderQuotaExceededError):
|
||||
DatasetsHitTestingBase.perform_hit_testing(Mock(), dataset, {"query": "hello"}, account, "tenant-1")
|
||||
|
||||
def test_model_not_supported(self, dataset, account):
|
||||
with patch.object(
|
||||
HitTestingService,
|
||||
"retrieve",
|
||||
side_effect=ModelCurrentlyNotSupportError(),
|
||||
):
|
||||
with pytest.raises(ProviderModelCurrentlyNotSupportError):
|
||||
DatasetsHitTestingBase.perform_hit_testing(Mock(), dataset, {"query": "hello"}, account, "tenant-1")
|
||||
|
||||
def test_llm_bad_request(self, dataset, account):
|
||||
with patch.object(
|
||||
HitTestingService,
|
||||
"retrieve",
|
||||
side_effect=LLMBadRequestError("bad request"),
|
||||
):
|
||||
with pytest.raises(ProviderNotInitializeError):
|
||||
DatasetsHitTestingBase.perform_hit_testing(Mock(), dataset, {"query": "hello"}, account, "tenant-1")
|
||||
|
||||
def test_invoke_error(self, dataset, account):
|
||||
with patch.object(
|
||||
HitTestingService,
|
||||
"retrieve",
|
||||
side_effect=InvokeError("invoke failed"),
|
||||
):
|
||||
with pytest.raises(CompletionRequestError):
|
||||
DatasetsHitTestingBase.perform_hit_testing(Mock(), dataset, {"query": "hello"}, account, "tenant-1")
|
||||
|
||||
def test_value_error(self, dataset, account):
|
||||
with patch.object(
|
||||
HitTestingService,
|
||||
"retrieve",
|
||||
side_effect=ValueError("bad args"),
|
||||
):
|
||||
with pytest.raises(ValueError, match="bad args"):
|
||||
DatasetsHitTestingBase.perform_hit_testing(Mock(), dataset, {"query": "hello"}, account, "tenant-1")
|
||||
|
||||
def test_unexpected_error(self, dataset, account):
|
||||
with patch.object(
|
||||
HitTestingService,
|
||||
"retrieve",
|
||||
side_effect=Exception("boom"),
|
||||
):
|
||||
with pytest.raises(InternalServerError) as exc_info:
|
||||
DatasetsHitTestingBase.perform_hit_testing(Mock(), dataset, {"query": "hello"}, account, "tenant-1")
|
||||
assert "boom" not in str(exc_info.value)
|
||||
@@ -0,0 +1,145 @@
|
||||
"""Console metadata routes use admission, explicit context, and response models."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock, create_autospec
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
from flask.testing import FlaskClient
|
||||
from flask_restx import Api
|
||||
|
||||
from controllers.common.rbac import DatasetId, RBACPermission
|
||||
from controllers.console import api as console_api
|
||||
from controllers.console import console_ns
|
||||
from controllers.console.datasets.metadata import DatasetMetadataCreateApi
|
||||
from enums import DeploymentEdition
|
||||
from libs.login import AccountWithTenant
|
||||
from models.account import Account, TenantAccountRole
|
||||
from services.errors.base import NoPermissionError
|
||||
from services.errors.metadata import MetadataResourceNotFoundError
|
||||
from services.knowledge.dataset_access import DatasetAccessDeniedError, DatasetNotFoundError
|
||||
from services.knowledge.metadata.application import MetadataService
|
||||
from services.knowledge.resource_scope import DatasetRef
|
||||
from tests.helpers.unit.controllers.rbac_introspection import rbac_checks
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def routes(monkeypatch: pytest.MonkeyPatch, config_overrides):
|
||||
config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD, LOGIN_DISABLED=True, RBAC_ENABLED=False)
|
||||
app = Flask(__name__)
|
||||
app.config.update(TESTING=False, PROPAGATE_EXCEPTIONS=False)
|
||||
api = Api(app)
|
||||
api.error_handlers = console_api.error_handlers.copy()
|
||||
api.add_namespace(console_ns, path="/console/api")
|
||||
account = Account(name="Owner", email="metadata@example.com")
|
||||
account.id = "account-1"
|
||||
account.role = TenantAccountRole.OWNER
|
||||
identity = AccountWithTenant(account=account, tenant_id="tenant-1")
|
||||
monkeypatch.setattr("controllers.console.flask_admission.current_account_with_tenant", lambda: identity)
|
||||
monkeypatch.setattr("controllers.console.wraps.current_account_with_tenant", lambda: identity)
|
||||
service = create_autospec(MetadataService, instance=True, spec_set=True)
|
||||
service.require_dataset.side_effect = lambda context, dataset_id: DatasetRef(
|
||||
context.active_workspace_id, dataset_id
|
||||
)
|
||||
app.extensions["application_services"] = SimpleNamespace(knowledge=SimpleNamespace(metadata=service))
|
||||
return app.test_client(), service
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("error", "status", "code", "message"),
|
||||
[
|
||||
(NoPermissionError("Access denied"), 400, "invalid_param", "Access denied"),
|
||||
(RuntimeError("backend detail"), 500, "unknown", "Internal Server Error"),
|
||||
],
|
||||
)
|
||||
def test_metadata_permission_error_http_contract(
|
||||
routes: tuple[FlaskClient, Mock], error: Exception, status: int, code: str, message: str
|
||||
) -> None:
|
||||
client, service = routes
|
||||
service.require_dataset.side_effect = error
|
||||
response = client.post(f"/console/api/datasets/{uuid4()}/metadata", json={"type": "string", "name": "author"})
|
||||
assert response.status_code == status
|
||||
assert response.json == {"code": code, "message": message, "status": status}
|
||||
service.create_metadata.assert_not_called()
|
||||
|
||||
|
||||
def test_create_serializes_result_and_passes_actor(routes):
|
||||
client, service = routes
|
||||
dataset_id = str(uuid4())
|
||||
service.create_metadata.return_value = {"id": str(uuid4()), "name": "author", "type": "string"}
|
||||
response = client.post(f"/console/api/datasets/{dataset_id}/metadata", json={"type": "string", "name": "author"})
|
||||
assert response.status_code == 201
|
||||
assert response.get_json()["name"] == "author"
|
||||
assert service.create_metadata.call_args.args[0] == DatasetRef("tenant-1", dataset_id)
|
||||
assert service.create_metadata.call_args.kwargs == {"actor_id": "account-1"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("payload", [{"name": None, "type": "string"}, {"name": "author", "type": None}])
|
||||
def test_create_rejects_invalid_payload_before_application(routes, payload):
|
||||
client, service = routes
|
||||
response = client.post(f"/console/api/datasets/{uuid4()}/metadata", json=payload)
|
||||
assert response.status_code == 422
|
||||
service.create_metadata.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("error", "status"), [(DatasetNotFoundError(), 404), (DatasetAccessDeniedError(), 403)])
|
||||
def test_get_maps_access_errors(routes, error, status):
|
||||
client, service = routes
|
||||
service.require_dataset.side_effect = error
|
||||
assert client.get(f"/console/api/datasets/{uuid4()}/metadata").status_code == status
|
||||
service.get_dataset_metadatas.assert_not_called()
|
||||
|
||||
|
||||
def test_get_serializes_field_counts(routes):
|
||||
client, service = routes
|
||||
service.get_dataset_metadatas.return_value = {"doc_metadata": [], "built_in_field_enabled": False}
|
||||
response = client.get(f"/console/api/datasets/{uuid4()}/metadata")
|
||||
assert response.status_code == 200
|
||||
assert response.get_json() == {"doc_metadata": [], "built_in_field_enabled": False}
|
||||
|
||||
|
||||
def test_rename_and_delete_use_explicit_metadata_id(routes):
|
||||
client, service = routes
|
||||
dataset_id, metadata_id = str(uuid4()), str(uuid4())
|
||||
path = f"/console/api/datasets/{dataset_id}/metadata/{metadata_id}"
|
||||
service.update_metadata_name.return_value = {"id": metadata_id, "name": "writer", "type": "string"}
|
||||
assert client.patch(path, json={"name": "writer"}).status_code == 200
|
||||
service.update_metadata_name.assert_called_once_with(
|
||||
DatasetRef("tenant-1", dataset_id), metadata_id, "writer", actor_id="account-1"
|
||||
)
|
||||
response = client.delete(path)
|
||||
assert response.status_code == 204
|
||||
assert response.data == b""
|
||||
service.delete_metadata.assert_called_once_with(DatasetRef("tenant-1", dataset_id), metadata_id)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("action", ["enable", "disable"])
|
||||
def test_builtin_toggle_is_dispatched(routes, action):
|
||||
client, service = routes
|
||||
dataset_id = str(uuid4())
|
||||
response = client.post(f"/console/api/datasets/{dataset_id}/metadata/built-in/{action}")
|
||||
assert response.status_code == 204
|
||||
operation = service.enable_built_in_field if action == "enable" else service.disable_built_in_field
|
||||
operation.assert_called_once_with(DatasetRef("tenant-1", dataset_id))
|
||||
|
||||
|
||||
def test_builtin_listing_uses_response_model(routes):
|
||||
client, service = routes
|
||||
service.get_built_in_fields.return_value = [{"name": "uploader", "type": "string"}]
|
||||
response = client.get("/console/api/datasets/metadata/built-in")
|
||||
assert response.status_code == 200
|
||||
assert response.get_json() == {"fields": [{"name": "uploader", "type": "string"}]}
|
||||
|
||||
|
||||
def test_document_metadata_missing_resource_is_not_found(routes):
|
||||
client, service = routes
|
||||
service.update_documents_metadata.side_effect = MetadataResourceNotFoundError("Document not found.")
|
||||
response = client.post(f"/console/api/datasets/{uuid4()}/documents/metadata", json={"operation_data": []})
|
||||
assert response.status_code == 404
|
||||
|
||||
|
||||
def test_get_requires_dataset_readonly_permission():
|
||||
[check] = rbac_checks(DatasetMetadataCreateApi.get)
|
||||
assert check.scene is RBACPermission.DATASET_READONLY
|
||||
assert isinstance(check.locator, DatasetId)
|
||||
@@ -0,0 +1,174 @@
|
||||
import inspect
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
from pytest_mock import MockerFixture
|
||||
|
||||
from controllers.console.datasets.error import WebsiteCrawlError
|
||||
from controllers.console.datasets.website import (
|
||||
WebsiteCrawlApi,
|
||||
WebsiteCrawlPayload,
|
||||
WebsiteCrawlStatusApi,
|
||||
WebsiteCrawlStatusQuery,
|
||||
)
|
||||
from services.data_source.website_service import (
|
||||
WebsiteCrawlApiRequest,
|
||||
WebsiteCrawlStatusApiRequest,
|
||||
WebsiteService,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app() -> Flask:
|
||||
app = Flask("test_website_crawl")
|
||||
app.config["TESTING"] = True
|
||||
return app
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def bypass_auth_and_setup(mocker: MockerFixture) -> None:
|
||||
"""Bypass setup/login/account decorators."""
|
||||
mocker.patch(
|
||||
"controllers.console.datasets.website.login_required",
|
||||
lambda f: f,
|
||||
)
|
||||
mocker.patch(
|
||||
"controllers.console.datasets.website.setup_required",
|
||||
lambda f: f,
|
||||
)
|
||||
mocker.patch(
|
||||
"controllers.console.datasets.website.account_initialization_required",
|
||||
lambda f: f,
|
||||
)
|
||||
|
||||
|
||||
class TestWebsiteCrawlApi:
|
||||
def test_crawl_success(self, app: Flask, mocker: MockerFixture) -> None:
|
||||
api = WebsiteCrawlApi()
|
||||
method = inspect.unwrap(api.post)
|
||||
|
||||
payload = {
|
||||
"provider": "firecrawl",
|
||||
"url": "https://example.com",
|
||||
"options": {"depth": 1},
|
||||
}
|
||||
req_data = WebsiteCrawlPayload.model_validate(payload)
|
||||
|
||||
with app.test_request_context("/", json=payload):
|
||||
crawl_url = mocker.patch.object(
|
||||
WebsiteService,
|
||||
"crawl_url",
|
||||
return_value={"job_id": "job-1"},
|
||||
)
|
||||
|
||||
result, status = method(api, req_data)
|
||||
|
||||
assert status == 200
|
||||
assert result["job_id"] == "job-1"
|
||||
crawl_url.assert_called_once_with(
|
||||
WebsiteCrawlApiRequest(provider="firecrawl", url="https://example.com", options={"depth": 1})
|
||||
)
|
||||
|
||||
def test_crawl_invalid_payload(self, app: Flask, mocker: MockerFixture) -> None:
|
||||
api = WebsiteCrawlApi()
|
||||
method = inspect.unwrap(api.post)
|
||||
|
||||
payload = {
|
||||
"provider": "firecrawl",
|
||||
"url": "bad-url",
|
||||
"options": dict[str, object](),
|
||||
}
|
||||
req_data = WebsiteCrawlPayload.model_validate(payload)
|
||||
|
||||
with app.test_request_context("/", json=payload):
|
||||
mocker.patch.object(
|
||||
WebsiteCrawlApiRequest,
|
||||
"from_args",
|
||||
side_effect=ValueError("invalid payload"),
|
||||
)
|
||||
|
||||
with pytest.raises(WebsiteCrawlError, match="invalid payload"):
|
||||
method(api, req_data)
|
||||
|
||||
def test_crawl_service_error(self, app: Flask, mocker: MockerFixture) -> None:
|
||||
api = WebsiteCrawlApi()
|
||||
method = inspect.unwrap(api.post)
|
||||
|
||||
payload = {
|
||||
"provider": "firecrawl",
|
||||
"url": "https://example.com",
|
||||
"options": {"limit": 1},
|
||||
}
|
||||
req_data = WebsiteCrawlPayload.model_validate(payload)
|
||||
|
||||
with app.test_request_context("/", json=payload):
|
||||
crawl_url = mocker.patch.object(
|
||||
WebsiteService,
|
||||
"crawl_url",
|
||||
side_effect=Exception("crawl failed"),
|
||||
)
|
||||
|
||||
with pytest.raises(WebsiteCrawlError, match="crawl failed"):
|
||||
method(api, req_data)
|
||||
|
||||
crawl_url.assert_called_once_with(
|
||||
WebsiteCrawlApiRequest(provider="firecrawl", url="https://example.com", options={"limit": 1})
|
||||
)
|
||||
|
||||
|
||||
class TestWebsiteCrawlStatusApi:
|
||||
def test_get_status_success(self, app: Flask, mocker: MockerFixture) -> None:
|
||||
api = WebsiteCrawlStatusApi()
|
||||
method = inspect.unwrap(api.get)
|
||||
|
||||
job_id = "job-123"
|
||||
req_data = WebsiteCrawlStatusQuery.model_validate({"provider": "firecrawl"})
|
||||
|
||||
with app.test_request_context("/?provider=firecrawl"):
|
||||
get_status = mocker.patch.object(
|
||||
WebsiteService,
|
||||
"get_crawl_status_typed",
|
||||
return_value={"status": "completed"},
|
||||
)
|
||||
|
||||
result, status = method(api, req_data, job_id)
|
||||
|
||||
assert status == 200
|
||||
assert result["status"] == "completed"
|
||||
get_status.assert_called_once_with(WebsiteCrawlStatusApiRequest(provider="firecrawl", job_id=job_id))
|
||||
|
||||
def test_get_status_invalid_provider(self, app: Flask, mocker: MockerFixture) -> None:
|
||||
api = WebsiteCrawlStatusApi()
|
||||
method = inspect.unwrap(api.get)
|
||||
|
||||
job_id = "job-123"
|
||||
req_data = WebsiteCrawlStatusQuery.model_validate({"provider": "firecrawl"})
|
||||
|
||||
with app.test_request_context("/?provider=firecrawl"):
|
||||
mocker.patch.object(
|
||||
WebsiteCrawlStatusApiRequest,
|
||||
"from_args",
|
||||
side_effect=ValueError("invalid provider"),
|
||||
)
|
||||
|
||||
with pytest.raises(WebsiteCrawlError, match="invalid provider"):
|
||||
method(api, req_data, job_id)
|
||||
|
||||
def test_get_status_service_error(self, app: Flask, mocker: MockerFixture) -> None:
|
||||
api = WebsiteCrawlStatusApi()
|
||||
method = inspect.unwrap(api.get)
|
||||
|
||||
job_id = "job-123"
|
||||
req_data = WebsiteCrawlStatusQuery.model_validate({"provider": "firecrawl"})
|
||||
|
||||
with app.test_request_context("/?provider=firecrawl"):
|
||||
get_status = mocker.patch.object(
|
||||
WebsiteService,
|
||||
"get_crawl_status_typed",
|
||||
side_effect=Exception("status lookup failed"),
|
||||
)
|
||||
|
||||
with pytest.raises(WebsiteCrawlError, match="status lookup failed"):
|
||||
method(api, req_data, job_id)
|
||||
|
||||
get_status.assert_called_once_with(WebsiteCrawlStatusApiRequest(provider="firecrawl", job_id=job_id))
|
||||
@@ -0,0 +1,168 @@
|
||||
from collections.abc import Iterator
|
||||
|
||||
import pytest
|
||||
from pytest_mock import MockerFixture
|
||||
from sqlalchemy.orm import Session, scoped_session, sessionmaker
|
||||
|
||||
from controllers.console.datasets.error import PipelineNotFoundError
|
||||
from controllers.console.datasets.wraps import get_rag_pipeline, load_rag_pipeline
|
||||
from extensions.ext_database import db
|
||||
from models.account import Account
|
||||
from models.dataset import Pipeline
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def database_session(
|
||||
monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session]
|
||||
) -> Iterator[scoped_session[Session]]:
|
||||
"""Install a real scoped SQLite session for the legacy decorator entrypoint."""
|
||||
session = scoped_session(sqlite_session_factory)
|
||||
monkeypatch.setattr(db, "session", session)
|
||||
yield session
|
||||
session.remove()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def current_account() -> Account:
|
||||
account = Account(name="Test User", email="test@example.com")
|
||||
account.id = "account-1"
|
||||
return account
|
||||
|
||||
|
||||
class TestGetRagPipeline:
|
||||
def test_missing_pipeline_id(self) -> None:
|
||||
@get_rag_pipeline
|
||||
def dummy_view(**_kwargs: object) -> str:
|
||||
return "ok"
|
||||
|
||||
with pytest.raises(ValueError, match="missing pipeline_id"):
|
||||
dummy_view()
|
||||
|
||||
def test_pipeline_not_found(
|
||||
self,
|
||||
mocker: MockerFixture,
|
||||
database_session: scoped_session[Session],
|
||||
current_account: Account,
|
||||
) -> None:
|
||||
@get_rag_pipeline
|
||||
def dummy_view(**_kwargs: object) -> str:
|
||||
return "ok"
|
||||
|
||||
mocker.patch(
|
||||
"controllers.console.datasets.wraps.current_account_with_tenant",
|
||||
return_value=(current_account, "tenant-1"),
|
||||
)
|
||||
|
||||
get_pipeline_by_id = mocker.patch(
|
||||
"controllers.console.datasets.wraps.RagPipelineService.get_pipeline_by_id",
|
||||
return_value=None,
|
||||
)
|
||||
|
||||
with pytest.raises(PipelineNotFoundError):
|
||||
dummy_view(pipeline_id="pipeline-1")
|
||||
get_pipeline_by_id.assert_called_once_with("pipeline-1", "tenant-1", session=database_session())
|
||||
|
||||
def test_pipeline_found_and_injected(
|
||||
self,
|
||||
mocker: MockerFixture,
|
||||
database_session: scoped_session[Session],
|
||||
current_account: Account,
|
||||
) -> None:
|
||||
pipeline = Pipeline(
|
||||
tenant_id="tenant-1",
|
||||
name="Test Pipeline",
|
||||
)
|
||||
pipeline.id = "pipeline-1"
|
||||
|
||||
@get_rag_pipeline
|
||||
def dummy_view(**kwargs: object) -> object:
|
||||
return kwargs["pipeline"]
|
||||
|
||||
mocker.patch(
|
||||
"controllers.console.datasets.wraps.current_account_with_tenant",
|
||||
return_value=(current_account, "tenant-1"),
|
||||
)
|
||||
|
||||
get_pipeline_by_id = mocker.patch(
|
||||
"controllers.console.datasets.wraps.RagPipelineService.get_pipeline_by_id",
|
||||
return_value=pipeline,
|
||||
)
|
||||
|
||||
result = dummy_view(pipeline_id="pipeline-1")
|
||||
|
||||
assert result is pipeline
|
||||
get_pipeline_by_id.assert_called_once_with("pipeline-1", "tenant-1", session=database_session())
|
||||
|
||||
def test_load_rag_pipeline_uses_provided_session(
|
||||
self, mocker: MockerFixture, sqlite_session: Session, current_account: Account
|
||||
) -> None:
|
||||
pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")
|
||||
|
||||
mocker.patch(
|
||||
"controllers.console.datasets.wraps.current_account_with_tenant",
|
||||
return_value=(current_account, "tenant-1"),
|
||||
)
|
||||
get_pipeline_by_id = mocker.patch(
|
||||
"controllers.console.datasets.wraps.RagPipelineService.get_pipeline_by_id",
|
||||
return_value=pipeline,
|
||||
)
|
||||
|
||||
result = load_rag_pipeline(sqlite_session, "pipeline-1")
|
||||
|
||||
assert result is pipeline
|
||||
get_pipeline_by_id.assert_called_once_with("pipeline-1", "tenant-1", session=sqlite_session)
|
||||
|
||||
def test_pipeline_id_removed_from_kwargs(
|
||||
self,
|
||||
mocker: MockerFixture,
|
||||
database_session: scoped_session[Session],
|
||||
current_account: Account,
|
||||
) -> None:
|
||||
pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")
|
||||
|
||||
@get_rag_pipeline
|
||||
def dummy_view(**kwargs: object) -> str:
|
||||
assert "pipeline_id" not in kwargs
|
||||
return "ok"
|
||||
|
||||
mocker.patch(
|
||||
"controllers.console.datasets.wraps.current_account_with_tenant",
|
||||
return_value=(current_account, "tenant-1"),
|
||||
)
|
||||
|
||||
mocker.patch(
|
||||
"controllers.console.datasets.wraps.RagPipelineService.get_pipeline_by_id",
|
||||
return_value=pipeline,
|
||||
)
|
||||
|
||||
result = dummy_view(pipeline_id="pipeline-1")
|
||||
|
||||
assert result == "ok"
|
||||
assert db.session is database_session
|
||||
|
||||
def test_pipeline_id_cast_to_string(
|
||||
self,
|
||||
mocker: MockerFixture,
|
||||
database_session: scoped_session[Session],
|
||||
current_account: Account,
|
||||
) -> None:
|
||||
pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")
|
||||
|
||||
@get_rag_pipeline
|
||||
def dummy_view(**kwargs: object) -> object:
|
||||
return kwargs["pipeline"]
|
||||
|
||||
mocker.patch(
|
||||
"controllers.console.datasets.wraps.current_account_with_tenant",
|
||||
return_value=(current_account, "tenant-1"),
|
||||
)
|
||||
|
||||
get_pipeline_by_id = mocker.patch(
|
||||
"controllers.console.datasets.wraps.RagPipelineService.get_pipeline_by_id",
|
||||
return_value=pipeline,
|
||||
)
|
||||
|
||||
result = dummy_view(pipeline_id=123)
|
||||
|
||||
assert result is pipeline
|
||||
get_pipeline_by_id.assert_called_once_with("123", "tenant-1", session=database_session())
|
||||
Reference in new issue
Block a user