chore: 纳入本地 Dify 1.19.0 源码及品牌与免登改动
演示目录只保留可重建的源码。运行密钥、数据卷、依赖和带口令的本地配置不进入仓库。 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
60158935f4
commit
6519682885
15831 files changed
+3063428
No files matched your search
Whitespace-only changes.
+132
@@ -0,0 +1,132 @@
|
||||
"""Unit coverage for ClickZetta configuration and path generation."""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from extensions.storage.clickzetta_volume.clickzetta_volume_storage import (
|
||||
ClickZettaVolumeConfig,
|
||||
ClickZettaVolumeStorage,
|
||||
)
|
||||
|
||||
|
||||
def _table_storage() -> ClickZettaVolumeStorage:
|
||||
config = ClickZettaVolumeConfig(
|
||||
username="test_user",
|
||||
password="test_pass",
|
||||
instance="test_instance",
|
||||
service="uat-api.clickzetta.com",
|
||||
workspace="quick_start",
|
||||
vcluster="default_ap",
|
||||
schema_name="dify",
|
||||
volume_type="table",
|
||||
table_prefix="test_dataset_",
|
||||
)
|
||||
storage = ClickZettaVolumeStorage.__new__(ClickZettaVolumeStorage)
|
||||
storage._config = config
|
||||
return storage
|
||||
|
||||
|
||||
def test_config_validation() -> None:
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match=(
|
||||
"1 validation error for ClickZettaVolumeConfig\\\n Value error, "
|
||||
"CLICKZETTA_VOLUME_USERNAME or CLICKZETTA_USERNAME is required"
|
||||
),
|
||||
):
|
||||
ClickZettaVolumeConfig(username="", password="pass", instance="instance")
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match=(
|
||||
"1 validation error for ClickZettaVolumeConfig\\\n Value error, "
|
||||
"CLICKZETTA_VOLUME_TYPE must be one of: table, user, external"
|
||||
),
|
||||
):
|
||||
ClickZettaVolumeConfig(username="user", password="pass", instance="instance", volume_type="invalid_type")
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match=(
|
||||
"1 validation error for ClickZettaVolumeConfig\\\n Value error, "
|
||||
"CLICKZETTA_VOLUME_NAME is required for external volume type"
|
||||
),
|
||||
):
|
||||
ClickZettaVolumeConfig(
|
||||
username="user",
|
||||
password="pass",
|
||||
instance="instance",
|
||||
volume_type="external",
|
||||
)
|
||||
|
||||
|
||||
def test_volume_path_generation() -> None:
|
||||
storage = _table_storage()
|
||||
|
||||
assert storage._get_volume_path("test.txt", "12345") == "test_dataset_12345/test.txt"
|
||||
assert storage._get_volume_path("12345/test.txt") == "12345/test.txt"
|
||||
|
||||
storage._config.volume_type = "user"
|
||||
assert storage._get_volume_path("test.txt") == "dify_km/test.txt"
|
||||
|
||||
|
||||
def test_sql_prefix_generation() -> None:
|
||||
storage = _table_storage()
|
||||
|
||||
assert storage._get_volume_sql_prefix("12345") == "TABLE VOLUME test_dataset_12345"
|
||||
|
||||
storage._config.volume_type = "user"
|
||||
assert storage._get_volume_sql_prefix() == "USER VOLUME"
|
||||
|
||||
storage._config.volume_type = "external"
|
||||
storage._config.volume_name = "my_external_volume"
|
||||
assert storage._get_volume_sql_prefix() == "VOLUME my_external_volume"
|
||||
|
||||
|
||||
def test_scan_table_prefix_matches_file_storage_path(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
storage = _table_storage()
|
||||
execute = MagicMock(return_value=[("app/trace.json",), ("another-app/trace.json",)])
|
||||
monkeypatch.setattr(storage, "_execute_sql", execute)
|
||||
assert storage.scan("ops_trace/app/") == ["ops_trace/app/trace.json"]
|
||||
execute.assert_called_once_with("LIST TABLE VOLUME test_dataset_ops_trace SUBDIRECTORY 'app/'", fetch=True)
|
||||
|
||||
|
||||
def test_scan_special_table_prefix_restores_logical_root(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
storage = _table_storage()
|
||||
execute = MagicMock(return_value=[("dify_km/tenant/file.txt",)])
|
||||
monkeypatch.setattr(storage, "_execute_sql", execute)
|
||||
assert storage.scan("upload_files/tenant/") == ["upload_files/tenant/file.txt"]
|
||||
execute.assert_called_once_with("LIST USER VOLUME SUBDIRECTORY 'dify_km/tenant/'", fetch=True)
|
||||
|
||||
|
||||
def test_scan_preserves_bare_dataset_root_behavior(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
storage = _table_storage()
|
||||
execute = MagicMock(return_value=[("file.txt",)])
|
||||
monkeypatch.setattr(storage, "_execute_sql", execute)
|
||||
assert storage.scan("12345") == ["file.txt"]
|
||||
execute.assert_called_once_with("LIST TABLE VOLUME test_dataset_12345", fetch=True)
|
||||
|
||||
|
||||
def test_scan_propagates_backend_failure(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
storage = _table_storage()
|
||||
monkeypatch.setattr(storage, "_execute_sql", MagicMock(side_effect=RuntimeError("backend unavailable")))
|
||||
with pytest.raises(RuntimeError, match="backend unavailable"):
|
||||
storage.scan("ops_trace/app/")
|
||||
|
||||
|
||||
def test_delete_propagates_failed_existence_check(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
storage = _table_storage()
|
||||
execute = MagicMock(side_effect=RuntimeError("backend unavailable"))
|
||||
monkeypatch.setattr(storage, "_execute_sql", execute)
|
||||
with pytest.raises(RuntimeError, match="backend unavailable"):
|
||||
storage.delete("ops_trace/app/trace.json")
|
||||
assert execute.call_count == 1
|
||||
|
||||
|
||||
def test_delete_absent_file_remains_idempotent(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
storage = _table_storage()
|
||||
execute = MagicMock(return_value=[])
|
||||
monkeypatch.setattr(storage, "_execute_sql", execute)
|
||||
storage.delete("ops_trace/app/trace.json")
|
||||
assert execute.call_count == 1
|
||||
@@ -0,0 +1,25 @@
|
||||
from typing import override
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from oss2 import Auth
|
||||
|
||||
from extensions.storage.aliyun_oss_storage import AliyunOssStorage
|
||||
from tests.helpers.unit.oss.__mock.base import (
|
||||
BaseStorageTest,
|
||||
get_example_bucket,
|
||||
get_example_folder,
|
||||
)
|
||||
|
||||
pytest_plugins = ("tests.helpers.unit.oss.__mock.aliyun_oss",)
|
||||
|
||||
|
||||
class TestAliyunOss(BaseStorageTest):
|
||||
@pytest.fixture(autouse=True)
|
||||
@override
|
||||
def setup_method(self, setup_aliyun_oss_mock: None) -> None:
|
||||
"""Executed before each test method."""
|
||||
with patch.object(Auth, "__init__", return_value=None):
|
||||
self.storage = AliyunOssStorage()
|
||||
self.storage.bucket_name = get_example_bucket()
|
||||
self.storage.folder = get_example_folder()
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Unit tests for the common AWS S3 storage adapter."""
|
||||
|
||||
from collections.abc import Callable
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from extensions.storage.aws_s3_storage import AwsS3Storage
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def s3_storage(config_overrides: Callable[..., None]) -> tuple[AwsS3Storage, MagicMock]:
|
||||
client = MagicMock()
|
||||
client.head_bucket.return_value = dict[str, object]()
|
||||
config_overrides(S3_USE_AWS_MANAGED_IAM=False, S3_BUCKET_NAME="dify-files")
|
||||
with patch("extensions.storage.aws_s3_storage.boto3.client", return_value=client):
|
||||
storage = AwsS3Storage()
|
||||
return storage, client
|
||||
|
||||
|
||||
def test_generate_presigned_url(s3_storage: tuple[AwsS3Storage, MagicMock]) -> None:
|
||||
storage, client = s3_storage
|
||||
client.generate_presigned_url.return_value = "https://s3.example.com/icon.png?signature=test"
|
||||
|
||||
result = storage.generate_presigned_url(
|
||||
"upload_files/tenant/icon.png",
|
||||
expires_in=300,
|
||||
content_type="image/png",
|
||||
)
|
||||
|
||||
assert result == "https://s3.example.com/icon.png?signature=test"
|
||||
client.generate_presigned_url.assert_called_once_with(
|
||||
"get_object",
|
||||
Params={
|
||||
"Bucket": "dify-files",
|
||||
"Key": "upload_files/tenant/icon.png",
|
||||
"ResponseContentType": "image/png",
|
||||
},
|
||||
ExpiresIn=300,
|
||||
)
|
||||
|
||||
|
||||
def test_scan_lists_recursive_files_and_derived_directories(
|
||||
s3_storage: tuple[AwsS3Storage, MagicMock],
|
||||
) -> None:
|
||||
storage, client = s3_storage
|
||||
paginator = client.get_paginator.return_value
|
||||
paginator.paginate.return_value = [
|
||||
{
|
||||
"Contents": [
|
||||
{"Key": "knowledge-fs/objects/tenant-1/a.txt"},
|
||||
{"Key": "knowledge-fs/objects/tenant-1/nested/b.txt"},
|
||||
{"Key": "knowledge-fs/objects/tenant-1/empty/"},
|
||||
]
|
||||
},
|
||||
{"Contents": [{"Key": "knowledge-fs/objects/tenant-1/nested/c.txt"}]},
|
||||
]
|
||||
|
||||
result = storage.scan(
|
||||
"knowledge-fs/objects/tenant-1",
|
||||
files=True,
|
||||
directories=True,
|
||||
)
|
||||
|
||||
assert result == [
|
||||
"knowledge-fs/objects/tenant-1/a.txt",
|
||||
"knowledge-fs/objects/tenant-1/empty/",
|
||||
"knowledge-fs/objects/tenant-1/nested/",
|
||||
"knowledge-fs/objects/tenant-1/nested/b.txt",
|
||||
"knowledge-fs/objects/tenant-1/nested/c.txt",
|
||||
]
|
||||
client.get_paginator.assert_called_once_with("list_objects_v2")
|
||||
paginator.paginate.assert_called_once_with(
|
||||
Bucket="dify-files",
|
||||
Prefix="knowledge-fs/objects/tenant-1/",
|
||||
)
|
||||
|
||||
|
||||
def test_scan_rejects_request_without_files_or_directories(
|
||||
s3_storage: tuple[AwsS3Storage, MagicMock],
|
||||
) -> None:
|
||||
storage, _ = s3_storage
|
||||
|
||||
with pytest.raises(ValueError, match="At least one"):
|
||||
storage.scan("knowledge-fs", files=False, directories=False)
|
||||
@@ -0,0 +1,62 @@
|
||||
from typing import override
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from baidubce.auth.bce_credentials import BceCredentials
|
||||
from baidubce.bce_client_configuration import BceClientConfiguration
|
||||
|
||||
from extensions.storage.baidu_obs_storage import BaiduObsStorage
|
||||
from tests.helpers.unit.oss.__mock.base import (
|
||||
BaseStorageTest,
|
||||
get_example_bucket,
|
||||
)
|
||||
|
||||
pytest_plugins = ("tests.helpers.unit.oss.__mock.baidu_obs",)
|
||||
|
||||
|
||||
class TestBaiduObs(BaseStorageTest):
|
||||
@pytest.fixture(autouse=True)
|
||||
@override
|
||||
def setup_method(self, setup_baidu_obs_mock: None) -> None:
|
||||
"""Executed before each test method."""
|
||||
with (
|
||||
patch.object(BceCredentials, "__init__", return_value=None),
|
||||
patch.object(BceClientConfiguration, "__init__", return_value=None),
|
||||
):
|
||||
self.storage = BaiduObsStorage()
|
||||
self.storage.bucket_name = get_example_bucket()
|
||||
|
||||
|
||||
class TestBaiduObsConfiguration:
|
||||
def test_init_with_config(self) -> None:
|
||||
mock_dify_config = MagicMock()
|
||||
mock_dify_config.BAIDU_OBS_BUCKET_NAME = "test-bucket"
|
||||
mock_dify_config.BAIDU_OBS_ACCESS_KEY = "test-access-key"
|
||||
mock_dify_config.BAIDU_OBS_SECRET_KEY = "test-secret-key"
|
||||
mock_dify_config.BAIDU_OBS_ENDPOINT = "https://bj.bcebos.com"
|
||||
|
||||
mock_credentials = MagicMock(name="credentials")
|
||||
mock_config = MagicMock(name="config")
|
||||
mock_client = MagicMock(name="client")
|
||||
|
||||
with (
|
||||
patch("extensions.storage.baidu_obs_storage.dify_config", mock_dify_config),
|
||||
patch("extensions.storage.baidu_obs_storage.BceCredentials", return_value=mock_credentials) as credentials,
|
||||
patch(
|
||||
"extensions.storage.baidu_obs_storage.BceClientConfiguration", return_value=mock_config
|
||||
) as configuration,
|
||||
patch("extensions.storage.baidu_obs_storage.BosClient", return_value=mock_client) as client_cls,
|
||||
):
|
||||
storage = BaiduObsStorage()
|
||||
|
||||
assert storage.bucket_name == "test-bucket"
|
||||
assert storage.client == mock_client
|
||||
credentials.assert_called_once_with(
|
||||
access_key_id="test-access-key",
|
||||
secret_access_key="test-secret-key",
|
||||
)
|
||||
configuration.assert_called_once_with(
|
||||
credentials=mock_credentials,
|
||||
endpoint="https://bj.bcebos.com",
|
||||
)
|
||||
client_cls.assert_called_once_with(config=mock_config)
|
||||
@@ -0,0 +1,143 @@
|
||||
from collections.abc import Generator
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from extensions.storage.opendal_storage import OpenDALStorage
|
||||
from tests.helpers.unit.oss.__mock.base import (
|
||||
get_example_data,
|
||||
get_example_filename,
|
||||
get_opendal_bucket,
|
||||
)
|
||||
|
||||
"""Unit coverage for OpenDAL filesystem root selection."""
|
||||
|
||||
|
||||
class TestOpenDALFsDefaultRoot:
|
||||
"""Test that OpenDALStorage with scheme='fs' works correctly when no root is provided."""
|
||||
|
||||
def test_fs_without_root_uses_default(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""When no root is specified, the default 'storage' should be used and passed to the Operator."""
|
||||
# Change to tmp_path so the default "storage" dir is created there
|
||||
monkeypatch.chdir(tmp_path)
|
||||
# Ensure no OPENDAL_FS_ROOT env var is set
|
||||
monkeypatch.delenv("OPENDAL_FS_ROOT", raising=False)
|
||||
|
||||
storage = OpenDALStorage(scheme="fs")
|
||||
|
||||
# The default directory should have been created
|
||||
assert (tmp_path / "storage").is_dir()
|
||||
# The storage should be functional
|
||||
storage.save("test_default_root.txt", b"hello")
|
||||
assert storage.exists("test_default_root.txt")
|
||||
assert storage.load_once("test_default_root.txt") == b"hello"
|
||||
|
||||
# Cleanup
|
||||
storage.delete("test_default_root.txt")
|
||||
|
||||
def test_fs_with_explicit_root(self, tmp_path: Path) -> None:
|
||||
"""When root is explicitly provided, it should be used."""
|
||||
custom_root = str(tmp_path / "custom_storage")
|
||||
storage = OpenDALStorage(scheme="fs", root=custom_root)
|
||||
|
||||
assert Path(custom_root).is_dir()
|
||||
storage.save("test_explicit_root.txt", b"world")
|
||||
assert storage.exists("test_explicit_root.txt")
|
||||
assert storage.load_once("test_explicit_root.txt") == b"world"
|
||||
|
||||
# Cleanup
|
||||
storage.delete("test_explicit_root.txt")
|
||||
|
||||
def test_fs_with_env_var_root(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""When OPENDAL_FS_ROOT env var is set, it should be picked up via _get_opendal_kwargs."""
|
||||
env_root = str(tmp_path / "env_storage")
|
||||
monkeypatch.setenv("OPENDAL_FS_ROOT", env_root)
|
||||
# Ensure .env file doesn't interfere
|
||||
monkeypatch.chdir(tmp_path)
|
||||
|
||||
storage = OpenDALStorage(scheme="fs")
|
||||
|
||||
assert Path(env_root).is_dir()
|
||||
storage.save("test_env_root.txt", b"env_data")
|
||||
assert storage.exists("test_env_root.txt")
|
||||
assert storage.load_once("test_env_root.txt") == b"env_data"
|
||||
|
||||
# Cleanup
|
||||
storage.delete("test_env_root.txt")
|
||||
|
||||
|
||||
class TestOpenDAL:
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_method(self) -> None:
|
||||
"""Executed before each test method."""
|
||||
self.storage = OpenDALStorage(
|
||||
scheme="fs",
|
||||
root=get_opendal_bucket(),
|
||||
)
|
||||
|
||||
@pytest.fixture(scope="class", autouse=True)
|
||||
def teardown_class(self) -> Generator[None]:
|
||||
"""Clean up after all tests in the class."""
|
||||
|
||||
yield
|
||||
|
||||
folder = Path(get_opendal_bucket())
|
||||
if folder.exists() and folder.is_dir():
|
||||
import shutil
|
||||
|
||||
shutil.rmtree(folder, ignore_errors=True)
|
||||
|
||||
def test_save_and_exists(self) -> None:
|
||||
"""Test saving data and checking existence."""
|
||||
filename = get_example_filename()
|
||||
data = get_example_data()
|
||||
|
||||
assert not self.storage.exists(filename)
|
||||
self.storage.save(filename, data)
|
||||
assert self.storage.exists(filename)
|
||||
|
||||
def test_load_once(self) -> None:
|
||||
"""Test loading data once."""
|
||||
filename = get_example_filename()
|
||||
data = get_example_data()
|
||||
|
||||
self.storage.save(filename, data)
|
||||
loaded_data = self.storage.load_once(filename)
|
||||
assert loaded_data == data
|
||||
|
||||
def test_load_stream(self) -> None:
|
||||
"""Test loading data as a stream."""
|
||||
filename = get_example_filename()
|
||||
chunks = 5
|
||||
chunk_size = 4096
|
||||
data = get_example_data(length=chunk_size * chunks)
|
||||
|
||||
self.storage.save(filename, data)
|
||||
generator = self.storage.load_stream(filename)
|
||||
assert isinstance(generator, Generator)
|
||||
for i in range(chunks):
|
||||
fetched = next(generator)
|
||||
assert len(fetched) == chunk_size
|
||||
assert fetched == data[i * chunk_size : (i + 1) * chunk_size]
|
||||
with pytest.raises(StopIteration):
|
||||
next(generator)
|
||||
|
||||
def test_download(self) -> None:
|
||||
"""Test downloading data to a file."""
|
||||
filename = get_example_filename()
|
||||
filepath = str(Path(get_opendal_bucket()) / filename)
|
||||
data = get_example_data()
|
||||
|
||||
self.storage.save(filename, data)
|
||||
self.storage.download(filename, filepath)
|
||||
|
||||
def test_delete(self) -> None:
|
||||
"""Test deleting a file."""
|
||||
filename = get_example_filename()
|
||||
data = get_example_data()
|
||||
|
||||
self.storage.save(filename, data)
|
||||
assert self.storage.exists(filename)
|
||||
|
||||
self.storage.delete(filename)
|
||||
assert not self.storage.exists(filename)
|
||||
@@ -0,0 +1,239 @@
|
||||
from collections.abc import Callable, Generator
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from extensions.storage.supabase_storage import SupabaseStorage
|
||||
|
||||
|
||||
class TestSupabaseStorage:
|
||||
"""Test suite for SupabaseStorage class."""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _supabase_config(self, config_overrides: Callable[..., None]) -> None:
|
||||
config_overrides(
|
||||
SUPABASE_URL="https://test.supabase.co",
|
||||
SUPABASE_API_KEY="test-api-key",
|
||||
SUPABASE_BUCKET_NAME="test-bucket",
|
||||
)
|
||||
|
||||
def test_init_success_with_all_config(self) -> None:
|
||||
"""Test successful initialization when all required config is provided."""
|
||||
with patch("extensions.storage.supabase_storage.Client", autospec=True) as mock_client_class:
|
||||
mock_client = Mock()
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
# Mock bucket_exists to return True so create_bucket is not called
|
||||
with patch.object(SupabaseStorage, "bucket_exists", return_value=True):
|
||||
storage = SupabaseStorage()
|
||||
|
||||
assert storage.bucket_name == "test-bucket"
|
||||
mock_client_class.assert_called_once_with(
|
||||
supabase_url="https://test.supabase.co", supabase_key="test-api-key"
|
||||
)
|
||||
|
||||
def test_init_raises_error_when_url_missing(self, config_overrides: Callable[..., None]) -> None:
|
||||
"""Test initialization raises ValueError when SUPABASE_URL is None."""
|
||||
config_overrides(SUPABASE_URL=None)
|
||||
with pytest.raises(ValueError, match="SUPABASE_URL is not set"):
|
||||
SupabaseStorage()
|
||||
|
||||
def test_init_raises_error_when_api_key_missing(self, config_overrides: Callable[..., None]) -> None:
|
||||
"""Test initialization raises ValueError when SUPABASE_API_KEY is None."""
|
||||
config_overrides(SUPABASE_API_KEY=None)
|
||||
with pytest.raises(ValueError, match="SUPABASE_API_KEY is not set"):
|
||||
SupabaseStorage()
|
||||
|
||||
def test_init_raises_error_when_bucket_name_missing(self, config_overrides: Callable[..., None]) -> None:
|
||||
"""Test initialization raises ValueError when SUPABASE_BUCKET_NAME is None."""
|
||||
config_overrides(SUPABASE_BUCKET_NAME=None)
|
||||
with pytest.raises(ValueError, match="SUPABASE_BUCKET_NAME is not set"):
|
||||
SupabaseStorage()
|
||||
|
||||
def test_create_bucket_when_not_exists(self) -> None:
|
||||
"""Test create_bucket creates bucket when it doesn't exist."""
|
||||
with patch("extensions.storage.supabase_storage.Client", autospec=True) as mock_client_class:
|
||||
mock_client = Mock()
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
with patch.object(SupabaseStorage, "bucket_exists", return_value=False):
|
||||
SupabaseStorage()
|
||||
|
||||
mock_client.storage.create_bucket.assert_called_once_with(id="test-bucket", name="test-bucket")
|
||||
|
||||
def test_create_bucket_when_exists(self) -> None:
|
||||
"""Test create_bucket does not create bucket when it already exists."""
|
||||
with patch("extensions.storage.supabase_storage.Client", autospec=True) as mock_client_class:
|
||||
mock_client = Mock()
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
with patch.object(SupabaseStorage, "bucket_exists", return_value=True):
|
||||
SupabaseStorage()
|
||||
|
||||
mock_client.storage.create_bucket.assert_not_called()
|
||||
|
||||
@pytest.fixture
|
||||
def storage_with_mock_client(self) -> Generator[tuple[SupabaseStorage, Mock]]:
|
||||
"""Fixture providing SupabaseStorage with mocked client."""
|
||||
with patch("extensions.storage.supabase_storage.Client", autospec=True) as mock_client_class:
|
||||
mock_client = Mock()
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
with patch.object(SupabaseStorage, "bucket_exists", return_value=True):
|
||||
storage = SupabaseStorage()
|
||||
# Create fresh mock for each test
|
||||
mock_client.reset_mock()
|
||||
yield storage, mock_client
|
||||
|
||||
def test_save(self, storage_with_mock_client: tuple[SupabaseStorage, Mock]) -> None:
|
||||
"""Test save calls client.storage.from_(bucket).upload(path, data)."""
|
||||
storage, mock_client = storage_with_mock_client
|
||||
|
||||
filename = "test.txt"
|
||||
data = b"test data"
|
||||
|
||||
storage.save(filename, data)
|
||||
|
||||
mock_client.storage.from_.assert_called_once_with("test-bucket")
|
||||
mock_client.storage.from_().upload.assert_called_once_with(filename, data)
|
||||
|
||||
def test_load_once_returns_bytes(self, storage_with_mock_client: tuple[SupabaseStorage, Mock]) -> None:
|
||||
"""Test load_once returns bytes."""
|
||||
storage, mock_client = storage_with_mock_client
|
||||
|
||||
expected_data = b"test content"
|
||||
mock_client.storage.from_().download.return_value = expected_data
|
||||
|
||||
result = storage.load_once("test.txt")
|
||||
|
||||
assert result == expected_data
|
||||
# Verify the correct calls were made
|
||||
assert "test-bucket" in [call[0][0] for call in mock_client.storage.from_.call_args_list if call[0]]
|
||||
mock_client.storage.from_().download.assert_called_with("test.txt")
|
||||
|
||||
def test_load_stream_yields_chunks(self, storage_with_mock_client: tuple[SupabaseStorage, Mock]) -> None:
|
||||
"""Test load_stream yields chunks."""
|
||||
storage, mock_client = storage_with_mock_client
|
||||
|
||||
test_data = b"test content for streaming"
|
||||
mock_client.storage.from_().download.return_value = test_data
|
||||
|
||||
result = storage.load_stream("test.txt")
|
||||
|
||||
assert isinstance(result, Generator)
|
||||
|
||||
# Collect all chunks
|
||||
chunks = list(result)
|
||||
|
||||
# Verify chunks contain the expected data
|
||||
assert b"".join(chunks) == test_data
|
||||
# Verify the correct calls were made
|
||||
assert "test-bucket" in [call[0][0] for call in mock_client.storage.from_.call_args_list if call[0]]
|
||||
mock_client.storage.from_().download.assert_called_with("test.txt")
|
||||
|
||||
def test_download_writes_bytes_to_disk(
|
||||
self, storage_with_mock_client: tuple[SupabaseStorage, Mock], tmp_path: Path
|
||||
) -> None:
|
||||
"""Test download writes expected bytes to disk."""
|
||||
storage, mock_client = storage_with_mock_client
|
||||
|
||||
test_data = b"test file content"
|
||||
mock_client.storage.from_().download.return_value = test_data
|
||||
|
||||
target_file = tmp_path / "downloaded_file.txt"
|
||||
|
||||
storage.download("test.txt", str(target_file))
|
||||
|
||||
# Verify file was written with correct content
|
||||
assert target_file.read_bytes() == test_data
|
||||
# Verify the correct calls were made
|
||||
assert "test-bucket" in [call[0][0] for call in mock_client.storage.from_.call_args_list if call[0]]
|
||||
mock_client.storage.from_().download.assert_called_with("test.txt")
|
||||
|
||||
def test_exists_returns_true_when_file_found(self, storage_with_mock_client: tuple[SupabaseStorage, Mock]) -> None:
|
||||
"""Test exists returns True when list() returns items."""
|
||||
storage, mock_client = storage_with_mock_client
|
||||
|
||||
mock_client.storage.from_().list.return_value = [{"name": "test.txt"}]
|
||||
|
||||
result = storage.exists("test.txt")
|
||||
|
||||
assert result is True
|
||||
assert "test-bucket" in [call[0][0] for call in mock_client.storage.from_.call_args_list if call[0]]
|
||||
mock_client.storage.from_().list.assert_called_with(path="test.txt")
|
||||
|
||||
def test_exists_returns_false_when_file_not_found(
|
||||
self, storage_with_mock_client: tuple[SupabaseStorage, Mock]
|
||||
) -> None:
|
||||
"""Test exists returns False when list() returns an empty list."""
|
||||
storage, mock_client = storage_with_mock_client
|
||||
|
||||
mock_client.storage.from_().list.return_value = []
|
||||
|
||||
result = storage.exists("test.txt")
|
||||
|
||||
assert result is False
|
||||
assert "test-bucket" in [call[0][0] for call in mock_client.storage.from_.call_args_list if call[0]]
|
||||
mock_client.storage.from_().list.assert_called_with(path="test.txt")
|
||||
|
||||
def test_delete_calls_remove_with_filename_in_list(
|
||||
self, storage_with_mock_client: tuple[SupabaseStorage, Mock]
|
||||
) -> None:
|
||||
"""Test delete calls remove([...]) (some client versions require a list)."""
|
||||
storage, mock_client = storage_with_mock_client
|
||||
|
||||
filename = "test.txt"
|
||||
|
||||
storage.delete(filename)
|
||||
|
||||
mock_client.storage.from_.assert_called_once_with("test-bucket")
|
||||
mock_client.storage.from_().remove.assert_called_once_with([filename])
|
||||
|
||||
def test_bucket_exists_returns_true_when_bucket_found(self) -> None:
|
||||
"""Test bucket_exists returns True when bucket is found in list."""
|
||||
with patch("extensions.storage.supabase_storage.Client", autospec=True) as mock_client_class:
|
||||
mock_client = Mock()
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
mock_bucket = Mock()
|
||||
mock_bucket.name = "test-bucket"
|
||||
mock_client.storage.list_buckets.return_value = [mock_bucket]
|
||||
storage = SupabaseStorage()
|
||||
result = storage.bucket_exists()
|
||||
|
||||
assert result is True
|
||||
assert mock_client.storage.list_buckets.call_count >= 1
|
||||
|
||||
def test_bucket_exists_returns_false_when_bucket_not_found(self) -> None:
|
||||
"""Test bucket_exists returns False when bucket is not found in list."""
|
||||
with patch("extensions.storage.supabase_storage.Client", autospec=True) as mock_client_class:
|
||||
mock_client = Mock()
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
# Mock different bucket
|
||||
mock_bucket = Mock()
|
||||
mock_bucket.name = "different-bucket"
|
||||
mock_client.storage.list_buckets.return_value = [mock_bucket]
|
||||
mock_client.storage.create_bucket = Mock()
|
||||
|
||||
storage = SupabaseStorage()
|
||||
result = storage.bucket_exists()
|
||||
|
||||
assert result is False
|
||||
assert mock_client.storage.list_buckets.call_count >= 1
|
||||
|
||||
def test_bucket_exists_returns_false_when_no_buckets(self) -> None:
|
||||
"""Test bucket_exists returns False when no buckets exist."""
|
||||
with patch("extensions.storage.supabase_storage.Client", autospec=True) as mock_client_class:
|
||||
mock_client = Mock()
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
mock_client.storage.list_buckets.return_value = list[object]()
|
||||
mock_client.storage.create_bucket = Mock()
|
||||
|
||||
storage = SupabaseStorage()
|
||||
result = storage.bucket_exists()
|
||||
|
||||
assert result is False
|
||||
assert mock_client.storage.list_buckets.call_count >= 1
|
||||
@@ -0,0 +1,102 @@
|
||||
from pathlib import Path
|
||||
from typing import override
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from qcloud_cos import CosConfig
|
||||
|
||||
from extensions.storage.tencent_cos_storage import TencentCosStorage
|
||||
from tests.helpers.unit.oss.__mock.base import (
|
||||
BaseStorageTest,
|
||||
get_example_bucket,
|
||||
get_example_data,
|
||||
get_example_filename,
|
||||
)
|
||||
|
||||
pytest_plugins = ("tests.helpers.unit.oss.__mock.tencent_cos",)
|
||||
|
||||
|
||||
class TestTencentCos(BaseStorageTest):
|
||||
@pytest.fixture(autouse=True)
|
||||
@override
|
||||
def setup_method(self, setup_tencent_cos_mock: None) -> None:
|
||||
"""Executed before each test method."""
|
||||
with patch.object(CosConfig, "__init__", return_value=None, autospec=True):
|
||||
self.storage = TencentCosStorage()
|
||||
self.storage.bucket_name = get_example_bucket()
|
||||
|
||||
# Replaces the base test so the download lands in tmp_path; the extra fixture argument narrows the signature.
|
||||
@override
|
||||
def test_download(self, tmp_path: Path) -> None: # pyrefly: ignore[bad-override]
|
||||
destination = tmp_path / "download.txt"
|
||||
self.storage.download(get_example_filename(), str(destination))
|
||||
assert destination.read_bytes() == get_example_data()
|
||||
|
||||
|
||||
class TestTencentCosConfiguration:
|
||||
"""Tests for TencentCosStorage initialization with different configurations."""
|
||||
|
||||
def test_init_with_custom_domain(self) -> None:
|
||||
"""Test initialization with custom domain configured."""
|
||||
# Mock dify_config to return custom domain configuration
|
||||
mock_dify_config = MagicMock()
|
||||
mock_dify_config.TENCENT_COS_CUSTOM_DOMAIN = "cos.example.com"
|
||||
mock_dify_config.TENCENT_COS_SECRET_ID = "test-secret-id"
|
||||
mock_dify_config.TENCENT_COS_SECRET_KEY = "test-secret-key"
|
||||
mock_dify_config.TENCENT_COS_SCHEME = "https"
|
||||
|
||||
# Mock CosConfig and CosS3Client
|
||||
mock_config_instance = MagicMock()
|
||||
mock_client = MagicMock()
|
||||
|
||||
with (
|
||||
patch("extensions.storage.tencent_cos_storage.dify_config", mock_dify_config),
|
||||
patch(
|
||||
"extensions.storage.tencent_cos_storage.CosConfig", return_value=mock_config_instance, autospec=True
|
||||
) as mock_cos_config,
|
||||
patch("extensions.storage.tencent_cos_storage.CosS3Client", return_value=mock_client, autospec=True),
|
||||
):
|
||||
TencentCosStorage()
|
||||
|
||||
# Verify CosConfig was called with Domain parameter (not Region)
|
||||
mock_cos_config.assert_called_once()
|
||||
call_kwargs = mock_cos_config.call_args[1]
|
||||
assert "Domain" in call_kwargs
|
||||
assert call_kwargs["Domain"] == "cos.example.com"
|
||||
assert "Region" not in call_kwargs
|
||||
assert call_kwargs["SecretId"] == "test-secret-id"
|
||||
assert call_kwargs["SecretKey"] == "test-secret-key"
|
||||
assert call_kwargs["Scheme"] == "https"
|
||||
|
||||
def test_init_with_region(self) -> None:
|
||||
"""Test initialization with region configured (no custom domain)."""
|
||||
# Mock dify_config to return region configuration
|
||||
mock_dify_config = MagicMock()
|
||||
mock_dify_config.TENCENT_COS_CUSTOM_DOMAIN = None
|
||||
mock_dify_config.TENCENT_COS_REGION = "ap-guangzhou"
|
||||
mock_dify_config.TENCENT_COS_SECRET_ID = "test-secret-id"
|
||||
mock_dify_config.TENCENT_COS_SECRET_KEY = "test-secret-key"
|
||||
mock_dify_config.TENCENT_COS_SCHEME = "https"
|
||||
|
||||
# Mock CosConfig and CosS3Client
|
||||
mock_config_instance = MagicMock()
|
||||
mock_client = MagicMock()
|
||||
|
||||
with (
|
||||
patch("extensions.storage.tencent_cos_storage.dify_config", mock_dify_config),
|
||||
patch(
|
||||
"extensions.storage.tencent_cos_storage.CosConfig", return_value=mock_config_instance, autospec=True
|
||||
) as mock_cos_config,
|
||||
patch("extensions.storage.tencent_cos_storage.CosS3Client", return_value=mock_client, autospec=True),
|
||||
):
|
||||
TencentCosStorage()
|
||||
|
||||
# Verify CosConfig was called with Region parameter (not Domain)
|
||||
mock_cos_config.assert_called_once()
|
||||
call_kwargs = mock_cos_config.call_args[1]
|
||||
assert "Region" in call_kwargs
|
||||
assert call_kwargs["Region"] == "ap-guangzhou"
|
||||
assert "Domain" not in call_kwargs
|
||||
assert call_kwargs["SecretId"] == "test-secret-id"
|
||||
assert call_kwargs["SecretKey"] == "test-secret-key"
|
||||
assert call_kwargs["Scheme"] == "https"
|
||||
@@ -0,0 +1,35 @@
|
||||
from collections.abc import Callable
|
||||
from typing import override
|
||||
|
||||
import pytest
|
||||
from tos import TosClientV2
|
||||
|
||||
from extensions.storage.volcengine_tos_storage import VolcengineTosStorage
|
||||
from tests.helpers.unit.oss.__mock.base import (
|
||||
BaseStorageTest,
|
||||
get_example_bucket,
|
||||
)
|
||||
|
||||
pytest_plugins = ("tests.helpers.unit.oss.__mock.volcengine_tos",)
|
||||
|
||||
|
||||
class TestVolcengineTos(BaseStorageTest):
|
||||
@pytest.fixture(autouse=True)
|
||||
@override
|
||||
def setup_method(self, setup_volcengine_tos_mock: None, config_overrides: Callable[..., None]) -> None:
|
||||
"""Executed before each test method."""
|
||||
config_overrides(
|
||||
VOLCENGINE_TOS_ACCESS_KEY="test_access_key",
|
||||
VOLCENGINE_TOS_SECRET_KEY="test_secret_key",
|
||||
VOLCENGINE_TOS_ENDPOINT="test_endpoint",
|
||||
VOLCENGINE_TOS_REGION="test_region",
|
||||
)
|
||||
self.storage = VolcengineTosStorage()
|
||||
|
||||
self.storage.bucket_name = get_example_bucket()
|
||||
self.storage.client = TosClientV2(
|
||||
ak="dify",
|
||||
sk="dify",
|
||||
endpoint="https://xxx.volces.com",
|
||||
region="cn-beijing",
|
||||
)
|
||||
Reference in new issue
Block a user