lib-x17-fintech 2.1.3__py3-none-any.whl

xfintech/connect/instance/tests/test_class_s3connect_all.py

@@ -0,0 +1,603 @@
+"""
+描述:
+- S3Connect 类的单元测试。
+- 测试 S3 连接实现的所有功能和边界情况。
+"""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from xfintech.connect.common.connect import ConnectLike
+from xfintech.connect.common.connectref import ConnectRef
+from xfintech.connect.instance.s3 import S3Connect
+
+# =============================================================================
+# Initialization Tests
+# =============================================================================
+
+
+def test_s3connect_init_default_client():
+    """测试使用默认客户端初始化"""
+    with patch("boto3.client") as mock_client_factory:
+        mock_client = MagicMock()
+        mock_client.put_object = MagicMock()
+        mock_client_factory.return_value = mock_client
+
+        connect = S3Connect()
+
+        assert connect.kind == "s3"
+        assert connect._client is mock_client
+        mock_client_factory.assert_called_once_with("s3")
+
+
+def test_s3connect_init_custom_client():
+    """测试使用自定义客户端初始化"""
+    mock_client = MagicMock()
+    mock_client.put_object = MagicMock()
+
+    connect = S3Connect(client=mock_client)
+
+    assert connect.kind == "s3"
+    assert connect._client is mock_client
+
+
+def test_s3connect_init_without_boto3():
+    """测试没有 boto3 时初始化失败"""
+    with patch("builtins.__import__", side_effect=ImportError("No module named 'boto3'")):
+        with pytest.raises(ImportError) as excinfo:
+            S3Connect()
+        assert "boto3 is required" in str(excinfo.value)
+        assert "pip install boto3" in str(excinfo.value)
+
+
+def test_s3connect_is_connectlike():
+    """测试 S3Connect 实现了 ConnectLike 协议"""
+    mock_client = MagicMock()
+    mock_client.put_object = MagicMock()
+
+    connect = S3Connect(client=mock_client)
+    assert isinstance(connect, ConnectLike)
+
+
+def test_s3connect_has_docstring():
+    """测试 S3Connect 有文档字符串"""
+    assert S3Connect.__doc__ is not None
+    assert len(S3Connect.__doc__) > 0
+
+
+# =============================================================================
+# Client Validation Tests
+# =============================================================================
+
+
+def test_resolve_client_valid_custom_client():
+    """测试解析有效的自定义客户端"""
+    mock_client = MagicMock()
+    mock_client.put_object = MagicMock()
+
+    connect = S3Connect(client=mock_client)
+    assert connect._client is mock_client
+
+
+def test_resolve_client_missing_put_object():
+    """测试客户端缺少 put_object 方法"""
+    with patch("boto3.client") as mock_client_factory:
+        mock_client = MagicMock(spec=[])  # No methods
+        mock_client_factory.return_value = mock_client
+
+        with pytest.raises(AttributeError) as excinfo:
+            S3Connect()
+        assert "missing put_object method" in str(excinfo.value)
+
+
+def test_resolve_client_put_object_not_callable():
+    """测试 put_object 不可调用"""
+    with patch("boto3.client") as mock_client_factory:
+        mock_client = MagicMock()
+        mock_client.put_object = "not_callable"
+        mock_client_factory.return_value = mock_client
+
+        with pytest.raises(TypeError) as excinfo:
+            S3Connect()
+        assert "put_object is not callable" in str(excinfo.value)
+
+
+# =============================================================================
+# URI Resolution Tests
+# =============================================================================
+
+
+def test_resolve_s3_uri_full_uri():
+    """测试解析完整 S3 URI"""
+    mock_client = MagicMock()
+    mock_client.put_object = MagicMock()
+
+    connect = S3Connect(client=mock_client)
+    result = connect._resolve_s3_uri("s3://bucket/key")
+
+    assert result.scheme == "s3"
+    assert result.netloc == "bucket"
+    assert result.path == "/key"
+
+
+def test_resolve_s3_uri_without_scheme():
+    """测试解析不带 scheme 的 URI"""
+    mock_client = MagicMock()
+    mock_client.put_object = MagicMock()
+
+    connect = S3Connect(client=mock_client)
+    result = connect._resolve_s3_uri("bucket/key")
+
+    assert result.scheme == "s3"
+    assert result.netloc == "bucket"
+    assert result.path == "/key"
+
+
+def test_resolve_s3_uri_nested_path():
+    """测试解析嵌套路径"""
+    mock_client = MagicMock()
+    mock_client.put_object = MagicMock()
+
+    connect = S3Connect(client=mock_client)
+    result = connect._resolve_s3_uri("s3://bucket/path/to/key")
+
+    assert result.netloc == "bucket"
+    assert result.path == "/path/to/key"
+
+
+def test_resolve_s3_uri_with_http_prefix():
+    """测试带 http 前缀的输入（会被转换）"""
+    mock_client = MagicMock()
+    mock_client.put_object = MagicMock()
+
+    connect = S3Connect(client=mock_client)
+    # http://bucket/key becomes s3://http://bucket/key, which is valid
+    result = connect._resolve_s3_uri("http://bucket/key")
+    assert result.scheme == "s3"
+    # The netloc becomes "http:" and path becomes "//bucket/key"
+    assert result.netloc == "http:"
+
+
+def test_resolve_s3_uri_missing_bucket():
+    """测试缺少 bucket 的 URI"""
+    mock_client = MagicMock()
+    mock_client.put_object = MagicMock()
+
+    connect = S3Connect(client=mock_client)
+    with pytest.raises(ValueError) as excinfo:
+        connect._resolve_s3_uri("s3:///key")
+    assert "missing bucket" in str(excinfo.value)
+
+
+def test_resolve_s3_uri_missing_key():
+    """测试缺少 key 的 URI"""
+    mock_client = MagicMock()
+    mock_client.put_object = MagicMock()
+
+    connect = S3Connect(client=mock_client)
+    with pytest.raises(ValueError) as excinfo:
+        connect._resolve_s3_uri("s3://bucket")
+    assert "missing key" in str(excinfo.value)
+
+
+# =============================================================================
+# Bucket Resolution Tests
+# =============================================================================
+
+
+def test_resolve_bucket_valid():
+    """测试解析有效的 bucket"""
+    mock_client = MagicMock()
+    mock_client.put_object = MagicMock()
+
+    connect = S3Connect(client=mock_client)
+    bucket = connect._resolve_bucket("s3://my-bucket/my-key")
+
+    assert bucket == "my-bucket"
+
+
+def test_resolve_bucket_with_nested_path():
+    """测试解析带嵌套路径的 bucket"""
+    mock_client = MagicMock()
+    mock_client.put_object = MagicMock()
+
+    connect = S3Connect(client=mock_client)
+    bucket = connect._resolve_bucket("s3://my-bucket/path/to/key")
+
+    assert bucket == "my-bucket"
+
+
+# =============================================================================
+# Key Resolution Tests
+# =============================================================================
+
+
+def test_resolve_key_valid():
+    """测试解析有效的 key"""
+    mock_client = MagicMock()
+    mock_client.put_object = MagicMock()
+
+    connect = S3Connect(client=mock_client)
+    key = connect._resolve_key("s3://bucket/my-key")
+
+    assert key == "my-key"
+
+
+def test_resolve_key_with_nested_path():
+    """测试解析嵌套路径的 key"""
+    mock_client = MagicMock()
+    mock_client.put_object = MagicMock()
+
+    connect = S3Connect(client=mock_client)
+    key = connect._resolve_key("s3://bucket/path/to/key.txt")
+
+    assert key == "path/to/key.txt"
+
+
+def test_resolve_key_strips_leading_slash():
+    """测试移除 key 开头的斜杠"""
+    mock_client = MagicMock()
+    mock_client.put_object = MagicMock()
+
+    connect = S3Connect(client=mock_client)
+    key = connect._resolve_key("s3://bucket/key")
+
+    # Path has leading slash, should be stripped
+    assert not key.startswith("/")
+
+
+# =============================================================================
+# put_bytes Tests
+# =============================================================================
+
+
+def test_put_bytes_success():
+    """测试成功写入字节"""
+    mock_client = MagicMock()
+    mock_client.put_object = MagicMock()
+
+    connect = S3Connect(client=mock_client)
+    connect.put_bytes(b"test data", "s3://bucket/key")
+
+    mock_client.put_object.assert_called_once()
+    call_kwargs = mock_client.put_object.call_args[1]
+    assert call_kwargs["Bucket"] == "bucket"
+    assert call_kwargs["Key"] == "key"
+    assert call_kwargs["Body"] == b"test data"
+
+
+def test_put_bytes_with_kwargs():
+    """测试传递额外参数"""
+    mock_client = MagicMock()
+    mock_client.put_object = MagicMock()
+
+    connect = S3Connect(client=mock_client)
+    connect.put_bytes(b"test data", "s3://bucket/key", ContentType="application/json", ACL="public-read")
+
+    call_kwargs = mock_client.put_object.call_args[1]
+    assert call_kwargs["ContentType"] == "application/json"
+    assert call_kwargs["ACL"] == "public-read"
+
+
+def test_put_bytes_nested_path():
+    """测试写入嵌套路径"""
+    mock_client = MagicMock()
+    mock_client.put_object = MagicMock()
+
+    connect = S3Connect(client=mock_client)
+    connect.put_bytes(b"data", "s3://bucket/path/to/file.txt")
+
+    call_kwargs = mock_client.put_object.call_args[1]
+    assert call_kwargs["Key"] == "path/to/file.txt"
+
+
+# =============================================================================
+# get_bytes Tests
+# =============================================================================
+
+
+def test_get_bytes_success():
+    """测试成功读取字节"""
+    mock_client = MagicMock()
+    mock_response = {"Body": MagicMock()}
+    mock_response["Body"].read.return_value = b"test data"
+    mock_client.get_object.return_value = mock_response
+
+    connect = S3Connect(client=mock_client)
+    data = connect.get_bytes("s3://bucket/key")
+
+    assert data == b"test data"
+    mock_client.get_object.assert_called_once()
+    call_kwargs = mock_client.get_object.call_args[1]
+    assert call_kwargs["Bucket"] == "bucket"
+    assert call_kwargs["Key"] == "key"
+
+
+def test_get_bytes_with_kwargs():
+    """测试传递额外参数"""
+    mock_client = MagicMock()
+    mock_response = {"Body": MagicMock()}
+    mock_response["Body"].read.return_value = b"data"
+    mock_client.get_object.return_value = mock_response
+
+    connect = S3Connect(client=mock_client)
+    connect.get_bytes("s3://bucket/key", VersionId="version123")
+
+    call_kwargs = mock_client.get_object.call_args[1]
+    assert call_kwargs["VersionId"] == "version123"
+
+
+def test_get_bytes_nested_path():
+    """测试读取嵌套路径"""
+    mock_client = MagicMock()
+    mock_response = {"Body": MagicMock()}
+    mock_response["Body"].read.return_value = b"data"
+    mock_client.get_object.return_value = mock_response
+
+    connect = S3Connect(client=mock_client)
+    connect.get_bytes("s3://bucket/path/to/file.txt")
+
+    call_kwargs = mock_client.get_object.call_args[1]
+    assert call_kwargs["Key"] == "path/to/file.txt"
+
+
+# =============================================================================
+# put_object Tests
+# =============================================================================
+
+
+def test_put_object_returns_connectref():
+    """测试 put_object 返回 ConnectRef"""
+    mock_client = MagicMock()
+    mock_client.put_object = MagicMock()
+
+    connect = S3Connect(client=mock_client)
+    ref = connect.put_object(b"data", "s3://bucket/key")
+
+    assert isinstance(ref, ConnectRef)
+    assert ref.location == "s3://bucket/key"
+    assert ref.kind == "s3"
+
+
+def test_put_object_calls_put_bytes():
+    """测试 put_object 调用 put_bytes"""
+    mock_client = MagicMock()
+    mock_client.put_object = MagicMock()
+
+    connect = S3Connect(client=mock_client)
+    connect.put_object(b"test", "s3://bucket/key")
+
+    mock_client.put_object.assert_called_once()
+
+
+def test_put_object_with_kwargs():
+    """测试 put_object 传递 kwargs"""
+    mock_client = MagicMock()
+    mock_client.put_object = MagicMock()
+
+    connect = S3Connect(client=mock_client)
+    connect.put_object(b"data", "s3://bucket/key", ContentType="text/plain")
+
+    call_kwargs = mock_client.put_object.call_args[1]
+    assert call_kwargs["ContentType"] == "text/plain"
+
+
+# =============================================================================
+# get_object Tests
+# =============================================================================
+
+
+def test_get_object_returns_bytes():
+    """测试 get_object 返回字节"""
+    mock_client = MagicMock()
+    mock_response = {"Body": MagicMock()}
+    mock_response["Body"].read.return_value = b"test data"
+    mock_client.get_object.return_value = mock_response
+
+    connect = S3Connect(client=mock_client)
+    data = connect.get_object("s3://bucket/key")
+
+    assert isinstance(data, bytes)
+    assert data == b"test data"
+
+
+def test_get_object_calls_get_bytes():
+    """测试 get_object 调用 get_bytes"""
+    mock_client = MagicMock()
+    mock_response = {"Body": MagicMock()}
+    mock_response["Body"].read.return_value = b"data"
+    mock_client.get_object.return_value = mock_response
+
+    connect = S3Connect(client=mock_client)
+    connect.get_object("s3://bucket/key")
+
+    mock_client.get_object.assert_called_once()
+
+
+def test_get_object_with_kwargs():
+    """测试 get_object 传递 kwargs"""
+    mock_client = MagicMock()
+    mock_response = {"Body": MagicMock()}
+    mock_response["Body"].read.return_value = b"data"
+    mock_client.get_object.return_value = mock_response
+
+    connect = S3Connect(client=mock_client)
+    connect.get_object("s3://bucket/key", Range="bytes=0-100")
+
+    call_kwargs = mock_client.get_object.call_args[1]
+    assert call_kwargs["Range"] == "bytes=0-100"
+
+
+# =============================================================================
+# Integration Tests
+# =============================================================================
+
+
+def test_round_trip():
+    """测试完整的读写流程"""
+    mock_client = MagicMock()
+    mock_client.put_object = MagicMock()
+    mock_response = {"Body": MagicMock()}
+    mock_response["Body"].read.return_value = b"test data"
+    mock_client.get_object.return_value = mock_response
+
+    connect = S3Connect(client=mock_client)
+
+    # Write
+    ref = connect.put_object(b"test data", "s3://bucket/key")
+    assert ref.location == "s3://bucket/key"
+
+    # Read
+    data = connect.get_object("s3://bucket/key")
+    assert data == b"test data"
+
+
+def test_multiple_operations():
+    """测试多次操作"""
+    mock_client = MagicMock()
+    mock_client.put_object = MagicMock()
+    mock_response = {"Body": MagicMock()}
+    mock_response["Body"].read.return_value = b"data"
+    mock_client.get_object.return_value = mock_response
+
+    connect = S3Connect(client=mock_client)
+
+    # Multiple writes
+    connect.put_object(b"data1", "s3://bucket/key1")
+    connect.put_object(b"data2", "s3://bucket/key2")
+
+    assert mock_client.put_object.call_count == 2
+
+
+def test_different_buckets():
+    """测试不同的 bucket"""
+    mock_client = MagicMock()
+    mock_client.put_object = MagicMock()
+
+    connect = S3Connect(client=mock_client)
+
+    connect.put_object(b"data", "s3://bucket1/key")
+    connect.put_object(b"data", "s3://bucket2/key")
+
+    calls = mock_client.put_object.call_args_list
+    assert calls[0][1]["Bucket"] == "bucket1"
+    assert calls[1][1]["Bucket"] == "bucket2"
+
+
+# =============================================================================
+# Edge Cases
+# =============================================================================
+
+
+def test_empty_data():
+    """测试空数据"""
+    mock_client = MagicMock()
+    mock_client.put_object = MagicMock()
+
+    connect = S3Connect(client=mock_client)
+    connect.put_object(b"", "s3://bucket/key")
+
+    call_kwargs = mock_client.put_object.call_args[1]
+    assert call_kwargs["Body"] == b""
+
+
+def test_large_data():
+    """测试大数据"""
+    mock_client = MagicMock()
+    mock_client.put_object = MagicMock()
+
+    large_data = b"x" * (10 * 1024 * 1024)  # 10MB
+
+    connect = S3Connect(client=mock_client)
+    connect.put_object(large_data, "s3://bucket/key")
+
+    call_kwargs = mock_client.put_object.call_args[1]
+    assert len(call_kwargs["Body"]) == 10 * 1024 * 1024
+
+
+def test_special_characters_in_key():
+    """测试 key 中的特殊字符"""
+    mock_client = MagicMock()
+    mock_client.put_object = MagicMock()
+
+    connect = S3Connect(client=mock_client)
+    connect.put_object(b"data", "s3://bucket/path with spaces/file.txt")
+
+    call_kwargs = mock_client.put_object.call_args[1]
+    assert "path with spaces" in call_kwargs["Key"]
+
+
+def test_unicode_in_key():
+    """测试 key 中的 Unicode 字符"""
+    mock_client = MagicMock()
+    mock_client.put_object = MagicMock()
+
+    connect = S3Connect(client=mock_client)
+    connect.put_object(b"data", "s3://bucket/文件/数据.txt")
+
+    call_kwargs = mock_client.put_object.call_args[1]
+    assert "文件" in call_kwargs["Key"]
+
+
+# =============================================================================
+# Error Handling Tests
+# =============================================================================
+
+
+def test_put_bytes_client_error():
+    """测试客户端错误"""
+    mock_client = MagicMock()
+    mock_client.put_object.side_effect = Exception("S3 Error")
+
+    connect = S3Connect(client=mock_client)
+
+    with pytest.raises(Exception) as excinfo:
+        connect.put_object(b"data", "s3://bucket/key")
+    assert "S3 Error" in str(excinfo.value)
+
+
+def test_get_bytes_client_error():
+    """测试读取时客户端错误"""
+    mock_client = MagicMock()
+    mock_client.get_object.side_effect = Exception("Not Found")
+
+    connect = S3Connect(client=mock_client)
+
+    with pytest.raises(Exception) as excinfo:
+        connect.get_object("s3://bucket/nonexistent")
+    assert "Not Found" in str(excinfo.value)
+
+
+# =============================================================================
+# Type Tests
+# =============================================================================
+
+
+def test_returns_correct_types():
+    """测试返回正确的类型"""
+    mock_client = MagicMock()
+    mock_client.put_object = MagicMock()
+    mock_response = {"Body": MagicMock()}
+    mock_response["Body"].read.return_value = b"data"
+    mock_client.get_object.return_value = mock_response
+
+    connect = S3Connect(client=mock_client)
+
+    # put_object returns ConnectRef
+    ref = connect.put_object(b"data", "s3://bucket/key")
+    assert isinstance(ref, ConnectRef)
+
+    # get_object returns bytes
+    data = connect.get_object("s3://bucket/key")
+    assert isinstance(data, bytes)
+
+    # put_bytes returns None
+    result = connect.put_bytes(b"data", "s3://bucket/key2")
+    assert result is None
+
+    # get_bytes returns bytes
+    data = connect.get_bytes("s3://bucket/key")
+    assert isinstance(data, bytes)

xfintech/data/__init__.py

@@ -0,0 +1,20 @@
+"""
+Data module for xfintech.
+
+This module provides the core data acquisition and processing infrastructure:
+
+Submodules:
+- common: Common utilities (Cache, Retry, Metric, Params, Paginate, Coolant)
+- job: Job system with registry and error handling
+- relay: Relay client for remote API access
+- source: Data source implementations (tushare, baostock)
+"""
+
+from . import common, job, relay, source
+
+__all__ = [
+    "common",
+    "job",
+    "relay",
+    "source",
+]

xfintech/data/common/__init__.py

@@ -0,0 +1,15 @@
+from .cache import Cache
+from .coolant import Coolant
+from .metric import Metric
+from .paginate import Paginate
+from .params import Params
+from .retry import Retry
+
+__all__ = [
+    "Cache",
+    "Metric",
+    "Params",
+    "Coolant",
+    "Paginate",
+    "Retry",
+]