lib-x17-fintech 2.1.3__py3-none-any.whl

xfintech/serde/serialiser/python.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from xfintech.serde.common.dataformat import DataFormat
+from xfintech.serde.common.error import (
+    SerialiserFailedError,
+    SerialiserInputTypeError,
+    SerialiserNotSupportedError,
+)
+from xfintech.serde.common.serialiserlike import SerialiserLike
+
+
+class PythonSerialiser(SerialiserLike):
+    """
+    描述:
+    - PythonDict序列化模块。
+    - DICT -> BYTES 的单向转换，适用于配置和轻量级数据交换。
+
+    方法:
+    - serialise(data, format, **kwargs): 通用序列化方法，仅支持 JSON 格式。
+    - to_json(data, **kwargs): 将数据编码为 JSON 字节流，编码默认 "utf-8"。
+
+    例子:
+    ```python
+        from xfintech.serde.serialiser.python import PythonSerialiser
+        from xfintech.serde.common.dataformat import DataFormat
+
+        # 使用通用序列化方法（推荐）
+        bytes = PythonSerialiser.serialise(
+            data,
+            format=DataFormat.JSON,
+        )
+    ```
+    """
+
+    @staticmethod
+    def serialise(
+        data: Any,
+        format: DataFormat | str,
+        **kwargs: Any,
+    ) -> bytes:
+        if not isinstance(data, dict):
+            msg = f"PythonSerialiser.serialise expects dict, got {type(data)}"
+            raise SerialiserInputTypeError(msg)
+        format = DataFormat.from_any(format)
+        if format == DataFormat.JSON:
+            return PythonSerialiser.to_json(data, **kwargs)
+        else:
+            msg = f"Unsupported DataFormat for serialization: {format}"
+            raise SerialiserNotSupportedError(msg)
+
+    @staticmethod
+    def to_json(
+        data: Any,
+        **kwargs: Any,
+    ) -> bytes:
+        encoding = kwargs.pop("encoding", "utf-8")
+        kwargs.setdefault("ensure_ascii", False)
+        kwargs.setdefault("indent", 4)
+        kwargs.setdefault("sort_keys", True)
+        try:
+            payload = json.dumps(
+                data,
+                **kwargs,
+            )
+            return payload.encode(encoding)
+        except Exception as e:
+            msg = f"Failed to serialize data: {e}"
+            raise SerialiserFailedError(msg) from e

xfintech/serde/serialiser/tests/__init__.py

@@ -0,0 +1 @@
+# Tests for serialiser modules

xfintech/serde/serialiser/tests/test_class_pandasserialiser_all.py

@@ -0,0 +1,474 @@
+from io import BytesIO
+
+import pandas as pd
+import pytest
+
+from xfintech.serde.common.dataformat import DataFormat
+from xfintech.serde.common.error import (
+    SerialiserFailedError,
+    SerialiserInputTypeError,
+)
+from xfintech.serde.serialiser.pandas import PandasSerialiser
+
+# ==================== Class Structure Tests ====================
+
+
+def test_pandasserialiser_has_serialise_method():
+    """Test PandasSerialiser has serialise method"""
+    assert hasattr(PandasSerialiser, "serialise")
+    assert callable(PandasSerialiser.serialise)
+
+
+def test_pandasserialiser_has_to_parquet_method():
+    """Test PandasSerialiser has to_parquet method"""
+    assert hasattr(PandasSerialiser, "to_parquet")
+    assert callable(PandasSerialiser.to_parquet)
+
+
+def test_pandasserialiser_has_to_csv_method():
+    """Test PandasSerialiser has to_csv method"""
+    assert hasattr(PandasSerialiser, "to_csv")
+    assert callable(PandasSerialiser.to_csv)
+
+
+def test_pandasserialiser_has_to_json_method():
+    """Test PandasSerialiser has to_json method"""
+    assert hasattr(PandasSerialiser, "to_json")
+    assert callable(PandasSerialiser.to_json)
+
+
+def test_pandasserialiser_methods_are_static():
+    """Test all methods are static methods"""
+    assert isinstance(PandasSerialiser.__dict__["serialise"], staticmethod)
+    assert isinstance(PandasSerialiser.__dict__["to_parquet"], staticmethod)
+    assert isinstance(PandasSerialiser.__dict__["to_csv"], staticmethod)
+    assert isinstance(PandasSerialiser.__dict__["to_json"], staticmethod)
+
+
+# ==================== serialise() Method Tests ====================
+
+
+def test_serialise_with_parquet_format_enum():
+    """Test serialise with PARQUET DataFormat enum"""
+    df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
+    result = PandasSerialiser.serialise(df, DataFormat.PARQUET)
+    assert isinstance(result, bytes)
+    assert len(result) > 0
+
+
+def test_serialise_with_parquet_format_string():
+    """Test serialise with 'parquet' string"""
+    df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
+    result = PandasSerialiser.serialise(df, "parquet")
+    assert isinstance(result, bytes)
+    assert len(result) > 0
+
+
+def test_serialise_with_csv_format_enum():
+    """Test serialise with CSV DataFormat enum"""
+    df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
+    result = PandasSerialiser.serialise(df, DataFormat.CSV)
+    assert isinstance(result, bytes)
+    assert b"a,b" in result
+
+
+def test_serialise_with_csv_format_string():
+    """Test serialise with 'csv' string"""
+    df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
+    result = PandasSerialiser.serialise(df, "csv")
+    assert isinstance(result, bytes)
+    assert b"a,b" in result
+
+
+def test_serialise_with_json_format_enum():
+    """Test serialise with JSON DataFormat enum"""
+    df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
+    result = PandasSerialiser.serialise(df, DataFormat.JSON)
+    assert isinstance(result, bytes)
+    assert b"a" in result and b"b" in result
+
+
+def test_serialise_with_json_format_string():
+    """Test serialise with 'json' string"""
+    df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
+    result = PandasSerialiser.serialise(df, "json")
+    assert isinstance(result, bytes)
+    assert b"a" in result and b"b" in result
+
+
+def test_serialise_with_uppercase_format_string():
+    """Test serialise with uppercase format string"""
+    df = pd.DataFrame({"a": [1, 2]})
+    result = PandasSerialiser.serialise(df, "PARQUET")
+    assert isinstance(result, bytes)
+
+
+def test_serialise_with_mixed_case_format_string():
+    """Test serialise with mixed case format string"""
+    df = pd.DataFrame({"a": [1, 2]})
+    result = PandasSerialiser.serialise(df, "CsV")
+    assert isinstance(result, bytes)
+
+
+def test_serialise_raises_on_non_dataframe():
+    """Test serialise raises error for non-DataFrame input"""
+    with pytest.raises(SerialiserInputTypeError) as exc_info:
+        PandasSerialiser.serialise([1, 2, 3], DataFormat.PARQUET)
+    assert "expects pd.DataFrame" in str(exc_info.value)
+
+
+def test_serialise_raises_on_dict_input():
+    """Test serialise raises error for dict input"""
+    with pytest.raises(SerialiserInputTypeError):
+        PandasSerialiser.serialise({"a": [1, 2]}, DataFormat.CSV)
+
+
+def test_serialise_raises_on_string_input():
+    """Test serialise raises error for string input"""
+    with pytest.raises(SerialiserInputTypeError):
+        PandasSerialiser.serialise("not a dataframe", DataFormat.JSON)
+
+
+def test_serialise_raises_on_none_input():
+    """Test serialise raises error for None input"""
+    with pytest.raises(SerialiserInputTypeError):
+        PandasSerialiser.serialise(None, DataFormat.PARQUET)
+
+
+def test_serialise_raises_on_unsupported_format():
+    """Test serialise raises error for unsupported format"""
+    df = pd.DataFrame({"a": [1, 2]})
+    with pytest.raises(ValueError) as exc_info:
+        PandasSerialiser.serialise(df, "xml")
+    assert "Unknown DataFormat" in str(exc_info.value)
+
+
+def test_serialise_empty_dataframe():
+    """Test serialise with empty DataFrame"""
+    df = pd.DataFrame()
+    result = PandasSerialiser.serialise(df, DataFormat.CSV)
+    assert isinstance(result, bytes)
+
+
+def test_serialise_single_column_dataframe():
+    """Test serialise with single column DataFrame"""
+    df = pd.DataFrame({"x": [1, 2, 3]})
+    result = PandasSerialiser.serialise(df, DataFormat.JSON)
+    assert isinstance(result, bytes)
+    assert b"x" in result
+
+
+def test_serialise_large_dataframe():
+    """Test serialise with large DataFrame"""
+    df = pd.DataFrame({"col": range(10000)})
+    result = PandasSerialiser.serialise(df, DataFormat.PARQUET)
+    assert isinstance(result, bytes)
+    assert len(result) > 0
+
+
+# ==================== to_parquet() Method Tests ====================
+
+
+def test_to_parquet_basic():
+    """Test to_parquet with basic DataFrame"""
+    df = pd.DataFrame({"a": [1, 2, 3], "b": ["x", "y", "z"]})
+    result = PandasSerialiser.to_parquet(df)
+    assert isinstance(result, bytes)
+    assert len(result) > 0
+
+
+def test_to_parquet_returns_valid_parquet_bytes():
+    """Test to_parquet returns valid parquet that can be read back"""
+    df = pd.DataFrame({"col1": [1, 2, 3], "col2": [4.5, 5.6, 6.7]})
+    result = PandasSerialiser.to_parquet(df)
+
+    # Verify it can be read back
+    buffer = BytesIO(result)
+    df_read = pd.read_parquet(buffer)
+    assert df.equals(df_read)
+
+
+def test_to_parquet_with_compression():
+    """Test to_parquet with compression argument"""
+    df = pd.DataFrame({"a": range(100)})
+    result = PandasSerialiser.to_parquet(df, compression="gzip")
+    assert isinstance(result, bytes)
+
+
+def test_to_parquet_with_index():
+    """Test to_parquet with index parameter"""
+    df = pd.DataFrame({"a": [1, 2, 3]})
+    result = PandasSerialiser.to_parquet(df, index=True)
+    assert isinstance(result, bytes)
+
+
+def test_to_parquet_without_index():
+    """Test to_parquet without index"""
+    df = pd.DataFrame({"a": [1, 2, 3]})
+    result = PandasSerialiser.to_parquet(df, index=False)
+    assert isinstance(result, bytes)
+
+
+def test_to_parquet_raises_on_non_dataframe():
+    """Test to_parquet raises error for non-DataFrame"""
+    with pytest.raises(SerialiserFailedError):
+        PandasSerialiser.to_parquet({"a": [1, 2]})
+
+
+def test_to_parquet_with_datetime_column():
+    """Test to_parquet with datetime column"""
+    df = pd.DataFrame({"date": pd.date_range("2024-01-01", periods=3)})
+    result = PandasSerialiser.to_parquet(df)
+    assert isinstance(result, bytes)
+
+
+def test_to_parquet_with_mixed_types():
+    """Test to_parquet with mixed data types"""
+    df = pd.DataFrame({"int": [1, 2, 3], "float": [1.1, 2.2, 3.3], "str": ["a", "b", "c"], "bool": [True, False, True]})
+    result = PandasSerialiser.to_parquet(df)
+    assert isinstance(result, bytes)
+
+
+# ==================== to_csv() Method Tests ====================
+
+
+def test_to_csv_basic():
+    """Test to_csv with basic DataFrame"""
+    df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
+    result = PandasSerialiser.to_csv(df)
+    assert isinstance(result, bytes)
+    assert b"a,b" in result
+    assert b"1,4" in result
+
+
+def test_to_csv_returns_valid_csv_bytes():
+    """Test to_csv returns valid CSV that can be read back"""
+    df = pd.DataFrame({"col1": [1, 2, 3], "col2": [4, 5, 6]})
+    result = PandasSerialiser.to_csv(df, index=False)
+
+    # Verify it's proper CSV format
+    text = result.decode("utf-8")
+    assert "col1,col2" in text
+    assert "1,4" in text
+
+
+def test_to_csv_with_index():
+    """Test to_csv with index included"""
+    df = pd.DataFrame({"a": [1, 2, 3]})
+    result = PandasSerialiser.to_csv(df, index=True)
+    assert isinstance(result, bytes)
+
+
+def test_to_csv_without_index():
+    """Test to_csv without index"""
+    df = pd.DataFrame({"a": [1, 2, 3]})
+    result = PandasSerialiser.to_csv(df, index=False)
+    text = result.decode("utf-8")
+    assert text.startswith("a\n") or text.startswith("a\r\n")
+
+
+def test_to_csv_with_custom_encoding():
+    """Test to_csv with custom encoding"""
+    df = pd.DataFrame({"a": ["中文", "日本語"]})
+    result = PandasSerialiser.to_csv(df, encoding="utf-8")
+    assert isinstance(result, bytes)
+
+
+def test_to_csv_with_custom_separator():
+    """Test to_csv with custom separator"""
+    df = pd.DataFrame({"a": [1, 2], "b": [3, 4]})
+    result = PandasSerialiser.to_csv(df, sep="|")
+    assert b"|" in result
+
+
+def test_to_csv_raises_on_non_dataframe():
+    """Test to_csv raises error for non-DataFrame"""
+    with pytest.raises(SerialiserFailedError):
+        PandasSerialiser.to_csv([1, 2, 3])
+
+
+def test_to_csv_empty_dataframe():
+    """Test to_csv with empty DataFrame"""
+    df = pd.DataFrame()
+    result = PandasSerialiser.to_csv(df)
+    assert isinstance(result, bytes)
+
+
+def test_to_csv_with_na_values():
+    """Test to_csv with NA values"""
+    df = pd.DataFrame({"a": [1, None, 3], "b": [4, 5, None]})
+    result = PandasSerialiser.to_csv(df)
+    assert isinstance(result, bytes)
+
+
+def test_to_csv_with_header_false():
+    """Test to_csv without header"""
+    df = pd.DataFrame({"a": [1, 2], "b": [3, 4]})
+    result = PandasSerialiser.to_csv(df, header=False, index=False)
+    text = result.decode("utf-8")
+    assert "a,b" not in text
+    assert "1,3" in text
+
+
+# ==================== to_json() Method Tests ====================
+
+
+def test_to_json_basic():
+    """Test to_json with basic DataFrame"""
+    df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
+    result = PandasSerialiser.to_json(df)
+    assert isinstance(result, bytes)
+    assert b"a" in result and b"b" in result
+
+
+def test_to_json_returns_valid_json_bytes():
+    """Test to_json returns valid JSON"""
+    df = pd.DataFrame({"col1": [1, 2], "col2": [3, 4]})
+    result = PandasSerialiser.to_json(df, orient="records")
+
+    # Verify it's valid JSON
+    import json
+
+    text = result.decode("utf-8")
+    parsed = json.loads(text)
+    assert isinstance(parsed, list)
+    assert len(parsed) == 2
+
+
+def test_to_json_with_orient_records():
+    """Test to_json with orient='records'"""
+    df = pd.DataFrame({"a": [1, 2], "b": [3, 4]})
+    result = PandasSerialiser.to_json(df, orient="records")
+    assert isinstance(result, bytes)
+
+
+def test_to_json_with_orient_split():
+    """Test to_json with orient='split'"""
+    df = pd.DataFrame({"a": [1, 2], "b": [3, 4]})
+    result = PandasSerialiser.to_json(df, orient="split")
+    assert isinstance(result, bytes)
+    assert b"columns" in result and b"data" in result
+
+
+def test_to_json_with_orient_index():
+    """Test to_json with orient='index'"""
+    df = pd.DataFrame({"a": [1, 2], "b": [3, 4]})
+    result = PandasSerialiser.to_json(df, orient="index")
+    assert isinstance(result, bytes)
+
+
+def test_to_json_with_orient_columns():
+    """Test to_json with orient='columns'"""
+    df = pd.DataFrame({"a": [1, 2], "b": [3, 4]})
+    result = PandasSerialiser.to_json(df, orient="columns")
+    assert isinstance(result, bytes)
+
+
+def test_to_json_with_orient_values():
+    """Test to_json with orient='values'"""
+    df = pd.DataFrame({"a": [1, 2], "b": [3, 4]})
+    result = PandasSerialiser.to_json(df, orient="values")
+    assert isinstance(result, bytes)
+
+
+def test_to_json_with_custom_encoding():
+    """Test to_json with custom encoding"""
+    df = pd.DataFrame({"a": [1, 2]})
+    result = PandasSerialiser.to_json(df, encoding="utf-8")
+    assert isinstance(result, bytes)
+
+
+def test_to_json_raises_on_non_dataframe():
+    """Test to_json raises error for non-DataFrame"""
+    with pytest.raises(SerialiserFailedError):
+        PandasSerialiser.to_json({"a": [1, 2]})
+
+
+def test_to_json_empty_dataframe():
+    """Test to_json with empty DataFrame"""
+    df = pd.DataFrame()
+    result = PandasSerialiser.to_json(df)
+    assert isinstance(result, bytes)
+
+
+def test_to_json_with_datetime():
+    """Test to_json with datetime values"""
+    df = pd.DataFrame({"date": pd.date_range("2024-01-01", periods=2)})
+    result = PandasSerialiser.to_json(df)
+    assert isinstance(result, bytes)
+
+
+def test_to_json_with_lines_format():
+    """Test to_json with lines=True (newline-delimited JSON)"""
+    df = pd.DataFrame({"a": [1, 2], "b": [3, 4]})
+    result = PandasSerialiser.to_json(df, orient="records", lines=True)
+    assert isinstance(result, bytes)
+    text = result.decode("utf-8")
+    lines = text.strip().split("\n")
+    assert len(lines) == 2
+
+
+# ==================== Integration and Edge Case Tests ====================
+
+
+def test_serialise_all_formats_produce_bytes():
+    """Test all formats produce bytes output"""
+    df = pd.DataFrame({"a": [1, 2, 3]})
+
+    for format in [DataFormat.PARQUET, DataFormat.CSV, DataFormat.JSON]:
+        result = PandasSerialiser.serialise(df, format)
+        assert isinstance(result, bytes)
+
+
+def test_serialise_same_dataframe_multiple_times():
+    """Test serialising same DataFrame multiple times produces consistent results"""
+    df = pd.DataFrame({"a": [1, 2, 3]})
+    result1 = PandasSerialiser.serialise(df, DataFormat.CSV, index=False)
+    result2 = PandasSerialiser.serialise(df, DataFormat.CSV, index=False)
+    assert result1 == result2
+
+
+def test_dataframe_with_special_characters():
+    """Test DataFrame with special characters in values"""
+    df = pd.DataFrame({"col": ["test,with,comma", "test\nwith\nnewline", 'test"quote"']})
+    result = PandasSerialiser.to_csv(df)
+    assert isinstance(result, bytes)
+
+
+def test_dataframe_with_unicode():
+    """Test DataFrame with Unicode characters"""
+    df = pd.DataFrame({"text": ["Hello 世界", "Привет", "مرحبا"]})
+    for format in [DataFormat.CSV, DataFormat.JSON]:
+        result = PandasSerialiser.serialise(df, format)
+        assert isinstance(result, bytes)
+
+
+def test_dataframe_with_large_strings():
+    """Test DataFrame with large string values"""
+    df = pd.DataFrame({"text": ["x" * 10000]})
+    result = PandasSerialiser.serialise(df, DataFormat.PARQUET)
+    assert isinstance(result, bytes)
+
+
+def test_serialise_with_kwargs_passthrough():
+    """Test that kwargs are passed through to underlying methods"""
+    df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
+    result = PandasSerialiser.serialise(df, DataFormat.CSV, index=False, sep="|")
+    assert b"|" in result
+
+
+def test_multiindex_dataframe():
+    """Test serialising DataFrame with MultiIndex"""
+    arrays = [["A", "A", "B", "B"], [1, 2, 1, 2]]
+    index = pd.MultiIndex.from_arrays(arrays, names=["letter", "number"])
+    df = pd.DataFrame({"value": [10, 20, 30, 40]}, index=index)
+
+    result = PandasSerialiser.to_parquet(df)
+    assert isinstance(result, bytes)
+
+
+def test_dataframe_with_categorical_data():
+    """Test DataFrame with categorical data"""
+    df = pd.DataFrame({"cat": pd.Categorical(["a", "b", "c", "a"])})
+    result = PandasSerialiser.to_parquet(df)
+    assert isinstance(result, bytes)