etlplus 0.11.12__tar.gz → 0.12.2__tar.gz

{etlplus-0.11.12/etlplus.egg-info → etlplus-0.12.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: etlplus
-Version: 0.11.12
+Version: 0.12.2
 Summary: A Swiss Army knife for simple ETL operations
 Home-page: https://github.com/Dagitali/ETLPlus
 Author: ETLPlus Team
@@ -17,8 +17,11 @@ Classifier: Programming Language :: Python :: 3.14
 Requires-Python: >=3.13,<3.15
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: fastavro>=1.12.1
 Requires-Dist: jinja2>=3.1.6
+Requires-Dist: openpyxl>=3.1.5
 Requires-Dist: pyodbc>=5.3.0
+Requires-Dist: pyarrow>=22.0.0
 Requires-Dist: python-dotenv>=1.2.1
 Requires-Dist: pandas>=2.3.3
 Requires-Dist: pydantic>=2.12.5
@@ -26,6 +29,8 @@ Requires-Dist: PyYAML>=6.0.3
 Requires-Dist: requests>=2.32.5
 Requires-Dist: SQLAlchemy>=2.0.45
 Requires-Dist: typer>=0.21.0
+Requires-Dist: xlrd>=2.0.2
+Requires-Dist: xlwt>=1.3.0
 Provides-Extra: dev
 Requires-Dist: black>=25.9.0; extra == "dev"
 Requires-Dist: build>=1.2.2; extra == "dev"

etlplus-0.12.2/etlplus/file/avro.py

@@ -0,0 +1,198 @@
+"""
+:mod:`etlplus.file.avro` module.
+
+Helpers for reading/writing Avro files.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+from typing import cast
+
+from ..types import JSONData
+from ..types import JSONDict
+from ..types import JSONList
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'read',
+    'write',
+]
+
+
+# SECTION: INTERNAL CONSTANTS =============================================== #
+
+
+_FASTAVRO_CACHE: dict[str, Any] = {}
+
+
+_PRIMITIVE_TYPES: tuple[type, ...] = (
+    bool,
+    int,
+    float,
+    str,
+    bytes,
+    bytearray,
+)
+
+
+# SECTION: INTERNAL FUNCTIONS =============================================== #
+
+
+def _get_fastavro() -> Any:
+    """
+    Return the fastavro module, importing it on first use.
+
+    Raises an informative ImportError if the optional dependency is missing.
+    """
+    mod = _FASTAVRO_CACHE.get('mod')
+    if mod is not None:  # pragma: no cover - tiny branch
+        return mod
+    try:
+        _fastavro = __import__('fastavro')  # type: ignore[assignment]
+    except ImportError as e:  # pragma: no cover
+        raise ImportError(
+            'AVRO support requires optional dependency "fastavro".\n'
+            'Install with: pip install fastavro',
+        ) from e
+    _FASTAVRO_CACHE['mod'] = _fastavro
+
+    return _fastavro
+
+
+def _normalize_records(data: JSONData) -> JSONList:
+    """
+    Normalize JSON payloads into a list of dictionaries.
+
+    Raises TypeError when payloads contain non-dict items.
+    """
+    if isinstance(data, list):
+        if not all(isinstance(item, dict) for item in data):
+            raise TypeError('AVRO payloads must contain only objects (dicts)')
+        return cast(JSONList, data)
+    return [cast(JSONDict, data)]
+
+
+def _infer_value_type(value: object) -> str | list[str]:
+    """
+    Infer the Avro type for a primitive value.
+
+    Raises TypeError for unsupported types.
+    """
+    if value is None:
+        return 'null'
+    if isinstance(value, bool):
+        return 'boolean'
+    if isinstance(value, int):
+        return 'long'
+    if isinstance(value, float):
+        return 'double'
+    if isinstance(value, str):
+        return 'string'
+    if isinstance(value, (bytes, bytearray)):
+        return 'bytes'
+    raise TypeError('AVRO payloads must contain only primitive values')
+
+
+def _merge_types(types: list[str]) -> str | list[str]:
+    """Return a stable Avro type union for a list of types."""
+    unique = list(dict.fromkeys(types))
+    if len(unique) == 1:
+        return unique[0]
+    ordered = ['null'] + sorted(t for t in unique if t != 'null')
+    return ordered
+
+
+def _infer_schema(records: JSONList) -> dict[str, Any]:
+    """
+    Infer a basic Avro schema from record payloads.
+
+    Only primitive field values are supported; complex values raise TypeError.
+    """
+    field_names = sorted({key for record in records for key in record})
+    fields: list[dict[str, Any]] = []
+    for name in field_names:
+        types: list[str] = []
+        for record in records:
+            value = record.get(name)
+            if value is None:
+                types.append('null')
+                continue
+            if isinstance(value, dict | list):
+                raise TypeError(
+                    'AVRO payloads must contain only primitive values',
+                )
+            if not isinstance(value, _PRIMITIVE_TYPES):
+                raise TypeError(
+                    'AVRO payloads must contain only primitive values',
+                )
+            types.append(cast(str, _infer_value_type(value)))
+        fields.append({'name': name, 'type': _merge_types(types)})
+
+    return {
+        'name': 'etlplus_record',
+        'type': 'record',
+        'fields': fields,
+    }
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def read(
+    path: Path,
+) -> JSONList:
+    """
+    Read AVRO content from ``path``.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the AVRO file on disk.
+
+    Returns
+    -------
+    JSONList
+        The list of dictionaries read from the AVRO file.
+    """
+    fastavro = _get_fastavro()
+    with path.open('rb') as handle:
+        reader = fastavro.reader(handle)
+        return [cast(JSONDict, record) for record in reader]
+
+
+def write(
+    path: Path,
+    data: JSONData,
+) -> int:
+    """
+    Write ``data`` to AVRO at ``path`` and return record count.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the AVRO file on disk.
+    data : JSONData
+        Data to write.
+
+    Returns
+    -------
+    int
+        Number of records written.
+    """
+    records = _normalize_records(data)
+    if not records:
+        return 0
+
+    fastavro = _get_fastavro()
+    schema = _infer_schema(records)
+    parsed_schema = fastavro.parse_schema(schema)
+
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open('wb') as handle:
+        fastavro.writer(handle, parsed_schema, records)
+
+    return len(records)

{etlplus-0.11.12 → etlplus-0.12.2}/etlplus/file/core.py

@@ -25,7 +25,7 @@ from . import xls
 from . import xlsx
 from . import xml
 from . import yaml
-from . import zip
+from . import zip as zip_
 from .enums import FileFormat
 from .enums import infer_file_format_and_compression
 
@@ -222,7 +222,7 @@ class File:
             case FileFormat.YAML:
                 return yaml.read(self.path)
             case FileFormat.ZIP:
-                return zip.read(self.path)
+                return zip_.read(self.path)
         raise ValueError(f'Unsupported format: {fmt}')
 
     def write(
@@ -283,5 +283,5 @@ class File:
             case FileFormat.YAML:
                 return yaml.write(self.path, data)
             case FileFormat.ZIP:
-                return zip.write(self.path, data)
+                return zip_.write(self.path, data)
         raise ValueError(f'Unsupported format: {fmt}')

{etlplus-0.11.12 → etlplus-0.12.2}/etlplus/file/csv.py

@@ -1,7 +1,7 @@
 """
 :mod:`etlplus.file.csv` module.
 
-CSV read/write helpers.
+Helpers for reading/writing CSV files.
 """
 
 from __future__ import annotations
@@ -14,6 +14,15 @@ from ..types import JSONData
 from ..types import JSONDict
 from ..types import JSONList
 
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'read',
+    'write',
+]
+
+
 # SECTION: FUNCTIONS ======================================================== #
 
 
@@ -21,7 +30,7 @@ def read(
     path: Path,
 ) -> JSONList:
     """
-    Load CSV content as a list of dictionaries.
+    Read CSV content from ``path``.
 
     Parameters
     ----------
@@ -48,7 +57,7 @@ def write(
     data: JSONData,
 ) -> int:
     """
-    Write CSV rows to ``path`` and return the number of rows.
+    Write ``data`` to CSV at ``path`` and return record count.
 
     Parameters
     ----------

etlplus-0.12.2/etlplus/file/feather.py

@@ -0,0 +1,144 @@
+"""
+:mod:`etlplus.file.feather` module.
+
+Helpers for reading/writing Feather files.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+from typing import cast
+
+from ..types import JSONData
+from ..types import JSONDict
+from ..types import JSONList
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'read',
+    'write',
+]
+
+
+# SECTION: INTERNAL CONSTANTS =============================================== #
+
+
+_PANDAS_CACHE: dict[str, Any] = {}
+
+
+# SECTION: INTERNAL FUNCTIONS =============================================== #
+
+
+def _get_pandas() -> Any:
+    """
+    Return the pandas module, importing it on first use.
+
+    Raises an informative ImportError if the optional dependency is missing.
+    """
+    mod = _PANDAS_CACHE.get('mod')
+    if mod is not None:  # pragma: no cover - tiny branch
+        return mod
+    try:
+        _pd = __import__('pandas')  # type: ignore[assignment]
+    except ImportError as e:  # pragma: no cover
+        raise ImportError(
+            'Feather support requires optional dependency "pandas".\n'
+            'Install with: pip install pandas',
+        ) from e
+    _PANDAS_CACHE['mod'] = _pd
+
+    return _pd
+
+
+def _normalize_records(data: JSONData) -> JSONList:
+    """
+    Normalize JSON payloads into a list of dictionaries.
+
+    Raises TypeError when payloads contain non-dict items.
+    """
+    if isinstance(data, list):
+        if not all(isinstance(item, dict) for item in data):
+            raise TypeError(
+                'Feather payloads must contain only objects (dicts)',
+            )
+        return cast(JSONList, data)
+    return [cast(JSONDict, data)]
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def read(
+    path: Path,
+) -> JSONList:
+    """
+    Read Feather content from ``path``.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the Feather file on disk.
+
+    Returns
+    -------
+    JSONList
+        The list of dictionaries read from the Feather file.
+
+    Raises
+    ------
+    ImportError
+        When optional dependency "pyarrow" is missing.
+    """
+    pandas = _get_pandas()
+    try:
+        frame = pandas.read_feather(path)
+    except ImportError as e:  # pragma: no cover
+        raise ImportError(
+            'Feather support requires optional dependency "pyarrow".\n'
+            'Install with: pip install pyarrow',
+        ) from e
+    return cast(JSONList, frame.to_dict(orient='records'))
+
+
+def write(
+    path: Path,
+    data: JSONData,
+) -> int:
+    """
+    Write ``data`` to Feather at ``path`` and return record count.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the Feather file on disk.
+    data : JSONData
+        Data to write.
+
+    Returns
+    -------
+    int
+        Number of records written.
+
+    Raises
+    ------
+    ImportError
+        When optional dependency "pyarrow" is missing.
+    """
+    records = _normalize_records(data)
+    if not records:
+        return 0
+
+    pandas = _get_pandas()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    frame = pandas.DataFrame.from_records(records)
+    try:
+        frame.to_feather(path)
+    except ImportError as e:  # pragma: no cover
+        raise ImportError(
+            'Feather support requires optional dependency "pyarrow".\n'
+            'Install with: pip install pyarrow',
+        ) from e
+    return len(records)

etlplus-0.12.2/etlplus/file/gz.py

@@ -0,0 +1,123 @@
+"""
+:mod:`etlplus.file.gz` module.
+
+Helpers for reading/writing GZ files.
+"""
+
+from __future__ import annotations
+
+import gzip
+import tempfile
+from pathlib import Path
+
+from ..types import JSONData
+from .enums import CompressionFormat
+from .enums import FileFormat
+from .enums import infer_file_format_and_compression
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'read',
+    'write',
+]
+
+
+# SECTION: INTERNAL FUNCTIONS =============================================== #
+
+
+def _resolve_format(
+    path: Path,
+) -> FileFormat:
+    """
+    Resolve the inner file format from a .gz filename.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the GZ file on disk.
+
+    Returns
+    -------
+    FileFormat
+        The inferred inner file format.
+
+    Raises
+    ------
+    ValueError
+        If the file format cannot be inferred from the filename.
+    """
+    fmt, compression = infer_file_format_and_compression(path)
+    if compression is not CompressionFormat.GZ:
+        raise ValueError(f'Not a gzip file: {path}')
+    if fmt is None:
+        raise ValueError(
+            f'Cannot infer file format from compressed file {path!r}',
+        )
+    return fmt
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def read(
+    path: Path,
+) -> JSONData:
+    """
+    Read GZ content from ``path`` and parse the inner payload.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the GZ file on disk.
+
+    Returns
+    -------
+    JSONData
+        Parsed payload.
+    """
+    fmt = _resolve_format(path)
+    with gzip.open(path, 'rb') as handle:
+        payload = handle.read()
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        tmp_path = Path(tmpdir) / f'payload.{fmt.value}'
+        tmp_path.write_bytes(payload)
+        from .core import File
+
+        return File(tmp_path, fmt).read()
+
+
+def write(
+    path: Path,
+    data: JSONData,
+) -> int:
+    """
+    Write ``data`` to GZ at ``path`` and return record count.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the GZ file on disk.
+    data : JSONData
+        Data to write.
+
+    Returns
+    -------
+    int
+        Number of records written.
+    """
+    fmt = _resolve_format(path)
+    with tempfile.TemporaryDirectory() as tmpdir:
+        tmp_path = Path(tmpdir) / f'payload.{fmt.value}'
+        from .core import File
+
+        count = File(tmp_path, fmt).write(data)
+        payload = tmp_path.read_bytes()
+
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with gzip.open(path, 'wb') as handle:
+        handle.write(payload)
+
+    return count

{etlplus-0.11.12 → etlplus-0.12.2}/etlplus/file/json.py

@@ -1,7 +1,7 @@
 """
 :mod:`etlplus.file.json` module.
 
-JSON read/write helpers.
+Helpers for reading/writing JSON files.
 """
 
 from __future__ import annotations
@@ -15,6 +15,15 @@ from ..types import JSONDict
 from ..types import JSONList
 from ..utils import count_records
 
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'read',
+    'write',
+]
+
+
 # SECTION: FUNCTIONS ======================================================== #
 
 
@@ -22,7 +31,9 @@ def read(
     path: Path,
 ) -> JSONData:
     """
-    Load and validate JSON payloads from ``path``.
+    Read JSON content from ``path``.
+
+    Validates that the JSON root is a dict or a list of dicts.
 
     Parameters
     ----------

etlplus-0.12.2/etlplus/file/ndjson.py

@@ -0,0 +1,109 @@
+"""
+:mod:`etlplus.file.ndjson` module.
+
+Helpers for reading/writing NDJSON files.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import cast
+
+from ..types import JSONData
+from ..types import JSONDict
+from ..types import JSONList
+from ..utils import count_records
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'read',
+    'write',
+]
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def read(
+    path: Path,
+) -> JSONList:
+    """
+    Read NDJSON content from ``path``.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the NDJSON file on disk.
+
+    Returns
+    -------
+    JSONList
+        The list of dictionaries read from the NDJSON file.
+
+    Raises
+    ------
+    TypeError
+        If any line in the NDJSON file is not a JSON object (dict).
+    """
+    rows: JSONList = []
+    with path.open('r', encoding='utf-8') as handle:
+        for idx, line in enumerate(handle, start=1):
+            text = line.strip()
+            if not text:
+                continue
+            payload = json.loads(text)
+            if not isinstance(payload, dict):
+                raise TypeError(
+                    f'NDJSON lines must be objects (dicts) (line {idx})',
+                )
+            rows.append(cast(JSONDict, payload))
+    return rows
+
+
+def write(
+    path: Path,
+    data: JSONData,
+) -> int:
+    """
+    Write ``data`` to NDJSON at ``path``.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the NDJSON file on disk.
+    data : JSONData
+        Data to write.
+
+    Returns
+    -------
+    int
+        Number of records written.
+
+    Raises
+    ------
+    TypeError
+        If ``data`` is a list containing non-dict items.
+    """
+    rows: JSONList
+    if isinstance(data, list):
+        if not all(isinstance(item, dict) for item in data):
+            raise TypeError(
+                'NDJSON payloads must contain only objects (dicts)',
+            )
+        rows = cast(JSONList, data)
+    else:
+        rows = [cast(JSONDict, data)]
+
+    if not rows:
+        return 0
+
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open('w', encoding='utf-8') as handle:
+        for row in rows:
+            handle.write(json.dumps(row, ensure_ascii=False))
+            handle.write('\n')
+
+    return count_records(rows)