etlplus 0.11.11__py3-none-any.whl → 0.12.3__py3-none-any.whl

etlplus/file/core.py

@@ -25,7 +25,7 @@ from . import xls
 from . import xlsx
 from . import xml
 from . import yaml
-from . import zip
+from . import zip as zip_
 from .enums import FileFormat
 from .enums import infer_file_format_and_compression
 
@@ -222,7 +222,7 @@ class File:
             case FileFormat.YAML:
                 return yaml.read(self.path)
             case FileFormat.ZIP:
-                return zip.read(self.path)
+                return zip_.read(self.path)
         raise ValueError(f'Unsupported format: {fmt}')
 
     def write(
@@ -283,5 +283,5 @@ class File:
             case FileFormat.YAML:
                 return yaml.write(self.path, data)
             case FileFormat.ZIP:
-                return zip.write(self.path, data)
+                return zip_.write(self.path, data)
         raise ValueError(f'Unsupported format: {fmt}')

etlplus/file/csv.py

@@ -1,18 +1,26 @@
 """
 :mod:`etlplus.file.csv` module.
 
-CSV read/write helpers.
+Helpers for reading/writing CSV files.
 """
 
 from __future__ import annotations
 
-import csv
 from pathlib import Path
-from typing import cast
 
 from ..types import JSONData
-from ..types import JSONDict
 from ..types import JSONList
+from ._io import read_delimited
+from ._io import write_delimited
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'read',
+    'write',
+]
+
 
 # SECTION: FUNCTIONS ======================================================== #
 
@@ -21,7 +29,7 @@ def read(
     path: Path,
 ) -> JSONList:
     """
-    Load CSV content as a list of dictionaries.
+    Read CSV content from ``path``.
 
     Parameters
     ----------
@@ -33,14 +41,7 @@ def read(
     JSONList
         The list of dictionaries read from the CSV file.
     """
-    with path.open('r', encoding='utf-8', newline='') as handle:
-        reader: csv.DictReader[str] = csv.DictReader(handle)
-        rows: JSONList = []
-        for row in reader:
-            if not any(row.values()):
-                continue
-            rows.append(cast(JSONDict, dict(row)))
-    return rows
+    return read_delimited(path, delimiter=',')
 
 
 def write(
@@ -48,7 +49,7 @@ def write(
     data: JSONData,
 ) -> int:
     """
-    Write CSV rows to ``path`` and return the number of rows.
+    Write ``data`` to CSV at ``path`` and return record count.
 
     Parameters
     ----------
@@ -63,20 +64,4 @@ def write(
     int
         The number of rows written to the CSV file.
     """
-    rows: list[JSONDict]
-    if isinstance(data, list):
-        rows = [row for row in data if isinstance(row, dict)]
-    else:
-        rows = [data]
-
-    if not rows:
-        return 0
-
-    fieldnames = sorted({key for row in rows for key in row})
-    with path.open('w', encoding='utf-8', newline='') as handle:
-        writer = csv.DictWriter(handle, fieldnames=fieldnames)
-        writer.writeheader()
-        for row in rows:
-            writer.writerow({field: row.get(field) for field in fieldnames})
-
-    return len(rows)
+    return write_delimited(path, data, delimiter=',')

etlplus/file/feather.py

@@ -1,48 +1,74 @@
 """
 :mod:`etlplus.file.feather` module.
 
-Stub helpers for FEATHER read/write.
+Helpers for reading/writing Feather files.
 """
 
 from __future__ import annotations
 
 from pathlib import Path
+from typing import cast
 
 from ..types import JSONData
+from ..types import JSONList
+from ._io import normalize_records
+from ._pandas import get_pandas
 
 # SECTION: EXPORTS ========================================================== #
 
 
-def read(path: Path) -> JSONData:
+__all__ = [
+    'read',
+    'write',
+]
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def read(
+    path: Path,
+) -> JSONList:
     """
-    Read FEATHER content from ``path``.
+    Read Feather content from ``path``.
 
     Parameters
     ----------
     path : Path
-        Path to the FEATHER file on disk.
+        Path to the Feather file on disk.
 
     Returns
     -------
-    JSONData
-        Parsed payload.
+    JSONList
+        The list of dictionaries read from the Feather file.
 
     Raises
     ------
-    NotImplementedError
-        FEATHER :func:`read` is not implemented yet.
+    ImportError
+        When optional dependency "pyarrow" is missing.
     """
-    raise NotImplementedError('FEATHER read is not implemented yet')
+    pandas = get_pandas('Feather')
+    try:
+        frame = pandas.read_feather(path)
+    except ImportError as e:  # pragma: no cover
+        raise ImportError(
+            'Feather support requires optional dependency "pyarrow".\n'
+            'Install with: pip install pyarrow',
+        ) from e
+    return cast(JSONList, frame.to_dict(orient='records'))
 
 
-def write(path: Path, data: JSONData) -> int:
+def write(
+    path: Path,
+    data: JSONData,
+) -> int:
     """
-    Write ``data`` to FEATHER at ``path``.
+    Write ``data`` to Feather at ``path`` and return record count.
 
     Parameters
     ----------
     path : Path
-        Path to the FEATHER file on disk.
+        Path to the Feather file on disk.
     data : JSONData
         Data to write.
 
@@ -53,7 +79,21 @@ def write(path: Path, data: JSONData) -> int:
 
     Raises
     ------
-    NotImplementedError
-        FEATHER :func:`write` is not implemented yet.
+    ImportError
+        When optional dependency "pyarrow" is missing.
     """
-    raise NotImplementedError('FEATHER write is not implemented yet')
+    records = normalize_records(data, 'Feather')
+    if not records:
+        return 0
+
+    pandas = get_pandas('Feather')
+    path.parent.mkdir(parents=True, exist_ok=True)
+    frame = pandas.DataFrame.from_records(records)
+    try:
+        frame.to_feather(path)
+    except ImportError as e:  # pragma: no cover
+        raise ImportError(
+            'Feather support requires optional dependency "pyarrow".\n'
+            'Install with: pip install pyarrow',
+        ) from e
+    return len(records)

etlplus/file/gz.py

@@ -1,21 +1,37 @@
 """
 :mod:`etlplus.file.gz` module.
 
-Stub helpers for GZ read/write.
+Helpers for reading/writing GZ files.
 """
 
 from __future__ import annotations
 
+import gzip
+import tempfile
 from pathlib import Path
 
 from ..types import JSONData
+from .enums import CompressionFormat
+from .enums import FileFormat
+from .enums import infer_file_format_and_compression
 
 # SECTION: EXPORTS ========================================================== #
 
 
-def read(path: Path) -> JSONData:
+__all__ = [
+    'read',
+    'write',
+]
+
+
+# SECTION: INTERNAL FUNCTIONS =============================================== #
+
+
+def _resolve_format(
+    path: Path,
+) -> FileFormat:
     """
-    Read GZ content from ``path``.
+    Resolve the inner file format from a .gz filename.
 
     Parameters
     ----------
@@ -24,20 +40,61 @@ def read(path: Path) -> JSONData:
 
     Returns
     -------
-    JSONData
-        Parsed payload.
+    FileFormat
+        The inferred inner file format.
 
     Raises
     ------
-    NotImplementedError
-        GZ :func:`read` is not implemented yet.
+    ValueError
+        If the file format cannot be inferred from the filename.
+    """
+    fmt, compression = infer_file_format_and_compression(path)
+    if compression is not CompressionFormat.GZ:
+        raise ValueError(f'Not a gzip file: {path}')
+    if fmt is None:
+        raise ValueError(
+            f'Cannot infer file format from compressed file {path!r}',
+        )
+    return fmt
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def read(
+    path: Path,
+) -> JSONData:
+    """
+    Read GZ content from ``path`` and parse the inner payload.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the GZ file on disk.
+
+    Returns
+    -------
+    JSONData
+        Parsed payload.
     """
-    raise NotImplementedError('GZ read is not implemented yet')
+    fmt = _resolve_format(path)
+    with gzip.open(path, 'rb') as handle:
+        payload = handle.read()
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        tmp_path = Path(tmpdir) / f'payload.{fmt.value}'
+        tmp_path.write_bytes(payload)
+        from .core import File
 
+        return File(tmp_path, fmt).read()
 
-def write(path: Path, data: JSONData) -> int:
+
+def write(
+    path: Path,
+    data: JSONData,
+) -> int:
     """
-    Write ``data`` to GZ at ``path``.
+    Write ``data`` to GZ at ``path`` and return record count.
 
     Parameters
     ----------
@@ -50,10 +107,17 @@ def write(path: Path, data: JSONData) -> int:
     -------
     int
         Number of records written.
-
-    Raises
-    ------
-    NotImplementedError
-        GZ :func:`write` is not implemented yet.
     """
-    raise NotImplementedError('GZ write is not implemented yet')
+    fmt = _resolve_format(path)
+    with tempfile.TemporaryDirectory() as tmpdir:
+        tmp_path = Path(tmpdir) / f'payload.{fmt.value}'
+        from .core import File
+
+        count = File(tmp_path, fmt).write(data)
+        payload = tmp_path.read_bytes()
+
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with gzip.open(path, 'wb') as handle:
+        handle.write(payload)
+
+    return count

etlplus/file/json.py

@@ -1,7 +1,7 @@
 """
 :mod:`etlplus.file.json` module.
 
-JSON read/write helpers.
+Helpers for reading/writing JSON files.
 """
 
 from __future__ import annotations
@@ -15,6 +15,15 @@ from ..types import JSONDict
 from ..types import JSONList
 from ..utils import count_records
 
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'read',
+    'write',
+]
+
+
 # SECTION: FUNCTIONS ======================================================== #
 
 
@@ -22,7 +31,9 @@ def read(
     path: Path,
 ) -> JSONData:
     """
-    Load and validate JSON payloads from ``path``.
+    Read JSON content from ``path``.
+
+    Validates that the JSON root is a dict or a list of dicts.
 
     Parameters
     ----------

etlplus/file/ndjson.py

@@ -1,19 +1,35 @@
 """
 :mod:`etlplus.file.ndjson` module.
 
-Stub helpers for NDJSON read/write.
+Helpers for reading/writing NDJSON files.
 """
 
 from __future__ import annotations
 
+import json
 from pathlib import Path
+from typing import cast
 
 from ..types import JSONData
+from ..types import JSONDict
+from ..types import JSONList
+from ..utils import count_records
 
 # SECTION: EXPORTS ========================================================== #
 
 
-def read(path: Path) -> JSONData:
+__all__ = [
+    'read',
+    'write',
+]
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def read(
+    path: Path,
+) -> JSONList:
     """
     Read NDJSON content from ``path``.
 
@@ -24,18 +40,33 @@ def read(path: Path) -> JSONData:
 
     Returns
     -------
-    JSONData
-        Parsed payload.
+    JSONList
+        The list of dictionaries read from the NDJSON file.
 
     Raises
     ------
-    NotImplementedError
-        NDJSON :func:`read` is not implemented yet.
+    TypeError
+        If any line in the NDJSON file is not a JSON object (dict).
     """
-    raise NotImplementedError('NDJSON read is not implemented yet')
+    rows: JSONList = []
+    with path.open('r', encoding='utf-8') as handle:
+        for idx, line in enumerate(handle, start=1):
+            text = line.strip()
+            if not text:
+                continue
+            payload = json.loads(text)
+            if not isinstance(payload, dict):
+                raise TypeError(
+                    f'NDJSON lines must be objects (dicts) (line {idx})',
+                )
+            rows.append(cast(JSONDict, payload))
+    return rows
 
 
-def write(path: Path, data: JSONData) -> int:
+def write(
+    path: Path,
+    data: JSONData,
+) -> int:
     """
     Write ``data`` to NDJSON at ``path``.
 
@@ -53,7 +84,26 @@ def write(path: Path, data: JSONData) -> int:
 
     Raises
     ------
-    NotImplementedError
-        NDJSON :func:`write` is not implemented yet.
+    TypeError
+        If ``data`` is a list containing non-dict items.
     """
-    raise NotImplementedError('NDJSON write is not implemented yet')
+    rows: JSONList
+    if isinstance(data, list):
+        if not all(isinstance(item, dict) for item in data):
+            raise TypeError(
+                'NDJSON payloads must contain only objects (dicts)',
+            )
+        rows = cast(JSONList, data)
+    else:
+        rows = [cast(JSONDict, data)]
+
+    if not rows:
+        return 0
+
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open('w', encoding='utf-8') as handle:
+        for row in rows:
+            handle.write(json.dumps(row, ensure_ascii=False))
+            handle.write('\n')
+
+    return count_records(rows)

etlplus/file/orc.py

@@ -1,19 +1,34 @@
 """
 :mod:`etlplus.file.orc` module.
 
-Stub helpers for ORC read/write.
+Helpers for reading/writing ORC files.
 """
 
 from __future__ import annotations
 
 from pathlib import Path
+from typing import cast
 
 from ..types import JSONData
+from ..types import JSONList
+from ._io import normalize_records
+from ._pandas import get_pandas
 
 # SECTION: EXPORTS ========================================================== #
 
 
-def read(path: Path) -> JSONData:
+__all__ = [
+    'read',
+    'write',
+]
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def read(
+    path: Path,
+) -> JSONList:
     """
     Read ORC content from ``path``.
 
@@ -24,20 +39,31 @@ def read(path: Path) -> JSONData:
 
     Returns
     -------
-    JSONData
-        Parsed payload.
+    JSONList
+        The list of dictionaries read from the ORC file.
 
     Raises
     ------
-    NotImplementedError
-        ORC :func:`read` is not implemented yet.
+    ImportError
+        When optional dependency "pyarrow" is missing.
     """
-    raise NotImplementedError('ORC read is not implemented yet')
+    pandas = get_pandas('ORC')
+    try:
+        frame = pandas.read_orc(path)
+    except ImportError as e:  # pragma: no cover
+        raise ImportError(
+            'ORC support requires optional dependency "pyarrow".\n'
+            'Install with: pip install pyarrow',
+        ) from e
+    return cast(JSONList, frame.to_dict(orient='records'))
 
 
-def write(path: Path, data: JSONData) -> int:
+def write(
+    path: Path,
+    data: JSONData,
+) -> int:
     """
-    Write ``data`` to ORC at ``path``.
+    Write ``data`` to ORC at ``path`` and return record count.
 
     Parameters
     ----------
@@ -53,7 +79,21 @@ def write(path: Path, data: JSONData) -> int:
 
     Raises
     ------
-    NotImplementedError
-        ORC :func:`write` is not implemented yet.
+    ImportError
+        When optional dependency "pyarrow" is missing.
     """
-    raise NotImplementedError('ORC write is not implemented yet')
+    records = normalize_records(data, 'ORC')
+    if not records:
+        return 0
+
+    pandas = get_pandas('ORC')
+    path.parent.mkdir(parents=True, exist_ok=True)
+    frame = pandas.DataFrame.from_records(records)
+    try:
+        frame.to_orc(path, index=False)
+    except ImportError as e:  # pragma: no cover
+        raise ImportError(
+            'ORC support requires optional dependency "pyarrow".\n'
+            'Install with: pip install pyarrow',
+        ) from e
+    return len(records)