etlplus 0.10.4__py3-none-any.whl → 0.12.2__py3-none-any.whl

etlplus/file/orc.py

@@ -0,0 +1,142 @@
+"""
+:mod:`etlplus.file.orc` module.
+
+Helpers for reading/writing ORC files.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+from typing import cast
+
+from ..types import JSONData
+from ..types import JSONDict
+from ..types import JSONList
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'read',
+    'write',
+]
+
+
+# SECTION: INTERNAL CONSTANTS =============================================== #
+
+
+_PANDAS_CACHE: dict[str, Any] = {}
+
+
+# SECTION: INTERNAL FUNCTIONS =============================================== #
+
+
+def _get_pandas() -> Any:
+    """
+    Return the pandas module, importing it on first use.
+
+    Raises an informative ImportError if the optional dependency is missing.
+    """
+    mod = _PANDAS_CACHE.get('mod')
+    if mod is not None:  # pragma: no cover - tiny branch
+        return mod
+    try:
+        _pd = __import__('pandas')  # type: ignore[assignment]
+    except ImportError as e:  # pragma: no cover
+        raise ImportError(
+            'ORC support requires optional dependency "pandas".\n'
+            'Install with: pip install pandas',
+        ) from e
+    _PANDAS_CACHE['mod'] = _pd
+
+    return _pd
+
+
+def _normalize_records(data: JSONData) -> JSONList:
+    """
+    Normalize JSON payloads into a list of dictionaries.
+
+    Raises TypeError when payloads contain non-dict items.
+    """
+    if isinstance(data, list):
+        if not all(isinstance(item, dict) for item in data):
+            raise TypeError('ORC payloads must contain only objects (dicts)')
+        return cast(JSONList, data)
+    return [cast(JSONDict, data)]
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def read(
+    path: Path,
+) -> JSONList:
+    """
+    Read ORC content from ``path``.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the ORC file on disk.
+
+    Returns
+    -------
+    JSONList
+        The list of dictionaries read from the ORC file.
+
+    Raises
+    ------
+    ImportError
+        When optional dependency "pyarrow" is missing.
+    """
+    pandas = _get_pandas()
+    try:
+        frame = pandas.read_orc(path)
+    except ImportError as e:  # pragma: no cover
+        raise ImportError(
+            'ORC support requires optional dependency "pyarrow".\n'
+            'Install with: pip install pyarrow',
+        ) from e
+    return cast(JSONList, frame.to_dict(orient='records'))
+
+
+def write(
+    path: Path,
+    data: JSONData,
+) -> int:
+    """
+    Write ``data`` to ORC at ``path`` and return record count.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the ORC file on disk.
+    data : JSONData
+        Data to write.
+
+    Returns
+    -------
+    int
+        Number of records written.
+
+    Raises
+    ------
+    ImportError
+        When optional dependency "pyarrow" is missing.
+    """
+    records = _normalize_records(data)
+    if not records:
+        return 0
+
+    pandas = _get_pandas()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    frame = pandas.DataFrame.from_records(records)
+    try:
+        frame.to_orc(path, index=False)
+    except ImportError as e:  # pragma: no cover
+        raise ImportError(
+            'ORC support requires optional dependency "pyarrow".\n'
+            'Install with: pip install pyarrow',
+        ) from e
+    return len(records)

etlplus/file/parquet.py

@@ -0,0 +1,146 @@
+"""
+:mod:`etlplus.file.parquet` module.
+
+Helpers for reading/writing Parquet files.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+from typing import cast
+
+from ..types import JSONData
+from ..types import JSONDict
+from ..types import JSONList
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'read',
+    'write',
+]
+
+
+# SECTION: INTERNAL CONSTANTS =============================================== #
+
+
+_PANDAS_CACHE: dict[str, Any] = {}
+
+
+# SECTION: INTERNAL FUNCTIONS =============================================== #
+
+
+def _get_pandas() -> Any:
+    """
+    Return the pandas module, importing it on first use.
+
+    Raises an informative ImportError if the optional dependency is missing.
+    """
+    mod = _PANDAS_CACHE.get('mod')
+    if mod is not None:  # pragma: no cover - tiny branch
+        return mod
+    try:
+        _pd = __import__('pandas')  # type: ignore[assignment]
+    except ImportError as e:  # pragma: no cover
+        raise ImportError(
+            'Parquet support requires optional dependency "pandas".\n'
+            'Install with: pip install pandas',
+        ) from e
+    _PANDAS_CACHE['mod'] = _pd
+
+    return _pd
+
+
+def _normalize_records(data: JSONData) -> JSONList:
+    """
+    Normalize JSON payloads into a list of dictionaries.
+
+    Raises TypeError when payloads contain non-dict items.
+    """
+    if isinstance(data, list):
+        if not all(isinstance(item, dict) for item in data):
+            raise TypeError(
+                'Parquet payloads must contain only objects (dicts)',
+            )
+        return cast(JSONList, data)
+    return [cast(JSONDict, data)]
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def read(
+    path: Path,
+) -> JSONList:
+    """
+    Read Parquet content from ``path``.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the PARQUET file on disk.
+
+    Returns
+    -------
+    JSONList
+        The list of dictionaries read from the Parquet file.
+
+    Raises
+    ------
+    ImportError
+        If optional dependencies for Parquet support are missing.
+    """
+    pandas = _get_pandas()
+    try:
+        frame = pandas.read_parquet(path)
+    except ImportError as e:  # pragma: no cover
+        raise ImportError(
+            'Parquet support requires optional dependency '
+            '"pyarrow" or "fastparquet".\n'
+            'Install with: pip install pyarrow',
+        ) from e
+    return cast(JSONList, frame.to_dict(orient='records'))
+
+
+def write(
+    path: Path,
+    data: JSONData,
+) -> int:
+    """
+    Write ``data`` to Parquet at ``path`` and return record count.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the PARQUET file on disk.
+    data : JSONData
+        Data to write.
+
+    Returns
+    -------
+    int
+        Number of records written.
+
+    Raises
+    ------
+    ImportError
+        If optional dependencies for Parquet support are missing.
+    """
+    records = _normalize_records(data)
+    if not records:
+        return 0
+
+    pandas = _get_pandas()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    frame = pandas.DataFrame.from_records(records)
+    try:
+        frame.to_parquet(path, index=False)
+    except ImportError as e:  # pragma: no cover
+        raise ImportError(
+            'Parquet support requires optional dependency '
+            '"pyarrow" or "fastparquet".\n'
+            'Install with: pip install pyarrow',
+        ) from e
+    return len(records)

etlplus/file/tsv.py

@@ -0,0 +1,91 @@
+"""
+:mod:`etlplus.file.tsv` module.
+
+Helpers for reading/writing TSV files.
+"""
+
+from __future__ import annotations
+
+import csv
+from pathlib import Path
+from typing import cast
+
+from ..types import JSONData
+from ..types import JSONDict
+from ..types import JSONList
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'read',
+    'write',
+]
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def read(
+    path: Path,
+) -> JSONList:
+    """
+    Read TSV content from ``path``.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the TSV file on disk.
+
+    Returns
+    -------
+    JSONList
+        The list of dictionaries read from the TSV file.
+    """
+    with path.open('r', encoding='utf-8', newline='') as handle:
+        reader: csv.DictReader[str] = csv.DictReader(handle, delimiter='\t')
+        rows: JSONList = []
+        for row in reader:
+            if not any(row.values()):
+                continue
+            rows.append(cast(JSONDict, dict(row)))
+    return rows
+
+
+def write(
+    path: Path,
+    data: JSONData,
+) -> int:
+    """
+    Write ``data`` to TSV at ``path`` and return record count.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the TSV file on disk.
+    data : JSONData
+        Data to write as TSV. Should be a list of dictionaries or a
+        single dictionary.
+
+    Returns
+    -------
+    int
+        The number of rows written to the TSV file.
+    """
+    rows: list[JSONDict]
+    if isinstance(data, list):
+        rows = [row for row in data if isinstance(row, dict)]
+    else:
+        rows = [data]
+
+    if not rows:
+        return 0
+
+    fieldnames = sorted({key for row in rows for key in row})
+    with path.open('w', encoding='utf-8', newline='') as handle:
+        writer = csv.DictWriter(handle, fieldnames=fieldnames, delimiter='\t')
+        writer.writeheader()
+        for row in rows:
+            writer.writerow({field: row.get(field) for field in fieldnames})
+
+    return len(rows)

etlplus/file/txt.py

@@ -0,0 +1,99 @@
+"""
+:mod:`etlplus.file.txt` module.
+
+Helpers for reading/writing text files.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import cast
+
+from ..types import JSONData
+from ..types import JSONDict
+from ..types import JSONList
+from ..utils import count_records
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'read',
+    'write',
+]
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def read(
+    path: Path,
+) -> JSONList:
+    """
+    Read TXT content from ``path``.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the TXT file on disk.
+
+    Returns
+    -------
+    JSONList
+        The list of dictionaries read from the TXT file.
+    """
+    rows: JSONList = []
+    with path.open('r', encoding='utf-8') as handle:
+        for line in handle:
+            text = line.rstrip('\n')
+            if text == '':
+                continue
+            rows.append({'text': text})
+    return rows
+
+
+def write(
+    path: Path,
+    data: JSONData,
+) -> int:
+    """
+    Write ``data`` to TXT at ``path`` and return record count.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the TXT file on disk.
+    data : JSONData
+        Data to write. Expects ``{'text': '...'} `` or a list of those.
+
+    Returns
+    -------
+    int
+        Number of records written.
+
+    Raises
+    ------
+    TypeError
+        If any item in ``data`` is not a dictionary or if any dictionary
+        does not contain a ``'text'`` key.
+    """
+    rows: JSONList
+    if isinstance(data, list):
+        if not all(isinstance(item, dict) for item in data):
+            raise TypeError('TXT payloads must contain only objects (dicts)')
+        rows = cast(JSONList, data)
+    else:
+        rows = [cast(JSONDict, data)]
+
+    if not rows:
+        return 0
+
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open('w', encoding='utf-8') as handle:
+        for row in rows:
+            if 'text' not in row:
+                raise TypeError('TXT payloads must include a "text" key')
+            handle.write(str(row['text']))
+            handle.write('\n')
+
+    return count_records(rows)

etlplus/file/xls.py

@@ -0,0 +1,132 @@
+"""
+:mod:`etlplus.file.xls` module.
+
+Helpers for reading/writing Excel XLS files.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+from typing import cast
+
+from ..types import JSONData
+from ..types import JSONDict
+from ..types import JSONList
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'read',
+    'write',
+]
+
+
+# SECTION: INTERNAL CONSTANTS =============================================== #
+
+
+_PANDAS_CACHE: dict[str, Any] = {}
+
+
+# SECTION: INTERNAL FUNCTIONS =============================================== #
+
+
+def _get_pandas() -> Any:
+    """
+    Return the pandas module, importing it on first use.
+
+    Raises an informative ImportError if the optional dependency is missing.
+    """
+    mod = _PANDAS_CACHE.get('mod')
+    if mod is not None:  # pragma: no cover - tiny branch
+        return mod
+    try:
+        _pd = __import__('pandas')  # type: ignore[assignment]
+    except ImportError as e:  # pragma: no cover
+        raise ImportError(
+            'XLS support requires optional dependency "pandas".\n'
+            'Install with: pip install pandas',
+        ) from e
+    _PANDAS_CACHE['mod'] = _pd
+
+    return _pd
+
+
+def _normalize_records(data: JSONData) -> JSONList:
+    """
+    Normalize JSON payloads into a list of dictionaries.
+
+    Raises TypeError when payloads contain non-dict items.
+    """
+    if isinstance(data, list):
+        if not all(isinstance(item, dict) for item in data):
+            raise TypeError('XLS payloads must contain only objects (dicts)')
+        return cast(JSONList, data)
+    return [cast(JSONDict, data)]
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def read(
+    path: Path,
+) -> JSONList:
+    """
+    Read XLS content from ``path``.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the XLS file on disk.
+
+    Returns
+    -------
+    JSONList
+        The list of dictionaries read from the XLS file.
+
+    Raises
+    ------
+    ImportError
+        If the optional dependency "xlrd" is not installed.
+    """
+    pandas = _get_pandas()
+    try:
+        frame = pandas.read_excel(path, engine='xlrd')
+    except ImportError as e:  # pragma: no cover
+        raise ImportError(
+            'XLS support requires optional dependency "xlrd".\n'
+            'Install with: pip install xlrd',
+        ) from e
+    return cast(JSONList, frame.to_dict(orient='records'))
+
+
+def write(
+    path: Path,
+    data: JSONData,
+) -> int:
+    """
+    Write ``data`` to XLS at ``path`` and return record count.
+
+    Notes
+    -----
+    XLS writing is not supported by pandas 2.x. Use XLSX for writes.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the XLS file on disk.
+    data : JSONData
+        Data to write.
+
+    Returns
+    -------
+    int
+        Number of records written.
+
+    Raises
+    ------
+    ImportError
+        If the optional dependency "xlwt" is not installed.
+    """
+    raise RuntimeError('XLS write is not supported; use XLSX instead')