etlplus 0.10.5__py3-none-any.whl → 0.11.2__py3-none-any.whl

etlplus/file/json.py

@@ -0,0 +1,87 @@
+"""
+:mod:`etlplus.file.json` module.
+
+JSON read/write helpers.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import cast
+
+from ..types import JSONData
+from ..types import JSONDict
+from ..types import JSONList
+from ..utils import count_records
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def read(
+    path: Path,
+) -> JSONData:
+    """
+    Load and validate JSON payloads from ``path``.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the JSON file on disk.
+
+    Returns
+    -------
+    JSONData
+        The structured data read from the JSON file.
+
+    Raises
+    ------
+    TypeError
+        If the JSON root is not an object or an array of objects.
+    """
+    with path.open('r', encoding='utf-8') as handle:
+        loaded = json.load(handle)
+
+    if isinstance(loaded, dict):
+        return cast(JSONDict, loaded)
+    if isinstance(loaded, list):
+        if all(isinstance(item, dict) for item in loaded):
+            return cast(JSONList, loaded)
+        raise TypeError(
+            'JSON array must contain only objects (dicts) when loading file',
+        )
+    raise TypeError(
+        'JSON root must be an object or an array of objects when loading file',
+    )
+
+
+def write(
+    path: Path,
+    data: JSONData,
+) -> int:
+    """
+    Write ``data`` as formatted JSON to ``path``.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the JSON file on disk.
+    data : JSONData
+        Data to serialize as JSON.
+
+    Returns
+    -------
+    int
+        The number of records written to the JSON file.
+    """
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open('w', encoding='utf-8') as handle:
+        json.dump(
+            data,
+            handle,
+            indent=2,
+            ensure_ascii=False,
+        )
+        handle.write('\n')
+
+    return count_records(data)

etlplus/file/xml.py

@@ -0,0 +1,165 @@
+"""
+:mod:`etlplus.file.xml` module.
+
+XML read/write helpers.
+"""
+
+from __future__ import annotations
+
+import xml.etree.ElementTree as ET
+from pathlib import Path
+from typing import Any
+
+from ..types import JSONData
+from ..types import JSONDict
+from ..utils import count_records
+
+# SECTION: CONSTANTS ======================================================== #
+
+
+DEFAULT_XML_ROOT = 'root'
+
+
+# SECTION: INTERNAL FUNCTIONS =============================================== #
+
+
+def _dict_to_element(
+    name: str,
+    payload: Any,
+) -> ET.Element:
+    """
+    Convert a dictionary-like payload into an XML element.
+
+    Parameters
+    ----------
+    name : str
+        Name of the XML element.
+    payload : Any
+        The data to include in the XML element.
+
+    Returns
+    -------
+    ET.Element
+        The constructed XML element.
+    """
+    element = ET.Element(name)
+
+    if isinstance(payload, dict):
+        text = payload.get('text')
+        if text is not None:
+            element.text = str(text)
+
+        for key, value in payload.items():
+            if key == 'text':
+                continue
+            if key.startswith('@'):
+                element.set(key[1:], str(value))
+                continue
+            if isinstance(value, list):
+                for item in value:
+                    element.append(_dict_to_element(key, item))
+            else:
+                element.append(_dict_to_element(key, value))
+    elif isinstance(payload, list):
+        for item in payload:
+            element.append(_dict_to_element('item', item))
+    elif payload is not None:
+        element.text = str(payload)
+
+    return element
+
+
+def _element_to_dict(
+    element: ET.Element,
+) -> JSONDict:
+    """
+    Convert an XML element into a nested dictionary.
+
+    Parameters
+    ----------
+    element : ET.Element
+        XML element to convert.
+
+    Returns
+    -------
+    JSONDict
+        Nested dictionary representation of the XML element.
+    """
+    result: JSONDict = {}
+    text = (element.text or '').strip()
+    if text:
+        result['text'] = text
+
+    for child in element:
+        child_data = _element_to_dict(child)
+        tag = child.tag
+        if tag in result:
+            existing = result[tag]
+            if isinstance(existing, list):
+                existing.append(child_data)
+            else:
+                result[tag] = [existing, child_data]
+        else:
+            result[tag] = child_data
+
+    for key, value in element.attrib.items():
+        if key in result:
+            result[f'@{key}'] = value
+        else:
+            result[key] = value
+    return result
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def read(
+    path: Path,
+) -> JSONDict:
+    """
+    Parse XML document at ``path`` into a nested dictionary.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the XML file on disk.
+
+    Returns
+    -------
+    JSONDict
+        Nested dictionary representation of the XML file.
+    """
+    tree = ET.parse(path)
+    root = tree.getroot()
+
+    return {root.tag: _element_to_dict(root)}
+
+
+def write(path: Path, data: JSONData, *, root_tag: str) -> int:
+    """
+    Write ``data`` as XML to ``path`` and return record count.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the XML file on disk.
+    data : JSONData
+        Data to write as XML.
+    root_tag : str
+        Root tag name to use when writing XML files.
+
+    Returns
+    -------
+    int
+        The number of records written to the XML file.
+    """
+    if isinstance(data, dict) and len(data) == 1:
+        root_name, payload = next(iter(data.items()))
+        root_element = _dict_to_element(str(root_name), payload)
+    else:
+        root_element = _dict_to_element(root_tag, data)
+
+    tree = ET.ElementTree(root_element)
+    tree.write(path, encoding='utf-8', xml_declaration=True)
+
+    return count_records(data)

etlplus/file/yaml.py

@@ -0,0 +1,125 @@
+"""
+:mod:`etlplus.file.yaml` module.
+
+Optional YAML read/write helpers.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+from typing import cast
+
+from ..types import JSONData
+from ..types import JSONDict
+from ..types import JSONList
+from ..utils import count_records
+
+# SECTION: INTERNAL CONSTANTS =============================================== #
+
+
+# Optional YAML support (lazy-loaded to avoid hard dependency)
+# Cached access function to avoid global statements.
+_YAML_CACHE: dict[str, Any] = {}
+
+
+# SECTION: INTERNAL FUNCTIONS =============================================== #
+
+
+def _get_yaml() -> Any:
+    """
+    Return the PyYAML module, importing it on first use.
+
+    Raises an informative ImportError if the optional dependency is missing.
+    """
+    mod = _YAML_CACHE.get('mod')
+    if mod is not None:  # pragma: no cover - tiny branch
+        return mod
+    try:
+        _yaml_mod = __import__('yaml')  # type: ignore[assignment]
+    except ImportError as e:  # pragma: no cover
+        raise ImportError(
+            'YAML support requires optional dependency "PyYAML".\n'
+            'Install with: pip install PyYAML',
+        ) from e
+    _YAML_CACHE['mod'] = _yaml_mod
+
+    return _yaml_mod
+
+
+def _require_yaml() -> None:
+    """Ensure PyYAML is available or raise an informative error."""
+    _get_yaml()
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def read(
+    path: Path,
+) -> JSONData:
+    """
+    Load and validate YAML payloads from ``path``.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the YAML file on disk.
+
+    Returns
+    -------
+    JSONData
+        The structured data read from the YAML file.
+
+    Raises
+    ------
+    TypeError
+        If the YAML root is not an object or an array of objects.
+    """
+    _require_yaml()
+
+    with path.open('r', encoding='utf-8') as handle:
+        loaded = _get_yaml().safe_load(handle)
+
+    if isinstance(loaded, dict):
+        return cast(JSONDict, loaded)
+    if isinstance(loaded, list):
+        if all(isinstance(item, dict) for item in loaded):
+            return cast(JSONList, loaded)
+        raise TypeError(
+            'YAML array must contain only objects (dicts) when loading',
+        )
+    raise TypeError(
+        'YAML root must be an object or an array of objects when loading',
+    )
+
+
+def write(
+    path: Path,
+    data: JSONData,
+) -> int:
+    """
+    Write ``data`` as YAML to ``path`` and return record count.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the YAML file on disk.
+    data : JSONData
+        Data to write as YAML.
+
+    Returns
+    -------
+    int
+        The number of records written.
+    """
+    _require_yaml()
+    with path.open('w', encoding='utf-8') as handle:
+        _get_yaml().safe_dump(
+            data,
+            handle,
+            sort_keys=False,
+            allow_unicode=True,
+            default_flow_style=False,
+        )
+    return count_records(data)

etlplus/load.py

@@ -15,12 +15,9 @@ from typing import cast
 import requests  # type: ignore[import]
 
 from .enums import DataConnectorType
-from .enums import FileFormat
 from .enums import HttpMethod
-from .enums import coerce_data_connector_type
-from .enums import coerce_file_format
-from .enums import coerce_http_method
 from .file import File
+from .file import FileFormat
 from .types import JSONData
 from .types import JSONDict
 from .types import JSONList
@@ -101,7 +98,7 @@ def load_data(
         return cast(JSONData, source)
 
     if isinstance(source, Path):
-        return File(source, FileFormat.JSON).read_json()
+        return File(source, FileFormat.JSON).read()
 
     if isinstance(source, str):
         # Special case: '-' means read JSON from STDIN (Unix convention).
@@ -111,7 +108,7 @@ def load_data(
         candidate = Path(source)
         if candidate.exists():
             try:
-                return File(candidate, FileFormat.JSON).read_json()
+                return File(candidate, FileFormat.JSON).read()
             except (OSError, json.JSONDecodeError, ValueError):
                 # Fall back to treating the string as raw JSON content.
                 pass
@@ -155,9 +152,9 @@ def load_to_file(
     if file_format is None:
         records = File(path).write(data)
         ext = path.suffix.lstrip('.').lower()
-        fmt = coerce_file_format(ext) if ext else FileFormat.JSON
+        fmt = FileFormat.coerce(ext) if ext else FileFormat.JSON
     else:
-        fmt = coerce_file_format(file_format)
+        fmt = FileFormat.coerce(file_format)
         records = File(path, fmt).write(data)
     if fmt is FileFormat.CSV and records == 0:
         message = 'No data to write'
@@ -242,7 +239,7 @@ def load_to_api(
     TypeError
         If the session object is not valid.
     """
-    http_method = coerce_http_method(method)
+    http_method = HttpMethod.coerce(method)
 
     # Apply a conservative timeout to guard against hanging requests.
     timeout = kwargs.pop('timeout', 10.0)
@@ -316,7 +313,7 @@ def load(
     """
     data = load_data(source)
 
-    match coerce_data_connector_type(target_type):
+    match DataConnectorType.coerce(target_type):
         case DataConnectorType.FILE:
             # Prefer explicit format if provided, else infer from filename.
             return load_to_file(data, target, file_format)
@@ -331,6 +328,6 @@ def load(
                 **kwargs,
             )
         case _:
-            # `coerce_data_connector_type` covers invalid entries, but keep
-            # explicit guard.
+            # :meth:`coerce` already raises for invalid connector types, but
+            # keep explicit guard for defensive programming.
             raise ValueError(f'Invalid target type: {target_type}')

etlplus/run.py

@@ -23,7 +23,6 @@ from .api import RetryPolicy
 from .api import Url
 from .config import load_pipeline_config
 from .enums import DataConnectorType
-from .enums import coerce_data_connector_type
 from .extract import extract
 from .load import load
 from .run_helpers import compose_api_request_env
@@ -185,8 +184,7 @@ def run(
 
     data: Any
     stype_raw = getattr(source_obj, 'type', None)
-    stype = coerce_data_connector_type(stype_raw or '')
-    match stype:
+    match DataConnectorType.coerce(stype_raw or ''):
         case DataConnectorType.FILE:
             path = getattr(source_obj, 'path', None)
             fmt = ex_opts.get('format') or getattr(
@@ -261,8 +259,8 @@ def run(
                     sleep_seconds=cast(float, env.get('sleep_seconds', 0.0)),
                 )
         case _:
-            # ``coerce_data_connector_type`` already raises for invalid
-            # connector types; this branch is defensive only.
+            # :meth:`coerce` already raises for invalid connector types, but
+            # keep explicit guard for defensive programming.
             raise ValueError(f'Unsupported source type: {stype_raw}')
 
     # DRY: unified validation helper (pre/post transform)
@@ -318,8 +316,7 @@ def run(
     overrides = job_obj.load.overrides or {}
 
     ttype_raw = getattr(target_obj, 'type', None)
-    ttype = coerce_data_connector_type(ttype_raw or '')
-    match ttype:
+    match DataConnectorType.coerce(ttype_raw or ''):
         case DataConnectorType.FILE:
             path = overrides.get('path') or getattr(target_obj, 'path', None)
             fmt = overrides.get('format') or getattr(
@@ -357,8 +354,8 @@ def run(
             )
             result = load(data, 'database', str(conn))
         case _:
-            # ``coerce_data_connector_type`` already raises for invalid
-            # connector types; this branch is defensive only.
+            # :meth:`coerce` already raises for invalid connector types, but
+            # keep explicit guard for defensive programming.
             raise ValueError(f'Unsupported target type: {ttype_raw}')
 
     # Return the terminal load result directly; callers (e.g., CLI) can wrap

{etlplus-0.10.5.dist-info → etlplus-0.11.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: etlplus
-Version: 0.10.5
+Version: 0.11.2
 Summary: A Swiss Army knife for simple ETL operations
 Home-page: https://github.com/Dagitali/ETLPlus
 Author: ETLPlus Team

{etlplus-0.10.5.dist-info → etlplus-0.11.2.dist-info}/RECORD

@@ -1,13 +1,12 @@
 etlplus/__init__.py,sha256=M2gScnyir6WOMAh_EuoQIiAzdcTls0_5hbd_Q6of8I0,1021
 etlplus/__main__.py,sha256=btoROneNiigyfBU7BSzPKZ1R9gzBMpxcpsbPwmuHwTM,479
 etlplus/__version__.py,sha256=1E0GMK_yUWCMQFKxXjTvyMwofi0qT2k4CDNiHWiymWE,327
-etlplus/enums.py,sha256=8hzprOLyeCCzlHaXpG4VfgmxPSEdlZeOnHLFzBneKNs,15969
-etlplus/extract.py,sha256=f44JdHhNTACxgn44USx05paKTwq7LQY-V4wANCW9hVM,6173
-etlplus/file.py,sha256=B-zebTrIFDKaaKzA9Fq5-L0JwDNYa2T--_6veR3N03s,17939
-etlplus/load.py,sha256=R_y0_vtsEo1bwxWVQu2bfhB5ZIJoIoWu2ycCdvY4RnE,8737
+etlplus/enums.py,sha256=WyxpUEUPdYdXlueKDXGaSEo7o9OqCXyzjDOOPqmW8tw,8326
+etlplus/extract.py,sha256=LOyL8_KCOaIGemTxSnKbN_ttfLWUljqT4OQxANe7G3k,6089
+etlplus/load.py,sha256=aufl-2CpuI_J1hKBY1uFsoVf9Gfl9bKQjs233dYFf00,8631
 etlplus/mixins.py,sha256=ifGpHwWv7U00yqGf-kN93vJax2IiK4jaGtTsPsO3Oak,1350
 etlplus/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-etlplus/run.py,sha256=X4kp5FQlIWVf1_d9oSrchKau7BFDCE1Zkscvu7WPaWw,12340
+etlplus/run.py,sha256=FjcMF56HUbSw2PAvB_dZWP-xTFP-Pa_QLYTsrjmFurw,12262
 etlplus/run_helpers.py,sha256=bj6MkaeFxjl3CeKG1HoXKx5DwAlXNERVW-GX-z1P_qQ,24373
 etlplus/transform.py,sha256=uAUVDDHYCgx7GpVez9IK3OAZM-CnCuMa9iox3vwGGJA,25296
 etlplus/types.py,sha256=1hsDlnF6r76zAwaUYay-i6pCM-Y0IU5nP7Crj8PLCQ4,6157
@@ -31,10 +30,10 @@ etlplus/api/rate_limiting/__init__.py,sha256=ZySB1dZettEDnWvI1EHf_TZ9L08M_kKsNR-
 etlplus/api/rate_limiting/config.py,sha256=2b4wIynblN-1EyMqI4aXa71SljzSjXYh5N1Nngr3jOg,9406
 etlplus/api/rate_limiting/rate_limiter.py,sha256=Uxozqd_Ej5Lsj-M-mLT2WexChgWh7x35_YP10yqYPQA,7159
 etlplus/cli/__init__.py,sha256=J97-Rv931IL1_b4AXnB7Fbbd7HKnHBpx18NQfC_kE6c,299
-etlplus/cli/commands.py,sha256=BK2qmFsser6AXOgEvpiadrYMIiwviAzqkSxMlBhRXRw,24670
-etlplus/cli/constants.py,sha256=KIZj7J2tNf5mJbkqAdZmu5FXYW2FQmxwgeOKWc3-3Hg,1944
+etlplus/cli/commands.py,sha256=g8_m3A8HEMyTRu2HctNiRoi2gtB5oSZCUEcyq-PIXos,24669
+etlplus/cli/constants.py,sha256=E6Uy4WauLa_0zkzxqImXh-bb1gKdb9sBZQVc8QOzr2Q,1943
 etlplus/cli/handlers.py,sha256=K0GazvrPgocJ-63HZqF0xhyJk8TB1Gcj-eIbWltXKRU,17759
-etlplus/cli/io.py,sha256=7sldiZz4-Geomge5IO_XYykXPa6UiORfUWzLCdQePG8,7846
+etlplus/cli/io.py,sha256=EFaBPYaBOyOllfMQWXgTjy-MPiGfNejicpD7ROrPyAE,7840
 etlplus/cli/main.py,sha256=IgeqxypixfwLHR-QcpgVMQ7vMZ865bXOh2oO9v-BWeM,5234
 etlplus/cli/options.py,sha256=vfXT3YLh7wG1iC-aTdSg6ItMC8l6n0Lozmy53XjqLbA,1199
 etlplus/cli/state.py,sha256=Pfd8ru0wYIN7eGp1_A0tioqs1LiCDZCuJ6AnjZb6yYQ,8027
@@ -42,7 +41,7 @@ etlplus/cli/types.py,sha256=tclhKVJXDqHzlTQBYKARfqMgDOcuBJ-Zej2pvFy96WM,652
 etlplus/config/__init__.py,sha256=VZWzOg7d2YR9NT6UwKTv44yf2FRUMjTHynkm1Dl5Qzo,1486
 etlplus/config/connector.py,sha256=0-TIwevHbKRHVmucvyGpPd-3tB1dKHB-dj0yJ6kq5eY,9809
 etlplus/config/jobs.py,sha256=hmzRCqt0OvCEZZR4ONKrd3lvSv0OmayjLc4yOBk3ug8,7399
-etlplus/config/pipeline.py,sha256=Va4MQY6KEyKqHGMKPmh09ZcGpx95br-iNUjpkqtzVbw,9500
+etlplus/config/pipeline.py,sha256=m4Jh0ctFcKrIx6zR7LEC0sYY5wq0o8NqOruWPlz6qmA,9494
 etlplus/config/profile.py,sha256=Ss2zedQGjkaGSpvBLTD4SZaWViMJ7TJPLB8Q2_BTpPg,1898
 etlplus/config/types.py,sha256=a0epJ3z16HQ5bY3Ctf8s_cQPa3f0HHcwdOcjCP2xoG4,4954
 etlplus/config/utils.py,sha256=4SUHMkt5bKBhMhiJm-DrnmE2Q4TfOgdNCKz8PJDS27o,3443
@@ -52,14 +51,21 @@ etlplus/database/engine.py,sha256=7rr7YndA8LwyWJL8k1YhQbqxxmW4gWEUQjp0NwQcYtc,40
 etlplus/database/orm.py,sha256=gCSqH-CjQz6tV9133-VqgiwokK5ylun0BwXaIWfImAo,10008
 etlplus/database/schema.py,sha256=HNTgglI8qvQLInr7gq--2lLmLKHzAZTL2MJUOIw9DlY,7025
 etlplus/database/types.py,sha256=_pkQyC14TzAlgyeIqZG4F5LWYknZbHw3TW68Auk7Ya0,795
+etlplus/file/__init__.py,sha256=xd_Tvtzx7_PrGVb4Cjqp-v8p3P2qTPA3cZ14VzA1-0g,539
+etlplus/file/core.py,sha256=NXTGSIKIo7HvLDlMtme37_d4NUhsf4RUNKp5mTj-wqU,8131
+etlplus/file/csv.py,sha256=VbMW_NaqCw03HlfvYzb9MoAgCXI3cl9qc4dASkTHoyw,1880
+etlplus/file/enums.py,sha256=NjgXQ0f53Xa1eyGKHvYkk58udjQI5TQfLVaoawfdXY0,7520
+etlplus/file/json.py,sha256=xSV5PkZ_tZQuZNdLr1FQUwuCQXyL7Ch3WRJ3hkw0p68,1911
+etlplus/file/xml.py,sha256=vjate5u9Z26LPlpvZsdzpqXsIUZRgen7oHa3ly-aIhs,3905
+etlplus/file/yaml.py,sha256=6KaWoG7oYB26EHX2TZ7LOgigO11Hoq3MH--adFq_Eck,3004
 etlplus/templates/__init__.py,sha256=tsniN7XJYs3NwYxJ6c2HD5upHP3CDkLx-bQCMt97UOM,106
 etlplus/templates/ddl.sql.j2,sha256=s8fMWvcb4eaJVXkifuib1aQPljtZ8buuyB_uA-ZdU3Q,4734
 etlplus/templates/view.sql.j2,sha256=Iy8DHfhq5yyvrUKDxqp_aHIEXY4Tm6j4wT7YDEFWAhk,2180
 etlplus/validation/__init__.py,sha256=Pe5Xg1_EA4uiNZGYu5WTF3j7odjmyxnAJ8rcioaplSQ,1254
 etlplus/validation/utils.py,sha256=Mtqg449VIke0ziy_wd2r6yrwJzQkA1iulZC87FzXMjo,10201
-etlplus-0.10.5.dist-info/licenses/LICENSE,sha256=MuNO63i6kWmgnV2pbP2SLqP54mk1BGmu7CmbtxMmT-U,1069
-etlplus-0.10.5.dist-info/METADATA,sha256=XQapJo5LDeyZJH8A1GOHdv2sdBeqMxw-FWBgVsAmhfQ,21036
-etlplus-0.10.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-etlplus-0.10.5.dist-info/entry_points.txt,sha256=6w-2-jzuPa55spzK34h-UKh2JTEShh38adFRONNP9QE,45
-etlplus-0.10.5.dist-info/top_level.txt,sha256=aWWF-udn_sLGuHTM6W6MLh99ArS9ROkUWO8Mi8y1_2U,8
-etlplus-0.10.5.dist-info/RECORD,,
+etlplus-0.11.2.dist-info/licenses/LICENSE,sha256=MuNO63i6kWmgnV2pbP2SLqP54mk1BGmu7CmbtxMmT-U,1069
+etlplus-0.11.2.dist-info/METADATA,sha256=3I2djs1HEPyMy31qvH5oSlolJRgXi6qKXcjcLdLhFig,21036
+etlplus-0.11.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+etlplus-0.11.2.dist-info/entry_points.txt,sha256=6w-2-jzuPa55spzK34h-UKh2JTEShh38adFRONNP9QE,45
+etlplus-0.11.2.dist-info/top_level.txt,sha256=aWWF-udn_sLGuHTM6W6MLh99ArS9ROkUWO8Mi8y1_2U,8
+etlplus-0.11.2.dist-info/RECORD,,