etlplus 0.14.3__py3-none-any.whl → 0.16.0__py3-none-any.whl

etlplus/file/yaml.py

@@ -18,18 +18,17 @@ Notes
 from __future__ import annotations
 
 from pathlib import Path
-from typing import cast
 
 from ..types import JSONData
-from ..types import JSONDict
-from ..types import JSONList
 from ..utils import count_records
 from ._imports import get_yaml
+from ._io import coerce_record_payload
 
 # SECTION: EXPORTS ========================================================== #
 
 
 __all__ = [
+    # Functions
     'read',
     'write',
 ]
@@ -42,7 +41,7 @@ def read(
     path: Path,
 ) -> JSONData:
     """
-    Read YAML content from ``path``.
+    Read YAML content from *path*.
 
     Validates that the YAML root is a dict or a list of dicts.
 
@@ -64,17 +63,7 @@ def read(
     with path.open('r', encoding='utf-8') as handle:
         loaded = get_yaml().safe_load(handle)
 
-    if isinstance(loaded, dict):
-        return cast(JSONDict, loaded)
-    if isinstance(loaded, list):
-        if all(isinstance(item, dict) for item in loaded):
-            return cast(JSONList, loaded)
-        raise TypeError(
-            'YAML array must contain only objects (dicts) when loading',
-        )
-    raise TypeError(
-        'YAML root must be an object or an array of objects when loading',
-    )
+    return coerce_record_payload(loaded, format_name='YAML')
 
 
 def write(
@@ -82,7 +71,7 @@ def write(
     data: JSONData,
 ) -> int:
     """
-    Write ``data`` as YAML to ``path`` and return record count.
+    Write *data* as YAML to *path* and return record count.
 
     Parameters
     ----------

etlplus/file/zip.py

@@ -20,6 +20,7 @@ from .enums import infer_file_format_and_compression
 
 
 __all__ = [
+    # Functions
     'read',
     'write',
 ]
@@ -89,7 +90,7 @@ def read(
     path: Path,
 ) -> JSONData:
     """
-    Read ZIP content from ``path`` and parse the inner payload(s).
+    Read ZIP content from *path* and parse the inner payload(s).
 
     Parameters
     ----------
@@ -140,7 +141,7 @@ def write(
     data: JSONData,
 ) -> int:
     """
-    Write ``data`` to ZIP at ``path`` and return record count.
+    Write *data* to ZIP at *path* and return record count.
 
     Parameters
     ----------

etlplus/file/zsav.py

@@ -27,6 +27,7 @@ from . import stub
 
 
 __all__ = [
+    # Functions
     'read',
     'write',
 ]
@@ -39,7 +40,7 @@ def read(
     path: Path,
 ) -> JSONList:
     """
-    Read ZSAV content from ``path``.
+    Read ZSAV content from *path*.
 
     Parameters
     ----------
@@ -59,7 +60,7 @@ def write(
     data: JSONData,
 ) -> int:
     """
-    Write ``data`` to ZSAV file at ``path`` and return record count.
+    Write *data* to ZSAV file at *path* and return record count.
 
     Parameters
     ----------

etlplus/ops/extract.py

@@ -12,7 +12,7 @@ from typing import cast
 
 from ..api import HttpMethod
 from ..api.utils import resolve_request
-from ..enums import DataConnectorType
+from ..connector import DataConnectorType
 from ..file import File
 from ..file import FileFormat
 from ..types import JSONData
@@ -20,6 +20,18 @@ from ..types import JSONDict
 from ..types import JSONList
 from ..types import StrPath
 
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Functions
+    'extract',
+    'extract_from_api',
+    'extract_from_database',
+    'extract_from_file',
+]
+
+
 # SECTION: FUNCTIONS ======================================================== #
 
 

etlplus/ops/load.py

@@ -14,7 +14,7 @@ from typing import cast
 
 from ..api import HttpMethod
 from ..api.utils import resolve_request
-from ..enums import DataConnectorType
+from ..connector import DataConnectorType
 from ..file import File
 from ..file import FileFormat
 from ..types import JSONData
@@ -23,6 +23,19 @@ from ..types import JSONList
 from ..types import StrPath
 from ..utils import count_records
 
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Functions
+    'load',
+    'load_data',
+    'load_to_api',
+    'load_to_database',
+    'load_to_file',
+]
+
+
 # SECTION: INTERNAL FUNCTIONS ============================================== #
 
 
@@ -30,7 +43,7 @@ def _parse_json_string(
     raw: str,
 ) -> JSONData:
     """
-    Parse JSON data from ``raw`` text.
+    Parse JSON data from *raw* text.
 
     Parameters
     ----------

etlplus/ops/run.py

@@ -20,8 +20,7 @@ from ..api import RequestOptions
 from ..api import compose_api_request_env
 from ..api import compose_api_target_env
 from ..api import paginate_with_client
-from ..config import load_pipeline_config
-from ..enums import DataConnectorType
+from ..connector import DataConnectorType
 from ..file import FileFormat
 from ..types import JSONData
 from ..types import JSONDict
@@ -29,6 +28,7 @@ from ..types import PipelineConfig
 from ..types import StrPath
 from ..types import Timeout
 from ..utils import print_json
+from ..workflow import load_pipeline_config
 from .extract import extract
 from .load import load
 from .transform import transform
@@ -94,7 +94,7 @@ def run(
     Run a pipeline job defined in a YAML configuration.
 
     By default it reads the configuration from ``in/pipeline.yml``, but callers
-    can provide an explicit ``config_path`` to override this.
+    can provide an explicit *config_path* to override this.
 
     Parameters
     ----------
@@ -162,9 +162,12 @@ def run(
                 # can monkeypatch this class on etlplus.ops.run.
                 ClientClass = EndpointClient  # noqa: N806
                 client = ClientClass(
-                    base_url=cast(str, env['base_url']),
+                    base_url=cast(str, env.get('base_url')),
                     base_path=cast(str | None, env.get('base_path')),
-                    endpoints=cast(dict[str, str], env['endpoints_map']),
+                    endpoints=cast(
+                        dict[str, str],
+                        env.get('endpoints_map', {}),
+                    ),
                     retry=env.get('retry'),
                     retry_network_errors=bool(
                         env.get('retry_network_errors', False),
@@ -173,7 +176,7 @@ def run(
                 )
                 data = paginate_with_client(
                     client,
-                    cast(str, env['endpoint_key']),
+                    cast(str, env.get('endpoint_key')),
                     env.get('params'),
                     env.get('headers'),
                     env.get('timeout'),
@@ -276,12 +279,14 @@ def run(
             if not url_t:
                 raise ValueError('API target missing "url"')
             kwargs_t: dict[str, Any] = {}
-            if env_t.get('headers'):
-                kwargs_t['headers'] = cast(dict[str, str], env_t['headers'])
+            headers = env_t.get('headers')
+            if headers:
+                kwargs_t['headers'] = cast(dict[str, str], headers)
             if env_t.get('timeout') is not None:
-                kwargs_t['timeout'] = env_t['timeout']
-            if env_t.get('session') is not None:
-                kwargs_t['session'] = env_t['session']
+                kwargs_t['timeout'] = env_t.get('timeout')
+            session = env_t.get('session')
+            if session is not None:
+                kwargs_t['session'] = session
             result = load(
                 data,
                 'api',
@@ -323,11 +328,11 @@ def run_pipeline(
     Parameters
     ----------
     source_type : DataConnectorType | str | None, optional
-        Connector type for extraction. When ``None``, ``source`` is assumed
+        Connector type for extraction. When ``None``, *source* is assumed
         to be pre-loaded data and extraction is skipped.
     source : StrPath | JSONData | None, optional
         Data source for extraction or the pre-loaded payload when
-        ``source_type`` is ``None``.
+        *source_type* is ``None``.
     operations : PipelineConfig | None, optional
         Transform configuration passed to :func:`etlplus.ops.transform`.
     target_type : DataConnectorType | str | None, optional

etlplus/ops/transform.py

@@ -110,7 +110,7 @@ def _agg_count(
     present: int,
 ) -> int:
     """
-    Return the provided presence count ``present``.
+    Return the provided presence count *present*.
 
     Parameters
     ----------
@@ -120,7 +120,7 @@ def _agg_count(
     Returns
     -------
     int
-        The provided presence count ``present``.
+        The provided presence count *present*.
     """
     return present
 

etlplus/ops/utils.py

@@ -7,13 +7,11 @@ The helpers defined here embrace a "high cohesion, low coupling" design by
 isolating normalization, configuration, and logging responsibilities. The
 resulting surface keeps ``maybe_validate`` focused on orchestration while
 offloading ancillary concerns to composable helpers.
-
 """
 
 from __future__ import annotations
 
 from collections.abc import Callable
-from collections.abc import Mapping
 from dataclasses import dataclass
 from types import MappingProxyType
 from typing import Any
@@ -23,7 +21,7 @@ from typing import TypedDict
 from typing import cast
 
 from ..types import StrAnyMap
-from ..utils import normalized_str
+from ..utils import normalize_choice
 
 # SECTION: TYPED DICTIONARIES =============================================== #
 
@@ -207,14 +205,14 @@ def maybe_validate(
     Returns
     -------
     Any
-        ``payload`` when validation is skipped or when severity is ``"warn"``
+        *payload* when validation is skipped or when severity is ``"warn"``
         and the validation fails. Returns the validator ``data`` payload when
         validation succeeds.
 
     Raises
     ------
     ValueError
-        Raised when validation fails and ``severity`` is ``"error"``.
+        Raised when validation fails and *severity* is ``"error"``.
 
     Examples
     --------
@@ -320,7 +318,7 @@ def _normalize_phase(
     """
     return cast(
         ValidationPhase,
-        _normalize_choice(
+        normalize_choice(
             value,
             mapping=_PHASE_CHOICES,
             default='before_transform',
@@ -346,7 +344,7 @@ def _normalize_severity(
     """
     return cast(
         ValidationSeverity,
-        _normalize_choice(
+        normalize_choice(
             value,
             mapping=_SEVERITY_CHOICES,
             default='error',
@@ -372,7 +370,7 @@ def _normalize_window(
     """
     return cast(
         ValidationWindow,
-        _normalize_choice(
+        normalize_choice(
             value,
             mapping=_WINDOW_CHOICES,
             default='both',
@@ -380,33 +378,6 @@ def _normalize_window(
     )
 
 
-def _normalize_choice(
-    value: str | None,
-    *,
-    mapping: Mapping[str, str],
-    default: str,
-) -> str:
-    """
-    Normalize a text value against a mapping with a default fallback.
-
-    Parameters
-    ----------
-    value : str | None
-        Input text to normalize.
-    mapping : Mapping[str, str]
-        Mapping of accepted values to normalized outputs.
-    default : str
-        Default to return when input is missing or unrecognized.
-
-    Returns
-    -------
-    str
-        Normalized value.
-    """
-    normalized = normalized_str(value)
-    return mapping.get(normalized, default)
-
-
 def _rule_name(
     rules: Ruleset,
 ) -> str | None:

etlplus/ops/validate.py

@@ -11,8 +11,8 @@ Highlights
 ----------
 - Centralized type map and helpers for clarity and reuse.
 - Consistent error wording; field and item paths like ``[2].email``.
-- Small, focused public API with ``load_data``, ``validate_field``,
-  ``validate``.
+- Small, focused public API with :func:`load_data`, :func:`validate_field`,
+    :func:`validate`.
 
 Examples
 --------
@@ -66,7 +66,7 @@ TYPE_MAP: Final[dict[str, type | tuple[type, ...]]] = {
 }
 
 
-# SECTION: CLASSES ========================================================== #
+# SECTION: TYPED DICTS ====================================================== #
 
 
 class FieldRules(TypedDict, total=False):

etlplus/templates/README.md

@@ -1,4 +1,4 @@
-# etlplus.templates subpackage
+# `etlplus.templates` Subpackage
 
 Documentation for the `etlplus.templates` subpackage: SQL and DDL template helpers.
 
@@ -8,7 +8,7 @@ Documentation for the `etlplus.templates` subpackage: SQL and DDL template helpe
 
 Back to project overview: see the top-level [README](../../README.md).
 
-- [etlplus.templates subpackage](#etlpustemplates-subpackage)
+- [`etlplus.templates` Subpackage](#etlplus-templates-subpackage)
     - [Available Templates](#available-templates)
     - [Rendering Templates](#rendering-templates)
     - [Example: Rendering a DDL Template](#example-rendering-a-ddl-template)

etlplus/types.py

@@ -11,8 +11,9 @@ Notes
 
 See Also
 --------
-- :mod:`etlplus.api.types` for HTTP-specific aliases
-- :mod:`etlplus.config.types` for TypedDict surfaces
+- :mod:`etlplus.api.types` for HTTP-specific aliases and data classes
+- :mod:`etlplus.connector.types` for connector-specific aliases and TypedDict
+    surfaces
 
 Examples
 --------