etlplus 0.12.10__py3-none-any.whl → 0.14.3__py3-none-any.whl

etlplus/{transform.py → ops/transform.py}

@@ -1,5 +1,5 @@
 """
-:mod:`etlplus.transform` module.
+:mod:`etlplus.ops.transform` module.
 
 Helpers to filter, map/rename, select, sort, aggregate, and otherwise
 transform JSON-like records (dicts and lists of dicts).
@@ -24,7 +24,7 @@ Basic pipeline with strings::
 
 Using enums for keys and functions::
 
-    from .enums import PipelineStep, OperatorName, AggregateName
+    from etlplus.enums import PipelineStep, OperatorName, AggregateName
     ops = {
         PipelineStep.FILTER: {
             'field': 'age', 'op': OperatorName.GTE, 'value': 18
@@ -44,28 +44,28 @@ from collections.abc import Sequence
 from typing import Any
 from typing import cast
 
-from .enums import AggregateName
-from .enums import OperatorName
-from .enums import PipelineStep
+from ..enums import AggregateName
+from ..enums import OperatorName
+from ..enums import PipelineStep
+from ..types import AggregateFunc
+from ..types import AggregateSpec
+from ..types import FieldName
+from ..types import Fields
+from ..types import FilterSpec
+from ..types import JSONData
+from ..types import JSONDict
+from ..types import JSONList
+from ..types import MapSpec
+from ..types import OperatorFunc
+from ..types import PipelineConfig
+from ..types import PipelineStepName
+from ..types import SortKey
+from ..types import StepApplier
+from ..types import StepOrSteps
+from ..types import StepSpec
+from ..types import StrPath
+from ..utils import to_number
 from .load import load_data
-from .types import AggregateFunc
-from .types import AggregateSpec
-from .types import FieldName
-from .types import Fields
-from .types import FilterSpec
-from .types import JSONData
-from .types import JSONDict
-from .types import JSONList
-from .types import MapSpec
-from .types import OperatorFunc
-from .types import PipelineConfig
-from .types import PipelineStepName
-from .types import SortKey
-from .types import StepApplier
-from .types import StepOrSteps
-from .types import StepSpec
-from .types import StrPath
-from .utils import to_number
 
 # SECTION: EXPORTS ========================================================== #
 
@@ -730,15 +730,16 @@ def _is_plain_fields_list(obj: Any) -> bool:
 
 
 _PIPELINE_STEPS: tuple[PipelineStepName, ...] = (
+    'aggregate',
     'filter',
     'map',
     'select',
     'sort',
-    'aggregate',
 )
 
 
 _STEP_APPLIERS: dict[PipelineStepName, StepApplier] = {
+    'aggregate': _apply_aggregate_step,
     'filter': _apply_filter_step,
     'map': _apply_map_step,
     'select': _apply_select_step,
@@ -746,7 +747,54 @@ _STEP_APPLIERS: dict[PipelineStepName, StepApplier] = {
 }
 
 
-# SECTION: EXPORTS ========================================================== #
+# SECTION: FUNCTIONS ======================================================== #
+
+
+# -- Helpers -- #
+
+
+def apply_aggregate(
+    records: JSONList,
+    operation: AggregateSpec,
+) -> JSONDict:
+    """
+    Aggregate a numeric field or count presence.
+
+    Parameters
+    ----------
+    records : JSONList
+        Records to aggregate.
+    operation : AggregateSpec
+        Dict with keys ``field`` and ``func``. ``func`` is one of
+        ``'sum'``, ``'avg'``, ``'min'``, ``'max'``, or ``'count'``.
+        A callable may also be supplied for ``func``. Optionally, set
+        ``alias`` to control the output key name.
+
+    Returns
+    -------
+    JSONDict
+        A single-row result like ``{"sum_age": 42}``.
+
+    Notes
+    -----
+    Numeric operations ignore non-numeric values but count their presence
+    for ``'count'``.
+    """
+    field = operation.get('field')
+    func = operation.get('func')
+    alias = operation.get('alias')
+
+    if not field or func is None:
+        return {'error': 'Invalid aggregation operation'}
+
+    try:
+        aggregator = _resolve_aggregator(func)
+    except TypeError:
+        return {'error': f'Unknown aggregation function: {func}'}
+
+    nums, present = _collect_numeric_and_presence(records, field)
+    key_name = _derive_agg_key(func, field, alias)
+    return {key_name: aggregator(nums, present)}
 
 
 def apply_filter(
@@ -894,48 +942,7 @@ def apply_sort(
     )
 
 
-def apply_aggregate(
-    records: JSONList,
-    operation: AggregateSpec,
-) -> JSONDict:
-    """
-    Aggregate a numeric field or count presence.
-
-    Parameters
-    ----------
-    records : JSONList
-        Records to aggregate.
-    operation : AggregateSpec
-        Dict with keys ``field`` and ``func``. ``func`` is one of
-        ``'sum'``, ``'avg'``, ``'min'``, ``'max'``, or ``'count'``.
-        A callable may also be supplied for ``func``. Optionally, set
-        ``alias`` to control the output key name.
-
-    Returns
-    -------
-    JSONDict
-        A single-row result like ``{"sum_age": 42}``.
-
-    Notes
-    -----
-    Numeric operations ignore non-numeric values but count their presence
-    for ``'count'``.
-    """
-    field = operation.get('field')
-    func = operation.get('func')
-    alias = operation.get('alias')
-
-    if not field or func is None:
-        return {'error': 'Invalid aggregation operation'}
-
-    try:
-        aggregator = _resolve_aggregator(func)
-    except TypeError:
-        return {'error': f'Unknown aggregation function: {func}'}
-
-    nums, present = _collect_numeric_and_presence(records, field)
-    key_name = _derive_agg_key(func, field, alias)
-    return {key_name: aggregator(nums, present)}
+# -- Orchestration -- #
 
 
 def transform(
@@ -982,7 +989,7 @@ def transform(
 
     Using enums for keys and functions::
 
-        from .enums import PipelineStep, OperatorName, AggregateName
+        from etlplus.enums import PipelineStep, OperatorName, AggregateName
         ops = {
             PipelineStep.FILTER: {
                 'field': 'age', 'op': OperatorName.GTE, 'value': 18

etlplus/{validation → ops}/utils.py

@@ -1,7 +1,7 @@
 """
-:mod:`etlplus.validation.utils` module.
+:mod:`etlplus.ops.utils` module.
 
-Utility helpers for conditional validation orchestration.
+Utility helpers for conditional data ops orchestration.
 
 The helpers defined here embrace a "high cohesion, low coupling" design by
 isolating normalization, configuration, and logging responsibilities. The
@@ -13,11 +13,14 @@ offloading ancillary concerns to composable helpers.
 from __future__ import annotations
 
 from collections.abc import Callable
+from collections.abc import Mapping
 from dataclasses import dataclass
+from types import MappingProxyType
 from typing import Any
 from typing import Literal
 from typing import Self
 from typing import TypedDict
+from typing import cast
 
 from ..types import StrAnyMap
 from ..utils import normalized_str
@@ -47,6 +50,30 @@ type ValidateFn = Callable[[Any, Ruleset], ValidationResult]
 type PrintFn = Callable[[Any], None]
 
 
+# SECTION: INTERNAL CONSTANTS ============================================== #
+
+
+_PHASE_CHOICES = MappingProxyType(
+    {
+        'before_transform': 'before_transform',
+        'after_transform': 'after_transform',
+    },
+)
+_SEVERITY_CHOICES = MappingProxyType(
+    {
+        'warn': 'warn',
+        'error': 'error',
+    },
+)
+_WINDOW_CHOICES = MappingProxyType(
+    {
+        'before_transform': 'before_transform',
+        'after_transform': 'after_transform',
+        'both': 'both',
+    },
+)
+
+
 # SECTION: DATA CLASSES ===================================================== #
 
 
@@ -291,11 +318,14 @@ def _normalize_phase(
         Normalized validation phase. Defaults to ``"before_transform"`` when
         unspecified.
     """
-    match normalized_str(value):
-        case 'after_transform':
-            return 'after_transform'
-        case _:
-            return 'before_transform'
+    return cast(
+        ValidationPhase,
+        _normalize_choice(
+            value,
+            mapping=_PHASE_CHOICES,
+            default='before_transform',
+        ),
+    )
 
 
 def _normalize_severity(
@@ -314,7 +344,14 @@ def _normalize_severity(
     ValidationSeverity
         Normalized severity. Defaults to ``"error"`` when unspecified.
     """
-    return 'warn' if normalized_str(value) == 'warn' else 'error'
+    return cast(
+        ValidationSeverity,
+        _normalize_choice(
+            value,
+            mapping=_SEVERITY_CHOICES,
+            default='error',
+        ),
+    )
 
 
 def _normalize_window(
@@ -333,13 +370,41 @@ def _normalize_window(
     ValidationWindow
         Normalized validation window. Defaults to ``"both"`` when unspecified.
     """
-    match normalized_str(value):
-        case 'before_transform':
-            return 'before_transform'
-        case 'after_transform':
-            return 'after_transform'
-        case _:
-            return 'both'
+    return cast(
+        ValidationWindow,
+        _normalize_choice(
+            value,
+            mapping=_WINDOW_CHOICES,
+            default='both',
+        ),
+    )
+
+
+def _normalize_choice(
+    value: str | None,
+    *,
+    mapping: Mapping[str, str],
+    default: str,
+) -> str:
+    """
+    Normalize a text value against a mapping with a default fallback.
+
+    Parameters
+    ----------
+    value : str | None
+        Input text to normalize.
+    mapping : Mapping[str, str]
+        Mapping of accepted values to normalized outputs.
+    default : str
+        Default to return when input is missing or unrecognized.
+
+    Returns
+    -------
+    str
+        Normalized value.
+    """
+    normalized = normalized_str(value)
+    return mapping.get(normalized, default)
 
 
 def _rule_name(

etlplus/{validate.py → ops/validate.py}

@@ -1,5 +1,5 @@
 """
-:mod:`etlplus.validation` module.
+:mod:`etlplus.ops.validate` module.
 
 Validate dicts and lists of dicts using simple, schema-like rules.
 
@@ -34,11 +34,11 @@ from typing import Final
 from typing import Literal
 from typing import TypedDict
 
+from ..types import JSONData
+from ..types import Record
+from ..types import StrAnyMap
+from ..types import StrPath
 from .load import load_data
-from .types import JSONData
-from .types import Record
-from .types import StrAnyMap
-from .types import StrPath
 
 # SECTION: EXPORTS ========================================================== #
 
@@ -279,11 +279,15 @@ def _type_matches(
     bool
         ``True`` if the value matches the expected type; ``False`` if not.
     """
-    py_type = TYPE_MAP.get(expected)
-    if py_type:
-        return isinstance(value, py_type)
+    if expected == 'number':
+        return _is_number(value)
+    if expected == 'integer':
+        return isinstance(value, int) and not isinstance(value, bool)
+    if expected == 'boolean':
+        return isinstance(value, bool)
 
-    return False
+    py_type = TYPE_MAP.get(expected)
+    return isinstance(value, py_type) if py_type else False
 
 
 def _validate_record(
@@ -330,6 +334,9 @@ def _validate_record(
 # SECTION: FUNCTIONS ======================================================== #
 
 
+# -- Helpers -- #
+
+
 def validate_field(
     value: Any,
     rules: StrAnyMap | FieldRules,
@@ -425,6 +432,9 @@ def validate_field(
     return {'valid': len(errors) == 0, 'errors': errors}
 
 
+# -- Orchestration -- #
+
+
 def validate(
     source: StrPath | JSONData,
     rules: RulesMap | None = None,

etlplus/types.py

@@ -193,8 +193,8 @@ type AggregateSpec = StrAnyMap
 
 # -- Pipelines-- #
 
-# Unified pipeline step spec consumed by :mod:`etlplus.transform`.
-type StepSpec = FilterSpec | MapSpec | SelectSpec | SortSpec | AggregateSpec
+# Unified pipeline step spec consumed by :mod:`etlplus.ops.transform`.
+type StepSpec = AggregateSpec | FilterSpec | MapSpec | SelectSpec | SortSpec
 
 # Collections of steps
 

{etlplus-0.12.10.dist-info → etlplus-0.14.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: etlplus
-Version: 0.12.10
+Version: 0.14.3
 Summary: A Swiss Army knife for simple ETL operations
 Home-page: https://github.com/Dagitali/ETLPlus
 Author: ETLPlus Team
@@ -79,8 +79,10 @@ package and command-line interface for data extraction, validation, transformati
       - [Binary Serialization and Interchange](#binary-serialization-and-interchange)
       - [Databases and Embedded Storage](#databases-and-embedded-storage)
       - [Spreadsheets](#spreadsheets)
-      - [Data Archives](#data-archives)
+      - [Statistical / Scientific / Numeric Computing](#statistical--scientific--numeric-computing)
       - [Logs and Event Streams](#logs-and-event-streams)
+      - [Data Archives](#data-archives)
+      - [Templates](#templates)
   - [Usage](#usage)
     - [Command Line Interface](#command-line-interface)
       - [Argument Order and Required Options](#argument-order-and-required-options)
@@ -194,7 +196,7 @@ etlplus extract file examples/data/sample.csv \
 [Python API](#python-api):
 
 ```python
-from etlplus import extract, transform, validate, load
+from etlplus.ops import extract, transform, validate, load
 
 data = extract("file", "input.csv")
 ops = {"filter": {"field": "age", "op": "gt", "value": 25}, "select": ["name", "email"]}
@@ -221,93 +223,122 @@ DDL can be rendered from table specs for migrations or schema checks.
 
 ### Files (`file`)
 
-File formats are grouped as in `FileFormat`. Support is marked as:
+Recognized file formats are listed in the tables below. Support for reading to or writing from a recognized file format is marked as:
 
 - **Y**: implemented (may require optional dependencies)
 - **N**: stubbed or not yet implemented
 
 #### Stubbed / Placeholder
 
-| Format | Supported | Description |
-| --- | --- | --- |
+| Format | Read | Write | Description |
+| --- | --- | --- | --- |
 | `stub` | N | Placeholder format for tests and future connectors. |
 
 #### Tabular & Delimited Text
 
-| Format | Supported | Description |
-| --- | --- | --- |
-| `csv` | Y | Comma-Separated Values |
-| `fwf` | N | Fixed-Width Fields |
-| `dat` | N | Generic data file, often delimited or fixed-width |
-| `psv` | N | Pipe-Separated Values |
-| `tab` | N | Often synonymous with TSV |
-| `tsv` | Y | Tab-Separated Values |
-| `txt` | Y | Plain text, often delimited or fixed-width |
+| Format | Read | Write | Description |
+| --- | --- | --- | --- |
+| `csv` | Y | Y | Comma-Separated Values |
+| `dat` | N | N | Generic data file, often delimited or fixed-width |
+| `fwf` | N | N | Fixed-Width Fields |
+| `psv` | N | N | Pipe-Separated Values |
+| `tab` | N | N | Often synonymous with TSV |
+| `tsv` | Y | Y | Tab-Separated Values |
+| `txt` | Y | Y | Plain text, often delimited or fixed-width |
 
 #### Semi-Structured Text
 
-| Format | Supported | Description |
-| --- | --- | --- |
-| `cfg` | N | Config-style key-value pairs |
-| `conf` | N | Config-style key-value pairs |
-| `ini` | N | Config-style key-value pairs |
-| `json` | Y | JavaScript Object Notation |
-| `ndjson` | Y | Newline-Delimited JSON |
-| `properties` | N | Java-style key-value pairs |
-| `toml` | N | Tom's Obvious Minimal Language |
-| `xml` | Y | Extensible Markup Language |
-| `yaml` | Y | YAML Ain't Markup Language |
+| Format | Read | Write | Description |
+| --- | --- | --- | --- |
+| `cfg` | N | N | Config-style key-value pairs |
+| `conf` | N | N | Config-style key-value pairs |
+| `ini` | N | N | Config-style key-value pairs |
+| `json` | Y | Y | JavaScript Object Notation |
+| `ndjson` | Y | Y | Newline-Delimited JSON |
+| `properties` | N | N | Java-style key-value pairs |
+| `toml` | N | N | Tom's Obvious Minimal Language |
+| `xml` | Y | Y | Extensible Markup Language |
+| `yaml` | Y | Y | YAML Ain't Markup Language |
 
 #### Columnar / Analytics-Friendly
 
-| Format | Supported | Description |
-| --- | --- | --- |
-| `arrow` | N | Apache Arrow IPC |
-| `feather` | Y | Apache Arrow Feather |
-| `orc` | Y | Optimized Row Columnar; common in Hadoop |
-| `parquet` | Y | Apache Parquet; common in Big Data |
+| Format | Read | Write | Description |
+| --- | --- | --- | --- |
+| `arrow` | N | N | Apache Arrow IPC |
+| `feather` | Y | Y | Apache Arrow Feather |
+| `orc` | Y | Y | Optimized Row Columnar; common in Hadoop |
+| `parquet` | Y | Y | Apache Parquet; common in Big Data |
 
 #### Binary Serialization and Interchange
 
-| Format | Supported | Description |
-| --- | --- | --- |
-| `avro` | Y | Apache Avro |
-| `bson` | N | Binary JSON; common with MongoDB exports/dumps |
-| `cbor` | N | Concise Binary Object Representation |
-| `ion` | N | Amazon Ion |
-| `msgpack` | N | MessagePack |
-| `pb` | N | Protocol Buffers (Google Protobuf) |
-| `pbf` | N | Protocolbuffer Binary Format; often for GIS data |
-| `proto` | N | Protocol Buffers schema; often in .pb / .bin |
+| Format | Read | Write | Description |
+| --- | --- | --- | --- |
+| `avro` | Y | Y | Apache Avro |
+| `bson` | N | N | Binary JSON; common with MongoDB exports/dumps |
+| `cbor` | N | N | Concise Binary Object Representation |
+| `ion` | N | N | Amazon Ion |
+| `msgpack` | N | N | MessagePack |
+| `pb` | N | N | Protocol Buffers (Google Protobuf) |
+| `pbf` | N | N | Protocolbuffer Binary Format; often for GIS data |
+| `proto` | N | N | Protocol Buffers schema; often in .pb / .bin |
 
 #### Databases and Embedded Storage
 
-| Format | Supported | Description |
-| --- | --- | --- |
-| `accdb` | N | Microsoft Access database file (newer format) |
-| `duckdb` | N | DuckDB database file |
-| `mdb` | N | Microsoft Access database file (older format) |
-| `sqlite` | N | SQLite database file |
+| Format | Read | Write | Description |
+| --- | --- | --- | --- |
+| `accdb` | N | N | Microsoft Access (newer format) |
+| `duckdb` | N | N | DuckDB |
+| `mdb` | N | N | Microsoft Access (older format) |
+| `sqlite` | N | N | SQLite |
 
 #### Spreadsheets
 
+| Format | Read | Write | Description |
+| --- | --- | --- | --- |
+| `numbers` | N | N | Apple Numbers |
+| `ods` | N | N | OpenDocument |
+| `wks` | N | N | Lotus 1-2-3  |
+| `xls` | Y | Y | Microsoft Excel (BIFF) |
+| `xlsm` | N | N | Microsoft Excel Macro-Enabled (Open XML) |
+| `xlsx` | Y | Y | Microsoft Excel (Open XML) |
+
+#### Statistical / Scientific / Numeric Computing
+
+| Format | Read | Write | Description |
+| --- | --- | --- | --- |
+| `dta` | N | N | Stata |
+| `hdf5` | N | N | Hierarchical Data Format |
+| `mat` | N | N | MATLAB |
+| `nc` | N | N | NetCDF |
+| `rda` | N | N | RData workspace/object |
+| `rds` | N | N | R data |
+| `sas7bdat` | N | N | SAS data |
+| `sav` | N | N | SPSS data |
+| `sylk` | N | N | Symbolic Link |
+| `xpt` | N | N | SAS Transport |
+| `zsav` | N | N | Compressed SPSS data |
+
+#### Logs and Event Streams
+
 | Format | Supported | Description |
 | --- | --- | --- |
-| `xls` | Y | Microsoft Excel (BIFF); read-only |
-| `xlsx` | Y | Microsoft Excel (Open XML) |
+| `log` | N | N | Generic log file |
 
 #### Data Archives
 
-| Format | Supported | Description |
-| --- | --- | --- |
-| `gz` | Y | Gzip-compressed file |
-| `zip` | Y | ZIP archive |
+| Format | Read | Write | Description |
+| --- | --- | --- | --- |
+| `gz` | Y | Y | Gzip-compressed file |
+| `zip` | Y | Y | ZIP archive |
 
-#### Logs and Event Streams
+#### Templates
 
-| Format | Supported | Description |
-| --- | --- | --- |
-| `log` | N | Generic log file |
+| Format | Read | Write | Description |
+| --- | --- | --- | --- |
+| `hbs` | N | N | Handlebars |
+| `jinja2` | N | N | Jinja2 |
+| `mustache` | N | N | Mustache |
+| `vm` | N | N | Apache Velocity |
 
 ## Usage
 
@@ -500,7 +531,7 @@ cat examples/data/sample.json \
 Use ETLPlus as a Python library:
 
 ```python
-from etlplus import extract, validate, transform, load
+from etlplus.ops import extract, validate, transform, load
 
 # Extract data
 data = extract("file", "data.json")
@@ -695,7 +726,7 @@ We split tests into two layers:
   pagination + rate limit defaults, file/API connector interactions) may touch temp files and use
   fake clients.
 
-If a test calls `etlplus.cli.main()` or `etlplus.run.run()` it’s integration by default.  Full
+If a test calls `etlplus.cli.main()` or `etlplus.ops.run.run()` it’s integration by default. Full
 criteria: [`CONTRIBUTING.md#testing`](CONTRIBUTING.md#testing).
 
 ### Code Coverage