etlplus 0.10.4__py3-none-any.whl → 0.12.2__py3-none-any.whl

etlplus/README.md

@@ -0,0 +1,37 @@
+# etlplus package
+
+The `etlplus` package provides a unified Python API and CLI for ETL operations: extraction,
+validation, transformation, and loading of data from files, APIs, and databases.
+
+- Top-level entry points for extract, validate, transform, and load
+- Utilities for pipeline orchestration and helpers
+- Exposes all subpackages for advanced usage
+
+Back to project overview: see the top-level [README](../README.md).
+
+## Subpackages
+
+- [etlplus.api](api/README.md): Lightweight HTTP client and paginated REST helpers
+- [etlplus.file](file/README.md): Unified file format support and helpers
+- [etlplus.config](config/README.md): Configuration helpers for connectors, pipelines, jobs, and
+  profiles
+- [etlplus.cli](cli/README.md): Command-line interface for ETLPlus workflows
+- [etlplus.database](database/README.md): Database engine, schema, and ORM helpers
+- [etlplus.templates](templates/README.md): SQL and DDL template helpers
+- [etlplus.validation](validation/README.md): Data validation utilities and helpers
+
+## Quickstart
+
+```python
+from etlplus import extract, validate, transform, load
+
+data = extract("file", "input.csv")
+filtered = transform(data, {"filter": {"field": "age", "op": "gt", "value": 25}})
+assert validate(filtered, {"age": {"type": "number", "min": 0}})["valid"]
+load(filtered, "file", "output.json", file_format="json")
+```
+
+## See Also
+
+- [Top-level project README](../README.md)
+- [API reference](../docs/README.md)

etlplus/api/README.md

@@ -1,7 +1,7 @@
-# etlplus.api module.
+# etlplus.api subpackage
 
-Focused documentation for the `etlplus.api` subpackage: a lightweight HTTP client and helpers for
-paginated REST endpoints.
+Documentation for the `etlplus.api` subpackage: a lightweight HTTP client and helpers for paginated
+REST endpoints.
 
 - Provides a small `EndpointClient` for calling JSON APIs
 - Supports page-, offset-, and cursor-based pagination via `PaginationConfig`
@@ -12,6 +12,20 @@ paginated REST endpoints.
 
 Back to project overview: see the top-level [README](../../README.md).
 
+- [etlplus.api subpackage](#etlplusapi-subpackage)
+  - [Installation](#installation)
+  - [Quickstart](#quickstart)
+    - [Overriding Rate Limits Per Call](#overriding-rate-limits-per-call)
+  - [Choosing `records_path` and `cursor_path`](#choosing-records_path-and-cursor_path)
+  - [Cursor-Based Pagination Example](#cursor-based-pagination-example)
+  - [Offset-based pagination example](#offset-based-pagination-example)
+  - [Authentication](#authentication)
+  - [Errors and Rate Limiting](#errors-and-rate-limiting)
+  - [Types and Transport](#types-and-transport)
+  - [Supporting Modules](#supporting-modules)
+  - [Minimal Contract](#minimal-contract)
+  - [See also](#see-also)
+
 ## Installation
 
 `etlplus.api` ships as part of the `etlplus` package. Install the package as usual:
@@ -233,3 +247,6 @@ providers can fall back to their own defaults. If you already possess a static t
 ## See also
 
 - Top-level CLI and library usage in the main [README](../../README.md)
+
+
+[def]: #installation

etlplus/cli/README.md

@@ -0,0 +1,40 @@
+# etlplus.cli subpackage
+
+Documentation for the `etlplus.cli` subpackage: command-line interface for ETLPlus workflows.
+
+- Provides a CLI for running ETL pipelines, jobs, and utilities
+- Supports commands for running, validating, and inspecting pipelines
+- Includes options for configuration, state, and output control
+- Exposes handlers for custom command integration
+
+Back to project overview: see the top-level [README](../../README.md).
+
+- [etlplus.cli subpackage](#etlpluscli-subpackage)
+  - [Available Commands](#available-commands)
+  - [Command Options](#command-options)
+  - [Example: Running a Pipeline](#example-running-a-pipeline)
+  - [See Also](#see-also)
+
+## Available Commands
+
+- **run**: Execute a pipeline or job
+- **validate**: Validate pipeline or config files
+- **inspect**: Show pipeline/job details
+
+## Command Options
+
+- `--config`: Path to config file
+- `--state`: Path to state file
+- `--output`: Output file or format
+
+## Example: Running a Pipeline
+
+```bash
+etlplus run --config configs/pipeline.yml --output results.json
+```
+
+## See Also
+
+- Top-level CLI and library usage in the main [README](../../README.md)
+- Command handlers in [handlers.py](handlers.py)
+- Command options in [options.py](options.py)

etlplus/cli/commands.py

@@ -36,7 +36,7 @@ from typing import cast
 import typer
 
 from .. import __version__
-from ..enums import FileFormat
+from ..file import FileFormat
 from . import handlers
 from .constants import CLI_DESCRIPTION
 from .constants import CLI_EPILOG

etlplus/cli/constants.py

@@ -9,7 +9,7 @@ from __future__ import annotations
 from typing import Final
 
 from ..enums import DataConnectorType
-from ..enums import FileFormat
+from ..file import FileFormat
 
 # SECTION: EXPORTS ========================================================== #
 

etlplus/cli/handlers.py

@@ -570,7 +570,7 @@ def transform_handler(
     data = transform(payload, cast(TransformOperations, operations_payload))
 
     if target and target != '-':
-        File.write_file(target, data, file_format=target_format)
+        File(target, file_format=target_format).write(data)
         print(f'Data transformed and saved to {target}')
         return 0
 

etlplus/cli/io.py

@@ -15,8 +15,8 @@ from pathlib import Path
 from typing import Any
 from typing import cast
 
-from ..enums import FileFormat
 from ..file import File
+from ..file import FileFormat
 from ..types import JSONData
 from ..utils import print_json
 
@@ -331,6 +331,6 @@ def write_json_output(
     """
     if not output_path or output_path == '-':
         return False
-    File(Path(output_path), FileFormat.JSON).write_json(data)
+    File(Path(output_path), FileFormat.JSON).write(data)
     print(f'{success_message} {output_path}')
     return True

etlplus/config/README.md

@@ -0,0 +1,52 @@
+# etlplus.config subpackage
+
+Documentation for the `etlplus.config` subpackage: configuration helpers for connectors, pipelines,
+jobs, and profiles.
+
+- Provides classes and utilities for managing ETL pipeline configuration
+- Supports YAML/JSON config loading and validation
+- Includes helpers for connectors, jobs, pipelines, and profiles
+- Exposes type definitions for config schemas
+
+Back to project overview: see the top-level [README](../../README.md).
+
+- [etlplus.config subpackage](#etlplusconfig-subpackage)
+  - [Supported Configuration Types](#supported-configuration-types)
+  - [Loading and Validating Configs](#loading-and-validating-configs)
+  - [Example: Loading a Pipeline Config](#example-loading-a-pipeline-config)
+  - [See Also](#see-also)
+
+## Supported Configuration Types
+
+- **Connector**: Connection details for databases, files, or APIs
+- **Job**: ETL job definitions and scheduling
+- **Pipeline**: End-to-end pipeline configuration
+- **Profile**: User or environment-specific settings
+
+## Loading and Validating Configs
+
+Use the provided classes to load and validate configuration files:
+
+```python
+from etlplus.config import PipelineConfig
+
+cfg = PipelineConfig.from_yaml("pipeline.yml")
+```
+
+- Supports YAML and JSON formats
+- Validates against expected schema
+
+## Example: Loading a Pipeline Config
+
+```python
+from etlplus.config import PipelineConfig
+
+pipeline = PipelineConfig.from_yaml("configs/pipeline.yml")
+print(pipeline)
+```
+
+## See Also
+
+- Top-level CLI and library usage in the main [README](../../README.md)
+- Config type definitions in [types.py](types.py)
+- Config utilities in [utils.py](utils.py)

etlplus/config/pipeline.py

@@ -24,8 +24,8 @@ from typing import Any
 from typing import Self
 
 from ..api import ApiConfig
-from ..enums import FileFormat
 from ..file import File
+from ..file import FileFormat
 from ..types import StrAnyMap
 from ..utils import coerce_dict
 from ..utils import maybe_mapping
@@ -246,7 +246,7 @@ class PipelineConfig:
         TypeError
             If the YAML root is not a mapping/object.
         """
-        raw = File(Path(path), FileFormat.YAML).read_yaml()
+        raw = File(Path(path), FileFormat.YAML).read()
         if not isinstance(raw, dict):
             raise TypeError('Pipeline YAML must have a mapping/object root')
 

etlplus/database/README.md

@@ -0,0 +1,48 @@
+# etlplus.database subpackage
+
+Documentation for the `etlplus.database` subpackage: database engine, schema, and ORM helpers.
+
+- Provides database engine and connection management
+- Supports schema definition and DDL generation
+- Includes lightweight ORM utilities for tabular data
+- Exposes type definitions for database objects
+
+Back to project overview: see the top-level [README](../../README.md).
+
+- [etlplus.database subpackage](#etlplusdatabase-subpackage)
+  - [Database Engine and Connections](#database-engine-and-connections)
+  - [Schema and DDL Helpers](#schema-and-ddl-helpers)
+  - [ORM Utilities](#orm-utilities)
+  - [Example: Creating a Table](#example-creating-a-table)
+  - [See Also](#see-also)
+
+## Database Engine and Connections
+
+- Manage connections to supported databases
+- Configure engines for different backends
+
+## Schema and DDL Helpers
+
+- Define table schemas and columns
+- Generate DDL statements for supported databases
+
+## ORM Utilities
+
+- Map rows to Python objects
+- Simple CRUD helpers for tabular data
+
+## Example: Creating a Table
+
+```python
+from etlplus.database import Schema, Engine
+
+engine = Engine.connect("sqlite:///example.db")
+schema = Schema.from_dict({"name": "users", "columns": [ ... ]})
+engine.create_table(schema)
+```
+
+## See Also
+
+- Top-level CLI and library usage in the main [README](../../README.md)
+- Schema helpers in [schema.py](schema.py)
+- ORM utilities in [orm.py](orm.py)

etlplus/database/ddl.py

@@ -203,7 +203,7 @@ def load_table_spec(
         raise ValueError('Spec must be .json, .yml, or .yaml')
 
     try:
-        spec = File.read_file(spec_path)
+        spec = File(spec_path).read()
     except ImportError as e:
         if suffix in {'.yml', '.yaml'}:
             raise RuntimeError(

etlplus/database/engine.py

@@ -113,7 +113,7 @@ def load_database_url_from_config(
     ValueError
         If no connection string/URL/DSN is found for the specified entry.
     """
-    cfg = File.read_file(Path(path))
+    cfg = File(Path(path)).read()
     if not isinstance(cfg, Mapping):
         raise TypeError('Database config must be a mapping')
 

etlplus/database/schema.py

@@ -260,7 +260,7 @@ def load_table_specs(
     list[TableSpec]
         A list of TableSpec instances parsed from the YAML file.
     """
-    data = File.read_file(Path(path))
+    data = File(Path(path)).read()
     if not data:
         return []
 

etlplus/enums.py

@@ -8,7 +8,6 @@ from __future__ import annotations
 
 import enum
 import operator as _op
-from pathlib import PurePath
 from statistics import fmean
 from typing import Self
 
@@ -23,18 +22,10 @@ __all__ = [
     # Enums
     'AggregateName',
     'CoercibleStrEnum',
-    'CompressionFormat',
     'DataConnectorType',
-    'FileFormat',
     'HttpMethod',
     'OperatorName',
     'PipelineStep',
-    # Functions
-    'coerce_compression_format',
-    'coerce_data_connector_type',
-    'coerce_file_format',
-    'coerce_http_method',
-    'infer_file_format_and_compression',
 ]
 
 
@@ -178,39 +169,6 @@ class AggregateName(CoercibleStrEnum):
         return lambda xs, n: (fmean(xs) if xs else 0.0)
 
 
-class CompressionFormat(CoercibleStrEnum):
-    """Supported compression formats for data files."""
-
-    # -- Constants -- #
-
-    GZ = 'gz'
-    ZIP = 'zip'
-
-    # -- Class Methods -- #
-
-    @classmethod
-    def aliases(cls) -> StrStrMap:
-        """
-        Return a mapping of common aliases for each enum member.
-
-        Returns
-        -------
-        StrStrMap
-            A mapping of alias names to their corresponding enum member names.
-        """
-        return {
-            # File extensions
-            '.gz': 'gz',
-            '.gzip': 'gz',
-            '.zip': 'zip',
-            # MIME types
-            'application/gzip': 'gz',
-            'application/x-gzip': 'gz',
-            'application/zip': 'zip',
-            'application/x-zip-compressed': 'zip',
-        }
-
-
 class DataConnectorType(CoercibleStrEnum):
     """Supported data connector types."""
 
@@ -242,99 +200,6 @@ class DataConnectorType(CoercibleStrEnum):
         }
 
 
-class FileFormat(CoercibleStrEnum):
-    """Supported file formats for extraction."""
-
-    # -- Constants -- #
-
-    AVRO = 'avro'
-    CSV = 'csv'
-    FEATHER = 'feather'
-    GZ = 'gz'
-    JSON = 'json'
-    NDJSON = 'ndjson'
-    ORC = 'orc'
-    PARQUET = 'parquet'
-    TSV = 'tsv'
-    TXT = 'txt'
-    XLS = 'xls'
-    XLSX = 'xlsx'
-    ZIP = 'zip'
-    XML = 'xml'
-    YAML = 'yaml'
-
-    # -- Class Methods -- #
-
-    @classmethod
-    def aliases(cls) -> StrStrMap:
-        """
-        Return a mapping of common aliases for each enum member.
-
-        Returns
-        -------
-        StrStrMap
-            A mapping of alias names to their corresponding enum member names.
-        """
-        return {
-            # Common shorthand
-            'parq': 'parquet',
-            'yml': 'yaml',
-            # File extensions
-            '.avro': 'avro',
-            '.csv': 'csv',
-            '.feather': 'feather',
-            '.gz': 'gz',
-            '.json': 'json',
-            '.jsonl': 'ndjson',
-            '.ndjson': 'ndjson',
-            '.orc': 'orc',
-            '.parquet': 'parquet',
-            '.pq': 'parquet',
-            '.tsv': 'tsv',
-            '.txt': 'txt',
-            '.xls': 'xls',
-            '.xlsx': 'xlsx',
-            '.zip': 'zip',
-            '.xml': 'xml',
-            '.yaml': 'yaml',
-            '.yml': 'yaml',
-            # MIME types
-            'application/avro': 'avro',
-            'application/csv': 'csv',
-            'application/feather': 'feather',
-            'application/gzip': 'gz',
-            'application/json': 'json',
-            'application/jsonlines': 'ndjson',
-            'application/ndjson': 'ndjson',
-            'application/orc': 'orc',
-            'application/parquet': 'parquet',
-            'application/vnd.apache.avro': 'avro',
-            'application/vnd.apache.parquet': 'parquet',
-            'application/vnd.apache.arrow.file': 'feather',
-            'application/vnd.apache.orc': 'orc',
-            'application/vnd.ms-excel': 'xls',
-            (
-                'application/vnd.openxmlformats-'
-                'officedocument.spreadsheetml.sheet'
-            ): 'xlsx',
-            'application/x-avro': 'avro',
-            'application/x-csv': 'csv',
-            'application/x-feather': 'feather',
-            'application/x-orc': 'orc',
-            'application/x-ndjson': 'ndjson',
-            'application/x-parquet': 'parquet',
-            'application/x-yaml': 'yaml',
-            'application/xml': 'xml',
-            'application/zip': 'zip',
-            'text/csv': 'csv',
-            'text/plain': 'txt',
-            'text/tab-separated-values': 'tsv',
-            'text/tsv': 'tsv',
-            'text/xml': 'xml',
-            'text/yaml': 'yaml',
-        }
-
-
 class HttpMethod(CoercibleStrEnum):
     """Supported HTTP verbs that accept JSON payloads."""
 
@@ -360,8 +225,8 @@ class HttpMethod(CoercibleStrEnum):
         Notes
         -----
         - RFCs do not strictly forbid bodies on some other methods (e.g.,
-          ``DELETE``), but many servers/clients do not expect them. We mark
-          ``POST``, ``PUT``, and ``PATCH`` as True.
+            ``DELETE``), but many servers/clients do not expect them. We mark
+            ``POST``, ``PUT``, and ``PATCH`` as True.
         """
         return self in {HttpMethod.POST, HttpMethod.PUT, HttpMethod.PATCH}
 
@@ -465,13 +330,6 @@ class PipelineStep(CoercibleStrEnum):
 # SECTION: INTERNAL CONSTANTS ============================================== #
 
 
-# Compression formats that are also file formats.
-_COMPRESSION_FILE_FORMATS: set[FileFormat] = {
-    FileFormat.GZ,
-    FileFormat.ZIP,
-}
-
-
 # Precomputed order index for PipelineStep; avoids recomputing on each access.
 _PIPELINE_ORDER_INDEX: dict[PipelineStep, int] = {
     PipelineStep.FILTER: 0,
@@ -480,129 +338,3 @@ _PIPELINE_ORDER_INDEX: dict[PipelineStep, int] = {
     PipelineStep.SORT: 3,
     PipelineStep.AGGREGATE: 4,
 }
-
-
-# SECTION: FUNCTIONS ======================================================== #
-
-
-def coerce_data_connector_type(
-    connector: DataConnectorType | str,
-) -> DataConnectorType:
-    """
-    Normalize textual data connector values to :class:`DataConnectorType`.
-
-    This thin wrapper is kept for backward compatibility; prefer
-    :meth:`DataConnectorType.coerce` going forward.
-    """
-    return DataConnectorType.coerce(connector)
-
-
-def coerce_file_format(
-    file_format: FileFormat | str,
-) -> FileFormat:
-    """
-    Normalize textual file format values to :class:`FileFormat`.
-
-    This thin wrapper is kept for backward compatibility; prefer
-    :meth:`FileFormat.coerce` going forward.
-    """
-    return FileFormat.coerce(file_format)
-
-
-def coerce_compression_format(
-    compression_format: CompressionFormat | str,
-) -> CompressionFormat:
-    """
-    Normalize textual compression format values to :class:`CompressionFormat`.
-
-    This thin wrapper is kept for backward compatibility; prefer
-    :meth:`CompressionFormat.coerce` going forward.
-    """
-    return CompressionFormat.coerce(compression_format)
-
-
-def coerce_http_method(
-    http_method: HttpMethod | str,
-) -> HttpMethod:
-    """
-    Normalize textual HTTP method values to :class:`HttpMethod`.
-
-    This thin wrapper is kept for backward compatibility; prefer
-    :meth:`HttpMethod.coerce` going forward.
-    """
-    return HttpMethod.coerce(http_method)
-
-
-def infer_file_format_and_compression(
-    value: object,
-    filename: object | None = None,
-) -> tuple[FileFormat | None, CompressionFormat | None]:
-    """
-    Infer data format and compression from a filename, extension, or MIME type.
-
-    Parameters
-    ----------
-    value : object
-        A filename, extension, MIME type, or existing enum member.
-    filename : object | None, optional
-        A filename to consult for extension-based inference (e.g. when
-        ``value`` is ``application/octet-stream``).
-
-    Returns
-    -------
-    tuple[FileFormat | None, CompressionFormat | None]
-        The inferred data format and compression, if any.
-    """
-    if isinstance(value, FileFormat):
-        if value in _COMPRESSION_FILE_FORMATS:
-            return None, CompressionFormat.coerce(value.value)
-        return value, None
-    if isinstance(value, CompressionFormat):
-        return None, value
-
-    text = str(value).strip()
-    if not text:
-        return None, None
-
-    normalized = text.casefold()
-    mime = normalized.split(';', 1)[0].strip()
-
-    is_octet_stream = mime == 'application/octet-stream'
-    compression = CompressionFormat.try_coerce(mime)
-    fmt = None if is_octet_stream else FileFormat.try_coerce(mime)
-
-    is_mime = mime.startswith(
-        (
-            'application/',
-            'text/',
-            'audio/',
-            'image/',
-            'video/',
-            'multipart/',
-        ),
-    )
-    suffix_source: object | None = filename if filename is not None else text
-    if is_mime and filename is None:
-        suffix_source = None
-
-    suffixes = (
-        PurePath(str(suffix_source)).suffixes
-        if suffix_source is not None
-        else []
-    )
-    if suffixes:
-        normalized_suffixes = [suffix.casefold() for suffix in suffixes]
-        compression = (
-            CompressionFormat.try_coerce(normalized_suffixes[-1])
-            or compression
-        )
-        if compression is not None:
-            normalized_suffixes = normalized_suffixes[:-1]
-        if normalized_suffixes:
-            fmt = FileFormat.try_coerce(normalized_suffixes[-1]) or fmt
-
-    if fmt in _COMPRESSION_FILE_FORMATS:
-        compression = compression or CompressionFormat.coerce(fmt.value)
-        fmt = None
-
-    return fmt, compression

etlplus/extract.py

@@ -13,11 +13,9 @@ from typing import cast
 import requests  # type: ignore[import]
 
 from .enums import DataConnectorType
-from .enums import FileFormat
 from .enums import HttpMethod
-from .enums import coerce_data_connector_type
-from .enums import coerce_file_format
 from .file import File
+from .file import FileFormat
 from .types import JSONData
 from .types import JSONDict
 from .types import JSONList
@@ -55,7 +53,7 @@ def extract_from_file(
     # If no explicit format is provided, let File infer from extension.
     if file_format is None:
         return File(path, None).read()
-    fmt = coerce_file_format(file_format)
+    fmt = FileFormat.coerce(file_format)
 
     # Let file module perform existence and format validation.
     return File(path, fmt).read()
@@ -202,7 +200,7 @@ def extract(
     ValueError
         If `source_type` is not one of the supported values.
     """
-    match coerce_data_connector_type(source_type):
+    match DataConnectorType.coerce(source_type):
         case DataConnectorType.FILE:
             # Prefer explicit format if provided, else infer from filename.
             return extract_from_file(source, file_format)
@@ -213,6 +211,6 @@ def extract(
             # ``file_format`` is ignored for APIs.
             return extract_from_api(str(source), **kwargs)
         case _:
-            # ``coerce_data_connector_type`` covers invalid entries, but keep
-            # explicit guard for defensive programming.
+            # :meth:`coerce` already raises for invalid connector types, but
+            # keep explicit guard for defensive programming.
             raise ValueError(f'Invalid source type: {source_type}')