etlplus 0.12.12__py3-none-any.whl → 0.15.0__py3-none-any.whl

etlplus/config/README.md

@@ -1,52 +1,50 @@
-# etlplus.config subpackage
+# `etlplus.config` Subpackage
 
-Documentation for the `etlplus.config` subpackage: configuration helpers for connectors, pipelines,
-jobs, and profiles.
+Documentation for the `etlplus.config` subpackage: type definitions and config shape helpers for
+ETLPlus.
 
-- Provides classes and utilities for managing ETL pipeline configuration
-- Supports YAML/JSON config loading and validation
-- Includes helpers for connectors, jobs, pipelines, and profiles
-- Exposes type definitions for config schemas
+- Exposes TypedDict-based config schemas for API profiles and endpoints
+- Provides exported type aliases for API configuration maps
+- Designed for Python 3.13 typing and editor assistance (runtime parsing lives elsewhere)
 
 Back to project overview: see the top-level [README](../../README.md).
 
-- [etlplus.config subpackage](#etlplusconfig-subpackage)
-  - [Supported Configuration Types](#supported-configuration-types)
-  - [Loading and Validating Configs](#loading-and-validating-configs)
-  - [Example: Loading a Pipeline Config](#example-loading-a-pipeline-config)
+- [`etlplus.config` Subpackage](#etlplusconfig-subpackage)
+  - [Modules](#modules)
+  - [Exported Types](#exported-types)
+  - [Example: Typing an API Config](#example-typing-an-api-config)
   - [See Also](#see-also)
 
-## Supported Configuration Types
+## Modules
 
-- **Connector**: Connection details for databases, files, or APIs
-- **Job**: ETL job definitions and scheduling
-- **Pipeline**: End-to-end pipeline configuration
-- **Profile**: User or environment-specific settings
+- `etlplus.config.__init__`: package exports and high-level package notes
+- `etlplus.config.types`: TypedDict-based config schemas
 
-## Loading and Validating Configs
+## Exported Types
 
-Use the provided classes to load and validate configuration files:
+- `ApiConfigMap`: top-level API config shape
+- `ApiProfileConfigMap`: per-profile API config shape
+- `ApiProfileDefaultsMap`: defaults block within a profile
+- `EndpointMap`: endpoint config shape
 
-```python
-from etlplus.config import PipelineConfig
-
-cfg = PipelineConfig.from_yaml("pipeline.yml")
-```
-
-- Supports YAML and JSON formats
-- Validates against expected schema
-
-## Example: Loading a Pipeline Config
+## Example: Typing an API Config
 
 ```python
-from etlplus.config import PipelineConfig
-
-pipeline = PipelineConfig.from_yaml("configs/pipeline.yml")
-print(pipeline)
+from etlplus.config import ApiConfigMap
+
+api_cfg: ApiConfigMap = {
+    "base_url": "https://example.test",
+    "headers": {"Authorization": "Bearer token"},
+    "endpoints": {
+        "users": {
+            "path": "/users",
+            "method": "GET",
+        },
+    },
+}
 ```
 
 ## See Also
 
 - Top-level CLI and library usage in the main [README](../../README.md)
 - Config type definitions in [types.py](types.py)
-- Config utilities in [utils.py](utils.py)

etlplus/config/__init__.py

@@ -16,41 +16,18 @@ Notes
 
 from __future__ import annotations
 
-from .connector import Connector
-from .connector import ConnectorApi
-from .connector import ConnectorDb
-from .connector import ConnectorFile
-from .connector import parse_connector
-from .jobs import ExtractRef
-from .jobs import JobConfig
-from .jobs import LoadRef
-from .jobs import TransformRef
-from .jobs import ValidationRef
-from .pipeline import PipelineConfig
-from .pipeline import load_pipeline_config
-from .profile import ProfileConfig
-from .types import ConnectorType
+from .types import ApiConfigMap
+from .types import ApiProfileConfigMap
+from .types import ApiProfileDefaultsMap
+from .types import EndpointMap
 
 # SECTION: EXPORTS ========================================================== #
 
 
 __all__ = [
-    # Connectors
-    'Connector',
-    'ConnectorType',
-    'ConnectorApi',
-    'ConnectorDb',
-    'ConnectorFile',
-    'parse_connector',
-    # Jobs / Refs
-    'ExtractRef',
-    'JobConfig',
-    'LoadRef',
-    'TransformRef',
-    'ValidationRef',
-    # Pipeline
-    'PipelineConfig',
-    'load_pipeline_config',
-    # Profile
-    'ProfileConfig',
+    # Typed Dicts
+    'ApiConfigMap',
+    'ApiProfileConfigMap',
+    'ApiProfileDefaultsMap',
+    'EndpointMap',
 ]

etlplus/config/types.py

@@ -33,7 +33,6 @@ from __future__ import annotations
 
 from collections.abc import Mapping
 from typing import Any
-from typing import Literal
 from typing import TypedDict
 
 from ..api import PaginationConfigMap
@@ -44,26 +43,17 @@ from ..types import StrAnyMap
 
 
 __all__ = [
-    # Type aliases
-    'ConnectorType',
-    # 'PaginationType',
     # TypedDicts
     'ApiProfileDefaultsMap',
     'ApiProfileConfigMap',
     'ApiConfigMap',
     'EndpointMap',
-    'ConnectorApiConfigMap',
-    'ConnectorDbConfigMap',
-    'ConnectorFileConfigMap',
 ]
 
 
 # SECTION: TYPE ALIASES ===================================================== #
 
 
-# Literal type for supported connector kinds
-type ConnectorType = Literal['api', 'database', 'file']
-
 # Literal type for supported pagination kinds
 # type PaginationType = Literal['page', 'offset', 'cursor']
 
@@ -129,60 +119,6 @@ class ApiProfileDefaultsMap(TypedDict, total=False):
     rate_limit: RateLimitConfigMap | StrAnyMap
 
 
-class ConnectorApiConfigMap(TypedDict, total=False):
-    """
-    Shape accepted by ConnectorApi.from_obj (all keys optional).
-
-    See Also
-    --------
-    - etlplus.config.connector.ConnectorApi.from_obj
-    """
-
-    name: str
-    type: ConnectorType
-    url: str
-    method: str
-    headers: StrAnyMap
-    query_params: StrAnyMap
-    pagination: PaginationConfigMap
-    rate_limit: RateLimitConfigMap
-    api: str
-    endpoint: str
-
-
-class ConnectorDbConfigMap(TypedDict, total=False):
-    """
-    Shape accepted by ConnectorDb.from_obj (all keys optional).
-
-    See Also
-    --------
-    - etlplus.config.connector.ConnectorDb.from_obj
-    """
-
-    name: str
-    type: ConnectorType
-    connection_string: str
-    query: str
-    table: str
-    mode: str
-
-
-class ConnectorFileConfigMap(TypedDict, total=False):
-    """
-    Shape accepted by ConnectorFile.from_obj (all keys optional).
-
-    See Also
-    --------
-    - etlplus.config.connector.ConnectorFile.from_obj
-    """
-
-    name: str
-    type: ConnectorType
-    format: str
-    path: str
-    options: StrAnyMap
-
-
 class EndpointMap(TypedDict, total=False):
     """
     Shape accepted by EndpointConfig.from_obj.

etlplus/dag.py

@@ -0,0 +1,103 @@
+"""
+:mod:`etlplus.dag` module.
+
+Lightweight directed acyclic graph (DAG) helpers for ordering jobs based on
+``depends_on``.
+"""
+
+from __future__ import annotations
+
+from collections import deque
+from dataclasses import dataclass
+
+from .config.jobs import JobConfig
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'DagError',
+    'topological_sort_jobs',
+]
+
+
+# SECTION: ERRORS =========================================================== #
+
+
+@dataclass(slots=True)
+class DagError(ValueError):
+    """
+    Raised when the job dependency graph is invalid.
+
+    Attributes
+    ----------
+    message : str
+        Error message.
+    """
+
+    # -- Attributes -- #
+
+    message: str
+
+    # -- Magic Methods (Object Representation) -- #
+
+    def __str__(self) -> str:
+        return self.message
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def topological_sort_jobs(
+    jobs: list[JobConfig],
+) -> list[JobConfig]:
+    """
+    Return jobs in topological order based on ``depends_on``.
+
+    Parameters
+    ----------
+    jobs : list[JobConfig]
+        List of job configurations to sort.
+
+    Returns
+    -------
+    list[JobConfig]
+        Jobs sorted in topological order.
+
+    Raises
+    ------
+    DagError
+        If a dependency is missing, self-referential, or when a cycle is
+        detected.
+    """
+    index = {job.name: job for job in jobs}
+    edges: dict[str, set[str]] = {name: set() for name in index}
+    indegree: dict[str, int] = {name: 0 for name in index}
+
+    for job in jobs:
+        for dep in job.depends_on:
+            if dep not in index:
+                raise DagError(
+                    f'Unknown dependency "{dep}" in job "{job.name}"',
+                )
+            if dep == job.name:
+                raise DagError(f'Job "{job.name}" depends on itself')
+            if job.name not in edges[dep]:
+                edges[dep].add(job.name)
+                indegree[job.name] += 1
+
+    queue = deque(sorted(name for name, deg in indegree.items() if deg == 0))
+    ordered: list[str] = []
+
+    while queue:
+        name = queue.popleft()
+        ordered.append(name)
+        for child in sorted(edges[name]):
+            indegree[child] -= 1
+            if indegree[child] == 0:
+                queue.append(child)
+
+    if len(ordered) != len(jobs):
+        raise DagError('Dependency cycle detected')
+
+    return [index[name] for name in ordered]

etlplus/database/README.md

@@ -1,4 +1,4 @@
-# etlplus.database subpackage
+# `etlplus.database` Subpackage
 
 Documentation for the `etlplus.database` subpackage: database engine, schema, and ORM helpers.
 
@@ -9,7 +9,7 @@ Documentation for the `etlplus.database` subpackage: database engine, schema, an
 
 Back to project overview: see the top-level [README](../../README.md).
 
-- [etlplus.database subpackage](#etlplusdatabase-subpackage)
+- [`etlplus.database` Subpackage](#etlplusdatabase-subpackage)
   - [Database Engine and Connections](#database-engine-and-connections)
   - [Schema and DDL Helpers](#schema-and-ddl-helpers)
   - [ORM Utilities](#orm-utilities)

etlplus/enums.py

@@ -23,7 +23,6 @@ __all__ = [
     'AggregateName',
     'CoercibleStrEnum',
     'DataConnectorType',
-    'HttpMethod',
     'OperatorName',
     'PipelineStep',
 ]
@@ -200,37 +199,6 @@ class DataConnectorType(CoercibleStrEnum):
         }
 
 
-class HttpMethod(CoercibleStrEnum):
-    """Supported HTTP verbs that accept JSON payloads."""
-
-    # -- Constants -- #
-
-    CONNECT = 'connect'
-    DELETE = 'delete'
-    GET = 'get'
-    HEAD = 'head'
-    OPTIONS = 'options'
-    PATCH = 'patch'
-    POST = 'post'
-    PUT = 'put'
-    TRACE = 'trace'
-
-    # -- Getters -- #
-
-    @property
-    def allows_body(self) -> bool:
-        """
-        Whether the method typically allows a request body.
-
-        Notes
-        -----
-        - RFCs do not strictly forbid bodies on some other methods (e.g.,
-            ``DELETE``), but many servers/clients do not expect them. We mark
-            ``POST``, ``PUT``, and ``PATCH`` as True.
-        """
-        return self in {HttpMethod.POST, HttpMethod.PUT, HttpMethod.PATCH}
-
-
 class OperatorName(CoercibleStrEnum):
     """Supported comparison operators with helpers."""
 

etlplus/file/README.md

@@ -1,4 +1,4 @@
-# etlplus.file subpackage
+# `etlplus.file` Subpackage
 
 Documentation for the `etlplus.file` subpackage: unified file format support and helpers for reading
 and writing data files.
@@ -11,7 +11,7 @@ and writing data files.
 
 Back to project overview: see the top-level [README](../../README.md).
 
-- [etlplus.file subpackage](#etlplusfile-subpackage)
+- [`etlplus.file` Subpackage](#etlplusfile-subpackage)
   - [Supported File Formats](#supported-file-formats)
   - [Inferring File Format and Compression](#inferring-file-format-and-compression)
   - [Reading and Writing Files](#reading-and-writing-files)

etlplus/file/enums.py

@@ -123,7 +123,7 @@ class FileFormat(CoercibleStrEnum):
     RDS = 'rds'  # R data file
     SAS7BDAT = 'sas7bdat'  # SAS data file
     SAV = 'sav'  # SPSS data file
-    SYLK = 'sylk'  # Symbolic Link (SYmbolic LinK)
+    SYLK = 'sylk'  # Symbolic Link
     XPT = 'xpt'  # SAS Transport file
     ZSAV = 'zsav'  # Compressed SPSS data file
 

etlplus/{validation → ops}/README.md

@@ -1,4 +1,4 @@
-# etlplus.validation subpackage
+# etlplus.ops subpackage
 
 Documentation for the `etlplus.validation` subpackage: data validation utilities and helpers.
 
@@ -8,7 +8,7 @@ Documentation for the `etlplus.validation` subpackage: data validation utilities
 
 Back to project overview: see the top-level [README](../../README.md).
 
-- [etlplus.validation subpackage](#etlplusvalidation-subpackage)
+- [etlplus.ops subpackage](#etlplusops-subpackage)
   - [Validation Features](#validation-features)
   - [Defining Validation Rules](#defining-validation-rules)
   - [Example: Validating Data](#example-validating-data)

etlplus/ops/__init__.py

@@ -0,0 +1,61 @@
+"""
+:mod:`etlplus.ops` package.
+
+Data operations helpers.
+
+Importing :mod:`etlplus.ops` exposes the coarse-grained helpers most users care
+about: ``extract``, ``transform``, ``load``, ``validate``, ``run``, and
+``run_pipeline``. Each helper delegates to the richer modules under
+``etlplus.ops.*`` while presenting a compact public API surface. Conditional
+validation orchestration is available via
+:func:`etlplus.ops.utils.maybe_validate`. The legacy compatibility module
+:mod:`etlplus.ops.__init__validation` is deprecated in favor of this package.
+
+Examples
+--------
+>>> from etlplus.ops import extract, transform
+>>> raw = extract('file', 'input.json')
+>>> curated = transform(raw, {'select': ['id', 'name']})
+
+>>> from etlplus.ops.utils import maybe_validate
+>>> payload = {'name': 'Alice'}
+>>> rules = {'required': ['name']}
+>>> def validator(data, config):
+...     missing = [field for field in config['required'] if field not in data]
+...     return {'valid': not missing, 'errors': missing, 'data': data}
+>>> maybe_validate(
+...     payload,
+...     when='both',
+...     enabled=True,
+...     rules=rules,
+...     phase='before_transform',
+...     severity='warn',
+...     validate_fn=validator,
+...     print_json_fn=lambda message: message,
+... )
+{'name': 'Alice'}
+
+See Also
+--------
+:mod:`etlplus.ops.run`
+:mod:`etlplus.ops.utils`
+"""
+
+from .extract import extract
+from .load import load
+from .run import run
+from .run import run_pipeline
+from .transform import transform
+from .validate import validate
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'extract',
+    'load',
+    'run',
+    'run_pipeline',
+    'transform',
+    'validate',
+]