etlplus 0.14.3__tar.gz → 0.15.0__tar.gz

{etlplus-0.14.3/etlplus.egg-info → etlplus-0.15.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: etlplus
-Version: 0.14.3
+Version: 0.15.0
 Summary: A Swiss Army knife for simple ETL operations
 Home-page: https://github.com/Dagitali/ETLPlus
 Author: ETLPlus Team

{etlplus-0.14.3 → etlplus-0.15.0}/etlplus/README.md

@@ -1,4 +1,4 @@
-# etlplus package
+# `etlplus` Package
 
 The `etlplus` package provides a unified Python API and CLI for ETL operations: extraction,
 validation, transformation, and loading of data from files, APIs, and databases.

{etlplus-0.14.3 → etlplus-0.15.0}/etlplus/api/README.md

@@ -1,4 +1,4 @@
-# etlplus.api subpackage
+# `etlplus.api` Subpackage
 
 Documentation for the `etlplus.api` subpackage: a lightweight HTTP client and helpers for paginated
 REST endpoints.
@@ -12,7 +12,7 @@ REST endpoints.
 
 Back to project overview: see the top-level [README](../../README.md).
 
-- [etlplus.api subpackage](#etlplusapi-subpackage)
+- [`etlplus.api` Subpackage](#etlplusapi-subpackage)
   - [Installation](#installation)
   - [Quickstart](#quickstart)
     - [Overriding Rate Limits Per Call](#overriding-rate-limits-per-call)

{etlplus-0.14.3 → etlplus-0.15.0}/etlplus/api/utils.py

@@ -892,4 +892,8 @@ def resolve_request(
             'Session object must supply a callable '
             f'"{http_method.value}" method',
         )
-    return request_callable, request_timeout, http_method
+    typed_request_callable = cast(
+        Callable[..., requests.Response],
+        request_callable,
+    )
+    return typed_request_callable, request_timeout, http_method

{etlplus-0.14.3 → etlplus-0.15.0}/etlplus/cli/README.md

@@ -1,4 +1,4 @@
-# etlplus.cli subpackage
+# `etlplus.cli` Subpackage
 
 Documentation for the `etlplus.cli` subpackage: command-line interface for ETLPlus workflows.
 
@@ -9,7 +9,7 @@ Documentation for the `etlplus.cli` subpackage: command-line interface for ETLPl
 
 Back to project overview: see the top-level [README](../../README.md).
 
-- [etlplus.cli subpackage](#etlpluscli-subpackage)
+- [`etlplus.cli` Subpackage](#etlpluscli-subpackage)
   - [Available Commands](#available-commands)
   - [Command Options](#command-options)
   - [Example: Running a Pipeline](#example-running-a-pipeline)

{etlplus-0.14.3 → etlplus-0.15.0}/etlplus/cli/handlers.py

@@ -14,8 +14,6 @@ from typing import Any
 from typing import Literal
 from typing import cast
 
-from ..config import PipelineConfig
-from ..config import load_pipeline_config
 from ..database import load_table_spec
 from ..database import render_tables
 from ..file import File
@@ -28,6 +26,8 @@ from ..ops import validate
 from ..ops.validate import FieldRules
 from ..types import JSONData
 from ..types import TemplateKey
+from ..workflow import PipelineConfig
+from ..workflow import load_pipeline_config
 from . import io as cli_io
 
 # SECTION: EXPORTS ========================================================== #

etlplus-0.15.0/etlplus/config/README.md

@@ -0,0 +1,50 @@
+# `etlplus.config` Subpackage
+
+Documentation for the `etlplus.config` subpackage: type definitions and config shape helpers for
+ETLPlus.
+
+- Exposes TypedDict-based config schemas for API profiles and endpoints
+- Provides exported type aliases for API configuration maps
+- Designed for Python 3.13 typing and editor assistance (runtime parsing lives elsewhere)
+
+Back to project overview: see the top-level [README](../../README.md).
+
+- [`etlplus.config` Subpackage](#etlplusconfig-subpackage)
+  - [Modules](#modules)
+  - [Exported Types](#exported-types)
+  - [Example: Typing an API Config](#example-typing-an-api-config)
+  - [See Also](#see-also)
+
+## Modules
+
+- `etlplus.config.__init__`: package exports and high-level package notes
+- `etlplus.config.types`: TypedDict-based config schemas
+
+## Exported Types
+
+- `ApiConfigMap`: top-level API config shape
+- `ApiProfileConfigMap`: per-profile API config shape
+- `ApiProfileDefaultsMap`: defaults block within a profile
+- `EndpointMap`: endpoint config shape
+
+## Example: Typing an API Config
+
+```python
+from etlplus.config import ApiConfigMap
+
+api_cfg: ApiConfigMap = {
+    "base_url": "https://example.test",
+    "headers": {"Authorization": "Bearer token"},
+    "endpoints": {
+        "users": {
+            "path": "/users",
+            "method": "GET",
+        },
+    },
+}
+```
+
+## See Also
+
+- Top-level CLI and library usage in the main [README](../../README.md)
+- Config type definitions in [types.py](types.py)

etlplus-0.15.0/etlplus/config/__init__.py

@@ -0,0 +1,33 @@
+"""
+:mod:`etlplus.config` package.
+
+Configuration models and helpers for ETLPlus.
+
+This package defines models for data sources/targets ("connectors"), APIs,
+pagination/rate limits, pipeline orchestration, and related utilities. The
+parsers are permissive (accepting ``Mapping[str, Any]``) and normalize to
+concrete types without raising on unknown/optional fields.
+
+Notes
+-----
+- The models use ``@dataclass(slots=True)`` and avoid mutating inputs.
+- TypedDicts are editor/type-checking hints and are not enforced at runtime.
+"""
+
+from __future__ import annotations
+
+from .types import ApiConfigMap
+from .types import ApiProfileConfigMap
+from .types import ApiProfileDefaultsMap
+from .types import EndpointMap
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Typed Dicts
+    'ApiConfigMap',
+    'ApiProfileConfigMap',
+    'ApiProfileDefaultsMap',
+    'EndpointMap',
+]

{etlplus-0.14.3 → etlplus-0.15.0}/etlplus/config/types.py

@@ -33,7 +33,6 @@ from __future__ import annotations
 
 from collections.abc import Mapping
 from typing import Any
-from typing import Literal
 from typing import TypedDict
 
 from ..api import PaginationConfigMap
@@ -44,26 +43,17 @@ from ..types import StrAnyMap
 
 
 __all__ = [
-    # Type aliases
-    'ConnectorType',
-    # 'PaginationType',
     # TypedDicts
     'ApiProfileDefaultsMap',
     'ApiProfileConfigMap',
     'ApiConfigMap',
     'EndpointMap',
-    'ConnectorApiConfigMap',
-    'ConnectorDbConfigMap',
-    'ConnectorFileConfigMap',
 ]
 
 
 # SECTION: TYPE ALIASES ===================================================== #
 
 
-# Literal type for supported connector kinds
-type ConnectorType = Literal['api', 'database', 'file']
-
 # Literal type for supported pagination kinds
 # type PaginationType = Literal['page', 'offset', 'cursor']
 
@@ -129,60 +119,6 @@ class ApiProfileDefaultsMap(TypedDict, total=False):
     rate_limit: RateLimitConfigMap | StrAnyMap
 
 
-class ConnectorApiConfigMap(TypedDict, total=False):
-    """
-    Shape accepted by ConnectorApi.from_obj (all keys optional).
-
-    See Also
-    --------
-    - etlplus.config.connector.ConnectorApi.from_obj
-    """
-
-    name: str
-    type: ConnectorType
-    url: str
-    method: str
-    headers: StrAnyMap
-    query_params: StrAnyMap
-    pagination: PaginationConfigMap
-    rate_limit: RateLimitConfigMap
-    api: str
-    endpoint: str
-
-
-class ConnectorDbConfigMap(TypedDict, total=False):
-    """
-    Shape accepted by ConnectorDb.from_obj (all keys optional).
-
-    See Also
-    --------
-    - etlplus.config.connector.ConnectorDb.from_obj
-    """
-
-    name: str
-    type: ConnectorType
-    connection_string: str
-    query: str
-    table: str
-    mode: str
-
-
-class ConnectorFileConfigMap(TypedDict, total=False):
-    """
-    Shape accepted by ConnectorFile.from_obj (all keys optional).
-
-    See Also
-    --------
-    - etlplus.config.connector.ConnectorFile.from_obj
-    """
-
-    name: str
-    type: ConnectorType
-    format: str
-    path: str
-    options: StrAnyMap
-
-
 class EndpointMap(TypedDict, total=False):
     """
     Shape accepted by EndpointConfig.from_obj.

{etlplus-0.14.3 → etlplus-0.15.0}/etlplus/database/README.md

@@ -1,4 +1,4 @@
-# etlplus.database subpackage
+# `etlplus.database` Subpackage
 
 Documentation for the `etlplus.database` subpackage: database engine, schema, and ORM helpers.
 
@@ -9,7 +9,7 @@ Documentation for the `etlplus.database` subpackage: database engine, schema, an
 
 Back to project overview: see the top-level [README](../../README.md).
 
-- [etlplus.database subpackage](#etlplusdatabase-subpackage)
+- [`etlplus.database` Subpackage](#etlplusdatabase-subpackage)
   - [Database Engine and Connections](#database-engine-and-connections)
   - [Schema and DDL Helpers](#schema-and-ddl-helpers)
   - [ORM Utilities](#orm-utilities)

{etlplus-0.14.3 → etlplus-0.15.0}/etlplus/file/README.md

@@ -1,4 +1,4 @@
-# etlplus.file subpackage
+# `etlplus.file` Subpackage
 
 Documentation for the `etlplus.file` subpackage: unified file format support and helpers for reading
 and writing data files.
@@ -11,7 +11,7 @@ and writing data files.
 
 Back to project overview: see the top-level [README](../../README.md).
 
-- [etlplus.file subpackage](#etlplusfile-subpackage)
+- [`etlplus.file` Subpackage](#etlplusfile-subpackage)
   - [Supported File Formats](#supported-file-formats)
   - [Inferring File Format and Compression](#inferring-file-format-and-compression)
   - [Reading and Writing Files](#reading-and-writing-files)

{etlplus-0.14.3 → etlplus-0.15.0}/etlplus/ops/run.py

@@ -20,7 +20,6 @@ from ..api import RequestOptions
 from ..api import compose_api_request_env
 from ..api import compose_api_target_env
 from ..api import paginate_with_client
-from ..config import load_pipeline_config
 from ..enums import DataConnectorType
 from ..file import FileFormat
 from ..types import JSONData
@@ -29,6 +28,7 @@ from ..types import PipelineConfig
 from ..types import StrPath
 from ..types import Timeout
 from ..utils import print_json
+from ..workflow import load_pipeline_config
 from .extract import extract
 from .load import load
 from .transform import transform
@@ -162,9 +162,12 @@ def run(
                 # can monkeypatch this class on etlplus.ops.run.
                 ClientClass = EndpointClient  # noqa: N806
                 client = ClientClass(
-                    base_url=cast(str, env['base_url']),
+                    base_url=cast(str, env.get('base_url')),
                     base_path=cast(str | None, env.get('base_path')),
-                    endpoints=cast(dict[str, str], env['endpoints_map']),
+                    endpoints=cast(
+                        dict[str, str],
+                        env.get('endpoints_map', {}),
+                    ),
                     retry=env.get('retry'),
                     retry_network_errors=bool(
                         env.get('retry_network_errors', False),
@@ -173,7 +176,7 @@ def run(
                 )
                 data = paginate_with_client(
                     client,
-                    cast(str, env['endpoint_key']),
+                    cast(str, env.get('endpoint_key')),
                     env.get('params'),
                     env.get('headers'),
                     env.get('timeout'),
@@ -276,12 +279,14 @@ def run(
             if not url_t:
                 raise ValueError('API target missing "url"')
             kwargs_t: dict[str, Any] = {}
-            if env_t.get('headers'):
-                kwargs_t['headers'] = cast(dict[str, str], env_t['headers'])
+            headers = env_t.get('headers')
+            if headers:
+                kwargs_t['headers'] = cast(dict[str, str], headers)
             if env_t.get('timeout') is not None:
-                kwargs_t['timeout'] = env_t['timeout']
-            if env_t.get('session') is not None:
-                kwargs_t['session'] = env_t['session']
+                kwargs_t['timeout'] = env_t.get('timeout')
+            session = env_t.get('session')
+            if session is not None:
+                kwargs_t['session'] = session
             result = load(
                 data,
                 'api',

{etlplus-0.14.3 → etlplus-0.15.0}/etlplus/templates/README.md

@@ -1,4 +1,4 @@
-# etlplus.templates subpackage
+# `etlplus.templates` Subpackage
 
 Documentation for the `etlplus.templates` subpackage: SQL and DDL template helpers.
 
@@ -8,7 +8,7 @@ Documentation for the `etlplus.templates` subpackage: SQL and DDL template helpe
 
 Back to project overview: see the top-level [README](../../README.md).
 
-- [etlplus.templates subpackage](#etlpustemplates-subpackage)
+- [`etlplus.templates` Subpackage](#etlplus-templates-subpackage)
     - [Available Templates](#available-templates)
     - [Rendering Templates](#rendering-templates)
     - [Example: Rendering a DDL Template](#example-rendering-a-ddl-template)

{etlplus-0.14.3/etlplus/config → etlplus-0.15.0/etlplus/workflow}/README.md

@@ -1,7 +1,7 @@
-# etlplus.config subpackage
+# `etlplus.workflow` Subpackage
 
-Documentation for the `etlplus.config` subpackage: configuration helpers for connectors, pipelines,
-jobs, and profiles.
+Documentation for the `etlplus.workflow` subpackage: configuration helpers for connectors,
+pipelines, jobs, and profiles.
 
 - Provides classes and utilities for managing ETL pipeline configuration
 - Supports YAML/JSON config loading and validation
@@ -10,7 +10,7 @@ jobs, and profiles.
 
 Back to project overview: see the top-level [README](../../README.md).
 
-- [etlplus.config subpackage](#etlplusconfig-subpackage)
+- [`etlplus.workflow` Subpackage](#etlplusworkflow-subpackage)
   - [Supported Configuration Types](#supported-configuration-types)
   - [Loading and Validating Configs](#loading-and-validating-configs)
   - [Example: Loading a Pipeline Config](#example-loading-a-pipeline-config)
@@ -28,7 +28,7 @@ Back to project overview: see the top-level [README](../../README.md).
 Use the provided classes to load and validate configuration files:
 
 ```python
-from etlplus.config import PipelineConfig
+from etlplus.workflow import PipelineConfig
 
 cfg = PipelineConfig.from_yaml("pipeline.yml")
 ```
@@ -39,7 +39,7 @@ cfg = PipelineConfig.from_yaml("pipeline.yml")
 ## Example: Loading a Pipeline Config
 
 ```python
-from etlplus.config import PipelineConfig
+from etlplus.workflow import PipelineConfig
 
 pipeline = PipelineConfig.from_yaml("configs/pipeline.yml")
 print(pipeline)

{etlplus-0.14.3/etlplus/config → etlplus-0.15.0/etlplus/workflow}/__init__.py

@@ -1,17 +1,7 @@
 """
-:mod:`etlplus.config` package.
+:mod:`etlplus.workflow` package.
 
-Configuration models and helpers for ETLPlus.
-
-This package defines models for data sources/targets ("connectors"), APIs,
-pagination/rate limits, pipeline orchestration, and related utilities. The
-parsers are permissive (accepting ``Mapping[str, Any]``) and normalize to
-concrete types without raising on unknown/optional fields.
-
-Notes
------
-- The models use ``@dataclass(slots=True)`` and avoid mutating inputs.
-- TypedDicts are editor/type-checking hints and are not enforced at runtime.
+Job workflow helpers.
 """
 
 from __future__ import annotations
@@ -21,6 +11,7 @@ from .connector import ConnectorApi
 from .connector import ConnectorDb
 from .connector import ConnectorFile
 from .connector import parse_connector
+from .dag import topological_sort_jobs
 from .jobs import ExtractRef
 from .jobs import JobConfig
 from .jobs import LoadRef
@@ -28,29 +19,25 @@ from .jobs import TransformRef
 from .jobs import ValidationRef
 from .pipeline import PipelineConfig
 from .pipeline import load_pipeline_config
-from .profile import ProfileConfig
-from .types import ConnectorType
 
 # SECTION: EXPORTS ========================================================== #
 
 
 __all__ = [
-    # Connectors
-    'Connector',
-    'ConnectorType',
+    # Data Classes
     'ConnectorApi',
     'ConnectorDb',
     'ConnectorFile',
-    'parse_connector',
-    # Jobs / Refs
     'ExtractRef',
     'JobConfig',
     'LoadRef',
+    'PipelineConfig',
     'TransformRef',
     'ValidationRef',
-    # Pipeline
-    'PipelineConfig',
+    # Functions
     'load_pipeline_config',
-    # Profile
-    'ProfileConfig',
+    'parse_connector',
+    'topological_sort_jobs',
+    # Type Aliases
+    'Connector',
 ]

{etlplus-0.14.3/etlplus/config → etlplus-0.15.0/etlplus/workflow}/connector.py

@@ -1,5 +1,5 @@
 """
-:mod:`etlplus.config.connector` module.
+:mod:`etlplus.workflow.connector` module.
 
 A module defining configuration types for data source/target connectors in ETL
 pipelines. A "connector" is any I/O endpoint:
@@ -11,18 +11,19 @@ pipelines. A "connector" is any I/O endpoint:
 
 Examples
 --------
-- Use ``ConnectorApi``/``ConnectorFile``/``ConnectorDb`` when you want the
-  concrete dataclasses.
-- Use the ``Connector`` union for typing a value that can be any connector.
-- Use ``parse_connector(obj)`` to construct a connector instance from a generic
-  mapping that includes a ``type`` key.
+- Use :class:`ConnectorApi`/:class:`ConnectorFile`/:class:`ConnectorDb` when
+    you want the concrete dataclasses.
+- Use the :class:`Connector` union for typing a value that can be any
+    connector.
+- Use :func:`parse_connector(obj)` to construct a connector instance from a
+    generic mapping that includes a *type* key.
 
 Notes
 -----
 - TypedDict shapes are editor hints; runtime parsing remains permissive
-  (from_obj accepts Mapping[str, Any]).
+    (from_obj accepts Mapping[str, Any]).
 - TypedDicts referenced in :mod:`etlplus.config.types` remain editor hints.
-  Runtime parsing stays permissive and tolerant.
+    Runtime parsing stays permissive and tolerant.
 
 See Also
 --------
@@ -59,7 +60,7 @@ if TYPE_CHECKING:  # Editor-only typing hints to avoid runtime imports
 
 
 __all__ = [
-    # Classes
+    # Data Classes
     'ConnectorApi',
     'ConnectorDb',
     'ConnectorFile',
@@ -83,12 +84,12 @@ class ConnectorApi:
     name : str
         Unique connector name.
     type : ConnectorType
-        Connector kind literal, always ``"api"``.
+        Connector kind literal, always ``'api'``.
     url : str | None
         Direct absolute URL (when not using ``service``/``endpoint`` refs).
     method : str | None
         Optional HTTP method; typically omitted for sources (defaults to
-        GET) and used for targets (e.g., ``"post"``).
+        GET) and used for targets (e.g., ``'post'``).
     headers : dict[str, str]
         Additional request headers.
     query_params : dict[str, Any]
@@ -111,7 +112,7 @@ class ConnectorApi:
 
     # Direct form
     url: str | None = None
-    # Optional HTTP method; typically omitted for sources (defaults to GET
+    # Optional HTTP method; typically omitted for sources (defaults to GET)
     # at runtime) and used for targets (e.g., 'post', 'put').
     method: str | None = None
     headers: dict[str, str] = field(default_factory=dict)
@@ -185,7 +186,7 @@ class ConnectorDb:
     name : str
         Unique connector name.
     type : ConnectorType
-        Connector kind literal, always ``"database"``.
+        Connector kind literal, always ``'database'``.
     connection_string : str | None
         Connection string/DSN for the database.
     query : str | None
@@ -193,7 +194,7 @@ class ConnectorDb:
     table : str | None
         Target/source table name (optional).
     mode : str | None
-        Load mode hint (e.g., ``"append"``, ``"replace"``) — future use.
+        Load mode hint (e.g., ``'append'``, ``'replace'``) — future use.
     """
 
     # -- Attributes -- #
@@ -262,9 +263,9 @@ class ConnectorFile:
     name : str
         Unique connector name.
     type : ConnectorType
-        Connector kind literal, always ``"file"``.
+        Connector kind literal, always ``'file'``.
     format : str | None
-        File format (e.g., ``"json"``, ``"csv"``).
+        File format (e.g., ``'json'``, ``'csv'``).
     path : str | None
         File path or URI.
     options : dict[str, Any]

etlplus-0.15.0/etlplus/workflow/dag.py

@@ -0,0 +1,105 @@
+"""
+:mod:`etlplus.workflow.dag` module.
+
+Lightweight directed acyclic graph (DAG) helpers for ordering jobs based on
+:attr:`depends_on`.
+"""
+
+from __future__ import annotations
+
+from collections import deque
+from dataclasses import dataclass
+
+from .jobs import JobConfig
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Errors
+    'DagError',
+    # Functions
+    'topological_sort_jobs',
+]
+
+
+# SECTION: ERRORS =========================================================== #
+
+
+@dataclass(slots=True)
+class DagError(ValueError):
+    """
+    Raised when the job dependency graph is invalid.
+
+    Attributes
+    ----------
+    message : str
+        Error message.
+    """
+
+    # -- Attributes -- #
+
+    message: str
+
+    # -- Magic Methods (Object Representation) -- #
+
+    def __str__(self) -> str:
+        return self.message
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def topological_sort_jobs(
+    jobs: list[JobConfig],
+) -> list[JobConfig]:
+    """
+    Return jobs in topological order based on :attr:`depends_on`.
+
+    Parameters
+    ----------
+    jobs : list[JobConfig]
+        List of job configurations to sort.
+
+    Returns
+    -------
+    list[JobConfig]
+        Jobs sorted in topological order.
+
+    Raises
+    ------
+    DagError
+        If a dependency is missing, self-referential, or when a cycle is
+        detected.
+    """
+    index = {job.name: job for job in jobs}
+    edges: dict[str, set[str]] = {name: set() for name in index}
+    indegree: dict[str, int] = {name: 0 for name in index}
+
+    for job in jobs:
+        for dep in job.depends_on:
+            if dep not in index:
+                raise DagError(
+                    f'Unknown dependency "{dep}" in job "{job.name}"',
+                )
+            if dep == job.name:
+                raise DagError(f'Job "{job.name}" depends on itself')
+            if job.name not in edges[dep]:
+                edges[dep].add(job.name)
+                indegree[job.name] += 1
+
+    queue = deque(sorted(name for name, deg in indegree.items() if deg == 0))
+    ordered: list[str] = []
+
+    while queue:
+        name = queue.popleft()
+        ordered.append(name)
+        for child in sorted(edges[name]):
+            indegree[child] -= 1
+            if indegree[child] == 0:
+                queue.append(child)
+
+    if len(ordered) != len(jobs):
+        raise DagError('Dependency cycle detected')
+
+    return [index[name] for name in ordered]