etlplus 0.16.3__tar.gz → 0.16.4__tar.gz

{etlplus-0.16.3 → etlplus-0.16.4}/CONTRIBUTING.md

@@ -133,5 +133,5 @@ If a test calls `etlplus.cli.main()` or `etlplus.ops.run.run()`, it is integrati
 - CLI tests: monkeypatch `sys.argv` and call `etlplus.cli.main()`; capture output with `capsys`.
 - File I/O: use `tmp_path` / `TemporaryDirectory()`; never write to the repo tree.
 - API flows: stub `EndpointClient` or transport layer via `monkeypatch` to avoid real HTTP.
-- Runner tests: monkeypatch `load_pipeline_config` to inject an in-memory `PipelineConfig`.
+- Runner tests: monkeypatch `load_config` to inject an in-memory `Config`.
 - Keep tests small and focused; prefer one behavior per test with clear assertions.

{etlplus-0.16.3/etlplus.egg-info → etlplus-0.16.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: etlplus
-Version: 0.16.3
+Version: 0.16.4
 Summary: A Swiss Army knife for simple ETL operations
 Home-page: https://github.com/Dagitali/ETLPlus
 Author: ETLPlus Team

{etlplus-0.16.3 → etlplus-0.16.4}/docs/pipeline-guide.md

@@ -497,8 +497,8 @@ For the HTTP client and pagination API, see `etlplus/api/README.md`.
 
 ## Design notes: Mapping inputs, dict outputs
 
-ETLPlus config constructors (e.g., `ApiConfig.from_obj`, `PipelineConfig.from_dict`) accept
-`Mapping[str, Any]` rather than `dict[str, Any]` for inputs. Why?
+ETLPlus config constructors (e.g., `ApiConfig.from_obj`, `Config.from_dict`) accept `Mapping[str,
+Any]` rather than `dict[str, Any]` for inputs. Why?
 
 - Flexibility: callers can pass any mapping-like object (e.g., YAML loaders that return custom
   mappings) without copying into a `dict` first.

{etlplus-0.16.3 → etlplus-0.16.4}/etlplus/README.md

@@ -31,6 +31,28 @@ assert validate(filtered, {"age": {"type": "number", "min": 0}})["valid"]
 load(filtered, "file", "output.json", file_format="json")
 ```
 
+## Loading and Validating Configs
+
+Use the provided classes to load and validate configuration files:
+
+```python
+from etlplus.workflow import Config
+
+cfg = Config.from_yaml("pipeline.yml")
+```
+
+- Supports YAML and JSON formats
+- Validates against expected schema
+
+## Example: Loading a Pipeline Config
+
+```python
+from etlplus.workflow import Config
+
+config = Config.from_yaml("configs/pipeline.yml")
+print(config)
+```
+
 ## See Also
 
 - [Top-level project README](../README.md)

{etlplus-0.16.3 → etlplus-0.16.4}/etlplus/__init__.py

@@ -5,6 +5,7 @@ Top-level facade for the ETLPlus toolkit.
 """
 
 from .__version__ import __version__
+from .config import Config
 
 __author__ = 'ETLPlus Team'
 
@@ -15,4 +16,5 @@ __author__ = 'ETLPlus Team'
 __all__ = [
     '__author__',
     '__version__',
+    'Config',
 ]

{etlplus-0.16.3 → etlplus-0.16.4}/etlplus/cli/commands.py

@@ -62,6 +62,16 @@ __all__ = ['app']
 # SECTION: TYPE ALIASES ==================================================== #
 
 
+ConfigOption = Annotated[
+    str,
+    typer.Option(
+        ...,
+        '--config',
+        metavar='PATH',
+        help='Path to YAML-formatted configuration file.',
+    ),
+]
+
 JobOption = Annotated[
     str | None,
     typer.Option(
@@ -97,16 +107,6 @@ OutputOption = Annotated[
     ),
 ]
 
-PipelineConfigOption = Annotated[
-    str,
-    typer.Option(
-        ...,
-        '--config',
-        metavar='PATH',
-        help='Path to pipeline YAML configuration file.',
-    ),
-]
-
 PipelineOption = Annotated[
     str | None,
     typer.Option(
@@ -407,7 +407,7 @@ def _root(
 @app.command('check')
 def check_cmd(
     ctx: typer.Context,
-    config: PipelineConfigOption,
+    config: ConfigOption,
     jobs: JobsOption = False,
     pipelines: PipelinesOption = False,
     sources: SourcesOption = False,
@@ -422,20 +422,20 @@ def check_cmd(
     ----------
     ctx : typer.Context
         The Typer context.
-    config : PipelineConfigOption
+    config : ConfigOption
         Path to pipeline YAML configuration file.
-    jobs : bool, optional
+    jobs : JobsOption, optional
         List available job names and exit. Default is ``False``.
-    pipelines : bool, optional
+    pipelines : PipelinesOption, optional
         List ETL pipelines. Default is ``False``.
-    sources : bool, optional
+    sources : SourcesOption, optional
         List data sources. Default is ``False``.
-    summary : bool, optional
+    summary : SummaryOption, optional
         Show pipeline summary (name, version, sources, targets, jobs). Default
         is ``False``.
-    targets : bool, optional
+    targets : TargetsOption, optional
         List data targets. Default is ``False``.
-    transforms : bool, optional
+    transforms : TransformsOption, optional
         List data transforms. Default is ``False``.
 
     Returns
@@ -725,7 +725,7 @@ def render_cmd(
 @app.command('run')
 def run_cmd(
     ctx: typer.Context,
-    config: PipelineConfigOption,
+    config: ConfigOption,
     job: JobOption = None,
     pipeline: PipelineOption = None,
 ) -> int:
@@ -736,11 +736,11 @@ def run_cmd(
     ----------
     ctx : typer.Context
         The Typer context.
-    config : PipelineConfigOption
+    config : ConfigOption
         Path to pipeline YAML configuration file.
-    job : str | None, optional
+    job : JobOption, optional
         Name of the job to run. Default is ``None``.
-    pipeline : str | None, optional
+    pipeline : PipelineOption, optional
         Name of the pipeline to run. Default is ``None``.
 
     Returns

{etlplus-0.16.3 → etlplus-0.16.4}/etlplus/cli/handlers.py

@@ -14,6 +14,7 @@ from typing import Any
 from typing import Literal
 from typing import cast
 
+from .. import Config
 from ..database import load_table_spec
 from ..database import render_tables
 from ..file import File
@@ -26,8 +27,6 @@ from ..ops import validate
 from ..ops.validate import FieldRules
 from ..types import JSONData
 from ..types import TemplateKey
-from ..workflow import PipelineConfig
-from ..workflow import load_pipeline_config
 from . import io as cli_io
 
 # SECTION: EXPORTS ========================================================== #
@@ -73,14 +72,14 @@ def _collect_table_specs(
         specs.append(dict(load_table_spec(Path(spec_path))))
 
     if config_path:
-        cfg = load_pipeline_config(config_path, substitute=True)
+        cfg = Config.from_yaml(config_path, substitute=True)
         specs.extend(getattr(cfg, 'table_schemas', []))
 
     return specs
 
 
 def _check_sections(
-    cfg: PipelineConfig,
+    cfg: Config,
     *,
     jobs: bool,
     pipelines: bool,
@@ -93,7 +92,7 @@ def _check_sections(
 
     Parameters
     ----------
-    cfg : PipelineConfig
+    cfg : Config
         The loaded pipeline configuration.
     jobs : bool
         Whether to include job metadata.
@@ -133,14 +132,14 @@ def _check_sections(
 
 
 def _pipeline_summary(
-    cfg: PipelineConfig,
+    cfg: Config,
 ) -> dict[str, Any]:
     """
     Return a human-friendly snapshot of a pipeline config.
 
     Parameters
     ----------
-    cfg : PipelineConfig
+    cfg : Config
         The loaded pipeline configuration.
 
     Returns
@@ -229,7 +228,7 @@ def check_handler(
         Zero on success.
 
     """
-    cfg = load_pipeline_config(config, substitute=substitute)
+    cfg = Config.from_yaml(config, substitute=substitute)
     if summary:
         cli_io.emit_json(_pipeline_summary(cfg), pretty=True)
         return 0
@@ -514,7 +513,7 @@ def run_handler(
     int
         Zero on success.
     """
-    cfg = load_pipeline_config(config, substitute=True)
+    cfg = Config.from_yaml(config, substitute=True)
 
     job_name = job or pipeline
     if job_name:

etlplus-0.16.3/etlplus/workflow/pipeline.py → etlplus-0.16.4/etlplus/config.py

@@ -1,7 +1,7 @@
 """
-:mod:`etlplus.workflow.pipeline` module.
+:mod:`etlplus.config` module.
 
-Pipeline configuration model and helpers for job orchestration.
+Configuration model and helpers for job pipeline orchestration.
 
 Notes
 -----
@@ -24,26 +24,24 @@ from pathlib import Path
 from typing import Any
 from typing import Self
 
-from ..api import ApiConfig
-from ..connector import Connector
-from ..connector import parse_connector
-from ..file import File
-from ..file import FileFormat
-from ..types import StrAnyMap
-from ..utils import coerce_dict
-from ..utils import deep_substitute
-from ..utils import maybe_mapping
-from .jobs import JobConfig
-from .profile import ProfileConfig
+from .api import ApiConfig
+from .connector import Connector
+from .connector import parse_connector
+from .file import File
+from .file import FileFormat
+from .types import StrAnyMap
+from .utils import coerce_dict
+from .utils import deep_substitute
+from .utils import maybe_mapping
+from .workflow.jobs import JobConfig
+from .workflow.profile import ProfileConfig
 
 # SECTION: EXPORTS ========================================================== #
 
 
 __all__ = [
     # Data Classes
-    'PipelineConfig',
-    # Functions
-    'load_pipeline_config',
+    'Config',
 ]
 
 
@@ -126,29 +124,11 @@ def _parse_connector_entry(
         return None
 
 
-# SECTION: FUNCTIONS ======================================================== #
-
-
-def load_pipeline_config(
-    path: Path | str,
-    *,
-    substitute: bool = False,
-    env: Mapping[str, str] | None = None,
-) -> PipelineConfig:
-    """
-    Load a pipeline YAML file into a ``PipelineConfig`` instance.
-
-    Delegates to ``PipelineConfig.from_yaml`` for construction and optional
-    variable substitution.
-    """
-    return PipelineConfig.from_yaml(path, substitute=substitute, env=env)
-
-
 # SECTION: DATA CLASSES ===================================================== #
 
 
 @dataclass(kw_only=True, slots=True)
-class PipelineConfig:
+class Config:
     """
     Configuration for the data processing pipeline.
 
@@ -211,7 +191,7 @@ class PipelineConfig:
         env: Mapping[str, str] | None = None,
     ) -> Self:
         """
-        Parse a YAML file into a ``PipelineConfig`` instance.
+        Parse a YAML file into a ``Config`` instance.
 
         Parameters
         ----------
@@ -259,7 +239,7 @@ class PipelineConfig:
         raw: StrAnyMap,
     ) -> Self:
         """
-        Parse a mapping into a ``PipelineConfig`` instance.
+        Parse a mapping into a ``Config`` instance.
 
         Parameters
         ----------

{etlplus-0.16.3 → etlplus-0.16.4}/etlplus/ops/run.py

@@ -11,6 +11,7 @@ from typing import Final
 from typing import cast
 
 from ..api import HttpMethod
+from ..config import Config
 from ..connector import DataConnectorType
 from ..file import FileFormat
 from ..ops.types import PipelineConfig
@@ -18,7 +19,6 @@ from ..types import JSONData
 from ..types import JSONDict
 from ..types import StrPath
 from ..utils import print_json
-from ..workflow import load_pipeline_config
 from .extract import extract
 from .extract import extract_from_api_source
 from .load import load
@@ -176,7 +176,7 @@ def run(
         If the job is not found or if there are configuration issues.
     """
     cfg_path = config_path or DEFAULT_CONFIG_PATH
-    cfg = load_pipeline_config(cfg_path, substitute=True)
+    cfg = Config.from_yaml(cfg_path, substitute=True)
 
     # Lookup job by name
     if not (job_obj := next((j for j in cfg.jobs if j.name == job), None)):

{etlplus-0.16.3 → etlplus-0.16.4}/etlplus/types.py

@@ -16,7 +16,8 @@ See Also
 
 Examples
 --------
->>> from etlplus.types import JSONDict, PipelineConfig
+>>> from etlplus.types import JSONDict
+>>> from etlplus.ops.types import PipelineConfig
 >>> payload: JSONDict = {'id': 1, 'name': 'Ada'}
 >>> isinstance(payload, dict)
 True

{etlplus-0.16.3 → etlplus-0.16.4}/etlplus/workflow/README.md

@@ -12,8 +12,6 @@ Back to project overview: see the top-level [README](../../README.md).
 
 - [`etlplus.workflow` Subpackage](#etlplusworkflow-subpackage)
   - [Supported Configuration Types](#supported-configuration-types)
-  - [Loading and Validating Configs](#loading-and-validating-configs)
-  - [Example: Loading a Pipeline Config](#example-loading-a-pipeline-config)
   - [See Also](#see-also)
 
 ## Supported Configuration Types
@@ -23,28 +21,6 @@ Back to project overview: see the top-level [README](../../README.md).
 - **Pipeline**: End-to-end pipeline configuration
 - **Profile**: User or environment-specific settings
 
-## Loading and Validating Configs
-
-Use the provided classes to load and validate configuration files:
-
-```python
-from etlplus.workflow import PipelineConfig
-
-cfg = PipelineConfig.from_yaml("pipeline.yml")
-```
-
-- Supports YAML and JSON formats
-- Validates against expected schema
-
-## Example: Loading a Pipeline Config
-
-```python
-from etlplus.workflow import PipelineConfig
-
-pipeline = PipelineConfig.from_yaml("configs/pipeline.yml")
-print(pipeline)
-```
-
 ## See Also
 
 - Top-level CLI and library usage in the main [README](../../README.md)

{etlplus-0.16.3 → etlplus-0.16.4}/etlplus/workflow/__init__.py

@@ -12,8 +12,6 @@ from .jobs import JobConfig
 from .jobs import LoadRef
 from .jobs import TransformRef
 from .jobs import ValidationRef
-from .pipeline import PipelineConfig
-from .pipeline import load_pipeline_config
 from .profile import ProfileConfig
 
 # SECTION: EXPORTS ========================================================== #
@@ -24,11 +22,9 @@ __all__ = [
     'ExtractRef',
     'JobConfig',
     'LoadRef',
-    'PipelineConfig',
     'ProfileConfig',
     'TransformRef',
     'ValidationRef',
     # Functions
-    'load_pipeline_config',
     'topological_sort_jobs',
 ]

{etlplus-0.16.3 → etlplus-0.16.4}/etlplus/workflow/jobs.py

@@ -6,8 +6,6 @@ transform, load).
 
 Notes
 -----
-- Lightweight references used inside :class:`PipelineConfig` to avoid storing
-    large nested structures.
 - All attributes are simple and optional where appropriate, keeping parsing
     tolerant.
 """

{etlplus-0.16.3 → etlplus-0.16.4/etlplus.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: etlplus
-Version: 0.16.3
+Version: 0.16.4
 Summary: A Swiss Army knife for simple ETL operations
 Home-page: https://github.com/Dagitali/ETLPlus
 Author: ETLPlus Team

{etlplus-0.16.3 → etlplus-0.16.4}/etlplus.egg-info/SOURCES.txt

@@ -26,6 +26,7 @@ etlplus/README.md
 etlplus/__init__.py
 etlplus/__main__.py
 etlplus/__version__.py
+etlplus/config.py
 etlplus/enums.py
 etlplus/mixins.py
 etlplus/py.typed
@@ -163,7 +164,6 @@ etlplus/workflow/README.md
 etlplus/workflow/__init__.py
 etlplus/workflow/dag.py
 etlplus/workflow/jobs.py
-etlplus/workflow/pipeline.py
 etlplus/workflow/profile.py
 examples/README.md
 examples/quickstart.py
@@ -178,23 +178,24 @@ tests/__init__.py
 tests/conftest.py
 tests/integration/conftest.py
 tests/integration/test_i_cli.py
+tests/integration/test_i_config_load.py
 tests/integration/test_i_examples_data_parity.py
 tests/integration/test_i_pagination_strategy.py
 tests/integration/test_i_pipeline_smoke.py
-tests/integration/test_i_pipeline_yaml_load.py
 tests/integration/test_i_run.py
 tests/integration/test_i_run_profile_pagination_defaults.py
 tests/integration/test_i_run_profile_rate_limit_defaults.py
 tests/unit/conftest.py
+tests/unit/test_u_config.py
 tests/unit/test_u_main.py
 tests/unit/test_u_mixins.py
 tests/unit/test_u_utils.py
 tests/unit/test_u_version.py
 tests/unit/api/conftest.py
+tests/unit/api/test_u_api_config.py
 tests/unit/api/test_u_api_enums.py
 tests/unit/api/test_u_api_utils.py
 tests/unit/api/test_u_auth.py
-tests/unit/api/test_u_config.py
 tests/unit/api/test_u_endpoint_client.py
 tests/unit/api/test_u_mocks.py
 tests/unit/api/test_u_pagination_client.py
@@ -228,5 +229,4 @@ tests/unit/ops/test_u_ops_transform.py
 tests/unit/ops/test_u_ops_utils.py
 tests/unit/ops/test_u_ops_validate.py
 tests/unit/workflow/test_u_workflow_jobs.py
-tests/unit/workflow/test_u_workflow_pipeline.py
 tools/update_demo_snippets.py

{etlplus-0.16.3 → etlplus-0.16.4}/tests/integration/conftest.py

@@ -19,6 +19,7 @@ from typing import Protocol
 
 import pytest
 
+from etlplus import Config
 from etlplus.api import ApiConfig
 from etlplus.api import ApiProfileConfig
 from etlplus.api import EndpointConfig
@@ -30,7 +31,6 @@ from etlplus.connector import ConnectorFile
 from etlplus.workflow import ExtractRef
 from etlplus.workflow import JobConfig
 from etlplus.workflow import LoadRef
-from etlplus.workflow import PipelineConfig
 
 # SECTION: HELPERS ========================================================== #
 
@@ -173,7 +173,7 @@ def fake_endpoint_client_fixture() -> tuple[
 def file_to_api_pipeline_factory_fixture(
     tmp_path: pathlib.Path,
     base_url: str,
-) -> Callable[..., PipelineConfig]:
+) -> Callable[..., Config]:
     """Build a pipeline wiring a JSON file source to an API target."""
 
     def _make(
@@ -187,7 +187,7 @@ def file_to_api_pipeline_factory_fixture(
         headers: dict[str, str] | None = None,
         job_name: str = 'send',
         target_name: str = 'ingest_out',
-    ) -> PipelineConfig:
+    ) -> Config:
         source_path = tmp_path / f'{job_name}_input.json'
         effective_payload = payload if payload is not None else {'ok': True}
         text = (
@@ -226,7 +226,7 @@ def file_to_api_pipeline_factory_fixture(
             headers=headers or {},
         )
 
-        return PipelineConfig(
+        return Config(
             apis={'svc': api},
             sources=[src],
             targets=[tgt],
@@ -246,9 +246,9 @@ def file_to_api_pipeline_factory_fixture(
 def pipeline_cfg_factory_fixture(
     tmp_path: pathlib.Path,
     base_url: str,
-) -> Callable[..., PipelineConfig]:
+) -> Callable[..., Config]:
     """
-    Factory to build a minimal PipelineConfig for runner tests.
+    Factory to build a minimal Config for runner tests.
 
     Accepts optional pagination and rate limit defaults at the API profile
     level. Creates a single job named 'job' with a source that references
@@ -263,8 +263,8 @@ def pipeline_cfg_factory_fixture(
 
     Returns
     -------
-    Callable[..., PipelineConfig]
-        Factory function to create :class:`PipelineConfig` instances.
+    Callable[..., Config]
+        Factory function to create :class:`Config` instances.
     """
 
     def _make(
@@ -272,7 +272,7 @@ def pipeline_cfg_factory_fixture(
         pagination_defaults: PaginationConfig | None = None,
         rate_limit_defaults: RateLimitConfig | None = None,
         extract_options: dict[str, Any] | None = None,
-    ) -> PipelineConfig:
+    ) -> Config:
         prof = ApiProfileConfig(
             base_url=base_url,
             headers={},
@@ -306,7 +306,7 @@ def pipeline_cfg_factory_fixture(
                 raise ValueError(msg)
             job.extract.options = extract_options
 
-        return PipelineConfig(
+        return Config(
             apis={'svc': api},
             sources=[src],
             targets=[tgt],
@@ -341,7 +341,7 @@ def run_patched_fixture(
     extract_mod = importlib.import_module('etlplus.ops.extract')
 
     def _run(
-        cfg: PipelineConfig,
+        cfg: Config,
         endpoint_client_cls: type,
         *,
         sleep_seconds: float | None = None,
@@ -350,8 +350,8 @@ def run_patched_fixture(
 
         # Patch config loader and EndpointClient.
         monkeypatch.setattr(
-            run_mod,
-            'load_pipeline_config',
+            run_mod.Config,
+            'from_yaml',
             lambda *_a, **_k: cfg,
         )
         monkeypatch.setattr(

etlplus-0.16.3/tests/integration/test_i_pipeline_yaml_load.py → etlplus-0.16.4/tests/integration/test_i_config_load.py

@@ -7,7 +7,7 @@ variable substitution enabled.
 
 Notes
 -----
-- Uses ``PipelineConfig.from_yaml`` on the repo's example config.
+- Uses ``Config.from_yaml`` on the repo's example config.
 - Asserts basic API modeling and presence of expected endpoints.
 """
 
@@ -17,7 +17,7 @@ from collections.abc import Mapping
 
 import pytest
 
-from etlplus.workflow import PipelineConfig
+from etlplus import Config
 
 # SECTION: HELPERS ========================================================== #
 
@@ -28,8 +28,8 @@ pytestmark = pytest.mark.integration
 # SECTION: TESTS ============================================================ #
 
 
-class TestPipelineYamlLoad:
-    """Integration test suite for pipeline YAML loading."""
+class TestConfigLoad:
+    """Integration test suite for configuration loading."""
 
     @pytest.mark.parametrize('substitute', [False, True])
     def test_load_repo_pipeline_yaml(
@@ -37,16 +37,16 @@ class TestPipelineYamlLoad:
         substitute: bool,
     ) -> None:
         """
-        Test loading the repository pipeline YAML with optional environment
-        variable substitution.
+        Test loading the configuration file with optional environment variable
+        substitution.
         """
 
-        # Ensure the repository pipeline YAML parses under current models.
-        cfg = PipelineConfig.from_yaml(
+        # Ensure the configuration file parses under current models.
+        cfg = Config.from_yaml(
             'examples/configs/pipeline.yml',
             substitute=substitute,
         )
-        assert isinstance(cfg, PipelineConfig)
+        assert isinstance(cfg, Config)
 
         # Basic sanity checks on REST API modeling.
         assert 'github' in cfg.apis

{etlplus-0.16.3 → etlplus-0.16.4}/tests/integration/test_i_pagination_strategy.py

@@ -32,8 +32,8 @@ import pytest
 
 import etlplus.api.request_manager as rm_module
 import etlplus.cli.handlers as cli_handlers
+from etlplus import Config
 from etlplus.cli import main
-from etlplus.workflow.pipeline import PipelineConfig
 from tests.integration.conftest import FakeEndpointClientProtocol
 
 # SECTION: HELPERS ========================================================== #
@@ -527,7 +527,7 @@ class TestPaginationStrategies:
     def test_pagination_edge_cases(
         self,
         scenario: PaginationEdgeCase,
-        pipeline_cfg_factory: Callable[..., PipelineConfig],
+        pipeline_cfg_factory: Callable[..., Config],
         fake_endpoint_client: tuple[
             type[FakeEndpointClientProtocol],
             list[FakeEndpointClientProtocol],