etlplus 0.16.3__py3-none-any.whl → 0.16.4__py3-none-any.whl

etlplus/README.md

@@ -31,6 +31,28 @@ assert validate(filtered, {"age": {"type": "number", "min": 0}})["valid"]
 load(filtered, "file", "output.json", file_format="json")
 ```
 
+## Loading and Validating Configs
+
+Use the provided classes to load and validate configuration files:
+
+```python
+from etlplus.workflow import Config
+
+cfg = Config.from_yaml("pipeline.yml")
+```
+
+- Supports YAML and JSON formats
+- Validates against expected schema
+
+## Example: Loading a Pipeline Config
+
+```python
+from etlplus.workflow import Config
+
+config = Config.from_yaml("configs/pipeline.yml")
+print(config)
+```
+
 ## See Also
 
 - [Top-level project README](../README.md)

etlplus/__init__.py

@@ -5,6 +5,7 @@ Top-level facade for the ETLPlus toolkit.
 """
 
 from .__version__ import __version__
+from .config import Config
 
 __author__ = 'ETLPlus Team'
 
@@ -15,4 +16,5 @@ __author__ = 'ETLPlus Team'
 __all__ = [
     '__author__',
     '__version__',
+    'Config',
 ]

etlplus/cli/commands.py

@@ -62,6 +62,16 @@ __all__ = ['app']
 # SECTION: TYPE ALIASES ==================================================== #
 
 
+ConfigOption = Annotated[
+    str,
+    typer.Option(
+        ...,
+        '--config',
+        metavar='PATH',
+        help='Path to YAML-formatted configuration file.',
+    ),
+]
+
 JobOption = Annotated[
     str | None,
     typer.Option(
@@ -97,16 +107,6 @@ OutputOption = Annotated[
     ),
 ]
 
-PipelineConfigOption = Annotated[
-    str,
-    typer.Option(
-        ...,
-        '--config',
-        metavar='PATH',
-        help='Path to pipeline YAML configuration file.',
-    ),
-]
-
 PipelineOption = Annotated[
     str | None,
     typer.Option(
@@ -407,7 +407,7 @@ def _root(
 @app.command('check')
 def check_cmd(
     ctx: typer.Context,
-    config: PipelineConfigOption,
+    config: ConfigOption,
     jobs: JobsOption = False,
     pipelines: PipelinesOption = False,
     sources: SourcesOption = False,
@@ -422,20 +422,20 @@ def check_cmd(
     ----------
     ctx : typer.Context
         The Typer context.
-    config : PipelineConfigOption
+    config : ConfigOption
         Path to pipeline YAML configuration file.
-    jobs : bool, optional
+    jobs : JobsOption, optional
         List available job names and exit. Default is ``False``.
-    pipelines : bool, optional
+    pipelines : PipelinesOption, optional
         List ETL pipelines. Default is ``False``.
-    sources : bool, optional
+    sources : SourcesOption, optional
         List data sources. Default is ``False``.
-    summary : bool, optional
+    summary : SummaryOption, optional
         Show pipeline summary (name, version, sources, targets, jobs). Default
         is ``False``.
-    targets : bool, optional
+    targets : TargetsOption, optional
         List data targets. Default is ``False``.
-    transforms : bool, optional
+    transforms : TransformsOption, optional
         List data transforms. Default is ``False``.
 
     Returns
@@ -725,7 +725,7 @@ def render_cmd(
 @app.command('run')
 def run_cmd(
     ctx: typer.Context,
-    config: PipelineConfigOption,
+    config: ConfigOption,
     job: JobOption = None,
     pipeline: PipelineOption = None,
 ) -> int:
@@ -736,11 +736,11 @@ def run_cmd(
     ----------
     ctx : typer.Context
         The Typer context.
-    config : PipelineConfigOption
+    config : ConfigOption
         Path to pipeline YAML configuration file.
-    job : str | None, optional
+    job : JobOption, optional
         Name of the job to run. Default is ``None``.
-    pipeline : str | None, optional
+    pipeline : PipelineOption, optional
         Name of the pipeline to run. Default is ``None``.
 
     Returns

etlplus/cli/handlers.py

@@ -14,6 +14,7 @@ from typing import Any
 from typing import Literal
 from typing import cast
 
+from .. import Config
 from ..database import load_table_spec
 from ..database import render_tables
 from ..file import File
@@ -26,8 +27,6 @@ from ..ops import validate
 from ..ops.validate import FieldRules
 from ..types import JSONData
 from ..types import TemplateKey
-from ..workflow import PipelineConfig
-from ..workflow import load_pipeline_config
 from . import io as cli_io
 
 # SECTION: EXPORTS ========================================================== #
@@ -73,14 +72,14 @@ def _collect_table_specs(
         specs.append(dict(load_table_spec(Path(spec_path))))
 
     if config_path:
-        cfg = load_pipeline_config(config_path, substitute=True)
+        cfg = Config.from_yaml(config_path, substitute=True)
         specs.extend(getattr(cfg, 'table_schemas', []))
 
     return specs
 
 
 def _check_sections(
-    cfg: PipelineConfig,
+    cfg: Config,
     *,
     jobs: bool,
     pipelines: bool,
@@ -93,7 +92,7 @@ def _check_sections(
 
     Parameters
     ----------
-    cfg : PipelineConfig
+    cfg : Config
         The loaded pipeline configuration.
     jobs : bool
         Whether to include job metadata.
@@ -133,14 +132,14 @@ def _check_sections(
 
 
 def _pipeline_summary(
-    cfg: PipelineConfig,
+    cfg: Config,
 ) -> dict[str, Any]:
     """
     Return a human-friendly snapshot of a pipeline config.
 
     Parameters
     ----------
-    cfg : PipelineConfig
+    cfg : Config
         The loaded pipeline configuration.
 
     Returns
@@ -229,7 +228,7 @@ def check_handler(
         Zero on success.
 
     """
-    cfg = load_pipeline_config(config, substitute=substitute)
+    cfg = Config.from_yaml(config, substitute=substitute)
     if summary:
         cli_io.emit_json(_pipeline_summary(cfg), pretty=True)
         return 0
@@ -514,7 +513,7 @@ def run_handler(
     int
         Zero on success.
     """
-    cfg = load_pipeline_config(config, substitute=True)
+    cfg = Config.from_yaml(config, substitute=True)
 
     job_name = job or pipeline
     if job_name:

etlplus/{workflow/pipeline.py → config.py}

@@ -1,7 +1,7 @@
 """
-:mod:`etlplus.workflow.pipeline` module.
+:mod:`etlplus.config` module.
 
-Pipeline configuration model and helpers for job orchestration.
+Configuration model and helpers for job pipeline orchestration.
 
 Notes
 -----
@@ -24,26 +24,24 @@ from pathlib import Path
 from typing import Any
 from typing import Self
 
-from ..api import ApiConfig
-from ..connector import Connector
-from ..connector import parse_connector
-from ..file import File
-from ..file import FileFormat
-from ..types import StrAnyMap
-from ..utils import coerce_dict
-from ..utils import deep_substitute
-from ..utils import maybe_mapping
-from .jobs import JobConfig
-from .profile import ProfileConfig
+from .api import ApiConfig
+from .connector import Connector
+from .connector import parse_connector
+from .file import File
+from .file import FileFormat
+from .types import StrAnyMap
+from .utils import coerce_dict
+from .utils import deep_substitute
+from .utils import maybe_mapping
+from .workflow.jobs import JobConfig
+from .workflow.profile import ProfileConfig
 
 # SECTION: EXPORTS ========================================================== #
 
 
 __all__ = [
     # Data Classes
-    'PipelineConfig',
-    # Functions
-    'load_pipeline_config',
+    'Config',
 ]
 
 
@@ -126,29 +124,11 @@ def _parse_connector_entry(
         return None
 
 
-# SECTION: FUNCTIONS ======================================================== #
-
-
-def load_pipeline_config(
-    path: Path | str,
-    *,
-    substitute: bool = False,
-    env: Mapping[str, str] | None = None,
-) -> PipelineConfig:
-    """
-    Load a pipeline YAML file into a ``PipelineConfig`` instance.
-
-    Delegates to ``PipelineConfig.from_yaml`` for construction and optional
-    variable substitution.
-    """
-    return PipelineConfig.from_yaml(path, substitute=substitute, env=env)
-
-
 # SECTION: DATA CLASSES ===================================================== #
 
 
 @dataclass(kw_only=True, slots=True)
-class PipelineConfig:
+class Config:
     """
     Configuration for the data processing pipeline.
 
@@ -211,7 +191,7 @@ class PipelineConfig:
         env: Mapping[str, str] | None = None,
     ) -> Self:
         """
-        Parse a YAML file into a ``PipelineConfig`` instance.
+        Parse a YAML file into a ``Config`` instance.
 
         Parameters
         ----------
@@ -259,7 +239,7 @@ class PipelineConfig:
         raw: StrAnyMap,
     ) -> Self:
         """
-        Parse a mapping into a ``PipelineConfig`` instance.
+        Parse a mapping into a ``Config`` instance.
 
         Parameters
         ----------

etlplus/ops/run.py

@@ -11,6 +11,7 @@ from typing import Final
 from typing import cast
 
 from ..api import HttpMethod
+from ..config import Config
 from ..connector import DataConnectorType
 from ..file import FileFormat
 from ..ops.types import PipelineConfig
@@ -18,7 +19,6 @@ from ..types import JSONData
 from ..types import JSONDict
 from ..types import StrPath
 from ..utils import print_json
-from ..workflow import load_pipeline_config
 from .extract import extract
 from .extract import extract_from_api_source
 from .load import load
@@ -176,7 +176,7 @@ def run(
         If the job is not found or if there are configuration issues.
     """
     cfg_path = config_path or DEFAULT_CONFIG_PATH
-    cfg = load_pipeline_config(cfg_path, substitute=True)
+    cfg = Config.from_yaml(cfg_path, substitute=True)
 
     # Lookup job by name
     if not (job_obj := next((j for j in cfg.jobs if j.name == job), None)):

etlplus/types.py

@@ -16,7 +16,8 @@ See Also
 
 Examples
 --------
->>> from etlplus.types import JSONDict, PipelineConfig
+>>> from etlplus.types import JSONDict
+>>> from etlplus.ops.types import PipelineConfig
 >>> payload: JSONDict = {'id': 1, 'name': 'Ada'}
 >>> isinstance(payload, dict)
 True

etlplus/workflow/README.md

@@ -12,8 +12,6 @@ Back to project overview: see the top-level [README](../../README.md).
 
 - [`etlplus.workflow` Subpackage](#etlplusworkflow-subpackage)
   - [Supported Configuration Types](#supported-configuration-types)
-  - [Loading and Validating Configs](#loading-and-validating-configs)
-  - [Example: Loading a Pipeline Config](#example-loading-a-pipeline-config)
   - [See Also](#see-also)
 
 ## Supported Configuration Types
@@ -23,28 +21,6 @@ Back to project overview: see the top-level [README](../../README.md).
 - **Pipeline**: End-to-end pipeline configuration
 - **Profile**: User or environment-specific settings
 
-## Loading and Validating Configs
-
-Use the provided classes to load and validate configuration files:
-
-```python
-from etlplus.workflow import PipelineConfig
-
-cfg = PipelineConfig.from_yaml("pipeline.yml")
-```
-
-- Supports YAML and JSON formats
-- Validates against expected schema
-
-## Example: Loading a Pipeline Config
-
-```python
-from etlplus.workflow import PipelineConfig
-
-pipeline = PipelineConfig.from_yaml("configs/pipeline.yml")
-print(pipeline)
-```
-
 ## See Also
 
 - Top-level CLI and library usage in the main [README](../../README.md)

etlplus/workflow/__init__.py

@@ -12,8 +12,6 @@ from .jobs import JobConfig
 from .jobs import LoadRef
 from .jobs import TransformRef
 from .jobs import ValidationRef
-from .pipeline import PipelineConfig
-from .pipeline import load_pipeline_config
 from .profile import ProfileConfig
 
 # SECTION: EXPORTS ========================================================== #
@@ -24,11 +22,9 @@ __all__ = [
     'ExtractRef',
     'JobConfig',
     'LoadRef',
-    'PipelineConfig',
     'ProfileConfig',
     'TransformRef',
     'ValidationRef',
     # Functions
-    'load_pipeline_config',
     'topological_sort_jobs',
 ]

etlplus/workflow/jobs.py

@@ -6,8 +6,6 @@ transform, load).
 
 Notes
 -----
-- Lightweight references used inside :class:`PipelineConfig` to avoid storing
-    large nested structures.
 - All attributes are simple and optional where appropriate, keeping parsing
     tolerant.
 """

{etlplus-0.16.3.dist-info → etlplus-0.16.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: etlplus
-Version: 0.16.3
+Version: 0.16.4
 Summary: A Swiss Army knife for simple ETL operations
 Home-page: https://github.com/Dagitali/ETLPlus
 Author: ETLPlus Team

{etlplus-0.16.3.dist-info → etlplus-0.16.4.dist-info}/RECORD

@@ -1,11 +1,12 @@
-etlplus/README.md,sha256=JaMSomnMsHrTruDnonHqe83Rv4K0-e7Wy46tMeVoleU,1468
-etlplus/__init__.py,sha256=mgTP4PJmRmsEjTCAizzzdtzAmhuHtarmPzphzdjvLgM,277
+etlplus/README.md,sha256=L3un9q7Q7Mstfh0dmSjpsZMZzsOz2tvlWw4_-Y_LZEs,1887
+etlplus/__init__.py,sha256=yHZt-sjjfPjB_CrNhcT9bcMlZCfwiul39ZhQ2VfxpRs,318
 etlplus/__main__.py,sha256=btoROneNiigyfBU7BSzPKZ1R9gzBMpxcpsbPwmuHwTM,479
 etlplus/__version__.py,sha256=1E0GMK_yUWCMQFKxXjTvyMwofi0qT2k4CDNiHWiymWE,327
+etlplus/config.py,sha256=6BCI9hC1yWYAy5WclIHJlg90FYeWn5vAVrT1NWUTwpE,8817
 etlplus/enums.py,sha256=MfQhy3XDpN7oqLrF7_WwZojl7n8cW3RAzsZGRnAbWgc,4073
 etlplus/mixins.py,sha256=ifGpHwWv7U00yqGf-kN93vJax2IiK4jaGtTsPsO3Oak,1350
 etlplus/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-etlplus/types.py,sha256=DC9424i5qNBuCVGrfNLY3Ha2sz1mU84I0btSRAH7vrc,3428
+etlplus/types.py,sha256=Op2H1dcmv0Srm9prFnBZjt7f1S4Mqrus7XrdsjoZkIM,3461
 etlplus/utils.py,sha256=X-k_Y8i6oDjlE5aQu9sw3gPw7O2ikiSn4uoheVv_ERc,17091
 etlplus/api/README.md,sha256=amxS_eIcsnNuVvD0x_w8nkyfedOTYbhlY0gGhaFg0DE,8705
 etlplus/api/__init__.py,sha256=PK2lQv1FbsE7ZZS_ejevFZQSuOUHGApBc22YfHAzMqA,4615
@@ -28,9 +29,9 @@ etlplus/api/rate_limiting/config.py,sha256=Byc_kmnwFmjjfDEFIdc_sHc7Wnjde1NGgsjPH
 etlplus/api/rate_limiting/rate_limiter.py,sha256=uYxn-l2qwLUKVclDQ3vJIIP3fozJx2JlHhz7_zyXVbA,7033
 etlplus/cli/README.md,sha256=8H_G2d3HteYIU6ReX9K9DM485QjWDT5vHMQbGD_vv20,1237
 etlplus/cli/__init__.py,sha256=J97-Rv931IL1_b4AXnB7Fbbd7HKnHBpx18NQfC_kE6c,299
-etlplus/cli/commands.py,sha256=HFlg29tO6Jwv1NXWAHmvniLCyRSlboL55Arn9B8nZAM,25028
+etlplus/cli/commands.py,sha256=Mbnu_YYUrOumbDjkul9x5VjP8VXW5u08xNi4nLF9Yyo,25048
 etlplus/cli/constants.py,sha256=0F7dXIQKWUhhVu2Us527GJeknJIWpBqz7CK2e5OQgcE,1947
-etlplus/cli/handlers.py,sha256=uvbAyF6Ux8_5C-obCWZOrOP0QP0oiT-Km1hPhE8tDx0,18558
+etlplus/cli/handlers.py,sha256=r06JatAUewxtZYGuu7lLWlcy4hKGQ8U-Oo_YtAuIhzI,18454
 etlplus/cli/io.py,sha256=tGGNQ4ecezqj-mD285fgBVrYdphdeqApsyV9VojOj1I,7836
 etlplus/cli/main.py,sha256=68_uJwmWajhOC9o4R_ns8IQloC9BFmAKC_9GlQOxKWg,5239
 etlplus/cli/options.py,sha256=vfXT3YLh7wG1iC-aTdSg6ItMC8l6n0Lozmy53XjqLbA,1199
@@ -120,7 +121,7 @@ etlplus/ops/__init__.py,sha256=r5_-pPhSLCD1nq1EbN0rQrLOGpudueeIxCH_JvT2bt0,1718
 etlplus/ops/enums.py,sha256=dC_8CfaTiB2i83Az-oG-2hkjMuAfDADNbcMF2f94UeU,4014
 etlplus/ops/extract.py,sha256=LOYiPrALRMF7JDBabnRF24_HKnnIcfTdfXesWdS3QZM,11020
 etlplus/ops/load.py,sha256=yicciVwomUKkdbhuRqbavKBNpT2Hg813BnQzG6IgF4o,10811
-etlplus/ops/run.py,sha256=2Z27ahRZGVULxDelHldHzUJ_vdbBCwlkwpm5KyKFP7U,11298
+etlplus/ops/run.py,sha256=4HWelMevW0pW_76lJkoMcbzeQMiThMbxzO09wx6yoHg,11278
 etlplus/ops/transform.py,sha256=-41uw_pwOGsMTUYxtXaeYOmTF_fTkN-L4Q9KT1OFe78,25671
 etlplus/ops/types.py,sha256=Cvp8AJzJhJ1iYjyHd7j9ZLioxE2NdK__3g6fOI0qq6Q,4198
 etlplus/ops/utils.py,sha256=lJmrO1KDob-xZU8Gc2SvZvMgdYLsVoaz-fTV42KkLVo,10835
@@ -129,15 +130,14 @@ etlplus/templates/README.md,sha256=IfPXlj1TGVA-uFWosHJhE2rabFW-znxOlOMazO9Z5cE,1
 etlplus/templates/__init__.py,sha256=tsniN7XJYs3NwYxJ6c2HD5upHP3CDkLx-bQCMt97UOM,106
 etlplus/templates/ddl.sql.j2,sha256=s8fMWvcb4eaJVXkifuib1aQPljtZ8buuyB_uA-ZdU3Q,4734
 etlplus/templates/view.sql.j2,sha256=Iy8DHfhq5yyvrUKDxqp_aHIEXY4Tm6j4wT7YDEFWAhk,2180
-etlplus/workflow/README.md,sha256=D1oloiJCOHiqpqgv3m3qpRSIUOMIQcWtIsOPv7KkNI0,1652
-etlplus/workflow/__init__.py,sha256=ueothwpLruyLgr3-2hW8VT1unNyFJxdmT-l_3eB2ejc,724
+etlplus/workflow/README.md,sha256=QelyVFGX-sZM9mx3v6BXkzX36hv6MI1yK4eCPSOKNwI,1050
+etlplus/workflow/__init__.py,sha256=XgCQr684om0rONrQZ61yQ0r4qqFQL0iLAAB2Mn2BRSE,594
 etlplus/workflow/dag.py,sha256=-f1x8N1eb-PUuiOwEvFLmJwfR7JaMDJihlCHlhrFhgE,2937
-etlplus/workflow/jobs.py,sha256=5DmAzmEZV6XXQ-xzowkLxFzplIh8Eno3wuCmjy79xHw,8818
-etlplus/workflow/pipeline.py,sha256=PA5zhcfrk--pAg3b3x4oBf29WMj5HqR8zOozz4oEmg8,9387
+etlplus/workflow/jobs.py,sha256=hLE9QJUzQaI0aOEon0P-xxxa6xHp997ANei4F310WRY,8711
 etlplus/workflow/profile.py,sha256=FQU3bzBZ9_yjKC9kCXKN1FQDS9zjNUjtWB1r3UL95_Q,1993
-etlplus-0.16.3.dist-info/licenses/LICENSE,sha256=MuNO63i6kWmgnV2pbP2SLqP54mk1BGmu7CmbtxMmT-U,1069
-etlplus-0.16.3.dist-info/METADATA,sha256=uV-JEWCscNgGsoF34kK-1TZWzWjYZKamLylPHGwhlwU,28114
-etlplus-0.16.3.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-etlplus-0.16.3.dist-info/entry_points.txt,sha256=6w-2-jzuPa55spzK34h-UKh2JTEShh38adFRONNP9QE,45
-etlplus-0.16.3.dist-info/top_level.txt,sha256=aWWF-udn_sLGuHTM6W6MLh99ArS9ROkUWO8Mi8y1_2U,8
-etlplus-0.16.3.dist-info/RECORD,,
+etlplus-0.16.4.dist-info/licenses/LICENSE,sha256=MuNO63i6kWmgnV2pbP2SLqP54mk1BGmu7CmbtxMmT-U,1069
+etlplus-0.16.4.dist-info/METADATA,sha256=Pe-3vIQcAfTasRzy2HPq0A4cin2I5PJChAmA23S9dgo,28114
+etlplus-0.16.4.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+etlplus-0.16.4.dist-info/entry_points.txt,sha256=6w-2-jzuPa55spzK34h-UKh2JTEShh38adFRONNP9QE,45
+etlplus-0.16.4.dist-info/top_level.txt,sha256=aWWF-udn_sLGuHTM6W6MLh99ArS9ROkUWO8Mi8y1_2U,8
+etlplus-0.16.4.dist-info/RECORD,,