etlplus 0.3.14__tar.gz → 0.11.8__tar.gz

etlplus-0.11.8/.coveragerc

@@ -0,0 +1,21 @@
+# .coveragerc
+# ETLPlus
+#
+# Copyright © 2025 Dagitali LLC. All rights reserved.
+#
+# An optional pytest-cov configuration file. Limits coverage measurement to the
+# ETLPlus package and ignore test modules.
+#
+# See:
+# 1. https://pytest-cov.readthedocs.io/en/latest/config.html
+
+[run]
+source = etlplus
+branch = true
+omit =
+    tests/*
+    */tests/*
+
+[report]
+skip_covered = true
+show_missing = true

{etlplus-0.3.14 → etlplus-0.11.8}/.github/workflows/ci.yml

@@ -74,9 +74,26 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           python-bootstrap: "-e .[dev,yaml]"
-      - name: Run tests
+      - name: Run tests (with coverage)
         run: |
-          pytest -q
+          pytest -q \
+            --cov \
+            --cov-branch \
+            --cov-config=.coveragerc \
+            --cov-report=term-missing \
+            --cov-report=xml \
+            tests/
+
+      - name: Upload coverage reports to Codecov
+        if: matrix.python-version == '3.13'
+        uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de  # Pinned v5.5.2
+        with:
+          fail_ci_if_error: true
+          files: coverage.xml
+          flags: unit
+          name: etlplus
+          token: ${{ secrets.CODECOV_TOKEN }}  # Omit for public repo
+          verbose: true
 
   build:
     name: Build distributions

{etlplus-0.3.14 → etlplus-0.11.8}/.pre-commit-config.yaml

@@ -159,7 +159,11 @@ repos:
     rev: v1.19.0
     hooks:
       - id: mypy
-        args: [--ignore-missing-imports, --install-types, --non-interactive]
+        args:
+          - --cache-dir=.mypy_cache/pre-commit
+          - --ignore-missing-imports
+          - --install-types
+          - --non-interactive
 
   - repo: https://github.com/pycqa/flake8
     rev: 7.3.0

{etlplus-0.3.14 → etlplus-0.11.8}/DEMO.md

@@ -58,7 +58,7 @@ John Doe,30,New York
 Jane Smith,25,Los Angeles
 CSVDATA
 
-$ etlplus extract file users.csv --format csv
+$ etlplus extract users.csv
 [
   {
     "name": "John Doe",
@@ -89,14 +89,14 @@ $ etlplus validate '{"email": "user@example.com", "age": 25}' \
 
 ### Filter and Select
 ```bash
-$ etlplus transform '[
+$ etlplus transform --operations '{
+    "filter": {"field": "age", "op": "gt", "value": 26},
+    "select": ["name", "age"]
+  }' '[
     {"name": "John", "age": 30, "city": "NYC"},
     {"name": "Jane", "age": 25, "city": "LA"},
     {"name": "Bob", "age": 35, "city": "Chicago"}
-  ]' --operations '{
-    "filter": {"field": "age", "op": "gt", "value": 26},
-    "select": ["name", "age"]
-  }'
+  ]'
 [
   {
     "name": "John",
@@ -111,24 +111,19 @@ $ etlplus transform '[
 
 ### Sort Data
 ```bash
-$ etlplus transform '[
-    {"name": "Charlie", "score": 85},
-    {"name": "Alice", "score": 95},
-    {"name": "Bob", "score": 90}
-  ]' --operations '{
-    "sort": {"field": "score", "reverse": true}
-  }'
+$ etlplus transform -\
+  -operations '{"sort": {"field": "score", "reverse": true}}' \
+  '[{"name": "Charlie", "score": 85}, {"name": "Alice", "score": 95}, {"name": "Bob", "score": 90}]'
 ```
 
 ### Aggregate Data
 ```bash
-$ etlplus transform '[
+$ etlplus transform --operations '{"aggregate": {"field": "sales", "func": "sum"}}' \
+  '[
     {"product": "A", "sales": 100},
     {"product": "B", "sales": 150},
     {"product": "C", "sales": 200}
-  ]' --operations '{
-    "aggregate": {"field": "sales", "func": "sum"}
-  }'
+  ]'
 {
   "sum_sales": 450
 }
@@ -138,7 +133,9 @@ $ etlplus transform '[
 
 ### Load to JSON File
 ```bash
-$ etlplus load '{"name": "John", "status": "active"}' file output.json
+$ etlplus load \
+  '{"name": "John", "status": "active"}' \
+  output.json --target-type file
 {
   "status": "success",
   "message": "Data loaded to output.json",
@@ -148,10 +145,12 @@ $ etlplus load '{"name": "John", "status": "active"}' file output.json
 
 ### Load to CSV File
 ```bash
-$ etlplus load '[
+$ etlplus load \
+  '[
     {"name": "John", "email": "john@example.com"},
     {"name": "Jane", "email": "jane@example.com"}
-  ]' file users.csv --format csv
+  ]' \
+  users.csv --target-type file
 {
   "status": "success",
   "message": "Data loaded to users.csv",
@@ -170,22 +169,25 @@ This example shows a complete ETL workflow:
 
 ```bash
 # Step 1: Extract
-$ etlplus extract file raw_data.csv --format csv -o extracted.json
+$ etlplus extract raw_data.csv > extracted.json
 
 # Step 2: Transform
-$ etlplus transform extracted.json \
+$ etlplus transform \
   --operations '{
     "filter": {"field": "age", "op": "gte", "value": 18},
     "select": ["name", "email", "age"]
-  }' -o transformed.json
+  }' \
+  extracted.json \
+  transformed.json
 
 # Step 3: Validate
-$ etlplus validate transformed.json \
+$ etlplus validate \
   --rules '{
     "name": {"type": "string", "required": true},
     "email": {"type": "string", "required": true, "pattern": "^[\\w.-]+@[\\w.-]+\\.\\w+$"},
     "age": {"type": "number", "min": 18, "max": 120}
-  }'
+  }' \
+  transformed.json
 
 # Step 4: Load
 $ etlplus load transformed.json file final_output.csv

etlplus-0.11.8/MANIFEST.in

@@ -0,0 +1,12 @@
+# MANIFEST.in
+# ETLPlus
+#
+# Copyright © 2026 Dagitali LLC. All rights reserved.
+#
+# Contains commands that allow lists of files to be discovered and manipulated.
+#
+# See:
+# 1. https://setuptools.pypa.io/en/latest/userguide/miscellaneous.html
+
+# Include Jinja template files in the etlplus package
+recursive-include etlplus/templates *.j2

{etlplus-0.3.14 → etlplus-0.11.8}/Makefile

@@ -253,7 +253,7 @@ venv: ## Create the virtual environment (at $(VENV_DIR))
 	else \
 		$(call ECHO_INFO, "Using existing venv: $(VENV_DIR)"); \
 	fi
-	@$(PYTHON) -m pip install --upgrade pip setuptool wheel >/dev/null
+	@$(PYTHON) -m pip install --upgrade pip setuptools wheel >/dev/null
 	@$(call ECHO_OK,"venv ready")
 
 ##@ CI

{etlplus-0.3.14/etlplus.egg-info → etlplus-0.11.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: etlplus
-Version: 0.3.14
+Version: 0.11.8
 Summary: A Swiss Army knife for simple ETL operations
 Home-page: https://github.com/Dagitali/ETLPlus
 Author: ETLPlus Team
@@ -21,7 +21,11 @@ Requires-Dist: jinja2>=3.1.6
 Requires-Dist: pyodbc>=5.3.0
 Requires-Dist: python-dotenv>=1.2.1
 Requires-Dist: pandas>=2.3.3
+Requires-Dist: pydantic>=2.12.5
+Requires-Dist: PyYAML>=6.0.3
 Requires-Dist: requests>=2.32.5
+Requires-Dist: SQLAlchemy>=2.0.45
+Requires-Dist: typer>=0.21.0
 Provides-Extra: dev
 Requires-Dist: black>=25.9.0; extra == "dev"
 Requires-Dist: build>=1.2.2; extra == "dev"
@@ -60,13 +64,16 @@ package and command-line interface for data extraction, validation, transformati
   - [Quickstart](#quickstart)
   - [Usage](#usage)
     - [Command Line Interface](#command-line-interface)
+      - [Argument Order and Required Options](#argument-order-and-required-options)
+      - [Check Pipelines](#check-pipelines)
+      - [Render SQL DDL](#render-sql-ddl)
       - [Extract Data](#extract-data)
       - [Validate Data](#validate-data)
       - [Transform Data](#transform-data)
       - [Load Data](#load-data)
     - [Python API](#python-api)
     - [Complete ETL Pipeline Example](#complete-etl-pipeline-example)
-    - [Environment Variables](#environment-variables)
+    - [Format Overrides](#format-overrides)
   - [Transformation Operations](#transformation-operations)
     - [Filter Operations](#filter-operations)
     - [Aggregation Functions](#aggregation-functions)
@@ -78,6 +85,8 @@ package and command-line interface for data extraction, validation, transformati
       - [Test Layers](#test-layers)
     - [Code Coverage](#code-coverage)
     - [Linting](#linting)
+    - [Updating Demo Snippets](#updating-demo-snippets)
+    - [Releasing to PyPI](#releasing-to-pypi)
   - [Links](#links)
   - [License](#license)
   - [Contributing](#contributing)
@@ -85,6 +94,14 @@ package and command-line interface for data extraction, validation, transformati
 
 ## Features
 
+- **Check** data pipeline definitions before running them:
+  - Summarize jobs, sources, targets, and transforms
+  - Confirm configuration changes by printing focused sections on demand
+
+- **Render** SQL DDL from shared table specs:
+  - Generate CREATE TABLE or view statements
+  - Swap templates or direct output to files for database migrations
+
 - **Extract** data from multiple sources:
   - Files (CSV, JSON, XML, YAML)
   - Databases (connection string support)
@@ -135,8 +152,8 @@ etlplus --version
 
 # One-liner: extract CSV, filter, select, and write JSON
 etlplus extract file examples/data/sample.csv \
-  | etlplus transform - --operations '{"filter": {"field": "age", "op": "gt", "value": 25}, "select": ["name", "email"]}' \
-  -o temp/sample_output.json
+  | etlplus transform --operations '{"filter": {"field": "age", "op": "gt", "value": 25}, "select": ["name", "email"]}' \
+  - temp/sample_output.json
 ```
 
 [Python API](#python-api):
@@ -166,11 +183,73 @@ etlplus --help
 etlplus --version
 ```
 
+The CLI is implemented with Typer (Click-based). There is no argparse compatibility layer, so rely
+on the documented commands/flags and run `etlplus <command> --help` for current options.
+
+**Example error messages:**
+
+- If you omit a required argument: `Error: Missing required argument 'SOURCE'.`
+- If you place an option before its argument: `Error: Option '--source-format' must follow the 'SOURCE' argument.`
+
+#### Argument Order and Required Options
+
+For each command, positional arguments must precede options. Required options must follow their
+associated argument:
+
+- **extract**: `etlplus extract SOURCE [--source-format ...] [--source-type ...]`
+  - `SOURCE` is required. `--source-format` and `--source-type` must follow `SOURCE`.
+- **transform**: `etlplus transform [--operations ...] SOURCE [--source-format ...] [--source-type ...] TARGET [--target-format ...] [--target-type ...]`
+  - `SOURCE` and `TARGET` are required. Format/type options must follow their respective argument.
+- **load**: `etlplus load TARGET [--target-format ...] [--target-type ...] [--source-format ...]`
+  - `TARGET` is required. `--target-format` and `--target-type` must follow `TARGET`.
+- **validate**: `etlplus validate SOURCE [--rules ...] [--source-format ...] [--source-type ...]`
+  - `SOURCE` is required. `--rules` and format/type options must follow `SOURCE`.
+
+If required arguments or options are missing, or if options are placed before their associated argument, the CLI will display a clear error message.
+
+#### Check Pipelines
+
+Use `etlplus check` to explore pipeline YAML definitions without running them. The command can print
+job names, summarize configured sources and targets, or drill into specific sections.
+
+List jobs and show a pipeline summary:
+```bash
+etlplus check --config examples/configs/pipeline.yml --jobs
+etlplus check --config examples/configs/pipeline.yml --summary
+```
+
+Show sources or transforms for troubleshooting:
+```bash
+etlplus check --config examples/configs/pipeline.yml --sources
+etlplus check --config examples/configs/pipeline.yml --transforms
+```
+
+#### Render SQL DDL
+
+Use `etlplus render` to turn table schema specs into ready-to-run SQL. Render from a pipeline config
+or from a standalone schema file, and choose the built-in `ddl` or `view` templates (or provide your
+own).
+
+Render all tables defined in a pipeline:
+```bash
+etlplus render --config examples/configs/pipeline.yml --template ddl
+```
+
+Render a single table in that pipeline:
+```bash
+etlplus render --config examples/configs/pipeline.yml --table customers --template view
+```
+
+Render from a standalone table spec to a file:
+```bash
+etlplus render --spec schemas/customer.yml --template view -o temp/customer_view.sql
+```
+
 #### Extract Data
 
-Note: For file sources, the format is inferred from the filename extension; the `--format` option is
-ignored.  To treat passing `--format` as an error for file sources, either set
-`ETLPLUS_FORMAT_BEHAVIOR=error` or pass the CLI flag `--strict-format`.
+Note: For file sources, the format is normally inferred from the filename extension. Use
+`--source-format` to override inference when a file lacks an extension or when you want to force a
+specific parser.
 
 Extract from JSON file:
 ```bash
@@ -194,7 +273,7 @@ etlplus extract api https://api.example.com/data
 
 Save extracted data to file:
 ```bash
-etlplus extract file examples/data/sample.csv -o temp/sample_output.json
+etlplus extract file examples/data/sample.csv > temp/sample_output.json
 ```
 
 #### Validate Data
@@ -211,42 +290,69 @@ etlplus validate examples/data/sample.json --rules '{"email": {"type": "string",
 
 #### Transform Data
 
+When piping data through `etlplus transform`, use `--source-format` whenever the SOURCE argument is
+`-` or a literal payload, mirroring the `etlplus extract` semantics. Use `--target-format` to
+control the emitted format for STDOUT or other non-file outputs, just like `etlplus load`. File
+paths continue to infer formats from their extensions. Use `--source-type` to override the inferred
+source connector type and `--target-type` to override the inferred target connector type, matching
+the `etlplus extract`/`etlplus load` behavior.
+
+Transform file inputs while overriding connector types:
+```bash
+etlplus transform \
+  --operations '{"select": ["name", "email"]}' \
+  examples/data/sample.json  --source-type file \
+  temp/selected_output.json --target-type file
+```
+
 Filter and select fields:
 ```bash
-etlplus transform '[{"name": "John", "age": 30}, {"name": "Jane", "age": 25}]' \
-  --operations '{"filter": {"field": "age", "op": "gt", "value": 26}, "select": ["name"]}'
+etlplus transform \
+  --operations '{"filter": {"field": "age", "op": "gt", "value": 26}, "select": ["name"]}' \
+  '[{"name": "John", "age": 30}, {"name": "Jane", "age": 25}]'
 ```
 
 Sort data:
 ```bash
-etlplus transform examples/data/sample.json --operations '{"sort": {"field": "age", "reverse": true}}'
+etlplus transform \
+  --operations '{"sort": {"field": "age", "reverse": true}}' \
+  examples/data/sample.json
 ```
 
 Aggregate data:
 ```bash
-etlplus transform examples/data/sample.json --operations '{"aggregate": {"field": "age", "func": "sum"}}'
+etlplus transform \
+  --operations '{"aggregate": {"field": "age", "func": "sum"}}' \
+  examples/data/sample.json
 ```
 
 Map/rename fields:
 ```bash
-etlplus transform examples/data/sample.json --operations '{"map": {"name": "new_name"}}'
+etlplus transform \
+  --operations '{"map": {"name": "new_name"}}' \
+  examples/data/sample.json
 ```
 
 #### Load Data
 
+`etlplus load` consumes JSON from STDIN; provide only the target argument plus optional flags.
+
 Load to JSON file:
 ```bash
-etlplus load '{"name": "John", "age": 30}' file temp/sample_output.json
+etlplus extract file examples/data/sample.json \
+  | etlplus load temp/sample_output.json --target-type file
 ```
 
 Load to CSV file:
 ```bash
-etlplus load '[{"name": "John", "age": 30}]' file temp/sample_output.csv
+etlplus extract file examples/data/sample.csv \
+  | etlplus load temp/sample_output.csv --target-type file
 ```
 
 Load to REST API:
 ```bash
-etlplus load examples/data/sample.json api https://api.example.com/endpoint
+cat examples/data/sample.json \
+  | etlplus load https://api.example.com/endpoint --target-type api
 ```
 
 ### Python API
@@ -284,57 +390,57 @@ For YAML-driven pipelines executed end-to-end (extract → validate → transfor
 - Authoring: [`docs/pipeline-guide.md`](docs/pipeline-guide.md)
 - Runner API and internals: [`docs/run-module.md`](docs/run-module.md)
 
+CLI quick reference for pipelines:
+
+```bash
+# List jobs or show a pipeline summary
+etlplus check --config examples/configs/pipeline.yml --jobs
+etlplus check --config examples/configs/pipeline.yml --summary
+
+# Run a job
+etlplus run --config examples/configs/pipeline.yml --job file_to_file_customers
+```
+
 ### Complete ETL Pipeline Example
 
 ```bash
 # 1. Extract from CSV
-etlplus extract file examples/data/sample.csv -o temp/sample_extracted.json
+etlplus extract file examples/data/sample.csv > temp/sample_extracted.json
 
 # 2. Transform (filter and select fields)
-etlplus transform temp/sample_extracted.json \
+etlplus transform \
   --operations '{"filter": {"field": "age", "op": "gt", "value": 25}, "select": ["name", "email"]}' \
-  -o temp/sample_transformed.json
+  temp/sample_extracted.json \
+  temp/sample_transformed.json
 
 # 3. Validate transformed data
-etlplus validate temp/sample_transformed.json \
-  --rules '{"name": {"type": "string", "required": true}, "email": {"type": "string", "required": true}}'
+etlplus validate \
+  --rules '{"name": {"type": "string", "required": true}, "email": {"type": "string", "required": true}}' \
+  temo/sample_transformed.json
 
 # 4. Load to CSV
-etlplus load temp/sample_transformed.json file temp/sample_output.csv
+cat temp/sample_transformed.json \
+  | etlplus load temp/sample_output.csv
 ```
 
-### Environment Variables
-
-ETLPlus honors a small number of environment toggles to refine CLI behavior:
+### Format Overrides
 
-- `ETLPLUS_FORMAT_BEHAVIOR`: controls what happens when `--format` is provided for
-  file sources or targets (extract/load) where the format is inferred from the
-  filename extension.
-  - `error|fail|strict`: treat as error (non-zero exit)
-  - `warn` (default): print a warning to stderr
-  - `ignore|silent`: no message
-- Precedence: the CLI flag `--strict-format` overrides the environment.
+`--source-format` and `--target-format` override whichever format would normally be inferred from a
+file extension. This is useful when an input lacks an extension (for example, `records.txt` that
+actually contains CSV) or when you intentionally want to treat a file as another format.
 
 Examples (zsh):
 
 ```zsh
-# Warn (default)
-etlplus extract file data.csv --format csv
-etlplus load data.json file out.csv --format csv
-
-# Enforce error via environment
-ETLPLUS_FORMAT_BEHAVIOR=error \
-  etlplus extract file data.csv --format csv
-ETLPLUS_FORMAT_BEHAVIOR=error \
-  etlplus load data.json file out.csv --format csv
-
-# Equivalent strict behavior via flag (overrides environment)
-etlplus extract file data.csv --format csv --strict-format
-etlplus load data.json file out.csv --format csv --strict-format
-
-# Recommended: rely on extension, no --format needed for files
-etlplus extract file data.csv
-etlplus load data.json file out.csv
+# Force CSV parsing for an extension-less file
+etlplus extract data.txt --source-type file --source-format csv
+
+# Write CSV to a file without the .csv suffix
+etlplus load output.bin --target-type file --target-format csv < data.json
+
+# Leave the flags off when extensions already match the desired format
+etlplus extract data.csv --source-type file
+etlplus load data.json --target-type file  < data.json
 ```
 
 ## Transformation Operations