etlpipe 2.0.0__tar.gz

etlpipe-2.0.0/.github/workflows/ci.yml

@@ -0,0 +1,78 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  lint:
+    name: Lint & Format Check
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: "3.12"
+    - name: Install ruff
+      run: pip install ruff
+    - name: Check formatting
+      run: ruff format --check src/ tests/
+    - name: Check linting rules
+      run: ruff check src/ tests/
+
+  security:
+    name: Security Scan
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: "3.12"
+    - name: Install tools
+      run: pip install pip-audit bandit
+    - name: Audit dependencies for known vulnerabilities
+      run: pip install -e ".[dev]" && pip-audit
+    - name: Static security analysis
+      run: bandit -r src/etlpipe/ -ll --skip B101
+
+  test:
+    name: Tests (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install package with dev dependencies
+      run: python -m pip install -e ".[dev]"
+    - name: Run test suite with coverage
+      run: |
+        pytest tests/ -v --cov=etlpipe --cov-report=term-missing --cov-report=xml \
+          -x --tb=short -q
+    - name: Upload coverage
+      if: matrix.python-version == '3.12'
+      uses: actions/upload-artifact@v4
+      with:
+        name: coverage-report
+        path: coverage.xml
+
+  spark-test:
+    name: Spark Integration Tests
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: "3.12"
+    - name: Install package with Spark dependencies
+      run: python -m pip install setuptools && python -m pip install -e ".[dev,spark]"
+    - name: Run Spark integration tests
+      run: |
+        pytest tests/test_spark_engine.py -v --tb=short -q

etlpipe-2.0.0/.github/workflows/publish-governance.yml

@@ -0,0 +1,37 @@
+name: Publish Governance to PyPI
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+
+jobs:
+  build-and-publish:
+    name: Build and publish etlpipe-governance ?? to PyPI
+    runs-on: ubuntu-latest
+    
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.11"
+
+    - name: Install build dependencies
+      run: python -m pip install --upgrade pip build
+
+    - name: Build governance package
+      run: |
+        cd src/etlpipe_governance
+        python -m build --outdir ../../dist_governance
+
+    - name: Publish governance package
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        packages-dir: dist_governance/

etlpipe-2.0.0/.github/workflows/publish-main.yml

@@ -0,0 +1,43 @@
+name: Publish Main to PyPI
+
+on:
+  workflow_run:
+    workflows: ["Publish Governance to PyPI"]
+    types:
+      - completed
+  workflow_dispatch:
+
+jobs:
+  build-and-publish:
+    name: Build and publish etlpipe ?? to PyPI
+    runs-on: ubuntu-latest
+    
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.11"
+
+    - name: Install build dependencies
+      run: python -m pip install --upgrade pip build
+
+    - name: Install package and test dependencies
+      run: python -m pip install -e ".[dev]"
+
+    - name: Run test suite
+      run: pytest tests/
+
+    - name: Build main package
+      run: python -m build --outdir dist_main/
+
+    - name: Publish main package
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        packages-dir: dist_main/

etlpipe-2.0.0/.gitignore

@@ -0,0 +1,47 @@
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Build and Distribution
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Python and Cache
+__pycache__/
+*.py[cod]
+*$py.class
+.pytest_cache/
+.coverage
+htmlcov/
+
+# Editor and IDEs
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# Mock data generated during E2E test & demo execution
+sales.csv
+regions.csv
+cli_test_output.csv
+e2e_demo.py
+demo/*_output.csv
+demo/pipeline_output_*.csv

etlpipe-2.0.0/CHANGELOG.md

@@ -0,0 +1,74 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [2.0.0] — 2026-07-22
+
+### Added
+- **`.yxmd` Workflow Converter**: New `YxmdConverter` class and `etlpipe-convert` CLI tool to convert proprietary visual ETL workflow XML files into Etlpipe YAML pipelines. Supports 15+ tool mappings with expression translation, topological sorting, and graceful degradation for unparseable tools.
+- **Enterprise Governance Sub-Package (`etlpipe-governance`)**: Extracted and expanded data quality toolkit into a standalone package:
+  - `scan_pii()` — Detects PII across 12 international patterns (email, phone, SSN, credit card, Aadhaar, IBAN, passport, etc.) with confidence scoring.
+  - `mask_pii()` — Three masking strategies: `redact`, `hash` (SHA-256), and `pseudonymise` (reversible labels with mapping). Competitive differentiator vs. Great Expectations/Pandera.
+  - `expect_schema()` / `infer_schema()` — Schema contract validation with dtype aliases, nullability checks, strict/non-strict modes.
+  - `profile()` — Statistical profiler with cardinality, null rates, min/max/mean/std, and top-N value distributions.
+  - `ContractSuite` — Batch contract runner for pipeline audit checkpoints with PASS/FAIL/SKIPPED/WARN/ERROR reporting.
+- **Structured Logging**: Replaced all `print()` statements in pipeline engine with `logging.getLogger()` using hierarchical names (`etlpipe.pipeline`, `etlpipe.engines.pandas`, etc.). Pipeline metrics emitted as structured JSON.
+- **Pipeline Execution Metrics**: Per-step timing (`duration_s`), row counts, output types, and status tracking. Metrics accessible via `Pipeline.metrics` after execution.
+- **Pipeline Event Hooks**: Four lifecycle callbacks (`on_step_start`, `on_step_complete`, `on_step_error`, `on_pipeline_complete`) for integrating with Slack, PagerDuty, Teams, or any alerting system without adding those as dependencies.
+- **Retry/Backoff for Downloads**: `Developer.download()` now supports configurable `max_retries` (default 3) and `retry_delay` (default 1.0s) with exponential backoff. Handles `URLError`, `TimeoutError`, `OSError`, and HTTP 5xx responses.
+- **Schema Contracts in YAML Pipelines**: Steps can declare `output_schema` to enforce data contracts as part of pipeline execution.
+- **Spark Integration Tests in CI**: Dedicated non-blocking CI job running Spark backend tests on Python 3.12.
+- **Security Policy**: Added `SECURITY.md` with vulnerability reporting process, response timeline, and security track record.
+- **Contributing Guide**: Added `CONTRIBUTING.md` with dev setup, code standards, testing requirements, and PR process.
+- **Architecture Decision Records**: Added `doc/adr/` with template and records for dual-backend architecture and eval() removal.
+- **Enterprise Governance Documentation**: New section in `USER_GUIDE.md` covering PII scanning, data contracts, event hooks, OpenLineage integration patterns, and data catalog recommendations.
+
+### Changed
+- **Version bump to 2.0.0** — reflects new sub-package, converter, and breaking packaging changes.
+- **Development Status upgraded from Alpha to Beta** in PyPI classifiers.
+- **Wheel now bundles `etlpipe_governance`** alongside `etlpipe` — governance features are available without separate installation.
+- **Governance tests included in default pytest** via `testpaths` configuration.
+- **Backward-compatible shim modules** (`etlpipe._contracts`, `etlpipe._pii`) re-export from `etlpipe_governance` with deprecation warnings targeting removal in 3.0.
+
+### Security
+- **Converter uses `defusedxml`** for safe XML parsing of `.yxmd` files (XXE/SSRF protection). Falls back with a hard error if `defusedxml` is not available (no silent downgrade).
+
+## [1.0.0] — 2026-07-04
+
+### Added
+- **Automated CI/CD**: Added a GitHub Actions workflow using OIDC (Trusted Publishing) for automated, secure PyPI deployments on new GitHub Releases.
+- **Enterprise Edge Case Protections**: Built a background transpiler to auto-backtick column headers containing spaces to support dirty real-world enterprise datasets without syntax crashes.
+
+### Changed
+- **Memory Optimization (Pandas)**: The Pandas backend now automatically requires the `pyarrow` multi-threaded C++ engine when loading large CSV files, heavily reducing physical RAM usage and preventing `MemoryError` crashes.
+- **Datatype Coercion**: `Join.join` now dynamically casts mismatching datatypes between left and right anchor columns, eliminating strict PySpark `AnalysisException` errors when migrating disparate schemas.
+
+### Fixed
+- **Critical Security Fix**: Eradicated an arbitrary code execution vulnerability (RCE) in `Preparation.formula` by removing the insecure `eval()` fallback. The engine is now entirely zero-trust and sandboxed via `pd.eval` or strict `lambda` callables.
+- **Spark Architecture (Sorting Death Trap)**: Completely refactored `Preparation.record_id` to utilize parallelized `F.monotonically_increasing_id()` instead of `Window.orderBy`, eliminating massive single-node memory bottlenecks when scaling Spark pipelines.
+- **Spark Architecture (Tuple Bombs)**: Injected native `.persist()` boundaries directly into the upstream nodes of `Join.join` to prevent PySpark from redundantly computing the entire historical DAG multiple times during joins.
+- **Regex Edge Cases**: Fixed an engine logic gap where Regex capture groups incorrectly counted non-capturing groups `(?:)` in the Spark backend.
+## [0.2.0] — 2026-07-04
+
+### Added
+- **Enterprise Scalability (Dual-Backend Architecture)**: Added native support for distributed execution on big-data clusters using the new `SparkEngine`.
+- **Thread-Safe Context Manager**: Added `Etlpipe.backend()` context manager to allow concurrent multi-threading with different execution engines.
+- **Dynamic Dispatch**: `Etlpipe.set_backend("spark")` now seamlessly reroutes all tool executions down to Spark SQL and Vectorized Pandas UDFs (PyArrow).
+
+### Changed
+- All 55 Core tools have been refactored into thin dispatchers to support multiple engine backends without altering user-facing APIs.
+- Updated `README.md` and created `USER_GUIDE.md` with complete documentation, tool reference tables, and declarative YAML pipeline configurations.
+
+## [0.1.0] — 2025-06-28
+
+- **InOut** class — `input_data`, `output_data`, `text_input`, `browse`, `directory`, `date_time_now`
+- **Preparation** class — `filter`, `formula`, `select`, `data_cleansing`, `sort`, `unique`, `sample`, `record_id`, `generate_rows`, `auto_field`, `multi_field_formula`, `multi_row_formula`, `tile`, `imputation`
+- **Join** class — `join`, `join_multiple`, `union`, `find_replace`, `append_fields`, `fuzzy_match`
+- **Transform** class — `summarize`, `transpose`, `cross_tab`, `running_total`, `count_records`
+- **Parse** class — `date_time`, `regex_match`, `regex_parse`, `regex_replace`, `regex_tokenize`, `text_to_columns`, `xml_parse`
+- **Developer** class — `base64_encode`, `base64_decode`, `download`, `column_info`, `dynamic_rename`
+- Full test suite with pytest
+- Type hints and Google-style docstrings on all public methods

etlpipe-2.0.0/CONTRIBUTING.md

@@ -0,0 +1,170 @@
+# Contributing to Etlpipe
+
+Thank you for your interest in contributing to Etlpipe! This guide will help you get started.
+
+---
+
+## Development Setup
+
+### Prerequisites
+
+- Python 3.10+
+- Git
+
+### Getting Started
+
+```bash
+# Clone the repository
+git clone https://github.com/tonystark7cris/etlpipe.git
+cd etlpipe
+
+# Create a virtual environment
+python -m venv venv
+source venv/bin/activate  # Linux/Mac
+venv\Scripts\activate     # Windows
+
+# Install with development dependencies
+pip install -e ".[dev]"
+
+# Verify installation
+pytest tests/ -v
+```
+
+---
+
+## Code Standards
+
+### Style & Formatting
+
+Etlpipe uses **[Ruff](https://docs.astral.sh/ruff/)** for both linting and formatting. Configuration is in [pyproject.toml](pyproject.toml):
+
+- **Target**: Python 3.10
+- **Line length**: 120 characters
+- **Lint rules**: E, F, I, W, UP, B, SIM
+
+```bash
+# Check formatting
+ruff format --check src/ tests/
+
+# Auto-format
+ruff format src/ tests/
+
+# Run linter
+ruff check src/ tests/
+
+# Auto-fix lint issues
+ruff check --fix src/ tests/
+```
+
+### Type Hints
+
+All public methods **must** have full type annotations. Use `from __future__ import annotations` at the top of every module.
+
+### Docstrings
+
+All public methods **must** have Google-style docstrings including:
+- One-line summary
+- `Args:` section with parameter descriptions
+- `Returns:` section
+- `Raises:` section (if applicable)
+- `Example:` section with a short usage snippet
+
+### Immutability
+
+All Etlpipe tool functions **must** be pure — they return new DataFrames and never mutate inputs. This is a core architectural invariant.
+
+---
+
+## Testing
+
+### Running Tests
+
+```bash
+# Full test suite with coverage
+pytest tests/ -v --cov=etlpipe --cov-report=term-missing
+
+# Run specific test file
+pytest tests/test_preparation.py -v
+
+# Run a single test
+pytest tests/test_contracts.py::TestExpectSchemaPass::test_round_trip -v
+```
+
+### Writing Tests
+
+- Place test files in `tests/` with the naming convention `test_<module>.py`
+- Use shared fixtures from `tests/conftest.py` where possible
+- Test both happy paths and error cases
+- For dual-engine features, ensure Pandas tests exist (Spark tests are optional but encouraged)
+
+### Coverage Requirements
+
+New code should maintain or improve overall test coverage. Critical paths (security, data contracts, PII scanning) require >90% coverage.
+
+---
+
+## Architecture
+
+### Engine Pattern
+
+Etlpipe uses an abstract **BackendEngine** pattern:
+
+1. **Public API** classes (`Preparation`, `Join`, etc.) are thin dispatchers
+2. They call `get_engine()` to obtain the active backend
+3. The backend (`PandasEngine` or `SparkEngine`) contains the actual implementation
+4. Both backends inherit from `BackendEngine` (ABC)
+
+When adding a new tool:
+1. Add the abstract method signature to `engines/base.py`
+2. Implement in `engines/pandas_engine.py`
+3. Implement in `engines/spark_engine.py`
+4. Add the public static method to the appropriate palette class
+5. Add tests for both backends
+
+### Architecture Decision Records
+
+Significant design decisions are documented as ADRs in `doc/adr/`. When proposing a major change, create a new ADR using the template at `doc/adr/000-template.md`.
+
+---
+
+## Pull Request Process
+
+1. **Branch**: Create a feature branch from `main` (e.g., `feature/add-pivot-tool`)
+2. **Implement**: Make your changes following the code standards above
+3. **Test**: Ensure all tests pass and add new tests for your changes
+4. **Lint**: Run `ruff format` and `ruff check` before pushing
+5. **PR**: Open a pull request with:
+   - Clear description of what changed and why
+   - Link to any related issues
+   - Screenshots/examples if applicable
+6. **Review**: Address review feedback promptly
+7. **Merge**: Squash-merge after approval
+
+---
+
+## Release Process
+
+Etlpipe follows [Semantic Versioning](https://semver.org/):
+
+- **PATCH** (1.0.x): Bug fixes, no API changes
+- **MINOR** (1.x.0): New features, backward-compatible
+- **MAJOR** (x.0.0): Breaking API changes
+
+### Release Checklist
+
+1. Update `src/etlpipe/_version.py` with the new version
+2. Update `CHANGELOG.md` following [Keep a Changelog](https://keepachangelog.com/) format
+3. Create a GitHub Release with a tag matching the version (e.g., `v1.1.0`)
+4. The CI/CD pipeline will automatically publish to PyPI via OIDC Trusted Publishing
+
+---
+
+## Security
+
+If you discover a security vulnerability, please report it privately. See [SECURITY.md](SECURITY.md) for details.
+
+---
+
+## License
+
+By contributing, you agree that your contributions will be licensed under the [MIT License](LICENSE).

etlpipe-2.0.0/DATABRICKS_GUIDE.md

@@ -0,0 +1,193 @@
+# End-to-End Etlpipe Implementation on Databricks
+
+This guide provides a step-by-step walkthrough for implementing a full, production-ready Etlpipe project on **Databricks**. 
+
+Because Etlpipe features a dynamic dual-engine architecture, you can write familiar Etlpipe-like logic in Python, and Etlpipe will translate it natively into distributed Spark execution under the hood.
+
+---
+
+## Architecture Overview
+
+1. **Ingestion**: Read raw data from Databricks Delta Tables (or DBFS/S3).
+2. **Transformation**: Execute Etlpipe logic utilizing the `spark` backend engine.
+3. **Validation**: Assert data quality before loading.
+4. **Load**: Write the transformed data back to a curated Delta Table.
+5. **Orchestration**: Schedule the notebook/script via Databricks Workflows (Jobs).
+
+---
+
+## Step 1: Cluster Setup
+
+To run Etlpipe on Databricks, you just need to install the core package. Databricks already provides the Spark backend natively.
+
+1. Navigate to your Databricks Workspace -> **Compute**.
+2. Select your target cluster (e.g., Databricks Runtime 13.3 LTS).
+3. Click the **Libraries** tab -> **Install New**.
+4. Select **PyPI** and enter: `etlpipe`
+5. Click **Install**. 
+
+Alternatively, if you are using Databricks Repos, add `etlpipe` to your `requirements.txt`.
+
+---
+
+## Step 2: Project Structure
+
+When using Databricks Repos (Git integration), we recommend structuring your Etlpipe project like a standard software engineering repository:
+
+```text
+/my-databricks-project
+├── notebooks/
+│   └── 01_run_pipeline.py     # Main entry point for Databricks Jobs
+├── src/
+│   ├── config.yaml            # Pipeline configuration
+│   └── pipeline.py            # Core Etlpipe logic
+├── tests/
+│   └── test_pipeline.py       # Unit tests (run locally via Pytest)
+└── requirements.txt           # Etlpipe[spark], etc.
+```
+
+---
+
+## Step 3: Core Pipeline Implementation (`src/pipeline.py`)
+
+Here is the core business logic. We explicitly configure Etlpipe to use the `spark` backend so that operations execute on the cluster rather than the driver node.
+
+```python
+import etlpipe
+from etlpipe import InOut, Preparation, Join, Transform, Developer
+
+def run_customer_360_pipeline():
+    # ==========================================
+    # 0. Set Backend to Spark
+    # ==========================================
+    etlpipe.set_backend("spark")
+    print("Executing Etlpipe Pipeline on Apache Spark Backend...")
+
+    # ==========================================
+    # 1. Ingest Data from Delta Tables
+    # ==========================================
+    # In Databricks, you can query Delta tables directly via Spark SQL syntax
+    # Etlpipe's spark backend treats SQL queries natively.
+    df_customers = InOut.input_data("dbfs:/mnt/lakehouse/raw/customers")
+    df_orders = InOut.input_data("dbfs:/mnt/lakehouse/raw/orders")
+    
+    # ==========================================
+    # 2. Data Cleansing & Preparation
+    # ==========================================
+    # Cleanse Customer data: upper case names, strip whitespace, handle nulls
+    df_customers_clean = Preparation.data_cleansing(
+        df_customers,
+        replace_nulls_with="",
+        strip_whitespace=True,
+        modify_case="upper"
+    )
+    
+    # Filter for completed orders
+    df_orders_valid, df_orders_invalid = Preparation.filter(
+        df_orders, 
+        "OrderStatus = 'COMPLETED'"
+    )
+    
+    # ==========================================
+    # 3. Join Datasets
+    # ==========================================
+    # Join (L: Unjoined Customers, J: Joined Data, R: Unjoined Orders)
+    left_unjoined, joined_data, right_unjoined = Join.join(
+        df_customers_clean, 
+        df_orders_valid, 
+        on="CustomerID"
+    )
+    
+    # ==========================================
+    # 4. Transform & Aggregate
+    # ==========================================
+    # Summarize Total Lifetime Value (LTV) and Order Count by Customer
+    summary_data = Transform.summarize(
+        joined_data,
+        group_by=["CustomerID", "CustomerName", "Region"],
+        aggregations={
+            "OrderAmount": ["sum", "mean"],
+            "OrderID": "count distinct"
+        }
+    )
+    
+    # Apply business logic using Formula
+    final_data = Preparation.formula(
+        summary_data,
+        column="CustomerTier",
+        expression="IF Sum_OrderAmount > 10000 THEN 'Platinum' ELSE 'Standard' ENDIF"
+    )
+    
+    # ==========================================
+    # 5. Data Quality Testing
+    # ==========================================
+    # Developer.test evaluates natively on the Spark DataFrame
+    Developer.test(
+        final_data,
+        condition_func=lambda df: df["Sum_OrderAmount"].min() >= 0,
+        error_msg="Data Quality Failure: Negative LTV detected."
+    )
+    
+    # ==========================================
+    # 6. Load Output to Delta Lake
+    # ==========================================
+    # Write the output back to the Databricks Lakehouse as a Delta table
+    InOut.output_data(
+        final_data, 
+        "dbfs:/mnt/lakehouse/curated/customer_360",
+        format="delta",
+        mode="overwrite"
+    )
+    
+    print("Pipeline executed successfully. Output written to Curated layer.")
+
+if __name__ == "__main__":
+    run_customer_360_pipeline()
+```
+
+---
+
+## Step 4: The Databricks Entrypoint (`notebooks/01_run_pipeline.py`)
+
+In Databricks, you typically create a notebook to serve as the entry point for your Workflows/Jobs. Because we defined our code in a `src` module, the notebook is extremely clean:
+
+```python
+# COMMAND ----------
+# MAGIC %pip install -r ../requirements.txt
+# COMMAND ----------
+
+import sys
+import os
+
+# Ensure the src directory is in the Python path
+sys.path.append(os.path.abspath("../src"))
+
+from pipeline import run_customer_360_pipeline
+
+# COMMAND ----------
+# Run the pipeline
+run_customer_360_pipeline()
+```
+
+---
+
+## Step 5: Scheduling via Databricks Workflows
+
+To run this pipeline automatically:
+
+1. Navigate to **Workflows** in the Databricks sidebar and click **Create Job**.
+2. Name the Job: `Etlpipe_Customer360_ETL`.
+3. In the task configuration:
+   - **Type**: Notebook
+   - **Source**: Workspace (or Git if using Repos).
+   - **Path**: Select `notebooks/01_run_pipeline.py`.
+   - **Compute**: Select the cluster you configured in Step 1 (or define a Job Cluster for cheaper, ephemeral execution).
+4. **Schedule**: Set a trigger (e.g., Daily at 2:00 AM) and configure Failure Alerts to notify your team via Email/Slack.
+
+---
+
+## Databricks Specific Best Practices
+
+- **Leverage Delta Lake**: When using `InOut.input_data` and `InOut.output_data`, specify paths starting with `dbfs:/` and utilize the `format="delta"` argument to leverage Databricks' optimized storage layer.
+- **Job Clusters vs. All-Purpose Clusters**: For scheduled Etlpipe pipelines, use **Job Clusters**. They are significantly cheaper and automatically terminate when the Etlpipe workflow completes.
+- **Avoid `.browse()` in Production**: While `InOut.browse()` is fantastic for debugging interactively in a Notebook, remove it from production scripts, as it forces Spark to collect data to the driver node, which can cause Out-Of-Memory (OOM) errors on massive datasets.

etlpipe-2.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 etlpipe Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.