redcliff 0.1.0__tar.gz

redcliff-0.1.0/.gitignore

@@ -0,0 +1,25 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+.venv/
+.pytest_cache/
+dist/
+build/
+
+# Node
+node_modules/
+web/dist/
+
+# Spike artifacts
+spikes/**/wheels/
+
+# Editors and OS
+.vscode/
+.idea/
+.DS_Store
+
+# Vendored by web/scripts/build-engine.mjs
+web/public/pyodide/
+web/public/engine/
+web/public/samples/

redcliff-0.1.0/PKG-INFO

@@ -0,0 +1,75 @@
+Metadata-Version: 2.5
+Name: redcliff
+Version: 0.1.0
+Summary: Open-source visual pipelines with lineage built in. Legacy ETL in, clean Spark out.
+Project-URL: Homepage, https://github.com/Fullcountr/redcliff
+Project-URL: Playground, https://fullcountr.github.io/redcliff/
+Project-URL: Documentation, https://github.com/Fullcountr/redcliff/tree/main/docs
+Project-URL: Issues, https://github.com/Fullcountr/redcliff/issues
+Author: Harishwar Reddy Korpol
+License-Expression: Apache-2.0
+Keywords: data lineage,data quality,dbt,duckdb,etl,informatica,migration,openlineage,powercenter,pyspark,spark
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Database
+Classifier: Topic :: Software Development :: Code Generators
+Requires-Python: >=3.11
+Requires-Dist: sqlglot>=30
+Provides-Extra: cli
+Requires-Dist: duckdb>=1.4; extra == 'cli'
+Requires-Dist: pyyaml>=6; extra == 'cli'
+Provides-Extra: dev
+Requires-Dist: duckdb>=1.4; extra == 'dev'
+Requires-Dist: jsonschema>=4; extra == 'dev'
+Requires-Dist: pytest>=8; extra == 'dev'
+Requires-Dist: pyyaml>=6; extra == 'dev'
+Provides-Extra: yaml
+Requires-Dist: pyyaml>=6; extra == 'yaml'
+Description-Content-Type: text/markdown
+
+# redcliff
+
+The Python engine behind [Redcliff](https://github.com/Fullcountr/redcliff), an open-source visual
+pipeline tool with lineage built in: **legacy ETL in, clean Spark out.**
+
+```text
+pip install "redcliff[cli]"
+```
+
+- **Import** legacy SQL (11 dialects, BTEQ scripts) and Informatica PowerCenter exports (mappings,
+  mapplets, sessions, workflows, parameter files) into a small YAML pipeline spec, with a
+  migration scorecard.
+- **Compile** the spec to Spark SQL, PySpark, DuckDB SQL, a dbt project or a Lakeflow
+  Declarative Pipeline.
+- **Trace** column-level lineage, a column's blast radius, and the source rows behind any row;
+  export OpenLineage.
+- **Check** data quality expectations, **diff** two versions of a pipeline, and let AI agents
+  edit specs through an **MCP server** that shows a diff before anything is written.
+
+```text
+redcliff import job.sql --dialect teradata -o pipeline.yaml
+redcliff import wf_export.xml --params wf.par -o pipeline.yaml --scorecard scorecard.md
+redcliff compile pipeline.yaml --to spark          # or pyspark, duckdb, dbt, lakeflow
+redcliff lineage pipeline.yaml --impact orders.status
+redcliff test pipeline.yaml                        # data quality expectations, in DuckDB
+redcliff run pipeline.yaml --var start_date=2026-09-01
+redcliff diff --git origin/main --data
+redcliff mcp --root .
+```
+
+The core depends only on [sqlglot](https://github.com/tobymao/sqlglot), so it also runs in the
+browser through Pyodide: try the [playground](https://fullcountr.github.io/redcliff/). The `cli`
+extra adds PyYAML and DuckDB for running pipelines locally.
+
+Documentation: [spec](https://github.com/Fullcountr/redcliff/blob/main/docs/spec.md),
+[outputs](https://github.com/Fullcountr/redcliff/blob/main/docs/outputs.md),
+[PowerCenter](https://github.com/Fullcountr/redcliff/blob/main/docs/powercenter.md),
+[pull request bot](https://github.com/Fullcountr/redcliff/blob/main/docs/pr-bot.md),
+[MCP server](https://github.com/Fullcountr/redcliff/blob/main/docs/mcp.md).
+Apache 2.0.

redcliff-0.1.0/README.md

@@ -0,0 +1,40 @@
+# redcliff
+
+The Python engine behind [Redcliff](https://github.com/Fullcountr/redcliff), an open-source visual
+pipeline tool with lineage built in: **legacy ETL in, clean Spark out.**
+
+```text
+pip install "redcliff[cli]"
+```
+
+- **Import** legacy SQL (11 dialects, BTEQ scripts) and Informatica PowerCenter exports (mappings,
+  mapplets, sessions, workflows, parameter files) into a small YAML pipeline spec, with a
+  migration scorecard.
+- **Compile** the spec to Spark SQL, PySpark, DuckDB SQL, a dbt project or a Lakeflow
+  Declarative Pipeline.
+- **Trace** column-level lineage, a column's blast radius, and the source rows behind any row;
+  export OpenLineage.
+- **Check** data quality expectations, **diff** two versions of a pipeline, and let AI agents
+  edit specs through an **MCP server** that shows a diff before anything is written.
+
+```text
+redcliff import job.sql --dialect teradata -o pipeline.yaml
+redcliff import wf_export.xml --params wf.par -o pipeline.yaml --scorecard scorecard.md
+redcliff compile pipeline.yaml --to spark          # or pyspark, duckdb, dbt, lakeflow
+redcliff lineage pipeline.yaml --impact orders.status
+redcliff test pipeline.yaml                        # data quality expectations, in DuckDB
+redcliff run pipeline.yaml --var start_date=2026-09-01
+redcliff diff --git origin/main --data
+redcliff mcp --root .
+```
+
+The core depends only on [sqlglot](https://github.com/tobymao/sqlglot), so it also runs in the
+browser through Pyodide: try the [playground](https://fullcountr.github.io/redcliff/). The `cli`
+extra adds PyYAML and DuckDB for running pipelines locally.
+
+Documentation: [spec](https://github.com/Fullcountr/redcliff/blob/main/docs/spec.md),
+[outputs](https://github.com/Fullcountr/redcliff/blob/main/docs/outputs.md),
+[PowerCenter](https://github.com/Fullcountr/redcliff/blob/main/docs/powercenter.md),
+[pull request bot](https://github.com/Fullcountr/redcliff/blob/main/docs/pr-bot.md),
+[MCP server](https://github.com/Fullcountr/redcliff/blob/main/docs/mcp.md).
+Apache 2.0.

redcliff-0.1.0/pyproject.toml

@@ -0,0 +1,67 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "redcliff"
+dynamic = ["version"]
+description = "Open-source visual pipelines with lineage built in. Legacy ETL in, clean Spark out."
+readme = "README.md"
+license = "Apache-2.0"
+requires-python = ">=3.11"
+authors = [{ name = "Harishwar Reddy Korpol" }]
+keywords = ["etl", "data lineage", "spark", "pyspark", "dbt", "informatica", "powercenter", "migration", "duckdb",
+            "openlineage", "data quality"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Information Technology",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Database",
+    "Topic :: Software Development :: Code Generators",
+]
+# Pure-Python dependencies only: this package must also run in Pyodide.
+dependencies = ["sqlglot>=30"]
+
+[project.optional-dependencies]
+# PyYAML is optional: JSON specs need nothing extra, and Pyodide ships PyYAML.
+yaml = ["pyyaml>=6"]
+# DuckDB runs previews, checks and traces; 1.4 added MERGE.
+cli = ["pyyaml>=6", "duckdb>=1.4"]
+dev = ["pytest>=8", "duckdb>=1.4", "pyyaml>=6", "jsonschema>=4"]
+
+[project.urls]
+Homepage = "https://github.com/Fullcountr/redcliff"
+Playground = "https://fullcountr.github.io/redcliff/"
+Documentation = "https://github.com/Fullcountr/redcliff/tree/main/docs"
+Issues = "https://github.com/Fullcountr/redcliff/issues"
+
+[project.scripts]
+redcliff = "redcliff.cli:main"
+
+[tool.hatch.version]
+path = "src/redcliff/__init__.py"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/redcliff"]
+
+[tool.hatch.build.targets.sdist]
+include = ["src/redcliff", "README.md", "pyproject.toml"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.ruff]
+line-length = 130
+target-version = "py311"
+
+[tool.ruff.lint]
+select = ["E", "F", "W", "B", "UP", "SIM"]
+ignore = ["SIM108", "B905"]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/*" = ["E501"]

redcliff-0.1.0/src/redcliff/__init__.py

@@ -0,0 +1,43 @@
+"""Redcliff: open-source visual pipelines with lineage built in."""
+
+from redcliff.columns import infer_columns
+from redcliff.compile import CompileError, compile_pyspark, compile_query, compile_script
+from redcliff.importer import ImportResult, SQLImportError, import_sql
+from redcliff.lineage import Ref, blast_radius, build_graph, column_issues, lineage_report, openlineage_facet
+from redcliff.powercenter import PowerCenterImportError, import_powercenter
+from redcliff.scorecard import Scorecard, ScoreItem
+from redcliff.ssis import SSISImportError, import_ssis
+from redcliff.spec import Issue, Node, Pipeline, SpecError, json_schema, load_spec, parse_spec, validate
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "CompileError",
+    "compile_pyspark",
+    "compile_query",
+    "compile_script",
+    "infer_columns",
+    "ImportResult",
+    "SQLImportError",
+    "import_sql",
+    "PowerCenterImportError",
+    "import_powercenter",
+    "SSISImportError",
+    "import_ssis",
+    "Scorecard",
+    "ScoreItem",
+    "Ref",
+    "blast_radius",
+    "build_graph",
+    "column_issues",
+    "lineage_report",
+    "openlineage_facet",
+    "Issue",
+    "Node",
+    "Pipeline",
+    "SpecError",
+    "json_schema",
+    "load_spec",
+    "parse_spec",
+    "validate",
+]