spark-cell-runner 0.7.0__tar.gz

spark_cell_runner-0.7.0/PKG-INFO

@@ -0,0 +1,111 @@
+Metadata-Version: 2.4
+Name: spark-cell-runner
+Version: 0.7.0
+Summary: Run Databricks notebook-source .py cells and SQL against Databricks (Databricks Connect) or a local Sail Spark pool, with Unity Catalog sync and a local Delta / Iceberg warehouse.
+Author: Iamprashanth-1
+License: MIT
+Keywords: databricks,notebook,pyspark,spark,sail,unity-catalog,delta-lake,iceberg
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: pysail>=0.4
+Requires-Dist: pyspark-client
+Requires-Dist: databricks-sdk
+Provides-Extra: databricks
+Requires-Dist: databricks-connect; extra == "databricks"
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+
+# spark-cell-runner (Python CLI)
+
+Run Databricks notebook-source `.py` cells and SQL from the terminal — against
+Databricks (Databricks Connect) or fully offline on a local
+[Sail](https://github.com/lakehq/sail) Spark pool — with Unity Catalog sync and
+a local Delta / Iceberg warehouse. This is the pip-installable twin of the
+[Spark Cell Runner VS Code extension](https://marketplace.visualstudio.com/items?itemName=PrashanthReddyMunagala.spark-cell-runner);
+pools, warehouses, and Databricks profiles are shared between the two.
+
+```bash
+pip install spark-cell-runner
+```
+
+## Quickstart
+
+```bash
+# 1. Create and start a local pool (offline engine, ~1s startup, no JVM)
+spark-cell-runner pool create dev
+spark-cell-runner pool start dev
+
+# 2. Run a Databricks notebook export, cell by cell
+spark-cell-runner run my_notebook.py --pool dev
+
+# 3. Run SQL
+spark-cell-runner sql query.sql --pool dev
+echo "SELECT 1" | spark-cell-runner sql - --pool dev
+
+# 4. Mirror Unity Catalog schemas/tables into the local warehouse
+spark-cell-runner sync --catalog main --schema sales --mode schema
+spark-cell-runner sync --catalog main --mode data --format iceberg --limit 10000
+
+# 5. Explore and preview (JSON / CSV)
+spark-cell-runner explore --source local --level tables --pool dev
+spark-cell-runner preview ~/.spark-cell-runner/warehouse/dev/demo.db/orders
+```
+
+## Open table formats: Delta + Iceberg
+
+The local warehouse supports both open table formats natively (requires
+`pysail >= 0.4`):
+
+| | Delta | Iceberg |
+|---|---|---|
+| Write (Python) | `df.write.format('delta').save(path)` | `df.write.format('iceberg').save(path)` |
+| Write (SQL) | `CREATE TABLE t ... USING delta LOCATION '...'` | `CREATE TABLE t ... USING iceberg LOCATION '...'` |
+| Time travel | `VERSION AS OF 0` / `versionAsOf` option | `snapshotId` / `branch` / `tag` options |
+| On-disk marker | `_delta_log/` | `metadata/*.metadata.json` |
+
+A directory with either marker is a table: the warehouse registry re-registers
+on-disk tables (both formats) into every new session, the Unity Catalog
+explorer shows the format per table, and previews auto-detect the format.
+The `sync` command and the extension's `sparkCellRunner.syncFormat` setting
+choose the local format for UC sync (default `delta`).
+
+## Running on Databricks instead
+
+```bash
+spark-cell-runner run my_notebook.py --serverless
+spark-cell-runner run my_notebook.py --cluster-id 1234-567890 --profile DEFAULT
+```
+
+Requires the `databricks-connect` extra: `pip install "spark-cell-runner[databricks]"`.
+
+## Shared state with the VS Code extension
+
+- Pools: `~/.spark-cell-runner/pools/<name>.json` (create from either side)
+- Warehouses: `~/.spark-cell-runner/warehouse/<pool>/`
+- Databricks profiles: `~/.databrickscfg`
+
+## Notebook magic support
+
+`%sql` (shown via `result.show()`), `%md` (skipped), `%python`, `%pip`
+(reported, not auto-installed), `%sh`, and `%run` (reported) are handled;
+plain Python cells execute against the shared session namespace.
+
+## Development
+
+```bash
+cd python
+pip install -e ".[dev]"
+pytest tests
+```
+
+MIT License — see the repository for details.

spark_cell_runner-0.7.0/README.md

@@ -0,0 +1,84 @@
+# spark-cell-runner (Python CLI)
+
+Run Databricks notebook-source `.py` cells and SQL from the terminal — against
+Databricks (Databricks Connect) or fully offline on a local
+[Sail](https://github.com/lakehq/sail) Spark pool — with Unity Catalog sync and
+a local Delta / Iceberg warehouse. This is the pip-installable twin of the
+[Spark Cell Runner VS Code extension](https://marketplace.visualstudio.com/items?itemName=PrashanthReddyMunagala.spark-cell-runner);
+pools, warehouses, and Databricks profiles are shared between the two.
+
+```bash
+pip install spark-cell-runner
+```
+
+## Quickstart
+
+```bash
+# 1. Create and start a local pool (offline engine, ~1s startup, no JVM)
+spark-cell-runner pool create dev
+spark-cell-runner pool start dev
+
+# 2. Run a Databricks notebook export, cell by cell
+spark-cell-runner run my_notebook.py --pool dev
+
+# 3. Run SQL
+spark-cell-runner sql query.sql --pool dev
+echo "SELECT 1" | spark-cell-runner sql - --pool dev
+
+# 4. Mirror Unity Catalog schemas/tables into the local warehouse
+spark-cell-runner sync --catalog main --schema sales --mode schema
+spark-cell-runner sync --catalog main --mode data --format iceberg --limit 10000
+
+# 5. Explore and preview (JSON / CSV)
+spark-cell-runner explore --source local --level tables --pool dev
+spark-cell-runner preview ~/.spark-cell-runner/warehouse/dev/demo.db/orders
+```
+
+## Open table formats: Delta + Iceberg
+
+The local warehouse supports both open table formats natively (requires
+`pysail >= 0.4`):
+
+| | Delta | Iceberg |
+|---|---|---|
+| Write (Python) | `df.write.format('delta').save(path)` | `df.write.format('iceberg').save(path)` |
+| Write (SQL) | `CREATE TABLE t ... USING delta LOCATION '...'` | `CREATE TABLE t ... USING iceberg LOCATION '...'` |
+| Time travel | `VERSION AS OF 0` / `versionAsOf` option | `snapshotId` / `branch` / `tag` options |
+| On-disk marker | `_delta_log/` | `metadata/*.metadata.json` |
+
+A directory with either marker is a table: the warehouse registry re-registers
+on-disk tables (both formats) into every new session, the Unity Catalog
+explorer shows the format per table, and previews auto-detect the format.
+The `sync` command and the extension's `sparkCellRunner.syncFormat` setting
+choose the local format for UC sync (default `delta`).
+
+## Running on Databricks instead
+
+```bash
+spark-cell-runner run my_notebook.py --serverless
+spark-cell-runner run my_notebook.py --cluster-id 1234-567890 --profile DEFAULT
+```
+
+Requires the `databricks-connect` extra: `pip install "spark-cell-runner[databricks]"`.
+
+## Shared state with the VS Code extension
+
+- Pools: `~/.spark-cell-runner/pools/<name>.json` (create from either side)
+- Warehouses: `~/.spark-cell-runner/warehouse/<pool>/`
+- Databricks profiles: `~/.databrickscfg`
+
+## Notebook magic support
+
+`%sql` (shown via `result.show()`), `%md` (skipped), `%python`, `%pip`
+(reported, not auto-installed), `%sh`, and `%run` (reported) are handled;
+plain Python cells execute against the shared session namespace.
+
+## Development
+
+```bash
+cd python
+pip install -e ".[dev]"
+pytest tests
+```
+
+MIT License — see the repository for details.

spark_cell_runner-0.7.0/pyproject.toml

@@ -0,0 +1,49 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "spark-cell-runner"
+version = "0.7.0"
+description = "Run Databricks notebook-source .py cells and SQL against Databricks (Databricks Connect) or a local Sail Spark pool, with Unity Catalog sync and a local Delta / Iceberg warehouse."
+readme = "README.md"
+requires-python = ">=3.9"
+license = { text = "MIT" }
+authors = [{ name = "Iamprashanth-1" }]
+keywords = [
+  "databricks",
+  "notebook",
+  "pyspark",
+  "spark",
+  "sail",
+  "unity-catalog",
+  "delta-lake",
+  "iceberg",
+]
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Environment :: Console",
+  "Intended Audience :: Developers",
+  "License :: OSI Approved :: MIT License",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Topic :: Software Development :: Libraries :: Python Modules",
+]
+dependencies = [
+  "pysail>=0.4",
+  "pyspark-client",
+  "databricks-sdk",
+]
+
+[project.optional-dependencies]
+databricks = ["databricks-connect"]
+dev = ["pytest"]
+
+[project.scripts]
+spark-cell-runner = "spark_cell_runner.cli:main"
+
+[tool.setuptools]
+packages = ["spark_cell_runner"]

spark_cell_runner-0.7.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

spark_cell_runner-0.7.0/spark_cell_runner/__init__.py

@@ -0,0 +1,10 @@
+"""spark-cell-runner — headless engine for Databricks notebook cells and SQL.
+
+Runs Databricks notebook-source .py cells and SQL against Databricks
+(Databricks Connect) or a local Sail Spark pool, with Unity Catalog sync and
+a local Delta / Iceberg warehouse. The VS Code extension of the same name
+ships these same modules and spawns them by path; this package is the
+pip-installable CLI form.
+"""
+
+__version__ = "0.7.0"