kenze 0.3.0__tar.gz

kenze-0.3.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Ken
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

kenze-0.3.0/PKG-INFO

@@ -0,0 +1,153 @@
+Metadata-Version: 2.4
+Name: kenze
+Version: 0.3.0
+Summary: Big-file data prep that never runs out of memory — no SQL required. Powered by DuckDB.
+Author: Ken
+License: MIT
+Project-URL: Homepage, https://pypi.org/project/kenze/
+Keywords: duckdb,data,csv,parquet,etl,cli,big-data,dataframe,data-cleaning
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Utilities
+Classifier: Environment :: Console
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: duckdb>=0.10
+Requires-Dist: psutil>=5.9
+Provides-Extra: cloud
+Requires-Dist: duckdb>=0.10; extra == "cloud"
+Dynamic: license-file
+
+# kenze
+
+**Big-file data prep that never runs out of memory — no SQL required.**
+
+`kenze` is a tiny command-line tool for cleaning and reshaping data files
+(CSV, Parquet, JSON) that are too big for pandas. It's a friendly front-end
+over [DuckDB](https://duckdb.org): DuckDB does the heavy lifting (streaming,
+disk-spill, all your CPU cores) and `kenze` makes it a one-liner — and
+auto-configures memory so your job doesn't crash.
+
+```bash
+pip install kenze
+```
+
+One name for everything: `pip install kenze` → the `kenze` command → `import kenze`.
+
+## Why
+
+- **It doesn't OOM.** Memory is capped to a fraction of *free* RAM and DuckDB
+  spills to disk instead of dying. Point it at a file bigger than your RAM; it's fine.
+- **No SQL, no pandas.** Simple verbs, or a readable recipe file.
+- **One streaming pass.** A whole recipe compiles to a single query — no
+  intermediate files, so it's fast and light.
+- **Any format, local or cloud.** CSV / Parquet / JSON, plain or `.gz`,
+  on disk or on `s3://` / `gs://` / `https://` — auto-detected.
+
+## One-liners
+
+```bash
+kenze profile  sales.parquet                          # schema + row count, instantly
+kenze peek     sales.parquet                           # first rows + types + null counts
+kenze stats    sales.parquet                           # per-column min/max/nulls/unique
+kenze check    sales.csv                               # is the file valid? any bad rows?
+
+kenze keep     sales.parquet --cols id,city,amount -o small.csv
+kenze drop     users.csv     --cols email,phone    -o clean.parquet
+kenze filter   sales.parquet --where "amount > 100" -o big.csv
+kenze rename   sales.csv     --map "amount:total"   -o out.csv
+kenze cast     users.csv     --types "zip:VARCHAR"  -o out.parquet   # keep leading zeros
+kenze fillna   users.csv     --with "city:Unknown"  -o out.csv
+kenze mask     users.csv     --cols email,ssn --method hash -o safe.csv
+kenze dedup    users.csv     --on id               -o unique.parquet
+kenze sample   sales.parquet --n 50000             -o sample.csv
+kenze clip     points.parquet --bbox -10,35,5,45    -o region.parquet
+
+kenze join     orders.csv users.parquet --on user_id -o joined.parquet
+kenze diff     old.csv new.csv --on id                # added / removed / changed
+kenze pivot    sales.csv --on city --values amount --agg sum --group region -o wide.csv
+kenze split    sales.parquet --by city -o by_city/    # one file per value
+kenze partition sales.parquet --by year -o lake/      # hive year=2026/ folders
+kenze convert  sales.parquet -o sales.csv             # just change format
+
+kenze sql  "SELECT *, lag(amount) OVER (ORDER BY ts) FROM 'sales.parquet'" -o out.csv
+```
+
+Read or write the cloud directly (nothing to download first):
+
+```bash
+kenze filter s3://bucket/huge.parquet --where "amount > 0" -o local.csv
+```
+
+Pipe like any Unix tool (use `-` for stdin/stdout):
+
+```bash
+cat data.csv | kenze filter - --where "x > 1" -o - | gzip > out.csv.gz
+```
+
+## Recipes
+
+Chain steps in a readable `.dq` file — they run as one streaming pass:
+
+```yaml
+# clean.dq
+input:  data/sales_${DAY}.parquet     # ${DAY} filled from --set or the environment
+keep:   [id, city, amount]
+types:  zip:VARCHAR
+filter: amount > 0
+fillna: city:Unknown
+dedup:  id
+sample: 50000
+output: out/clean.csv
+```
+
+```bash
+kenze run clean.dq --set DAY=2026-07-14
+kenze recipe                 # show every valid recipe step
+kenze eject clean.dq --to sql    # print the raw DuckDB SQL (no lock-in)
+```
+
+## From Python
+
+```python
+import kenze
+kenze.sift("big.parquet", "clean.csv", keep=["id", "city"], filter="amount > 0", sample=50000)
+rows = kenze.sql("SELECT city, count(*) FROM 'big.parquet' GROUP BY 1")
+kenze.profile("big.parquet")
+```
+
+## Handy flags
+
+- `--memory-limit 8` — pin the RAM budget (GB) for reproducible / SLA runs.
+- `--temp-dir D:/spill` — put disk-spill where there's room.
+- `--skip-bad-lines` — ignore malformed rows in a dirty CSV.
+- `--log run.json` — write a run manifest (inputs, rows, timing).
+- Writes are **atomic** — a cancelled run never leaves a half-written file.
+
+## Commands
+
+`profile` · `peek` · `stats` · `check` · `validate` · `keep` · `drop` · `rename` · `cast` ·
+`fillna` · `mask` · `filter` · `dedup` · `sample` · `head` · `clip` · `convert` · `join` ·
+`diff` · `pivot` · `split` · `partition` · `sql` · `eject` · `run` · `recipe`
+
+## Where it stops (on purpose)
+
+kenze is one dependency and one machine — that's the whole point. It maxes out
+your cores and spills to disk so a single laptop or VM can chew through files far
+bigger than its RAM. It does **not** run a cluster. If you've genuinely outgrown
+one machine (multi-terabyte, distributed pipelines with SLAs and lineage
+tracking), reach for Spark/Dask — kenze is the tool you use *before* you need those.
+
+## Troubleshooting
+
+**`'kenze' is not recognized` / `kenze: command not found`?**
+`pip` installed kenze correctly — the command just landed in a folder that isn't on
+your PATH (this affects every pip-installed CLI). Options:
+
+- **Use it now, no setup:** `python -m kenze --help`
+- **Fix it for good:** reinstall Python from [python.org](https://www.python.org/downloads/)
+  with **"Add python.exe to PATH"** ticked, or use `python -m pipx install kenze`.
+
+MIT licensed.

kenze-0.3.0/README.md

@@ -0,0 +1,131 @@
+# kenze
+
+**Big-file data prep that never runs out of memory — no SQL required.**
+
+`kenze` is a tiny command-line tool for cleaning and reshaping data files
+(CSV, Parquet, JSON) that are too big for pandas. It's a friendly front-end
+over [DuckDB](https://duckdb.org): DuckDB does the heavy lifting (streaming,
+disk-spill, all your CPU cores) and `kenze` makes it a one-liner — and
+auto-configures memory so your job doesn't crash.
+
+```bash
+pip install kenze
+```
+
+One name for everything: `pip install kenze` → the `kenze` command → `import kenze`.
+
+## Why
+
+- **It doesn't OOM.** Memory is capped to a fraction of *free* RAM and DuckDB
+  spills to disk instead of dying. Point it at a file bigger than your RAM; it's fine.
+- **No SQL, no pandas.** Simple verbs, or a readable recipe file.
+- **One streaming pass.** A whole recipe compiles to a single query — no
+  intermediate files, so it's fast and light.
+- **Any format, local or cloud.** CSV / Parquet / JSON, plain or `.gz`,
+  on disk or on `s3://` / `gs://` / `https://` — auto-detected.
+
+## One-liners
+
+```bash
+kenze profile  sales.parquet                          # schema + row count, instantly
+kenze peek     sales.parquet                           # first rows + types + null counts
+kenze stats    sales.parquet                           # per-column min/max/nulls/unique
+kenze check    sales.csv                               # is the file valid? any bad rows?
+
+kenze keep     sales.parquet --cols id,city,amount -o small.csv
+kenze drop     users.csv     --cols email,phone    -o clean.parquet
+kenze filter   sales.parquet --where "amount > 100" -o big.csv
+kenze rename   sales.csv     --map "amount:total"   -o out.csv
+kenze cast     users.csv     --types "zip:VARCHAR"  -o out.parquet   # keep leading zeros
+kenze fillna   users.csv     --with "city:Unknown"  -o out.csv
+kenze mask     users.csv     --cols email,ssn --method hash -o safe.csv
+kenze dedup    users.csv     --on id               -o unique.parquet
+kenze sample   sales.parquet --n 50000             -o sample.csv
+kenze clip     points.parquet --bbox -10,35,5,45    -o region.parquet
+
+kenze join     orders.csv users.parquet --on user_id -o joined.parquet
+kenze diff     old.csv new.csv --on id                # added / removed / changed
+kenze pivot    sales.csv --on city --values amount --agg sum --group region -o wide.csv
+kenze split    sales.parquet --by city -o by_city/    # one file per value
+kenze partition sales.parquet --by year -o lake/      # hive year=2026/ folders
+kenze convert  sales.parquet -o sales.csv             # just change format
+
+kenze sql  "SELECT *, lag(amount) OVER (ORDER BY ts) FROM 'sales.parquet'" -o out.csv
+```
+
+Read or write the cloud directly (nothing to download first):
+
+```bash
+kenze filter s3://bucket/huge.parquet --where "amount > 0" -o local.csv
+```
+
+Pipe like any Unix tool (use `-` for stdin/stdout):
+
+```bash
+cat data.csv | kenze filter - --where "x > 1" -o - | gzip > out.csv.gz
+```
+
+## Recipes
+
+Chain steps in a readable `.dq` file — they run as one streaming pass:
+
+```yaml
+# clean.dq
+input:  data/sales_${DAY}.parquet     # ${DAY} filled from --set or the environment
+keep:   [id, city, amount]
+types:  zip:VARCHAR
+filter: amount > 0
+fillna: city:Unknown
+dedup:  id
+sample: 50000
+output: out/clean.csv
+```
+
+```bash
+kenze run clean.dq --set DAY=2026-07-14
+kenze recipe                 # show every valid recipe step
+kenze eject clean.dq --to sql    # print the raw DuckDB SQL (no lock-in)
+```
+
+## From Python
+
+```python
+import kenze
+kenze.sift("big.parquet", "clean.csv", keep=["id", "city"], filter="amount > 0", sample=50000)
+rows = kenze.sql("SELECT city, count(*) FROM 'big.parquet' GROUP BY 1")
+kenze.profile("big.parquet")
+```
+
+## Handy flags
+
+- `--memory-limit 8` — pin the RAM budget (GB) for reproducible / SLA runs.
+- `--temp-dir D:/spill` — put disk-spill where there's room.
+- `--skip-bad-lines` — ignore malformed rows in a dirty CSV.
+- `--log run.json` — write a run manifest (inputs, rows, timing).
+- Writes are **atomic** — a cancelled run never leaves a half-written file.
+
+## Commands
+
+`profile` · `peek` · `stats` · `check` · `validate` · `keep` · `drop` · `rename` · `cast` ·
+`fillna` · `mask` · `filter` · `dedup` · `sample` · `head` · `clip` · `convert` · `join` ·
+`diff` · `pivot` · `split` · `partition` · `sql` · `eject` · `run` · `recipe`
+
+## Where it stops (on purpose)
+
+kenze is one dependency and one machine — that's the whole point. It maxes out
+your cores and spills to disk so a single laptop or VM can chew through files far
+bigger than its RAM. It does **not** run a cluster. If you've genuinely outgrown
+one machine (multi-terabyte, distributed pipelines with SLAs and lineage
+tracking), reach for Spark/Dask — kenze is the tool you use *before* you need those.
+
+## Troubleshooting
+
+**`'kenze' is not recognized` / `kenze: command not found`?**
+`pip` installed kenze correctly — the command just landed in a folder that isn't on
+your PATH (this affects every pip-installed CLI). Options:
+
+- **Use it now, no setup:** `python -m kenze --help`
+- **Fix it for good:** reinstall Python from [python.org](https://www.python.org/downloads/)
+  with **"Add python.exe to PATH"** ticked, or use `python -m pipx install kenze`.
+
+MIT licensed.

kenze-0.3.0/pyproject.toml

@@ -0,0 +1,35 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "kenze"
+version = "0.3.0"
+description = "Big-file data prep that never runs out of memory — no SQL required. Powered by DuckDB."
+readme = "README.md"
+requires-python = ">=3.9"
+license = { text = "MIT" }
+authors = [{ name = "Ken" }]
+keywords = ["duckdb", "data", "csv", "parquet", "etl", "cli", "big-data", "dataframe", "data-cleaning"]
+classifiers = [
+  "Programming Language :: Python :: 3",
+  "License :: OSI Approved :: MIT License",
+  "Topic :: Scientific/Engineering",
+  "Topic :: Utilities",
+  "Environment :: Console",
+]
+# psutil is core: the never-OOM memory sizing is the headline feature, so it
+# ships by default rather than hiding behind an optional extra.
+dependencies = ["duckdb>=0.10", "psutil>=5.9"]
+
+[project.optional-dependencies]
+cloud = ["duckdb>=0.10"]   # S3/GCS/Azure work out of the box via DuckDB httpfs
+
+[project.urls]
+Homepage = "https://pypi.org/project/kenze/"
+
+[project.scripts]
+kenze = "kenze.cli:main"
+
+[tool.setuptools.packages.find]
+where = ["src"]

kenze-0.3.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

kenze-0.3.0/src/kenze/__init__.py

@@ -0,0 +1,70 @@
+"""kenze - big-file data prep that never runs out of memory.
+
+One `pip install`, then simple commands. DuckDB does the heavy lifting under
+the hood (streaming, disk-spill, all cores); kenze just makes it a one-liner
+and auto-configures memory so it doesn't OOM.
+
+You can also use it from Python:
+
+    import kenze
+    kenze.sift("big.parquet", "clean.csv", keep=["id", "city"],
+               filter="amount > 0", sample=50000)
+    rows = kenze.sql("SELECT city, count(*) FROM 'big.parquet' GROUP BY 1")
+    kenze.profile("big.parquet")
+"""
+from __future__ import annotations
+
+from .engine import connect
+from .ops import (
+    build_query,
+    check,
+    diff,
+    join,
+    partition,
+    peek,
+    pivot,
+    profile,
+    run_spec,
+    run_sql,
+    split,
+    stats,
+    validate,
+)
+
+__version__ = "0.3.0"
+
+
+def sift(input, output, con=None, quiet=True, **steps):
+    """Run one streaming pass. steps = keep/drop/filter/bbox/types/fillna/
+    mask/rename/dedup/sample/head (same words as the recipe / CLI)."""
+    spec = {"input": input, "output": output, **steps}
+    return run_spec(spec, con=con, quiet=quiet)
+
+
+def run(recipe_path, variables=None, con=None, quiet=True):
+    """Run a .dq recipe file from Python."""
+    from .recipe import parse
+
+    with open(recipe_path, encoding="utf-8") as f:
+        spec = parse(f.read(), variables=variables)
+    return run_spec(spec, con=con, quiet=quiet)
+
+
+def sql(query, con=None):
+    """Run any DuckDB SQL and get the rows back (window functions, aggregates)."""
+    own = con is None
+    con = con or connect()
+    try:
+        return con.execute(query).fetchall()
+    finally:
+        if own:
+            con.close()
+
+
+__all__ = [
+    "connect", "sift", "run", "sql",
+    "run_spec", "build_query", "run_sql",
+    "profile", "stats", "peek", "check", "validate",
+    "join", "diff", "split", "partition", "pivot",
+    "__version__",
+]

kenze-0.3.0/src/kenze/__main__.py

@@ -0,0 +1,4 @@
+from .cli import main
+
+if __name__ == "__main__":
+    main()