ducl 0.1.0__tar.gz

ducl-0.1.0/.gitignore

@@ -0,0 +1,12 @@
+__pycache__/
+*.pyc
+*.egg-info/
+dist/
+build/
+.eggs/
+*.feather
+*.parquet
+src/ducl/pwalk2/pwalk2
+!src/ducl/pwalk2/pwalk2.c
+.pytest_cache/
+.venv/

ducl-0.1.0/PKG-INFO

@@ -0,0 +1,15 @@
+Metadata-Version: 2.4
+Name: ducl
+Version: 0.1.0
+Summary: Disk Usage Command Line toolkit — scan, dashboard, query
+License-Expression: MIT
+Requires-Python: >=3.10
+Requires-Dist: click>=8.0
+Requires-Dist: numpy>=1.24
+Requires-Dist: polars>=1.0
+Requires-Dist: pyarrow>=14.0
+Provides-Extra: dev
+Requires-Dist: boto3>=1.28; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Provides-Extra: s3
+Requires-Dist: boto3>=1.28; extra == 's3'

ducl-0.1.0/README.md

@@ -0,0 +1,47 @@
+# ducl — Disk Usage Command Line toolkit
+
+Scan filesystems (or S3 buckets), build interactive dashboards, and query scan data — all from one CLI.
+
+## Install
+
+```bash
+pip install ducl            # from PyPI
+pip install ducl[s3]        # with S3 support (boto3)
+pip install -e ".[dev]"     # development (editable + pytest)
+```
+
+On Linux the bundled **pwalk2** C binary is compiled automatically at install time.
+
+## Usage
+
+```bash
+# Scan a filesystem → Feather file + dashboard
+ducl scan /mnt/data -o scan.feather
+
+# Scan an S3 bucket
+ducl scan my-bucket -o bucket.feather --s3
+
+# Build dashboard from existing Feather
+ducl dashboard scan.feather ./output/
+
+# Incremental update after rescanning a subtree
+ducl update ./output/ subtree.feather
+
+# Query scan data
+ducl query scan.feather --under /mnt/data/models/ --ext wav --top 10
+
+# Run bundled pwalk2 directly
+ducl pwalk2 /mnt/data --threads 64
+```
+
+## Development
+
+```bash
+cd v3
+pip install -e ".[dev]"
+pytest -v
+```
+
+## License
+
+MIT

ducl-0.1.0/hatch_build.py

@@ -0,0 +1,21 @@
+"""Hatchling build hook -- compiles pwalk2 from bundled C sources."""
+
+import os
+import platform
+import subprocess
+
+from hatchling.builders.hooks.plugin.interface import BuildHookInterface
+
+
+class CustomBuildHook(BuildHookInterface):
+    def initialize(self, version, build_data):
+        if platform.system() != "Linux":
+            return  # pwalk2 is Linux-only; skip silently
+        pw2_dir = os.path.join(self.root, "src", "ducl", "pwalk2")
+        if not os.path.isdir(pw2_dir):
+            return
+        subprocess.check_call(["make", "-C", pw2_dir, "pwalk2"])
+        # Include compiled binary in wheel
+        binary = os.path.join(pw2_dir, "pwalk2")
+        if os.path.isfile(binary):
+            build_data.setdefault("force-include", {})[binary] = "ducl/pwalk2/pwalk2"

ducl-0.1.0/pyproject.toml

@@ -0,0 +1,35 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "ducl"
+version = "0.1.0"
+description = "Disk Usage Command Line toolkit — scan, dashboard, query"
+requires-python = ">=3.10"
+license = "MIT"
+dependencies = [
+    "click>=8.0",
+    "polars>=1.0",
+    "pyarrow>=14.0",
+    "numpy>=1.24",
+]
+
+[project.optional-dependencies]
+s3 = ["boto3>=1.28"]
+dev = ["pytest>=7.0", "boto3>=1.28"]
+
+[project.scripts]
+ducl = "ducl.cli:cli"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/ducl"]
+
+[tool.hatch.build.targets.wheel.hooks.custom]
+path = "hatch_build.py"
+
+[tool.hatch.build.targets.sdist]
+include = ["src/ducl/**", "tests/**", "pyproject.toml", "README.md", "hatch_build.py"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

ducl-0.1.0/src/ducl/__init__.py

@@ -0,0 +1,3 @@
+"""ducl — Disk Usage Command Line toolkit."""
+
+__version__ = "0.1.0"

ducl-0.1.0/src/ducl/agg.py

@@ -0,0 +1,88 @@
+"""Batch processing for aggregation sidecars."""
+
+from __future__ import annotations
+
+import numpy as np
+import polars as pl
+import pyarrow as pa
+
+from .schema import AGG_GROUP_COLS, EXAMPLES_TOP_K, HIST_EDGES
+from .ext import normalize_ext_expr
+
+
+def process_batch(batch: pa.RecordBatch) -> tuple[pl.DataFrame, pl.DataFrame]:
+    """Process a single pyarrow RecordBatch into agg and example DataFrames.
+
+    Returns (agg_df, examples_df).
+    """
+    df = pl.from_arrow(batch)
+
+    # Filter to files only
+    df = df.filter(pl.col("child_count") == -1)
+    if df.height == 0:
+        agg_empty = pl.DataFrame(schema={
+            "dir_path": pl.Utf8, "ext": pl.Utf8, "leaf_folder": pl.Utf8,
+            "size_bin": pl.Int8, "n_components": pl.Int32,
+            "file_count": pl.Int64, "total_size": pl.Int64,
+        })
+        ex_empty = pl.DataFrame(schema={
+            "dir_path": pl.Utf8, "size_bin": pl.Int8, "filename": pl.Utf8,
+            "ext": pl.Utf8, "size": pl.Int64,
+        })
+        return agg_empty, ex_empty
+
+    # Compute derived columns
+    df = df.select("path", "ext", "size").with_columns(
+        pl.col("path").str.replace(r"/[^/]+$", "").alias("dir_path"),
+        pl.col("path").str.extract(r"([^/]+)/[^/]+$").alias("leaf_folder"),
+        pl.col("path").str.extract(r"([^/]+)$").alias("filename"),
+    ).with_columns(
+        normalize_ext_expr(),
+        (pl.col("dir_path").str.count_matches("/") + 2).cast(pl.Int32).alias("n_components"),
+    )
+
+    # Assign size bins
+    sizes_np = df["size"].to_numpy().astype(float)
+    bins = np.digitize(sizes_np, HIST_EDGES).astype(np.int8)
+    df = df.with_columns(pl.lit(pl.Series("size_bin", bins)).alias("size_bin"))
+
+    # Agg: group by (dir_path, ext, leaf_folder, size_bin, n_components)
+    agg_df = (
+        df.group_by(AGG_GROUP_COLS)
+        .agg(
+            pl.len().cast(pl.Int64).alias("file_count"),
+            pl.col("size").sum().alias("total_size"),
+        )
+    )
+
+    # Examples: top-3 largest files per (dir_path, size_bin)
+    examples_df = (
+        df.sort("size", descending=True)
+        .group_by(["dir_path", "size_bin"])
+        .head(EXAMPLES_TOP_K)
+        .select("dir_path", "size_bin", "filename", "ext", "size")
+    )
+
+    return agg_df, examples_df
+
+
+def compact_agg(batches: list[pl.DataFrame]) -> pl.DataFrame:
+    """Re-aggregate accumulated agg batches to bound memory."""
+    combined = pl.concat(batches)
+    return (
+        combined.group_by(AGG_GROUP_COLS)
+        .agg(
+            pl.col("file_count").sum().alias("file_count"),
+            pl.col("total_size").sum().alias("total_size"),
+        )
+    )
+
+
+def compact_examples(batches: list[pl.DataFrame]) -> pl.DataFrame:
+    """Re-pick top-3 from accumulated example batches."""
+    combined = pl.concat(batches)
+    return (
+        combined.sort("size", descending=True)
+        .group_by(["dir_path", "size_bin"])
+        .head(EXAMPLES_TOP_K)
+    )

ducl-0.1.0/src/ducl/cli.py

@@ -0,0 +1,113 @@
+"""ducl CLI — one entry point for disk-usage operations."""
+
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+
+import click
+
+
+@click.group()
+@click.version_option(package_name="ducl")
+def cli():
+    """Disk Usage Command Line toolkit."""
+
+
+@cli.command()
+@click.argument("path")
+@click.option("-o", "--output", required=True, help="Output .feather file path")
+@click.option("--s3", "use_s3", is_flag=True, help="Treat PATH as an S3 bucket name")
+@click.option("--no-agg", is_flag=True, help="Skip sidecar aggregation")
+@click.option("--no-dashboard", is_flag=True, help="Skip automatic dashboard build")
+@click.option("--block-size", type=int, default=256 << 20, help="CSV read block size (default 256 MiB)")
+@click.option("-w", "--workers", type=int, default=32, help="S3 parallel listing threads")
+@click.option("--discover-depth", type=int, default=2, help="S3 prefix discovery depth")
+@click.option("--endpoint-url", help="S3-compatible endpoint")
+@click.option("--profile", help="AWS profile name")
+@click.option("--region", help="AWS region")
+def scan(path, output, use_s3, no_agg, no_dashboard, block_size, workers,
+         discover_depth, endpoint_url, profile, region):
+    """Scan a filesystem or S3 bucket into a Feather file."""
+    do_agg = not no_agg
+
+    if use_s3:
+        from .s3scan import scan_bucket
+        scan_bucket(
+            bucket=path,
+            output=output,
+            workers=workers,
+            discover_depth=discover_depth,
+            do_agg=do_agg,
+            endpoint_url=endpoint_url,
+            profile=profile,
+            region=region,
+        )
+    else:
+        from .scan import scan_filesystem
+        scan_filesystem(path, output, do_agg=do_agg, block_size=block_size)
+
+    if not no_dashboard and os.path.exists(output):
+        out_dir = Path(output).parent / (Path(output).stem + "_dashboard")
+        click.echo(f"Building dashboard in {out_dir} ...", err=True)
+        from .dashboard import build
+        build(output, out_dir)
+
+
+@cli.command()
+@click.argument("feather")
+@click.argument("output_dir")
+def dashboard(feather, output_dir):
+    """Build a dashboard from a Feather scan file."""
+    from .dashboard import build
+    build(feather, output_dir)
+
+
+@cli.command("update")
+@click.argument("dashboard_dir")
+@click.argument("subtree_feather")
+def update_cmd(dashboard_dir, subtree_feather):
+    """Incrementally update a dashboard after a subtree rescan."""
+    from .dashboard import update
+    update(dashboard_dir, subtree_feather)
+
+
+@cli.command()
+@click.argument("feather")
+@click.option("--under", help="Path prefix filter")
+@click.option("--name", help="Exact match on any path component")
+@click.option("--name-glob", help="Glob pattern on any path component")
+@click.option("--ext", multiple=True, help="Extension filter (repeatable)")
+@click.option("--min-size", help="Minimum file size (K/M/G/T suffixes)")
+@click.option("--max-size", help="Maximum file size (K/M/G/T suffixes)")
+@click.option("--path-contains", help="Substring match on full path")
+@click.option("--dirs", is_flag=True, help="Directories only (default: files only)")
+@click.option("--top", type=int, help="Show N largest entries")
+@click.option("--delete", is_flag=True, help="Delete matching files from disk")
+def query(feather, **kwargs):
+    """Query a .feather scan file with du-style output."""
+    from .query import run_query
+    ext_vals = list(kwargs.pop("ext")) or None
+    run_query(feather, ext=ext_vals, **kwargs)
+
+
+@cli.command(
+    "pwalk2",
+    context_settings=dict(
+        ignore_unknown_options=True,
+        allow_extra_args=True,
+    ),
+)
+@click.argument("args", nargs=-1, type=click.UNPROCESSED)
+def pwalk2_cmd(args):
+    """Run the bundled pwalk2 binary (all arguments forwarded)."""
+    binary = Path(__file__).parent / "pwalk2" / "pwalk2"
+    if not binary.exists():
+        click.echo(
+            "Error: pwalk2 binary not found. "
+            "pwalk2 requires Linux and is compiled at install time.",
+            err=True,
+        )
+        sys.exit(1)
+    os.execvp(str(binary), [str(binary)] + list(args))