Qubx 1.3.2__tar.gz → 1.3.3.dev2__tar.gz

{qubx-1.3.2 → qubx-1.3.3.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: Qubx
-Version: 1.3.2
+Version: 1.3.3.dev2
 Summary: Qubx - Quantitative Trading Framework
 Project-URL: homepage, https://xlydian.com
 Project-URL: repository, https://github.com/xLydianSoftware/Qubx

{qubx-1.3.2 → qubx-1.3.3.dev2}/src/qubx/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.3.2'
-__version_tuple__ = version_tuple = (1, 3, 2)
+__version__ = version = '1.3.3.dev2'
+__version_tuple__ = version_tuple = (1, 3, 3, 'dev2')
 
 __commit_id__ = commit_id = None

{qubx-1.3.2 → qubx-1.3.3.dev2}/src/qubx/cli/commands.py

@@ -1060,6 +1060,25 @@ def s3_rm_cmd(path: str, recursive: bool, yes: bool):
     s3_rm(path, recursive=recursive)
 
 
+@s3.command("parquet-stats")
+@click.argument("path", type=str)
+@click.option("--per-column", is_flag=True, help="Show per-column chunk sizes for row group 0.")
+def s3_parquet_stats_cmd(path: str, per_column: bool):
+    """
+    Print parquet file-level summary and row-group size distribution.
+
+    Accepts both S3 paths (account:bucket/key) and local paths. Reads only
+    the footer, so the full file is NOT downloaded.
+
+    Examples:\n
+      qubx s3 parquet-stats r2:dvault-features/.../part-2026-03.parquet\n
+      qubx s3 parquet-stats ./local.parquet --per-column
+    """
+    from .s3 import s3_parquet_stats
+
+    s3_parquet_stats(path, per_column=per_column)
+
+
 @s3.command("cp")
 @click.argument("src", type=str)
 @click.argument("dst", type=str)

{qubx-1.3.2 → qubx-1.3.3.dev2}/src/qubx/cli/release.py

@@ -1454,12 +1454,16 @@ def _bundle_source_overrides(
                     subprocess.run(["git", "clone", git_url, checkout_dir], check=True, capture_output=True, text=True)
                     subprocess.run(["git", "checkout", commit_sha], cwd=checkout_dir, check=True, capture_output=True, text=True)
 
-            logger.info(f"  Bundling {pkg_name}=={pkg_ver} from {checkout_dir} ...")
+            # Honor [tool.uv.sources] `subdirectory` for monorepo git sources
+            subdir = source.get("subdirectory")
+            build_cwd = os.path.join(checkout_dir, subdir) if subdir else checkout_dir
+
+            logger.info(f"  Bundling {pkg_name}=={pkg_ver} from {build_cwd} ...")
             os.makedirs(wheels_dir, exist_ok=True)
             try:
                 result = subprocess.run(
                     ["uv", "build", "--wheel", ".", "--out-dir", wheels_dir],
-                    cwd=checkout_dir,
+                    cwd=build_cwd,
                     check=True,
                     capture_output=True,
                     text=True,

{qubx-1.3.2 → qubx-1.3.3.dev2}/src/qubx/cli/s3.py

@@ -109,6 +109,98 @@ def s3_cp(src: str, dst: str, recursive: bool = False) -> None:
         raise click.Abort()
 
 
+def s3_parquet_stats(path: str, per_column: bool = False) -> None:
+    """Print file-level summary + row-group size distribution for one parquet file.
+
+    Accepts either an S3 URI (``account:bucket/key``) or a local filesystem path.
+    Uses pyarrow for metadata parsing so no data pages are fetched.
+    """
+    import pyarrow.parquet as pq
+    from pyarrow.fs import LocalFileSystem
+
+    # Heuristic: an S3 URI has "account:..." where account is non-empty and
+    # contains no path separator. Fall back to local for anything else.
+    is_s3 = ":" in path and "/" not in path.split(":", 1)[0] and not path.startswith("/")
+    if is_s3:
+        try:
+            client, s3_path = S3Client.from_uri(path)
+        except ValueError as e:
+            raise click.BadParameter(str(e))
+        fs = client.fs
+        open_path = s3_path
+    else:
+        fs = LocalFileSystem()
+        open_path = path
+
+    try:
+        with fs.open_input_file(open_path) as f:
+            meta = pq.ParquetFile(f).metadata
+    except Exception as e:
+        click.echo(click.style(f"Error reading {path}: {e}", fg="red"), err=True)
+        raise click.Abort()
+
+    click.echo(click.style(f"File: {path}", fg="cyan", bold=True))
+    click.echo(f"  num_rows:       {meta.num_rows:,}")
+    click.echo(f"  num_row_groups: {meta.num_row_groups}")
+    click.echo(f"  num_columns:    {meta.num_columns}")
+    click.echo(f"  format_version: {meta.format_version}")
+    if meta.created_by:
+        click.echo(f"  created_by:     {meta.created_by}")
+
+    if meta.num_row_groups == 0:
+        return
+
+    # Parquet spec: RowGroup.total_byte_size is the SUM of uncompressed
+    # column sizes. The compressed on-disk size is the sum of
+    # ColumnChunk.total_compressed_size across columns.
+    rows: list[int] = []
+    per_rg_compressed: list[int] = []
+    per_rg_uncompressed: list[int] = []
+    compressions: set[str] = set()
+    for i in range(meta.num_row_groups):
+        rg = meta.row_group(i)
+        rows.append(rg.num_rows)
+        c_sz = 0
+        u_sz = 0
+        for c in range(rg.num_columns):
+            col = rg.column(c)
+            c_sz += col.total_compressed_size
+            u_sz += col.total_uncompressed_size
+            compressions.add(str(col.compression))
+        per_rg_compressed.append(c_sz)
+        per_rg_uncompressed.append(u_sz)
+
+    total_compressed = sum(per_rg_compressed)
+    total_uncompressed = sum(per_rg_uncompressed)
+    click.echo()
+    click.echo(click.style("Row groups:", fg="cyan", bold=True))
+    click.echo(f"  rows per rg:   min={min(rows):,}  max={max(rows):,}  avg={sum(rows)//len(rows):,}")
+    click.echo(
+        f"  cmp per rg:    min={_human_size(min(per_rg_compressed))}  "
+        f"max={_human_size(max(per_rg_compressed))}  avg={_human_size(sum(per_rg_compressed) // len(per_rg_compressed))}"
+    )
+    click.echo(f"  total:         compressed={_human_size(total_compressed)}  uncompressed={_human_size(total_uncompressed)}")
+    if total_uncompressed > 0:
+        ratio = 100 * total_compressed / total_uncompressed
+        click.echo(f"  compression:   {', '.join(sorted(compressions))}  ({ratio:.1f}% of uncompressed)")
+    else:
+        click.echo(f"  compression:   {', '.join(sorted(compressions))}")
+
+    if per_column:
+        click.echo()
+        click.echo(click.style("Columns (row group 0):", fg="cyan", bold=True))
+        rg0 = meta.row_group(0)
+        click.echo(f"  {'path':30s}  {'type':12s}  {'compressed':>11s}  {'uncompressed':>12s}   pct  compression")
+        for c in range(rg0.num_columns):
+            col = rg0.column(c)
+            pct = (100 * col.total_compressed_size / col.total_uncompressed_size) if col.total_uncompressed_size else 0.0
+            click.echo(
+                f"  {col.path_in_schema:30s}  {str(col.physical_type):12s}  "
+                f"{_human_size(col.total_compressed_size):>11s}  {_human_size(col.total_uncompressed_size):>12s}  "
+                f"{pct:5.1f}%  {col.compression}"
+            )
+
+
 def s3_accounts() -> None:
     s = get_settings()
     if not s.s3: