pybcsv 1.5.7__tar.gz → 1.5.10.dev0__tar.gz

{pybcsv-1.5.7 → pybcsv-1.5.10.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pybcsv
-Version: 1.5.7
+Version: 1.5.10.dev0
 Summary: High-performance Python bindings for the BCSV (Binary CSV) library with pandas integration
 Keywords: csv,binary,data-science,compression,performance,pandas,numpy,big-data,data-processing
 Author-Email: Tobias Weber <weber.tobias.md@gmail.com>
@@ -156,6 +156,44 @@ with pybcsv.ReaderDirectAccess() as da:
     print(da.read(100))  # alternative syntax
 ```
 
+## Parquet Conversion Tools (CLI)
+
+Installing `pybcsv` provides two streaming command-line converters (require the
+`arrow` extra: `pip install pybcsv[arrow]`). They stream in bounded batches, so
+they handle files larger than memory.
+
+```bash
+# Parquet → BCSV
+parquet2bcsv input.parquet -o output.bcsv
+#   --row-codec {delta,zoh,flat}          row codec (default: delta)
+#   --file-codec {packet_lz4_batch,...}   file codec (default: packet_lz4_batch)
+#   --chunk-size N                        rows per streamed batch (default: 512000)
+#   -f/--force                            overwrite an existing output
+
+# BCSV → Parquet
+bcsv2parquet input.bcsv -o output.parquet
+#   --columns "a,b,c"       select/reorder columns (order is honored)
+#   --slice 10:100          Python-style row slice
+#   --unflatten (default)   reconstruct nested structs from dotted/bracketed names
+#   --no-unflatten          keep flat columns (names like 'a.b', 'vals[0]')
+#   --parquet-compression {none,snappy,gzip,zstd,lz4}
+```
+
+**Schema mapping.** Parquet structs and fixed-size lists are *flattened* to BCSV
+columns using dotted (`location.lat`) and bracketed (`vals[0]`) names;
+`bcsv2parquet --unflatten` reverses this. Notes and limitations:
+
+- **No nulls:** BCSV has no null representation — a null in any converted column
+  is rejected with the offending row number. Filter nulls before converting.
+- **Type widening:** `float16`/`bfloat16` widen to `float32`; `large_string`
+  maps to `string`.
+- **Unsupported types** (variable-length lists, maps, timestamps, decimals,
+  dictionaries) are rejected with a clear error.
+- **Column names ending in `_`** are rejected (the unflatten escape protocol
+  reserves trailing underscores).
+- **Colliding names:** if a literal dotted column (`a.b`) and a struct path both
+  map to the same nested path, `--unflatten` fails loudly; use `--no-unflatten`.
+
 ## Available Types
 
 | Constant | Description |

{pybcsv-1.5.7 → pybcsv-1.5.10.dev0}/README.md

@@ -110,6 +110,44 @@ with pybcsv.ReaderDirectAccess() as da:
     print(da.read(100))  # alternative syntax
 ```
 
+## Parquet Conversion Tools (CLI)
+
+Installing `pybcsv` provides two streaming command-line converters (require the
+`arrow` extra: `pip install pybcsv[arrow]`). They stream in bounded batches, so
+they handle files larger than memory.
+
+```bash
+# Parquet → BCSV
+parquet2bcsv input.parquet -o output.bcsv
+#   --row-codec {delta,zoh,flat}          row codec (default: delta)
+#   --file-codec {packet_lz4_batch,...}   file codec (default: packet_lz4_batch)
+#   --chunk-size N                        rows per streamed batch (default: 512000)
+#   -f/--force                            overwrite an existing output
+
+# BCSV → Parquet
+bcsv2parquet input.bcsv -o output.parquet
+#   --columns "a,b,c"       select/reorder columns (order is honored)
+#   --slice 10:100          Python-style row slice
+#   --unflatten (default)   reconstruct nested structs from dotted/bracketed names
+#   --no-unflatten          keep flat columns (names like 'a.b', 'vals[0]')
+#   --parquet-compression {none,snappy,gzip,zstd,lz4}
+```
+
+**Schema mapping.** Parquet structs and fixed-size lists are *flattened* to BCSV
+columns using dotted (`location.lat`) and bracketed (`vals[0]`) names;
+`bcsv2parquet --unflatten` reverses this. Notes and limitations:
+
+- **No nulls:** BCSV has no null representation — a null in any converted column
+  is rejected with the offending row number. Filter nulls before converting.
+- **Type widening:** `float16`/`bfloat16` widen to `float32`; `large_string`
+  maps to `string`.
+- **Unsupported types** (variable-length lists, maps, timestamps, decimals,
+  dictionaries) are rejected with a clear error.
+- **Column names ending in `_`** are rejected (the unflatten escape protocol
+  reserves trailing underscores).
+- **Colliding names:** if a literal dotted column (`a.b`) and a struct path both
+  map to the same nested path, `--unflatten` fails loudly; use `--no-unflatten`.
+
 ## Available Types
 
 | Constant | Description |

pybcsv-1.5.10.dev0/VERSION.txt

@@ -0,0 +1 @@
+1.5.9

{pybcsv-1.5.7 → pybcsv-1.5.10.dev0}/pybcsv/__init__.py

@@ -1,8 +1,8 @@
 # Copyright (c) 2025-2026 Tobias Weber <weber.tobias.md@gmail.com>
-# 
+#
 # This file is part of the BCSV library.
-# 
-# Licensed under the MIT License. See LICENSE file in the project root 
+#
+# Licensed under the MIT License. See LICENSE file in the project root
 # for full license information.
 
 """pybcsv - Python bindings for the BCSV library."""
@@ -12,10 +12,12 @@ from .__version__ import __version__
 # Import the compiled extension module — fail immediately if not available
 try:
     from ._bcsv import *
+
     _BINDINGS_AVAILABLE = True
 except ImportError:
     try:
-        from _bcsv import *   # fallback for legacy in-tree builds
+        from _bcsv import *  # fallback for legacy in-tree builds
+
         _BINDINGS_AVAILABLE = True
     except ImportError as _exc:
         raise ImportError(
@@ -27,35 +29,86 @@ except ImportError:
 # Try to import pandas utilities if pandas is available
 try:
     from .pandas_utils import write_dataframe, read_dataframe, to_csv, from_csv
+
     _PANDAS_UTILS_AVAILABLE = True
 except ImportError:
     _PANDAS_UTILS_AVAILABLE = False
-    
+
     # Create stub functions that raise ImportError
     def write_dataframe(*args, **kwargs):
-        raise ImportError("pandas is not available. Please install pandas to use DataFrame functions.")
-    
+        raise ImportError(
+            "pandas is not available. Please install pandas to use DataFrame functions."
+        )
+
     def read_dataframe(*args, **kwargs):
-        raise ImportError("pandas is not available. Please install pandas to use DataFrame functions.")
-    
+        raise ImportError(
+            "pandas is not available. Please install pandas to use DataFrame functions."
+        )
+
     def to_csv(*args, **kwargs):
-        raise ImportError("pandas is not available. Please install pandas to use CSV conversion functions.")
-    
+        raise ImportError(
+            "pandas is not available. Please install pandas to use CSV conversion functions."
+        )
+
     def from_csv(*args, **kwargs):
-        raise ImportError("pandas is not available. Please install pandas to use CSV conversion functions.")
+        raise ImportError(
+            "pandas is not available. Please install pandas to use CSV conversion functions."
+        )
+
 
 # Try to import Polars utilities if polars + pyarrow are available
 try:
     from .polars_utils import read_polars, write_polars
+
     _POLARS_UTILS_AVAILABLE = True
 except ImportError:
     _POLARS_UTILS_AVAILABLE = False
 
     def read_polars(*args, **kwargs):
-        raise ImportError("polars and pyarrow are required. Install with: pip install pybcsv[polars,arrow]")
+        raise ImportError(
+            "polars and pyarrow are required. Install with: pip install pybcsv[polars,arrow]"
+        )
 
     def write_polars(*args, **kwargs):
-        raise ImportError("polars and pyarrow are required. Install with: pip install pybcsv[polars,arrow]")
+        raise ImportError(
+            "polars and pyarrow are required. Install with: pip install pybcsv[polars,arrow]"
+        )
+
+
+# Try to import Parquet utilities if pyarrow is available
+try:
+    from .parquet_utils import parquet_to_bcsv, bcsv_to_parquet
+
+    _PARQUET_AVAILABLE = True
+except ImportError:
+    _PARQUET_AVAILABLE = False
+
+    def parquet_to_bcsv(*args, **kwargs):
+        raise ImportError(
+            "pyarrow is required for Parquet conversion. "
+            "Install with: pip install pybcsv[arrow]"
+        )
+
+    def bcsv_to_parquet(*args, **kwargs):
+        raise ImportError(
+            "pyarrow is required for Parquet conversion. "
+            "Install with: pip install pybcsv[arrow]"
+        )
+
+
+def iter_arrow_batches(reader, *, batch_size=512000, columns=None, start_row=0):
+    """Yield pa.RecordBatch objects from an already-open ReaderDirectAccess.
+
+    Memory-bounded: exactly one batch is held at a time. `reader` must be open.
+    """
+    pos = start_row
+    while True:
+        batch = reader.read_arrow_batch(pos, batch_size, columns)
+        if batch is None:
+            break
+        yield batch
+        pos += batch.num_rows
+
 
 __all__ = [
     "__version__",
@@ -80,6 +133,10 @@ __all__ = [
     # Arrow interop
     "read_to_arrow",
     "write_from_arrow",
+    "iter_arrow_batches",
+    # Parquet interop
+    "parquet_to_bcsv",
+    "bcsv_to_parquet",
     # Polars interop
     "read_polars",
     "write_polars",
@@ -89,4 +146,4 @@ __all__ = [
     "read_dataframe",
     "to_csv",
     "from_csv",
-]
+]

{pybcsv-1.5.7 → pybcsv-1.5.10.dev0}/pybcsv/__init__.pyi

@@ -48,15 +48,12 @@ def write_dataframe(
     type_hints: Optional[Dict[str, ColumnType]] = None,
     strict: bool = False,
 ) -> None: ...
-
 def read_dataframe(
     filename: str,
     columns: Optional[list] = None,
     optimize_dtypes: bool = True,
 ) -> object: ...
-
 def to_csv(bcsv_filename: str, csv_filename: str, **csv_kwargs: object) -> None: ...
-
 def from_csv(
     csv_filename: str,
     bcsv_filename: str,
@@ -64,16 +61,21 @@ def from_csv(
     type_hints: Optional[Dict[str, ColumnType]] = None,
     **csv_kwargs: object,
 ) -> None: ...
-
 def read_polars(
     filename: str,
     columns: object = None,
     chunk_size: int = 0,
 ) -> object: ...
-
 def write_polars(
     df: object,
     filename: str,
     row_codec: str = "delta",
     compression_level: int = 1,
 ) -> None: ...
+def iter_arrow_batches(
+    reader: ReaderDirectAccess,
+    *,
+    batch_size: int = 512000,
+    columns: Optional[list] = None,
+    start_row: int = 0,
+) -> object: ...  # yields pa.RecordBatch

{pybcsv-1.5.7 → pybcsv-1.5.10.dev0}/pybcsv/_bcsv.pyi

@@ -4,7 +4,6 @@ from collections.abc import Sequence
 import enum
 from typing import overload
 
-
 class ColumnType(enum.Enum):
     BOOL = 0
 
@@ -56,54 +55,35 @@ STRING: ColumnType = ColumnType.STRING
 
 class ColumnDefinition:
     def __init__(self, name: str, type: ColumnType) -> None: ...
-
     @property
     def name(self) -> str: ...
-
     @name.setter
     def name(self, arg: str, /) -> None: ...
-
     @property
     def type(self) -> ColumnType: ...
-
     @type.setter
     def type(self, arg: ColumnType, /) -> None: ...
-
     def __repr__(self) -> str: ...
 
 class Layout:
     @overload
     def __init__(self) -> None: ...
-
     @overload
     def __init__(self, arg: Sequence[ColumnDefinition], /) -> None: ...
-
     @overload
     def add_column(self, column: ColumnDefinition) -> None: ...
-
     @overload
     def add_column(self, name: str, type: ColumnType) -> None: ...
-
     def column_count(self) -> int: ...
-
     def column_name(self, index: int) -> str: ...
-
     def column_type(self, index: int) -> ColumnType: ...
-
     def has_column(self, name: str) -> bool: ...
-
     def column_index(self, name: str) -> int: ...
-
     def get_column_names(self) -> list[str]: ...
-
     def get_column_types(self) -> list[ColumnType]: ...
-
     def get_column(self, index: int) -> ColumnDefinition: ...
-
     def __len__(self) -> int: ...
-
     def __getitem__(self, arg: int, /) -> ColumnDefinition: ...
-
     def __repr__(self) -> str: ...
 
 class FileFlags(enum.IntEnum):
@@ -120,9 +100,7 @@ class FileFlags(enum.IntEnum):
     DELTA_ENCODING = 16
 
     def __or__(self, arg: FileFlags, /) -> int: ...
-
     def __and__(self, arg: FileFlags, /) -> int: ...
-
     def __invert__(self) -> int: ...
 
 NONE: FileFlags = FileFlags.NONE
@@ -138,58 +116,43 @@ BATCH_COMPRESS: FileFlags = FileFlags.BATCH_COMPRESS
 DELTA_ENCODING: FileFlags = FileFlags.DELTA_ENCODING
 
 class Writer:
-    def __init__(self, layout: Layout, row_codec: str = 'delta') -> None: ...
-
-    def open(self, filename: str, overwrite: bool = False, compression_level: int = 1, block_size_kb: int = 8192, flags: FileFlags = FileFlags.BATCH_COMPRESS) -> bool: ...
-
+    def __init__(self, layout: Layout, row_codec: str = "delta") -> None: ...
+    def open(
+        self,
+        filename: str,
+        overwrite: bool = False,
+        compression_level: int = 1,
+        block_size_kb: int = 8192,
+        flags: FileFlags = FileFlags.BATCH_COMPRESS,
+    ) -> bool: ...
     def write_row(self, arg: list, /) -> None: ...
-
     def write_rows(self, arg: list, /) -> None:
         """Write multiple rows efficiently with batching"""
 
     def close(self) -> None: ...
-
     def flush(self) -> None: ...
-
     def is_open(self) -> bool: ...
-
     def row_count(self) -> int: ...
-
     def row_codec(self) -> str: ...
-
     def compression_level(self) -> int: ...
-
     def layout(self) -> Layout: ...
-
+    def write_batch(self, batch: object) -> None: ...  # batch: pa.RecordBatch
     def __enter__(self) -> Writer: ...
-
     def __exit__(self, *args) -> None: ...
-
     def __repr__(self) -> str: ...
 
 class Reader:
     def __init__(self) -> None: ...
-
     def open(self, filename: str) -> bool: ...
-
     def layout(self) -> Layout: ...
-
     def read_next(self) -> bool: ...
-
     def close(self) -> None: ...
-
     def is_open(self) -> bool: ...
-
     def file_flags(self) -> int: ...
-
     def compression_level(self) -> int: ...
-
     def row_pos(self) -> int: ...
-
     def version_string(self) -> str: ...
-
     def creation_time(self) -> int: ...
-
     def count_rows(self) -> int:
         """Count the total number of rows in the file"""
 
@@ -205,115 +168,79 @@ class Reader:
         """
 
     def read_row(self) -> object: ...
-
     def read_all(self) -> list: ...
-
     def __enter__(self) -> Reader: ...
-
     def __exit__(self, *args) -> None: ...
-
     def __iter__(self) -> Reader: ...
-
     def __next__(self) -> object: ...
-
     def __repr__(self) -> str: ...
 
 def type_to_string(arg: ColumnType, /) -> str:
     """Convert ColumnType to string"""
 
 class CsvWriter:
-    def __init__(self, layout: Layout, delimiter: str = ',', decimal_sep: str = '.') -> None: ...
-
-    def open(self, filename: str, overwrite: bool = False, include_header: bool = True) -> bool: ...
-
+    def __init__(
+        self, layout: Layout, delimiter: str = ",", decimal_sep: str = "."
+    ) -> None: ...
+    def open(
+        self, filename: str, overwrite: bool = False, include_header: bool = True
+    ) -> bool: ...
     def close(self) -> None: ...
-
     def is_open(self) -> bool: ...
-
     def row_count(self) -> int: ...
-
     def layout(self) -> Layout: ...
-
     def delimiter(self) -> str: ...
-
     def decimal_separator(self) -> str: ...
-
     def __enter__(self) -> CsvWriter: ...
-
     def __exit__(self, *args) -> None: ...
-
     def write_row(self, arg: list, /) -> None: ...
-
     def write_rows(self, arg: list, /) -> None:
         """Write multiple rows efficiently"""
 
 class CsvReader:
-    def __init__(self, layout: Layout, delimiter: str = ',', decimal_sep: str = '.') -> None: ...
-
+    def __init__(
+        self, layout: Layout, delimiter: str = ",", decimal_sep: str = "."
+    ) -> None: ...
     def open(self, filename: str, has_header: bool = True) -> bool: ...
-
     def read_next(self) -> bool: ...
-
     def close(self) -> None: ...
-
     def is_open(self) -> bool: ...
-
     def row_pos(self) -> int: ...
-
     def file_line(self) -> int: ...
-
     def layout(self) -> Layout: ...
-
     def delimiter(self) -> str: ...
-
     def decimal_separator(self) -> str: ...
-
     def error_msg(self) -> str: ...
-
     def read_row(self) -> object: ...
-
     def read_all(self) -> list: ...
-
     def __enter__(self) -> CsvReader: ...
-
     def __exit__(self, *args) -> None: ...
-
     def __iter__(self) -> CsvReader: ...
-
     def __next__(self) -> object: ...
 
 class ReaderDirectAccess:
     def __init__(self) -> None: ...
-
     def open(self, filename: str, rebuild_footer: bool = False) -> bool: ...
-
     def read(self, index: int) -> list: ...
-
     def row_count(self) -> int: ...
-
     def layout(self) -> Layout: ...
-
     def close(self) -> None: ...
-
     def is_open(self) -> bool: ...
-
     def file_flags(self) -> int: ...
-
     def compression_level(self) -> int: ...
-
     def version_string(self) -> str: ...
-
     def creation_time(self) -> int: ...
-
     def __enter__(self) -> ReaderDirectAccess: ...
-
     def __exit__(self, *args) -> None: ...
-
     def __len__(self) -> int: ...
-
     def __getitem__(self, arg: int, /) -> list: ...
-
     def __repr__(self) -> str: ...
+    def read_arrow_batch(
+        self,
+        start_row: int = 0,
+        batch_size: int = 512000,
+        columns: list[str] | None = None,
+    ) -> object: ...  # returns pa.RecordBatch or None at EOF
 
 class SamplerMode(enum.Enum):
     TRUNCATE = 0
@@ -340,56 +267,33 @@ SATURATE: SamplerErrorPolicy = SamplerErrorPolicy.SATURATE
 class SamplerCompileResult:
     @property
     def success(self) -> bool: ...
-
     @property
     def error_msg(self) -> str: ...
-
     @property
     def error_position(self) -> int: ...
-
     def __bool__(self) -> bool: ...
-
     def __repr__(self) -> str: ...
 
 class Sampler:
     def __init__(self, reader: Reader) -> None: ...
-
     def set_conditional(self, expr: str) -> SamplerCompileResult: ...
-
     def get_conditional(self) -> str: ...
-
     def set_selection(self, expr: str) -> SamplerCompileResult: ...
-
     def get_selection(self) -> str: ...
-
     def set_mode(self, mode: SamplerMode) -> None: ...
-
     def get_mode(self) -> SamplerMode: ...
-
     def set_error_policy(self, policy: SamplerErrorPolicy) -> None: ...
-
     def get_error_policy(self) -> SamplerErrorPolicy: ...
-
     def output_layout(self) -> Layout: ...
-
     def next(self) -> bool: ...
-
     def row(self) -> list: ...
-
     def source_row_pos(self) -> int: ...
-
     def bulk(self) -> list: ...
-
     def is_conditional_passthrough(self) -> bool: ...
-
     def is_selection_passthrough(self) -> bool: ...
-
     def window_capacity(self) -> int: ...
-
     def disassemble(self) -> str: ...
-
     def __iter__(self) -> Sampler: ...
-
     def __next__(self) -> object: ...
 
 def read_columns(filename: str) -> dict:
@@ -397,14 +301,30 @@ def read_columns(filename: str) -> dict:
     Read a BCSV file into a dict of numpy arrays (numeric) and lists (strings)
     """
 
-def write_columns(filename: str, columns: dict, col_order: list, col_types: list, row_codec: str = 'delta', compression_level: int = 1, flags: FileFlags = FileFlags.BATCH_COMPRESS) -> None:
+def write_columns(
+    filename: str,
+    columns: dict,
+    col_order: list,
+    col_types: list,
+    row_codec: str = "delta",
+    compression_level: int = 1,
+    flags: FileFlags = FileFlags.BATCH_COMPRESS,
+) -> None:
     """Write a dict of numpy arrays/lists to a BCSV file"""
 
-def read_to_arrow(filename: str, columns: list | None = None, chunk_size: int = 0) -> object:
+def read_to_arrow(
+    filename: str, columns: list | None = None, chunk_size: int = 0
+) -> object:
     """
     Read a BCSV file into a pyarrow.Table via Arrow C Data Interface (zero-copy).
     Set chunk_size > 0 for chunked reading (returns Table from multiple batches).
     """
 
-def write_from_arrow(filename: str, table: object, row_codec: str = 'delta', compression_level: int = 1, flags: FileFlags = FileFlags.BATCH_COMPRESS) -> None:
+def write_from_arrow(
+    filename: str,
+    table: object,
+    row_codec: str = "delta",
+    compression_level: int = 1,
+    flags: FileFlags = FileFlags.BATCH_COMPRESS,
+) -> None:
     """Write a pyarrow Table/RecordBatch to a BCSV file"""

{pybcsv-1.5.7 → pybcsv-1.5.10.dev0}/pybcsv/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.5.7'
-__version_tuple__ = version_tuple = (1, 5, 7)
+__version__ = version = '1.5.10.dev0'
+__version_tuple__ = version_tuple = (1, 5, 10, 'dev0')
 
-__commit_id__ = commit_id = 'g1303a72af'
+__commit_id__ = commit_id = 'g6ec074145'