pybcsv 1.5.8__tar.gz → 1.5.10.dev0__tar.gz

{pybcsv-1.5.8 → pybcsv-1.5.10.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pybcsv
-Version: 1.5.8
+Version: 1.5.10.dev0
 Summary: High-performance Python bindings for the BCSV (Binary CSV) library with pandas integration
 Keywords: csv,binary,data-science,compression,performance,pandas,numpy,big-data,data-processing
 Author-Email: Tobias Weber <weber.tobias.md@gmail.com>
@@ -156,6 +156,44 @@ with pybcsv.ReaderDirectAccess() as da:
     print(da.read(100))  # alternative syntax
 ```
 
+## Parquet Conversion Tools (CLI)
+
+Installing `pybcsv` provides two streaming command-line converters (require the
+`arrow` extra: `pip install pybcsv[arrow]`). They stream in bounded batches, so
+they handle files larger than memory.
+
+```bash
+# Parquet → BCSV
+parquet2bcsv input.parquet -o output.bcsv
+#   --row-codec {delta,zoh,flat}          row codec (default: delta)
+#   --file-codec {packet_lz4_batch,...}   file codec (default: packet_lz4_batch)
+#   --chunk-size N                        rows per streamed batch (default: 512000)
+#   -f/--force                            overwrite an existing output
+
+# BCSV → Parquet
+bcsv2parquet input.bcsv -o output.parquet
+#   --columns "a,b,c"       select/reorder columns (order is honored)
+#   --slice 10:100          Python-style row slice
+#   --unflatten (default)   reconstruct nested structs from dotted/bracketed names
+#   --no-unflatten          keep flat columns (names like 'a.b', 'vals[0]')
+#   --parquet-compression {none,snappy,gzip,zstd,lz4}
+```
+
+**Schema mapping.** Parquet structs and fixed-size lists are *flattened* to BCSV
+columns using dotted (`location.lat`) and bracketed (`vals[0]`) names;
+`bcsv2parquet --unflatten` reverses this. Notes and limitations:
+
+- **No nulls:** BCSV has no null representation — a null in any converted column
+  is rejected with the offending row number. Filter nulls before converting.
+- **Type widening:** `float16`/`bfloat16` widen to `float32`; `large_string`
+  maps to `string`.
+- **Unsupported types** (variable-length lists, maps, timestamps, decimals,
+  dictionaries) are rejected with a clear error.
+- **Column names ending in `_`** are rejected (the unflatten escape protocol
+  reserves trailing underscores).
+- **Colliding names:** if a literal dotted column (`a.b`) and a struct path both
+  map to the same nested path, `--unflatten` fails loudly; use `--no-unflatten`.
+
 ## Available Types
 
 | Constant | Description |

{pybcsv-1.5.8 → pybcsv-1.5.10.dev0}/README.md

@@ -110,6 +110,44 @@ with pybcsv.ReaderDirectAccess() as da:
     print(da.read(100))  # alternative syntax
 ```
 
+## Parquet Conversion Tools (CLI)
+
+Installing `pybcsv` provides two streaming command-line converters (require the
+`arrow` extra: `pip install pybcsv[arrow]`). They stream in bounded batches, so
+they handle files larger than memory.
+
+```bash
+# Parquet → BCSV
+parquet2bcsv input.parquet -o output.bcsv
+#   --row-codec {delta,zoh,flat}          row codec (default: delta)
+#   --file-codec {packet_lz4_batch,...}   file codec (default: packet_lz4_batch)
+#   --chunk-size N                        rows per streamed batch (default: 512000)
+#   -f/--force                            overwrite an existing output
+
+# BCSV → Parquet
+bcsv2parquet input.bcsv -o output.parquet
+#   --columns "a,b,c"       select/reorder columns (order is honored)
+#   --slice 10:100          Python-style row slice
+#   --unflatten (default)   reconstruct nested structs from dotted/bracketed names
+#   --no-unflatten          keep flat columns (names like 'a.b', 'vals[0]')
+#   --parquet-compression {none,snappy,gzip,zstd,lz4}
+```
+
+**Schema mapping.** Parquet structs and fixed-size lists are *flattened* to BCSV
+columns using dotted (`location.lat`) and bracketed (`vals[0]`) names;
+`bcsv2parquet --unflatten` reverses this. Notes and limitations:
+
+- **No nulls:** BCSV has no null representation — a null in any converted column
+  is rejected with the offending row number. Filter nulls before converting.
+- **Type widening:** `float16`/`bfloat16` widen to `float32`; `large_string`
+  maps to `string`.
+- **Unsupported types** (variable-length lists, maps, timestamps, decimals,
+  dictionaries) are rejected with a clear error.
+- **Column names ending in `_`** are rejected (the unflatten escape protocol
+  reserves trailing underscores).
+- **Colliding names:** if a literal dotted column (`a.b`) and a struct path both
+  map to the same nested path, `--unflatten` fails loudly; use `--no-unflatten`.
+
 ## Available Types
 
 | Constant | Description |

pybcsv-1.5.10.dev0/VERSION.txt

@@ -0,0 +1 @@
+1.5.9

{pybcsv-1.5.8 → pybcsv-1.5.10.dev0}/pybcsv/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.5.8'
-__version_tuple__ = version_tuple = (1, 5, 8)
+__version__ = version = '1.5.10.dev0'
+__version_tuple__ = version_tuple = (1, 5, 10, 'dev0')
 
-__commit_id__ = commit_id = 'g255e3b738'
+__commit_id__ = commit_id = 'g6ec074145'

{pybcsv-1.5.8 → pybcsv-1.5.10.dev0}/pybcsv/parquet_utils.py

@@ -11,6 +11,7 @@ import time
 from typing import List, Optional, Set, Tuple, Union
 
 import pyarrow as pa
+import pyarrow.compute as pc
 import pyarrow.parquet as pq
 
 import pybcsv
@@ -252,8 +253,24 @@ def flatten_batch(
 
 
 def _extract_flat_array(batch: pa.RecordBatch, flat_name: str) -> pa.Array:
-    """Navigate into nested columns to extract a flat array."""
-    parts = _decompose_name(flat_name)
+    """Navigate into nested columns to extract a flat array.
+
+    For already-flattened Parquet files whose column names contain literal
+    dots, tries an exact match first.  If found, returns the column
+    directly without decomposition.
+    """
+    # Fast path: flat_name matches a literal column name (common for
+    # pre-flattened Parquet files with dot-separated identifiers).
+    for i in range(len(batch.schema)):
+        if batch.schema.field(i).name == flat_name:
+            return batch.column(i)
+
+    # Undo collision-escape underscores before navigating: a struct path whose
+    # flattened name collided with a literal dotted column was escaped (e.g.
+    # 'a.b' -> 'a_.b') during flattening.  The batch's actual struct is named
+    # 'a', so strip the escape suffixes to navigate it.  Real identifiers can
+    # never end with '_' (rejected at flatten time), so this is unambiguous.
+    parts = _strip_escape_suffixes(_decompose_name(flat_name))
     root_name: str = parts[0]  # type: ignore[assignment]
 
     col_idx = -1
@@ -275,9 +292,17 @@ def _extract_flat_array(batch: pa.RecordBatch, flat_name: str) -> pa.Array:
                         f"List index {part} out of range [0, {list_size}) "
                         f"for column '{flat_name}'."
                     )
+                # FixedSizeList values are row-major: element `i` of row `r` lives
+                # at child[r*list_size + i].  Extracting element `part` across all
+                # rows is therefore a *strided* gather, not a contiguous slice.
+                # flatten() yields the logical, offset-adjusted row-major child.
                 num_rows = len(arr)
-                child_arr = arr.values
-                arr = child_arr.slice(part * num_rows, num_rows)
+                flat_values = arr.flatten()
+                gather = pa.array(
+                    [r * list_size + part for r in range(num_rows)],
+                    type=pa.int64(),
+                )
+                arr = flat_values.take(gather)
             else:
                 arr = arr[part]
         else:
@@ -451,6 +476,21 @@ def unflatten_schema_to_arrow(
             )
     for n in names:
         _check_underscore_name(n)
+    # Two distinct flat names that collapse to the same nested path (e.g. a
+    # literal 'a.b' column and a struct path escaped to 'a_.b') cannot be
+    # reconstructed unambiguously — the trie would silently merge them and drop
+    # data.  Detect this and fail loudly; --no-unflatten preserves the columns.
+    _seen_paths: dict = {}
+    for n in names:
+        key = tuple(_strip_escape_suffixes(_decompose_name(n)))
+        if key in _seen_paths:
+            path_str = ".".join(str(p) for p in key)
+            raise ValueError(
+                f"Cannot unflatten: columns '{_seen_paths[key]}' and '{n}' both "
+                f"map to nested path '{path_str}'. Re-run with --no-unflatten to "
+                "keep the flat columns."
+            )
+        _seen_paths[key] = n
     trie = _build_trie(names, arrow_types)
     fields = _trie_to_arrow_field(trie)
     return pa.schema(fields)
@@ -524,8 +564,20 @@ def _build_nested_array(
                 chunks.append(arr)
             else:
                 raise ValueError(f"Cannot find array for '{bracket}'")
+        # `chunks[i]` holds element `i` of every row (element-major).  The Arrow
+        # FixedSizeList child must be row-major (child[r*list_size + i]), so
+        # re-interleave: child[r*list_size + i] = chunks[i][r] = merged[i*num_rows + r].
         merged = pa.concat_arrays(chunks)
-        return pa.FixedSizeListArray.from_arrays(merged, list_size)
+        num_rows = len(chunks[0])
+        gather = pa.array(
+            [
+                (p % list_size) * num_rows + (p // list_size)
+                for p in range(num_rows * list_size)
+            ],
+            type=pa.int64(),
+        )
+        child = merged.take(gather)
+        return pa.FixedSizeListArray.from_arrays(child, list_size)
 
     struct_fields = field.type
     child_arrays: List[pa.Array] = []
@@ -616,41 +668,28 @@ def validate_parquet_schema(pf: pq.ParquetFile) -> None:
 
 
 def _check_nulls(batch: pa.RecordBatch, row_offset: int) -> None:
-    """Raise ValueError on first null using bitmap scan clamped to batch rows."""
+    """Raise ValueError on the first null in any column of the batch.
+
+    Uses pyarrow-native null location, which is offset-safe for sliced arrays
+    (a hand-rolled bitmap scan mis-attributes the row number when the array has
+    a non-zero offset, e.g. an element extracted from a FixedSizeList).
+    """
     for i in range(len(batch.schema)):
         col = batch.column(i)
         if col.null_count == 0:
             continue
 
         field_name = batch.schema.field(i).name
-        offset = col.offset
-        validity_buffer = col.buffers()[0]
-
-        if validity_buffer is not None:
-            bitmap = validity_buffer.to_pybytes()
-            max_bits = offset + len(col)
-            max_bytes = (max_bits + 7) // 8
-
-            for byte_idx in range(max_bytes):
-                byte_val = bitmap[byte_idx]
-                if byte_val != 0xFF:
-                    for bit_pos in range(8):
-                        absolute_bit = byte_idx * 8 + bit_pos
-                        if absolute_bit >= max_bits:
-                            break
-                        logical_row = absolute_bit - offset
-                        if not (byte_val & (1 << bit_pos)):
-                            raise ValueError(
-                                f"Null value detected in column '{field_name}' "
-                                f"at row {row_offset + logical_row}. "
-                                "BCSV does not support nulls. Filter before conversion."
-                            )
-        else:
-            raise ValueError(
-                f"Null value detected in column '{field_name}' "
-                f"(null_count={col.null_count}). "
-                "BCSV does not support nulls."
-            )
+        first_null = pc.index(pc.is_null(col), True).as_py()
+        location = (
+            f" at row {row_offset + first_null}"
+            if first_null is not None and first_null >= 0
+            else ""
+        )
+        raise ValueError(
+            f"Null value detected in column '{field_name}'{location}. "
+            "BCSV does not support nulls. Filter before conversion."
+        )
 
 
 # ---- Selection & helpers ----
@@ -859,14 +898,20 @@ def bcsv_to_parquet(
     subset_layout: Optional[pybcsv.Layout] = None
     unflatten_schema: Optional[pa.Schema] = None
     if unflatten:
-        if col_names and set(col_names) != set(bcsv_layout.get_column_names()):
+        if col_names and col_names != list(bcsv_layout.get_column_names()):
+            # Build the subset layout in the user-requested column order (col_names),
+            # NOT file order — otherwise the streamed output and the empty-file
+            # fallback (which uses arrow_schema, built from col_names) disagree, and
+            # a requested reordering is silently ignored.
+            type_by_name = dict(
+                zip(
+                    bcsv_layout.get_column_names(),
+                    bcsv_layout.get_column_types(),
+                )
+            )
             subset_layout = pybcsv.Layout()
-            col_set = set(col_names)
-            for n, t in zip(
-                bcsv_layout.get_column_names(), bcsv_layout.get_column_types()
-            ):
-                if n in col_set:
-                    subset_layout.add_column(n, t)
+            for n in col_names:
+                subset_layout.add_column(n, type_by_name[n])
         else:
             subset_layout = bcsv_layout
         layout_names = list(subset_layout.get_column_names())
@@ -1135,7 +1180,8 @@ def bcsv2parquet_cli() -> None:
         "--row-group-size",
         type=int,
         default=None,
-        help="Row group size in PyArrow write_table (not supported in ParquetWriter streaming mode)",
+        help="Max rows per Parquet row group (applied per streamed batch; "
+        "values larger than --chunk-size are bounded by the batch size)",
     )
     mut_unflatten = parser.add_mutually_exclusive_group()
     mut_unflatten.add_argument(

{pybcsv-1.5.8 → pybcsv-1.5.10.dev0}/tests/test_parquet_conversion.py

@@ -278,8 +278,9 @@ class TestFlatSchemaToBcsvLayout(unittest.TestCase):
                 continue  # widened to float32 before reaching this function
             flat = [("x", arrow_type)]
             layout = _flat_schema_to_bcsv_layout(flat)
-            self.assertEqual(len(layout.get_column_names()), 1,
-                             f"Missing mapping for {arrow_type}")
+            self.assertEqual(
+                len(layout.get_column_names()), 1, f"Missing mapping for {arrow_type}"
+            )
 
     def test_large_string_maps_to_string(self):
         """pa.large_string() must map to BCSV STRING (not raise)."""
@@ -477,6 +478,41 @@ class TestFlattenBatch(unittest.TestCase):
         self.assertEqual(result.schema.field(0).name, "b")
         self.assertEqual(result.schema.field(1).name, "a")
 
+    def test_flatten_already_flat_with_dots(self):
+        """Parquet file where columns are already flat with dot-separated names.
+
+        This is a regression test for column names like 'sim.ve.counter' where
+        the Parquet schema has no nesting but the name contains dots.  The
+        flatten_batch function must recognize the name as a literal column name
+        and not decompose it into a non-existent struct path.
+        """
+        batch = pa.RecordBatch.from_arrays(
+            [
+                pa.array([1, 2, 3]),
+                pa.array([10.0, 20.0, 30.0]),
+                pa.array([100, 200, 300]),
+            ],
+            schema=pa.schema(
+                [
+                    ("sim.ve.counter", pa.int64()),
+                    ("sim.ve.value", pa.float64()),
+                    ("time", pa.int64()),
+                ]
+            ),
+        )
+        flat_schema = [
+            ("sim.ve.counter", pa.int64()),
+            ("sim.ve.value", pa.float64()),
+            ("time", pa.int64()),
+        ]
+        result = flatten_batch(batch, flat_schema)
+        self.assertEqual(result.num_rows, 3)
+        self.assertEqual(result.num_columns, 3)
+        # Values should be identical since no transformation needed
+        self.assertEqual(result.column("sim.ve.counter").to_pylist(), [1, 2, 3])
+        self.assertEqual(result.column("sim.ve.value").to_pylist(), [10.0, 20.0, 30.0])
+        self.assertEqual(result.column("time").to_pylist(), [100, 200, 300])
+
 
 if __name__ == "__main__":
     unittest.main()

pybcsv-1.5.8/VERSION.txt

@@ -1 +0,0 @@
-1.5.8