dirigent-block-parquet 0.17.2__tar.gz

dirigent_block_parquet-0.17.2/LICENSE

@@ -0,0 +1,18 @@
+Copyright (c) 2026 Morten Olav Hansen <morten@winterop.com>. All rights reserved.
+
+This source code and accompanying documentation are the property of
+Morten Olav Hansen. No license, express or implied, is granted to use, copy,
+modify, merge, publish, distribute, sublicense, or sell copies of this
+software or its derivatives.
+
+The source is published for reference only. Any use beyond reading
+requires written permission from the copyright holder.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES,
+OR OTHER LIABILITY ARISING FROM THE USE OF THE SOFTWARE.
+
+Third-party components redistributed with this software, and the licences they
+carry, are listed in THIRD_PARTY_NOTICES.md.

dirigent_block_parquet-0.17.2/PKG-INFO

@@ -0,0 +1,44 @@
+Metadata-Version: 2.4
+Name: dirigent-block-parquet
+Version: 0.17.2
+Summary: The parquet block family for dirigent: the convert.arrow codec, on pyarrow.
+License-Expression: LicenseRef-Proprietary
+License-File: LICENSE
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: dirigent-common==0.17.2
+Requires-Dist: dirigent-plugin==0.17.2
+Requires-Dist: pyarrow>=21.0.0
+Requires-Python: >=3.13
+Description-Content-Type: text/markdown
+
+# dirigent-block-parquet
+
+The parquet block family: one codec, `convert.arrow`, trading parquet with the
+three text spellings `convert.std` already trades between.
+
+| From | To | What it does |
+| --- | --- | --- |
+| `parquet` | `json`, `ndjson`, `csv` | Each row becomes one record, every value in its JSON spelling. |
+| `json`, `ndjson`, `csv` | `parquet` | Each record becomes one row, under a schema inferred from the records. |
+
+The record model is `convert.std`'s: a sequence of flat records. What parquet adds is
+types, and the codec is honest about them in both directions:
+
+- **Writing** infers one type per column -- boolean, int64, float64, or string -- from the
+  records. Integers and floats unify to a float column, because JSON calls both a number;
+  any other mix is refused naming the column. A column every row leaves null is written as
+  a nullable string column. A csv source carries no types, so csv to parquet writes string
+  columns and nothing else.
+- **Reading** gives every value its JSON spelling: timestamps, dates and times come back as
+  ISO strings, decimals as strings, and a float that is NaN or infinite as null. A nested
+  column, or one holding raw bytes, is refused naming it -- flattening is a reshape, and a
+  reshape belongs to a jq step that knows what the flattening should mean.
+
+A conversion is a storage-object operation, like `storage.copy`: it reads the object at
+`source` and writes the one at `target`, and neither side is ever a value the step carries.
+A run that means to look at the records reads them with `storage.read`.
+
+The family ships separately because `convert.std` is deliberately on the standard library
+and nothing else; this codec stands on [pyarrow](https://arrow.apache.org/docs/python/).
+Nothing else in the workspace may depend on it.

dirigent_block_parquet-0.17.2/README.md

@@ -0,0 +1,30 @@
+# dirigent-block-parquet
+
+The parquet block family: one codec, `convert.arrow`, trading parquet with the
+three text spellings `convert.std` already trades between.
+
+| From | To | What it does |
+| --- | --- | --- |
+| `parquet` | `json`, `ndjson`, `csv` | Each row becomes one record, every value in its JSON spelling. |
+| `json`, `ndjson`, `csv` | `parquet` | Each record becomes one row, under a schema inferred from the records. |
+
+The record model is `convert.std`'s: a sequence of flat records. What parquet adds is
+types, and the codec is honest about them in both directions:
+
+- **Writing** infers one type per column -- boolean, int64, float64, or string -- from the
+  records. Integers and floats unify to a float column, because JSON calls both a number;
+  any other mix is refused naming the column. A column every row leaves null is written as
+  a nullable string column. A csv source carries no types, so csv to parquet writes string
+  columns and nothing else.
+- **Reading** gives every value its JSON spelling: timestamps, dates and times come back as
+  ISO strings, decimals as strings, and a float that is NaN or infinite as null. A nested
+  column, or one holding raw bytes, is refused naming it -- flattening is a reshape, and a
+  reshape belongs to a jq step that knows what the flattening should mean.
+
+A conversion is a storage-object operation, like `storage.copy`: it reads the object at
+`source` and writes the one at `target`, and neither side is ever a value the step carries.
+A run that means to look at the records reads them with `storage.read`.
+
+The family ships separately because `convert.std` is deliberately on the standard library
+and nothing else; this codec stands on [pyarrow](https://arrow.apache.org/docs/python/).
+Nothing else in the workspace may depend on it.

dirigent_block_parquet-0.17.2/pyproject.toml

@@ -0,0 +1,30 @@
+[project]
+name = "dirigent-block-parquet"
+version = "0.17.2"
+description = "The parquet block family for dirigent: the convert.arrow codec, on pyarrow."
+readme = "README.md"
+requires-python = ">=3.13"
+license = "LicenseRef-Proprietary"
+license-files = ["LICENSE"]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.13",
+]
+dependencies = [
+    "dirigent-common==0.17.2",
+    "dirigent-plugin==0.17.2",
+    "pyarrow>=21.0.0",
+]
+
+[project.entry-points."dirigent.plugins.v1"]
+block-parquet = "dirigent_block_parquet:plugin"
+
+[build-system]
+requires = ["uv_build>=0.12.0,<0.13.0"]
+build-backend = "uv_build"
+
+[tool.uv.sources.dirigent-plugin]
+workspace = true
+
+[tool.uv.sources.dirigent-common]
+workspace = true

dirigent_block_parquet-0.17.2/pyproject.toml.orig

@@ -0,0 +1,28 @@
+[project]
+name = "dirigent-block-parquet"
+version = "0.17.2"
+description = "The parquet block family for dirigent: the convert.arrow codec, on pyarrow."
+readme = "README.md"
+requires-python = ">=3.13"
+license = "LicenseRef-Proprietary"
+license-files = ["LICENSE"]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.13",
+]
+dependencies = [
+    "dirigent-common==0.17.2",
+    "dirigent-plugin==0.17.2",
+    "pyarrow>=21.0.0",
+]
+
+[project.entry-points."dirigent.plugins.v1"]
+block-parquet = "dirigent_block_parquet:plugin"
+
+[build-system]
+requires = ["uv_build>=0.12.0,<0.13.0"]
+build-backend = "uv_build"
+
+[tool.uv.sources]
+dirigent-plugin = { workspace = true }
+dirigent-common = { workspace = true }

dirigent_block_parquet-0.17.2/src/dirigent_block_parquet/__init__.py

@@ -0,0 +1,24 @@
+"""The parquet block family: the ``convert.arrow`` codec, on pyarrow."""
+
+from dirigent_block_parquet.arrow import TEXT_FORMATS, UNIT, ArrowConverter
+from dirigent_plugin import Contribution, extension
+
+
+class ParquetBlocks:
+    """The plugin object the host discovers under the dirigent.plugins.v1 entry-point group."""
+
+    @extension
+    def contribute(self) -> Contribution:
+        """Contribute the ``convert.arrow`` codec."""
+        return Contribution(operators=[ArrowConverter()])
+
+
+plugin = ParquetBlocks()
+
+__all__ = [
+    "TEXT_FORMATS",
+    "UNIT",
+    "ArrowConverter",
+    "ParquetBlocks",
+    "plugin",
+]

dirigent_block_parquet-0.17.2/src/dirigent_block_parquet/arrow.py

@@ -0,0 +1,261 @@
+"""The ``arrow`` engine for the convert verb: parquet against the text formats, on pyarrow."""
+
+import csv
+import io
+import json
+from typing import Final
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+from pydantic import JsonValue
+
+from dirigent_block_parquet.messages import (
+    BINARY_COLUMN,
+    CSV_HEADER_REPEATED,
+    CSV_HEADER_UNNAMED,
+    CSV_ROW_TOO_WIDE,
+    LINE_NOT_JSON,
+    MIXED_COLUMN,
+    NESTED_COLUMN,
+    NESTED_VALUE,
+    NO_JSON_SPELLING,
+    NOT_A_JSON_ARRAY,
+    NOT_JSON,
+    NOT_PARQUET,
+    NOT_UTF8,
+    RECORD_NOT_AN_OBJECT,
+)
+from dirigent_common import spelled
+from dirigent_plugin import Converter, TransformError
+
+#: The text spellings this engine trades parquet with.
+TEXT_FORMATS: Final = ("json", "ndjson", "csv")
+
+#: What a target format calls one element of the sequence it writes.
+UNIT: Final = {"json": "element", "ndjson": "line", "csv": "row", "parquet": "row"}
+
+
+class ArrowConverter(Converter):
+    """Re-encodes a sequence of records between parquet and the text formats.
+
+    The record model is ``convert.std``'s: a sequence of flat records, read out of one
+    spelling and written in another. What parquet adds is types -- a column knows whether it
+    holds numbers or text -- so writing infers a schema from the records, and a column whose
+    rows disagree about their type is refused naming it rather than coerced. Reading gives
+    every value its JSON spelling: timestamps, dates and times come back as ISO strings,
+    decimals as strings, and a float that is NaN or infinite as null, because JSON has no
+    other words for them.
+
+    A csv carries no types, so csv to parquet writes string columns and nothing else.
+    """
+
+    kind = "arrow"
+    summary = "Convert between parquet and the text formats."
+    pairs = frozenset({("parquet", text) for text in TEXT_FORMATS} | {(text, "parquet") for text in TEXT_FORMATS})
+
+    def convert(self, source: bytes, *, source_format: str, target_format: str) -> bytes:
+        """Read the records out of the source format and write them in the target format."""
+        if source_format == "parquet":
+            records = _read_parquet(source, target_format)
+        else:
+            records = _read_text(source, source_format, target_format)
+        if target_format == "parquet":
+            return _write_parquet(records)
+        return _write_text(records, target_format).encode()
+
+
+def _read_parquet(source: bytes, target_format: str) -> list[dict[str, JsonValue]]:
+    """Read a parquet payload as records, refusing what the target has no spelling for."""
+    try:
+        table = pq.read_table(pa.BufferReader(source))  # pyright: ignore[reportUnknownMemberType]
+    except pa.ArrowInvalid as error:
+        raise TransformError(NOT_PARQUET.render(detail=str(error))) from error
+    for name, kind in zip(table.schema.names, table.schema.types, strict=True):
+        if pa.types.is_nested(kind):
+            raise TransformError(
+                NESTED_COLUMN.render(
+                    column=repr(name), kind=kind, target_format=target_format, unit=UNIT[target_format]
+                )
+            )
+        if pa.types.is_binary(kind) or pa.types.is_large_binary(kind) or pa.types.is_fixed_size_binary(kind):
+            raise TransformError(BINARY_COLUMN.render(column=repr(name)))
+    return [{key: _spelled(value) for key, value in row.items()} for row in table.to_pylist()]
+
+
+def _spelled(value: object) -> JsonValue:
+    """Give one arrow value its JSON spelling, in the one house conversion."""
+    try:
+        return spelled(value)
+    except ValueError as error:
+        # A schema check above rules out nested and binary columns, so nothing else arrives.
+        raise TransformError(NO_JSON_SPELLING.render(detail=str(error))) from error
+
+
+def _read_text(source: bytes, source_format: str, target_format: str) -> list[dict[str, JsonValue]]:
+    """Read the sequence of records a text format spells out, refusing rows that nest."""
+    try:
+        text = source.decode("utf-8")
+    except UnicodeDecodeError as error:
+        raise TransformError(NOT_UTF8.render(reason=error.reason, position=error.start)) from error
+    if source_format == "csv":
+        return _read_csv(text)
+    values = _read_json_values(text, source_format, target_format)
+    records: list[dict[str, JsonValue]] = []
+    for number, value in enumerate(values, start=1):
+        if not isinstance(value, dict):
+            raise TransformError(RECORD_NOT_AN_OBJECT.render(unit=UNIT[source_format], number=number))
+        records.append(value)
+    return records
+
+
+def _read_json_values(text: str, source_format: str, target_format: str) -> list[JsonValue]:
+    """Read a JSON array, or one JSON value per line, the way convert.std reads them."""
+    if source_format == "json":
+        try:
+            parsed: JsonValue = json.loads(text)
+        except ValueError as error:
+            raise TransformError(NOT_JSON.render(detail=str(error))) from error
+        if not isinstance(parsed, list):
+            raise TransformError(NOT_A_JSON_ARRAY.render(target_format=target_format, unit=UNIT[target_format]))
+        return parsed
+    values: list[JsonValue] = []
+    for number, line in enumerate(text.splitlines(), start=1):
+        if not line.strip():
+            continue
+        try:
+            values.append(json.loads(line))
+        except ValueError as error:
+            raise TransformError(LINE_NOT_JSON.render(number=number, detail=str(error))) from error
+    return values
+
+
+def _read_csv(text: str) -> list[dict[str, JsonValue]]:
+    """Read the header row and its rows, as objects whose every value is a string."""
+    rows = csv.reader(io.StringIO(text, newline=""))
+    header = next(rows, None)
+    if header is None:
+        return []
+    _check_header(header)
+    records: list[dict[str, JsonValue]] = []
+    for number, row in enumerate(rows, start=1):
+        if not row:
+            continue
+        if len(row) > len(header):
+            raise TransformError(CSV_ROW_TOO_WIDE.render(number=number))
+        records.append({name: row[index] if index < len(row) else "" for index, name in enumerate(header)})
+    return records
+
+
+def _columns_phrase(positions: list[int]) -> str:
+    """Word one or more column positions, counting from one."""
+    listed = ", ".join(str(position) for position in positions)
+    return f"column {listed}" if len(positions) == 1 else f"columns {listed}"
+
+
+def _check_header(header: list[str]) -> None:
+    """Refuse a header that leaves a name empty or repeats one.
+
+    A record key names its column, so neither has a record spelling: keeping one of two
+    columns that share a name, or keying a column by the empty string, drops content while
+    reporting success.
+    """
+    positions: dict[str, list[int]] = {}
+    for index, name in enumerate(header, start=1):
+        positions.setdefault(name, []).append(index)
+    unnamed = positions.pop("", None)
+    if unnamed is not None:
+        raise TransformError(CSV_HEADER_UNNAMED.render(columns=_columns_phrase(unnamed)))
+    repeated = [(name, where) for name, where in positions.items() if len(where) > 1]
+    if repeated:
+        listed = "; ".join(f"{name!r} at {_columns_phrase(where)}" for name, where in repeated)
+        raise TransformError(CSV_HEADER_REPEATED.render(listed=listed))
+
+
+#: The arrow type each column kind writes as. A column every row leaves null carries no
+#: kind at all and is written as a nullable string column.
+KIND_TYPES: Final[dict[str, pa.DataType]] = {
+    "boolean": pa.bool_(),
+    "integer": pa.int64(),
+    "number": pa.float64(),
+    "string": pa.string(),
+}
+
+
+def _write_parquet(records: list[dict[str, JsonValue]]) -> bytes:
+    """Infer one type per column from the records, then write them as parquet bytes."""
+    schema = pa.schema([(name, KIND_TYPES[kind]) for name, kind in _columns(records).items()])
+    sink = io.BytesIO()
+    pq.write_table(pa.Table.from_pylist(records, schema=schema), sink)  # pyright: ignore[reportUnknownMemberType]
+    return sink.getvalue()
+
+
+def _columns(records: list[dict[str, JsonValue]]) -> dict[str, str]:
+    """Name each column's one kind, refusing a column whose rows disagree.
+
+    Integers and floats unify to a float column, because JSON calls both a number; any
+    other mix is two types in one column, which parquet does not write and a codec that
+    coerced would not preserve.
+    """
+    kinds: dict[str, str] = {}
+    for number, record in enumerate(records, start=1):
+        for key, value in record.items():
+            kind = _kind(value, number, key)
+            if kind is None:
+                continue
+            settled = kinds.get(key)
+            if settled is None:
+                kinds[key] = kind
+            elif {settled, kind} == {"integer", "number"}:
+                kinds[key] = "number"
+            elif settled != kind:
+                raise TransformError(MIXED_COLUMN.render(column=repr(key), settled=settled, kind=kind))
+    for key in {name for record in records for name in record}:
+        kinds.setdefault(key, "string")
+    return kinds
+
+
+def _kind(value: JsonValue, number: int, key: str) -> str | None:
+    """Name one value's column kind, refusing the nested ones parquet is not asked to hold."""
+    if value is None:
+        return None
+    if isinstance(value, dict | list):
+        raise TransformError(NESTED_VALUE.render(number=number, key=repr(key)))
+    # Before the integer check, because a bool is an int in Python and is not one in JSON.
+    if isinstance(value, bool):
+        return "boolean"
+    if isinstance(value, int):
+        return "integer"
+    if isinstance(value, float):
+        return "number"
+    return "string"
+
+
+def _write_text(records: list[dict[str, JsonValue]], target_format: str) -> str:
+    """Write the records in a text spelling, the way convert.std writes them."""
+    if target_format == "json":
+        return json.dumps(records, separators=(",", ":"))
+    if target_format == "ndjson":
+        return "".join(f"{json.dumps(record, separators=(',', ':'))}\n" for record in records)
+    return _write_csv(records)
+
+
+def _write_csv(records: list[dict[str, JsonValue]]) -> str:
+    """Write a header of every key any row has, in first-seen order, and one row per record."""
+    header: list[str] = []
+    for record in records:
+        header.extend(key for key in record if key not in header)
+    out = io.StringIO(newline="")
+    writer = csv.writer(out, lineterminator="\n")
+    writer.writerow(header)
+    for record in records:
+        writer.writerow([_cell(record.get(key)) for key in header])
+    return out.getvalue()
+
+
+def _cell(value: JsonValue) -> str:
+    """Render one value as csv text. Reading ruled the nested ones out already."""
+    if value is None:
+        return ""
+    if isinstance(value, str):
+        return value
+    return json.dumps(value)

dirigent_block_parquet-0.17.2/src/dirigent_block_parquet/messages.py

@@ -0,0 +1,70 @@
+"""Every refusal the parquet codec makes, catalogued under the ``parquet`` prefix.
+
+A codec refusal reaches an attempt as ``plugin.transform_failed``: the convert frame owns
+the failure, and the codec's sentence rides on it as the ``detail`` param.
+"""
+
+from dirigent_common import Catalogue
+
+PARQUET = Catalogue("parquet")
+
+NOT_PARQUET = PARQUET.define("not_parquet", "the input is not parquet: {detail}")
+
+NESTED_COLUMN = PARQUET.define(
+    "nested_column",
+    "column {column} is {kind}, and a nested column has no {target_format} {unit} spelling; "
+    "flatten it before converting",
+)
+
+BINARY_COLUMN = PARQUET.define(
+    "binary_column",
+    "column {column} holds raw bytes, which have no JSON spelling; decode or drop it before converting",
+)
+
+NO_JSON_SPELLING = PARQUET.define("no_json_spelling", "{detail}")
+
+NOT_UTF8 = PARQUET.define(
+    "not_utf8",
+    "the input is not UTF-8 text: {reason} at byte {position}; a text format is UTF-8",
+)
+
+RECORD_NOT_AN_OBJECT = PARQUET.define(
+    "record_not_an_object",
+    "{unit} {number} of the input is not an object, and a parquet row is a flat object",
+)
+
+NOT_JSON = PARQUET.define("not_json", "the input is not JSON: {detail}")
+
+NOT_A_JSON_ARRAY = PARQUET.define(
+    "not_a_json_array",
+    "json to {target_format} writes one {unit} per element, so the input has to be a JSON array",
+)
+
+LINE_NOT_JSON = PARQUET.define("line_not_json", "line {number} of the input is not JSON: {detail}")
+
+CSV_ROW_TOO_WIDE = PARQUET.define(
+    "csv_row_too_wide",
+    "row {number} of the csv has more cells than the header names columns, "
+    "and a cell no column names has nowhere to go",
+)
+
+CSV_HEADER_UNNAMED = PARQUET.define(
+    "csv_header_unnamed",
+    "the csv header has no name at {columns}, and a record key names its column; name it before converting",
+)
+
+CSV_HEADER_REPEATED = PARQUET.define(
+    "csv_header_repeated",
+    "the csv header repeats a column name: {listed}; a record key names one column, so rename them before converting",
+)
+
+MIXED_COLUMN = PARQUET.define(
+    "mixed_column",
+    "column {column} holds both {settled} and {kind} values, and a parquet "
+    "column carries one type; reshape it before converting",
+)
+
+NESTED_VALUE = PARQUET.define(
+    "nested_value",
+    "row {number} has a nested value at {key}, and this codec writes flat columns; flatten it before converting",
+)