protowire-python 0.70.0__tar.gz → 0.75.0__tar.gz

{protowire_python-0.70.0 → protowire_python-0.75.0}/.github/workflows/ci.yml

@@ -16,12 +16,11 @@ concurrency:
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 env:
-  # Pin the sibling C++ checkout to a specific commit. cpp@v0.70.0
-  # predates the __int128 → checked_arith refactor (MSVC), the
-  # protobuf-API-skew shim, and the MSVC source-charset fix; 9af2ec0
-  # is the first commit with all of those. Bump to a v0.70.x tag
-  # once cpp cuts one that includes them.
-  PROTOWIRE_CPP_REF: 9af2ec04918a417933848de1577cd61f83a710b0
+  # Pin the sibling C++ checkout to a specific tag. v0.75.0 carries the
+  # PXF v0.72-series feature set (@<name> / @entry / @table directive
+  # grammar, schema validator, Result accessors, TableReader streaming)
+  # the Python port wraps. Bump in lockstep with cpp release cuts.
+  PROTOWIRE_CPP_REF: v0.75.0
 
 jobs:
   # ---------------------------------------------------------------------

{protowire_python-0.70.0 → protowire_python-0.75.0}/.github/workflows/codeql.yml

@@ -16,8 +16,8 @@ permissions:
   security-events: write
 
 env:
-  # See ci.yml for why this is a SHA, not v0.70.0.
-  PROTOWIRE_CPP_REF: 9af2ec04918a417933848de1577cd61f83a710b0
+  # See ci.yml for the rationale on this pin.
+  PROTOWIRE_CPP_REF: v0.75.0
 
 jobs:
   analyze:

{protowire_python-0.70.0 → protowire_python-0.75.0}/.github/workflows/publish.yml

@@ -24,13 +24,11 @@ env:
   # frozen FFI surface, so it must be an immutable ref — never a
   # branch.
   #
-  # Using a SHA (not the v0.70.0 tag) because cpp@v0.70.0 predates
-  # the f1d3eb0 __int128 → checked_arith refactor needed for MSVC,
-  # plus the protobuf-API-skew shim and MSVC source-charset fixes.
-  # 9af2ec0 (cpp main, ci: MSVC source-charset + skip pxf_escapes)
-  # is the first commit with all of those. Bump to v0.70.x once cpp
-  # cuts a tag that includes them.
-  PROTOWIRE_CPP_REF: 9af2ec04918a417933848de1577cd61f83a710b0
+  # Pinned to a tagged C++ release. v0.75.0 ships the PXF v0.72-series
+  # feature set (@<name> / @entry / @table grammar, schema validator,
+  # Result accessors, TableReader streaming) that this Python port
+  # wraps. Bump in lockstep with cpp release cuts.
+  PROTOWIRE_CPP_REF: v0.75.0
 
 jobs:
   # ---------------------------------------------------------------------

protowire_python-0.75.0/CHANGELOG.md

@@ -0,0 +1,117 @@
+# Changelog
+
+All notable changes to `protowire-python` are documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+The version number is kept aligned with the rest of the `protowire-*`
+stack — releases bump in lockstep across language ports when the wire
+format changes.
+
+## [Unreleased]
+
+## [0.75.0] — 2026-05-12
+
+First release after the v0.70.0 baseline. Wraps the
+[`protowire-cpp` v0.75.0](https://github.com/trendvidia/protowire-cpp/releases/tag/v0.75.0)
+PXF v0.72-series feature set into the Python port, bringing the
+`@<name>` / `@entry` / `@table` directive grammar, schema reserved-name
+validator, `Result.directives` / `Result.tables` accessors, and the
+`TableReader` streaming row reader through the nanobind FFI. The
+Python port skips intermediate version numbers and lands the bundled
+feature set directly on v0.75.0 to match the active wire revision
+across the `protowire-*` stack.
+
+### Added
+
+- **`pxf.TableReader` and `pxf.bind_row`** (draft §3.4.4). Streaming
+  consumption for the `@table` directive, alternative to materializing
+  every row into `Result.tables` up front. Construct via
+  `pxf.TableReader.from_bytes(data)`; iterate with the standard for
+  loop or call `next_or_none()` until it returns `None`. The reader
+  exposes the header `type` / `columns` / `directives` properties and
+  a `tail()` method that returns the unconsumed buffer for chaining a
+  second reader on multi-`@table` documents. `bind_row(msg, columns,
+  row)` is the per-row binder used by `scan()` and exposed as a
+  free function for callers iterating `Result.tables[i].rows` from
+  the materializing path. Strategy is format-and-reparse, matching
+  the C++ port: cells are rendered as a synthetic PXF body and run
+  through `unmarshal`, reusing every branch of the existing decoder
+  (WKT timestamps / durations, wrapper-nullability, enum-by-name,
+  `pxf.required` / `pxf.default`, oneof). PR-2 takes input as bytes;
+  a file-like / chunked-IO bridge is a possible follow-up.
+
+### Changed
+
+- **CI pin to protowire-cpp v0.75.0.** The cpp sibling now ships the
+  PXF v0.72-series feature set (directive grammar, schema validator,
+  Result accessors, TableReader streaming). The pin moves from the
+  pre-v0.72 commit `9af2ec0` to the `v0.75.0` tag so the Python
+  wrapper exposes the new surface.
+
+### Added
+
+- **`pxf.Result.directives` / `pxf.Result.tables`** — the document-root
+  directives the decoder saw at `unmarshal_full` time, exposed as
+  immutable dataclasses:
+  - `pxf.Directive(name, prefixes, type, body, has_body, line, column)`
+    for generic `@<name> *(prefix) [{ ... }]` blocks. `body` is the
+    raw bytes between `{` and `}` (verbatim), suitable for handing to
+    a follow-up `pxf.unmarshal` against the consumer's message type.
+    `type` keeps the v0.72.0 single-prefix back-compat shape.
+  - `pxf.TableDirective(type, columns, rows)` for `@table` directives,
+    with cells modeled as `None` (absent) or a `(kind, value)` 2-tuple
+    where kind ∈ {`"null"`, `"string"`, `"int"`, `"float"`, `"bool"`,
+    `"bytes"`, `"ident"`, `"timestamp"`, `"duration"`} — faithful to
+    the three-state cell grammar (absent / present-but-null /
+    present-with-value, draft §3.4.4).
+- **`pxf.validate_descriptor(msg)` + `pxf.Violation`** — schema
+  reserved-name check (draft §3.13). Returns the list of fields,
+  oneofs, and enum values whose names case-sensitively match a PXF
+  value keyword (`null` / `true` / `false`). Sorted by element FQN.
+- **`skip_validate` keyword** on `pxf.unmarshal` and
+  `pxf.unmarshal_full` (and the `_bytes` variants) — opt-out of the
+  per-call schema validator when the caller has already validated the
+  descriptor at registry-load time.
+
+## [0.70.0]
+
+Initial public release. The version number aligns this port with the rest
+of the `protowire-*` stack, which targets the 0.70.x series for the first
+coordinated public release. The wire codec is provided by
+[`protowire-cpp`](https://github.com/trendvidia/protowire-cpp) and reaches
+Python through a [nanobind](https://github.com/wjakob/nanobind) FFI; this
+port's behaviour follows the C++ port's at every wire-level question.
+
+### Added
+
+- **PyPI distribution** as the `protowire-python` package (the bare
+  `protowire` was taken by an unrelated 2021 CLI; the import name stays
+  `import protowire`). Binary wheels built by CI for CPython 3.10–3.13
+  on Linux × {x86_64, aarch64}, macOS × {x86_64, arm64}, and Windows ×
+  x86_64. Wheels are published through PyPI OIDC trusted publishing
+  with Sigstore provenance attestations.
+- **Comprehensive CI matrix**: build + test on Python 3.10/3.11/3.12/3.13
+  across Linux/macOS/Windows, plus a `cibuildwheel` smoke build on every
+  PR to catch packaging regressions early. Weekly CodeQL SAST.
+- **Governance scaffolding**: `LICENSE` (MIT), `CONTRIBUTING.md`,
+  `SECURITY.md` (security@trendvidia.com), `GOVERNANCE.md`,
+  `CODE_OF_CONDUCT.md`, `.github/CODEOWNERS`, issue + PR templates,
+  Dependabot for GitHub Actions and pip.
+
+### Changed (breaking)
+
+- **PXF parser stricter on key forms**, mirroring the upstream grammar
+  tightening in
+  [`trendvidia/protowire@8262bbb`](https://github.com/trendvidia/protowire/commit/8262bbb)
+  (`docs/grammar.ebnf`, `docs/draft-trendvidia-protowire-00.txt`):
+  - `=` (field assignment) and `{ … }` (submessage) now require an
+    identifier key. Inputs like `123 = 234` or `child { 123 = 123 }`
+    now raise `pxf.ParseError` with
+    `"field assignment with '=' requires an identifier key, got integer
+    (\"123\"); use ':' for map entries"`.
+  - `:` (map entry) is rejected at document top level — the document
+    represents a proto message, never a `map<K,V>`. Use `=` for
+    top-level field assignments. Map literals (`field = { 1: "x" }`)
+    still work because `:` remains valid inside `{ … }` blocks.

{protowire_python-0.70.0 → protowire_python-0.75.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: protowire-python
-Version: 0.70.0
+Version: 0.75.0
 Summary: Python wrapper around protowire-cpp — PXF text, SBE binary, and envelope codecs.
 Keywords: protobuf,pxf,sbe,wire-format,fix,trading
 Author-Email: "TrendVidia, LLC" <open-source@trendvidia.com>

{protowire_python-0.70.0 → protowire_python-0.75.0}/pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "scikit_build_core.build"
 # 2021 CLI tool). The import name stays `import protowire` — these two
 # names are independent (cf. python-dateutil → import dateutil).
 name = "protowire-python"
-version = "0.70.0"
+version = "0.75.0"
 description = "Python wrapper around protowire-cpp — PXF text, SBE binary, and envelope codecs."
 readme = "README.md"
 requires-python = ">=3.10"

{protowire_python-0.70.0 → protowire_python-0.75.0}/src/_protowire/module.cc

@@ -17,10 +17,13 @@
 
 #include <cstdint>
 #include <memory>
+#include <optional>
 #include <span>
+#include <sstream>
 #include <string>
 #include <string_view>
 #include <utility>
+#include <variant>
 #include <vector>
 
 #include <google/protobuf/descriptor.h>
@@ -77,9 +80,59 @@ const pbuf::Descriptor* FindDescriptor(const SchemaBundle& s,
 
 // --- pxf bindings ---------------------------------------------------------
 
+// CellToPyTuple converts a single AST cell value (or std::nullopt for an
+// absent cell) into the FFI shape consumed by pxf.py — `None` for absent,
+// `(kind, value)` otherwise. Used by PxfUnmarshalFull for @table rows.
+//
+// kind values mirror the AST variant tags:
+//   "null"      → nb::none()
+//   "string"    → str  (already-unescaped UTF-8)
+//   "int"       → str  (raw integer text — Python wrapper decides parse)
+//   "float"     → str  (raw float text)
+//   "bool"      → bool
+//   "bytes"     → bytes
+//   "ident"     → str
+//   "timestamp" → str  (raw RFC3339)
+//   "duration"  → str  (raw duration)
+nb::object CellToPyTuple(const std::optional<protowire::pxf::ValuePtr>& cell) {
+  if (!cell.has_value()) return nb::none();
+  using namespace protowire::pxf;
+  return std::visit(
+      [](const auto& p) -> nb::object {
+        using T = std::decay_t<decltype(*p)>;
+        if constexpr (std::is_same_v<T, NullVal>) {
+          return nb::make_tuple(std::string("null"), nb::none());
+        } else if constexpr (std::is_same_v<T, StringVal>) {
+          return nb::make_tuple(std::string("string"), p->value);
+        } else if constexpr (std::is_same_v<T, IntVal>) {
+          return nb::make_tuple(std::string("int"), p->raw);
+        } else if constexpr (std::is_same_v<T, FloatVal>) {
+          return nb::make_tuple(std::string("float"), p->raw);
+        } else if constexpr (std::is_same_v<T, BoolVal>) {
+          return nb::make_tuple(std::string("bool"), p->value);
+        } else if constexpr (std::is_same_v<T, BytesVal>) {
+          return nb::make_tuple(
+              std::string("bytes"),
+              nb::bytes(reinterpret_cast<const char*>(p->value.data()), p->value.size()));
+        } else if constexpr (std::is_same_v<T, IdentVal>) {
+          return nb::make_tuple(std::string("ident"), p->name);
+        } else if constexpr (std::is_same_v<T, TimestampVal>) {
+          return nb::make_tuple(std::string("timestamp"), p->raw);
+        } else if constexpr (std::is_same_v<T, DurationVal>) {
+          return nb::make_tuple(std::string("duration"), p->raw);
+        } else {
+          // List / Block are rejected at @table cell-parse time, so this
+          // branch is unreachable for cells. Surface as a clean error.
+          return nb::make_tuple(std::string("unknown"), nb::none());
+        }
+      },
+      *cell);
+}
+
 // PXF text -> binary proto bytes.
 nb::bytes PxfUnmarshal(nb::bytes text, nb::bytes fds_bytes,
-                       const std::string& full_name, bool discard_unknown) {
+                       const std::string& full_name, bool discard_unknown,
+                       bool skip_validate) {
   auto schema = BuildSchema(std::string_view(fds_bytes.c_str(), fds_bytes.size()));
   const auto* desc = FindDescriptor(schema, full_name);
   std::unique_ptr<pbuf::Message> msg(
@@ -87,6 +140,7 @@ nb::bytes PxfUnmarshal(nb::bytes text, nb::bytes fds_bytes,
 
   protowire::pxf::UnmarshalOptions opts;
   opts.discard_unknown = discard_unknown;
+  opts.skip_validate = skip_validate;
   auto st = protowire::pxf::Unmarshal(
       std::string_view(text.c_str(), text.size()), msg.get(), opts);
   if (!st.ok()) {
@@ -99,10 +153,20 @@ nb::bytes PxfUnmarshal(nb::bytes text, nb::bytes fds_bytes,
   return nb::bytes(out.data(), out.size());
 }
 
-// PXF text -> (binary proto bytes, set_paths, null_paths).
-std::tuple<nb::bytes, std::vector<std::string>, std::vector<std::string>>
+// Directive FFI shape: (name, prefixes, type, body, has_body, line, column).
+using PyDirective = std::tuple<std::string, std::vector<std::string>, std::string,
+                               nb::bytes, bool, int, int>;
+// TableDirective FFI shape: (type, columns, rows) where rows is a list of
+// lists of cells (each cell None or (kind, value); see CellToPyTuple).
+using PyTableDirective = std::tuple<std::string, std::vector<std::string>,
+                                    std::vector<std::vector<nb::object>>>;
+
+// PXF text -> (binary proto bytes, set_paths, null_paths, directives, tables).
+std::tuple<nb::bytes, std::vector<std::string>, std::vector<std::string>,
+           std::vector<PyDirective>, std::vector<PyTableDirective>>
 PxfUnmarshalFull(nb::bytes text, nb::bytes fds_bytes,
-                 const std::string& full_name, bool discard_unknown) {
+                 const std::string& full_name, bool discard_unknown,
+                 bool skip_validate) {
   auto schema = BuildSchema(std::string_view(fds_bytes.c_str(), fds_bytes.size()));
   const auto* desc = FindDescriptor(schema, full_name);
   std::unique_ptr<pbuf::Message> msg(
@@ -110,6 +174,7 @@ PxfUnmarshalFull(nb::bytes text, nb::bytes fds_bytes,
 
   protowire::pxf::UnmarshalOptions opts;
   opts.discard_unknown = discard_unknown;
+  opts.skip_validate = skip_validate;
   auto r = protowire::pxf::UnmarshalFull(
       std::string_view(text.c_str(), text.size()), msg.get(), opts);
   if (!r.ok()) {
@@ -119,11 +184,141 @@ PxfUnmarshalFull(nb::bytes text, nb::bytes fds_bytes,
   if (!msg->SerializeToString(&out)) {
     throw nb::value_error("pxf.unmarshal_full: proto serialization failed");
   }
+  // Marshal directives.
+  std::vector<PyDirective> py_dirs;
+  py_dirs.reserve(r->Directives().size());
+  for (const auto& d : r->Directives()) {
+    py_dirs.emplace_back(
+        d.name, d.prefixes, d.type,
+        nb::bytes(d.body.data(), d.body.size()),
+        d.has_body, d.pos.line, d.pos.column);
+  }
+  // Marshal tables.
+  std::vector<PyTableDirective> py_tables;
+  py_tables.reserve(r->Tables().size());
+  for (const auto& t : r->Tables()) {
+    std::vector<std::vector<nb::object>> py_rows;
+    py_rows.reserve(t.rows.size());
+    for (const auto& row : t.rows) {
+      std::vector<nb::object> py_cells;
+      py_cells.reserve(row.cells.size());
+      for (const auto& cell : row.cells) py_cells.push_back(CellToPyTuple(cell));
+      py_rows.push_back(std::move(py_cells));
+    }
+    py_tables.emplace_back(t.type, t.columns, std::move(py_rows));
+  }
   return {nb::bytes(out.data(), out.size()),
           r->SetFields(),
-          r->NullFields()};
+          r->NullFields(),
+          std::move(py_dirs),
+          std::move(py_tables)};
+}
+
+// PXF schema reserved-name check (draft §3.13). Returns a list of
+// (kind, element, name, file) tuples. Empty list ⇒ conformant schema.
+// kind values: "field" / "oneof" / "enum_value".
+std::vector<std::tuple<std::string, std::string, std::string, std::string>>
+PxfValidateDescriptor(nb::bytes fds_bytes, const std::string& full_name) {
+  auto schema = BuildSchema(std::string_view(fds_bytes.c_str(), fds_bytes.size()));
+  const auto* desc = FindDescriptor(schema, full_name);
+  auto vs = protowire::pxf::ValidateDescriptor(desc);
+  std::vector<std::tuple<std::string, std::string, std::string, std::string>> out;
+  out.reserve(vs.size());
+  for (const auto& v : vs) {
+    std::string kind;
+    switch (v.kind) {
+      case protowire::pxf::ViolationKind::kField:     kind = "field";      break;
+      case protowire::pxf::ViolationKind::kOneof:     kind = "oneof";      break;
+      case protowire::pxf::ViolationKind::kEnumValue: kind = "enum_value"; break;
+    }
+    out.emplace_back(std::move(kind), v.element, v.name, v.file);
+  }
+  return out;
 }
 
+// --- PyTableReader: streaming @table consumption -------------------------
+//
+// Wraps protowire::pxf::TableReader. The reader takes a std::istream*; we
+// hold the istringstream alongside the reader so its lifetime is bound to
+// the Python object. Input is provided as bytes (PR-2 scope); a file-like
+// streambuf bridge is a possible follow-up.
+class PyTableReader {
+ public:
+  static std::unique_ptr<PyTableReader> FromBytes(nb::bytes data) {
+    auto out = std::unique_ptr<PyTableReader>(new PyTableReader());
+    out->stream_ = std::make_unique<std::istringstream>(
+        std::string(data.c_str(), data.size()));
+    auto tr = protowire::pxf::TableReader::Create(out->stream_.get());
+    if (!tr.ok()) {
+      throw nb::value_error(("pxf.TableReader: " + tr.status().ToString()).c_str());
+    }
+    out->reader_ = std::move(*tr);
+    // Marshal the side-channel directives once at construction; they're
+    // fixed for the reader's lifetime.
+    for (const auto& d : out->reader_->Directives()) {
+      out->directives_.emplace_back(
+          d.name, d.prefixes, d.type,
+          nb::bytes(d.body.data(), d.body.size()),
+          d.has_body, d.pos.line, d.pos.column);
+    }
+    return out;
+  }
+
+  const std::string& Type() const { return reader_->Type(); }
+  const std::vector<std::string>& Columns() const { return reader_->Columns(); }
+  const std::vector<PyDirective>& Directives() const { return directives_; }
+  bool Done() const { return reader_->Done(); }
+
+  // Returns the next row as a Python list of cells, or None at EOF.
+  // Raises ValueError on parse error.
+  nb::object NextOrNone() {
+    if (reader_->Done()) return nb::none();
+    protowire::pxf::TableRow row;
+    auto s = reader_->Next(&row);
+    if (!s.ok()) {
+      throw nb::value_error(("pxf.TableReader.next: " + s.ToString()).c_str());
+    }
+    if (reader_->Done()) return nb::none();
+    return RowToList(row);
+  }
+
+  // Iterator protocol: __next__ raises StopIteration at EOF.
+  nb::object Next() {
+    if (reader_->Done()) throw nb::stop_iteration();
+    protowire::pxf::TableRow row;
+    auto s = reader_->Next(&row);
+    if (!s.ok()) {
+      throw nb::value_error(("pxf.TableReader.next: " + s.ToString()).c_str());
+    }
+    if (reader_->Done()) throw nb::stop_iteration();
+    return RowToList(row);
+  }
+
+  // Drains the remaining buffered + underlying bytes. Only meaningful
+  // after Done(); the Python wrapper exposes this as a method that
+  // returns bytes so callers can chain a second TableReader on
+  // multi-@table documents.
+  nb::bytes Tail() {
+    auto t = reader_->Tail();
+    std::ostringstream buf;
+    buf << t->rdbuf();
+    std::string s = buf.str();
+    return nb::bytes(s.data(), s.size());
+  }
+
+ private:
+  static nb::object RowToList(const protowire::pxf::TableRow& row) {
+    std::vector<nb::object> cells;
+    cells.reserve(row.cells.size());
+    for (const auto& cell : row.cells) cells.push_back(CellToPyTuple(cell));
+    return nb::cast(cells);
+  }
+
+  std::unique_ptr<std::istringstream> stream_;
+  std::unique_ptr<protowire::pxf::TableReader> reader_;
+  std::vector<PyDirective> directives_;
+};
+
 // Binary proto bytes -> PXF text.
 std::string PxfMarshal(nb::bytes msg_bytes, nb::bytes fds_bytes,
                        const std::string& full_name) {
@@ -301,10 +496,22 @@ NB_MODULE(_protowire, m) {
   m.doc() = "protowire native extension (nanobind shim around protowire-cpp)";
 
   m.def("pxf_unmarshal", &PxfUnmarshal, "text"_a, "fds"_a, "full_name"_a,
-        "discard_unknown"_a = false);
+        "discard_unknown"_a = false, "skip_validate"_a = false);
   m.def("pxf_unmarshal_full", &PxfUnmarshalFull, "text"_a, "fds"_a,
-        "full_name"_a, "discard_unknown"_a = false);
+        "full_name"_a, "discard_unknown"_a = false, "skip_validate"_a = false);
   m.def("pxf_marshal", &PxfMarshal, "msg_bytes"_a, "fds"_a, "full_name"_a);
+  m.def("pxf_validate_descriptor", &PxfValidateDescriptor, "fds"_a, "full_name"_a);
+
+  nb::class_<PyTableReader>(m, "PxfTableReader")
+      .def_static("from_bytes", &PyTableReader::FromBytes, "data"_a)
+      .def_prop_ro("type", &PyTableReader::Type)
+      .def_prop_ro("columns", &PyTableReader::Columns)
+      .def_prop_ro("directives", &PyTableReader::Directives)
+      .def_prop_ro("done", &PyTableReader::Done)
+      .def("next_or_none", &PyTableReader::NextOrNone)
+      .def("tail", &PyTableReader::Tail)
+      .def("__iter__", [](PyTableReader& self) -> PyTableReader& { return self; })
+      .def("__next__", &PyTableReader::Next);
 
   nb::class_<SbeCodec>(m, "SbeCodec")
       .def_static("create", &SbeCodec::Create, "fds"_a, "file_names"_a)

{protowire_python-0.70.0 → protowire_python-0.75.0}/src/protowire/__init__.py

@@ -5,4 +5,4 @@
 from . import envelope, pxf, sbe
 
 __all__ = ["pxf", "sbe", "envelope"]
-__version__ = "0.70.0"
+__version__ = "0.75.0"