struct-frame 0.10.8__tar.gz → 0.10.10__tar.gz

struct_frame-0.10.10/.github/workflows/coverage.yml

@@ -0,0 +1,68 @@
+name: Coverage
+
+on:
+  push:
+    branches: [main, develop]
+  pull_request:
+    branches: [main, develop]
+
+concurrency:
+  group: coverage-${{ github.sha }}
+  cancel-in-progress: true
+
+# Tier C item 1: coverage instrumentation.  This workflow lands the Codecov
+# plumbing using Python (`coverage.py`) as the first language; gcov+lcov,
+# c8/nyc, coverlet, and cargo-llvm-cov plug into the same `codecov-action`
+# step in follow-up PRs.
+
+jobs:
+  python-coverage:
+    name: Python (coverage.py)
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+        with: { submodules: true }
+      - uses: actions/setup-python@v5
+        with: { python-version: "3.11" }
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install proto-schema-parser coverage hypothesis pytest
+      - name: Generate code (required by property tests)
+        run: python test_all.py --only-generate --skip-lang csharp --skip-lang rust
+      - name: Run Python suite under coverage
+        # IMPORTANT — what this number does and does NOT measure.
+        # Only *in-process* execution of the generator (src/struct_frame) is
+        # instrumented. Two large sources of real exercise are NOT captured here,
+        # so the reported percentage UNDERSTATES actual coverage:
+        #   1. Tests that invoke the generator as a subprocess (`python main.py`)
+        #      — e.g. test_generator_validation.py — are not instrumented by the
+        #      parent `coverage` process.
+        #   2. The per-language code emitters (c_gen, cpp_gen, ts_gen, js_gen,
+        #      csharp_gen, rust_gen, gql_gen) are only imported when generating
+        #      those languages, which the full cross-language suite in
+        #      tests/run_tests.py does — but that suite is not run under coverage.
+        # test_magic_bytes.py is included because its `_load_model` parses and
+        # validates in-process, exercising the parser/option/validation front end.
+        # Follow-up to make this honest: enable coverage subprocess capture
+        # (COVERAGE_PROCESS_START + parallel mode + `coverage combine`) so the
+        # subprocess generator runs and per-language emitters are measured too.
+        run: |
+          coverage run --source=src/struct_frame --branch \
+              -m pytest tests/test_property_roundtrip.py -v
+          # These files have no ``__main__`` block, so they must run under
+          # pytest; invoking them as scripts would execute zero tests.
+          coverage run --source=src/struct_frame --branch --append \
+              -m pytest tests/test_generator_validation.py tests/test_magic_bytes.py -v
+          coverage xml -o coverage.xml
+          coverage report --skip-covered --skip-empty
+      - name: Upload to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          files: coverage.xml
+          flags: python
+          fail_ci_if_error: false
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

struct_frame-0.10.10/.github/workflows/doc-samples.yml

@@ -0,0 +1,44 @@
+name: Documentation Samples
+
+on:
+  push:
+    branches: [main, develop]
+  pull_request:
+    branches: [main, develop]
+
+concurrency:
+  group: doc-samples-${{ github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  doc-samples:
+    name: Doctest-style sample compilation
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+        with: { submodules: true }
+      - uses: actions/setup-python@v5
+        with: { python-version: "3.11" }
+      - uses: actions/setup-node@v4
+        with: { node-version: "20" }
+      - uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: "8.0.x"
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y gcc g++ build-essential
+      - name: Install Python dependencies
+        run: pip install proto-schema-parser
+      - name: Install TypeScript dependencies
+        run: npm ci
+        working-directory: tests/ts
+      - name: Extract samples
+        run: python tests/docs/extract_samples.py
+      - uses: dtolnay/rust-toolchain@stable
+      - name: Compile documentation samples
+        # Full mode: C, C++, TypeScript, Rust and C# samples are compiled too. Snippets that depend
+        # on the reader's own code carry a `no-compile` marker in their code fence.
+        run: python tests/docs/run_samples.py

struct_frame-0.10.10/.github/workflows/fuzz.yml

@@ -0,0 +1,152 @@
+name: Fuzz (short)
+
+on:
+  push:
+    branches: [main, develop]
+  pull_request:
+    branches: [main, develop]
+
+concurrency:
+  group: fuzz-${{ github.sha }}
+  cancel-in-progress: true
+
+# Short per-push smoke fuzzer: 60 seconds per harness, intended only to catch
+# regressions that surface immediately.  Long-running corpus growth happens
+# upstream once the project is submitted to OSS-Fuzz (tracked as a follow-up).
+
+jobs:
+  fuzz-c-parser:
+    name: libFuzzer / C accumulating-reader
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+        with: { submodules: true }
+      - name: Install clang
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y clang llvm
+      - name: Build C fuzz harness
+        run: |
+          mkdir -p tests/c/build
+          clang -O1 -g -fsanitize=fuzzer,address,undefined \
+                -fno-sanitize-recover=all \
+                -I src/struct_frame/boilerplate/c \
+                tests/c/fuzz_parser.c -o tests/c/build/fuzz_parser
+      - name: Run libFuzzer for 60 s
+        env:
+          ASAN_OPTIONS: "abort_on_error=1:halt_on_error=1"
+          UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1"
+        run: |
+          mkdir -p tests/c/build/fuzz_corpus
+          python3 tests/fuzz/make_seed_corpus.py tests/c/build/fuzz_corpus
+          tests/c/build/fuzz_parser -max_total_time=60 -max_len=4096 \
+                                    tests/c/build/fuzz_corpus
+      - name: Upload corpus and any crashes
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: fuzz-c-corpus
+          path: |
+            tests/c/build/fuzz_corpus
+            crash-*
+            leak-*
+            timeout-*
+          if-no-files-found: ignore
+          retention-days: 7
+
+  fuzz-py-parser:
+    name: atheris / Python AccumulatingReader
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+        with: { submodules: true }
+      - uses: actions/setup-python@v5
+        with: { python-version: "3.11" }
+      - name: Install atheris
+        # atheris ships a wheel for cpython 3.11 on linux/x86_64.
+        run: pip install atheris
+      - name: Run atheris for 60 s
+        run: |
+          python tests/fuzz/make_seed_corpus.py fuzz_seeds
+          python tests/py/fuzz_parser.py -max_total_time=60 -max_len=4096 fuzz_seeds
+
+  fuzz-cpp-parser:
+    name: libFuzzer / C++ AccumulatingReader
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+        with: { submodules: true }
+      - uses: actions/setup-python@v5
+        with: { python-version: "3.11" }
+      - name: Install clang
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y clang llvm
+      - name: Build C++ fuzz harness
+        run: |
+          mkdir -p tests/cpp/build
+          clang++ -std=c++20 -O1 -g -fsanitize=fuzzer,address,undefined                   -fno-sanitize-recover=all                   -I src/struct_frame/boilerplate/cpp                   tests/cpp/fuzz_parser.cpp -o tests/cpp/build/fuzz_parser_cpp
+      - name: Run libFuzzer for 60 s
+        env:
+          ASAN_OPTIONS: "abort_on_error=1:halt_on_error=1"
+          UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1"
+        run: |
+          mkdir -p tests/cpp/build/fuzz_corpus
+          python tests/fuzz/make_seed_corpus.py tests/cpp/build/fuzz_corpus
+          tests/cpp/build/fuzz_parser_cpp -max_total_time=60 -max_len=4096 tests/cpp/build/fuzz_corpus
+
+  fuzz-rust-parser:
+    name: cargo-fuzz / Rust AccumulatingReader
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+        with: { submodules: true }
+      - uses: actions/setup-python@v5
+        with: { python-version: "3.11" }
+      - uses: dtolnay/rust-toolchain@nightly
+      - name: Install cargo-fuzz
+        run: cargo install cargo-fuzz --locked
+      - name: Run cargo-fuzz for 60 s
+        working-directory: tests/rust/fuzz
+        env:
+          RUSTFLAGS: "-Coverflow-checks=on"
+        run: |
+          python ../../fuzz/make_seed_corpus.py corpus/parser
+          cargo +nightly fuzz run parser -- -max_total_time=60 -max_len=4096
+
+  # JavaScript/TypeScript and C# have no libFuzzer-style engine in this repo. Their parsers
+  # are fuzzed differentially instead: the stream differential test feeds identical
+  # clean, corrupted and heavily mutated streams to every language at several chunk sizes
+  # and requires identical output.
+  stream-differential:
+    name: Cross-language stream differential
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+        with: { submodules: true }
+      - uses: actions/setup-python@v5
+        with: { python-version: "3.11" }
+      - uses: actions/setup-node@v4
+        with: { node-version: "20" }
+      - uses: actions/setup-dotnet@v4
+        with: { dotnet-version: "8.0.x" }
+      - uses: dtolnay/rust-toolchain@stable
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y gcc g++
+          pip install pytest proto-schema-parser
+      - name: Run differential stream test
+        env:
+          SF_REQUIRE_LANGS: py,c,cpp,js,rust,csharp
+        run: python -m pytest tests/test_stream_cross_lang.py -v

struct_frame-0.10.10/.github/workflows/sanitizers.yml

@@ -0,0 +1,226 @@
+name: Sanitizers
+
+on:
+  push:
+    branches: [main, develop]
+  pull_request:
+    branches: [main, develop]
+
+concurrency:
+  group: sanitizers-${{ github.sha }}
+  cancel-in-progress: true
+
+# All jobs run the full test suite via `python test_all.py`.  C/C++ compile
+# commands in tests/run_tests.py honor $CC/$CXX/$CFLAGS/$CXXFLAGS/$LDFLAGS, so
+# the only thing each job has to do is export the right toolchain + flags.
+#
+# We deliberately skip language slices that have nothing to do with the
+# instrumented C/C++ binaries (Rust/.NET) using --skip-lang to keep wall time
+# manageable and avoid noise from unrelated toolchains.
+
+jobs:
+  asan-ubsan:
+    name: AddressSanitizer + UndefinedBehaviorSanitizer (C / C++)
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    env:
+      CC: clang
+      CXX: clang++
+      # -fno-sanitize-recover so any UB aborts the test process with a
+      # non-zero status and is reported by run_tests.py.
+      CFLAGS: "-O1 -g -fno-omit-frame-pointer -fsanitize=address,undefined -fno-sanitize-recover=all"
+      CXXFLAGS: "-O1 -g -fno-omit-frame-pointer -fsanitize=address,undefined -fno-sanitize-recover=all"
+      LDFLAGS: "-fsanitize=address,undefined"
+      ASAN_OPTIONS: "detect_leaks=1:abort_on_error=1:halt_on_error=1:strict_string_checks=1:detect_stack_use_after_return=1"
+      UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1"
+    steps:
+      - uses: actions/checkout@v4
+        with: { submodules: true }
+      - uses: actions/setup-python@v5
+        with: { python-version: "3.11" }
+      - uses: actions/setup-node@v4
+        with: { node-version: "20" }
+      - name: Install toolchain
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y clang llvm
+          pip install proto-schema-parser pytest hypothesis
+          (cd tests/ts && npm ci)
+      - name: Run C / C++ suite under ASan + UBSan
+        run: python test_all.py --skip-lang csharp --skip-lang rust
+
+  tsan:
+    name: ThreadSanitizer (C++ SDK)
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    env:
+      CC: clang
+      CXX: clang++
+      CFLAGS:   "-O1 -g -fno-omit-frame-pointer -fsanitize=thread"
+      CXXFLAGS: "-O1 -g -fno-omit-frame-pointer -fsanitize=thread"
+      LDFLAGS:  "-fsanitize=thread"
+      TSAN_OPTIONS: "halt_on_error=1:second_deadlock_stack=1"
+    steps:
+      - uses: actions/checkout@v4
+        with: { submodules: true }
+      - uses: actions/setup-python@v5
+        with: { python-version: "3.11" }
+      - uses: actions/setup-node@v4
+        with: { node-version: "20" }
+      - name: Install toolchain
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y clang llvm
+          pip install proto-schema-parser pytest hypothesis
+          (cd tests/ts && npm ci)
+      - name: Run C / C++ suite under TSan
+        # TSan is mutually exclusive with ASan; the SDK subscribe / streaming
+        # tests are where the threading races would surface.
+        run: python test_all.py --skip-lang csharp --skip-lang rust
+
+  valgrind-c:
+    name: Valgrind memcheck (C only)
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    env:
+      CC: gcc
+      # No sanitizer flags — Valgrind instruments at run time.  -O0 -g keeps
+      # the line numbers in error reports useful.
+      CFLAGS: "-O0 -g"
+    steps:
+      - uses: actions/checkout@v4
+        with: { submodules: true }
+      - uses: actions/setup-python@v5
+        with: { python-version: "3.11" }
+      - name: Install Valgrind
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y valgrind gcc build-essential
+          pip install proto-schema-parser
+      - name: Build C binaries (compile only, no run)
+        # Only the C binaries are executed under Valgrind, so build only C and let any
+        # failure fail the job (the previous "|| echo" swallowed every build error).
+        run: python test_all.py --only-compile --skip-lang cpp --skip-lang py --skip-lang ts --skip-lang js --skip-lang csharp --skip-lang rust --skip-lang gql --require-langs c
+      - name: Run C binaries under Valgrind
+        # Run every produced C test binary under memcheck.  --error-exitcode=1
+        # converts any leak / invalid-access into a non-zero CI status.
+        run: |
+          set -e
+          shopt -s nullglob
+          found=0
+          tmp_dir="$(mktemp -d)"
+          trap 'rm -rf "$tmp_dir"' EXIT
+          for exe in tests/c/build/test_*; do
+            [ -x "$exe" ] || continue
+            found=1
+            echo "=== valgrind $exe ==="
+            case "$(basename "$exe")" in
+              test_standard|test_standard.exe)
+                valgrind --error-exitcode=1 --leak-check=full --show-leak-kinds=all \
+                         --errors-for-leak-kinds=definite,possible \
+                         --track-origins=yes "$exe" encode standard "$tmp_dir/standard.bin"
+                valgrind --error-exitcode=1 --leak-check=full --show-leak-kinds=all \
+                         --errors-for-leak-kinds=definite,possible \
+                         --track-origins=yes "$exe" decode standard "$tmp_dir/standard.bin"
+                ;;
+              test_extended|test_extended.exe)
+                valgrind --error-exitcode=1 --leak-check=full --show-leak-kinds=all \
+                         --errors-for-leak-kinds=definite,possible \
+                         --track-origins=yes "$exe" encode bulk "$tmp_dir/extended.bin"
+                valgrind --error-exitcode=1 --leak-check=full --show-leak-kinds=all \
+                         --errors-for-leak-kinds=definite,possible \
+                         --track-origins=yes "$exe" decode bulk "$tmp_dir/extended.bin"
+                ;;
+              test_variable_flag|test_variable_flag.exe)
+                valgrind --error-exitcode=1 --leak-check=full --show-leak-kinds=all \
+                         --errors-for-leak-kinds=definite,possible \
+                         --track-origins=yes "$exe" encode bulk "$tmp_dir/variable.bin"
+                valgrind --error-exitcode=1 --leak-check=full --show-leak-kinds=all \
+                         --errors-for-leak-kinds=definite,possible \
+                         --track-origins=yes "$exe" decode bulk "$tmp_dir/variable.bin"
+                ;;
+              test_wire_evolution_file_io|test_wire_evolution_file_io.exe)
+                valgrind --error-exitcode=1 --leak-check=full --show-leak-kinds=all \
+                         --errors-for-leak-kinds=definite,possible \
+                         --track-origins=yes "$exe" encode v1 "$tmp_dir/wire_evo_v1.bin"
+                valgrind --error-exitcode=1 --leak-check=full --show-leak-kinds=all \
+                         --errors-for-leak-kinds=definite,possible \
+                         --track-origins=yes "$exe" decode v1 "$tmp_dir/wire_evo_v1.bin"
+                valgrind --error-exitcode=1 --leak-check=full --show-leak-kinds=all \
+                         --errors-for-leak-kinds=definite,possible \
+                         --track-origins=yes "$exe" encode v2 "$tmp_dir/wire_evo_v2.bin"
+                valgrind --error-exitcode=1 --leak-check=full --show-leak-kinds=all \
+                         --errors-for-leak-kinds=definite,possible \
+                         --track-origins=yes "$exe" decode v2 "$tmp_dir/wire_evo_v2.bin"
+                ;;
+              *)
+                valgrind --error-exitcode=1 --leak-check=full --show-leak-kinds=all \
+                         --errors-for-leak-kinds=definite,possible \
+                         --track-origins=yes "$exe"
+                ;;
+            esac
+          done
+          if [ "$found" -eq 0 ]; then
+            echo "No C test binaries were built; nothing to check." >&2
+            exit 1
+          fi
+
+  rust-miri:
+    name: Miri / Rust stream parser
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+        with: { submodules: true }
+      - uses: actions/setup-python@v5
+        with: { python-version: "3.11" }
+      - uses: dtolnay/rust-toolchain@nightly
+        with: { components: miri }
+      - name: Run the stream driver under Miri
+        env:
+          # The driver reads its seed stream from disk, which Miri blocks under its default isolation.
+          MIRIFLAGS: "-Zmiri-disable-isolation"
+        # Miri interprets the parser and reports undefined behaviour and out-of-bounds reads
+        # that a normal run would not notice. The inputs are short seed streams built from the
+        # shared message table, fed at a few chunk sizes, so it stays fast.
+        run: |
+          set -e
+          python tests/fuzz/make_seed_corpus.py "$RUNNER_TEMP/seeds"
+          cargo +nightly miri setup
+          for f in standard_clean_1 standard_noisy_2 bulk_corrupt_1 network_noisy_3 sensor_clean_1 ipc_noisy_2; do
+            profile="${f%%_*}"
+            for chunk in 1 13; do
+              cargo +nightly miri run --quiet --manifest-path tests/fuzz/drivers/rust/Cargo.toml \
+                -- "$profile" "$RUNNER_TEMP/seeds/$f.bin" "$chunk" > /dev/null
+            done
+          done
+
+  rust-asan:
+    name: AddressSanitizer / Rust stream parser
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+        with: { submodules: true }
+      - uses: actions/setup-python@v5
+        with: { python-version: "3.11" }
+      - uses: dtolnay/rust-toolchain@nightly
+      - name: Run the stream driver with AddressSanitizer and overflow checks
+        env:
+          RUSTFLAGS: "-Zsanitizer=address -Coverflow-checks=on"
+        run: |
+          set -e
+          python tests/fuzz/make_seed_corpus.py "$RUNNER_TEMP/seeds"
+          cargo +nightly build --manifest-path tests/fuzz/drivers/rust/Cargo.toml \
+            --target x86_64-unknown-linux-gnu --target-dir "$RUNNER_TEMP/target"
+          for f in "$RUNNER_TEMP"/seeds/*.bin; do
+            base="$(basename "$f" .bin)"
+            for chunk in 1 13 512; do
+              "$RUNNER_TEMP/target/x86_64-unknown-linux-gnu/debug/stream_driver" "${base%%_*}" "$f" "$chunk" > /dev/null
+            done
+          done