graphsense-lib 2.16.2__tar.gz → 2.16.4__tar.gz

{graphsense_lib-2.16.2 → graphsense_lib-2.16.4}/.github/copilot-instructions.md

@@ -12,9 +12,16 @@
 - Format: `make format`
 - Fast pre-commit gate: `make pre-commit`
 - Test suite:
-  - Local default: `make test`
-  - CI-like (no slow tests): `make test-ci`
+  - What the pre-commit hook runs: `make test-fast` (~33s). A **subset**: 1307 of
+    2327 tests, skipping the `slow` marks and the four testcontainer directories
+    (`tests/web`, `tests/db`, `tests/tagstore`, `tests/integration`). Green here
+    does not mean green in CI.
+  - Full suite, `slow` included, and what CI runs: `make test-ci` (~105s).
+  - Local default (full suite + coverage + forced dep sync): `make test` (~145s).
   - Base-dependency matrix variant: `make test-with-base-dependencies-ci`
+  - Rust extension: `make test-rust` (pre-commit, gated on `rust/**`; and CI).
+  - Scala pipeline: `make test-spark`. No hook runs it; CI runs it on every push
+    and PR.
 - Run a single test:
   - `uv run --exact --all-extras pytest tests/path/to/test_file.py::test_name -x -vv`
 

graphsense_lib-2.16.4/.github/dependabot.yml

@@ -0,0 +1,138 @@
+# Dependabot configuration. See:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  # Root project. Version updates are grouped: without a group each bump is its
+  # own PR, the default open-pull-requests-limit of 5 fills up, and Dependabot
+  # then stops opening uv PRs entirely (that is how gitpython/aiohttp sat on
+  # 5 open version PRs while security alerts piled up). One weekly PR for
+  # minor+patch keeps the slot free; majors still come individually.
+  - package-ecosystem: "uv"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    groups:
+      uv-minor-patch:
+        applies-to: version-updates
+        patterns:
+          - "*"
+        update-types:
+          - "minor"
+          - "patch"
+
+  # The generated API client is a separate uv project with its own lockfile;
+  # without an entry here nothing ever bumps it and its alerts (urllib3,
+  # requests, pytest) only ever got a one-shot security PR.
+  - package-ecosystem: "uv"
+    directory: "/clients/python"
+    schedule:
+      interval: "weekly"
+    groups:
+      uv-minor-patch:
+        applies-to: version-updates
+        patterns:
+          - "*"
+        update-types:
+          - "minor"
+          - "patch"
+
+  # Nightly regression harness — own uv project, own lockfile.
+  - package-ecosystem: "uv"
+    directory: "/tests/regressions"
+    schedule:
+      interval: "weekly"
+    groups:
+      uv-minor-patch:
+        applies-to: version-updates
+        patterns:
+          - "*"
+        update-types:
+          - "minor"
+          - "patch"
+
+  # The pyo3 extension. pyo3 and arrow's `pyarrow` feature both link the native
+  # `python` library, so cargo only resolves when the two move together — a
+  # single-crate security bump of pyo3 always fails to resolve. Group them so
+  # Dependabot proposes the pair.
+  - package-ecosystem: "cargo"
+    directory: "/rust/gs_clustering"
+    schedule:
+      interval: "weekly"
+    groups:
+      pyo3-arrow:
+        patterns:
+          - "pyo3"
+          - "arrow"
+
+  # Base and tool images pinned in the Dockerfile (python, uv, temurin).
+  # They are pinned on purpose — a moving tag silently re-ships the big
+  # runtime layers to prod — so something has to bump them; this is it.
+  # Only FROM lines are parsed, which is why the uv/temurin pins live in
+  # named stages rather than inline COPY --from refs.
+  # One grouped PR per week keeps the noise down; CI builds the image on
+  # the PR, so a bad bump is visible before merge.
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    groups:
+      docker-images:
+        patterns:
+          - "*"
+    # Two of the three pinned images must not move on their own:
+    #   * python — the project is `requires-python = ">=3.10, <3.14"`, the
+    #     runtime stage prunes a hardcoded /usr/local/lib/python3.13 path, and
+    #     psycopg2-binary/pyspark have no wheels beyond 3.13. A 3.14 bump fails
+    #     the image build (PR #127); raise it deliberately together with
+    #     pyproject and the Dockerfile paths.
+    #   * pyspark — pinned `>=3.5.0,<4.0.0` in pyproject.toml because the
+    #     cluster runs Spark 3.5.8; a 4.x PR cannot merge against that
+    #     constraint and cannot be taken without upgrading the cluster first.
+    #   * eclipse-temurin — the `java11` stage exists *because* prod Spark
+    #     executors run Java 11 (shared hosts capped there by Cassandra 4.x);
+    #     a Java-17+ driver breaks Kryo task-result deserialization. Only the
+    #     digest may move, never the major.
+    ignore:
+      - dependency-name: "pyspark"
+        update-types:
+          - "version-update:semver-major"
+      - dependency-name: "python"
+        update-types:
+          - "version-update:semver-major"
+          - "version-update:semver-minor"
+      - dependency-name: "eclipse-temurin"
+        update-types:
+          - "version-update:semver-major"
+          - "version-update:semver-minor"
+
+  # `directory: "/"` above covers the two root Dockerfiles but not
+  # subdirectories, so the Spark image had no coverage at all. Its base is
+  # `eclipse-temurin:11-jdk` — same Java 11 constraint as the runtime image's
+  # `java11` stage (prod Spark executors are capped at 11 by Cassandra 4.x,
+  # and a Java-17+ driver breaks Kryo task-result deserialization), so major
+  # and minor are ignored here too and only the tag's digest may move.
+  - package-ecosystem: "docker"
+    directory: "/spark"
+    schedule:
+      interval: "weekly"
+    groups:
+      docker-images:
+        patterns:
+          - "*"
+    ignore:
+      - dependency-name: "eclipse-temurin"
+        update-types:
+          - "version-update:semver-major"
+          - "version-update:semver-minor"
+
+  # Workflow actions are pinned to commit SHAs (supply-chain hardening);
+  # without this they would silently rot at whatever SHA they were pinned to.
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    groups:
+      github-actions:
+        patterns:
+          - "*"

{graphsense_lib-2.16.2 → graphsense_lib-2.16.4}/.github/workflows/docker-build.yml

@@ -31,7 +31,7 @@ jobs:
       # inline caches.
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
+        uses: docker/setup-buildx-action@594f3bf4285d9ea8dc53c9a0c9c4092420091003 # v4.4.0
 
       - name: Log into ghcr.io for cache pulls
         uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
@@ -74,7 +74,7 @@ jobs:
       # A missing :buildcache tag (first run, or after cache tag cleanup) is a
       # warning, not an error — the build just runs uncached.
       - name: Docker Image smoke test
-        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
+        uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
         with:
           context: .
           push: false

{graphsense_lib-2.16.2 → graphsense_lib-2.16.4}/.github/workflows/github-packages-publish.yaml

@@ -83,7 +83,7 @@ jobs:
       # https://github.com/docker/setup-buildx-action
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
+        uses: docker/setup-buildx-action@594f3bf4285d9ea8dc53c9a0c9c4092420091003 # v4.4.0
 
       # Login against a Docker registry except on PR
       # https://github.com/docker/login-action
@@ -167,7 +167,7 @@ jobs:
       # https://github.com/docker/build-push-action
       - name: Build and push Docker image
         id: build-and-push
-        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
+        uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
         with:
           context: .
           push: ${{ github.event_name != 'pull_request' }}

{graphsense_lib-2.16.2 → graphsense_lib-2.16.4}/.github/workflows/run_tests_client.yaml

@@ -21,8 +21,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # Client pins requires-python = ">=3.9"; cover the full matrix.
-        python: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        # Client pins requires-python = ">=3.10"; cover the full matrix.
+        python: ["3.10", "3.11", "3.12", "3.13"]
 
     steps:
       - uses: actions/checkout@v7

graphsense_lib-2.16.4/.github/workflows/spark_dependency_graph.yml

@@ -0,0 +1,56 @@
+name: Submit Spark Dependency Graph
+
+# Dependabot has no sbt ecosystem — its JVM support is maven and gradle only —
+# so spark/build.sbt is invisible to the dependency graph and has never raised
+# a single alert. This action resolves the sbt build and pushes the result
+# through GitHub's Dependency Submission API, which is what the graph (and
+# therefore Dependabot alerts) reads.
+#
+# What this does and does NOT buy us:
+#   * alerts on the Scala/Spark dependency tree — spark-sql, spark-graphx,
+#     spark-cassandra-connector, scallop, scalatest — yes.
+#   * automated bump PRs — no. Dependabot still cannot update sbt, so
+#     remediation is a manual edit to spark/build.sbt. Treat an alert here as
+#     a ticket, not as something that will fix itself.
+#
+# Only the default branch is submitted: the dependency graph tracks one
+# snapshot per repo, and submitting from feature branches would make it
+# flap between whatever happens to be building.
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - 'spark/build.sbt'
+      - 'spark/project/**'
+      - '.github/workflows/spark_dependency_graph.yml'
+  workflow_dispatch:
+
+# Required by the Dependency Submission API.
+permissions:
+  contents: write
+
+jobs:
+  dependency-graph:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v7
+    # Java 11 to match spark_tests.yml and the runtime the jar is built for;
+    # resolution is version-sensitive, so a different JDK could snapshot a
+    # different tree than the one we ship.
+    - name: Setup JDK
+      uses: actions/setup-java@v6
+      with:
+        distribution: temurin
+        java-version: 11
+    - name: Setup sbt launcher
+      uses: sbt/setup-sbt@v1
+    - name: Submit dependency graph
+      uses: scalacenter/sbt-dependency-submission@d84eef4c09e633bcf5f113bcad7fd5e9af1baee9 # v3.2.3
+      with:
+        working-directory: spark
+        # A resolution hiccup should not fail the build on master — submit what
+        # resolved and surface the rest as a warning.
+        on-resolve-failure: warning

{graphsense_lib-2.16.2 → graphsense_lib-2.16.4}/.github/workflows/spark_docker_publish.yml

@@ -54,15 +54,15 @@ jobs:
           echo "Publishing ${IMAGE_NAME}:${version}"
 
       - name: Install cosign
-        uses: sigstore/cosign-installer@v3.4.0
+        uses: sigstore/cosign-installer@v4.1.2
         with:
           cosign-release: 'v2.2.2'
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
+        uses: docker/setup-buildx-action@594f3bf4285d9ea8dc53c9a0c9c4092420091003 # v4.4.0
 
       - name: Log into registry ${{ env.REGISTRY }}
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
+        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -70,7 +70,7 @@ jobs:
 
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0
+        uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           # Both tags are explicit. metadata-action's default `latest=auto`
@@ -85,7 +85,7 @@ jobs:
       # build.sbt and friends by paths relative to that tree.
       - name: Build and push Docker image
         id: build-and-push
-        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
+        uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
         with:
           context: ./spark
           push: true

{graphsense_lib-2.16.2 → graphsense_lib-2.16.4}/.github/workflows/spark_release.yml

@@ -24,11 +24,21 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v7
+      # Java 11, matching spark_tests.yml and the cluster — Cassandra 4.1 is
+      # co-located with the Spark workers and supports 8/11 only, so the
+      # executors and driver both run 11. This built on 17 for a long time and
+      # produced a loadable jar (Scala 2.12 emits Java 8 bytecode by default and
+      # no -release is set), but the source was still compiled against the JDK 17
+      # class library: a call to any Java 12+ stdlib method would compile clean
+      # here and throw NoSuchMethodError on the executors. This job runs no
+      # tests, and spark_tests.yml triggers on branches and PRs rather than
+      # tags, so the release artifact was the one build never exercised on the
+      # runtime JVM.
       - name: Setup JDK
-        uses: actions/setup-java@v5
+        uses: actions/setup-java@v6
         with:
           distribution: temurin
-          java-version: 17
+          java-version: 11
       - name: Setup sbt launcher
         uses: sbt/setup-sbt@v1
       # build.sbt derives the jar version from GITHUB_REF_NAME by switching on

graphsense_lib-2.16.4/.github/workflows/spark_tests.yml

@@ -0,0 +1,48 @@
+name: Run Spark Unit-Tests
+
+# The Scala pipeline lives in spark/. It runs unconditionally rather than on a
+# path filter: the delta updater and spark/ must produce the same derived tables
+# (see CLAUDE.md), and a path list is exactly the thing that goes stale when a
+# new file starts feeding that contract -- the drift this job exists to catch
+# would then slip through unnoticed. sbt is not run locally at all, so this is
+# the only gate on the Scala side.
+on:
+  push:
+    branches:
+      - master
+      - develop
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  # Not "build": three other workflows already expose a check by that name, so
+  # the Spark run is indistinguishable in a PR's check list.
+  spark-test:
+    runs-on: ubuntu-latest
+    # 11 is what production runs (Cassandra 4.1 is co-located with the Spark
+    # workers and supports 8/11 only) and what spark_release.yml builds on. 17
+    # is the highest JVM Spark 3.5 supports, and is tested so that "runs on a
+    # newer JVM" is an observed fact rather than an assumption — the jar itself
+    # is Java 8 bytecode and loads anywhere, but module-system encapsulation
+    # and reflection behaviour differ, which is what this catches. Java 21
+    # would need Spark 4.0.
+    strategy:
+      fail-fast: false
+      matrix:
+        java: [11, 17]
+    defaults:
+      run:
+        working-directory: spark
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v7
+    - name: Setup JDK
+      uses: actions/setup-java@v6
+      with:
+        distribution: temurin
+        java-version: ${{ matrix.java }}
+    - name: Setup sbt launcher
+      uses: sbt/setup-sbt@v1
+    - name: Test with SBT
+      run: sbt test

{graphsense_lib-2.16.2 → graphsense_lib-2.16.4}/.pre-commit-config.yaml

@@ -53,25 +53,35 @@ repos:
         language: system
         pass_filenames: false
         always_run: true
-        stages: [pre-commit, pre-push]
     -   id: update-api-version
         name: Sync API version from Makefile to version.py
         entry: make update-api-version
         language: system
         pass_filenames: false
         always_run: true
+    -   id: sync-spark-packages
+        name: Sync Scala job packages from spark/build.sbt to config.py
+        entry: make sync-spark-packages
+        language: system
+        pass_filenames: false
+        always_run: true
     -   id: type-check
         name: type-check
         entry: make type-check
         language: system
         pass_filenames: false
         always_run: true
+    # 7s of Docker + the Java openapi-generator. Its only inputs are
+    # graphsenselib.web.app (via scripts/generate_openapi_spec.py), the mustache
+    # templates and the compat patches, so it ran on every commit that could not
+    # possibly change the spec. CI regenerates unconditionally, so drift still
+    # surfaces there if this filter is ever too narrow.
     -   id: codegen
         name: Ensure Generated Code is up to date
         entry: make run-codegen
         language: system
         pass_filenames: false
-        always_run: true
+        files: ^(src/graphsenselib/web/|clients/python/templates/|clients/python/compat/)
     -   id: sync-gs-files
         name: Sync gs_files into the python client
         entry: make -C clients/python sync-gs-files
@@ -90,10 +100,20 @@ repos:
         language: system
         pass_filenames: false
         always_run: true
-        stages: [pre-commit, pre-push]
+    # cargo test is 3.3s cold / 0.06s warm -- cheap enough to gate on commit,
+    # and until now the Rust extension was only ever tested in CI.
+    -   id: rust-test
+        name: cargo test (gs_clustering)
+        entry: make test-rust
+        language: system
+        pass_filenames: false
+        files: ^rust/
+    # The fast gate: no coverage, no slow marks, no testcontainers (~33s vs
+    # ~226s). The container-backed and slow tests are covered by `make test-ci`
+    # in .github/workflows/run_tests.yaml, which runs the suite in full.
     -   id: pytest-check
-        name: pytest-check
-        entry: make test
+        name: 'pytest: fast subset only (no slow/container tests)'
+        entry: make test-fast
         language: system
         pass_filenames: false
         always_run: true