graphsense-lib 2.16.2__tar.gz → 2.16.3__tar.gz

graphsense_lib-2.16.3/.github/dependabot.yml

@@ -0,0 +1,132 @@
+# Dependabot configuration. See:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  # Root project. Version updates are grouped: without a group each bump is its
+  # own PR, the default open-pull-requests-limit of 5 fills up, and Dependabot
+  # then stops opening uv PRs entirely (that is how gitpython/aiohttp sat on
+  # 5 open version PRs while security alerts piled up). One weekly PR for
+  # minor+patch keeps the slot free; majors still come individually.
+  - package-ecosystem: "uv"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    groups:
+      uv-minor-patch:
+        applies-to: version-updates
+        patterns:
+          - "*"
+        update-types:
+          - "minor"
+          - "patch"
+
+  # The generated API client is a separate uv project with its own lockfile;
+  # without an entry here nothing ever bumps it and its alerts (urllib3,
+  # requests, pytest) only ever got a one-shot security PR.
+  - package-ecosystem: "uv"
+    directory: "/clients/python"
+    schedule:
+      interval: "weekly"
+    groups:
+      uv-minor-patch:
+        applies-to: version-updates
+        patterns:
+          - "*"
+        update-types:
+          - "minor"
+          - "patch"
+
+  # Nightly regression harness — own uv project, own lockfile.
+  - package-ecosystem: "uv"
+    directory: "/tests/regressions"
+    schedule:
+      interval: "weekly"
+    groups:
+      uv-minor-patch:
+        applies-to: version-updates
+        patterns:
+          - "*"
+        update-types:
+          - "minor"
+          - "patch"
+
+  # The pyo3 extension. pyo3 and arrow's `pyarrow` feature both link the native
+  # `python` library, so cargo only resolves when the two move together — a
+  # single-crate security bump of pyo3 always fails to resolve. Group them so
+  # Dependabot proposes the pair.
+  - package-ecosystem: "cargo"
+    directory: "/rust/gs_clustering"
+    schedule:
+      interval: "weekly"
+    groups:
+      pyo3-arrow:
+        patterns:
+          - "pyo3"
+          - "arrow"
+
+  # Base and tool images pinned in the Dockerfile (python, uv, temurin).
+  # They are pinned on purpose — a moving tag silently re-ships the big
+  # runtime layers to prod — so something has to bump them; this is it.
+  # Only FROM lines are parsed, which is why the uv/temurin pins live in
+  # named stages rather than inline COPY --from refs.
+  # One grouped PR per week keeps the noise down; CI builds the image on
+  # the PR, so a bad bump is visible before merge.
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    groups:
+      docker-images:
+        patterns:
+          - "*"
+    # Two of the three pinned images must not move on their own:
+    #   * python — the project is `requires-python = ">=3.10, <3.14"`, the
+    #     runtime stage prunes a hardcoded /usr/local/lib/python3.13 path, and
+    #     psycopg2-binary/pyspark have no wheels beyond 3.13. A 3.14 bump fails
+    #     the image build (PR #127); raise it deliberately together with
+    #     pyproject and the Dockerfile paths.
+    #   * eclipse-temurin — the `java11` stage exists *because* prod Spark
+    #     executors run Java 11 (shared hosts capped there by Cassandra 4.x);
+    #     a Java-17+ driver breaks Kryo task-result deserialization. Only the
+    #     digest may move, never the major.
+    ignore:
+      - dependency-name: "python"
+        update-types:
+          - "version-update:semver-major"
+          - "version-update:semver-minor"
+      - dependency-name: "eclipse-temurin"
+        update-types:
+          - "version-update:semver-major"
+          - "version-update:semver-minor"
+
+  # `directory: "/"` above covers the two root Dockerfiles but not
+  # subdirectories, so the Spark image had no coverage at all. Its base is
+  # `eclipse-temurin:11-jdk` — same Java 11 constraint as the runtime image's
+  # `java11` stage (prod Spark executors are capped at 11 by Cassandra 4.x,
+  # and a Java-17+ driver breaks Kryo task-result deserialization), so major
+  # and minor are ignored here too and only the tag's digest may move.
+  - package-ecosystem: "docker"
+    directory: "/spark"
+    schedule:
+      interval: "weekly"
+    groups:
+      docker-images:
+        patterns:
+          - "*"
+    ignore:
+      - dependency-name: "eclipse-temurin"
+        update-types:
+          - "version-update:semver-major"
+          - "version-update:semver-minor"
+
+  # Workflow actions are pinned to commit SHAs (supply-chain hardening);
+  # without this they would silently rot at whatever SHA they were pinned to.
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    groups:
+      github-actions:
+        patterns:
+          - "*"

{graphsense_lib-2.16.2 → graphsense_lib-2.16.3}/.github/workflows/run_tests_client.yaml

@@ -21,8 +21,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # Client pins requires-python = ">=3.9"; cover the full matrix.
-        python: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        # Client pins requires-python = ">=3.10"; cover the full matrix.
+        python: ["3.10", "3.11", "3.12", "3.13"]
 
     steps:
       - uses: actions/checkout@v7

graphsense_lib-2.16.3/.github/workflows/spark_dependency_graph.yml

@@ -0,0 +1,56 @@
+name: Submit Spark Dependency Graph
+
+# Dependabot has no sbt ecosystem — its JVM support is maven and gradle only —
+# so spark/build.sbt is invisible to the dependency graph and has never raised
+# a single alert. This action resolves the sbt build and pushes the result
+# through GitHub's Dependency Submission API, which is what the graph (and
+# therefore Dependabot alerts) reads.
+#
+# What this does and does NOT buy us:
+#   * alerts on the Scala/Spark dependency tree — spark-sql, spark-graphx,
+#     spark-cassandra-connector, scallop, scalatest — yes.
+#   * automated bump PRs — no. Dependabot still cannot update sbt, so
+#     remediation is a manual edit to spark/build.sbt. Treat an alert here as
+#     a ticket, not as something that will fix itself.
+#
+# Only the default branch is submitted: the dependency graph tracks one
+# snapshot per repo, and submitting from feature branches would make it
+# flap between whatever happens to be building.
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - 'spark/build.sbt'
+      - 'spark/project/**'
+      - '.github/workflows/spark_dependency_graph.yml'
+  workflow_dispatch:
+
+# Required by the Dependency Submission API.
+permissions:
+  contents: write
+
+jobs:
+  dependency-graph:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v7
+    # Java 11 to match spark_tests.yml and the runtime the jar is built for;
+    # resolution is version-sensitive, so a different JDK could snapshot a
+    # different tree than the one we ship.
+    - name: Setup JDK
+      uses: actions/setup-java@v5
+      with:
+        distribution: temurin
+        java-version: 11
+    - name: Setup sbt launcher
+      uses: sbt/setup-sbt@v1
+    - name: Submit dependency graph
+      uses: scalacenter/sbt-dependency-submission@d84eef4c09e633bcf5f113bcad7fd5e9af1baee9 # v3.2.3
+      with:
+        working-directory: spark
+        # A resolution hiccup should not fail the build on master — submit what
+        # resolved and surface the rest as a warning.
+        on-resolve-failure: warning

{graphsense_lib-2.16.2 → graphsense_lib-2.16.3}/.github/workflows/spark_docker_publish.yml

@@ -54,7 +54,7 @@ jobs:
           echo "Publishing ${IMAGE_NAME}:${version}"
 
       - name: Install cosign
-        uses: sigstore/cosign-installer@v3.4.0
+        uses: sigstore/cosign-installer@v4.1.2
         with:
           cosign-release: 'v2.2.2'
 
@@ -62,7 +62,7 @@ jobs:
         uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
 
       - name: Log into registry ${{ env.REGISTRY }}
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
+        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -70,7 +70,7 @@ jobs:
 
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0
+        uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           # Both tags are explicit. metadata-action's default `latest=auto`
@@ -85,7 +85,7 @@ jobs:
       # build.sbt and friends by paths relative to that tree.
       - name: Build and push Docker image
         id: build-and-push
-        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
+        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
         with:
           context: ./spark
           push: true

{graphsense_lib-2.16.2 → graphsense_lib-2.16.3}/CHANGELOG.md

@@ -10,6 +10,32 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 Use one changelog file, but separate entries by track in each release window.
 
+## [2.16.3] - 2026-08-28
+
+### Library
+
+#### Security
+- **Dependency bumps closing eight open Dependabot alerts.** `gitpython` 3.1.50 → 3.1.59 (six advisories, including arbitrary git-repo creation outside the working tree via an unvalidated `.gitmodules` submodule name), `aiohttp` 3.14.2 → 3.14.3, `cryptography` 48.0.1 → 50.0.0, and `pyo3` 0.23 → 0.29 in the Rust clustering extension (GHSA-36hh-v3qg-5jq4 out-of-bounds read in `nth`/`nth_back` for `PyList`/`PyTuple` iterators, plus two lesser ones). The `pyo3` bump could not be done on its own: `arrow`'s `pyarrow` feature also links the native `python` library, and cargo refuses a graph with two crates claiming the same `links` value, so `arrow` had to move 54.3 → 59 in the same step. That in turn needed three call-site changes — `Python::allow_threads` is now `Python::detach`, the `PyObject` alias is gone, and `ToPyArrow::to_pyarrow` returns a `Bound<'py, PyAny>` rather than an owned `Py`. The remaining open alert, `ecdsa` GHSA-wj6h-64fc-37mp (Minerva timing attack on P-256), has no patched version in any release and cannot be bumped away; `ecdsa` is only the `fast=False` fallback behind coincurve in pubkey validation.
+
+#### Added
+- **The Scala/Spark dependency tree is now visible to the dependency graph.** Dependabot has no sbt ecosystem — its JVM support is `maven` and `gradle` only — so `spark/build.sbt` had never produced a dependency-graph entry or a single alert in the repo's history, leaving `spark-sql`/`spark-graphx` 3.5.8, `spark-cassandra-connector` 3.5.1, `scallop`, `scalatest` and the sbt plugins entirely unmonitored. A new `spark_dependency_graph.yml` workflow runs `scalacenter/sbt-dependency-submission` on pushes to `master` that touch the sbt build, submitting the resolved graph through GitHub's Dependency Submission API so alerts fire normally. This buys **alerts only** — Dependabot still cannot update sbt, so remediation stays a manual `spark/build.sbt` edit.
+
+#### Changed
+- **Freshly transformed account keyspaces now store real 64-bit tx counts.** The v2.16.2 schema widening (`no_incoming_txs`/`no_outgoing_txs` and their `_zero_value` variants to `bigint`) was inert: the Spark aggregations still saturated at `Int.MaxValue`. The four tx-count aggregations now cast to Long (`TransformHelpers.saturateToLong`), so the next fresh transform stores true counts (TRON USDT's ~3.7e9 incoming txs instead of a capped 2147483647); degrees and relation counts stay 32-bit-saturated, bounded by the address universe rather than by time. The Python delta updater now resolves its `min(value, 2^31-1)` cap from the live schema instead of applying it unconditionally: `prepare_entities_for_ingest` takes a `count_cap`, and `resolve_tx_count_cap` reads `address.no_incoming_txs` out of `system_schema.columns` and passes `None` when it is `bigint`. On a widened keyspace the cap was not merely lossy but destructive — the merge reads the stored total, adds the delta and writes the result back, so a capped write returns a *smaller* number than it read, and TRON USDT's 3.7e9 would collapse to 2147483647 on the first batch that touched it. Keyspaces created before the widening keep their 32-bit columns forever (Cassandra cannot `ALTER int -> bigint`) and still cap, because an over-2^31 write there fails the whole ingest batch; the probe falls back to the cap whenever the column type cannot be established, which is the safe direction. The UTXO path is unaffected: it uses the separate `generic.prepare_entities_for_ingest`, which never capped.
+- **Dependency bumps taken from the open Dependabot PRs**: `protobuf` 6.33.6 → 7.36.0 with `grpcio`/`grpcio-tools` 1.81.1 → 1.83.0, plus `pydantic-settings` 2.15.0, `filelock` 3.32.4 and `orjson` 3.12.0. Applied on `develop` rather than merged, since every one of those PRs was based on `master`. `protobuf` 7 is a major bump and the TRON gRPC `_pb2` modules are imported function-locally, so a green suite proves nothing about them — they were imported explicitly and round-tripped against the new runtime instead. The checked-in generated code is left as it is: it loads and serializes correctly under protobuf 7, and `make run-codegen` (the pre-commit codegen hook) only regenerates the OpenAPI client, not the TRON stubs, which come from the separate `generate-tron-grpc-code` target.
+- **The Spark image publish workflow's actions are no longer years behind the main image build.** `spark_docker_publish.yml` was still on `docker/login-action` v3.0.0, `docker/metadata-action` v5.0.0 and `docker/build-push-action` v5.0.0 while `docker-build.yml` had already moved to v4.6.0 / v7.3.0; `sigstore/cosign-installer` goes 3.4.0 → 4.1.2. Same pins the grouped Dependabot PR proposed, SHA-verified against their tags.
+
+#### Fixed
+- **Exchange-rate fetches can no longer hang an ingest forever.** Every HTTP call in `rates/` was issued without a timeout — the four provider requests (`coingecko`, `coinmarketcap`, `cryptocompare`, `coindesk`, plus the token-rates contract lookup) and the ECB FX download, which `pd.read_csv(FX_URL)` performed through `urllib.request.urlopen` with the socket default of `None`. With no read timeout and no `SO_KEEPALIVE`, a connection that is established and then silently dropped (a NAT/firewall/LB reaping an idle flow) parks in `recv()` with no OS-level timer that will ever wake it. This was worse than a stalled run: `ingest` performs the exchange-rates step *before* it acquires the ingest lock, so a wedged process held nothing, blocked nothing, and was invisible to the lock's own stale-holder alerting — the next cron tick simply started another ingest that ran fine while the corpses accumulated, unnoticed until someone looked at `ps`. All six call sites now pass a shared `HTTP_TIMEOUT` of `(5, 60)` `(connect, read)`, and the ECB zip is fetched with `requests` into a buffer (`rates.utils.read_csv_url`) rather than by pandas. The retrying adapter the providers already mounted is now shared as `rates.utils.rates_session()`; because it retries read timeouts on GET, a transient stall is still retried and only a persistent one fails, in bounded time. Note the read timeout bounds *silence between reads*, not total request duration — which is exactly the failure being fixed, not a new cap on legitimately slow responses.
+- **Two remaining reader paths no longer misread wrapped 32-bit tx counts.** `get_addresses_light` (the delta updater's change-detection read) fed raw counts into its "did anything change" comparison, and the top-untagged job summed raw `no_incoming_txs + no_outgoing_txs` for its ranking and `min_txs` filter — a wrapped negative (TRON USDT: −591M for ~3.7e9 real txs) made the busiest address look nearly idle in both. Both now saturate first (`saturate_int32_count` on the read path, a `when(col < 0, Int.MaxValue)` column expression Spark-side), matching the REST read path's existing normalization; true bigint values from rebuilt keyspaces pass through unchanged.
+- **Dependabot stopped proposing dependency updates it had already been asked for.** Version updates for the `uv` ecosystem hit the default `open-pull-requests-limit` of 5 and stalled, so nothing was bumped while advisories accumulated; and three manifests were not configured at all — `clients/python` and `tests/regressions` (each its own uv project with its own lockfile) and `rust/gs_clustering` (cargo), which is why the `pyo3` alerts never produced a pull request. `.github/dependabot.yml` now covers all three, groups uv minor+patch bumps into one weekly PR per project so the PR slots stay free, and groups `pyo3` with `arrow` for cargo, since the two only resolve when they move together. `spark/` gained coverage too: `directory: "/"` picks up the two root Dockerfiles but not subdirectories, so `spark/Dockerfile` (`eclipse-temurin:11-jdk`) had none — it now has its own `docker` entry, carrying the same Java-11 major/minor `ignore` as the runtime image's `java11` stage. Note that closing a Dependabot PR unmerged tells Dependabot not to re-propose that release — a manual bump of the same dependency is the way to clear the alert.
+- **A Redis ingest lock left behind by a killed process no longer blocks ingest silently.** Two independent gaps let an eth ingest fall 32h behind without a single notification. (1) The Redis lock was released only in a `finally`, so a holder that died hard (OOM kill during auto-compaction, SIGKILL, reboot) never released it, and the key — an opaque uuid — survived until someone deleted it by hand, naming neither who held it nor since when. The lock value now identifies its holder as `{host, pid, acquired_at, nonce}`, readable with `redis-cli GET <key>` (host and pid are for the human reading the alert — in a container they are a discarded container id and pid 1, and nothing decides anything from them; the nonce keeps the value unique, since `release()` is a compare-and-delete against it), and the holder stamps a heartbeat in a sibling `<key>:heartbeat` while it works. A contender that finds a holder which has gone quiet for more than 15 minutes sends a Slack notification to the `exceptions` topic naming the host, the pid, how long it has been silent and the `redis-cli DEL` needed to break the lock — rate-limited to one report per lock per hour via a `SET NX EX` marker. Because the heartbeat measures *silence* rather than runtime, a legitimately hour-long job keeps stamping and is never reported. Nothing ever takes the lock away from its owner: there is deliberately **no TTL** (a lease must be renewed, and any renewal stall — Redis unreachable, the process stopped or thrashing, a long GIL-holding call — expires the lock while the holder is still writing, putting two ingest processes on one delta table), and no automatic reclaim (proving a pid is dead only works on the same host and in the same pid namespace, and gets it *wrong* elsewhere). A stalled heartbeat can therefore only cause a false alarm, never a released lock. (2) `ingest from-node` exited with code 911 on lock contention *before* reaching its post-ingest staleness check — and 911 is deliberately whitelisted in the Slack exception context, since contention normally just means the previous run is still going. A stuck lock therefore produced one log line per cron run and nothing else. The staleness check now runs on the contention path too (exit code 911 unchanged), so a lock that stays stuck past the configured `raw_ingest_staleness_threshold` raises the same alert as any other stalled ingest.
+
+### Web API + Python client
+
+#### Changed
+- **The `graphsense-python` client now requires Python 3.10+** (was 3.9+). Python 3.9 has been end-of-life since October 2025, and supporting it forked the client's lockfile: uv had to keep a second, older resolution pinned under `python_full_version < '3.10'`, which is where the open `urllib3` (GHSA-qccp-gfcp-xxvc, GHSA-mf9v-mfxr-j63j), `requests` (GHSA-gc5v-m9x4-r6x2) and `pytest` (GHSA-6w46-j5rx-g56g) advisories lived — unfixable without dropping 3.9, since the patched releases all require 3.10. With the floor raised the fork collapses to a single resolution on `urllib3` 2.7.0, `requests` 2.34.2 and `pytest` 9.1.1. The library itself has required `>=3.10` all along. Needs a `WEBAPISEM` bump before release.
+
 ## [2.16.2] - 2026-08-24
 
 ### Library

{graphsense_lib-2.16.2 → graphsense_lib-2.16.3}/Makefile

@@ -1,12 +1,12 @@
 SHELL := /bin/bash
 PROJECT := graphsense-lib
 VENV := venv
-RELEASESEM := 'v2.16.2'
+RELEASESEM := 'v2.16.3'
 WEBAPISEM := 'v2.15.3'
 # Spark pipeline release track (spark/, tag shape spark-vX.Y.Z). Kept apart
 # from the library track so a jar is only rebuilt and republished when the
 # Scala side actually changes — see VERSIONING.md.
-SPARKSEM := 'v26.08.1'
+SPARKSEM := 'v26.08.2'
 
 
 -include .env

{graphsense_lib-2.16.2 → graphsense_lib-2.16.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: graphsense-lib
-Version: 2.16.2
+Version: 2.16.3
 Summary: Graphsense backend lib and automation cli
 Author-email: Iknaio Cryptoasset Analytics GmbH <contact@iknaio.com>
 Project-URL: Homepage, https://graphsense.org/

{graphsense_lib-2.16.2 → graphsense_lib-2.16.3}/clients/python/README.md

@@ -49,7 +49,7 @@ For more information, please visit [https://www.iknaio.com/](https://www.iknaio.
 
 ## Requirements.
 
-Python 3.9+
+Python 3.10+
 
 ## Installation & Usage
 

{graphsense_lib-2.16.2 → graphsense_lib-2.16.3}/clients/python/pyproject.toml

@@ -8,7 +8,7 @@ authors = [
 ]
 readme = "README.md"
 keywords = ["OpenAPI", "OpenAPI-Generator", "GraphSense API"]
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 
 dependencies = [
   "urllib3 (>=2.1.0,<3.0.0)",

{graphsense_lib-2.16.2 → graphsense_lib-2.16.3}/clients/python/templates/README.mustache

@@ -18,7 +18,7 @@ For more information, please visit [{{{infoUrl}}}]({{{infoUrl}}})
 
 ## Requirements.
 
-Python {{{generatorLanguageVersion}}}
+Python 3.10+
 
 ## Installation & Usage