graphsense-python 2.13.2__py3-none-any.whl → 2.15.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. graphsense/__init__.py +44 -2
  2. graphsense/address_scan/__init__.py +11 -0
  3. graphsense/address_scan/decompress.py +175 -0
  4. graphsense/address_scan/detectors.py +124 -0
  5. graphsense/address_scan/scanner.py +182 -0
  6. graphsense/address_scan/validators.py +383 -0
  7. graphsense/api/__init__.py +1 -0
  8. graphsense/api/addresses_api.py +16 -16
  9. graphsense/api/blocks_api.py +1 -1
  10. graphsense/api/bulk_api.py +8 -8
  11. graphsense/api/clusters_api.py +19 -19
  12. graphsense/api/entities_api.py +19 -19
  13. graphsense/api/general_api.py +1 -1
  14. graphsense/api/graph_api.py +692 -0
  15. graphsense/api/rates_api.py +1 -1
  16. graphsense/api/tags_api.py +7 -7
  17. graphsense/api/tokens_api.py +1 -1
  18. graphsense/api/txs_api.py +4 -4
  19. graphsense/api_client.py +2 -2
  20. graphsense/cli/main.py +2 -0
  21. graphsense/cli/scan.py +118 -0
  22. graphsense/configuration.py +3 -3
  23. graphsense/exceptions.py +1 -1
  24. graphsense/gs_files/__init__.py +6 -0
  25. graphsense/gs_files/encoder.py +470 -48
  26. graphsense/models/__init__.py +21 -1
  27. graphsense/models/actor.py +1 -1
  28. graphsense/models/actor_context.py +1 -1
  29. graphsense/models/address.py +3 -3
  30. graphsense/models/address_output.py +1 -1
  31. graphsense/models/address_tag.py +1 -1
  32. graphsense/models/address_tags.py +1 -1
  33. graphsense/models/address_tx.py +1 -1
  34. graphsense/models/address_tx_utxo.py +1 -1
  35. graphsense/models/address_txs.py +1 -1
  36. graphsense/models/block.py +1 -1
  37. graphsense/models/block_at_date.py +1 -1
  38. graphsense/models/change_heuristics.py +1 -1
  39. graphsense/models/cluster.py +1 -1
  40. graphsense/models/cluster_addresses.py +1 -1
  41. graphsense/models/coin_join_consensus.py +1 -1
  42. graphsense/models/coin_join_heuristics.py +1 -1
  43. graphsense/models/comparison_signal.py +107 -0
  44. graphsense/models/comparison_verdict.py +109 -0
  45. graphsense/models/concept.py +1 -1
  46. graphsense/models/consensus_entry.py +1 -1
  47. graphsense/models/currency_stats.py +1 -1
  48. graphsense/models/direct_change_heuristic.py +1 -1
  49. graphsense/models/entity.py +1 -1
  50. graphsense/models/entity_addresses.py +1 -1
  51. graphsense/models/external_conversion.py +1 -1
  52. graphsense/models/graph_address_network_summary.py +132 -0
  53. graphsense/models/graph_address_overall.py +140 -0
  54. graphsense/models/graph_address_ref.py +88 -0
  55. graphsense/models/graph_address_summary.py +99 -0
  56. graphsense/models/graph_compare_note.py +94 -0
  57. graphsense/models/graph_compare_request.py +107 -0
  58. graphsense/models/graph_compare_signal.py +113 -0
  59. graphsense/models/graph_compare_verdict.py +115 -0
  60. graphsense/models/graph_compared_tx.py +100 -0
  61. graphsense/models/graph_comparison.py +120 -0
  62. graphsense/models/graph_lineage_edge.py +101 -0
  63. graphsense/models/graph_note.py +99 -0
  64. graphsense/models/graph_summary.py +96 -0
  65. graphsense/models/graph_summary_request.py +104 -0
  66. graphsense/models/graph_tx_characteristics.py +108 -0
  67. graphsense/models/graph_tx_network_summary.py +120 -0
  68. graphsense/models/graph_tx_overall.py +109 -0
  69. graphsense/models/graph_tx_ref.py +88 -0
  70. graphsense/models/graph_tx_summary.py +99 -0
  71. graphsense/models/http_validation_error.py +1 -1
  72. graphsense/models/join_market_heuristic.py +1 -1
  73. graphsense/models/label_summary.py +1 -1
  74. graphsense/models/labeled_item_ref.py +1 -1
  75. graphsense/models/lineage_edge.py +101 -0
  76. graphsense/models/link.py +1 -1
  77. graphsense/models/link_utxo.py +1 -1
  78. graphsense/models/links.py +1 -1
  79. graphsense/models/links_inner.py +1 -1
  80. graphsense/models/location_inner.py +1 -1
  81. graphsense/models/multi_input_change_heuristic.py +1 -1
  82. graphsense/models/neighbor_address.py +1 -1
  83. graphsense/models/neighbor_addresses.py +1 -1
  84. graphsense/models/neighbor_cluster.py +1 -1
  85. graphsense/models/neighbor_clusters.py +1 -1
  86. graphsense/models/neighbor_entities.py +1 -1
  87. graphsense/models/neighbor_entity.py +1 -1
  88. graphsense/models/one_time_change_heuristic.py +1 -1
  89. graphsense/models/per_tx_inner.py +193 -0
  90. graphsense/models/rate.py +1 -1
  91. graphsense/models/rates.py +1 -1
  92. graphsense/models/related_address.py +1 -1
  93. graphsense/models/related_addresses.py +1 -1
  94. graphsense/models/search_result.py +1 -1
  95. graphsense/models/search_result_by_currency.py +1 -1
  96. graphsense/models/search_result_level1.py +1 -1
  97. graphsense/models/search_result_level2.py +1 -1
  98. graphsense/models/search_result_level3.py +1 -1
  99. graphsense/models/search_result_level4.py +1 -1
  100. graphsense/models/search_result_level5.py +1 -1
  101. graphsense/models/search_result_level6.py +1 -1
  102. graphsense/models/stats.py +1 -1
  103. graphsense/models/tag.py +1 -1
  104. graphsense/models/tag_cloud_entry.py +1 -1
  105. graphsense/models/tag_summary.py +1 -1
  106. graphsense/models/taxonomy.py +1 -1
  107. graphsense/models/token_config.py +1 -1
  108. graphsense/models/token_configs.py +1 -1
  109. graphsense/models/transaction_comparison.py +120 -0
  110. graphsense/models/tx.py +1 -1
  111. graphsense/models/tx_account.py +1 -1
  112. graphsense/models/tx_characteristics.py +108 -0
  113. graphsense/models/tx_compared_item.py +100 -0
  114. graphsense/models/tx_ref.py +1 -1
  115. graphsense/models/tx_summary.py +1 -1
  116. graphsense/models/tx_utxo.py +1 -1
  117. graphsense/models/tx_value.py +1 -1
  118. graphsense/models/user_reported_tag.py +1 -1
  119. graphsense/models/user_tag_report_response.py +1 -1
  120. graphsense/models/utxo_heuristics.py +1 -1
  121. graphsense/models/validation_error.py +1 -1
  122. graphsense/models/values.py +1 -1
  123. graphsense/models/wasabi_heuristic.py +1 -1
  124. graphsense/models/whirlpool_coin_join_heuristic.py +1 -1
  125. graphsense/models/whirlpool_tx0_heuristic.py +1 -1
  126. graphsense/rest.py +1 -1
  127. {graphsense_python-2.13.2.dist-info → graphsense_python-2.15.0.dist-info}/METADATA +44 -4
  128. graphsense_python-2.15.0.dist-info/RECORD +209 -0
  129. {graphsense_python-2.13.2.dist-info → graphsense_python-2.15.0.dist-info}/WHEEL +1 -1
  130. graphsense_python-2.13.2.dist-info/RECORD +0 -176
  131. {graphsense_python-2.13.2.dist-info → graphsense_python-2.15.0.dist-info}/entry_points.txt +0 -0
  132. {graphsense_python-2.13.2.dist-info → graphsense_python-2.15.0.dist-info}/top_level.txt +0 -0
graphsense/__init__.py CHANGED
@@ -5,14 +5,14 @@
5
5
  """
6
6
  GraphSense API
7
7
 
8
- GraphSense API provides programmatic access to blockchain analytics data across multiple ledgers. Use it to explore addresses, clusters, blocks, transactions, tags, token activity, and exchange-rate context, and to integrate investigation workflows into your own applications and automation. ## Versioning and deprecation policy The API follows semantic versioning. Minor releases are additive and backwards-compatible; breaking changes only happen in major releases, which are rare and announced in advance. Deprecated endpoints and fields remain fully functional for at least six months after they are marked deprecated. During that window they are highlighted with a strikethrough in the docs and in generated clients, and responses from deprecated endpoints carry a `Deprecation` HTTP header that client tooling can detect. Replacement endpoints and fields are always introduced before the deprecated surface is removed. See the [full versioning and deprecation policy](https://github.com/graphsense/graphsense-lib/blob/master/README.md#rest-api-evolution-and-deprecation-policy) for details. # noqa: E501
8
+ GraphSense API provides programmatic access to blockchain analytics data across multiple ledgers. Use it to explore addresses, clusters, blocks, transactions, tags, token activity, and exchange-rate context, and to integrate investigation workflows into your own applications and automation. ## Versioning & deprecation The API follows semantic versioning. Minor releases are additive and backwards-compatible; breaking changes only happen in major releases, which are rare and announced in advance. Deprecated endpoints and fields remain fully functional for at least six months after they are marked deprecated. During that window they are highlighted with a strikethrough in the docs and in generated clients, and responses from deprecated endpoints carry a `Deprecation` HTTP header that client tooling can detect. Replacement endpoints and fields are always introduced before the deprecated surface is removed. See the [full versioning and deprecation policy](https://github.com/graphsense/graphsense-lib/blob/master/README.md#rest-api-evolution-and-deprecation-policy) for details. ## AI assistant access (MCP) In addition to this REST API, this deployment exposes a **Model Context Protocol (MCP)** interface that lets AI assistants query GraphSense directly in natural language, without writing code against the endpoints below. The MCP interface is available at the `/mcp` path of this server, relative to the address this documentation is served from. It offers a curated selection of the most useful operations, so an assistant can answer questions about addresses, transactions, clusters, and tags on your behalf. To use it, add that `/mcp` URL to an MCP-capable assistant as a new connector or tool source; the assistant then handles the rest. **Authentication:** the MCP endpoint is protected by OAuth. When adding the connector in your assistant, use the client ID `iknaio-mcp`. # noqa: E501
9
9
 
10
10
  Contact: contact@iknaio.com
11
11
  Generated by: https://openapi-generator.tech
12
12
  """
13
13
 
14
14
 
15
- __version__ = "2.13.2"
15
+ __version__ = "2.15.0"
16
16
 
17
17
  # Define package exports
18
18
  __all__ = [
@@ -22,6 +22,7 @@ __all__ = [
22
22
  "ClustersApi",
23
23
  "EntitiesApi",
24
24
  "GeneralApi",
25
+ "GraphApi",
25
26
  "RatesApi",
26
27
  "TagsApi",
27
28
  "TokensApi",
@@ -58,6 +59,25 @@ __all__ = [
58
59
  "Entity",
59
60
  "EntityAddresses",
60
61
  "ExternalConversion",
62
+ "GraphAddressNetworkSummary",
63
+ "GraphAddressOverall",
64
+ "GraphAddressRef",
65
+ "GraphAddressSummary",
66
+ "GraphCompareNote",
67
+ "GraphCompareRequest",
68
+ "GraphCompareSignal",
69
+ "GraphCompareVerdict",
70
+ "GraphComparedTx",
71
+ "GraphComparison",
72
+ "GraphLineageEdge",
73
+ "GraphNote",
74
+ "GraphSummary",
75
+ "GraphSummaryRequest",
76
+ "GraphTxCharacteristics",
77
+ "GraphTxNetworkSummary",
78
+ "GraphTxOverall",
79
+ "GraphTxRef",
80
+ "GraphTxSummary",
61
81
  "HTTPValidationError",
62
82
  "JoinMarketHeuristic",
63
83
  "LabelSummary",
@@ -75,6 +95,7 @@ __all__ = [
75
95
  "NeighborEntities",
76
96
  "NeighborEntity",
77
97
  "OneTimeChangeHeuristic",
98
+ "PerTxInner",
78
99
  "Rate",
79
100
  "Rates",
80
101
  "RelatedAddress",
@@ -120,6 +141,7 @@ from graphsense.api.bulk_api import BulkApi as BulkApi
120
141
  from graphsense.api.clusters_api import ClustersApi as ClustersApi
121
142
  from graphsense.api.entities_api import EntitiesApi as EntitiesApi
122
143
  from graphsense.api.general_api import GeneralApi as GeneralApi
144
+ from graphsense.api.graph_api import GraphApi as GraphApi
123
145
  from graphsense.api.rates_api import RatesApi as RatesApi
124
146
  from graphsense.api.tags_api import TagsApi as TagsApi
125
147
  from graphsense.api.tokens_api import TokensApi as TokensApi
@@ -160,6 +182,25 @@ from graphsense.models.direct_change_heuristic import DirectChangeHeuristic as D
160
182
  from graphsense.models.entity import Entity as Entity
161
183
  from graphsense.models.entity_addresses import EntityAddresses as EntityAddresses
162
184
  from graphsense.models.external_conversion import ExternalConversion as ExternalConversion
185
+ from graphsense.models.graph_address_network_summary import GraphAddressNetworkSummary as GraphAddressNetworkSummary
186
+ from graphsense.models.graph_address_overall import GraphAddressOverall as GraphAddressOverall
187
+ from graphsense.models.graph_address_ref import GraphAddressRef as GraphAddressRef
188
+ from graphsense.models.graph_address_summary import GraphAddressSummary as GraphAddressSummary
189
+ from graphsense.models.graph_compare_note import GraphCompareNote as GraphCompareNote
190
+ from graphsense.models.graph_compare_request import GraphCompareRequest as GraphCompareRequest
191
+ from graphsense.models.graph_compare_signal import GraphCompareSignal as GraphCompareSignal
192
+ from graphsense.models.graph_compare_verdict import GraphCompareVerdict as GraphCompareVerdict
193
+ from graphsense.models.graph_compared_tx import GraphComparedTx as GraphComparedTx
194
+ from graphsense.models.graph_comparison import GraphComparison as GraphComparison
195
+ from graphsense.models.graph_lineage_edge import GraphLineageEdge as GraphLineageEdge
196
+ from graphsense.models.graph_note import GraphNote as GraphNote
197
+ from graphsense.models.graph_summary import GraphSummary as GraphSummary
198
+ from graphsense.models.graph_summary_request import GraphSummaryRequest as GraphSummaryRequest
199
+ from graphsense.models.graph_tx_characteristics import GraphTxCharacteristics as GraphTxCharacteristics
200
+ from graphsense.models.graph_tx_network_summary import GraphTxNetworkSummary as GraphTxNetworkSummary
201
+ from graphsense.models.graph_tx_overall import GraphTxOverall as GraphTxOverall
202
+ from graphsense.models.graph_tx_ref import GraphTxRef as GraphTxRef
203
+ from graphsense.models.graph_tx_summary import GraphTxSummary as GraphTxSummary
163
204
  from graphsense.models.http_validation_error import HTTPValidationError as HTTPValidationError
164
205
  from graphsense.models.join_market_heuristic import JoinMarketHeuristic as JoinMarketHeuristic
165
206
  from graphsense.models.label_summary import LabelSummary as LabelSummary
@@ -177,6 +218,7 @@ from graphsense.models.neighbor_clusters import NeighborClusters as NeighborClus
177
218
  from graphsense.models.neighbor_entities import NeighborEntities as NeighborEntities
178
219
  from graphsense.models.neighbor_entity import NeighborEntity as NeighborEntity
179
220
  from graphsense.models.one_time_change_heuristic import OneTimeChangeHeuristic as OneTimeChangeHeuristic
221
+ from graphsense.models.per_tx_inner import PerTxInner as PerTxInner
180
222
  from graphsense.models.rate import Rate as Rate
181
223
  from graphsense.models.rates import Rates as Rates
182
224
  from graphsense.models.related_address import RelatedAddress as RelatedAddress
@@ -0,0 +1,11 @@
1
+ # AUTO-GENERATED — DO NOT EDIT.
2
+ # Synced from src/graphsenselib/convert/address_scan/__init__.py via
3
+ # clients/python/scripts/sync_address_scan.py. Edit the source and re-run
4
+ # `make -C clients/python sync-address-scan`.
5
+ """Scan text/SQL files (and compressed containers) for cryptocurrency addresses.
6
+
7
+ Extraction is deliberately permissive; the real filter is checksum validation
8
+ (reusing :mod:`graphsenselib.utils.address`), which removes the many
9
+ address-shaped false positives found in database dumps (hashes, session ids,
10
+ base64 blobs, filenames).
11
+ """
@@ -0,0 +1,175 @@
1
+ # AUTO-GENERATED — DO NOT EDIT.
2
+ # Synced from src/graphsenselib/convert/address_scan/decompress.py via
3
+ # clients/python/scripts/sync_address_scan.py. Edit the source and re-run
4
+ # `make -C clients/python sync-address-scan`.
5
+ """Transparently unwrap compressed containers before scanning.
6
+
7
+ Addresses often hide inside compressed data (.gz dumps, zip exports, or raw
8
+ zlib/gzip blobs embedded in a binary). We unwrap known containers, recurse
9
+ (so .tar.gz works), and optionally carve embedded streams. A byte budget guards
10
+ against decompression bombs.
11
+
12
+ Supported: gzip, zlib, bz2, xz, zip, tar (recursively), GraphSense ``.gs`` save
13
+ files (LZW + base64 + JSON, decoded via
14
+ :func:`graphsenselib.convert.gs_files.parser.lzw_unpack`), and -- with
15
+ ``carve=True`` -- zlib/gzip streams embedded anywhere in a binary.
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import base64
21
+ import bz2
22
+ import io
23
+ import lzma
24
+ import re
25
+ import struct
26
+ import tarfile
27
+ import zipfile
28
+ import zlib
29
+ from typing import Iterator
30
+
31
+ from graphsense.gs_files.parser import lzw_unpack
32
+
33
+ MAX_DEPTH = 6
34
+ DEFAULT_BUDGET = 1024 * 1024 * 1024 # 1 GiB of decompressed output per file
35
+ GS_AUTO_MAX = 64 * 1024 * 1024 # only auto-probe .gs on smallish files
36
+
37
+
38
+ def _decode_gs(data: bytes, name: str) -> "bytes | None":
39
+ """Return the inner JSON bytes of a GraphSense ``.gs`` file, or None.
40
+
41
+ Headerless format, so we validate structurally: size is a non-zero multiple
42
+ of 4, the first LZW code is a base-dictionary char (<256), and the payload
43
+ base64-decodes to something that starts like JSON. That combination makes
44
+ false positives on arbitrary binaries essentially nil.
45
+ """
46
+ if not data or len(data) % 4 != 0:
47
+ return None
48
+ forced = name.lower().endswith(".gs")
49
+ if not forced and len(data) > GS_AUTO_MAX:
50
+ return None # avoid an expensive LZW pass on large non-.gs blobs
51
+ codes = struct.unpack(f"<{len(data) // 4}I", data)
52
+ if codes[0] >= 256:
53
+ return None
54
+ try:
55
+ payload = base64.b64decode(lzw_unpack(list(codes)), validate=True)
56
+ except Exception:
57
+ return None
58
+ return payload if payload.lstrip()[:1] in (b"[", b"{") else None
59
+
60
+
61
+ def _bounded(decompressor: object, data: bytes, limit: int) -> bytes:
62
+ """Streaming-decompress ``data``, stopping once ``limit`` bytes are produced."""
63
+ out = bytearray()
64
+ step = 1 << 20
65
+ try:
66
+ for pos in range(0, len(data), step):
67
+ out += decompressor.decompress(data[pos : pos + step]) # type: ignore[attr-defined]
68
+ if len(out) >= limit:
69
+ return bytes(out[:limit])
70
+ flush = getattr(decompressor, "flush", None)
71
+ if callable(flush):
72
+ out += flush()
73
+ except Exception:
74
+ pass # truncated/garbage tail -> keep whatever decoded so far
75
+ return bytes(out[:limit])
76
+
77
+
78
+ def _is_zlib(data: bytes) -> bool:
79
+ return (
80
+ len(data) > 2 and (data[0] & 0x0F) == 8 and ((data[0] << 8) | data[1]) % 31 == 0
81
+ )
82
+
83
+
84
+ def _decompress_layer(
85
+ data: bytes, name: str, budget: list[int]
86
+ ) -> Iterator[tuple[str, bytes]]:
87
+ """Yield (label, bytes) for each container format ``data`` unwraps into."""
88
+ if budget[0] <= 0:
89
+ return
90
+ limit = budget[0]
91
+
92
+ if data[:2] == b"\x1f\x8b": # gzip
93
+ out = _bounded(zlib.decompressobj(zlib.MAX_WBITS | 16), data, limit)
94
+ if out:
95
+ budget[0] -= len(out)
96
+ yield f"{name} :: gzip", out
97
+ elif _is_zlib(data): # raw zlib
98
+ out = _bounded(zlib.decompressobj(), data, limit)
99
+ if out:
100
+ budget[0] -= len(out)
101
+ yield f"{name} :: zlib", out
102
+ elif data[:3] == b"BZh": # bzip2
103
+ out = _bounded(bz2.BZ2Decompressor(), data, limit)
104
+ if out:
105
+ budget[0] -= len(out)
106
+ yield f"{name} :: bz2", out
107
+ elif data[:6] == b"\xfd7zXZ\x00": # xz
108
+ out = _bounded(lzma.LZMADecompressor(), data, limit)
109
+ if out:
110
+ budget[0] -= len(out)
111
+ yield f"{name} :: xz", out
112
+
113
+ if data[:4] in (b"PK\x03\x04", b"PK\x05\x06"): # zip
114
+ try:
115
+ with zipfile.ZipFile(io.BytesIO(data)) as zf:
116
+ for info in zf.infolist():
117
+ if info.is_dir() or budget[0] <= 0:
118
+ continue
119
+ with zf.open(info) as fh:
120
+ out = fh.read(budget[0] + 1)[: budget[0]]
121
+ budget[0] -= len(out)
122
+ yield f"{name} :: zip[{info.filename}]", out
123
+ except Exception:
124
+ pass
125
+ elif tarfile.is_tarfile(io.BytesIO(data)): # tar (incl. plain, not gz here)
126
+ try:
127
+ with tarfile.open(fileobj=io.BytesIO(data)) as tf:
128
+ for member in tf:
129
+ if not member.isfile() or budget[0] <= 0:
130
+ continue
131
+ fh = tf.extractfile(member)
132
+ if fh is None:
133
+ continue
134
+ out = fh.read(budget[0] + 1)[: budget[0]]
135
+ budget[0] -= len(out)
136
+ yield f"{name} :: tar[{member.name}]", out
137
+ except Exception:
138
+ pass
139
+
140
+ gs = _decode_gs(data, name) # GraphSense .gs (LZW + base64 + JSON)
141
+ if gs is not None and budget[0] > 0:
142
+ gs = gs[: budget[0]]
143
+ budget[0] -= len(gs)
144
+ yield f"{name} :: gs/lzw", gs
145
+
146
+
147
+ def _carve_zlib(
148
+ data: bytes, name: str, budget: list[int]
149
+ ) -> Iterator[tuple[str, bytes]]:
150
+ """Best-effort: find gzip/zlib streams anywhere in ``data`` and inflate them."""
151
+ seen = 0
152
+ for m in re.finditer(rb"\x1f\x8b\x08|\x78[\x01\x9c\xda]", data):
153
+ if budget[0] <= 0 or seen >= 200:
154
+ break
155
+ off = m.start()
156
+ wbits = zlib.MAX_WBITS | 16 if data[off] == 0x1F else zlib.MAX_WBITS
157
+ out = _bounded(zlib.decompressobj(wbits), data[off:], budget[0])
158
+ if len(out) >= 16: # ignore trivially short / bogus inflations
159
+ budget[0] -= len(out)
160
+ seen += 1
161
+ yield f"{name} :: carved@{off}", out
162
+
163
+
164
+ def iter_streams(
165
+ data: bytes, name: str, carve: bool, budget: list[int], depth: int = 0
166
+ ) -> Iterator[tuple[str, bytes]]:
167
+ """Yield the raw file plus every decompressed stream, recursively."""
168
+ yield name, data
169
+ if depth >= MAX_DEPTH:
170
+ return
171
+ for label, inner in _decompress_layer(data, name, budget):
172
+ yield from iter_streams(inner, label, carve, budget, depth + 1)
173
+ if carve:
174
+ for label, inner in _carve_zlib(data, name, budget):
175
+ yield from iter_streams(inner, label, carve, budget, depth + 1)
@@ -0,0 +1,124 @@
1
+ # AUTO-GENERATED — DO NOT EDIT.
2
+ # Synced from src/graphsenselib/convert/address_scan/detectors.py via
3
+ # clients/python/scripts/sync_address_scan.py. Edit the source and re-run
4
+ # `make -C clients/python sync-address-scan`.
5
+ """Regex detectors and validators for the address scanner.
6
+
7
+ Each detector pairs a permissive regex (to *extract* address-shaped runs) with
8
+ a validator (to *confirm* them). The validators reuse the checksum logic in
9
+ :mod:`graphsenselib.utils.address` wherever possible; only XMR (no cheap
10
+ checksum) and the opt-in tx-hash candidates are format-level heuristics.
11
+
12
+ Supported with full checksum validation:
13
+ - Bitcoin : legacy (1.../3...), bech32/bech32m (bc1...)
14
+ - Litecoin : legacy (L/M...), bech32 (ltc1...)
15
+ - Ethereum : 0x + 40 hex (EIP-55 mixed-case checksum verified when present)
16
+ - TRON : T... base58check
17
+ - Zcash : transparent t1/t3 base58check
18
+ - XRP : classic r-addresses (base58check, ripple alphabet)
19
+ Reported as candidates only (no cheap checksum):
20
+ - Monero : 4.../8... 95-char addresses
21
+ """
22
+
23
+ from __future__ import annotations
24
+
25
+ from typing import Callable, NamedTuple, Optional
26
+
27
+ from graphsense.address_scan.validators import validate_address, validate_xrp_address
28
+
29
+
30
+ def decodes_to_text(hex_body: str, threshold: float = 0.7) -> bool:
31
+ """True if ``hex_body`` decodes mostly to printable ASCII.
32
+
33
+ SQL dumps store blobs as ``0x...``/hex literals; a run that decodes to
34
+ readable text is encoded content (e.g. stored HTML/JS), not an address.
35
+ Real ETH/XMR addresses are effectively random bytes and fail this test.
36
+ """
37
+ try:
38
+ raw = bytes.fromhex(hex_body)
39
+ except ValueError:
40
+ return False
41
+ if not raw:
42
+ return False
43
+ printable = sum(1 for b in raw if 0x20 <= b <= 0x7E)
44
+ return printable / len(raw) >= threshold
45
+
46
+
47
+ def _eth_valid(addr: str) -> bool:
48
+ # Drop hex-encoded text blobs before the (stricter) EIP-55 check.
49
+ if decodes_to_text(addr[2:]):
50
+ return False
51
+ return validate_address("eth", addr)
52
+
53
+
54
+ def _xmr_candidate_valid(s: str) -> bool:
55
+ """Monero has no cheap checksum, so reject the common false positives:
56
+ pure-hex strings (SQL blob literals) and hex that decodes to text."""
57
+ if all(c in "0123456789abcdefABCDEF" for c in s):
58
+ return False # a real base58 Monero address is ~never all-hex
59
+ if decodes_to_text(s):
60
+ return False
61
+ return True
62
+
63
+
64
+ def txhash_valid(s: str) -> bool:
65
+ """A tx hash is 32 random bytes with NO checksum -- unverifiable by form.
66
+ We can only drop the obvious non-hashes: hex that decodes to ASCII text."""
67
+ return not decodes_to_text(s[2:] if s[:2].lower() == "0x" else s)
68
+
69
+
70
+ class Detector(NamedTuple):
71
+ label: str
72
+ pattern: str
73
+ validator: Optional[Callable[[str], bool]]
74
+
75
+
76
+ # Lookarounds keep matches from being slices of a longer hex/base58 run.
77
+ _H = r"[a-fA-F0-9]"
78
+
79
+ DETECTORS: list[Detector] = [
80
+ Detector(
81
+ "BTC legacy/P2SH",
82
+ r"[13][1-9A-HJ-NP-Za-km-z]{25,34}",
83
+ lambda m: validate_address("btc", m),
84
+ ),
85
+ Detector(
86
+ "BTC bech32", r"bc1[0-9ac-hj-np-z]{11,87}", lambda m: validate_address("btc", m)
87
+ ),
88
+ Detector(
89
+ "LTC legacy",
90
+ r"[LM][1-9A-HJ-NP-Za-km-z]{25,34}",
91
+ lambda m: validate_address("ltc", m),
92
+ ),
93
+ Detector(
94
+ "LTC bech32",
95
+ r"ltc1[0-9ac-hj-np-z]{11,87}",
96
+ lambda m: validate_address("ltc", m),
97
+ ),
98
+ Detector("ETH", rf"(?<!{_H})0x{_H}{{40}}(?!{_H})", _eth_valid),
99
+ Detector("TRX", r"T[1-9A-HJ-NP-Za-km-z]{33}", lambda m: validate_address("trx", m)),
100
+ Detector(
101
+ "ZEC transparent",
102
+ r"t[13][1-9A-HJ-NP-Za-km-z]{33}",
103
+ lambda m: validate_address("zec", m),
104
+ ),
105
+ Detector("XRP", r"r[1-9A-HJ-NP-Za-km-z]{24,34}", validate_xrp_address),
106
+ Detector(
107
+ "XMR (candidate)", r"[48][0-9AB][1-9A-HJ-NP-Za-km-z]{93}", _xmr_candidate_valid
108
+ ),
109
+ ]
110
+
111
+ # Opt-in only (--tx-hashes): format-level, NOT checksum-verifiable. Every 64-hex
112
+ # digest (SHA-256 file hashes, tokens, etc.) matches, so results are candidates.
113
+ # The 0x form is matched first; the bare form excludes an 'x'-prefixed run so a
114
+ # single 0x<64hex> is not double-counted.
115
+ TX_DETECTORS: list[Detector] = [
116
+ Detector(
117
+ "TX-hash candidate (0x+64hex)", rf"(?<!{_H})0x{_H}{{64}}(?!{_H})", txhash_valid
118
+ ),
119
+ Detector(
120
+ "TX-hash candidate (64hex)",
121
+ rf"(?<![a-fA-F0-9xX]){_H}{{64}}(?!{_H})",
122
+ txhash_valid,
123
+ ),
124
+ ]
@@ -0,0 +1,182 @@
1
+ # AUTO-GENERATED — DO NOT EDIT.
2
+ # Synced from src/graphsenselib/convert/address_scan/scanner.py via
3
+ # clients/python/scripts/sync_address_scan.py. Edit the source and re-run
4
+ # `make -C clients/python sync-address-scan`.
5
+ """Scan streams for crypto addresses and build a structured report.
6
+
7
+ The report is a plain dict (JSON-serialisable); the CLI renders it either as
8
+ JSON or human-readable text. Keeping report-building separate from rendering
9
+ makes both the ``--json`` output and the tests trivial.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import re
15
+ from collections import Counter, defaultdict
16
+
17
+ from .decompress import iter_streams
18
+ from .detectors import DETECTORS, TX_DETECTORS
19
+
20
+
21
+ def scan(
22
+ text: str, tx_hashes: bool = False
23
+ ) -> tuple[dict[str, Counter], dict[str, Counter]]:
24
+ """Return (validated, rejected) counters keyed by detector label.
25
+
26
+ ``rejected`` holds regex matches dropped by the checksum/heuristic validator
27
+ (the bulk of them in a DB dump: hashes, ids, hex blobs). With ``tx_hashes``,
28
+ format-level (unverifiable) tx-hash candidates are added too.
29
+ """
30
+ detectors = DETECTORS + (TX_DETECTORS if tx_hashes else [])
31
+ found: dict[str, Counter] = defaultdict(Counter)
32
+ rejected: dict[str, Counter] = defaultdict(Counter)
33
+ for label, pattern, validator in detectors:
34
+ for match in re.findall(pattern, text):
35
+ if validator is None or validator(match):
36
+ found[label][match] += 1
37
+ else:
38
+ rejected[label][match] += 1
39
+ return found, rejected
40
+
41
+
42
+ def context_lines(text: str, addr: str, width: int = 50) -> str:
43
+ i = text.find(addr)
44
+ snippet = text[max(0, i - width) : i + len(addr) + width]
45
+ return snippet.replace("\n", " ").replace("\r", " ")
46
+
47
+
48
+ def _group_obj(counter: Counter, text: str, limit: int, context: bool) -> dict:
49
+ items = []
50
+ for value, n in counter.most_common(limit or None):
51
+ entry = {"value": value, "count": n}
52
+ if context:
53
+ entry["context"] = context_lines(text, value)
54
+ items.append(entry)
55
+ return {
56
+ "unique": len(counter),
57
+ "occurrences": sum(counter.values()),
58
+ "shown": len(items),
59
+ "items": items,
60
+ }
61
+
62
+
63
+ def build_report(
64
+ paths: list[str],
65
+ *,
66
+ decompress: bool = True,
67
+ carve: bool = False,
68
+ max_decompressed_mb: int = 1024,
69
+ tx_hashes: bool = False,
70
+ context: bool = False,
71
+ hide_rejected: bool = False,
72
+ rejected_limit: int = 20,
73
+ ) -> dict:
74
+ """Scan every path and return a JSON-serialisable report dict."""
75
+ global_seen: set[str] = set() # distinct addresses across all files/streams
76
+ global_tx: set[str] = set() # distinct tx-hash candidates
77
+ report: dict = {"files": []}
78
+
79
+ for path in paths:
80
+ try:
81
+ with open(path, "rb") as fh:
82
+ raw = fh.read()
83
+ except OSError as exc:
84
+ report["files"].append({"path": path, "error": str(exc)})
85
+ continue
86
+
87
+ if not decompress:
88
+ streams = [(path, raw)]
89
+ else:
90
+ budget = [max_decompressed_mb * 1024 * 1024]
91
+ streams = list(iter_streams(raw, path, carve, budget))
92
+
93
+ file_entry: dict = {
94
+ "path": path,
95
+ "bytes": len(raw),
96
+ "stream_count": len(streams),
97
+ "streams": [],
98
+ }
99
+ for label, sdata in streams:
100
+ # latin-1 maps every byte 0x00-0xFF to one code point, losslessly.
101
+ # ASCII runs (all address encodings are ASCII) survive intact, and
102
+ # non-text bytes become non-matching separators -> works on binaries.
103
+ text = sdata.decode("latin-1")
104
+ results, rejected = scan(text, tx_hashes=tx_hashes)
105
+ for kind, counter in results.items():
106
+ target = global_tx if kind.startswith("TX-hash") else global_seen
107
+ target.update(counter)
108
+ if not results and (hide_rejected or not rejected):
109
+ continue # keep decompressed-but-empty streams quiet
110
+
111
+ stream_obj: dict = {
112
+ "stream": label,
113
+ "bytes": len(sdata),
114
+ "validated": {
115
+ k: _group_obj(c, text, 0, context) for k, c in results.items()
116
+ },
117
+ }
118
+ if not hide_rejected and rejected:
119
+ stream_obj["rejected"] = {
120
+ k: _group_obj(c, text, rejected_limit, context)
121
+ for k, c in rejected.items()
122
+ }
123
+ file_entry["streams"].append(stream_obj)
124
+ report["files"].append(file_entry)
125
+
126
+ report["summary"] = {
127
+ "unique_valid_addresses": len(global_seen),
128
+ "unique_tx_hash_candidates": len(global_tx),
129
+ }
130
+ return report
131
+
132
+
133
+ def render_text(report: dict, echo) -> None:
134
+ """Render ``report`` as human-readable text via the ``echo`` callable."""
135
+
136
+ def print_items(group: dict) -> None:
137
+ for it in group["items"]:
138
+ echo(f" {it['value']} x{it['count']}")
139
+ if "context" in it:
140
+ echo(f" … {it['context']} …")
141
+ hidden = group["unique"] - group["shown"]
142
+ if hidden > 0:
143
+ echo(f" … {hidden} more (raise --rejected-limit to see them)")
144
+
145
+ for f in report["files"]:
146
+ if "error" in f:
147
+ continue # errors are reported to stderr by the caller
148
+ echo(
149
+ f"\n=== {f['path']} ({f['bytes']:,} bytes, {f['stream_count']} stream(s)) ==="
150
+ )
151
+ for stream in f["streams"]:
152
+ if stream["stream"] != f["path"]:
153
+ echo(
154
+ f"\n ── stream: {stream['stream']} ({stream['bytes']:,} bytes) ──"
155
+ )
156
+ for kind, group in stream["validated"].items():
157
+ echo(
158
+ f"\n [{kind}] {group['unique']} unique / "
159
+ f"{group['occurrences']} occurrences"
160
+ )
161
+ print_items(group)
162
+ if "rejected" in stream:
163
+ echo("\n --- rejected candidates (failed checksum/heuristic) ---")
164
+ for kind, group in stream["rejected"].items():
165
+ echo(
166
+ f"\n [{kind}] {group['unique']} unique rejected / "
167
+ f"{group['occurrences']} occurrences"
168
+ )
169
+ print_items(group)
170
+ if not any(s["validated"] for s in f["streams"]):
171
+ echo(" no valid crypto addresses found")
172
+
173
+ summary = report["summary"]
174
+ echo(
175
+ f"\nTotal unique valid addresses across all files: "
176
+ f"{summary['unique_valid_addresses']}"
177
+ )
178
+ if summary["unique_tx_hash_candidates"]:
179
+ echo(
180
+ f"Total unique tx-hash candidates (unverified): "
181
+ f"{summary['unique_tx_hash_candidates']}"
182
+ )