sightingdb-client 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sightingdb_client-0.0.1/LICENSE +21 -0
- sightingdb_client-0.0.1/MANIFEST.in +7 -0
- sightingdb_client-0.0.1/PKG-INFO +249 -0
- sightingdb_client-0.0.1/README.md +215 -0
- sightingdb_client-0.0.1/pyproject.toml +63 -0
- sightingdb_client-0.0.1/samples/async_feed.py +58 -0
- sightingdb_client-0.0.1/samples/everything.py +107 -0
- sightingdb_client-0.0.1/setup.cfg +4 -0
- sightingdb_client-0.0.1/sightingdb/__init__.py +85 -0
- sightingdb_client-0.0.1/sightingdb/__version__.py +10 -0
- sightingdb_client-0.0.1/sightingdb/_async_client.py +306 -0
- sightingdb_client-0.0.1/sightingdb/_client.py +399 -0
- sightingdb_client-0.0.1/sightingdb/_protocol.py +349 -0
- sightingdb_client-0.0.1/sightingdb/config.py +278 -0
- sightingdb_client-0.0.1/sightingdb/errors.py +150 -0
- sightingdb_client-0.0.1/sightingdb/models.py +275 -0
- sightingdb_client-0.0.1/sightingdb/py.typed +0 -0
- sightingdb_client-0.0.1/sightingdb_client.egg-info/PKG-INFO +249 -0
- sightingdb_client-0.0.1/sightingdb_client.egg-info/SOURCES.txt +25 -0
- sightingdb_client-0.0.1/sightingdb_client.egg-info/dependency_links.txt +1 -0
- sightingdb_client-0.0.1/sightingdb_client.egg-info/requires.txt +8 -0
- sightingdb_client-0.0.1/sightingdb_client.egg-info/top_level.txt +1 -0
- sightingdb_client-0.0.1/tests/conftest.py +73 -0
- sightingdb_client-0.0.1/tests/test_async_client.py +131 -0
- sightingdb_client-0.0.1/tests/test_client.py +429 -0
- sightingdb_client-0.0.1/tests/test_config.py +149 -0
- sightingdb_client-0.0.1/tests/test_live.py +155 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (C) 2020-2026 Sebastien Tricaud
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
# The sdist carries what is needed to check the package out and test it, so a
|
|
2
|
+
# distribution rebuilt from PyPI can be verified without cloning the repository.
|
|
3
|
+
include LICENSE
|
|
4
|
+
include README.md
|
|
5
|
+
recursive-include tests *.py
|
|
6
|
+
recursive-include samples *.py
|
|
7
|
+
global-exclude __pycache__/* *.py[cod]
|
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: sightingdb-client
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: The Python Client Library for SightingDB
|
|
5
|
+
Author-email: Sebastien Tricaud <sebastien@honeynet.org>
|
|
6
|
+
Maintainer-email: Sebastien Tricaud <sebastien@honeynet.org>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Project-URL: Homepage, https://github.com/stricaud/sightingdb-client
|
|
9
|
+
Project-URL: Source, https://github.com/stricaud/sightingdb-client
|
|
10
|
+
Project-URL: Server, https://github.com/stricaud/sightingdb
|
|
11
|
+
Keywords: sightingdb,sightings,threat-intelligence,security
|
|
12
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Intended Audience :: Information Technology
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
23
|
+
Classifier: Topic :: Security
|
|
24
|
+
Classifier: Typing :: Typed
|
|
25
|
+
Requires-Python: >=3.10
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
License-File: LICENSE
|
|
28
|
+
Requires-Dist: httpx>=0.24
|
|
29
|
+
Requires-Dist: tomli>=1.1.0; python_version < "3.11"
|
|
30
|
+
Provides-Extra: dev
|
|
31
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
32
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == "dev"
|
|
33
|
+
Dynamic: license-file
|
|
34
|
+
|
|
35
|
+
# SightingDB Python Client
|
|
36
|
+
|
|
37
|
+
[SightingDB](https://github.com/stricaud/sightingdb) counts things: how many
|
|
38
|
+
times a value was seen, when it was first seen, when it was last seen, and how
|
|
39
|
+
many namespaces hold it. This library is its REST API as Python objects, for
|
|
40
|
+
both synchronous and asyncio code.
|
|
41
|
+
|
|
42
|
+
```
|
|
43
|
+
pip install sightingdb-client
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
The distribution is `sightingdb-client`; the module it installs is
|
|
47
|
+
`sightingdb`, so `import sightingdb` is what you write. Requires Python 3.10+
|
|
48
|
+
and talks to SightingDB 0.5 or later.
|
|
49
|
+
|
|
50
|
+
## Writing
|
|
51
|
+
|
|
52
|
+
```python
|
|
53
|
+
import sightingdb
|
|
54
|
+
|
|
55
|
+
with sightingdb.SightingDB("https://localhost:9999", apikey="changeme") as db:
|
|
56
|
+
db.write("feeds/misp/ips", "127.0.0.1") # returns the new count
|
|
57
|
+
db.write("feeds/misp/ips", "127.0.0.1", ttl=86400) # expires a day after last seen
|
|
58
|
+
db.write("feeds/misp/ips", "10.0.0.1", timestamp=1587364370) # seen in the past
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Make the client once and keep it: it holds a connection pool, and a client per
|
|
62
|
+
call re-runs the TLS handshake every time. `timestamp` takes Unix seconds or a
|
|
63
|
+
`datetime`; `ttl` takes seconds or a `timedelta`.
|
|
64
|
+
|
|
65
|
+
Many values go in one request:
|
|
66
|
+
|
|
67
|
+
```python
|
|
68
|
+
result = db.write_many([
|
|
69
|
+
("feeds/misp/ips", "8.8.8.8"),
|
|
70
|
+
sightingdb.Sighting("feeds/misp/ips", "1.1.1.1", ttl=86400),
|
|
71
|
+
{"namespace": "feeds/misp/domains", "value": "example.com"},
|
|
72
|
+
])
|
|
73
|
+
print(result.written)
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Anything the server refuses raises `BulkWriteError`, naming the items that did
|
|
77
|
+
not land — pass `strict=False` to get them in `result.errors` instead.
|
|
78
|
+
|
|
79
|
+
For a feed you are streaming rather than holding in memory, batch it:
|
|
80
|
+
|
|
81
|
+
```python
|
|
82
|
+
with db.batch(chunk_size=1000) as batch: # one request per 1000 sightings
|
|
83
|
+
for value in feed:
|
|
84
|
+
batch.add("feeds/misp/ips", value)
|
|
85
|
+
print(batch.result.written)
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
## Reading
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
attribute = db.read("feeds/misp/ips", "127.0.0.1")
|
|
92
|
+
attribute.count # 2
|
|
93
|
+
attribute.consensus # how many namespaces hold this value
|
|
94
|
+
attribute.first_seen # 1566624658
|
|
95
|
+
attribute.first_seen_at # datetime(2019, 8, 24, 5, 30, 58, tzinfo=timezone.utc)
|
|
96
|
+
attribute.expires_at # None when the value has no TTL
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
`read` raises `NotFoundError` for a value that was never seen. When you are
|
|
100
|
+
enriching a list, ask for them together instead — there a miss is an answer,
|
|
101
|
+
not an error:
|
|
102
|
+
|
|
103
|
+
```python
|
|
104
|
+
for result in db.read_many([("feeds/misp/ips", v) for v in indicators]):
|
|
105
|
+
if result.found:
|
|
106
|
+
print(result.value, result.count, result.consensus)
|
|
107
|
+
else:
|
|
108
|
+
print(result.value, result.error) # "Value not found" / "Path not found"
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
Reading is itself recorded, as a "shadow sighting" under `_shadow/`, so that you
|
|
112
|
+
can see how often a value was searched for. Pass `shadow=False` (or
|
|
113
|
+
`Sighting(..., noshadow=True)` in a bulk read) to look without leaving a trace.
|
|
114
|
+
|
|
115
|
+
Other reads:
|
|
116
|
+
|
|
117
|
+
```python
|
|
118
|
+
db.read("feeds/misp/ips", "127.0.0.1", stats=True).stats_by_hour # hourly histogram
|
|
119
|
+
db.list_values("feeds/misp/ips") # every value in a namespace
|
|
120
|
+
db.exists("feeds/misp/ips", "8.8.8.8")
|
|
121
|
+
db.delete("feeds/misp/ips") # the whole namespace, with care
|
|
122
|
+
db.info() # what the server says it is
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
Namespaces are normalized for you: `feeds/ips`, `/feeds/ips` and `/feeds/ips/`
|
|
126
|
+
are the same namespace here. To the server they are not — a trailing slash is a
|
|
127
|
+
different namespace with its own counts.
|
|
128
|
+
|
|
129
|
+
## asyncio
|
|
130
|
+
|
|
131
|
+
`AsyncSightingDB` is the same API, awaited:
|
|
132
|
+
|
|
133
|
+
```python
|
|
134
|
+
import asyncio, sightingdb
|
|
135
|
+
|
|
136
|
+
async def main():
|
|
137
|
+
async with sightingdb.AsyncSightingDB(apikey="changeme") as db:
|
|
138
|
+
async with db.batch(chunk_size=1000) as batch:
|
|
139
|
+
for value in feed:
|
|
140
|
+
await batch.add("feeds/misp/ips", value)
|
|
141
|
+
|
|
142
|
+
results = await db.read_many([("feeds/misp/ips", v) for v in indicators])
|
|
143
|
+
counts = await asyncio.gather(*(db.write("feeds/live", v) for v in values))
|
|
144
|
+
|
|
145
|
+
asyncio.run(main())
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
## Configuration
|
|
149
|
+
|
|
150
|
+
Settings are resolved per field, most specific first:
|
|
151
|
+
|
|
152
|
+
1. what you pass to the client,
|
|
153
|
+
2. the environment: `SIGHTINGDB_URL` (or `SIGHTINGDB_HOST`, `SIGHTINGDB_PORT`,
|
|
154
|
+
`SIGHTINGDB_SSL`), `SIGHTINGDB_APIKEY`, `SIGHTINGDB_VERIFY`,
|
|
155
|
+
`SIGHTINGDB_TIMEOUT`, `SIGHTINGDB_MAX_RETRIES`,
|
|
156
|
+
3. a TOML file — `$SIGHTINGDB_CONFIG`, else `~/.sightingdb/client.toml`.
|
|
157
|
+
|
|
158
|
+
```toml
|
|
159
|
+
# ~/.sightingdb/client.toml
|
|
160
|
+
[client]
|
|
161
|
+
url = "https://sightingdb.example.com:9999"
|
|
162
|
+
apikey = "changeme"
|
|
163
|
+
verify = "/etc/ssl/sightingdb-ca.pem"
|
|
164
|
+
timeout = 10.0
|
|
165
|
+
max_retries = 2
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
So a config file can hold the URL while the key comes from the environment,
|
|
169
|
+
without either repeating the other. `use_env=False` and `use_file=False` cut a
|
|
170
|
+
client off from both, which is what you want in tests.
|
|
171
|
+
|
|
172
|
+
### TLS
|
|
173
|
+
|
|
174
|
+
`verify` defaults to `True`. SightingDB generates a self-signed certificate on
|
|
175
|
+
first run, which nothing will verify, so either point `verify` at that
|
|
176
|
+
certificate — `verify="/Users/you/.sightingdb/ssl/cert.pem"` — or, knowing what
|
|
177
|
+
it costs, pass `verify=False`.
|
|
178
|
+
|
|
179
|
+
## Errors
|
|
180
|
+
|
|
181
|
+
Everything raised derives from `SightingDBError`.
|
|
182
|
+
|
|
183
|
+
| Exception | When |
|
|
184
|
+
|---|---|
|
|
185
|
+
| `ConfigurationError` | The settings do not make sense. |
|
|
186
|
+
| `TransportError`, `TimeoutError` | No answer arrived: DNS, TCP, TLS, timeout. |
|
|
187
|
+
| `ProtocolError` | The answer was not what the API promises — usually a wrong URL, or a server too old for an endpoint. |
|
|
188
|
+
| `BadRequestError` (400) | The server could not make sense of the request. |
|
|
189
|
+
| `AuthenticationError` (401) | No API key was sent. |
|
|
190
|
+
| `PermissionDeniedError` (403) | The key is unknown, or not granted this namespace. |
|
|
191
|
+
| `NotFoundError` (404) | No such namespace, or no such value. Carries `.namespace` and `.value`. |
|
|
192
|
+
| `ServerError` (5xx) | The server failed to handle the request. |
|
|
193
|
+
| `BulkWriteError` | A bulk write landed only in part. Carries the full `.result`. |
|
|
194
|
+
|
|
195
|
+
Requests that fail in a way a retry could fix are retried with backoff. Writes
|
|
196
|
+
are only retried when the connection was never established: SightingDB counts,
|
|
197
|
+
and a retried write that did land would count twice.
|
|
198
|
+
|
|
199
|
+
## Development
|
|
200
|
+
|
|
201
|
+
```
|
|
202
|
+
pip install -e '.[dev]'
|
|
203
|
+
pytest # unit tests, no server needed
|
|
204
|
+
|
|
205
|
+
SIGHTINGDB_TEST_URL=https://localhost:9999 \
|
|
206
|
+
SIGHTINGDB_TEST_APIKEY=changeme \
|
|
207
|
+
SIGHTINGDB_TEST_VERIFY=false \
|
|
208
|
+
pytest -m live # against a real server
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
`samples/everything.py` walks through the whole API, and
|
|
212
|
+
`samples/async_feed.py` shows the asyncio client ingesting and enriching a feed.
|
|
213
|
+
|
|
214
|
+
CI runs the unit tests on 3.10 through 3.14, builds the distribution, and runs
|
|
215
|
+
the live tests against a real SightingDB downloaded from the server's own
|
|
216
|
+
releases — because every bug this client has had was a disagreement with the
|
|
217
|
+
server that mocks could not have caught.
|
|
218
|
+
|
|
219
|
+
### Releasing
|
|
220
|
+
|
|
221
|
+
Tag it:
|
|
222
|
+
|
|
223
|
+
```
|
|
224
|
+
git tag python-v1.0.1 && git push origin python-v1.0.1
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
`.github/workflows/release.yml` does the rest: if `__version__.py` disagrees
|
|
228
|
+
with the tag it bumps, commits and moves the tag onto that commit, then tests,
|
|
229
|
+
builds, publishes to PyPI over Trusted Publishing, and cuts a GitHub Release
|
|
230
|
+
with the sdist and wheel attached. `python-v1.0.1` and `v1.0.1` are both
|
|
231
|
+
accepted, and `1.0.1rc1` publishes as a prerelease.
|
|
232
|
+
|
|
233
|
+
## Upgrading from 0.0.x
|
|
234
|
+
|
|
235
|
+
The 0.0.x API is gone. It predates several versions of the server, reported
|
|
236
|
+
failures as successes, and its `auth` object called endpoints that no longer
|
|
237
|
+
exist — API keys now live in the server's `acl.toml`.
|
|
238
|
+
|
|
239
|
+
| 0.0.x | 1.0 |
|
|
240
|
+
|---|---|
|
|
241
|
+
| `sightingdb.connection(host=..., apikey=...)` | `sightingdb.SightingDB(url, apikey=...)` |
|
|
242
|
+
| `writer.add(...)` then `writer.commit()` | `db.write_many([...])` or `with db.batch() as b: b.add(...)` |
|
|
243
|
+
| `writer.write_one(ns, v)` | `db.write(ns, v)` |
|
|
244
|
+
| `reader.add(...)` then `reader.fetch()` | `db.read_many([...])` |
|
|
245
|
+
| `reader.read_one(ns, v)` | `db.read(ns, v)` |
|
|
246
|
+
| `reader.read_one_with_stats(ns, v)` | `db.read(ns, v, stats=True)` |
|
|
247
|
+
| `delete(con).delete(ns)` | `db.delete(ns)` |
|
|
248
|
+
| `con.disable_ssl_warnings()` | nothing to disable; `httpx` does not warn |
|
|
249
|
+
| `sightingdb.auth(con)` | removed — keys are configured server-side |
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
# SightingDB Python Client
|
|
2
|
+
|
|
3
|
+
[SightingDB](https://github.com/stricaud/sightingdb) counts things: how many
|
|
4
|
+
times a value was seen, when it was first seen, when it was last seen, and how
|
|
5
|
+
many namespaces hold it. This library is its REST API as Python objects, for
|
|
6
|
+
both synchronous and asyncio code.
|
|
7
|
+
|
|
8
|
+
```
|
|
9
|
+
pip install sightingdb-client
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
The distribution is `sightingdb-client`; the module it installs is
|
|
13
|
+
`sightingdb`, so `import sightingdb` is what you write. Requires Python 3.10+
|
|
14
|
+
and talks to SightingDB 0.5 or later.
|
|
15
|
+
|
|
16
|
+
## Writing
|
|
17
|
+
|
|
18
|
+
```python
|
|
19
|
+
import sightingdb
|
|
20
|
+
|
|
21
|
+
with sightingdb.SightingDB("https://localhost:9999", apikey="changeme") as db:
|
|
22
|
+
db.write("feeds/misp/ips", "127.0.0.1") # returns the new count
|
|
23
|
+
db.write("feeds/misp/ips", "127.0.0.1", ttl=86400) # expires a day after last seen
|
|
24
|
+
db.write("feeds/misp/ips", "10.0.0.1", timestamp=1587364370) # seen in the past
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
Make the client once and keep it: it holds a connection pool, and a client per
|
|
28
|
+
call re-runs the TLS handshake every time. `timestamp` takes Unix seconds or a
|
|
29
|
+
`datetime`; `ttl` takes seconds or a `timedelta`.
|
|
30
|
+
|
|
31
|
+
Many values go in one request:
|
|
32
|
+
|
|
33
|
+
```python
|
|
34
|
+
result = db.write_many([
|
|
35
|
+
("feeds/misp/ips", "8.8.8.8"),
|
|
36
|
+
sightingdb.Sighting("feeds/misp/ips", "1.1.1.1", ttl=86400),
|
|
37
|
+
{"namespace": "feeds/misp/domains", "value": "example.com"},
|
|
38
|
+
])
|
|
39
|
+
print(result.written)
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Anything the server refuses raises `BulkWriteError`, naming the items that did
|
|
43
|
+
not land — pass `strict=False` to get them in `result.errors` instead.
|
|
44
|
+
|
|
45
|
+
For a feed you are streaming rather than holding in memory, batch it:
|
|
46
|
+
|
|
47
|
+
```python
|
|
48
|
+
with db.batch(chunk_size=1000) as batch: # one request per 1000 sightings
|
|
49
|
+
for value in feed:
|
|
50
|
+
batch.add("feeds/misp/ips", value)
|
|
51
|
+
print(batch.result.written)
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## Reading
|
|
55
|
+
|
|
56
|
+
```python
|
|
57
|
+
attribute = db.read("feeds/misp/ips", "127.0.0.1")
|
|
58
|
+
attribute.count # 2
|
|
59
|
+
attribute.consensus # how many namespaces hold this value
|
|
60
|
+
attribute.first_seen # 1566624658
|
|
61
|
+
attribute.first_seen_at # datetime(2019, 8, 24, 5, 30, 58, tzinfo=timezone.utc)
|
|
62
|
+
attribute.expires_at # None when the value has no TTL
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
`read` raises `NotFoundError` for a value that was never seen. When you are
|
|
66
|
+
enriching a list, ask for them together instead — there a miss is an answer,
|
|
67
|
+
not an error:
|
|
68
|
+
|
|
69
|
+
```python
|
|
70
|
+
for result in db.read_many([("feeds/misp/ips", v) for v in indicators]):
|
|
71
|
+
if result.found:
|
|
72
|
+
print(result.value, result.count, result.consensus)
|
|
73
|
+
else:
|
|
74
|
+
print(result.value, result.error) # "Value not found" / "Path not found"
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Reading is itself recorded, as a "shadow sighting" under `_shadow/`, so that you
|
|
78
|
+
can see how often a value was searched for. Pass `shadow=False` (or
|
|
79
|
+
`Sighting(..., noshadow=True)` in a bulk read) to look without leaving a trace.
|
|
80
|
+
|
|
81
|
+
Other reads:
|
|
82
|
+
|
|
83
|
+
```python
|
|
84
|
+
db.read("feeds/misp/ips", "127.0.0.1", stats=True).stats_by_hour # hourly histogram
|
|
85
|
+
db.list_values("feeds/misp/ips") # every value in a namespace
|
|
86
|
+
db.exists("feeds/misp/ips", "8.8.8.8")
|
|
87
|
+
db.delete("feeds/misp/ips") # the whole namespace, with care
|
|
88
|
+
db.info() # what the server says it is
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
Namespaces are normalized for you: `feeds/ips`, `/feeds/ips` and `/feeds/ips/`
|
|
92
|
+
are the same namespace here. To the server they are not — a trailing slash is a
|
|
93
|
+
different namespace with its own counts.
|
|
94
|
+
|
|
95
|
+
## asyncio
|
|
96
|
+
|
|
97
|
+
`AsyncSightingDB` is the same API, awaited:
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
import asyncio, sightingdb
|
|
101
|
+
|
|
102
|
+
async def main():
|
|
103
|
+
async with sightingdb.AsyncSightingDB(apikey="changeme") as db:
|
|
104
|
+
async with db.batch(chunk_size=1000) as batch:
|
|
105
|
+
for value in feed:
|
|
106
|
+
await batch.add("feeds/misp/ips", value)
|
|
107
|
+
|
|
108
|
+
results = await db.read_many([("feeds/misp/ips", v) for v in indicators])
|
|
109
|
+
counts = await asyncio.gather(*(db.write("feeds/live", v) for v in values))
|
|
110
|
+
|
|
111
|
+
asyncio.run(main())
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
## Configuration
|
|
115
|
+
|
|
116
|
+
Settings are resolved per field, most specific first:
|
|
117
|
+
|
|
118
|
+
1. what you pass to the client,
|
|
119
|
+
2. the environment: `SIGHTINGDB_URL` (or `SIGHTINGDB_HOST`, `SIGHTINGDB_PORT`,
|
|
120
|
+
`SIGHTINGDB_SSL`), `SIGHTINGDB_APIKEY`, `SIGHTINGDB_VERIFY`,
|
|
121
|
+
`SIGHTINGDB_TIMEOUT`, `SIGHTINGDB_MAX_RETRIES`,
|
|
122
|
+
3. a TOML file — `$SIGHTINGDB_CONFIG`, else `~/.sightingdb/client.toml`.
|
|
123
|
+
|
|
124
|
+
```toml
|
|
125
|
+
# ~/.sightingdb/client.toml
|
|
126
|
+
[client]
|
|
127
|
+
url = "https://sightingdb.example.com:9999"
|
|
128
|
+
apikey = "changeme"
|
|
129
|
+
verify = "/etc/ssl/sightingdb-ca.pem"
|
|
130
|
+
timeout = 10.0
|
|
131
|
+
max_retries = 2
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
So a config file can hold the URL while the key comes from the environment,
|
|
135
|
+
without either repeating the other. `use_env=False` and `use_file=False` cut a
|
|
136
|
+
client off from both, which is what you want in tests.
|
|
137
|
+
|
|
138
|
+
### TLS
|
|
139
|
+
|
|
140
|
+
`verify` defaults to `True`. SightingDB generates a self-signed certificate on
|
|
141
|
+
first run, which nothing will verify, so either point `verify` at that
|
|
142
|
+
certificate — `verify="/Users/you/.sightingdb/ssl/cert.pem"` — or, knowing what
|
|
143
|
+
it costs, pass `verify=False`.
|
|
144
|
+
|
|
145
|
+
## Errors
|
|
146
|
+
|
|
147
|
+
Everything raised derives from `SightingDBError`.
|
|
148
|
+
|
|
149
|
+
| Exception | When |
|
|
150
|
+
|---|---|
|
|
151
|
+
| `ConfigurationError` | The settings do not make sense. |
|
|
152
|
+
| `TransportError`, `TimeoutError` | No answer arrived: DNS, TCP, TLS, timeout. |
|
|
153
|
+
| `ProtocolError` | The answer was not what the API promises — usually a wrong URL, or a server too old for an endpoint. |
|
|
154
|
+
| `BadRequestError` (400) | The server could not make sense of the request. |
|
|
155
|
+
| `AuthenticationError` (401) | No API key was sent. |
|
|
156
|
+
| `PermissionDeniedError` (403) | The key is unknown, or not granted this namespace. |
|
|
157
|
+
| `NotFoundError` (404) | No such namespace, or no such value. Carries `.namespace` and `.value`. |
|
|
158
|
+
| `ServerError` (5xx) | The server failed to handle the request. |
|
|
159
|
+
| `BulkWriteError` | A bulk write landed only in part. Carries the full `.result`. |
|
|
160
|
+
|
|
161
|
+
Requests that fail in a way a retry could fix are retried with backoff. Writes
|
|
162
|
+
are only retried when the connection was never established: SightingDB counts,
|
|
163
|
+
and a retried write that did land would count twice.
|
|
164
|
+
|
|
165
|
+
## Development
|
|
166
|
+
|
|
167
|
+
```
|
|
168
|
+
pip install -e '.[dev]'
|
|
169
|
+
pytest # unit tests, no server needed
|
|
170
|
+
|
|
171
|
+
SIGHTINGDB_TEST_URL=https://localhost:9999 \
|
|
172
|
+
SIGHTINGDB_TEST_APIKEY=changeme \
|
|
173
|
+
SIGHTINGDB_TEST_VERIFY=false \
|
|
174
|
+
pytest -m live # against a real server
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
`samples/everything.py` walks through the whole API, and
|
|
178
|
+
`samples/async_feed.py` shows the asyncio client ingesting and enriching a feed.
|
|
179
|
+
|
|
180
|
+
CI runs the unit tests on 3.10 through 3.14, builds the distribution, and runs
|
|
181
|
+
the live tests against a real SightingDB downloaded from the server's own
|
|
182
|
+
releases — because every bug this client has had was a disagreement with the
|
|
183
|
+
server that mocks could not have caught.
|
|
184
|
+
|
|
185
|
+
### Releasing
|
|
186
|
+
|
|
187
|
+
Tag it:
|
|
188
|
+
|
|
189
|
+
```
|
|
190
|
+
git tag python-v1.0.1 && git push origin python-v1.0.1
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
`.github/workflows/release.yml` does the rest: if `__version__.py` disagrees
|
|
194
|
+
with the tag it bumps, commits and moves the tag onto that commit, then tests,
|
|
195
|
+
builds, publishes to PyPI over Trusted Publishing, and cuts a GitHub Release
|
|
196
|
+
with the sdist and wheel attached. `python-v1.0.1` and `v1.0.1` are both
|
|
197
|
+
accepted, and `1.0.1rc1` publishes as a prerelease.
|
|
198
|
+
|
|
199
|
+
## Upgrading from 0.0.x
|
|
200
|
+
|
|
201
|
+
The 0.0.x API is gone. It predates several versions of the server, reported
|
|
202
|
+
failures as successes, and its `auth` object called endpoints that no longer
|
|
203
|
+
exist — API keys now live in the server's `acl.toml`.
|
|
204
|
+
|
|
205
|
+
| 0.0.x | 1.0 |
|
|
206
|
+
|---|---|
|
|
207
|
+
| `sightingdb.connection(host=..., apikey=...)` | `sightingdb.SightingDB(url, apikey=...)` |
|
|
208
|
+
| `writer.add(...)` then `writer.commit()` | `db.write_many([...])` or `with db.batch() as b: b.add(...)` |
|
|
209
|
+
| `writer.write_one(ns, v)` | `db.write(ns, v)` |
|
|
210
|
+
| `reader.add(...)` then `reader.fetch()` | `db.read_many([...])` |
|
|
211
|
+
| `reader.read_one(ns, v)` | `db.read(ns, v)` |
|
|
212
|
+
| `reader.read_one_with_stats(ns, v)` | `db.read(ns, v, stats=True)` |
|
|
213
|
+
| `delete(con).delete(ns)` | `db.delete(ns)` |
|
|
214
|
+
| `con.disable_ssl_warnings()` | nothing to disable; `httpx` does not warn |
|
|
215
|
+
| `sightingdb.auth(con)` | removed — keys are configured server-side |
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
# 77 is where the SPDX `license = "MIT"` expression and `license-files` became
|
|
3
|
+
# valid metadata, replacing the deprecated License:: classifiers.
|
|
4
|
+
requires = ["setuptools>=77"]
|
|
5
|
+
build-backend = "setuptools.build_meta"
|
|
6
|
+
|
|
7
|
+
[project]
|
|
8
|
+
name = "sightingdb-client"
|
|
9
|
+
dynamic = ["version"]
|
|
10
|
+
description = "The Python Client Library for SightingDB"
|
|
11
|
+
readme = "README.md"
|
|
12
|
+
requires-python = ">=3.10"
|
|
13
|
+
license = "MIT"
|
|
14
|
+
license-files = ["LICENSE"]
|
|
15
|
+
authors = [{ name = "Sebastien Tricaud", email = "sebastien@honeynet.org" }]
|
|
16
|
+
maintainers = [{ name = "Sebastien Tricaud", email = "sebastien@honeynet.org" }]
|
|
17
|
+
keywords = ["sightingdb", "sightings", "threat-intelligence", "security"]
|
|
18
|
+
classifiers = [
|
|
19
|
+
"Development Status :: 5 - Production/Stable",
|
|
20
|
+
"Intended Audience :: Developers",
|
|
21
|
+
"Intended Audience :: Information Technology",
|
|
22
|
+
"Operating System :: OS Independent",
|
|
23
|
+
"Programming Language :: Python :: 3",
|
|
24
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
25
|
+
"Programming Language :: Python :: 3.10",
|
|
26
|
+
"Programming Language :: Python :: 3.11",
|
|
27
|
+
"Programming Language :: Python :: 3.12",
|
|
28
|
+
"Programming Language :: Python :: 3.13",
|
|
29
|
+
"Programming Language :: Python :: 3.14",
|
|
30
|
+
"Topic :: Security",
|
|
31
|
+
"Typing :: Typed",
|
|
32
|
+
]
|
|
33
|
+
dependencies = [
|
|
34
|
+
"httpx>=0.24",
|
|
35
|
+
"tomli>=1.1.0; python_version < '3.11'",
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
[project.optional-dependencies]
|
|
39
|
+
dev = [
|
|
40
|
+
"pytest>=7",
|
|
41
|
+
"pytest-asyncio>=0.23",
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
[project.urls]
|
|
45
|
+
Homepage = "https://github.com/stricaud/sightingdb-client"
|
|
46
|
+
Source = "https://github.com/stricaud/sightingdb-client"
|
|
47
|
+
Server = "https://github.com/stricaud/sightingdb"
|
|
48
|
+
|
|
49
|
+
[tool.setuptools]
|
|
50
|
+
packages = ["sightingdb"]
|
|
51
|
+
|
|
52
|
+
[tool.setuptools.dynamic]
|
|
53
|
+
version = { attr = "sightingdb.__version__.__version__" }
|
|
54
|
+
|
|
55
|
+
[tool.setuptools.package-data]
|
|
56
|
+
sightingdb = ["py.typed"]
|
|
57
|
+
|
|
58
|
+
[tool.pytest.ini_options]
|
|
59
|
+
testpaths = ["tests"]
|
|
60
|
+
asyncio_mode = "auto"
|
|
61
|
+
markers = [
|
|
62
|
+
"live: talks to a real SightingDB server; needs SIGHTINGDB_TEST_URL",
|
|
63
|
+
]
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Ingesting a feed and enriching it, with the asyncio client.
|
|
3
|
+
|
|
4
|
+
SIGHTINGDB_APIKEY=changeme ./samples/async_feed.py
|
|
5
|
+
|
|
6
|
+
The point of the async client is here: one connection pool, many requests in
|
|
7
|
+
flight, and lookups that would otherwise be a serial round trip each.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import asyncio
|
|
11
|
+
import os
|
|
12
|
+
import sys
|
|
13
|
+
|
|
14
|
+
import sightingdb
|
|
15
|
+
|
|
16
|
+
NAMESPACE = "_tests/samples/feed"
|
|
17
|
+
|
|
18
|
+
FEED = [f"198.51.100.{n}" for n in range(50)]
|
|
19
|
+
TO_ENRICH = ["198.51.100.7", "198.51.100.42", "203.0.113.1"]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
async def main() -> int:
|
|
23
|
+
async with sightingdb.AsyncSightingDB(
|
|
24
|
+
os.environ.get("SIGHTINGDB_URL", "https://localhost:9999"),
|
|
25
|
+
apikey=os.environ.get("SIGHTINGDB_APIKEY", "changeme"),
|
|
26
|
+
verify=False,
|
|
27
|
+
) as db:
|
|
28
|
+
# Ingest: one request per thousand sightings, not one per sighting.
|
|
29
|
+
async with db.batch(chunk_size=1000) as batch:
|
|
30
|
+
for value in FEED:
|
|
31
|
+
await batch.add(NAMESPACE, value, ttl=86400)
|
|
32
|
+
print(f"ingested {batch.result.written} sightings")
|
|
33
|
+
|
|
34
|
+
# Enrich: one bulk read answers all of them, misses included.
|
|
35
|
+
results = await db.read_many(
|
|
36
|
+
[sightingdb.Sighting(NAMESPACE, v, noshadow=True) for v in TO_ENRICH]
|
|
37
|
+
)
|
|
38
|
+
for value, result in zip(TO_ENRICH, results):
|
|
39
|
+
if result.found:
|
|
40
|
+
print(f" {value}: seen {result.count}x, consensus {result.consensus}")
|
|
41
|
+
else:
|
|
42
|
+
print(f" {value}: {result.error}")
|
|
43
|
+
|
|
44
|
+
# When the calls are genuinely independent, run them at once.
|
|
45
|
+
counts = await asyncio.gather(
|
|
46
|
+
*(db.write(NAMESPACE, value) for value in TO_ENRICH[:2])
|
|
47
|
+
)
|
|
48
|
+
print(f"concurrent writes now count {counts}")
|
|
49
|
+
|
|
50
|
+
await db.delete(NAMESPACE, missing_ok=True)
|
|
51
|
+
return 0
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
if __name__ == "__main__":
|
|
55
|
+
try:
|
|
56
|
+
sys.exit(asyncio.run(main()))
|
|
57
|
+
except sightingdb.SightingDBError as exc:
|
|
58
|
+
sys.exit(f"SightingDB: {exc}")
|