internetdata 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- internetdata-1.0.0/.gitignore +17 -0
- internetdata-1.0.0/LICENSE +21 -0
- internetdata-1.0.0/PKG-INFO +184 -0
- internetdata-1.0.0/README.md +150 -0
- internetdata-1.0.0/pyproject.toml +97 -0
- internetdata-1.0.0/scripts/download-spec.sh +21 -0
- internetdata-1.0.0/scripts/generate.sh +52 -0
- internetdata-1.0.0/scripts/publish.sh +47 -0
- internetdata-1.0.0/scripts/v2_subset.py +110 -0
- internetdata-1.0.0/spec/openapi.yaml +741 -0
- internetdata-1.0.0/src/internetdata/__init__.py +49 -0
- internetdata-1.0.0/src/internetdata/_core.py +287 -0
- internetdata-1.0.0/src/internetdata/_generated/__init__.py +8 -0
- internetdata-1.0.0/src/internetdata/_generated/api/__init__.py +1 -0
- internetdata-1.0.0/src/internetdata/_generated/api/database_v_2/__init__.py +1 -0
- internetdata-1.0.0/src/internetdata/_generated/api/database_v_2/database_checksum_v2.py +203 -0
- internetdata-1.0.0/src/internetdata/_generated/api/database_v_2/database_metadata_v2.py +205 -0
- internetdata-1.0.0/src/internetdata/_generated/api/database_v_2/download_database_v2.py +213 -0
- internetdata-1.0.0/src/internetdata/_generated/api/database_v_2/list_databases.py +162 -0
- internetdata-1.0.0/src/internetdata/_generated/api/database_v_2/list_downloads.py +178 -0
- internetdata-1.0.0/src/internetdata/_generated/client.py +272 -0
- internetdata-1.0.0/src/internetdata/_generated/errors.py +16 -0
- internetdata-1.0.0/src/internetdata/_generated/models/__init__.py +53 -0
- internetdata-1.0.0/src/internetdata/_generated/models/database.py +233 -0
- internetdata-1.0.0/src/internetdata/_generated/models/database_checksum_v2_format.py +9 -0
- internetdata-1.0.0/src/internetdata/_generated/models/database_checksum_v2_response_200.py +85 -0
- internetdata-1.0.0/src/internetdata/_generated/models/database_checksum_v2_response_200_format.py +9 -0
- internetdata-1.0.0/src/internetdata/_generated/models/database_metadata.py +132 -0
- internetdata-1.0.0/src/internetdata/_generated/models/database_metadata_column.py +80 -0
- internetdata-1.0.0/src/internetdata/_generated/models/database_metadata_sample.py +78 -0
- internetdata-1.0.0/src/internetdata/_generated/models/database_metadata_sample_additional_property_item.py +45 -0
- internetdata-1.0.0/src/internetdata/_generated/models/database_metadata_schema.py +72 -0
- internetdata-1.0.0/src/internetdata/_generated/models/database_metadata_size.py +47 -0
- internetdata-1.0.0/src/internetdata/_generated/models/database_redistribution_type_1.py +10 -0
- internetdata-1.0.0/src/internetdata/_generated/models/database_redistribution_type_2_type_1.py +10 -0
- internetdata-1.0.0/src/internetdata/_generated/models/database_redistribution_type_3_type_1.py +10 -0
- internetdata-1.0.0/src/internetdata/_generated/models/database_standing.py +10 -0
- internetdata-1.0.0/src/internetdata/_generated/models/database_version.py +97 -0
- internetdata-1.0.0/src/internetdata/_generated/models/database_version_formats_item.py +9 -0
- internetdata-1.0.0/src/internetdata/_generated/models/db_checksums.py +85 -0
- internetdata-1.0.0/src/internetdata/_generated/models/download.py +161 -0
- internetdata-1.0.0/src/internetdata/_generated/models/download_database_v2_format.py +9 -0
- internetdata-1.0.0/src/internetdata/_generated/models/download_outcome.py +13 -0
- internetdata-1.0.0/src/internetdata/_generated/models/error.py +62 -0
- internetdata-1.0.0/src/internetdata/_generated/models/list_databases_response_200.py +75 -0
- internetdata-1.0.0/src/internetdata/_generated/models/list_downloads_response_200.py +75 -0
- internetdata-1.0.0/src/internetdata/_generated/types.py +54 -0
- internetdata-1.0.0/src/internetdata/aio.py +268 -0
- internetdata-1.0.0/src/internetdata/client.py +278 -0
- internetdata-1.0.0/src/internetdata/errors.py +128 -0
- internetdata-1.0.0/src/internetdata/models.py +197 -0
- internetdata-1.0.0/src/internetdata/py.typed +0 -0
- internetdata-1.0.0/testdata/testdata.json +146 -0
- internetdata-1.0.0/tests/conftest.py +16 -0
- internetdata-1.0.0/tests/helpers.py +175 -0
- internetdata-1.0.0/tests/test_client.py +238 -0
- internetdata-1.0.0/tests/test_conformance.py +223 -0
- internetdata-1.0.0/tests/test_download.py +291 -0
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
__pycache__/
|
|
2
|
+
*.py[cod]
|
|
3
|
+
|
|
4
|
+
# Build output.
|
|
5
|
+
build/
|
|
6
|
+
dist/
|
|
7
|
+
*.egg-info/
|
|
8
|
+
|
|
9
|
+
# Virtualenvs, including any made to run codegen. Keep them out of the tree entirely.
|
|
10
|
+
.venv/
|
|
11
|
+
venv/
|
|
12
|
+
.venv-*/
|
|
13
|
+
|
|
14
|
+
# Tool caches.
|
|
15
|
+
.pytest_cache/
|
|
16
|
+
.mypy_cache/
|
|
17
|
+
.ruff_cache/
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Mslm Dev
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: internetdata
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Official Python client library for the InternetData API. Download and verify licensed IP datasets.
|
|
5
|
+
Project-URL: Homepage, https://internetdata.io
|
|
6
|
+
Project-URL: Documentation, https://docs.internetdata.io
|
|
7
|
+
Project-URL: Source, https://github.com/internetdata/sdk-python
|
|
8
|
+
Project-URL: Issues, https://github.com/internetdata/sdk-python/issues
|
|
9
|
+
Author-email: Mslm Dev <support@internetdata.io>
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: asn,bogon,dataset,geolocation,hosting,internetdata,ip-database,ip-intelligence,mmdb,proxy,vpn
|
|
13
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
20
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
21
|
+
Classifier: Topic :: Internet
|
|
22
|
+
Classifier: Topic :: Security
|
|
23
|
+
Classifier: Typing :: Typed
|
|
24
|
+
Requires-Python: >=3.11
|
|
25
|
+
Requires-Dist: attrs>=22.2.0
|
|
26
|
+
Requires-Dist: httpx<1,>=0.27
|
|
27
|
+
Provides-Extra: dev
|
|
28
|
+
Requires-Dist: mypy>=1.11; extra == 'dev'
|
|
29
|
+
Requires-Dist: openapi-python-client<0.30,>=0.29; extra == 'dev'
|
|
30
|
+
Requires-Dist: pytest-asyncio>=0.24; extra == 'dev'
|
|
31
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
32
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
33
|
+
Description-Content-Type: text/markdown
|
|
34
|
+
|
|
35
|
+
# [<img src="https://s3.internetdata.io/internetdata-public/brand/mark.svg" alt="InternetData" width="24"/>](https://internetdata.io/) InternetData Python Client Library
|
|
36
|
+
|
|
37
|
+
[](https://pypi.org/project/internetdata/)
|
|
38
|
+
[](LICENSE)
|
|
39
|
+
|
|
40
|
+
The official Python client library for the [InternetData](https://internetdata.io) API.
|
|
41
|
+
|
|
42
|
+
The library helps you browse the databases your organization is licensed for, check what is in each build before you fetch it, and download and verify the files.
|
|
43
|
+
|
|
44
|
+
## Getting Started
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
pip install internetdata
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Requires Python 3.11 or newer. Type hints are included, and the package ships `py.typed`.
|
|
51
|
+
|
|
52
|
+
## Usage
|
|
53
|
+
|
|
54
|
+
Every call needs an API key carrying the `db.download` scope. Create one in the console, then:
|
|
55
|
+
|
|
56
|
+
```python
|
|
57
|
+
import os
|
|
58
|
+
from internetdata import InternetData
|
|
59
|
+
|
|
60
|
+
client = InternetData(os.environ["INTERNETDATA_API_KEY"])
|
|
61
|
+
|
|
62
|
+
for family in client.database.list():
|
|
63
|
+
print(family.base, family.standing, [v.id for v in family.versions])
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
The client holds an HTTP connection pool, so use it as a context manager, or call `client.close()` when you are done with it:
|
|
67
|
+
|
|
68
|
+
```python
|
|
69
|
+
with InternetData(api_key) as client:
|
|
70
|
+
print(len(client.database.list()))
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
### The catalog
|
|
74
|
+
|
|
75
|
+
`list` answers database FAMILIES. A licence is held against a family, while a download names a specific version, so the ids the other calls take come from each family's `versions`:
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
for family in client.database.list():
|
|
79
|
+
if family.standing != "licensed":
|
|
80
|
+
continue
|
|
81
|
+
for version in family.versions:
|
|
82
|
+
print(version.id, version.formats) # 'bogon_ip_v1' ('csvgz', 'mmdb')
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
`standing` is `licensed`, `expired` or `unlicensed`, and `redistribution` is what your licence lets you do with the data (`evaluation`, `internal`, `redistribute`, or `None` when there is no licence). A family you have never bought is still listed, as `unlicensed`, so you can see what else exists.
|
|
86
|
+
|
|
87
|
+
**The catalog is not the same document for every key.** Databases commissioned for a single customer are absent from the listing entirely for everyone else, rather than shown as `unlicensed`. The server decides that per key, so treat what you get back as the answer for the key you asked with: this library never caches a listing, never reuses one across clients, and has no built-in list of database ids to fall back on.
|
|
88
|
+
|
|
89
|
+
### What is inside a build
|
|
90
|
+
|
|
91
|
+
`metadata` is cheap enough to poll. It answers when the build was generated, how many rows it has, its columns and a few real rows, and the size of each format in bytes, all without moving the file:
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
meta = client.database.metadata("bogon_ip_v1")
|
|
95
|
+
|
|
96
|
+
print(meta.updated) # datetime.date(2026, 9, 4)
|
|
97
|
+
print(meta.entries) # 1234
|
|
98
|
+
print(meta.size["csvgz"]) # 760
|
|
99
|
+
print([c.name for c in meta.schema["csvgz"]])
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
Checking `size` before a download is worth the round trip: the catalog spans a few hundred bytes to several gigabytes.
|
|
103
|
+
|
|
104
|
+
### Downloading
|
|
105
|
+
|
|
106
|
+
Three ways, depending on what you want to do with the bytes:
|
|
107
|
+
|
|
108
|
+
```python
|
|
109
|
+
url = client.database.download_url("bogon_ip_v1", "csvgz")
|
|
110
|
+
raw = client.database.download_bytes("bogon_ip_v1", "csvgz")
|
|
111
|
+
written = client.database.download("bogon_ip_v1", "mmdb", "./bogon_ip_v1.mmdb")
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
`download_url` hands back a time-limited link straight to object storage. It carries its own signature and none of your API key, so you can pass it to `curl`, a job runner or anything else that speaks HTTP. The link authorizes the START of a transfer, so one already running is not cut off when it lapses.
|
|
115
|
+
|
|
116
|
+
`download` streams to a file, so nothing larger than a chunk is ever held in memory. The bytes land in a neighboring `.part` file that is moved into place only once the whole transfer has arrived, so a failed refresh leaves the copy you already had untouched. `download_bytes` holds the whole file in memory, so reach for it only at the small end of the catalog.
|
|
117
|
+
|
|
118
|
+
Either way a short transfer fails rather than handing you a truncated file, and the key is never sent to object storage.
|
|
119
|
+
|
|
120
|
+
Verify what you fetched against the published digests:
|
|
121
|
+
|
|
122
|
+
```python
|
|
123
|
+
import hashlib
|
|
124
|
+
|
|
125
|
+
digests = client.database.checksums("bogon_ip_v1", "csvgz")
|
|
126
|
+
assert hashlib.sha256(raw).hexdigest() == digests["sha256"]
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
### Download history
|
|
130
|
+
|
|
131
|
+
`downloads` answers your organization's recent attempts, newest first, refusals included. A denial is what answers "it stopped working", and its absence answers nothing:
|
|
132
|
+
|
|
133
|
+
```python
|
|
134
|
+
for attempt in client.database.downloads(limit=20):
|
|
135
|
+
print(attempt.created, attempt.dataset_id, attempt.outcome, attempt.http_status)
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
### Async
|
|
139
|
+
|
|
140
|
+
Everything above works the same way under asyncio, with `AsyncInternetData`:
|
|
141
|
+
|
|
142
|
+
```python
|
|
143
|
+
import asyncio
|
|
144
|
+
from internetdata import AsyncInternetData
|
|
145
|
+
|
|
146
|
+
async def main():
|
|
147
|
+
async with AsyncInternetData(api_key) as client:
|
|
148
|
+
meta = await client.database.metadata("bogon_ip_v1")
|
|
149
|
+
await client.database.download("bogon_ip_v1", "csvgz", "./bogon_ip_v1.csv.gz")
|
|
150
|
+
print(meta.entries)
|
|
151
|
+
|
|
152
|
+
asyncio.run(main())
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
### Errors
|
|
156
|
+
|
|
157
|
+
Failures raise an `InternetDataError` carrying a `kind` and a `retryable` flag:
|
|
158
|
+
|
|
159
|
+
```python
|
|
160
|
+
from internetdata import InternetDataError
|
|
161
|
+
|
|
162
|
+
try:
|
|
163
|
+
client.database.download_url("vpn_ip_v1", "mmdb")
|
|
164
|
+
except InternetDataError as err:
|
|
165
|
+
print(err.kind, err.retryable, err.message) # forbidden False NOT_LICENSED
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
`kind` is one of `bad_request`, `unauthorized`, `forbidden`, `rate_limited`, `quota_exceeded`, `server_error` or `network`. `message` is the API's own result code, passed through as it was sent, so you can switch on `NOT_LICENSED` against `LICENSE_EXPIRED` without reading the status.
|
|
169
|
+
|
|
170
|
+
Note that `rate_limited` and `quota_exceeded` both arrive as HTTP 429 and are not the same thing. A rate limit is the API facing a traffic burst, and retrying later works; a spent quota needs your allowance raised or the window to roll over. The library retries rate limits for you, and server and network failures, but never a spent quota or anything else you sent.
|
|
171
|
+
|
|
172
|
+
## Other Libraries
|
|
173
|
+
|
|
174
|
+
There are official InternetData client libraries available for many languages including PHP, Python, Go, Java, Ruby, and many popular frameworks such as Django, Rails, and Laravel. See our GitHub at https://github.com/internetdata for more.
|
|
175
|
+
|
|
176
|
+
## About InternetData
|
|
177
|
+
|
|
178
|
+
InternetData: IP and network intelligence databases, published as CSV and MMDB, licensed by contract and downloadable on demand.
|
|
179
|
+
|
|
180
|
+
[<img src="https://s3.internetdata.io/internetdata-public/brand/mark.svg" alt="InternetData" width="96"/>](https://internetdata.io/)
|
|
181
|
+
|
|
182
|
+
## License
|
|
183
|
+
|
|
184
|
+
This project is licensed under the [MIT License](LICENSE).
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
# [<img src="https://s3.internetdata.io/internetdata-public/brand/mark.svg" alt="InternetData" width="24"/>](https://internetdata.io/) InternetData Python Client Library
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/internetdata/)
|
|
4
|
+
[](LICENSE)
|
|
5
|
+
|
|
6
|
+
The official Python client library for the [InternetData](https://internetdata.io) API.
|
|
7
|
+
|
|
8
|
+
The library helps you browse the databases your organization is licensed for, check what is in each build before you fetch it, and download and verify the files.
|
|
9
|
+
|
|
10
|
+
## Getting Started
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
pip install internetdata
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
Requires Python 3.11 or newer. Type hints are included, and the package ships `py.typed`.
|
|
17
|
+
|
|
18
|
+
## Usage
|
|
19
|
+
|
|
20
|
+
Every call needs an API key carrying the `db.download` scope. Create one in the console, then:
|
|
21
|
+
|
|
22
|
+
```python
|
|
23
|
+
import os
|
|
24
|
+
from internetdata import InternetData
|
|
25
|
+
|
|
26
|
+
client = InternetData(os.environ["INTERNETDATA_API_KEY"])
|
|
27
|
+
|
|
28
|
+
for family in client.database.list():
|
|
29
|
+
print(family.base, family.standing, [v.id for v in family.versions])
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
The client holds an HTTP connection pool, so use it as a context manager, or call `client.close()` when you are done with it:
|
|
33
|
+
|
|
34
|
+
```python
|
|
35
|
+
with InternetData(api_key) as client:
|
|
36
|
+
print(len(client.database.list()))
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
### The catalog
|
|
40
|
+
|
|
41
|
+
`list` answers database FAMILIES. A licence is held against a family, while a download names a specific version, so the ids the other calls take come from each family's `versions`:
|
|
42
|
+
|
|
43
|
+
```python
|
|
44
|
+
for family in client.database.list():
|
|
45
|
+
if family.standing != "licensed":
|
|
46
|
+
continue
|
|
47
|
+
for version in family.versions:
|
|
48
|
+
print(version.id, version.formats) # 'bogon_ip_v1' ('csvgz', 'mmdb')
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
`standing` is `licensed`, `expired` or `unlicensed`, and `redistribution` is what your licence lets you do with the data (`evaluation`, `internal`, `redistribute`, or `None` when there is no licence). A family you have never bought is still listed, as `unlicensed`, so you can see what else exists.
|
|
52
|
+
|
|
53
|
+
**The catalog is not the same document for every key.** Databases commissioned for a single customer are absent from the listing entirely for everyone else, rather than shown as `unlicensed`. The server decides that per key, so treat what you get back as the answer for the key you asked with: this library never caches a listing, never reuses one across clients, and has no built-in list of database ids to fall back on.
|
|
54
|
+
|
|
55
|
+
### What is inside a build
|
|
56
|
+
|
|
57
|
+
`metadata` is cheap enough to poll. It answers when the build was generated, how many rows it has, its columns and a few real rows, and the size of each format in bytes, all without moving the file:
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
meta = client.database.metadata("bogon_ip_v1")
|
|
61
|
+
|
|
62
|
+
print(meta.updated) # datetime.date(2026, 9, 4)
|
|
63
|
+
print(meta.entries) # 1234
|
|
64
|
+
print(meta.size["csvgz"]) # 760
|
|
65
|
+
print([c.name for c in meta.schema["csvgz"]])
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
Checking `size` before a download is worth the round trip: the catalog spans a few hundred bytes to several gigabytes.
|
|
69
|
+
|
|
70
|
+
### Downloading
|
|
71
|
+
|
|
72
|
+
Three ways, depending on what you want to do with the bytes:
|
|
73
|
+
|
|
74
|
+
```python
|
|
75
|
+
url = client.database.download_url("bogon_ip_v1", "csvgz")
|
|
76
|
+
raw = client.database.download_bytes("bogon_ip_v1", "csvgz")
|
|
77
|
+
written = client.database.download("bogon_ip_v1", "mmdb", "./bogon_ip_v1.mmdb")
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
`download_url` hands back a time-limited link straight to object storage. It carries its own signature and none of your API key, so you can pass it to `curl`, a job runner or anything else that speaks HTTP. The link authorizes the START of a transfer, so one already running is not cut off when it lapses.
|
|
81
|
+
|
|
82
|
+
`download` streams to a file, so nothing larger than a chunk is ever held in memory. The bytes land in a neighboring `.part` file that is moved into place only once the whole transfer has arrived, so a failed refresh leaves the copy you already had untouched. `download_bytes` holds the whole file in memory, so reach for it only at the small end of the catalog.
|
|
83
|
+
|
|
84
|
+
Either way a short transfer fails rather than handing you a truncated file, and the key is never sent to object storage.
|
|
85
|
+
|
|
86
|
+
Verify what you fetched against the published digests:
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
import hashlib
|
|
90
|
+
|
|
91
|
+
digests = client.database.checksums("bogon_ip_v1", "csvgz")
|
|
92
|
+
assert hashlib.sha256(raw).hexdigest() == digests["sha256"]
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
### Download history
|
|
96
|
+
|
|
97
|
+
`downloads` answers your organization's recent attempts, newest first, refusals included. A denial is what answers "it stopped working", and its absence answers nothing:
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
for attempt in client.database.downloads(limit=20):
|
|
101
|
+
print(attempt.created, attempt.dataset_id, attempt.outcome, attempt.http_status)
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
### Async
|
|
105
|
+
|
|
106
|
+
Everything above works the same way under asyncio, with `AsyncInternetData`:
|
|
107
|
+
|
|
108
|
+
```python
|
|
109
|
+
import asyncio
|
|
110
|
+
from internetdata import AsyncInternetData
|
|
111
|
+
|
|
112
|
+
async def main():
|
|
113
|
+
async with AsyncInternetData(api_key) as client:
|
|
114
|
+
meta = await client.database.metadata("bogon_ip_v1")
|
|
115
|
+
await client.database.download("bogon_ip_v1", "csvgz", "./bogon_ip_v1.csv.gz")
|
|
116
|
+
print(meta.entries)
|
|
117
|
+
|
|
118
|
+
asyncio.run(main())
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
### Errors
|
|
122
|
+
|
|
123
|
+
Failures raise an `InternetDataError` carrying a `kind` and a `retryable` flag:
|
|
124
|
+
|
|
125
|
+
```python
|
|
126
|
+
from internetdata import InternetDataError
|
|
127
|
+
|
|
128
|
+
try:
|
|
129
|
+
client.database.download_url("vpn_ip_v1", "mmdb")
|
|
130
|
+
except InternetDataError as err:
|
|
131
|
+
print(err.kind, err.retryable, err.message) # forbidden False NOT_LICENSED
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
`kind` is one of `bad_request`, `unauthorized`, `forbidden`, `rate_limited`, `quota_exceeded`, `server_error` or `network`. `message` is the API's own result code, passed through as it was sent, so you can switch on `NOT_LICENSED` against `LICENSE_EXPIRED` without reading the status.
|
|
135
|
+
|
|
136
|
+
Note that `rate_limited` and `quota_exceeded` both arrive as HTTP 429 and are not the same thing. A rate limit is the API facing a traffic burst, and retrying later works; a spent quota needs your allowance raised or the window to roll over. The library retries rate limits for you, and server and network failures, but never a spent quota or anything else you sent.
|
|
137
|
+
|
|
138
|
+
## Other Libraries
|
|
139
|
+
|
|
140
|
+
There are official InternetData client libraries available for many languages including PHP, Python, Go, Java, Ruby, and many popular frameworks such as Django, Rails, and Laravel. See our GitHub at https://github.com/internetdata for more.
|
|
141
|
+
|
|
142
|
+
## About InternetData
|
|
143
|
+
|
|
144
|
+
InternetData: IP and network intelligence databases, published as CSV and MMDB, licensed by contract and downloadable on demand.
|
|
145
|
+
|
|
146
|
+
[<img src="https://s3.internetdata.io/internetdata-public/brand/mark.svg" alt="InternetData" width="96"/>](https://internetdata.io/)
|
|
147
|
+
|
|
148
|
+
## License
|
|
149
|
+
|
|
150
|
+
This project is licensed under the [MIT License](LICENSE).
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.27"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "internetdata"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Official Python client library for the InternetData API. Download and verify licensed IP datasets."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
requires-python = ">=3.11"
|
|
13
|
+
authors = [{ name = "Mslm Dev", email = "support@internetdata.io" }]
|
|
14
|
+
keywords = [
|
|
15
|
+
"internetdata",
|
|
16
|
+
"ip-database",
|
|
17
|
+
"ip-intelligence",
|
|
18
|
+
"mmdb",
|
|
19
|
+
"geolocation",
|
|
20
|
+
"asn",
|
|
21
|
+
"bogon",
|
|
22
|
+
"vpn",
|
|
23
|
+
"proxy",
|
|
24
|
+
"hosting",
|
|
25
|
+
"dataset",
|
|
26
|
+
]
|
|
27
|
+
classifiers = [
|
|
28
|
+
"Development Status :: 5 - Production/Stable",
|
|
29
|
+
"Intended Audience :: Developers",
|
|
30
|
+
"Programming Language :: Python :: 3",
|
|
31
|
+
"Programming Language :: Python :: 3.11",
|
|
32
|
+
"Programming Language :: Python :: 3.12",
|
|
33
|
+
"Programming Language :: Python :: 3.13",
|
|
34
|
+
"Programming Language :: Python :: 3.14",
|
|
35
|
+
"Programming Language :: Python :: Implementation :: CPython",
|
|
36
|
+
"Topic :: Internet",
|
|
37
|
+
"Topic :: Security",
|
|
38
|
+
"Typing :: Typed",
|
|
39
|
+
]
|
|
40
|
+
dependencies = [
|
|
41
|
+
"httpx>=0.27,<1",
|
|
42
|
+
"attrs>=22.2.0",
|
|
43
|
+
]
|
|
44
|
+
|
|
45
|
+
[project.optional-dependencies]
|
|
46
|
+
dev = [
|
|
47
|
+
"pytest>=8.0",
|
|
48
|
+
"pytest-asyncio>=0.24",
|
|
49
|
+
"mypy>=1.11",
|
|
50
|
+
"ruff>=0.6",
|
|
51
|
+
"openapi-python-client>=0.29,<0.30",
|
|
52
|
+
]
|
|
53
|
+
|
|
54
|
+
[project.urls]
|
|
55
|
+
Homepage = "https://internetdata.io"
|
|
56
|
+
Documentation = "https://docs.internetdata.io"
|
|
57
|
+
Source = "https://github.com/internetdata/sdk-python"
|
|
58
|
+
Issues = "https://github.com/internetdata/sdk-python/issues"
|
|
59
|
+
|
|
60
|
+
[tool.hatch.version]
|
|
61
|
+
path = "src/internetdata/__init__.py"
|
|
62
|
+
|
|
63
|
+
[tool.hatch.build.targets.wheel]
|
|
64
|
+
packages = ["src/internetdata"]
|
|
65
|
+
|
|
66
|
+
# Root-anchored, because a bare "scripts" also matches integration/scripts and would ship
|
|
67
|
+
# a runner without the suite it drives.
|
|
68
|
+
[tool.hatch.build.targets.sdist]
|
|
69
|
+
include = ["/src", "/tests", "/testdata", "/spec", "/scripts", "/README.md", "/LICENSE"]
|
|
70
|
+
|
|
71
|
+
[tool.pytest.ini_options]
|
|
72
|
+
testpaths = ["tests"]
|
|
73
|
+
asyncio_mode = "auto"
|
|
74
|
+
|
|
75
|
+
# Read by `ruff`, which openapi-python-client runs over its own output, so the generated
|
|
76
|
+
# client and the hand-written layer are formatted to the same width.
|
|
77
|
+
[tool.ruff]
|
|
78
|
+
line-length = 100
|
|
79
|
+
|
|
80
|
+
[tool.ruff.lint.per-file-ignores]
|
|
81
|
+
# The generated tree is replaced wholesale by scripts/generate.sh, so a lint fix here
|
|
82
|
+
# would be undone by the next spec refresh. This one fires because the spec spells
|
|
83
|
+
# `redistribution` as a nullable enum and the generator renders that as three separate
|
|
84
|
+
# single-member classes, hence three `isinstance` arms it will not merge.
|
|
85
|
+
"src/internetdata/_generated/**/*.py" = ["SIM101"]
|
|
86
|
+
|
|
87
|
+
[tool.mypy]
|
|
88
|
+
python_version = "3.11"
|
|
89
|
+
strict = true
|
|
90
|
+
# The integration suite is checked by a second invocation rather than listed here: it
|
|
91
|
+
# carries a conftest.py of its own, and mypy cannot hold two modules of one name at once.
|
|
92
|
+
files = ["src/internetdata", "tests"]
|
|
93
|
+
|
|
94
|
+
# Generated from spec/openapi.yaml. It is checked by being regenerated, not by mypy.
|
|
95
|
+
[[tool.mypy.overrides]]
|
|
96
|
+
module = "internetdata._generated.*"
|
|
97
|
+
ignore_errors = true
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
|
|
3
|
+
# Refreshes the pinned OpenAPI spec from the published one.
|
|
4
|
+
#
|
|
5
|
+
# The copy in spec/ is what codegen reads, so a build stays reproducible and
|
|
6
|
+
# offline and the diff shows exactly which spec version produced the client. Run
|
|
7
|
+
# this deliberately, then commit the spec change alongside the regenerated
|
|
8
|
+
# client, so a reviewer sees both.
|
|
9
|
+
#
|
|
10
|
+
# The whole published document is pinned, v1 included, so the diff is against
|
|
11
|
+
# what the API actually publishes. `generate.sh` is what narrows it to v2.
|
|
12
|
+
|
|
13
|
+
set -euo pipefail
|
|
14
|
+
|
|
15
|
+
cd "$(dirname "$0")/.."
|
|
16
|
+
|
|
17
|
+
SPEC_URL="${SPEC_URL:-https://s3.internetdata.io/internetdata-public/openapi/openapi.yaml}"
|
|
18
|
+
|
|
19
|
+
curl -fsS "$SPEC_URL" -o spec/openapi.yaml
|
|
20
|
+
echo "spec/openapi.yaml <- ${SPEC_URL}"
|
|
21
|
+
grep -m1 '^ version:' spec/openapi.yaml
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
|
|
3
|
+
# Regenerates src/internetdata/_generated/ from the PINNED spec in this repo.
|
|
4
|
+
#
|
|
5
|
+
# Unlike most of the SDKs, the generated client is COMMITTED here: a Python
|
|
6
|
+
# package is installed from source, so a gitignored generated tree would mean a
|
|
7
|
+
# `pip install` of this repo produced a package that cannot import itself.
|
|
8
|
+
# Refresh it deliberately and commit the diff alongside any spec change.
|
|
9
|
+
#
|
|
10
|
+
# The generator lives in a throwaway virtualenv under /tmp rather than in the
|
|
11
|
+
# working tree, because a venv is thousands of files and one committed by
|
|
12
|
+
# accident is far more annoying to remove than to recreate.
|
|
13
|
+
#
|
|
14
|
+
# ./scripts/generate.sh
|
|
15
|
+
|
|
16
|
+
set -euo pipefail
|
|
17
|
+
|
|
18
|
+
cd "$(dirname "$0")/.."
|
|
19
|
+
|
|
20
|
+
VENV="${VENV:-/tmp/internetdata-sdk-python-codegen}"
|
|
21
|
+
GENERATOR_VERSION="${GENERATOR_VERSION:-0.29.1}"
|
|
22
|
+
SUBSET="${SUBSET:-/tmp/internetdata-sdk-python-v2.yaml}"
|
|
23
|
+
|
|
24
|
+
if [ ! -x "$VENV/bin/openapi-python-client" ] ; then
|
|
25
|
+
python3 -m venv "$VENV"
|
|
26
|
+
"$VENV/bin/pip" install --quiet --upgrade pip
|
|
27
|
+
"$VENV/bin/pip" install --quiet "openapi-python-client==${GENERATOR_VERSION}"
|
|
28
|
+
fi
|
|
29
|
+
|
|
30
|
+
# ruff must be on PATH: the generator shells out to it to format and lint its
|
|
31
|
+
# own output, and silently skips both when it cannot find it. It picks up this
|
|
32
|
+
# repo's pyproject.toml, so `requires-python` decides whether the models import
|
|
33
|
+
# `Self` from typing or from typing_extensions - which is the difference between
|
|
34
|
+
# three runtime dependencies and four.
|
|
35
|
+
export PATH="$VENV/bin:$PATH"
|
|
36
|
+
|
|
37
|
+
# The pinned spec carries v1 as well, and this client does not target it. See
|
|
38
|
+
# scripts/v2_subset.py for why the narrowing happens here and not in spec/.
|
|
39
|
+
python3 scripts/v2_subset.py spec/openapi.yaml "$SUBSET"
|
|
40
|
+
|
|
41
|
+
rm -rf src/internetdata/_generated
|
|
42
|
+
openapi-python-client generate \
|
|
43
|
+
--path "$SUBSET" \
|
|
44
|
+
--meta none \
|
|
45
|
+
--output-path src/internetdata/_generated
|
|
46
|
+
|
|
47
|
+
# The generator's own docstring templates carry em-dashes, which this codebase does
|
|
48
|
+
# not use anywhere. Normalizing them here keeps generation deterministic.
|
|
49
|
+
find src/internetdata/_generated -name '*.py' -exec sed -i 's/\xe2\x80\x94/ - /g' {} +
|
|
50
|
+
|
|
51
|
+
echo "src/internetdata/_generated <- spec/openapi.yaml"
|
|
52
|
+
grep -m1 '^ version:' spec/openapi.yaml
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
|
|
3
|
+
# Publishes the package to PyPI from inside the official Python image, so a
|
|
4
|
+
# release needs nothing installed locally beyond docker and works identically on
|
|
5
|
+
# any machine. The release workflow does the same steps on a tag; this is the
|
|
6
|
+
# manual path for a first release or when Actions is not an option.
|
|
7
|
+
#
|
|
8
|
+
# PYPI_TOKEN=... ./scripts/publish.sh # publish
|
|
9
|
+
# DRY_RUN=1 ./scripts/publish.sh # build + check, upload nothing
|
|
10
|
+
#
|
|
11
|
+
# A token is only needed for the FIRST publish: PyPI configures trusted
|
|
12
|
+
# publishing against an EXISTING project, so there is nothing to attach an OIDC
|
|
13
|
+
# publisher to until the name is taken. Once release.yml has published once with
|
|
14
|
+
# a trusted publisher configured, the token can be deleted.
|
|
15
|
+
#
|
|
16
|
+
# The dist/ directory is an anonymous volume so a build inside the container
|
|
17
|
+
# cannot leave root-owned artifacts in the working tree.
|
|
18
|
+
|
|
19
|
+
set -euo pipefail
|
|
20
|
+
|
|
21
|
+
cd "$(dirname "$0")/.."
|
|
22
|
+
|
|
23
|
+
PYTHON_IMAGE="${PYTHON_IMAGE:-python:3.13-slim}"
|
|
24
|
+
DRY_RUN="${DRY_RUN:-}"
|
|
25
|
+
|
|
26
|
+
if [ -z "$DRY_RUN" ] ; then
|
|
27
|
+
: "${PYPI_TOKEN:?set PYPI_TOKEN to a PyPI API token, or set DRY_RUN=1 to rehearse}"
|
|
28
|
+
upload="twine upload dist/*"
|
|
29
|
+
else
|
|
30
|
+
PYPI_TOKEN=""
|
|
31
|
+
upload="echo 'DRY_RUN: built and checked, uploading nothing'"
|
|
32
|
+
fi
|
|
33
|
+
|
|
34
|
+
docker run --rm \
|
|
35
|
+
-v "$PWD:/w" -v /w/dist \
|
|
36
|
+
-w /w \
|
|
37
|
+
-e TWINE_USERNAME=__token__ \
|
|
38
|
+
-e TWINE_PASSWORD="$PYPI_TOKEN" \
|
|
39
|
+
-e PIP_ROOT_USER_ACTION=ignore \
|
|
40
|
+
"$PYTHON_IMAGE" sh -euc "
|
|
41
|
+
pip install --quiet --upgrade pip build twine
|
|
42
|
+
pip install --quiet -e '.[dev]'
|
|
43
|
+
pytest -q
|
|
44
|
+
python -m build
|
|
45
|
+
twine check dist/*
|
|
46
|
+
$upload
|
|
47
|
+
"
|