urbandata 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- urbandata-0.0.1/PKG-INFO +267 -0
- urbandata-0.0.1/README.md +253 -0
- urbandata-0.0.1/pyproject.toml +25 -0
- urbandata-0.0.1/setup.cfg +4 -0
- urbandata-0.0.1/setup.py +17 -0
- urbandata-0.0.1/src/udmp/__init__.py +14 -0
- urbandata-0.0.1/src/urbandata/__init__.py +29 -0
- urbandata-0.0.1/src/urbandata/api/__init__.py +21 -0
- urbandata-0.0.1/src/urbandata/api/base.py +3 -0
- urbandata-0.0.1/src/urbandata/api/datasets.py +37 -0
- urbandata-0.0.1/src/urbandata/api/extensions.py +39 -0
- urbandata-0.0.1/src/urbandata/api/files.py +111 -0
- urbandata-0.0.1/src/urbandata/api/folders.py +21 -0
- urbandata-0.0.1/src/urbandata/api/processing.py +56 -0
- urbandata-0.0.1/src/urbandata/api/resources.py +29 -0
- urbandata-0.0.1/src/urbandata/api/sensors.py +44 -0
- urbandata-0.0.1/src/urbandata/api/serve.py +35 -0
- urbandata-0.0.1/src/urbandata/api/tables.py +46 -0
- urbandata-0.0.1/src/urbandata/auth/__init__.py +5 -0
- urbandata-0.0.1/src/urbandata/auth/oidc.py +107 -0
- urbandata-0.0.1/src/urbandata/auth/password.py +27 -0
- urbandata-0.0.1/src/urbandata/auth/token.py +8 -0
- urbandata-0.0.1/src/urbandata/client.py +99 -0
- urbandata-0.0.1/src/urbandata/config.py +92 -0
- urbandata-0.0.1/src/urbandata/errors.py +61 -0
- urbandata-0.0.1/src/urbandata/progress.py +52 -0
- urbandata-0.0.1/src/urbandata/transport.py +36 -0
- urbandata-0.0.1/src/urbandata/utils/__init__.py +18 -0
- urbandata-0.0.1/src/urbandata/utils/http.py +34 -0
- urbandata-0.0.1/src/urbandata/utils/network.py +20 -0
- urbandata-0.0.1/src/urbandata/utils/records.py +12 -0
- urbandata-0.0.1/src/urbandata/utils/timestamps.py +17 -0
- urbandata-0.0.1/src/urbandata.egg-info/PKG-INFO +267 -0
- urbandata-0.0.1/src/urbandata.egg-info/SOURCES.txt +51 -0
- urbandata-0.0.1/src/urbandata.egg-info/dependency_links.txt +1 -0
- urbandata-0.0.1/src/urbandata.egg-info/not-zip-safe +1 -0
- urbandata-0.0.1/src/urbandata.egg-info/requires.txt +5 -0
- urbandata-0.0.1/src/urbandata.egg-info/top_level.txt +2 -0
- urbandata-0.0.1/tests/test_auth.py +242 -0
- urbandata-0.0.1/tests/test_catalog.py +195 -0
- urbandata-0.0.1/tests/test_client.py +230 -0
- urbandata-0.0.1/tests/test_compat.py +25 -0
- urbandata-0.0.1/tests/test_config.py +116 -0
- urbandata-0.0.1/tests/test_errors.py +56 -0
- urbandata-0.0.1/tests/test_examples.py +75 -0
- urbandata-0.0.1/tests/test_files.py +250 -0
- urbandata-0.0.1/tests/test_processing.py +64 -0
- urbandata-0.0.1/tests/test_progress.py +52 -0
- urbandata-0.0.1/tests/test_sensors.py +81 -0
- urbandata-0.0.1/tests/test_serve.py +53 -0
- urbandata-0.0.1/tests/test_tables.py +46 -0
- urbandata-0.0.1/tests/test_transport.py +82 -0
- urbandata-0.0.1/tests/test_utils.py +95 -0
urbandata-0.0.1/PKG-INFO
ADDED
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: urbandata
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: Python SDK for the Urban Data Management Platform (UDMP)
|
|
5
|
+
Home-page: https://github.com/SLG-European-Projects/udmp-sdk-python
|
|
6
|
+
Author-email: Iason Sotiropoulos <isotiropoulos@singularlogic.eu>, Panagiotis Iatrou <piatrou@singularlogic.eu>
|
|
7
|
+
Requires-Python: >=3.9
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
Requires-Dist: requests>=2.31
|
|
10
|
+
Requires-Dist: tqdm>=4.66
|
|
11
|
+
Provides-Extra: dev
|
|
12
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
13
|
+
Dynamic: home-page
|
|
14
|
+
|
|
15
|
+
# urbandata
|
|
16
|
+
|
|
17
|
+
Python SDK for the Urban Data Management Platform (UDMP): datasets, folders, resources, processing
|
|
18
|
+
chains, files, sensor measurements and feature tables, on the control plane and the data plane.
|
|
19
|
+
|
|
20
|
+
- Interactive login against the platform's OIDC provider; the token is kept and refreshed for you
|
|
21
|
+
- Streamed uploads and resumable downloads, with progress bars
|
|
22
|
+
- Typed errors, and API objects that read as dicts or as attributes
|
|
23
|
+
|
|
24
|
+
## Install
|
|
25
|
+
|
|
26
|
+
Python 3.9 or newer.
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
git clone <repository-url> udmp-sdk-python
|
|
30
|
+
cd udmp-sdk-python
|
|
31
|
+
pip install -e .
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Or only the dependencies (`requests`, `tqdm`), to use the sources in place:
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
pip install -r requirements.txt
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## Configuration
|
|
41
|
+
|
|
42
|
+
Where to log in is not passed in code. The SDK reads it from the environment:
|
|
43
|
+
|
|
44
|
+
| Variable | Holds |
|
|
45
|
+
|---|---|
|
|
46
|
+
| `UDMP_OIDC_URL` | the issuer URL of the realm (`http://keycloak.localhost:8080/realms/ENHANCE`), or its token endpoint |
|
|
47
|
+
| `UDMP_CLIENT_ID` | the OIDC client to log in through (`control-plane` on a standard installation) |
|
|
48
|
+
| `UDMP_CLIENT_SECRET` | optional: the secret, when the client is confidential |
|
|
49
|
+
|
|
50
|
+
or from a JSON file named `.udmp`, in the working directory or in your home folder
|
|
51
|
+
(copy [`.udmp.example`](.udmp.example)):
|
|
52
|
+
|
|
53
|
+
```json
|
|
54
|
+
{
|
|
55
|
+
"oidc_url": "http://keycloak.localhost:8080/realms/ENHANCE",
|
|
56
|
+
"client_id": "control-plane",
|
|
57
|
+
"client_secret": "only-for-a-confidential-client"
|
|
58
|
+
}
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Each setting is taken from the first place that has it: an argument to `UrbanDataClient` (`oidc_url=`,
|
|
62
|
+
`client_id=`, `client_secret=`), the environment variable, the file. `UDMP_CONFIG_FILE` or
|
|
63
|
+
`UrbanDataClient(..., config_file=...)` names a file elsewhere. When the OIDC URL or the client id is in none
|
|
64
|
+
of them, `UrbanDataClient(...)` raises `ConfigError`, which says to set the environment variables or the
|
|
65
|
+
`.udmp` file.
|
|
66
|
+
|
|
67
|
+
`.udmp` is in `.gitignore`: it can hold a secret, so keep it out of version control.
|
|
68
|
+
|
|
69
|
+
## Quick start
|
|
70
|
+
|
|
71
|
+
```python
|
|
72
|
+
from urbandata import UrbanDataClient
|
|
73
|
+
|
|
74
|
+
client = UrbanDataClient(
|
|
75
|
+
control_plane_url="http://control-plane.localhost:8080",
|
|
76
|
+
data_plane_url="http://data-plane.localhost:8080",
|
|
77
|
+
)
|
|
78
|
+
client.login() # asks: UDMP username, UDMP password
|
|
79
|
+
|
|
80
|
+
dataset = client.datasets.create("Chlorophyll", owner_org="slg", visibility="private")
|
|
81
|
+
resource = client.resources.create(dataset.identifier, "mosaic", processing=["raster-to-tiles"])
|
|
82
|
+
|
|
83
|
+
client.files.upload(resource.identifier, "mosaic_chl_oc3.tif")
|
|
84
|
+
client.processing.wait(resource.identifier)
|
|
85
|
+
client.files.download_all(resource.identifier, "out/", step="raster-to-tiles")
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
## Authentication
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
client = UrbanDataClient(control_plane_url, data_plane_url)
|
|
92
|
+
|
|
93
|
+
client.login() # asks for the username and the password
|
|
94
|
+
client.login("alice") # asks for the password only
|
|
95
|
+
client.login("alice", "secret") # asks for nothing (scripts, CI)
|
|
96
|
+
|
|
97
|
+
client.token # the access token; None before login()
|
|
98
|
+
client.logged_in # True once there is a token
|
|
99
|
+
client.whoami() # user_id, username, email, name, roles, organizations
|
|
100
|
+
client.logout() # forgets the token
|
|
101
|
+
client.ping() # both planes answer? needs no login
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
- The password is not echoed when typed on a terminal. IDE run consoles and notebooks have no
|
|
105
|
+
terminal, so there it is read with a plain prompt and is visible.
|
|
106
|
+
- The token is sent with every call and refreshed on its own shortly before it expires. When the
|
|
107
|
+
session itself has expired, the next call raises `UnauthorizedError` and `login()` is needed again.
|
|
108
|
+
- A call made before `login()` raises `UnauthorizedError` without contacting the server.
|
|
109
|
+
- `login()` returns the access token.
|
|
110
|
+
|
|
111
|
+
A token obtained elsewhere can be used instead of logging in. It needs no OIDC settings:
|
|
112
|
+
|
|
113
|
+
```python
|
|
114
|
+
client = UrbanDataClient(control_plane_url, data_plane_url, token="eyJ...")
|
|
115
|
+
client.token = "eyJ..." # or set it later
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
Other parameters of `UrbanDataClient`: `timeout` in seconds (default 30), `session` (a `requests.Session` of
|
|
119
|
+
your own) and `progress` (see below).
|
|
120
|
+
|
|
121
|
+
Auth objects can still be passed, as `auth=` or as the third argument:
|
|
122
|
+
|
|
123
|
+
| Object | Use |
|
|
124
|
+
|---|---|
|
|
125
|
+
| `OidcAuth(oidc_url, client_id, client_secret=None, token=None)` | what `UrbanDataClient` creates from the configuration: `login()`, `refresh()`, `logout()` |
|
|
126
|
+
| `PasswordAuth(keycloak_url, realm, username, password, client_id="control-plane", client_secret=None)` | credentials known up front, token fetched on first use |
|
|
127
|
+
| `TokenAuth(token)` | a token string, or a callable returning one |
|
|
128
|
+
|
|
129
|
+
```python
|
|
130
|
+
from urbandata import UrbanDataClient, PasswordAuth
|
|
131
|
+
|
|
132
|
+
client = UrbanDataClient(control_plane_url, data_plane_url,
|
|
133
|
+
PasswordAuth("http://keycloak.localhost:8080", "ENHANCE", "alice", "secret"))
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
## API
|
|
137
|
+
|
|
138
|
+
| Attribute | Covers | Methods |
|
|
139
|
+
|---|---|---|
|
|
140
|
+
| `client.datasets` | datasets and their shares | `list` `get` `create` `update` `delete` `restore` `shares` `share` `unshare` |
|
|
141
|
+
| `client.folders` | folders | `list` `get` `create` `rename` `delete` |
|
|
142
|
+
| `client.resources` | resources, including sensor resources | `list` `get` `create` `update` `delete` |
|
|
143
|
+
| `client.extensions` | the extension registry (admin) | `list` `get` `register` `update` `delete` |
|
|
144
|
+
| `client.processing` | a resource's chain and its runs | `get` `set` `enable` `runs` `run` `trigger` `cancel` `resume` `wait` |
|
|
145
|
+
| `client.files` | uploads and downloads, raw or per `step` | `list` `upload` `download_url` `download` `download_all` `delete_all` |
|
|
146
|
+
| `client.sensors` | device, current values, measurements | `device` `current` `measurements` `measurement` `add_measurement` `add_measurements` `delete_measurement` |
|
|
147
|
+
| `client.tables` | tables, GeoJSON features, vector tiles | `list` `get` `features` `tile` `create` `insert_features` `drop` |
|
|
148
|
+
| `client.serve` | the routes a step's extension serves over its outputs (tiles, metadata, statistics) | `routes` `get` `json` `content` |
|
|
149
|
+
|
|
150
|
+
Objects come back as `Record`: the API's JSON as a dict whose keys also read as attributes
|
|
151
|
+
(`dataset["identifier"]` and `dataset.identifier`).
|
|
152
|
+
|
|
153
|
+
Notes:
|
|
154
|
+
|
|
155
|
+
- `step=` on `files`, `tables` and `sensors` selects what that extension of the processing chain
|
|
156
|
+
produced, instead of the resource's own data.
|
|
157
|
+
- `files.upload` streams the file from disk. `files.download` writes to `<dest>.part`, resumes
|
|
158
|
+
from it after an interruption (up to `max_retries` attempts) and renames it when complete.
|
|
159
|
+
- `processing.wait` blocks until the chain completes and returns its state.
|
|
160
|
+
- `sensors.add_measurement` takes a `datetime` or an ISO 8601 UTC string for the observation time;
|
|
161
|
+
`sensors.add_measurements` sends a list in one call.
|
|
162
|
+
- `serve.routes(resource_id)` lists what each step offers; `serve.json(resource_id, step, "info")` and
|
|
163
|
+
`serve.content(resource_id, step, "tiles/8/140/98", rescale=["0,3000"])` call a route, with the
|
|
164
|
+
query parameters it declares as keywords.
|
|
165
|
+
- `delete` on datasets and resources is a soft delete unless `permanent=True`; `datasets.restore`
|
|
166
|
+
undoes a soft delete.
|
|
167
|
+
- `resources.list` needs the dataset: `client.resources.list(dataset_id, folder_id=...)`.
|
|
168
|
+
- Rights: creating a dataset needs membership of the owning organisation; updating, sharing and
|
|
169
|
+
deleting it need an admin of that organisation; restoring and deleting permanently need a platform
|
|
170
|
+
admin (realm role `admin`). Without the right, the call raises `ForbiddenError`.
|
|
171
|
+
|
|
172
|
+
## Helpers
|
|
173
|
+
|
|
174
|
+
`urbandata.utils` holds the functions the package shares, usable in your own scripts:
|
|
175
|
+
|
|
176
|
+
| Function | Does |
|
|
177
|
+
|---|---|
|
|
178
|
+
| `fix_localhost()` | resolves `*.localhost` to 127.0.0.1 for this process; needed on Windows for a platform on a local cluster |
|
|
179
|
+
| `utc_timestamp(when=None)` | a `datetime` (now by default) as the `2026-10-01T12:00:00Z` form the platform takes |
|
|
180
|
+
| `Record`, `records(items)` | the dict with attribute access that the API objects are |
|
|
181
|
+
| `clean`, `csv`, `seg`, `quote_path`, `presigned_headers` | building query parameters, URL paths and storage requests |
|
|
182
|
+
|
|
183
|
+
## Progress bars
|
|
184
|
+
|
|
185
|
+
`files.upload`, `files.download`, `files.download_all` and `processing.wait` show a
|
|
186
|
+
[tqdm](https://tqdm.github.io) progress bar (bytes for transfers, completed steps for processing).
|
|
187
|
+
It renders in terminals and in notebooks.
|
|
188
|
+
|
|
189
|
+
```python
|
|
190
|
+
client = UrbanDataClient(control_plane_url, data_plane_url, progress=False) # off for this client
|
|
191
|
+
client.files.upload(resource_id, "big.tif", progress=True) # on for this call
|
|
192
|
+
client.files.download_all(resource_id, "out/", progress=False) # off for this call
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
## Errors
|
|
196
|
+
|
|
197
|
+
| Exception | Raised when |
|
|
198
|
+
|---|---|
|
|
199
|
+
| `UrbanDataError` | any failed call; base class of the rest. Carries `status`, `code`, `body` |
|
|
200
|
+
| `ConfigError` | the OIDC URL or the client id is in neither the environment nor a `.udmp` file |
|
|
201
|
+
| `UnauthorizedError` | 401, a failed login, a call before `login()`, an expired session |
|
|
202
|
+
| `ForbiddenError` | 403 |
|
|
203
|
+
| `NotFoundError` | 404 |
|
|
204
|
+
| `ConflictError` | 409 |
|
|
205
|
+
| `ProcessingFailed` | `processing.wait()` and the chain failed. Carries `resource_id`, `state` |
|
|
206
|
+
| `ProcessingTimeout` | `processing.wait()` and the chain did not finish within `timeout_seconds` |
|
|
207
|
+
|
|
208
|
+
```python
|
|
209
|
+
from urbandata import NotFoundError, UrbanDataError
|
|
210
|
+
|
|
211
|
+
try:
|
|
212
|
+
client.datasets.get(dataset_id)
|
|
213
|
+
except NotFoundError:
|
|
214
|
+
...
|
|
215
|
+
except UrbanDataError as e:
|
|
216
|
+
print(e.status, e.code, e.body)
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
## Examples
|
|
220
|
+
|
|
221
|
+
Runnable scripts in [`examples/`](examples). Besides the configuration above, they read
|
|
222
|
+
`UDMP_CONTROL_PLANE_URL` and `UDMP_DATA_PLANE_URL` (default: the local development stack) and
|
|
223
|
+
optionally `UDMP_USERNAME` and `UDMP_ORG`, and ask for the password.
|
|
224
|
+
|
|
225
|
+
| Script | Shows |
|
|
226
|
+
|---|---|
|
|
227
|
+
| `01_login.py` | ping, interactive login, the kept token, whoami, reusing a token, logout |
|
|
228
|
+
| `02_datasets_and_folders.py` | datasets, nested folders, resources, shares, soft and permanent delete |
|
|
229
|
+
| `03_upload_process_download.py` | upload, wait for processing, download the outputs, with progress bars |
|
|
230
|
+
| `04_sensors.py` | a sensor resource, adding measurements one by one and in a batch, queries, paging |
|
|
231
|
+
| `05_tables.py` | creating a table, inserting GeoJSON, bbox queries, vector tiles |
|
|
232
|
+
| `06_extensions.py` | the extension registry and a resource's processing chain |
|
|
233
|
+
| `07_errors.py` | the exceptions and how to handle them |
|
|
234
|
+
| `08_serve.py` | listing and calling the routes of a processing step |
|
|
235
|
+
|
|
236
|
+
```bash
|
|
237
|
+
python examples/01_login.py
|
|
238
|
+
python examples/03_upload_process_download.py mosaic_chl_oc3.tif
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
## Development
|
|
242
|
+
|
|
243
|
+
```bash
|
|
244
|
+
pip install -e .
|
|
245
|
+
pip install -r requirements-dev.txt
|
|
246
|
+
pytest
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
The tests need no running platform: HTTP is replaced by fakes (`tests/conftest.py`). They cover
|
|
250
|
+
every module of the package, and `tests/test_examples.py` runs the numbered examples against a
|
|
251
|
+
stand-in platform so that they stay in step with the SDK.
|
|
252
|
+
|
|
253
|
+
```
|
|
254
|
+
src/urbandata/
|
|
255
|
+
client.py UrbanDataClient: the URLs, login(), token, whoami(), ping(), and the API attributes
|
|
256
|
+
config.py the OIDC settings from the environment or a .udmp file
|
|
257
|
+
transport.py HTTP to the two planes, bearer token
|
|
258
|
+
errors.py the exceptions
|
|
259
|
+
progress.py tqdm bars
|
|
260
|
+
auth/ OidcAuth (login), PasswordAuth, TokenAuth
|
|
261
|
+
api/ one module per API attribute: datasets, folders, resources, extensions,
|
|
262
|
+
processing, files, sensors, tables, serve
|
|
263
|
+
utils/ shared helpers: records, http, timestamps, network
|
|
264
|
+
tests/ pytest suite
|
|
265
|
+
examples/ runnable scripts
|
|
266
|
+
src/udmp/ the old import name, kept working: Udmp, UdmpError
|
|
267
|
+
```
|
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
# urbandata
|
|
2
|
+
|
|
3
|
+
Python SDK for the Urban Data Management Platform (UDMP): datasets, folders, resources, processing
|
|
4
|
+
chains, files, sensor measurements and feature tables, on the control plane and the data plane.
|
|
5
|
+
|
|
6
|
+
- Interactive login against the platform's OIDC provider; the token is kept and refreshed for you
|
|
7
|
+
- Streamed uploads and resumable downloads, with progress bars
|
|
8
|
+
- Typed errors, and API objects that read as dicts or as attributes
|
|
9
|
+
|
|
10
|
+
## Install
|
|
11
|
+
|
|
12
|
+
Python 3.9 or newer.
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
git clone <repository-url> udmp-sdk-python
|
|
16
|
+
cd udmp-sdk-python
|
|
17
|
+
pip install -e .
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
Or only the dependencies (`requests`, `tqdm`), to use the sources in place:
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
pip install -r requirements.txt
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## Configuration
|
|
27
|
+
|
|
28
|
+
Where to log in is not passed in code. The SDK reads it from the environment:
|
|
29
|
+
|
|
30
|
+
| Variable | Holds |
|
|
31
|
+
|---|---|
|
|
32
|
+
| `UDMP_OIDC_URL` | the issuer URL of the realm (`http://keycloak.localhost:8080/realms/ENHANCE`), or its token endpoint |
|
|
33
|
+
| `UDMP_CLIENT_ID` | the OIDC client to log in through (`control-plane` on a standard installation) |
|
|
34
|
+
| `UDMP_CLIENT_SECRET` | optional: the secret, when the client is confidential |
|
|
35
|
+
|
|
36
|
+
or from a JSON file named `.udmp`, in the working directory or in your home folder
|
|
37
|
+
(copy [`.udmp.example`](.udmp.example)):
|
|
38
|
+
|
|
39
|
+
```json
|
|
40
|
+
{
|
|
41
|
+
"oidc_url": "http://keycloak.localhost:8080/realms/ENHANCE",
|
|
42
|
+
"client_id": "control-plane",
|
|
43
|
+
"client_secret": "only-for-a-confidential-client"
|
|
44
|
+
}
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Each setting is taken from the first place that has it: an argument to `UrbanDataClient` (`oidc_url=`,
|
|
48
|
+
`client_id=`, `client_secret=`), the environment variable, the file. `UDMP_CONFIG_FILE` or
|
|
49
|
+
`UrbanDataClient(..., config_file=...)` names a file elsewhere. When the OIDC URL or the client id is in none
|
|
50
|
+
of them, `UrbanDataClient(...)` raises `ConfigError`, which says to set the environment variables or the
|
|
51
|
+
`.udmp` file.
|
|
52
|
+
|
|
53
|
+
`.udmp` is in `.gitignore`: it can hold a secret, so keep it out of version control.
|
|
54
|
+
|
|
55
|
+
## Quick start
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
from urbandata import UrbanDataClient
|
|
59
|
+
|
|
60
|
+
client = UrbanDataClient(
|
|
61
|
+
control_plane_url="http://control-plane.localhost:8080",
|
|
62
|
+
data_plane_url="http://data-plane.localhost:8080",
|
|
63
|
+
)
|
|
64
|
+
client.login() # asks: UDMP username, UDMP password
|
|
65
|
+
|
|
66
|
+
dataset = client.datasets.create("Chlorophyll", owner_org="slg", visibility="private")
|
|
67
|
+
resource = client.resources.create(dataset.identifier, "mosaic", processing=["raster-to-tiles"])
|
|
68
|
+
|
|
69
|
+
client.files.upload(resource.identifier, "mosaic_chl_oc3.tif")
|
|
70
|
+
client.processing.wait(resource.identifier)
|
|
71
|
+
client.files.download_all(resource.identifier, "out/", step="raster-to-tiles")
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
## Authentication
|
|
75
|
+
|
|
76
|
+
```python
|
|
77
|
+
client = UrbanDataClient(control_plane_url, data_plane_url)
|
|
78
|
+
|
|
79
|
+
client.login() # asks for the username and the password
|
|
80
|
+
client.login("alice") # asks for the password only
|
|
81
|
+
client.login("alice", "secret") # asks for nothing (scripts, CI)
|
|
82
|
+
|
|
83
|
+
client.token # the access token; None before login()
|
|
84
|
+
client.logged_in # True once there is a token
|
|
85
|
+
client.whoami() # user_id, username, email, name, roles, organizations
|
|
86
|
+
client.logout() # forgets the token
|
|
87
|
+
client.ping() # both planes answer? needs no login
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
- The password is not echoed when typed on a terminal. IDE run consoles and notebooks have no
|
|
91
|
+
terminal, so there it is read with a plain prompt and is visible.
|
|
92
|
+
- The token is sent with every call and refreshed on its own shortly before it expires. When the
|
|
93
|
+
session itself has expired, the next call raises `UnauthorizedError` and `login()` is needed again.
|
|
94
|
+
- A call made before `login()` raises `UnauthorizedError` without contacting the server.
|
|
95
|
+
- `login()` returns the access token.
|
|
96
|
+
|
|
97
|
+
A token obtained elsewhere can be used instead of logging in. It needs no OIDC settings:
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
client = UrbanDataClient(control_plane_url, data_plane_url, token="eyJ...")
|
|
101
|
+
client.token = "eyJ..." # or set it later
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
Other parameters of `UrbanDataClient`: `timeout` in seconds (default 30), `session` (a `requests.Session` of
|
|
105
|
+
your own) and `progress` (see below).
|
|
106
|
+
|
|
107
|
+
Auth objects can still be passed, as `auth=` or as the third argument:
|
|
108
|
+
|
|
109
|
+
| Object | Use |
|
|
110
|
+
|---|---|
|
|
111
|
+
| `OidcAuth(oidc_url, client_id, client_secret=None, token=None)` | what `UrbanDataClient` creates from the configuration: `login()`, `refresh()`, `logout()` |
|
|
112
|
+
| `PasswordAuth(keycloak_url, realm, username, password, client_id="control-plane", client_secret=None)` | credentials known up front, token fetched on first use |
|
|
113
|
+
| `TokenAuth(token)` | a token string, or a callable returning one |
|
|
114
|
+
|
|
115
|
+
```python
|
|
116
|
+
from urbandata import UrbanDataClient, PasswordAuth
|
|
117
|
+
|
|
118
|
+
client = UrbanDataClient(control_plane_url, data_plane_url,
|
|
119
|
+
PasswordAuth("http://keycloak.localhost:8080", "ENHANCE", "alice", "secret"))
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
## API
|
|
123
|
+
|
|
124
|
+
| Attribute | Covers | Methods |
|
|
125
|
+
|---|---|---|
|
|
126
|
+
| `client.datasets` | datasets and their shares | `list` `get` `create` `update` `delete` `restore` `shares` `share` `unshare` |
|
|
127
|
+
| `client.folders` | folders | `list` `get` `create` `rename` `delete` |
|
|
128
|
+
| `client.resources` | resources, including sensor resources | `list` `get` `create` `update` `delete` |
|
|
129
|
+
| `client.extensions` | the extension registry (admin) | `list` `get` `register` `update` `delete` |
|
|
130
|
+
| `client.processing` | a resource's chain and its runs | `get` `set` `enable` `runs` `run` `trigger` `cancel` `resume` `wait` |
|
|
131
|
+
| `client.files` | uploads and downloads, raw or per `step` | `list` `upload` `download_url` `download` `download_all` `delete_all` |
|
|
132
|
+
| `client.sensors` | device, current values, measurements | `device` `current` `measurements` `measurement` `add_measurement` `add_measurements` `delete_measurement` |
|
|
133
|
+
| `client.tables` | tables, GeoJSON features, vector tiles | `list` `get` `features` `tile` `create` `insert_features` `drop` |
|
|
134
|
+
| `client.serve` | the routes a step's extension serves over its outputs (tiles, metadata, statistics) | `routes` `get` `json` `content` |
|
|
135
|
+
|
|
136
|
+
Objects come back as `Record`: the API's JSON as a dict whose keys also read as attributes
|
|
137
|
+
(`dataset["identifier"]` and `dataset.identifier`).
|
|
138
|
+
|
|
139
|
+
Notes:
|
|
140
|
+
|
|
141
|
+
- `step=` on `files`, `tables` and `sensors` selects what that extension of the processing chain
|
|
142
|
+
produced, instead of the resource's own data.
|
|
143
|
+
- `files.upload` streams the file from disk. `files.download` writes to `<dest>.part`, resumes
|
|
144
|
+
from it after an interruption (up to `max_retries` attempts) and renames it when complete.
|
|
145
|
+
- `processing.wait` blocks until the chain completes and returns its state.
|
|
146
|
+
- `sensors.add_measurement` takes a `datetime` or an ISO 8601 UTC string for the observation time;
|
|
147
|
+
`sensors.add_measurements` sends a list in one call.
|
|
148
|
+
- `serve.routes(resource_id)` lists what each step offers; `serve.json(resource_id, step, "info")` and
|
|
149
|
+
`serve.content(resource_id, step, "tiles/8/140/98", rescale=["0,3000"])` call a route, with the
|
|
150
|
+
query parameters it declares as keywords.
|
|
151
|
+
- `delete` on datasets and resources is a soft delete unless `permanent=True`; `datasets.restore`
|
|
152
|
+
undoes a soft delete.
|
|
153
|
+
- `resources.list` needs the dataset: `client.resources.list(dataset_id, folder_id=...)`.
|
|
154
|
+
- Rights: creating a dataset needs membership of the owning organisation; updating, sharing and
|
|
155
|
+
deleting it need an admin of that organisation; restoring and deleting permanently need a platform
|
|
156
|
+
admin (realm role `admin`). Without the right, the call raises `ForbiddenError`.
|
|
157
|
+
|
|
158
|
+
## Helpers
|
|
159
|
+
|
|
160
|
+
`urbandata.utils` holds the functions the package shares, usable in your own scripts:
|
|
161
|
+
|
|
162
|
+
| Function | Does |
|
|
163
|
+
|---|---|
|
|
164
|
+
| `fix_localhost()` | resolves `*.localhost` to 127.0.0.1 for this process; needed on Windows for a platform on a local cluster |
|
|
165
|
+
| `utc_timestamp(when=None)` | a `datetime` (now by default) as the `2026-10-01T12:00:00Z` form the platform takes |
|
|
166
|
+
| `Record`, `records(items)` | the dict with attribute access that the API objects are |
|
|
167
|
+
| `clean`, `csv`, `seg`, `quote_path`, `presigned_headers` | building query parameters, URL paths and storage requests |
|
|
168
|
+
|
|
169
|
+
## Progress bars
|
|
170
|
+
|
|
171
|
+
`files.upload`, `files.download`, `files.download_all` and `processing.wait` show a
|
|
172
|
+
[tqdm](https://tqdm.github.io) progress bar (bytes for transfers, completed steps for processing).
|
|
173
|
+
It renders in terminals and in notebooks.
|
|
174
|
+
|
|
175
|
+
```python
|
|
176
|
+
client = UrbanDataClient(control_plane_url, data_plane_url, progress=False) # off for this client
|
|
177
|
+
client.files.upload(resource_id, "big.tif", progress=True) # on for this call
|
|
178
|
+
client.files.download_all(resource_id, "out/", progress=False) # off for this call
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
## Errors
|
|
182
|
+
|
|
183
|
+
| Exception | Raised when |
|
|
184
|
+
|---|---|
|
|
185
|
+
| `UrbanDataError` | any failed call; base class of the rest. Carries `status`, `code`, `body` |
|
|
186
|
+
| `ConfigError` | the OIDC URL or the client id is in neither the environment nor a `.udmp` file |
|
|
187
|
+
| `UnauthorizedError` | 401, a failed login, a call before `login()`, an expired session |
|
|
188
|
+
| `ForbiddenError` | 403 |
|
|
189
|
+
| `NotFoundError` | 404 |
|
|
190
|
+
| `ConflictError` | 409 |
|
|
191
|
+
| `ProcessingFailed` | `processing.wait()` and the chain failed. Carries `resource_id`, `state` |
|
|
192
|
+
| `ProcessingTimeout` | `processing.wait()` and the chain did not finish within `timeout_seconds` |
|
|
193
|
+
|
|
194
|
+
```python
|
|
195
|
+
from urbandata import NotFoundError, UrbanDataError
|
|
196
|
+
|
|
197
|
+
try:
|
|
198
|
+
client.datasets.get(dataset_id)
|
|
199
|
+
except NotFoundError:
|
|
200
|
+
...
|
|
201
|
+
except UrbanDataError as e:
|
|
202
|
+
print(e.status, e.code, e.body)
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
## Examples
|
|
206
|
+
|
|
207
|
+
Runnable scripts in [`examples/`](examples). Besides the configuration above, they read
|
|
208
|
+
`UDMP_CONTROL_PLANE_URL` and `UDMP_DATA_PLANE_URL` (default: the local development stack) and
|
|
209
|
+
optionally `UDMP_USERNAME` and `UDMP_ORG`, and ask for the password.
|
|
210
|
+
|
|
211
|
+
| Script | Shows |
|
|
212
|
+
|---|---|
|
|
213
|
+
| `01_login.py` | ping, interactive login, the kept token, whoami, reusing a token, logout |
|
|
214
|
+
| `02_datasets_and_folders.py` | datasets, nested folders, resources, shares, soft and permanent delete |
|
|
215
|
+
| `03_upload_process_download.py` | upload, wait for processing, download the outputs, with progress bars |
|
|
216
|
+
| `04_sensors.py` | a sensor resource, adding measurements one by one and in a batch, queries, paging |
|
|
217
|
+
| `05_tables.py` | creating a table, inserting GeoJSON, bbox queries, vector tiles |
|
|
218
|
+
| `06_extensions.py` | the extension registry and a resource's processing chain |
|
|
219
|
+
| `07_errors.py` | the exceptions and how to handle them |
|
|
220
|
+
| `08_serve.py` | listing and calling the routes of a processing step |
|
|
221
|
+
|
|
222
|
+
```bash
|
|
223
|
+
python examples/01_login.py
|
|
224
|
+
python examples/03_upload_process_download.py mosaic_chl_oc3.tif
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
## Development
|
|
228
|
+
|
|
229
|
+
```bash
|
|
230
|
+
pip install -e .
|
|
231
|
+
pip install -r requirements-dev.txt
|
|
232
|
+
pytest
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
The tests need no running platform: HTTP is replaced by fakes (`tests/conftest.py`). They cover
|
|
236
|
+
every module of the package, and `tests/test_examples.py` runs the numbered examples against a
|
|
237
|
+
stand-in platform so that they stay in step with the SDK.
|
|
238
|
+
|
|
239
|
+
```
|
|
240
|
+
src/urbandata/
|
|
241
|
+
client.py UrbanDataClient: the URLs, login(), token, whoami(), ping(), and the API attributes
|
|
242
|
+
config.py the OIDC settings from the environment or a .udmp file
|
|
243
|
+
transport.py HTTP to the two planes, bearer token
|
|
244
|
+
errors.py the exceptions
|
|
245
|
+
progress.py tqdm bars
|
|
246
|
+
auth/ OidcAuth (login), PasswordAuth, TokenAuth
|
|
247
|
+
api/ one module per API attribute: datasets, folders, resources, extensions,
|
|
248
|
+
processing, files, sensors, tables, serve
|
|
249
|
+
utils/ shared helpers: records, http, timestamps, network
|
|
250
|
+
tests/ pytest suite
|
|
251
|
+
examples/ runnable scripts
|
|
252
|
+
src/udmp/ the old import name, kept working: Udmp, UdmpError
|
|
253
|
+
```
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=64"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "urbandata"
|
|
7
|
+
version = "0.0.1"
|
|
8
|
+
authors = [
|
|
9
|
+
{ name="Iason Sotiropoulos", email="isotiropoulos@singularlogic.eu"},
|
|
10
|
+
{ name="Panagiotis Iatrou", email="piatrou@singularlogic.eu"},
|
|
11
|
+
]
|
|
12
|
+
description = "Python SDK for the Urban Data Management Platform (UDMP)"
|
|
13
|
+
readme = "README.md"
|
|
14
|
+
requires-python = ">=3.9"
|
|
15
|
+
dependencies = ["requests>=2.31", "tqdm>=4.66"]
|
|
16
|
+
|
|
17
|
+
[project.optional-dependencies]
|
|
18
|
+
dev = ["pytest>=7"]
|
|
19
|
+
|
|
20
|
+
[tool.setuptools.packages.find]
|
|
21
|
+
where = ["src"]
|
|
22
|
+
|
|
23
|
+
[tool.pytest.ini_options]
|
|
24
|
+
testpaths = ["tests"]
|
|
25
|
+
pythonpath = ["src"]
|
urbandata-0.0.1/setup.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
from setuptools import setup, find_packages
|
|
2
|
+
from os import path
|
|
3
|
+
|
|
4
|
+
working_directory = path.abspath(path.dirname(__file__))
|
|
5
|
+
|
|
6
|
+
with open(path.join(working_directory, 'README.md'), encoding='utf-8') as f:
|
|
7
|
+
long_description = f.read()
|
|
8
|
+
|
|
9
|
+
setup(
|
|
10
|
+
name='urbandata', # name of packe which will be package dir below project
|
|
11
|
+
url='https://github.com/SLG-European-Projects/udmp-sdk-python',
|
|
12
|
+
long_description=long_description,
|
|
13
|
+
long_description_content_type='text/markdown',
|
|
14
|
+
packages=find_packages('src'),
|
|
15
|
+
package_dir={'': 'src'},
|
|
16
|
+
zip_safe=False
|
|
17
|
+
)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""The package was renamed to urbandata, and Udmp to UrbanDataClient. This keeps code
|
|
2
|
+
written against the old names working: from udmp import Udmp, PasswordAuth, ..."""
|
|
3
|
+
import warnings
|
|
4
|
+
|
|
5
|
+
from urbandata import * # noqa: F401,F403
|
|
6
|
+
from urbandata import UrbanDataClient, UrbanDataError
|
|
7
|
+
from urbandata import __all__ as _names
|
|
8
|
+
|
|
9
|
+
Udmp = UrbanDataClient
|
|
10
|
+
UdmpError = UrbanDataError
|
|
11
|
+
__all__ = [*_names, "Udmp", "UdmpError"]
|
|
12
|
+
|
|
13
|
+
warnings.warn("the udmp package is now urbandata (Udmp is UrbanDataClient, UdmpError is UrbanDataError)",
|
|
14
|
+
DeprecationWarning, stacklevel=2)
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
from .auth import OidcAuth, PasswordAuth, TokenAuth
|
|
2
|
+
from .client import UrbanDataClient
|
|
3
|
+
from .errors import (
|
|
4
|
+
ConfigError,
|
|
5
|
+
ConflictError,
|
|
6
|
+
ForbiddenError,
|
|
7
|
+
NotFoundError,
|
|
8
|
+
ProcessingFailed,
|
|
9
|
+
ProcessingTimeout,
|
|
10
|
+
UrbanDataError,
|
|
11
|
+
UnauthorizedError,
|
|
12
|
+
)
|
|
13
|
+
from .transport import Record
|
|
14
|
+
|
|
15
|
+
__all__ = [
|
|
16
|
+
"UrbanDataClient",
|
|
17
|
+
"OidcAuth",
|
|
18
|
+
"PasswordAuth",
|
|
19
|
+
"TokenAuth",
|
|
20
|
+
"Record",
|
|
21
|
+
"UrbanDataError",
|
|
22
|
+
"ConfigError",
|
|
23
|
+
"UnauthorizedError",
|
|
24
|
+
"ForbiddenError",
|
|
25
|
+
"NotFoundError",
|
|
26
|
+
"ConflictError",
|
|
27
|
+
"ProcessingFailed",
|
|
28
|
+
"ProcessingTimeout",
|
|
29
|
+
]
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
from .datasets import Datasets
|
|
2
|
+
from .extensions import Extensions
|
|
3
|
+
from .files import Files
|
|
4
|
+
from .folders import Folders
|
|
5
|
+
from .processing import Processing
|
|
6
|
+
from .resources import Resources
|
|
7
|
+
from .sensors import Sensors
|
|
8
|
+
from .serve import Serve
|
|
9
|
+
from .tables import Tables
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"Datasets",
|
|
13
|
+
"Folders",
|
|
14
|
+
"Resources",
|
|
15
|
+
"Extensions",
|
|
16
|
+
"Processing",
|
|
17
|
+
"Files",
|
|
18
|
+
"Sensors",
|
|
19
|
+
"Tables",
|
|
20
|
+
"Serve",
|
|
21
|
+
]
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
from ..utils import clean, records, seg
|
|
2
|
+
from .base import _Api
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class Datasets(_Api):
|
|
6
|
+
def list(self, owner_org=None, visibility=None, search=None, include_deleted=None):
|
|
7
|
+
params = dict(owner_org=owner_org, visibility=visibility, search=search, include_deleted=include_deleted)
|
|
8
|
+
return records(self._t.json("control", "GET", "datasets", params=params).get("data"))
|
|
9
|
+
|
|
10
|
+
def get(self, dataset_id, include_deleted=None):
|
|
11
|
+
return self._t.record("control", "GET", f"datasets/{seg(dataset_id)}",
|
|
12
|
+
params={"include_deleted": include_deleted})
|
|
13
|
+
|
|
14
|
+
def create(self, title, owner_org, visibility="private", **fields):
|
|
15
|
+
body = {"title": title, "owner_org": owner_org, "visibility": visibility, **fields}
|
|
16
|
+
return self._t.record("control", "POST", "datasets", json=body)
|
|
17
|
+
|
|
18
|
+
def update(self, dataset_id, **fields):
|
|
19
|
+
return self._t.record("control", "PUT", f"datasets/{seg(dataset_id)}", json=fields)
|
|
20
|
+
|
|
21
|
+
def delete(self, dataset_id, permanent=None):
|
|
22
|
+
return self._t.record("control", "DELETE", f"datasets/{seg(dataset_id)}", params={"permanent": permanent})
|
|
23
|
+
|
|
24
|
+
def restore(self, dataset_id):
|
|
25
|
+
return self._t.record("control", "PATCH", f"datasets/{seg(dataset_id)}/restore")
|
|
26
|
+
|
|
27
|
+
def shares(self, dataset_id, permission=None, status=None):
|
|
28
|
+
params = dict(permission=permission, status=status)
|
|
29
|
+
return records(self._t.json("control", "GET", f"datasets/{seg(dataset_id)}/shares", params=params).get("data"))
|
|
30
|
+
|
|
31
|
+
def share(self, dataset_id, permission, grantee_org=None, grantee_user_id=None, expires_at=None, note=None):
|
|
32
|
+
body = clean(dict(permission=permission, grantee_org=grantee_org, grantee_user_id=grantee_user_id,
|
|
33
|
+
expires_at=expires_at, note=note))
|
|
34
|
+
return self._t.record("control", "POST", f"datasets/{seg(dataset_id)}/shares", json=body)
|
|
35
|
+
|
|
36
|
+
def unshare(self, dataset_id, share_id):
|
|
37
|
+
return self._t.record("control", "DELETE", f"datasets/{seg(dataset_id)}/shares/{seg(share_id)}")
|