cdsetool 0.2.13__tar.gz → 0.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cdsetool-0.2.13 → cdsetool-0.3.2}/LICENSE +1 -1
- {cdsetool-0.2.13/src/cdsetool.egg-info → cdsetool-0.3.2}/PKG-INFO +147 -34
- {cdsetool-0.2.13 → cdsetool-0.3.2}/README.md +138 -25
- {cdsetool-0.2.13 → cdsetool-0.3.2}/pyproject.toml +22 -8
- cdsetool-0.3.2/src/cdsetool/_attributes.py +768 -0
- {cdsetool-0.2.13 → cdsetool-0.3.2}/src/cdsetool/_processing.py +1 -2
- cdsetool-0.3.2/src/cdsetool/cli.py +172 -0
- {cdsetool-0.2.13 → cdsetool-0.3.2}/src/cdsetool/credentials.py +4 -4
- cdsetool-0.3.2/src/cdsetool/download.py +279 -0
- {cdsetool-0.2.13 → cdsetool-0.3.2}/src/cdsetool/logger.py +6 -4
- {cdsetool-0.2.13 → cdsetool-0.3.2}/src/cdsetool/monitor.py +15 -15
- cdsetool-0.3.2/src/cdsetool/query.py +599 -0
- {cdsetool-0.2.13 → cdsetool-0.3.2/src/cdsetool.egg-info}/PKG-INFO +147 -34
- {cdsetool-0.2.13 → cdsetool-0.3.2}/src/cdsetool.egg-info/SOURCES.txt +1 -0
- cdsetool-0.3.2/src/cdsetool.egg-info/requires.txt +12 -0
- cdsetool-0.2.13/src/cdsetool/cli.py +0 -126
- cdsetool-0.2.13/src/cdsetool/download.py +0 -159
- cdsetool-0.2.13/src/cdsetool/query.py +0 -298
- cdsetool-0.2.13/src/cdsetool.egg-info/requires.txt +0 -13
- {cdsetool-0.2.13 → cdsetool-0.3.2}/setup.cfg +0 -0
- {cdsetool-0.2.13 → cdsetool-0.3.2}/src/cdsetool/__init__.py +0 -0
- {cdsetool-0.2.13 → cdsetool-0.3.2}/src/cdsetool/py.typed +0 -0
- {cdsetool-0.2.13 → cdsetool-0.3.2}/src/cdsetool.egg-info/dependency_links.txt +0 -0
- {cdsetool-0.2.13 → cdsetool-0.3.2}/src/cdsetool.egg-info/entry_points.txt +0 -0
- {cdsetool-0.2.13 → cdsetool-0.3.2}/src/cdsetool.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
MIT License
|
|
2
2
|
|
|
3
|
-
Copyright (c) 2023
|
|
3
|
+
Copyright (c) 2023-2025 CDSETool Contributors
|
|
4
4
|
|
|
5
5
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
6
|
of this software and associated documentation files (the "Software"), to deal
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: cdsetool
|
|
3
|
-
Version: 0.2
|
|
3
|
+
Version: 0.3.2
|
|
4
4
|
Summary: Tools & CLI for interacting with CDSE product APIs
|
|
5
5
|
Author-email: Jacob Vejby <javej@sdfi.dk>
|
|
6
6
|
Project-URL: Homepage, https://github.com/CDSETool/CDSETool
|
|
@@ -11,17 +11,17 @@ Requires-Python: >=3.9
|
|
|
11
11
|
Description-Content-Type: text/markdown
|
|
12
12
|
License-File: LICENSE
|
|
13
13
|
Requires-Dist: typer<1,>=0.9
|
|
14
|
-
Requires-Dist: rich<
|
|
14
|
+
Requires-Dist: rich<16,>=13.6
|
|
15
15
|
Requires-Dist: requests<3,>=2.28.1
|
|
16
|
-
Requires-Dist: pyjwt[crypto]<2.
|
|
16
|
+
Requires-Dist: pyjwt[crypto]<2.15,>=2.8
|
|
17
17
|
Requires-Dist: geopandas>=0.13.2
|
|
18
18
|
Provides-Extra: test
|
|
19
|
-
Requires-Dist: pylint==3.3.
|
|
20
|
-
Requires-Dist: pytest==8.
|
|
21
|
-
Requires-Dist: pytest-cov==
|
|
19
|
+
Requires-Dist: pylint==3.3.9; extra == "test"
|
|
20
|
+
Requires-Dist: pytest==8.4.2; extra == "test"
|
|
21
|
+
Requires-Dist: pytest-cov==7.1.0; extra == "test"
|
|
22
22
|
Requires-Dist: requests-mock==1.12.1; extra == "test"
|
|
23
|
-
Requires-Dist:
|
|
24
|
-
|
|
23
|
+
Requires-Dist: pytest-mock==3.15.1; extra == "test"
|
|
24
|
+
Dynamic: license-file
|
|
25
25
|
|
|
26
26
|
# CDSETool
|
|
27
27
|
|
|
@@ -38,12 +38,11 @@ from cdsetool.monitor import StatusMonitor
|
|
|
38
38
|
from datetime import date
|
|
39
39
|
|
|
40
40
|
features = query_features(
|
|
41
|
-
"
|
|
41
|
+
"SENTINEL-1",
|
|
42
42
|
{
|
|
43
|
-
"
|
|
44
|
-
"
|
|
43
|
+
"contentDateStartGe": "2020-12-20",
|
|
44
|
+
"contentDateStartLe": date(2020, 12, 25),
|
|
45
45
|
"processingLevel": "LEVEL1",
|
|
46
|
-
"sensorMode": "IW",
|
|
47
46
|
"productType": "IW_GRDH_1S",
|
|
48
47
|
"geometry": shape_to_wkt("path/to/shapefile.shp"),
|
|
49
48
|
},
|
|
@@ -65,9 +64,9 @@ list(
|
|
|
65
64
|
Or use the CLI:
|
|
66
65
|
|
|
67
66
|
```bash
|
|
68
|
-
cdsetool query search
|
|
67
|
+
cdsetool query search SENTINEL-2 --search-term contentDateStartGe=2020-01-01 --search-term contentDateStartLe=2020-01-10 --search-term productType=S2MSI2A
|
|
69
68
|
|
|
70
|
-
cdsetool download
|
|
69
|
+
cdsetool download SENTINEL-2 PATH/TO/DIR --concurrency 4 --search-term contentDateStartGe=2020-01-01 --search-term contentDateStartLe=2020-01-10 --search-term productType=S2MSI2A
|
|
71
70
|
```
|
|
72
71
|
|
|
73
72
|
## Table of Contents
|
|
@@ -76,7 +75,7 @@ cdsetool download Sentinel2 PATH/TO/DIR --concurrency 4 --search-term startDate=
|
|
|
76
75
|
* [About CDSETool](#about-cdsetool)
|
|
77
76
|
* [Quick Start](#quick-start)
|
|
78
77
|
* [Table of Contents](#table-of-contents)
|
|
79
|
-
* [
|
|
78
|
+
* [Installation](#installation)
|
|
80
79
|
* [Usage](#usage)
|
|
81
80
|
+ [Querying features](#querying-features)
|
|
82
81
|
- [Querying by shapes](#querying-by-shapes)
|
|
@@ -96,29 +95,31 @@ cdsetool download Sentinel2 PATH/TO/DIR --concurrency 4 --search-term startDate=
|
|
|
96
95
|
Install `cdsetool` using pip:
|
|
97
96
|
|
|
98
97
|
```bash
|
|
99
|
-
pip install cdsetool==0.2
|
|
98
|
+
pip install cdsetool==0.3.2
|
|
100
99
|
```
|
|
101
100
|
|
|
102
101
|
## Usage
|
|
103
102
|
|
|
104
103
|
### Querying features
|
|
105
104
|
|
|
106
|
-
Querying is always done in batches, returning `len(results) <=
|
|
105
|
+
Querying is always done in batches, returning `len(results) <= top` records each time.
|
|
107
106
|
A local buffer is filled and gradually emptied as results are yielded. When the buffer is empty,
|
|
108
107
|
more results will be requested and the process repeated until no more results are available, or
|
|
109
108
|
the iterator is discarded.
|
|
110
109
|
|
|
110
|
+
**Important**: The API has a pagination limit of 10,000 results per query. If your query returns more results, you'll need to narrow your search criteria (e.g., use smaller date ranges).
|
|
111
|
+
|
|
111
112
|
Since downloading features is the most common use-case, `query_features` assumes that the query will run till the end.
|
|
112
|
-
Because of this, the batch size is set to `
|
|
113
|
+
Because of this, the batch size is set to `1000`, which is the size limit set by CDSE.
|
|
113
114
|
|
|
114
115
|
```python
|
|
115
116
|
from cdsetool.query import query_features
|
|
116
117
|
|
|
117
|
-
collection = "
|
|
118
|
+
collection = "SENTINEL-2"
|
|
118
119
|
search_terms = {
|
|
119
|
-
"
|
|
120
|
-
"
|
|
121
|
-
"
|
|
120
|
+
"top": 100, # batch size, between 1 and 1000 (default: 1000)
|
|
121
|
+
"contentDateStartGe": "2024-01-01",
|
|
122
|
+
"productType": "S2MSI1C"
|
|
122
123
|
}
|
|
123
124
|
|
|
124
125
|
# wait for a single batch to finish, yield results immediately
|
|
@@ -136,7 +137,53 @@ featureB = next(iterator)
|
|
|
136
137
|
# ...
|
|
137
138
|
```
|
|
138
139
|
|
|
139
|
-
####
|
|
140
|
+
#### Query Options
|
|
141
|
+
|
|
142
|
+
**Product Format**
|
|
143
|
+
|
|
144
|
+
Query results are returned directly from the Copernicus API. Each product has the following structure:
|
|
145
|
+
|
|
146
|
+
```python
|
|
147
|
+
product = {
|
|
148
|
+
"Id": "uuid-string",
|
|
149
|
+
"Name": "S2A_MSIL2A_20240110T105421_...",
|
|
150
|
+
"Collection": "SENTINEL-2",
|
|
151
|
+
"ContentDate": {"Start": "2024-01-10T10:54:21Z", "End": "..."},
|
|
152
|
+
"Online": True,
|
|
153
|
+
"ContentLength": 1043654649,
|
|
154
|
+
"GeoFootprint": {"type": "Polygon", "coordinates": [...]},
|
|
155
|
+
"Attributes": [
|
|
156
|
+
{"Name": "productType", "Value": "S2MSI2A"},
|
|
157
|
+
{"Name": "cloudCover", "Value": 5.2},
|
|
158
|
+
...,
|
|
159
|
+
],
|
|
160
|
+
}
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
**Expand Product Attributes**
|
|
164
|
+
|
|
165
|
+
By default, query results do not include product attributes (productType, cloudCover, platform, instrument, etc.). To include product attributes, you need to request this from the server using the option `expand_attributes` and can then access the attributes using the function `get_product_attribute()`:
|
|
166
|
+
|
|
167
|
+
```python
|
|
168
|
+
from cdsetool.query import query_features, get_product_attribute
|
|
169
|
+
|
|
170
|
+
features = query_features(
|
|
171
|
+
"SENTINEL-2",
|
|
172
|
+
{"contentDateStartGe": "2024-01-01"},
|
|
173
|
+
options={"expand_attributes": True},
|
|
174
|
+
)
|
|
175
|
+
feature = features[0]
|
|
176
|
+
|
|
177
|
+
# Access basic properties directly
|
|
178
|
+
print(feature["Name"]) # Product name
|
|
179
|
+
print(feature["Id"]) # Product UUID
|
|
180
|
+
|
|
181
|
+
# Access attributes using helper function
|
|
182
|
+
cloud_cover = get_product_attribute(feature, "cloudCover")
|
|
183
|
+
product_type = get_product_attribute(feature, "productType")
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
#### Querying by geometry
|
|
140
187
|
|
|
141
188
|
To query by shapes, you must first convert your shape to Well Known Text (WKT). The included
|
|
142
189
|
`shape_to_wkt` can solve this.
|
|
@@ -145,8 +192,7 @@ To query by shapes, you must first convert your shape to Well Known Text (WKT).
|
|
|
145
192
|
from cdsetool.query import query_features, shape_to_wkt
|
|
146
193
|
|
|
147
194
|
geometry = shape_to_wkt("path/to/shape.shp")
|
|
148
|
-
|
|
149
|
-
features = query_features("Sentinel3", {"geometry": geometry})
|
|
195
|
+
features = query_features("SENTINEL-3", {"geometry": geometry})
|
|
150
196
|
```
|
|
151
197
|
|
|
152
198
|
#### Querying by lists of parameters
|
|
@@ -160,7 +206,7 @@ from cdsetool.query import query_features
|
|
|
160
206
|
tile_ids = ["32TPT", "32UPU", "32UPU", "31RFL", "37XDA"]
|
|
161
207
|
|
|
162
208
|
for tile_id in tile_ids:
|
|
163
|
-
features = query_features("
|
|
209
|
+
features = query_features("SENTINEL-2", {"tileId": tile_id})
|
|
164
210
|
for feature in features:
|
|
165
211
|
# do things with feature
|
|
166
212
|
```
|
|
@@ -169,14 +215,44 @@ for tile_id in tile_ids:
|
|
|
169
215
|
|
|
170
216
|
Its quite common to query for features created before, after or between dates.
|
|
171
217
|
|
|
218
|
+
Search terms support comparison operator suffixes:
|
|
219
|
+
|
|
220
|
+
| Suffix | Meaning | Example |
|
|
221
|
+
|--------|----------------------------|----------------------|
|
|
222
|
+
| `Eq` | equals (=) | `contentDateStartEq` |
|
|
223
|
+
| `Gt` | greater than (>) | `contentDateStartGt` |
|
|
224
|
+
| `Ge` | greater than or equal (>=) | `contentDateStartGe` |
|
|
225
|
+
| `Lt` | less than (<) | `contentDateStartLt` |
|
|
226
|
+
| `Le` | less than or equal (<=) | `contentDateStartLe` |
|
|
227
|
+
|
|
228
|
+
`Eq` can be applied on any field but the other suffixes can only be applied to numeric and date fields.
|
|
229
|
+
|
|
230
|
+
Interval syntax is only allowed on the base name, not on suffixed variants, and only on numeric and date fields.
|
|
231
|
+
|
|
232
|
+
| Interval notation | Suffixes to combine | Meaning |
|
|
233
|
+
|-------------------|---------------------|--------------------------------|
|
|
234
|
+
| [a, b] | `Ge` + `Le` | a <= value <= b (closed) |
|
|
235
|
+
| (a, b) | `Gt` + `Lt` | a < value < b (open) |
|
|
236
|
+
| [a, b) | `Ge` + `Lt` | a <= value < b (half-open) |
|
|
237
|
+
| (a, b] | `Gt` + `Le` | a < value <= b (half-open) |
|
|
238
|
+
|
|
172
239
|
```python
|
|
173
240
|
from cdsetool.query import query_features
|
|
174
241
|
from datetime import date, datetime
|
|
175
242
|
|
|
176
|
-
date_from = date(
|
|
243
|
+
date_from = date(
|
|
244
|
+
2020, 1, 1
|
|
245
|
+
) # or datetime(2020, 1, 1, 23, 59, 59, 123456) or "2020-01-01" or "2020-01-01T23:59:59.123456Z"
|
|
177
246
|
date_to = date(2020, 12, 31)
|
|
178
247
|
|
|
179
|
-
features = query_features(
|
|
248
|
+
features = query_features(
|
|
249
|
+
"SENTINEL-2",
|
|
250
|
+
{
|
|
251
|
+
"contentDateStartGe": date_from,
|
|
252
|
+
"contentDateStartLe": date_to,
|
|
253
|
+
"cloudCover": "[0, 30]",
|
|
254
|
+
},
|
|
255
|
+
)
|
|
180
256
|
```
|
|
181
257
|
|
|
182
258
|
#### Listing search terms
|
|
@@ -187,13 +263,13 @@ use the CLI:
|
|
|
187
263
|
```python
|
|
188
264
|
from cdsetool.query import describe_collection
|
|
189
265
|
|
|
190
|
-
search_terms = describe_collection("
|
|
266
|
+
search_terms = describe_collection("SENTINEL-2").keys()
|
|
191
267
|
print(search_terms)
|
|
192
268
|
```
|
|
193
269
|
|
|
194
270
|
And the CLI:
|
|
195
271
|
```bash
|
|
196
|
-
$ cdsetool query search-terms
|
|
272
|
+
$ cdsetool query search-terms SENTINEL-2
|
|
197
273
|
```
|
|
198
274
|
|
|
199
275
|
### Downloading features
|
|
@@ -240,7 +316,7 @@ Credentials can be validated using the `validate_credentials` function which wil
|
|
|
240
316
|
```python
|
|
241
317
|
from cdsetool.credentials import validate_credentials
|
|
242
318
|
|
|
243
|
-
validate_credentials(username=
|
|
319
|
+
validate_credentials(username="user", password="password")
|
|
244
320
|
```
|
|
245
321
|
|
|
246
322
|
If None are passed to username and password, `validate_credentials` will validate `.netrc`
|
|
@@ -257,7 +333,10 @@ The downloaded feature ids are yielded, so its required to await the results.
|
|
|
257
333
|
from cdsetool.query import query_features
|
|
258
334
|
from cdsetool.download import download_features
|
|
259
335
|
|
|
260
|
-
features = query_features(
|
|
336
|
+
features = query_features(
|
|
337
|
+
"SENTINEL-2",
|
|
338
|
+
{"contentDateStartGe": "2024-01-01", "contentDateStartLe": "2024-01-10"},
|
|
339
|
+
)
|
|
261
340
|
|
|
262
341
|
download_path = "/path/to/download/folder"
|
|
263
342
|
downloads = download_features(features, download_path, {"concurrency": 4})
|
|
@@ -278,13 +357,47 @@ Its possible to download features sequentially in a single thread if desired.
|
|
|
278
357
|
from cdsetool.query import query_features
|
|
279
358
|
from cdsetool.download import download_feature
|
|
280
359
|
|
|
281
|
-
features = query_features(
|
|
360
|
+
features = query_features(
|
|
361
|
+
"SENTINEL-2",
|
|
362
|
+
{"contentDateStartGe": "2024-01-01", "contentDateStartLe": "2024-01-10"},
|
|
363
|
+
)
|
|
282
364
|
|
|
283
365
|
download_path = "/path/to/download/folder"
|
|
284
366
|
for feature in features:
|
|
285
367
|
download_feature(feature, download_path)
|
|
286
368
|
```
|
|
287
369
|
|
|
370
|
+
#### Download specific files within features
|
|
371
|
+
|
|
372
|
+
It's possible to download specific files within products bundles using Unix filename pattern matching.
|
|
373
|
+
|
|
374
|
+
It can be used in CDSETool:
|
|
375
|
+
|
|
376
|
+
- Through the `filter_pattern` option of `download_features` and `download_feature`:
|
|
377
|
+
```python
|
|
378
|
+
from cdsetool.query import query_features
|
|
379
|
+
from cdsetool.download import download_features
|
|
380
|
+
|
|
381
|
+
features = query_features(
|
|
382
|
+
"SENTINEL-2",
|
|
383
|
+
{"contentDateStartGe": "2024-01-01", "contentDateStartLe": "2024-01-10"},
|
|
384
|
+
)
|
|
385
|
+
|
|
386
|
+
download_path = "/path/to/download/folder"
|
|
387
|
+
filter_pattern = "*TCI.jp2"
|
|
388
|
+
downloads = download_features(
|
|
389
|
+
features, download_path, {"filter_pattern": filter_pattern}
|
|
390
|
+
)
|
|
391
|
+
|
|
392
|
+
for id in downloads:
|
|
393
|
+
print(f"feature {id} downloaded")
|
|
394
|
+
```
|
|
395
|
+
|
|
396
|
+
- Or through the CLI:
|
|
397
|
+
```
|
|
398
|
+
cdsetool download SENTINEL-2 PATH/TO/DIR --filter-pattern *TCI.jp2 --concurrency 4 --search-term contentDateStartGe=2024-01-01 --search-term contentDateStartLe=2024-01-10 --search-term productType=S2MSI2A
|
|
399
|
+
```
|
|
400
|
+
|
|
288
401
|
## Roadmap
|
|
289
402
|
|
|
290
403
|
- [X] Query schema validation
|
|
@@ -13,12 +13,11 @@ from cdsetool.monitor import StatusMonitor
|
|
|
13
13
|
from datetime import date
|
|
14
14
|
|
|
15
15
|
features = query_features(
|
|
16
|
-
"
|
|
16
|
+
"SENTINEL-1",
|
|
17
17
|
{
|
|
18
|
-
"
|
|
19
|
-
"
|
|
18
|
+
"contentDateStartGe": "2020-12-20",
|
|
19
|
+
"contentDateStartLe": date(2020, 12, 25),
|
|
20
20
|
"processingLevel": "LEVEL1",
|
|
21
|
-
"sensorMode": "IW",
|
|
22
21
|
"productType": "IW_GRDH_1S",
|
|
23
22
|
"geometry": shape_to_wkt("path/to/shapefile.shp"),
|
|
24
23
|
},
|
|
@@ -40,9 +39,9 @@ list(
|
|
|
40
39
|
Or use the CLI:
|
|
41
40
|
|
|
42
41
|
```bash
|
|
43
|
-
cdsetool query search
|
|
42
|
+
cdsetool query search SENTINEL-2 --search-term contentDateStartGe=2020-01-01 --search-term contentDateStartLe=2020-01-10 --search-term productType=S2MSI2A
|
|
44
43
|
|
|
45
|
-
cdsetool download
|
|
44
|
+
cdsetool download SENTINEL-2 PATH/TO/DIR --concurrency 4 --search-term contentDateStartGe=2020-01-01 --search-term contentDateStartLe=2020-01-10 --search-term productType=S2MSI2A
|
|
46
45
|
```
|
|
47
46
|
|
|
48
47
|
## Table of Contents
|
|
@@ -51,7 +50,7 @@ cdsetool download Sentinel2 PATH/TO/DIR --concurrency 4 --search-term startDate=
|
|
|
51
50
|
* [About CDSETool](#about-cdsetool)
|
|
52
51
|
* [Quick Start](#quick-start)
|
|
53
52
|
* [Table of Contents](#table-of-contents)
|
|
54
|
-
* [
|
|
53
|
+
* [Installation](#installation)
|
|
55
54
|
* [Usage](#usage)
|
|
56
55
|
+ [Querying features](#querying-features)
|
|
57
56
|
- [Querying by shapes](#querying-by-shapes)
|
|
@@ -71,29 +70,31 @@ cdsetool download Sentinel2 PATH/TO/DIR --concurrency 4 --search-term startDate=
|
|
|
71
70
|
Install `cdsetool` using pip:
|
|
72
71
|
|
|
73
72
|
```bash
|
|
74
|
-
pip install cdsetool==0.2
|
|
73
|
+
pip install cdsetool==0.3.2
|
|
75
74
|
```
|
|
76
75
|
|
|
77
76
|
## Usage
|
|
78
77
|
|
|
79
78
|
### Querying features
|
|
80
79
|
|
|
81
|
-
Querying is always done in batches, returning `len(results) <=
|
|
80
|
+
Querying is always done in batches, returning `len(results) <= top` records each time.
|
|
82
81
|
A local buffer is filled and gradually emptied as results are yielded. When the buffer is empty,
|
|
83
82
|
more results will be requested and the process repeated until no more results are available, or
|
|
84
83
|
the iterator is discarded.
|
|
85
84
|
|
|
85
|
+
**Important**: The API has a pagination limit of 10,000 results per query. If your query returns more results, you'll need to narrow your search criteria (e.g., use smaller date ranges).
|
|
86
|
+
|
|
86
87
|
Since downloading features is the most common use-case, `query_features` assumes that the query will run till the end.
|
|
87
|
-
Because of this, the batch size is set to `
|
|
88
|
+
Because of this, the batch size is set to `1000`, which is the size limit set by CDSE.
|
|
88
89
|
|
|
89
90
|
```python
|
|
90
91
|
from cdsetool.query import query_features
|
|
91
92
|
|
|
92
|
-
collection = "
|
|
93
|
+
collection = "SENTINEL-2"
|
|
93
94
|
search_terms = {
|
|
94
|
-
"
|
|
95
|
-
"
|
|
96
|
-
"
|
|
95
|
+
"top": 100, # batch size, between 1 and 1000 (default: 1000)
|
|
96
|
+
"contentDateStartGe": "2024-01-01",
|
|
97
|
+
"productType": "S2MSI1C"
|
|
97
98
|
}
|
|
98
99
|
|
|
99
100
|
# wait for a single batch to finish, yield results immediately
|
|
@@ -111,7 +112,53 @@ featureB = next(iterator)
|
|
|
111
112
|
# ...
|
|
112
113
|
```
|
|
113
114
|
|
|
114
|
-
####
|
|
115
|
+
#### Query Options
|
|
116
|
+
|
|
117
|
+
**Product Format**
|
|
118
|
+
|
|
119
|
+
Query results are returned directly from the Copernicus API. Each product has the following structure:
|
|
120
|
+
|
|
121
|
+
```python
|
|
122
|
+
product = {
|
|
123
|
+
"Id": "uuid-string",
|
|
124
|
+
"Name": "S2A_MSIL2A_20240110T105421_...",
|
|
125
|
+
"Collection": "SENTINEL-2",
|
|
126
|
+
"ContentDate": {"Start": "2024-01-10T10:54:21Z", "End": "..."},
|
|
127
|
+
"Online": True,
|
|
128
|
+
"ContentLength": 1043654649,
|
|
129
|
+
"GeoFootprint": {"type": "Polygon", "coordinates": [...]},
|
|
130
|
+
"Attributes": [
|
|
131
|
+
{"Name": "productType", "Value": "S2MSI2A"},
|
|
132
|
+
{"Name": "cloudCover", "Value": 5.2},
|
|
133
|
+
...,
|
|
134
|
+
],
|
|
135
|
+
}
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
**Expand Product Attributes**
|
|
139
|
+
|
|
140
|
+
By default, query results do not include product attributes (productType, cloudCover, platform, instrument, etc.). To include product attributes, you need to request this from the server using the option `expand_attributes` and can then access the attributes using the function `get_product_attribute()`:
|
|
141
|
+
|
|
142
|
+
```python
|
|
143
|
+
from cdsetool.query import query_features, get_product_attribute
|
|
144
|
+
|
|
145
|
+
features = query_features(
|
|
146
|
+
"SENTINEL-2",
|
|
147
|
+
{"contentDateStartGe": "2024-01-01"},
|
|
148
|
+
options={"expand_attributes": True},
|
|
149
|
+
)
|
|
150
|
+
feature = features[0]
|
|
151
|
+
|
|
152
|
+
# Access basic properties directly
|
|
153
|
+
print(feature["Name"]) # Product name
|
|
154
|
+
print(feature["Id"]) # Product UUID
|
|
155
|
+
|
|
156
|
+
# Access attributes using helper function
|
|
157
|
+
cloud_cover = get_product_attribute(feature, "cloudCover")
|
|
158
|
+
product_type = get_product_attribute(feature, "productType")
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
#### Querying by geometry
|
|
115
162
|
|
|
116
163
|
To query by shapes, you must first convert your shape to Well Known Text (WKT). The included
|
|
117
164
|
`shape_to_wkt` can solve this.
|
|
@@ -120,8 +167,7 @@ To query by shapes, you must first convert your shape to Well Known Text (WKT).
|
|
|
120
167
|
from cdsetool.query import query_features, shape_to_wkt
|
|
121
168
|
|
|
122
169
|
geometry = shape_to_wkt("path/to/shape.shp")
|
|
123
|
-
|
|
124
|
-
features = query_features("Sentinel3", {"geometry": geometry})
|
|
170
|
+
features = query_features("SENTINEL-3", {"geometry": geometry})
|
|
125
171
|
```
|
|
126
172
|
|
|
127
173
|
#### Querying by lists of parameters
|
|
@@ -135,7 +181,7 @@ from cdsetool.query import query_features
|
|
|
135
181
|
tile_ids = ["32TPT", "32UPU", "32UPU", "31RFL", "37XDA"]
|
|
136
182
|
|
|
137
183
|
for tile_id in tile_ids:
|
|
138
|
-
features = query_features("
|
|
184
|
+
features = query_features("SENTINEL-2", {"tileId": tile_id})
|
|
139
185
|
for feature in features:
|
|
140
186
|
# do things with feature
|
|
141
187
|
```
|
|
@@ -144,14 +190,44 @@ for tile_id in tile_ids:
|
|
|
144
190
|
|
|
145
191
|
Its quite common to query for features created before, after or between dates.
|
|
146
192
|
|
|
193
|
+
Search terms support comparison operator suffixes:
|
|
194
|
+
|
|
195
|
+
| Suffix | Meaning | Example |
|
|
196
|
+
|--------|----------------------------|----------------------|
|
|
197
|
+
| `Eq` | equals (=) | `contentDateStartEq` |
|
|
198
|
+
| `Gt` | greater than (>) | `contentDateStartGt` |
|
|
199
|
+
| `Ge` | greater than or equal (>=) | `contentDateStartGe` |
|
|
200
|
+
| `Lt` | less than (<) | `contentDateStartLt` |
|
|
201
|
+
| `Le` | less than or equal (<=) | `contentDateStartLe` |
|
|
202
|
+
|
|
203
|
+
`Eq` can be applied on any field but the other suffixes can only be applied to numeric and date fields.
|
|
204
|
+
|
|
205
|
+
Interval syntax is only allowed on the base name, not on suffixed variants, and only on numeric and date fields.
|
|
206
|
+
|
|
207
|
+
| Interval notation | Suffixes to combine | Meaning |
|
|
208
|
+
|-------------------|---------------------|--------------------------------|
|
|
209
|
+
| [a, b] | `Ge` + `Le` | a <= value <= b (closed) |
|
|
210
|
+
| (a, b) | `Gt` + `Lt` | a < value < b (open) |
|
|
211
|
+
| [a, b) | `Ge` + `Lt` | a <= value < b (half-open) |
|
|
212
|
+
| (a, b] | `Gt` + `Le` | a < value <= b (half-open) |
|
|
213
|
+
|
|
147
214
|
```python
|
|
148
215
|
from cdsetool.query import query_features
|
|
149
216
|
from datetime import date, datetime
|
|
150
217
|
|
|
151
|
-
date_from = date(
|
|
218
|
+
date_from = date(
|
|
219
|
+
2020, 1, 1
|
|
220
|
+
) # or datetime(2020, 1, 1, 23, 59, 59, 123456) or "2020-01-01" or "2020-01-01T23:59:59.123456Z"
|
|
152
221
|
date_to = date(2020, 12, 31)
|
|
153
222
|
|
|
154
|
-
features = query_features(
|
|
223
|
+
features = query_features(
|
|
224
|
+
"SENTINEL-2",
|
|
225
|
+
{
|
|
226
|
+
"contentDateStartGe": date_from,
|
|
227
|
+
"contentDateStartLe": date_to,
|
|
228
|
+
"cloudCover": "[0, 30]",
|
|
229
|
+
},
|
|
230
|
+
)
|
|
155
231
|
```
|
|
156
232
|
|
|
157
233
|
#### Listing search terms
|
|
@@ -162,13 +238,13 @@ use the CLI:
|
|
|
162
238
|
```python
|
|
163
239
|
from cdsetool.query import describe_collection
|
|
164
240
|
|
|
165
|
-
search_terms = describe_collection("
|
|
241
|
+
search_terms = describe_collection("SENTINEL-2").keys()
|
|
166
242
|
print(search_terms)
|
|
167
243
|
```
|
|
168
244
|
|
|
169
245
|
And the CLI:
|
|
170
246
|
```bash
|
|
171
|
-
$ cdsetool query search-terms
|
|
247
|
+
$ cdsetool query search-terms SENTINEL-2
|
|
172
248
|
```
|
|
173
249
|
|
|
174
250
|
### Downloading features
|
|
@@ -215,7 +291,7 @@ Credentials can be validated using the `validate_credentials` function which wil
|
|
|
215
291
|
```python
|
|
216
292
|
from cdsetool.credentials import validate_credentials
|
|
217
293
|
|
|
218
|
-
validate_credentials(username=
|
|
294
|
+
validate_credentials(username="user", password="password")
|
|
219
295
|
```
|
|
220
296
|
|
|
221
297
|
If None are passed to username and password, `validate_credentials` will validate `.netrc`
|
|
@@ -232,7 +308,10 @@ The downloaded feature ids are yielded, so its required to await the results.
|
|
|
232
308
|
from cdsetool.query import query_features
|
|
233
309
|
from cdsetool.download import download_features
|
|
234
310
|
|
|
235
|
-
features = query_features(
|
|
311
|
+
features = query_features(
|
|
312
|
+
"SENTINEL-2",
|
|
313
|
+
{"contentDateStartGe": "2024-01-01", "contentDateStartLe": "2024-01-10"},
|
|
314
|
+
)
|
|
236
315
|
|
|
237
316
|
download_path = "/path/to/download/folder"
|
|
238
317
|
downloads = download_features(features, download_path, {"concurrency": 4})
|
|
@@ -253,13 +332,47 @@ Its possible to download features sequentially in a single thread if desired.
|
|
|
253
332
|
from cdsetool.query import query_features
|
|
254
333
|
from cdsetool.download import download_feature
|
|
255
334
|
|
|
256
|
-
features = query_features(
|
|
335
|
+
features = query_features(
|
|
336
|
+
"SENTINEL-2",
|
|
337
|
+
{"contentDateStartGe": "2024-01-01", "contentDateStartLe": "2024-01-10"},
|
|
338
|
+
)
|
|
257
339
|
|
|
258
340
|
download_path = "/path/to/download/folder"
|
|
259
341
|
for feature in features:
|
|
260
342
|
download_feature(feature, download_path)
|
|
261
343
|
```
|
|
262
344
|
|
|
345
|
+
#### Download specific files within features
|
|
346
|
+
|
|
347
|
+
It's possible to download specific files within products bundles using Unix filename pattern matching.
|
|
348
|
+
|
|
349
|
+
It can be used in CDSETool:
|
|
350
|
+
|
|
351
|
+
- Through the `filter_pattern` option of `download_features` and `download_feature`:
|
|
352
|
+
```python
|
|
353
|
+
from cdsetool.query import query_features
|
|
354
|
+
from cdsetool.download import download_features
|
|
355
|
+
|
|
356
|
+
features = query_features(
|
|
357
|
+
"SENTINEL-2",
|
|
358
|
+
{"contentDateStartGe": "2024-01-01", "contentDateStartLe": "2024-01-10"},
|
|
359
|
+
)
|
|
360
|
+
|
|
361
|
+
download_path = "/path/to/download/folder"
|
|
362
|
+
filter_pattern = "*TCI.jp2"
|
|
363
|
+
downloads = download_features(
|
|
364
|
+
features, download_path, {"filter_pattern": filter_pattern}
|
|
365
|
+
)
|
|
366
|
+
|
|
367
|
+
for id in downloads:
|
|
368
|
+
print(f"feature {id} downloaded")
|
|
369
|
+
```
|
|
370
|
+
|
|
371
|
+
- Or through the CLI:
|
|
372
|
+
```
|
|
373
|
+
cdsetool download SENTINEL-2 PATH/TO/DIR --filter-pattern *TCI.jp2 --concurrency 4 --search-term contentDateStartGe=2024-01-01 --search-term contentDateStartLe=2024-01-10 --search-term productType=S2MSI2A
|
|
374
|
+
```
|
|
375
|
+
|
|
263
376
|
## Roadmap
|
|
264
377
|
|
|
265
378
|
- [X] Query schema validation
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "cdsetool"
|
|
7
|
-
version = "0.2
|
|
7
|
+
version = "0.3.2"
|
|
8
8
|
authors = [
|
|
9
9
|
{ name="Jacob Vejby", email="javej@sdfi.dk" },
|
|
10
10
|
]
|
|
@@ -17,19 +17,24 @@ classifiers = [
|
|
|
17
17
|
]
|
|
18
18
|
dependencies = [
|
|
19
19
|
"typer >= 0.9,< 1",
|
|
20
|
-
"rich >= 13.6,<
|
|
20
|
+
"rich >= 13.6,< 16",
|
|
21
21
|
"requests >= 2.28.1,< 3",
|
|
22
|
-
"pyjwt[crypto] >= 2.8,< 2.
|
|
22
|
+
"pyjwt[crypto] >= 2.8,< 2.15",
|
|
23
23
|
"geopandas >= 0.13.2",
|
|
24
24
|
]
|
|
25
|
+
[dependency-groups]
|
|
26
|
+
dev = [
|
|
27
|
+
"ruff==0.16.7",
|
|
28
|
+
]
|
|
29
|
+
# (Jan 30 2025) Pip does not support dependency-groups yet, so
|
|
30
|
+
# leave the optional dependencies for everything except Ruff.
|
|
25
31
|
[project.optional-dependencies]
|
|
26
32
|
test = [
|
|
27
|
-
"pylint==3.3.
|
|
28
|
-
"pytest==8.
|
|
29
|
-
"pytest-cov==
|
|
33
|
+
"pylint==3.3.9",
|
|
34
|
+
"pytest==8.4.2",
|
|
35
|
+
"pytest-cov==7.1.0",
|
|
30
36
|
"requests-mock==1.12.1",
|
|
31
|
-
"
|
|
32
|
-
"pytest-mock==3.14.0",
|
|
37
|
+
"pytest-mock==3.15.1",
|
|
33
38
|
]
|
|
34
39
|
|
|
35
40
|
[project.urls]
|
|
@@ -46,11 +51,15 @@ max-locals = 20
|
|
|
46
51
|
max-line-length = "88"
|
|
47
52
|
disable="fixme"
|
|
48
53
|
|
|
54
|
+
[tool.pylint.imports]
|
|
55
|
+
disable="wrong-import-order"
|
|
56
|
+
|
|
49
57
|
[tool.pyright]
|
|
50
58
|
exclude = ["**/__pycache__", ".venv"]
|
|
51
59
|
|
|
52
60
|
# Can be strict too, see more fine-grained settings at:
|
|
53
61
|
# https://github.com/microsoft/pyright/blob/main/docs/configuration.md#diagnostic-settings-defaults
|
|
62
|
+
# reportMissingParameterType = true
|
|
54
63
|
typeCheckingMode = "standard"
|
|
55
64
|
|
|
56
65
|
executionEnvironments = [
|
|
@@ -61,3 +70,8 @@ executionEnvironments = [
|
|
|
61
70
|
addopts = [
|
|
62
71
|
"--import-mode=importlib",
|
|
63
72
|
]
|
|
73
|
+
|
|
74
|
+
[tool.ruff.lint]
|
|
75
|
+
select = [
|
|
76
|
+
"I", # isort
|
|
77
|
+
]
|