cdsetool 0.2.12__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cdsetool-0.2.12 → cdsetool-0.3.1}/LICENSE +1 -1
- {cdsetool-0.2.12/src/cdsetool.egg-info → cdsetool-0.3.1}/PKG-INFO +126 -33
- {cdsetool-0.2.12 → cdsetool-0.3.1}/README.md +116 -23
- {cdsetool-0.2.12 → cdsetool-0.3.1}/pyproject.toml +23 -9
- cdsetool-0.3.1/src/cdsetool/_attributes.py +768 -0
- {cdsetool-0.2.12 → cdsetool-0.3.1}/src/cdsetool/_processing.py +1 -3
- cdsetool-0.3.1/src/cdsetool/cli.py +172 -0
- {cdsetool-0.2.12 → cdsetool-0.3.1}/src/cdsetool/credentials.py +4 -4
- cdsetool-0.3.1/src/cdsetool/download.py +279 -0
- {cdsetool-0.2.12 → cdsetool-0.3.1}/src/cdsetool/logger.py +6 -4
- {cdsetool-0.2.12 → cdsetool-0.3.1}/src/cdsetool/monitor.py +15 -15
- cdsetool-0.3.1/src/cdsetool/query.py +599 -0
- {cdsetool-0.2.12 → cdsetool-0.3.1/src/cdsetool.egg-info}/PKG-INFO +126 -33
- {cdsetool-0.2.12 → cdsetool-0.3.1}/src/cdsetool.egg-info/SOURCES.txt +1 -0
- cdsetool-0.3.1/src/cdsetool.egg-info/requires.txt +12 -0
- cdsetool-0.2.12/src/cdsetool/cli.py +0 -126
- cdsetool-0.2.12/src/cdsetool/download.py +0 -156
- cdsetool-0.2.12/src/cdsetool/query.py +0 -275
- cdsetool-0.2.12/src/cdsetool.egg-info/requires.txt +0 -13
- {cdsetool-0.2.12 → cdsetool-0.3.1}/setup.cfg +0 -0
- {cdsetool-0.2.12 → cdsetool-0.3.1}/src/cdsetool/__init__.py +0 -0
- {cdsetool-0.2.12 → cdsetool-0.3.1}/src/cdsetool/py.typed +0 -0
- {cdsetool-0.2.12 → cdsetool-0.3.1}/src/cdsetool.egg-info/dependency_links.txt +0 -0
- {cdsetool-0.2.12 → cdsetool-0.3.1}/src/cdsetool.egg-info/entry_points.txt +0 -0
- {cdsetool-0.2.12 → cdsetool-0.3.1}/src/cdsetool.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
MIT License
|
|
2
2
|
|
|
3
|
-
Copyright (c) 2023
|
|
3
|
+
Copyright (c) 2023-2025 CDSETool Contributors
|
|
4
4
|
|
|
5
5
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
6
|
of this software and associated documentation files (the "Software"), to deal
|
|
@@ -1,27 +1,27 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: cdsetool
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.1
|
|
4
4
|
Summary: Tools & CLI for interacting with CDSE product APIs
|
|
5
5
|
Author-email: Jacob Vejby <javej@sdfi.dk>
|
|
6
6
|
Project-URL: Homepage, https://github.com/CDSETool/CDSETool
|
|
7
7
|
Project-URL: Bug Tracker, https://github.com/CDSETool/CDSETool/issues
|
|
8
8
|
Classifier: Programming Language :: Python :: 3
|
|
9
9
|
Classifier: Operating System :: OS Independent
|
|
10
|
-
Requires-Python: >=3.
|
|
10
|
+
Requires-Python: >=3.9
|
|
11
11
|
Description-Content-Type: text/markdown
|
|
12
12
|
License-File: LICENSE
|
|
13
13
|
Requires-Dist: typer<1,>=0.9
|
|
14
|
-
Requires-Dist: rich<
|
|
14
|
+
Requires-Dist: rich<16,>=13.6
|
|
15
15
|
Requires-Dist: requests<3,>=2.28.1
|
|
16
|
-
Requires-Dist: pyjwt[crypto]
|
|
16
|
+
Requires-Dist: pyjwt[crypto]<2.13,>=2.8
|
|
17
17
|
Requires-Dist: geopandas>=0.13.2
|
|
18
18
|
Provides-Extra: test
|
|
19
|
-
Requires-Dist:
|
|
20
|
-
Requires-Dist:
|
|
21
|
-
Requires-Dist: pytest==
|
|
22
|
-
Requires-Dist: pytest-cov==5.0.0; extra == "test"
|
|
19
|
+
Requires-Dist: pylint==3.3.9; extra == "test"
|
|
20
|
+
Requires-Dist: pytest==8.4.2; extra == "test"
|
|
21
|
+
Requires-Dist: pytest-cov==7.1.0; extra == "test"
|
|
23
22
|
Requires-Dist: requests-mock==1.12.1; extra == "test"
|
|
24
|
-
Requires-Dist: pytest-mock==3.
|
|
23
|
+
Requires-Dist: pytest-mock==3.15.1; extra == "test"
|
|
24
|
+
Dynamic: license-file
|
|
25
25
|
|
|
26
26
|
# CDSETool
|
|
27
27
|
|
|
@@ -38,12 +38,11 @@ from cdsetool.monitor import StatusMonitor
|
|
|
38
38
|
from datetime import date
|
|
39
39
|
|
|
40
40
|
features = query_features(
|
|
41
|
-
"
|
|
41
|
+
"SENTINEL-1",
|
|
42
42
|
{
|
|
43
|
-
"
|
|
44
|
-
"
|
|
43
|
+
"contentDateStartGe": "2020-12-20",
|
|
44
|
+
"contentDateStartLe": date(2020, 12, 25),
|
|
45
45
|
"processingLevel": "LEVEL1",
|
|
46
|
-
"sensorMode": "IW",
|
|
47
46
|
"productType": "IW_GRDH_1S",
|
|
48
47
|
"geometry": shape_to_wkt("path/to/shapefile.shp"),
|
|
49
48
|
},
|
|
@@ -65,9 +64,9 @@ list(
|
|
|
65
64
|
Or use the CLI:
|
|
66
65
|
|
|
67
66
|
```bash
|
|
68
|
-
cdsetool query search
|
|
67
|
+
cdsetool query search SENTINEL-2 --search-term contentDateStartGe=2020-01-01 --search-term contentDateStartLe=2020-01-10 --search-term productType=S2MSI2A
|
|
69
68
|
|
|
70
|
-
cdsetool download
|
|
69
|
+
cdsetool download SENTINEL-2 PATH/TO/DIR --concurrency 4 --search-term contentDateStartGe=2020-01-01 --search-term contentDateStartLe=2020-01-10 --search-term productType=S2MSI2A
|
|
71
70
|
```
|
|
72
71
|
|
|
73
72
|
## Table of Contents
|
|
@@ -76,7 +75,7 @@ cdsetool download Sentinel2 PATH/TO/DIR --concurrency 4 --search-term startDate=
|
|
|
76
75
|
* [About CDSETool](#about-cdsetool)
|
|
77
76
|
* [Quick Start](#quick-start)
|
|
78
77
|
* [Table of Contents](#table-of-contents)
|
|
79
|
-
* [
|
|
78
|
+
* [Installation](#installation)
|
|
80
79
|
* [Usage](#usage)
|
|
81
80
|
+ [Querying features](#querying-features)
|
|
82
81
|
- [Querying by shapes](#querying-by-shapes)
|
|
@@ -96,29 +95,31 @@ cdsetool download Sentinel2 PATH/TO/DIR --concurrency 4 --search-term startDate=
|
|
|
96
95
|
Install `cdsetool` using pip:
|
|
97
96
|
|
|
98
97
|
```bash
|
|
99
|
-
pip install cdsetool==0.
|
|
98
|
+
pip install cdsetool==0.3.1
|
|
100
99
|
```
|
|
101
100
|
|
|
102
101
|
## Usage
|
|
103
102
|
|
|
104
103
|
### Querying features
|
|
105
104
|
|
|
106
|
-
Querying is always done in batches, returning `len(results) <=
|
|
105
|
+
Querying is always done in batches, returning `len(results) <= top` records each time.
|
|
107
106
|
A local buffer is filled and gradually emptied as results are yielded. When the buffer is empty,
|
|
108
107
|
more results will be requested and the process repeated until no more results are available, or
|
|
109
108
|
the iterator is discarded.
|
|
110
109
|
|
|
110
|
+
**Important**: The API has a pagination limit of 10,000 results per query. If your query returns more results, you'll need to narrow your search criteria (e.g., use smaller date ranges).
|
|
111
|
+
|
|
111
112
|
Since downloading features is the most common use-case, `query_features` assumes that the query will run till the end.
|
|
112
|
-
Because of this, the batch size is set to `
|
|
113
|
+
Because of this, the batch size is set to `1000`, which is the size limit set by CDSE.
|
|
113
114
|
|
|
114
115
|
```python
|
|
115
116
|
from cdsetool.query import query_features
|
|
116
117
|
|
|
117
|
-
collection = "
|
|
118
|
+
collection = "SENTINEL-2"
|
|
118
119
|
search_terms = {
|
|
119
|
-
"
|
|
120
|
-
"
|
|
121
|
-
"
|
|
120
|
+
"top": 100, # batch size, between 1 and 1000 (default: 1000)
|
|
121
|
+
"contentDateStartGe": "2024-01-01",
|
|
122
|
+
"productType": "S2MSI1C"
|
|
122
123
|
}
|
|
123
124
|
|
|
124
125
|
# wait for a single batch to finish, yield results immediately
|
|
@@ -136,7 +137,53 @@ featureB = next(iterator)
|
|
|
136
137
|
# ...
|
|
137
138
|
```
|
|
138
139
|
|
|
139
|
-
####
|
|
140
|
+
#### Query Options
|
|
141
|
+
|
|
142
|
+
**Product Format**
|
|
143
|
+
|
|
144
|
+
Query results are returned directly from the Copernicus API. Each product has the following structure:
|
|
145
|
+
|
|
146
|
+
```python
|
|
147
|
+
product = {
|
|
148
|
+
"Id": "uuid-string",
|
|
149
|
+
"Name": "S2A_MSIL2A_20240110T105421_...",
|
|
150
|
+
"Collection": "SENTINEL-2",
|
|
151
|
+
"ContentDate": {"Start": "2024-01-10T10:54:21Z", "End": "..."},
|
|
152
|
+
"Online": True,
|
|
153
|
+
"ContentLength": 1043654649,
|
|
154
|
+
"GeoFootprint": {"type": "Polygon", "coordinates": [...]},
|
|
155
|
+
"Attributes": [
|
|
156
|
+
{"Name": "productType", "Value": "S2MSI2A"},
|
|
157
|
+
{"Name": "cloudCover", "Value": 5.2},
|
|
158
|
+
...
|
|
159
|
+
]
|
|
160
|
+
}
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
**Expand Product Attributes**
|
|
164
|
+
|
|
165
|
+
By default, query results do not include product attributes (productType, cloudCover, platform, instrument, etc.). To include product attributes, you need to request this from the server using the option `expand_attributes` and can then access the attributes using the function `get_product_attribute()`:
|
|
166
|
+
|
|
167
|
+
```python
|
|
168
|
+
from cdsetool.query import query_features, get_product_attribute
|
|
169
|
+
|
|
170
|
+
features = query_features(
|
|
171
|
+
"SENTINEL-2",
|
|
172
|
+
{"contentDateStartGe": "2024-01-01"},
|
|
173
|
+
options={"expand_attributes": True}
|
|
174
|
+
)
|
|
175
|
+
feature = features[0]
|
|
176
|
+
|
|
177
|
+
# Access basic properties directly
|
|
178
|
+
print(feature["Name"]) # Product name
|
|
179
|
+
print(feature["Id"]) # Product UUID
|
|
180
|
+
|
|
181
|
+
# Access attributes using helper function
|
|
182
|
+
cloud_cover = get_product_attribute(feature, "cloudCover")
|
|
183
|
+
product_type = get_product_attribute(feature, "productType")
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
#### Querying by geometry
|
|
140
187
|
|
|
141
188
|
To query by shapes, you must first convert your shape to Well Known Text (WKT). The included
|
|
142
189
|
`shape_to_wkt` can solve this.
|
|
@@ -145,8 +192,7 @@ To query by shapes, you must first convert your shape to Well Known Text (WKT).
|
|
|
145
192
|
from cdsetool.query import query_features, shape_to_wkt
|
|
146
193
|
|
|
147
194
|
geometry = shape_to_wkt("path/to/shape.shp")
|
|
148
|
-
|
|
149
|
-
features = query_features("Sentinel3", {"geometry": geometry})
|
|
195
|
+
features = query_features("SENTINEL-3", {"geometry": geometry})
|
|
150
196
|
```
|
|
151
197
|
|
|
152
198
|
#### Querying by lists of parameters
|
|
@@ -160,7 +206,7 @@ from cdsetool.query import query_features
|
|
|
160
206
|
tile_ids = ["32TPT", "32UPU", "32UPU", "31RFL", "37XDA"]
|
|
161
207
|
|
|
162
208
|
for tile_id in tile_ids:
|
|
163
|
-
features = query_features("
|
|
209
|
+
features = query_features("SENTINEL-2", {"tileId": tile_id})
|
|
164
210
|
for feature in features:
|
|
165
211
|
# do things with feature
|
|
166
212
|
```
|
|
@@ -169,6 +215,27 @@ for tile_id in tile_ids:
|
|
|
169
215
|
|
|
170
216
|
Its quite common to query for features created before, after or between dates.
|
|
171
217
|
|
|
218
|
+
Search terms support comparison operator suffixes:
|
|
219
|
+
|
|
220
|
+
| Suffix | Meaning | Example |
|
|
221
|
+
|--------|----------------------------|----------------------|
|
|
222
|
+
| `Eq` | equals (=) | `contentDateStartEq` |
|
|
223
|
+
| `Gt` | greater than (>) | `contentDateStartGt` |
|
|
224
|
+
| `Ge` | greater than or equal (>=) | `contentDateStartGe` |
|
|
225
|
+
| `Lt` | less than (<) | `contentDateStartLt` |
|
|
226
|
+
| `Le` | less than or equal (<=) | `contentDateStartLe` |
|
|
227
|
+
|
|
228
|
+
`Eq` can be applied on any field but the other suffixes can only be applied to numeric and date fields.
|
|
229
|
+
|
|
230
|
+
Interval syntax is only allowed on the base name, not on suffixed variants, and only on numeric and date fields.
|
|
231
|
+
|
|
232
|
+
| Interval notation | Suffixes to combine | Meaning |
|
|
233
|
+
|-------------------|---------------------|--------------------------------|
|
|
234
|
+
| [a, b] | `Ge` + `Le` | a <= value <= b (closed) |
|
|
235
|
+
| (a, b) | `Gt` + `Lt` | a < value < b (open) |
|
|
236
|
+
| [a, b) | `Ge` + `Lt` | a <= value < b (half-open) |
|
|
237
|
+
| (a, b] | `Gt` + `Le` | a < value <= b (half-open) |
|
|
238
|
+
|
|
172
239
|
```python
|
|
173
240
|
from cdsetool.query import query_features
|
|
174
241
|
from datetime import date, datetime
|
|
@@ -176,7 +243,7 @@ from datetime import date, datetime
|
|
|
176
243
|
date_from = date(2020, 1, 1) # or datetime(2020, 1, 1, 23, 59, 59, 123456) or "2020-01-01" or "2020-01-01T23:59:59.123456Z"
|
|
177
244
|
date_to = date(2020, 12, 31)
|
|
178
245
|
|
|
179
|
-
features = query_features("
|
|
246
|
+
features = query_features("SENTINEL-2", {"contentDateStartGe": date_from, "contentDateStartLe": date_to, "cloudCover": "[0, 30]"})
|
|
180
247
|
```
|
|
181
248
|
|
|
182
249
|
#### Listing search terms
|
|
@@ -187,13 +254,13 @@ use the CLI:
|
|
|
187
254
|
```python
|
|
188
255
|
from cdsetool.query import describe_collection
|
|
189
256
|
|
|
190
|
-
search_terms = describe_collection("
|
|
257
|
+
search_terms = describe_collection("SENTINEL-2").keys()
|
|
191
258
|
print(search_terms)
|
|
192
259
|
```
|
|
193
260
|
|
|
194
261
|
And the CLI:
|
|
195
262
|
```bash
|
|
196
|
-
$ cdsetool query search-terms
|
|
263
|
+
$ cdsetool query search-terms SENTINEL-2
|
|
197
264
|
```
|
|
198
265
|
|
|
199
266
|
### Downloading features
|
|
@@ -257,7 +324,7 @@ The downloaded feature ids are yielded, so its required to await the results.
|
|
|
257
324
|
from cdsetool.query import query_features
|
|
258
325
|
from cdsetool.download import download_features
|
|
259
326
|
|
|
260
|
-
features = query_features("
|
|
327
|
+
features = query_features("SENTINEL-2", {"contentDateStartGe": "2024-01-01", "contentDateStartLe": "2024-01-10"})
|
|
261
328
|
|
|
262
329
|
download_path = "/path/to/download/folder"
|
|
263
330
|
downloads = download_features(features, download_path, {"concurrency": 4})
|
|
@@ -278,13 +345,39 @@ Its possible to download features sequentially in a single thread if desired.
|
|
|
278
345
|
from cdsetool.query import query_features
|
|
279
346
|
from cdsetool.download import download_feature
|
|
280
347
|
|
|
281
|
-
features = query_features("
|
|
348
|
+
features = query_features("SENTINEL-2", {"contentDateStartGe": "2024-01-01", "contentDateStartLe": "2024-01-10"})
|
|
282
349
|
|
|
283
350
|
download_path = "/path/to/download/folder"
|
|
284
351
|
for feature in features:
|
|
285
352
|
download_feature(feature, download_path)
|
|
286
353
|
```
|
|
287
354
|
|
|
355
|
+
#### Download specific files within features
|
|
356
|
+
|
|
357
|
+
It's possible to download specific files within products bundles using Unix filename pattern matching.
|
|
358
|
+
|
|
359
|
+
It can be used in CDSETool:
|
|
360
|
+
|
|
361
|
+
- Through the `filter_pattern` option of `download_features` and `download_feature`:
|
|
362
|
+
```python
|
|
363
|
+
from cdsetool.query import query_features
|
|
364
|
+
from cdsetool.download import download_features
|
|
365
|
+
|
|
366
|
+
features = query_features("SENTINEL-2", {"contentDateStartGe": "2024-01-01", "contentDateStartLe": "2024-01-10"})
|
|
367
|
+
|
|
368
|
+
download_path = "/path/to/download/folder"
|
|
369
|
+
filter_pattern = "*TCI.jp2"
|
|
370
|
+
downloads = download_features(features, download_path, {"filter_pattern": filter_pattern})
|
|
371
|
+
|
|
372
|
+
for id in downloads:
|
|
373
|
+
print(f"feature {id} downloaded")
|
|
374
|
+
```
|
|
375
|
+
|
|
376
|
+
- Or through the CLI:
|
|
377
|
+
```
|
|
378
|
+
cdsetool download SENTINEL-2 PATH/TO/DIR --filter-pattern *TCI.jp2 --concurrency 4 --search-term contentDateStartGe=2024-01-01 --search-term contentDateStartLe=2024-01-10 --search-term productType=S2MSI2A
|
|
379
|
+
```
|
|
380
|
+
|
|
288
381
|
## Roadmap
|
|
289
382
|
|
|
290
383
|
- [X] Query schema validation
|
|
@@ -13,12 +13,11 @@ from cdsetool.monitor import StatusMonitor
|
|
|
13
13
|
from datetime import date
|
|
14
14
|
|
|
15
15
|
features = query_features(
|
|
16
|
-
"
|
|
16
|
+
"SENTINEL-1",
|
|
17
17
|
{
|
|
18
|
-
"
|
|
19
|
-
"
|
|
18
|
+
"contentDateStartGe": "2020-12-20",
|
|
19
|
+
"contentDateStartLe": date(2020, 12, 25),
|
|
20
20
|
"processingLevel": "LEVEL1",
|
|
21
|
-
"sensorMode": "IW",
|
|
22
21
|
"productType": "IW_GRDH_1S",
|
|
23
22
|
"geometry": shape_to_wkt("path/to/shapefile.shp"),
|
|
24
23
|
},
|
|
@@ -40,9 +39,9 @@ list(
|
|
|
40
39
|
Or use the CLI:
|
|
41
40
|
|
|
42
41
|
```bash
|
|
43
|
-
cdsetool query search
|
|
42
|
+
cdsetool query search SENTINEL-2 --search-term contentDateStartGe=2020-01-01 --search-term contentDateStartLe=2020-01-10 --search-term productType=S2MSI2A
|
|
44
43
|
|
|
45
|
-
cdsetool download
|
|
44
|
+
cdsetool download SENTINEL-2 PATH/TO/DIR --concurrency 4 --search-term contentDateStartGe=2020-01-01 --search-term contentDateStartLe=2020-01-10 --search-term productType=S2MSI2A
|
|
46
45
|
```
|
|
47
46
|
|
|
48
47
|
## Table of Contents
|
|
@@ -51,7 +50,7 @@ cdsetool download Sentinel2 PATH/TO/DIR --concurrency 4 --search-term startDate=
|
|
|
51
50
|
* [About CDSETool](#about-cdsetool)
|
|
52
51
|
* [Quick Start](#quick-start)
|
|
53
52
|
* [Table of Contents](#table-of-contents)
|
|
54
|
-
* [
|
|
53
|
+
* [Installation](#installation)
|
|
55
54
|
* [Usage](#usage)
|
|
56
55
|
+ [Querying features](#querying-features)
|
|
57
56
|
- [Querying by shapes](#querying-by-shapes)
|
|
@@ -71,29 +70,31 @@ cdsetool download Sentinel2 PATH/TO/DIR --concurrency 4 --search-term startDate=
|
|
|
71
70
|
Install `cdsetool` using pip:
|
|
72
71
|
|
|
73
72
|
```bash
|
|
74
|
-
pip install cdsetool==0.
|
|
73
|
+
pip install cdsetool==0.3.1
|
|
75
74
|
```
|
|
76
75
|
|
|
77
76
|
## Usage
|
|
78
77
|
|
|
79
78
|
### Querying features
|
|
80
79
|
|
|
81
|
-
Querying is always done in batches, returning `len(results) <=
|
|
80
|
+
Querying is always done in batches, returning `len(results) <= top` records each time.
|
|
82
81
|
A local buffer is filled and gradually emptied as results are yielded. When the buffer is empty,
|
|
83
82
|
more results will be requested and the process repeated until no more results are available, or
|
|
84
83
|
the iterator is discarded.
|
|
85
84
|
|
|
85
|
+
**Important**: The API has a pagination limit of 10,000 results per query. If your query returns more results, you'll need to narrow your search criteria (e.g., use smaller date ranges).
|
|
86
|
+
|
|
86
87
|
Since downloading features is the most common use-case, `query_features` assumes that the query will run till the end.
|
|
87
|
-
Because of this, the batch size is set to `
|
|
88
|
+
Because of this, the batch size is set to `1000`, which is the size limit set by CDSE.
|
|
88
89
|
|
|
89
90
|
```python
|
|
90
91
|
from cdsetool.query import query_features
|
|
91
92
|
|
|
92
|
-
collection = "
|
|
93
|
+
collection = "SENTINEL-2"
|
|
93
94
|
search_terms = {
|
|
94
|
-
"
|
|
95
|
-
"
|
|
96
|
-
"
|
|
95
|
+
"top": 100, # batch size, between 1 and 1000 (default: 1000)
|
|
96
|
+
"contentDateStartGe": "2024-01-01",
|
|
97
|
+
"productType": "S2MSI1C"
|
|
97
98
|
}
|
|
98
99
|
|
|
99
100
|
# wait for a single batch to finish, yield results immediately
|
|
@@ -111,7 +112,53 @@ featureB = next(iterator)
|
|
|
111
112
|
# ...
|
|
112
113
|
```
|
|
113
114
|
|
|
114
|
-
####
|
|
115
|
+
#### Query Options
|
|
116
|
+
|
|
117
|
+
**Product Format**
|
|
118
|
+
|
|
119
|
+
Query results are returned directly from the Copernicus API. Each product has the following structure:
|
|
120
|
+
|
|
121
|
+
```python
|
|
122
|
+
product = {
|
|
123
|
+
"Id": "uuid-string",
|
|
124
|
+
"Name": "S2A_MSIL2A_20240110T105421_...",
|
|
125
|
+
"Collection": "SENTINEL-2",
|
|
126
|
+
"ContentDate": {"Start": "2024-01-10T10:54:21Z", "End": "..."},
|
|
127
|
+
"Online": True,
|
|
128
|
+
"ContentLength": 1043654649,
|
|
129
|
+
"GeoFootprint": {"type": "Polygon", "coordinates": [...]},
|
|
130
|
+
"Attributes": [
|
|
131
|
+
{"Name": "productType", "Value": "S2MSI2A"},
|
|
132
|
+
{"Name": "cloudCover", "Value": 5.2},
|
|
133
|
+
...
|
|
134
|
+
]
|
|
135
|
+
}
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
**Expand Product Attributes**
|
|
139
|
+
|
|
140
|
+
By default, query results do not include product attributes (productType, cloudCover, platform, instrument, etc.). To include product attributes, you need to request this from the server using the option `expand_attributes` and can then access the attributes using the function `get_product_attribute()`:
|
|
141
|
+
|
|
142
|
+
```python
|
|
143
|
+
from cdsetool.query import query_features, get_product_attribute
|
|
144
|
+
|
|
145
|
+
features = query_features(
|
|
146
|
+
"SENTINEL-2",
|
|
147
|
+
{"contentDateStartGe": "2024-01-01"},
|
|
148
|
+
options={"expand_attributes": True}
|
|
149
|
+
)
|
|
150
|
+
feature = features[0]
|
|
151
|
+
|
|
152
|
+
# Access basic properties directly
|
|
153
|
+
print(feature["Name"]) # Product name
|
|
154
|
+
print(feature["Id"]) # Product UUID
|
|
155
|
+
|
|
156
|
+
# Access attributes using helper function
|
|
157
|
+
cloud_cover = get_product_attribute(feature, "cloudCover")
|
|
158
|
+
product_type = get_product_attribute(feature, "productType")
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
#### Querying by geometry
|
|
115
162
|
|
|
116
163
|
To query by shapes, you must first convert your shape to Well Known Text (WKT). The included
|
|
117
164
|
`shape_to_wkt` can solve this.
|
|
@@ -120,8 +167,7 @@ To query by shapes, you must first convert your shape to Well Known Text (WKT).
|
|
|
120
167
|
from cdsetool.query import query_features, shape_to_wkt
|
|
121
168
|
|
|
122
169
|
geometry = shape_to_wkt("path/to/shape.shp")
|
|
123
|
-
|
|
124
|
-
features = query_features("Sentinel3", {"geometry": geometry})
|
|
170
|
+
features = query_features("SENTINEL-3", {"geometry": geometry})
|
|
125
171
|
```
|
|
126
172
|
|
|
127
173
|
#### Querying by lists of parameters
|
|
@@ -135,7 +181,7 @@ from cdsetool.query import query_features
|
|
|
135
181
|
tile_ids = ["32TPT", "32UPU", "32UPU", "31RFL", "37XDA"]
|
|
136
182
|
|
|
137
183
|
for tile_id in tile_ids:
|
|
138
|
-
features = query_features("
|
|
184
|
+
features = query_features("SENTINEL-2", {"tileId": tile_id})
|
|
139
185
|
for feature in features:
|
|
140
186
|
# do things with feature
|
|
141
187
|
```
|
|
@@ -144,6 +190,27 @@ for tile_id in tile_ids:
|
|
|
144
190
|
|
|
145
191
|
Its quite common to query for features created before, after or between dates.
|
|
146
192
|
|
|
193
|
+
Search terms support comparison operator suffixes:
|
|
194
|
+
|
|
195
|
+
| Suffix | Meaning | Example |
|
|
196
|
+
|--------|----------------------------|----------------------|
|
|
197
|
+
| `Eq` | equals (=) | `contentDateStartEq` |
|
|
198
|
+
| `Gt` | greater than (>) | `contentDateStartGt` |
|
|
199
|
+
| `Ge` | greater than or equal (>=) | `contentDateStartGe` |
|
|
200
|
+
| `Lt` | less than (<) | `contentDateStartLt` |
|
|
201
|
+
| `Le` | less than or equal (<=) | `contentDateStartLe` |
|
|
202
|
+
|
|
203
|
+
`Eq` can be applied on any field but the other suffixes can only be applied to numeric and date fields.
|
|
204
|
+
|
|
205
|
+
Interval syntax is only allowed on the base name, not on suffixed variants, and only on numeric and date fields.
|
|
206
|
+
|
|
207
|
+
| Interval notation | Suffixes to combine | Meaning |
|
|
208
|
+
|-------------------|---------------------|--------------------------------|
|
|
209
|
+
| [a, b] | `Ge` + `Le` | a <= value <= b (closed) |
|
|
210
|
+
| (a, b) | `Gt` + `Lt` | a < value < b (open) |
|
|
211
|
+
| [a, b) | `Ge` + `Lt` | a <= value < b (half-open) |
|
|
212
|
+
| (a, b] | `Gt` + `Le` | a < value <= b (half-open) |
|
|
213
|
+
|
|
147
214
|
```python
|
|
148
215
|
from cdsetool.query import query_features
|
|
149
216
|
from datetime import date, datetime
|
|
@@ -151,7 +218,7 @@ from datetime import date, datetime
|
|
|
151
218
|
date_from = date(2020, 1, 1) # or datetime(2020, 1, 1, 23, 59, 59, 123456) or "2020-01-01" or "2020-01-01T23:59:59.123456Z"
|
|
152
219
|
date_to = date(2020, 12, 31)
|
|
153
220
|
|
|
154
|
-
features = query_features("
|
|
221
|
+
features = query_features("SENTINEL-2", {"contentDateStartGe": date_from, "contentDateStartLe": date_to, "cloudCover": "[0, 30]"})
|
|
155
222
|
```
|
|
156
223
|
|
|
157
224
|
#### Listing search terms
|
|
@@ -162,13 +229,13 @@ use the CLI:
|
|
|
162
229
|
```python
|
|
163
230
|
from cdsetool.query import describe_collection
|
|
164
231
|
|
|
165
|
-
search_terms = describe_collection("
|
|
232
|
+
search_terms = describe_collection("SENTINEL-2").keys()
|
|
166
233
|
print(search_terms)
|
|
167
234
|
```
|
|
168
235
|
|
|
169
236
|
And the CLI:
|
|
170
237
|
```bash
|
|
171
|
-
$ cdsetool query search-terms
|
|
238
|
+
$ cdsetool query search-terms SENTINEL-2
|
|
172
239
|
```
|
|
173
240
|
|
|
174
241
|
### Downloading features
|
|
@@ -232,7 +299,7 @@ The downloaded feature ids are yielded, so its required to await the results.
|
|
|
232
299
|
from cdsetool.query import query_features
|
|
233
300
|
from cdsetool.download import download_features
|
|
234
301
|
|
|
235
|
-
features = query_features("
|
|
302
|
+
features = query_features("SENTINEL-2", {"contentDateStartGe": "2024-01-01", "contentDateStartLe": "2024-01-10"})
|
|
236
303
|
|
|
237
304
|
download_path = "/path/to/download/folder"
|
|
238
305
|
downloads = download_features(features, download_path, {"concurrency": 4})
|
|
@@ -253,13 +320,39 @@ Its possible to download features sequentially in a single thread if desired.
|
|
|
253
320
|
from cdsetool.query import query_features
|
|
254
321
|
from cdsetool.download import download_feature
|
|
255
322
|
|
|
256
|
-
features = query_features("
|
|
323
|
+
features = query_features("SENTINEL-2", {"contentDateStartGe": "2024-01-01", "contentDateStartLe": "2024-01-10"})
|
|
257
324
|
|
|
258
325
|
download_path = "/path/to/download/folder"
|
|
259
326
|
for feature in features:
|
|
260
327
|
download_feature(feature, download_path)
|
|
261
328
|
```
|
|
262
329
|
|
|
330
|
+
#### Download specific files within features
|
|
331
|
+
|
|
332
|
+
It's possible to download specific files within products bundles using Unix filename pattern matching.
|
|
333
|
+
|
|
334
|
+
It can be used in CDSETool:
|
|
335
|
+
|
|
336
|
+
- Through the `filter_pattern` option of `download_features` and `download_feature`:
|
|
337
|
+
```python
|
|
338
|
+
from cdsetool.query import query_features
|
|
339
|
+
from cdsetool.download import download_features
|
|
340
|
+
|
|
341
|
+
features = query_features("SENTINEL-2", {"contentDateStartGe": "2024-01-01", "contentDateStartLe": "2024-01-10"})
|
|
342
|
+
|
|
343
|
+
download_path = "/path/to/download/folder"
|
|
344
|
+
filter_pattern = "*TCI.jp2"
|
|
345
|
+
downloads = download_features(features, download_path, {"filter_pattern": filter_pattern})
|
|
346
|
+
|
|
347
|
+
for id in downloads:
|
|
348
|
+
print(f"feature {id} downloaded")
|
|
349
|
+
```
|
|
350
|
+
|
|
351
|
+
- Or through the CLI:
|
|
352
|
+
```
|
|
353
|
+
cdsetool download SENTINEL-2 PATH/TO/DIR --filter-pattern *TCI.jp2 --concurrency 4 --search-term contentDateStartGe=2024-01-01 --search-term contentDateStartLe=2024-01-10 --search-term productType=S2MSI2A
|
|
354
|
+
```
|
|
355
|
+
|
|
263
356
|
## Roadmap
|
|
264
357
|
|
|
265
358
|
- [X] Query schema validation
|
|
@@ -4,32 +4,37 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "cdsetool"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.3.1"
|
|
8
8
|
authors = [
|
|
9
9
|
{ name="Jacob Vejby", email="javej@sdfi.dk" },
|
|
10
10
|
]
|
|
11
11
|
description = "Tools & CLI for interacting with CDSE product APIs"
|
|
12
12
|
readme = "README.md"
|
|
13
|
-
requires-python = ">=3.
|
|
13
|
+
requires-python = ">=3.9"
|
|
14
14
|
classifiers = [
|
|
15
15
|
"Programming Language :: Python :: 3",
|
|
16
16
|
"Operating System :: OS Independent",
|
|
17
17
|
]
|
|
18
18
|
dependencies = [
|
|
19
19
|
"typer >= 0.9,< 1",
|
|
20
|
-
"rich >= 13.6,<
|
|
20
|
+
"rich >= 13.6,< 16",
|
|
21
21
|
"requests >= 2.28.1,< 3",
|
|
22
|
-
"pyjwt[crypto]
|
|
22
|
+
"pyjwt[crypto] >= 2.8,< 2.13",
|
|
23
23
|
"geopandas >= 0.13.2",
|
|
24
24
|
]
|
|
25
|
+
[dependency-groups]
|
|
26
|
+
dev = [
|
|
27
|
+
"ruff==0.15.11",
|
|
28
|
+
]
|
|
29
|
+
# (Jan 30 2025) Pip does not support dependency-groups yet, so
|
|
30
|
+
# leave the optional dependencies for everything except Ruff.
|
|
25
31
|
[project.optional-dependencies]
|
|
26
32
|
test = [
|
|
27
|
-
"
|
|
28
|
-
"
|
|
29
|
-
"pytest==
|
|
30
|
-
"pytest-cov==5.0.0",
|
|
33
|
+
"pylint==3.3.9",
|
|
34
|
+
"pytest==8.4.2",
|
|
35
|
+
"pytest-cov==7.1.0",
|
|
31
36
|
"requests-mock==1.12.1",
|
|
32
|
-
"pytest-mock==3.
|
|
37
|
+
"pytest-mock==3.15.1",
|
|
33
38
|
]
|
|
34
39
|
|
|
35
40
|
[project.urls]
|
|
@@ -46,11 +51,15 @@ max-locals = 20
|
|
|
46
51
|
max-line-length = "88"
|
|
47
52
|
disable="fixme"
|
|
48
53
|
|
|
54
|
+
[tool.pylint.imports]
|
|
55
|
+
disable="wrong-import-order"
|
|
56
|
+
|
|
49
57
|
[tool.pyright]
|
|
50
58
|
exclude = ["**/__pycache__", ".venv"]
|
|
51
59
|
|
|
52
60
|
# Can be strict too, see more fine-grained settings at:
|
|
53
61
|
# https://github.com/microsoft/pyright/blob/main/docs/configuration.md#diagnostic-settings-defaults
|
|
62
|
+
# reportMissingParameterType = true
|
|
54
63
|
typeCheckingMode = "standard"
|
|
55
64
|
|
|
56
65
|
executionEnvironments = [
|
|
@@ -61,3 +70,8 @@ executionEnvironments = [
|
|
|
61
70
|
addopts = [
|
|
62
71
|
"--import-mode=importlib",
|
|
63
72
|
]
|
|
73
|
+
|
|
74
|
+
[tool.ruff.lint]
|
|
75
|
+
select = [
|
|
76
|
+
"I", # isort
|
|
77
|
+
]
|