cdsetool 0.2.13__tar.gz → 0.3.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. {cdsetool-0.2.13 → cdsetool-0.3.2}/LICENSE +1 -1
  2. {cdsetool-0.2.13/src/cdsetool.egg-info → cdsetool-0.3.2}/PKG-INFO +147 -34
  3. {cdsetool-0.2.13 → cdsetool-0.3.2}/README.md +138 -25
  4. {cdsetool-0.2.13 → cdsetool-0.3.2}/pyproject.toml +22 -8
  5. cdsetool-0.3.2/src/cdsetool/_attributes.py +768 -0
  6. {cdsetool-0.2.13 → cdsetool-0.3.2}/src/cdsetool/_processing.py +1 -2
  7. cdsetool-0.3.2/src/cdsetool/cli.py +172 -0
  8. {cdsetool-0.2.13 → cdsetool-0.3.2}/src/cdsetool/credentials.py +4 -4
  9. cdsetool-0.3.2/src/cdsetool/download.py +279 -0
  10. {cdsetool-0.2.13 → cdsetool-0.3.2}/src/cdsetool/logger.py +6 -4
  11. {cdsetool-0.2.13 → cdsetool-0.3.2}/src/cdsetool/monitor.py +15 -15
  12. cdsetool-0.3.2/src/cdsetool/query.py +599 -0
  13. {cdsetool-0.2.13 → cdsetool-0.3.2/src/cdsetool.egg-info}/PKG-INFO +147 -34
  14. {cdsetool-0.2.13 → cdsetool-0.3.2}/src/cdsetool.egg-info/SOURCES.txt +1 -0
  15. cdsetool-0.3.2/src/cdsetool.egg-info/requires.txt +12 -0
  16. cdsetool-0.2.13/src/cdsetool/cli.py +0 -126
  17. cdsetool-0.2.13/src/cdsetool/download.py +0 -159
  18. cdsetool-0.2.13/src/cdsetool/query.py +0 -298
  19. cdsetool-0.2.13/src/cdsetool.egg-info/requires.txt +0 -13
  20. {cdsetool-0.2.13 → cdsetool-0.3.2}/setup.cfg +0 -0
  21. {cdsetool-0.2.13 → cdsetool-0.3.2}/src/cdsetool/__init__.py +0 -0
  22. {cdsetool-0.2.13 → cdsetool-0.3.2}/src/cdsetool/py.typed +0 -0
  23. {cdsetool-0.2.13 → cdsetool-0.3.2}/src/cdsetool.egg-info/dependency_links.txt +0 -0
  24. {cdsetool-0.2.13 → cdsetool-0.3.2}/src/cdsetool.egg-info/entry_points.txt +0 -0
  25. {cdsetool-0.2.13 → cdsetool-0.3.2}/src/cdsetool.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  MIT License
2
2
 
3
- Copyright (c) 2023 Styrelsen for Dataforsyning og Infrastruktur
3
+ Copyright (c) 2023-2025 CDSETool Contributors
4
4
 
5
5
  Permission is hereby granted, free of charge, to any person obtaining a copy
6
6
  of this software and associated documentation files (the "Software"), to deal
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.4
2
2
  Name: cdsetool
3
- Version: 0.2.13
3
+ Version: 0.3.2
4
4
  Summary: Tools & CLI for interacting with CDSE product APIs
5
5
  Author-email: Jacob Vejby <javej@sdfi.dk>
6
6
  Project-URL: Homepage, https://github.com/CDSETool/CDSETool
@@ -11,17 +11,17 @@ Requires-Python: >=3.9
11
11
  Description-Content-Type: text/markdown
12
12
  License-File: LICENSE
13
13
  Requires-Dist: typer<1,>=0.9
14
- Requires-Dist: rich<14,>=13.6
14
+ Requires-Dist: rich<16,>=13.6
15
15
  Requires-Dist: requests<3,>=2.28.1
16
- Requires-Dist: pyjwt[crypto]<2.10,>=2.8
16
+ Requires-Dist: pyjwt[crypto]<2.15,>=2.8
17
17
  Requires-Dist: geopandas>=0.13.2
18
18
  Provides-Extra: test
19
- Requires-Dist: pylint==3.3.1; extra == "test"
20
- Requires-Dist: pytest==8.3.3; extra == "test"
21
- Requires-Dist: pytest-cov==5.0.0; extra == "test"
19
+ Requires-Dist: pylint==3.3.9; extra == "test"
20
+ Requires-Dist: pytest==8.4.2; extra == "test"
21
+ Requires-Dist: pytest-cov==7.1.0; extra == "test"
22
22
  Requires-Dist: requests-mock==1.12.1; extra == "test"
23
- Requires-Dist: ruff==0.6.6; extra == "test"
24
- Requires-Dist: pytest-mock==3.14.0; extra == "test"
23
+ Requires-Dist: pytest-mock==3.15.1; extra == "test"
24
+ Dynamic: license-file
25
25
 
26
26
  # CDSETool
27
27
 
@@ -38,12 +38,11 @@ from cdsetool.monitor import StatusMonitor
38
38
  from datetime import date
39
39
 
40
40
  features = query_features(
41
- "Sentinel1",
41
+ "SENTINEL-1",
42
42
  {
43
- "startDate": "2020-12-20",
44
- "completionDate": date(2020, 12, 25),
43
+ "contentDateStartGe": "2020-12-20",
44
+ "contentDateStartLe": date(2020, 12, 25),
45
45
  "processingLevel": "LEVEL1",
46
- "sensorMode": "IW",
47
46
  "productType": "IW_GRDH_1S",
48
47
  "geometry": shape_to_wkt("path/to/shapefile.shp"),
49
48
  },
@@ -65,9 +64,9 @@ list(
65
64
  Or use the CLI:
66
65
 
67
66
  ```bash
68
- cdsetool query search Sentinel2 --search-term startDate=2020-01-01 --search-term completionDate=2020-01-10 --search-term processingLevel=S2MSI2A --search-term box="4","51","4.5","52"
67
+ cdsetool query search SENTINEL-2 --search-term contentDateStartGe=2020-01-01 --search-term contentDateStartLe=2020-01-10 --search-term productType=S2MSI2A
69
68
 
70
- cdsetool download Sentinel2 PATH/TO/DIR --concurrency 4 --search-term startDate=2020-01-01 --search-term completionDate=2020-01-10 --search-term processingLevel=S2MSI2A --search-term box="4","51","4.5","52"
69
+ cdsetool download SENTINEL-2 PATH/TO/DIR --concurrency 4 --search-term contentDateStartGe=2020-01-01 --search-term contentDateStartLe=2020-01-10 --search-term productType=S2MSI2A
71
70
  ```
72
71
 
73
72
  ## Table of Contents
@@ -76,7 +75,7 @@ cdsetool download Sentinel2 PATH/TO/DIR --concurrency 4 --search-term startDate=
76
75
  * [About CDSETool](#about-cdsetool)
77
76
  * [Quick Start](#quick-start)
78
77
  * [Table of Contents](#table-of-contents)
79
- * [Installatson](#installation)
78
+ * [Installation](#installation)
80
79
  * [Usage](#usage)
81
80
  + [Querying features](#querying-features)
82
81
  - [Querying by shapes](#querying-by-shapes)
@@ -96,29 +95,31 @@ cdsetool download Sentinel2 PATH/TO/DIR --concurrency 4 --search-term startDate=
96
95
  Install `cdsetool` using pip:
97
96
 
98
97
  ```bash
99
- pip install cdsetool==0.2.13
98
+ pip install cdsetool==0.3.2
100
99
  ```
101
100
 
102
101
  ## Usage
103
102
 
104
103
  ### Querying features
105
104
 
106
- Querying is always done in batches, returning `len(results) <= maxRecords` records each time.
105
+ Querying is always done in batches, returning `len(results) <= top` records each time.
107
106
  A local buffer is filled and gradually emptied as results are yielded. When the buffer is empty,
108
107
  more results will be requested and the process repeated until no more results are available, or
109
108
  the iterator is discarded.
110
109
 
110
+ **Important**: The API has a pagination limit of 10,000 results per query. If your query returns more results, you'll need to narrow your search criteria (e.g., use smaller date ranges).
111
+
111
112
  Since downloading features is the most common use-case, `query_features` assumes that the query will run till the end.
112
- Because of this, the batch size is set to `2000`, which is the size limit set by CDSE.
113
+ Because of this, the batch size is set to `1000`, which is the size limit set by CDSE.
113
114
 
114
115
  ```python
115
116
  from cdsetool.query import query_features
116
117
 
117
- collection = "Sentinel2"
118
+ collection = "SENTINEL-2"
118
119
  search_terms = {
119
- "maxRecords": "100", # batch size, between 1 and 2000 (default: 2000).
120
- "startDate": "1999-01-01",
121
- "processingLevel": "S2MSI1C"
120
+ "top": 100, # batch size, between 1 and 1000 (default: 1000)
121
+ "contentDateStartGe": "2024-01-01",
122
+ "productType": "S2MSI1C"
122
123
  }
123
124
 
124
125
  # wait for a single batch to finish, yield results immediately
@@ -136,7 +137,53 @@ featureB = next(iterator)
136
137
  # ...
137
138
  ```
138
139
 
139
- #### Querying by shapes
140
+ #### Query Options
141
+
142
+ **Product Format**
143
+
144
+ Query results are returned directly from the Copernicus API. Each product has the following structure:
145
+
146
+ ```python
147
+ product = {
148
+ "Id": "uuid-string",
149
+ "Name": "S2A_MSIL2A_20240110T105421_...",
150
+ "Collection": "SENTINEL-2",
151
+ "ContentDate": {"Start": "2024-01-10T10:54:21Z", "End": "..."},
152
+ "Online": True,
153
+ "ContentLength": 1043654649,
154
+ "GeoFootprint": {"type": "Polygon", "coordinates": [...]},
155
+ "Attributes": [
156
+ {"Name": "productType", "Value": "S2MSI2A"},
157
+ {"Name": "cloudCover", "Value": 5.2},
158
+ ...,
159
+ ],
160
+ }
161
+ ```
162
+
163
+ **Expand Product Attributes**
164
+
165
+ By default, query results do not include product attributes (productType, cloudCover, platform, instrument, etc.). To include product attributes, you need to request this from the server using the option `expand_attributes` and can then access the attributes using the function `get_product_attribute()`:
166
+
167
+ ```python
168
+ from cdsetool.query import query_features, get_product_attribute
169
+
170
+ features = query_features(
171
+ "SENTINEL-2",
172
+ {"contentDateStartGe": "2024-01-01"},
173
+ options={"expand_attributes": True},
174
+ )
175
+ feature = features[0]
176
+
177
+ # Access basic properties directly
178
+ print(feature["Name"]) # Product name
179
+ print(feature["Id"]) # Product UUID
180
+
181
+ # Access attributes using helper function
182
+ cloud_cover = get_product_attribute(feature, "cloudCover")
183
+ product_type = get_product_attribute(feature, "productType")
184
+ ```
185
+
186
+ #### Querying by geometry
140
187
 
141
188
  To query by shapes, you must first convert your shape to Well Known Text (WKT). The included
142
189
  `shape_to_wkt` can solve this.
@@ -145,8 +192,7 @@ To query by shapes, you must first convert your shape to Well Known Text (WKT).
145
192
  from cdsetool.query import query_features, shape_to_wkt
146
193
 
147
194
  geometry = shape_to_wkt("path/to/shape.shp")
148
-
149
- features = query_features("Sentinel3", {"geometry": geometry})
195
+ features = query_features("SENTINEL-3", {"geometry": geometry})
150
196
  ```
151
197
 
152
198
  #### Querying by lists of parameters
@@ -160,7 +206,7 @@ from cdsetool.query import query_features
160
206
  tile_ids = ["32TPT", "32UPU", "32UPU", "31RFL", "37XDA"]
161
207
 
162
208
  for tile_id in tile_ids:
163
- features = query_features("Sentinel2", {"tileId": tile_id})
209
+ features = query_features("SENTINEL-2", {"tileId": tile_id})
164
210
  for feature in features:
165
211
  # do things with feature
166
212
  ```
@@ -169,14 +215,44 @@ for tile_id in tile_ids:
169
215
 
170
216
  Its quite common to query for features created before, after or between dates.
171
217
 
218
+ Search terms support comparison operator suffixes:
219
+
220
+ | Suffix | Meaning | Example |
221
+ |--------|----------------------------|----------------------|
222
+ | `Eq` | equals (=) | `contentDateStartEq` |
223
+ | `Gt` | greater than (>) | `contentDateStartGt` |
224
+ | `Ge` | greater than or equal (>=) | `contentDateStartGe` |
225
+ | `Lt` | less than (<) | `contentDateStartLt` |
226
+ | `Le` | less than or equal (<=) | `contentDateStartLe` |
227
+
228
+ `Eq` can be applied on any field but the other suffixes can only be applied to numeric and date fields.
229
+
230
+ Interval syntax is only allowed on the base name, not on suffixed variants, and only on numeric and date fields.
231
+
232
+ | Interval notation | Suffixes to combine | Meaning |
233
+ |-------------------|---------------------|--------------------------------|
234
+ | [a, b] | `Ge` + `Le` | a <= value <= b (closed) |
235
+ | (a, b) | `Gt` + `Lt` | a < value < b (open) |
236
+ | [a, b) | `Ge` + `Lt` | a <= value < b (half-open) |
237
+ | (a, b] | `Gt` + `Le` | a < value <= b (half-open) |
238
+
172
239
  ```python
173
240
  from cdsetool.query import query_features
174
241
  from datetime import date, datetime
175
242
 
176
- date_from = date(2020, 1, 1) # or datetime(2020, 1, 1, 23, 59, 59, 123456) or "2020-01-01" or "2020-01-01T23:59:59.123456Z"
243
+ date_from = date(
244
+ 2020, 1, 1
245
+ ) # or datetime(2020, 1, 1, 23, 59, 59, 123456) or "2020-01-01" or "2020-01-01T23:59:59.123456Z"
177
246
  date_to = date(2020, 12, 31)
178
247
 
179
- features = query_features("Sentinel2", {"startDate": date_from, "completionDate": date_to})
248
+ features = query_features(
249
+ "SENTINEL-2",
250
+ {
251
+ "contentDateStartGe": date_from,
252
+ "contentDateStartLe": date_to,
253
+ "cloudCover": "[0, 30]",
254
+ },
255
+ )
180
256
  ```
181
257
 
182
258
  #### Listing search terms
@@ -187,13 +263,13 @@ use the CLI:
187
263
  ```python
188
264
  from cdsetool.query import describe_collection
189
265
 
190
- search_terms = describe_collection("Sentinel2").keys()
266
+ search_terms = describe_collection("SENTINEL-2").keys()
191
267
  print(search_terms)
192
268
  ```
193
269
 
194
270
  And the CLI:
195
271
  ```bash
196
- $ cdsetool query search-terms Sentinel2
272
+ $ cdsetool query search-terms SENTINEL-2
197
273
  ```
198
274
 
199
275
  ### Downloading features
@@ -240,7 +316,7 @@ Credentials can be validated using the `validate_credentials` function which wil
240
316
  ```python
241
317
  from cdsetool.credentials import validate_credentials
242
318
 
243
- validate_credentials(username='user', password='password')
319
+ validate_credentials(username="user", password="password")
244
320
  ```
245
321
 
246
322
  If None are passed to username and password, `validate_credentials` will validate `.netrc`
@@ -257,7 +333,10 @@ The downloaded feature ids are yielded, so its required to await the results.
257
333
  from cdsetool.query import query_features
258
334
  from cdsetool.download import download_features
259
335
 
260
- features = query_features("Sentinel2")
336
+ features = query_features(
337
+ "SENTINEL-2",
338
+ {"contentDateStartGe": "2024-01-01", "contentDateStartLe": "2024-01-10"},
339
+ )
261
340
 
262
341
  download_path = "/path/to/download/folder"
263
342
  downloads = download_features(features, download_path, {"concurrency": 4})
@@ -278,13 +357,47 @@ Its possible to download features sequentially in a single thread if desired.
278
357
  from cdsetool.query import query_features
279
358
  from cdsetool.download import download_feature
280
359
 
281
- features = query_features("Sentinel2")
360
+ features = query_features(
361
+ "SENTINEL-2",
362
+ {"contentDateStartGe": "2024-01-01", "contentDateStartLe": "2024-01-10"},
363
+ )
282
364
 
283
365
  download_path = "/path/to/download/folder"
284
366
  for feature in features:
285
367
  download_feature(feature, download_path)
286
368
  ```
287
369
 
370
+ #### Download specific files within features
371
+
372
+ It's possible to download specific files within products bundles using Unix filename pattern matching.
373
+
374
+ It can be used in CDSETool:
375
+
376
+ - Through the `filter_pattern` option of `download_features` and `download_feature`:
377
+ ```python
378
+ from cdsetool.query import query_features
379
+ from cdsetool.download import download_features
380
+
381
+ features = query_features(
382
+ "SENTINEL-2",
383
+ {"contentDateStartGe": "2024-01-01", "contentDateStartLe": "2024-01-10"},
384
+ )
385
+
386
+ download_path = "/path/to/download/folder"
387
+ filter_pattern = "*TCI.jp2"
388
+ downloads = download_features(
389
+ features, download_path, {"filter_pattern": filter_pattern}
390
+ )
391
+
392
+ for id in downloads:
393
+ print(f"feature {id} downloaded")
394
+ ```
395
+
396
+ - Or through the CLI:
397
+ ```
398
+ cdsetool download SENTINEL-2 PATH/TO/DIR --filter-pattern *TCI.jp2 --concurrency 4 --search-term contentDateStartGe=2024-01-01 --search-term contentDateStartLe=2024-01-10 --search-term productType=S2MSI2A
399
+ ```
400
+
288
401
  ## Roadmap
289
402
 
290
403
  - [X] Query schema validation
@@ -13,12 +13,11 @@ from cdsetool.monitor import StatusMonitor
13
13
  from datetime import date
14
14
 
15
15
  features = query_features(
16
- "Sentinel1",
16
+ "SENTINEL-1",
17
17
  {
18
- "startDate": "2020-12-20",
19
- "completionDate": date(2020, 12, 25),
18
+ "contentDateStartGe": "2020-12-20",
19
+ "contentDateStartLe": date(2020, 12, 25),
20
20
  "processingLevel": "LEVEL1",
21
- "sensorMode": "IW",
22
21
  "productType": "IW_GRDH_1S",
23
22
  "geometry": shape_to_wkt("path/to/shapefile.shp"),
24
23
  },
@@ -40,9 +39,9 @@ list(
40
39
  Or use the CLI:
41
40
 
42
41
  ```bash
43
- cdsetool query search Sentinel2 --search-term startDate=2020-01-01 --search-term completionDate=2020-01-10 --search-term processingLevel=S2MSI2A --search-term box="4","51","4.5","52"
42
+ cdsetool query search SENTINEL-2 --search-term contentDateStartGe=2020-01-01 --search-term contentDateStartLe=2020-01-10 --search-term productType=S2MSI2A
44
43
 
45
- cdsetool download Sentinel2 PATH/TO/DIR --concurrency 4 --search-term startDate=2020-01-01 --search-term completionDate=2020-01-10 --search-term processingLevel=S2MSI2A --search-term box="4","51","4.5","52"
44
+ cdsetool download SENTINEL-2 PATH/TO/DIR --concurrency 4 --search-term contentDateStartGe=2020-01-01 --search-term contentDateStartLe=2020-01-10 --search-term productType=S2MSI2A
46
45
  ```
47
46
 
48
47
  ## Table of Contents
@@ -51,7 +50,7 @@ cdsetool download Sentinel2 PATH/TO/DIR --concurrency 4 --search-term startDate=
51
50
  * [About CDSETool](#about-cdsetool)
52
51
  * [Quick Start](#quick-start)
53
52
  * [Table of Contents](#table-of-contents)
54
- * [Installatson](#installation)
53
+ * [Installation](#installation)
55
54
  * [Usage](#usage)
56
55
  + [Querying features](#querying-features)
57
56
  - [Querying by shapes](#querying-by-shapes)
@@ -71,29 +70,31 @@ cdsetool download Sentinel2 PATH/TO/DIR --concurrency 4 --search-term startDate=
71
70
  Install `cdsetool` using pip:
72
71
 
73
72
  ```bash
74
- pip install cdsetool==0.2.13
73
+ pip install cdsetool==0.3.2
75
74
  ```
76
75
 
77
76
  ## Usage
78
77
 
79
78
  ### Querying features
80
79
 
81
- Querying is always done in batches, returning `len(results) <= maxRecords` records each time.
80
+ Querying is always done in batches, returning `len(results) <= top` records each time.
82
81
  A local buffer is filled and gradually emptied as results are yielded. When the buffer is empty,
83
82
  more results will be requested and the process repeated until no more results are available, or
84
83
  the iterator is discarded.
85
84
 
85
+ **Important**: The API has a pagination limit of 10,000 results per query. If your query returns more results, you'll need to narrow your search criteria (e.g., use smaller date ranges).
86
+
86
87
  Since downloading features is the most common use-case, `query_features` assumes that the query will run till the end.
87
- Because of this, the batch size is set to `2000`, which is the size limit set by CDSE.
88
+ Because of this, the batch size is set to `1000`, which is the size limit set by CDSE.
88
89
 
89
90
  ```python
90
91
  from cdsetool.query import query_features
91
92
 
92
- collection = "Sentinel2"
93
+ collection = "SENTINEL-2"
93
94
  search_terms = {
94
- "maxRecords": "100", # batch size, between 1 and 2000 (default: 2000).
95
- "startDate": "1999-01-01",
96
- "processingLevel": "S2MSI1C"
95
+ "top": 100, # batch size, between 1 and 1000 (default: 1000)
96
+ "contentDateStartGe": "2024-01-01",
97
+ "productType": "S2MSI1C"
97
98
  }
98
99
 
99
100
  # wait for a single batch to finish, yield results immediately
@@ -111,7 +112,53 @@ featureB = next(iterator)
111
112
  # ...
112
113
  ```
113
114
 
114
- #### Querying by shapes
115
+ #### Query Options
116
+
117
+ **Product Format**
118
+
119
+ Query results are returned directly from the Copernicus API. Each product has the following structure:
120
+
121
+ ```python
122
+ product = {
123
+ "Id": "uuid-string",
124
+ "Name": "S2A_MSIL2A_20240110T105421_...",
125
+ "Collection": "SENTINEL-2",
126
+ "ContentDate": {"Start": "2024-01-10T10:54:21Z", "End": "..."},
127
+ "Online": True,
128
+ "ContentLength": 1043654649,
129
+ "GeoFootprint": {"type": "Polygon", "coordinates": [...]},
130
+ "Attributes": [
131
+ {"Name": "productType", "Value": "S2MSI2A"},
132
+ {"Name": "cloudCover", "Value": 5.2},
133
+ ...,
134
+ ],
135
+ }
136
+ ```
137
+
138
+ **Expand Product Attributes**
139
+
140
+ By default, query results do not include product attributes (productType, cloudCover, platform, instrument, etc.). To include product attributes, you need to request this from the server using the option `expand_attributes` and can then access the attributes using the function `get_product_attribute()`:
141
+
142
+ ```python
143
+ from cdsetool.query import query_features, get_product_attribute
144
+
145
+ features = query_features(
146
+ "SENTINEL-2",
147
+ {"contentDateStartGe": "2024-01-01"},
148
+ options={"expand_attributes": True},
149
+ )
150
+ feature = features[0]
151
+
152
+ # Access basic properties directly
153
+ print(feature["Name"]) # Product name
154
+ print(feature["Id"]) # Product UUID
155
+
156
+ # Access attributes using helper function
157
+ cloud_cover = get_product_attribute(feature, "cloudCover")
158
+ product_type = get_product_attribute(feature, "productType")
159
+ ```
160
+
161
+ #### Querying by geometry
115
162
 
116
163
  To query by shapes, you must first convert your shape to Well Known Text (WKT). The included
117
164
  `shape_to_wkt` can solve this.
@@ -120,8 +167,7 @@ To query by shapes, you must first convert your shape to Well Known Text (WKT).
120
167
  from cdsetool.query import query_features, shape_to_wkt
121
168
 
122
169
  geometry = shape_to_wkt("path/to/shape.shp")
123
-
124
- features = query_features("Sentinel3", {"geometry": geometry})
170
+ features = query_features("SENTINEL-3", {"geometry": geometry})
125
171
  ```
126
172
 
127
173
  #### Querying by lists of parameters
@@ -135,7 +181,7 @@ from cdsetool.query import query_features
135
181
  tile_ids = ["32TPT", "32UPU", "32UPU", "31RFL", "37XDA"]
136
182
 
137
183
  for tile_id in tile_ids:
138
- features = query_features("Sentinel2", {"tileId": tile_id})
184
+ features = query_features("SENTINEL-2", {"tileId": tile_id})
139
185
  for feature in features:
140
186
  # do things with feature
141
187
  ```
@@ -144,14 +190,44 @@ for tile_id in tile_ids:
144
190
 
145
191
  Its quite common to query for features created before, after or between dates.
146
192
 
193
+ Search terms support comparison operator suffixes:
194
+
195
+ | Suffix | Meaning | Example |
196
+ |--------|----------------------------|----------------------|
197
+ | `Eq` | equals (=) | `contentDateStartEq` |
198
+ | `Gt` | greater than (>) | `contentDateStartGt` |
199
+ | `Ge` | greater than or equal (>=) | `contentDateStartGe` |
200
+ | `Lt` | less than (<) | `contentDateStartLt` |
201
+ | `Le` | less than or equal (<=) | `contentDateStartLe` |
202
+
203
+ `Eq` can be applied on any field but the other suffixes can only be applied to numeric and date fields.
204
+
205
+ Interval syntax is only allowed on the base name, not on suffixed variants, and only on numeric and date fields.
206
+
207
+ | Interval notation | Suffixes to combine | Meaning |
208
+ |-------------------|---------------------|--------------------------------|
209
+ | [a, b] | `Ge` + `Le` | a <= value <= b (closed) |
210
+ | (a, b) | `Gt` + `Lt` | a < value < b (open) |
211
+ | [a, b) | `Ge` + `Lt` | a <= value < b (half-open) |
212
+ | (a, b] | `Gt` + `Le` | a < value <= b (half-open) |
213
+
147
214
  ```python
148
215
  from cdsetool.query import query_features
149
216
  from datetime import date, datetime
150
217
 
151
- date_from = date(2020, 1, 1) # or datetime(2020, 1, 1, 23, 59, 59, 123456) or "2020-01-01" or "2020-01-01T23:59:59.123456Z"
218
+ date_from = date(
219
+ 2020, 1, 1
220
+ ) # or datetime(2020, 1, 1, 23, 59, 59, 123456) or "2020-01-01" or "2020-01-01T23:59:59.123456Z"
152
221
  date_to = date(2020, 12, 31)
153
222
 
154
- features = query_features("Sentinel2", {"startDate": date_from, "completionDate": date_to})
223
+ features = query_features(
224
+ "SENTINEL-2",
225
+ {
226
+ "contentDateStartGe": date_from,
227
+ "contentDateStartLe": date_to,
228
+ "cloudCover": "[0, 30]",
229
+ },
230
+ )
155
231
  ```
156
232
 
157
233
  #### Listing search terms
@@ -162,13 +238,13 @@ use the CLI:
162
238
  ```python
163
239
  from cdsetool.query import describe_collection
164
240
 
165
- search_terms = describe_collection("Sentinel2").keys()
241
+ search_terms = describe_collection("SENTINEL-2").keys()
166
242
  print(search_terms)
167
243
  ```
168
244
 
169
245
  And the CLI:
170
246
  ```bash
171
- $ cdsetool query search-terms Sentinel2
247
+ $ cdsetool query search-terms SENTINEL-2
172
248
  ```
173
249
 
174
250
  ### Downloading features
@@ -215,7 +291,7 @@ Credentials can be validated using the `validate_credentials` function which wil
215
291
  ```python
216
292
  from cdsetool.credentials import validate_credentials
217
293
 
218
- validate_credentials(username='user', password='password')
294
+ validate_credentials(username="user", password="password")
219
295
  ```
220
296
 
221
297
  If None are passed to username and password, `validate_credentials` will validate `.netrc`
@@ -232,7 +308,10 @@ The downloaded feature ids are yielded, so its required to await the results.
232
308
  from cdsetool.query import query_features
233
309
  from cdsetool.download import download_features
234
310
 
235
- features = query_features("Sentinel2")
311
+ features = query_features(
312
+ "SENTINEL-2",
313
+ {"contentDateStartGe": "2024-01-01", "contentDateStartLe": "2024-01-10"},
314
+ )
236
315
 
237
316
  download_path = "/path/to/download/folder"
238
317
  downloads = download_features(features, download_path, {"concurrency": 4})
@@ -253,13 +332,47 @@ Its possible to download features sequentially in a single thread if desired.
253
332
  from cdsetool.query import query_features
254
333
  from cdsetool.download import download_feature
255
334
 
256
- features = query_features("Sentinel2")
335
+ features = query_features(
336
+ "SENTINEL-2",
337
+ {"contentDateStartGe": "2024-01-01", "contentDateStartLe": "2024-01-10"},
338
+ )
257
339
 
258
340
  download_path = "/path/to/download/folder"
259
341
  for feature in features:
260
342
  download_feature(feature, download_path)
261
343
  ```
262
344
 
345
+ #### Download specific files within features
346
+
347
+ It's possible to download specific files within products bundles using Unix filename pattern matching.
348
+
349
+ It can be used in CDSETool:
350
+
351
+ - Through the `filter_pattern` option of `download_features` and `download_feature`:
352
+ ```python
353
+ from cdsetool.query import query_features
354
+ from cdsetool.download import download_features
355
+
356
+ features = query_features(
357
+ "SENTINEL-2",
358
+ {"contentDateStartGe": "2024-01-01", "contentDateStartLe": "2024-01-10"},
359
+ )
360
+
361
+ download_path = "/path/to/download/folder"
362
+ filter_pattern = "*TCI.jp2"
363
+ downloads = download_features(
364
+ features, download_path, {"filter_pattern": filter_pattern}
365
+ )
366
+
367
+ for id in downloads:
368
+ print(f"feature {id} downloaded")
369
+ ```
370
+
371
+ - Or through the CLI:
372
+ ```
373
+ cdsetool download SENTINEL-2 PATH/TO/DIR --filter-pattern *TCI.jp2 --concurrency 4 --search-term contentDateStartGe=2024-01-01 --search-term contentDateStartLe=2024-01-10 --search-term productType=S2MSI2A
374
+ ```
375
+
263
376
  ## Roadmap
264
377
 
265
378
  - [X] Query schema validation
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "cdsetool"
7
- version = "0.2.13"
7
+ version = "0.3.2"
8
8
  authors = [
9
9
  { name="Jacob Vejby", email="javej@sdfi.dk" },
10
10
  ]
@@ -17,19 +17,24 @@ classifiers = [
17
17
  ]
18
18
  dependencies = [
19
19
  "typer >= 0.9,< 1",
20
- "rich >= 13.6,< 14",
20
+ "rich >= 13.6,< 16",
21
21
  "requests >= 2.28.1,< 3",
22
- "pyjwt[crypto] >= 2.8,< 2.10",
22
+ "pyjwt[crypto] >= 2.8,< 2.15",
23
23
  "geopandas >= 0.13.2",
24
24
  ]
25
+ [dependency-groups]
26
+ dev = [
27
+ "ruff==0.16.7",
28
+ ]
29
+ # (Jan 30 2025) Pip does not support dependency-groups yet, so
30
+ # leave the optional dependencies for everything except Ruff.
25
31
  [project.optional-dependencies]
26
32
  test = [
27
- "pylint==3.3.1",
28
- "pytest==8.3.3",
29
- "pytest-cov==5.0.0",
33
+ "pylint==3.3.9",
34
+ "pytest==8.4.2",
35
+ "pytest-cov==7.1.0",
30
36
  "requests-mock==1.12.1",
31
- "ruff==0.6.6",
32
- "pytest-mock==3.14.0",
37
+ "pytest-mock==3.15.1",
33
38
  ]
34
39
 
35
40
  [project.urls]
@@ -46,11 +51,15 @@ max-locals = 20
46
51
  max-line-length = "88"
47
52
  disable="fixme"
48
53
 
54
+ [tool.pylint.imports]
55
+ disable="wrong-import-order"
56
+
49
57
  [tool.pyright]
50
58
  exclude = ["**/__pycache__", ".venv"]
51
59
 
52
60
  # Can be strict too, see more fine-grained settings at:
53
61
  # https://github.com/microsoft/pyright/blob/main/docs/configuration.md#diagnostic-settings-defaults
62
+ # reportMissingParameterType = true
54
63
  typeCheckingMode = "standard"
55
64
 
56
65
  executionEnvironments = [
@@ -61,3 +70,8 @@ executionEnvironments = [
61
70
  addopts = [
62
71
  "--import-mode=importlib",
63
72
  ]
73
+
74
+ [tool.ruff.lint]
75
+ select = [
76
+ "I", # isort
77
+ ]