freva-client 2502.0.0__tar.gz → 2505.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {freva_client-2502.0.0 → freva_client-2505.0.0}/PKG-INFO +1 -1
- {freva_client-2502.0.0 → freva_client-2505.0.0}/src/freva_client/__init__.py +1 -1
- {freva_client-2502.0.0 → freva_client-2505.0.0}/src/freva_client/cli/cli_parser.py +2 -0
- {freva_client-2502.0.0 → freva_client-2505.0.0}/src/freva_client/cli/databrowser_cli.py +267 -25
- {freva_client-2502.0.0 → freva_client-2505.0.0}/src/freva_client/query.py +166 -8
- {freva_client-2502.0.0 → freva_client-2505.0.0}/src/freva_client/utils/databrowser_utils.py +5 -0
- {freva_client-2502.0.0 → freva_client-2505.0.0}/MANIFEST.in +0 -0
- {freva_client-2502.0.0 → freva_client-2505.0.0}/README.md +0 -0
- {freva_client-2502.0.0 → freva_client-2505.0.0}/assets/share/freva/freva.toml +0 -0
- {freva_client-2502.0.0 → freva_client-2505.0.0}/pyproject.toml +0 -0
- {freva_client-2502.0.0 → freva_client-2505.0.0}/src/freva_client/__main__.py +0 -0
- {freva_client-2502.0.0 → freva_client-2505.0.0}/src/freva_client/auth.py +0 -0
- {freva_client-2502.0.0 → freva_client-2505.0.0}/src/freva_client/cli/__init__.py +0 -0
- {freva_client-2502.0.0 → freva_client-2505.0.0}/src/freva_client/cli/auth_cli.py +0 -0
- {freva_client-2502.0.0 → freva_client-2505.0.0}/src/freva_client/cli/cli_app.py +0 -0
- {freva_client-2502.0.0 → freva_client-2505.0.0}/src/freva_client/cli/cli_utils.py +0 -0
- {freva_client-2502.0.0 → freva_client-2505.0.0}/src/freva_client/py.typed +0 -0
- {freva_client-2502.0.0 → freva_client-2505.0.0}/src/freva_client/utils/__init__.py +0 -0
- {freva_client-2502.0.0 → freva_client-2505.0.0}/src/freva_client/utils/logger.py +0 -0
|
@@ -7,7 +7,7 @@ import json
|
|
|
7
7
|
from enum import Enum
|
|
8
8
|
from pathlib import Path
|
|
9
9
|
from tempfile import NamedTemporaryFile
|
|
10
|
-
from typing import Dict, List, Literal, Optional, Union, cast
|
|
10
|
+
from typing import Dict, List, Literal, Optional, Tuple, Union, cast
|
|
11
11
|
|
|
12
12
|
import typer
|
|
13
13
|
import xarray as xr
|
|
@@ -47,7 +47,7 @@ class Flavours(str, Enum):
|
|
|
47
47
|
user = "user"
|
|
48
48
|
|
|
49
49
|
|
|
50
|
-
class
|
|
50
|
+
class SelectMethod(str, Enum):
|
|
51
51
|
"""Literal implementation for the cli."""
|
|
52
52
|
|
|
53
53
|
strict = "strict"
|
|
@@ -55,18 +55,44 @@ class TimeSelect(str, Enum):
|
|
|
55
55
|
file = "file"
|
|
56
56
|
|
|
57
57
|
@staticmethod
|
|
58
|
-
def get_help() -> str:
|
|
59
|
-
"""Generate the help string.
|
|
58
|
+
def get_help(context: str) -> str:
|
|
59
|
+
"""Generate the help string for time or bbox selection methods.
|
|
60
|
+
|
|
61
|
+
Parameters
|
|
62
|
+
----------
|
|
63
|
+
context: str, default: "time"
|
|
64
|
+
Either "time" or "bbox" to generate appropriate help text.
|
|
65
|
+
"""
|
|
66
|
+
examples = {
|
|
67
|
+
"time": ("2000 to 2012", "2010 to 2020"),
|
|
68
|
+
"bbox": ("-10 10 -10 10", "0 5 0 5")
|
|
69
|
+
}
|
|
70
|
+
descriptions = {
|
|
71
|
+
"time": {
|
|
72
|
+
"unit": "time period",
|
|
73
|
+
"start_end": "start or end period",
|
|
74
|
+
"subset": "time period"
|
|
75
|
+
},
|
|
76
|
+
"bbox": {
|
|
77
|
+
"unit": "spatial extent",
|
|
78
|
+
"start_end": "any part of the extent",
|
|
79
|
+
"subset": "spatial extent"
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
context_info = descriptions.get(context, descriptions["time"])
|
|
84
|
+
example = examples.get(context, examples["time"])
|
|
85
|
+
|
|
60
86
|
return (
|
|
61
|
-
"Operator that specifies how the
|
|
87
|
+
f"Operator that specifies how the {context_info['unit']} is selected. "
|
|
62
88
|
"Choose from flexible (default), strict or file. "
|
|
63
89
|
"``strict`` returns only those files that have the *entire* "
|
|
64
|
-
"
|
|
65
|
-
"not select files containing data from
|
|
66
|
-
"the ``strict`` method. ``flexible`` will select those files "
|
|
67
|
-
"
|
|
68
|
-
"
|
|
69
|
-
"
|
|
90
|
+
f"{context_info['unit']} covered. The {context} search ``{example[0]}`` "
|
|
91
|
+
f"will not select files containing data from {example[1]} with "
|
|
92
|
+
"the ``strict`` method. ``flexible`` will select those files as "
|
|
93
|
+
f"``flexible`` returns those files that have {context_info['start_end']} "
|
|
94
|
+
f"covered. ``file`` will only return files where the entire "
|
|
95
|
+
f"{context_info['subset']} is contained within *one single* file."
|
|
70
96
|
)
|
|
71
97
|
|
|
72
98
|
|
|
@@ -124,11 +150,11 @@ def metadata_search(
|
|
|
124
150
|
"of climate datasets to query."
|
|
125
151
|
),
|
|
126
152
|
),
|
|
127
|
-
time_select:
|
|
153
|
+
time_select: SelectMethod = typer.Option(
|
|
128
154
|
"flexible",
|
|
129
155
|
"-ts",
|
|
130
156
|
"--time-select",
|
|
131
|
-
help=
|
|
157
|
+
help=SelectMethod.get_help("time"),
|
|
132
158
|
),
|
|
133
159
|
time: Optional[str] = typer.Option(
|
|
134
160
|
None,
|
|
@@ -144,6 +170,24 @@ def metadata_search(
|
|
|
144
170
|
" valid."
|
|
145
171
|
),
|
|
146
172
|
),
|
|
173
|
+
bbox: Optional[Tuple[float, float, float, float]] = typer.Option(
|
|
174
|
+
None,
|
|
175
|
+
"-b",
|
|
176
|
+
"--bbox",
|
|
177
|
+
help=(
|
|
178
|
+
"Special search facet to refine/subset search results by spatial "
|
|
179
|
+
"extent. This can be a string representation of a bounding box. "
|
|
180
|
+
"The bounding box has to follow the format ``min_lon max_lon "
|
|
181
|
+
"min_lat,max_lat``. Valid strings are ``-10 10 -10 10`` to "
|
|
182
|
+
"``0 5 0 5``."
|
|
183
|
+
),
|
|
184
|
+
),
|
|
185
|
+
bbox_select: SelectMethod = typer.Option(
|
|
186
|
+
"flexible",
|
|
187
|
+
"-bs",
|
|
188
|
+
"--bbox-select",
|
|
189
|
+
help=SelectMethod.get_help("bbox"),
|
|
190
|
+
),
|
|
147
191
|
extended_search: bool = typer.Option(
|
|
148
192
|
False,
|
|
149
193
|
"-e",
|
|
@@ -188,9 +232,9 @@ def metadata_search(
|
|
|
188
232
|
result = databrowser.metadata_search(
|
|
189
233
|
*(facets or []),
|
|
190
234
|
time=time or "",
|
|
191
|
-
time_select=cast(
|
|
192
|
-
|
|
193
|
-
),
|
|
235
|
+
time_select=cast(Literal["file", "flexible", "strict"], time_select.value),
|
|
236
|
+
bbox=bbox or None,
|
|
237
|
+
bbox_select=cast(Literal["file", "flexible", "strict"], bbox_select.value),
|
|
194
238
|
flavour=cast(
|
|
195
239
|
Literal["freva", "cmip6", "cmip5", "cordex", "nextgems", "user"],
|
|
196
240
|
flavour.value,
|
|
@@ -248,11 +292,11 @@ def data_search(
|
|
|
248
292
|
"of climate datasets to query."
|
|
249
293
|
),
|
|
250
294
|
),
|
|
251
|
-
time_select:
|
|
295
|
+
time_select: SelectMethod = typer.Option(
|
|
252
296
|
"flexible",
|
|
253
297
|
"-ts",
|
|
254
298
|
"--time-select",
|
|
255
|
-
help=
|
|
299
|
+
help=SelectMethod.get_help("time"),
|
|
256
300
|
),
|
|
257
301
|
zarr: bool = typer.Option(False, "--zarr", help="Create zarr stream files."),
|
|
258
302
|
access_token: Optional[str] = typer.Option(
|
|
@@ -277,6 +321,24 @@ def data_search(
|
|
|
277
321
|
" valid."
|
|
278
322
|
),
|
|
279
323
|
),
|
|
324
|
+
bbox: Optional[Tuple[float, float, float, float]] = typer.Option(
|
|
325
|
+
None,
|
|
326
|
+
"-b",
|
|
327
|
+
"--bbox",
|
|
328
|
+
help=(
|
|
329
|
+
"Special search facet to refine/subset search results by spatial "
|
|
330
|
+
"extent. This can be a string representation of a bounding box. "
|
|
331
|
+
"The bounding box has to follow the format ``min_lon max_lon "
|
|
332
|
+
"min_lat max_lat``. Valid strings are ``-10 10 -10 10`` to "
|
|
333
|
+
"``0 5 0 5``."
|
|
334
|
+
),
|
|
335
|
+
),
|
|
336
|
+
bbox_select: SelectMethod = typer.Option(
|
|
337
|
+
"flexible",
|
|
338
|
+
"-bs",
|
|
339
|
+
"--bbox-select",
|
|
340
|
+
help=SelectMethod.get_help("bbox"),
|
|
341
|
+
),
|
|
280
342
|
parse_json: bool = typer.Option(
|
|
281
343
|
False, "-j", "--json", help="Parse output in json format."
|
|
282
344
|
),
|
|
@@ -314,6 +376,8 @@ def data_search(
|
|
|
314
376
|
*(facets or []),
|
|
315
377
|
time=time or "",
|
|
316
378
|
time_select=cast(Literal["file", "flexible", "strict"], time_select),
|
|
379
|
+
bbox=bbox or None,
|
|
380
|
+
bbox_select=cast(Literal["file", "flexible", "strict"], bbox_select),
|
|
317
381
|
flavour=cast(
|
|
318
382
|
Literal["freva", "cmip6", "cmip5", "cordex", "nextgems", "user"],
|
|
319
383
|
flavour.value,
|
|
@@ -374,11 +438,11 @@ def intake_catalogue(
|
|
|
374
438
|
"of climate datasets to query."
|
|
375
439
|
),
|
|
376
440
|
),
|
|
377
|
-
time_select:
|
|
441
|
+
time_select: SelectMethod = typer.Option(
|
|
378
442
|
"flexible",
|
|
379
443
|
"-ts",
|
|
380
444
|
"--time-select",
|
|
381
|
-
help=
|
|
445
|
+
help=SelectMethod.get_help("time"),
|
|
382
446
|
),
|
|
383
447
|
time: Optional[str] = typer.Option(
|
|
384
448
|
None,
|
|
@@ -394,6 +458,24 @@ def intake_catalogue(
|
|
|
394
458
|
" valid."
|
|
395
459
|
),
|
|
396
460
|
),
|
|
461
|
+
bbox: Optional[Tuple[float, float, float, float]] = typer.Option(
|
|
462
|
+
None,
|
|
463
|
+
"-b",
|
|
464
|
+
"--bbox",
|
|
465
|
+
help=(
|
|
466
|
+
"Special search facet to refine/subset search results by spatial "
|
|
467
|
+
"extent. This can be a string representation of a bounding box. "
|
|
468
|
+
"The bounding box has to follow the format ``min_lon max_lon "
|
|
469
|
+
"min_lat max_lat``. Valid strings are ``-10 10 -10 10`` to "
|
|
470
|
+
"``0 5 0 5``."
|
|
471
|
+
),
|
|
472
|
+
),
|
|
473
|
+
bbox_select: SelectMethod = typer.Option(
|
|
474
|
+
"flexible",
|
|
475
|
+
"-bs",
|
|
476
|
+
"--bbox-select",
|
|
477
|
+
help=SelectMethod.get_help("bbox"),
|
|
478
|
+
),
|
|
397
479
|
zarr: bool = typer.Option(
|
|
398
480
|
False, "--zarr", help="Create zarr stream files, as catalogue targets."
|
|
399
481
|
),
|
|
@@ -445,6 +527,8 @@ def intake_catalogue(
|
|
|
445
527
|
*(facets or []),
|
|
446
528
|
time=time or "",
|
|
447
529
|
time_select=cast(Literal["file", "flexible", "strict"], time_select),
|
|
530
|
+
bbox=bbox or None,
|
|
531
|
+
bbox_select=cast(Literal["file", "flexible", "strict"], bbox_select),
|
|
448
532
|
flavour=cast(
|
|
449
533
|
Literal["freva", "cmip6", "cmip5", "cordex", "nextgems", "user"],
|
|
450
534
|
flavour.value,
|
|
@@ -464,6 +548,142 @@ def intake_catalogue(
|
|
|
464
548
|
print(Path(temp_f.name).read_text())
|
|
465
549
|
|
|
466
550
|
|
|
551
|
+
@databrowser_app.command(
|
|
552
|
+
name="stac-catalogue",
|
|
553
|
+
help="Create a static STAC catalogue from the search."
|
|
554
|
+
)
|
|
555
|
+
@exception_handler
|
|
556
|
+
def stac_catalogue(
|
|
557
|
+
search_keys: Optional[List[str]] = typer.Argument(
|
|
558
|
+
default=None,
|
|
559
|
+
help="Refine your data search with this `key=value` pair search "
|
|
560
|
+
"parameters. The parameters could be, depending on the DRS standard, "
|
|
561
|
+
"flavour product, project model etc.",
|
|
562
|
+
),
|
|
563
|
+
facets: Optional[List[str]] = typer.Option(
|
|
564
|
+
None,
|
|
565
|
+
"--facet",
|
|
566
|
+
help=(
|
|
567
|
+
"If you are not sure about the correct search key's you can use"
|
|
568
|
+
" the ``--facet`` flag to search of any matching entries. For "
|
|
569
|
+
"example --facet 'era5' would allow you to search for any entries"
|
|
570
|
+
" containing era5, regardless of project, product etc."
|
|
571
|
+
),
|
|
572
|
+
),
|
|
573
|
+
uniq_key: UniqKeys = typer.Option(
|
|
574
|
+
"file",
|
|
575
|
+
"--uniq-key",
|
|
576
|
+
"-u",
|
|
577
|
+
help=(
|
|
578
|
+
"The type of search result, which can be either “file” "
|
|
579
|
+
"or “uri”. This parameter determines whether the search will be "
|
|
580
|
+
"based on file paths or Uniform Resource Identifiers"
|
|
581
|
+
),
|
|
582
|
+
),
|
|
583
|
+
flavour: Flavours = typer.Option(
|
|
584
|
+
"freva",
|
|
585
|
+
"--flavour",
|
|
586
|
+
"-f",
|
|
587
|
+
help=(
|
|
588
|
+
"The Data Reference Syntax (DRS) standard specifying the type "
|
|
589
|
+
"of climate datasets to query."
|
|
590
|
+
),
|
|
591
|
+
),
|
|
592
|
+
time_select: SelectMethod = typer.Option(
|
|
593
|
+
"flexible",
|
|
594
|
+
"-ts",
|
|
595
|
+
"--time-select",
|
|
596
|
+
help=SelectMethod.get_help("time"),
|
|
597
|
+
),
|
|
598
|
+
time: Optional[str] = typer.Option(
|
|
599
|
+
None,
|
|
600
|
+
"-t",
|
|
601
|
+
"--time",
|
|
602
|
+
help=(
|
|
603
|
+
"Special search facet to refine/subset search results by time. "
|
|
604
|
+
"This can be a string representation of a time range or a single "
|
|
605
|
+
"time step. The time steps have to follow ISO-8601. Valid strings "
|
|
606
|
+
"are ``%Y-%m-%dT%H:%M`` to ``%Y-%m-%dT%H:%M`` for time ranges and "
|
|
607
|
+
"``%Y-%m-%dT%H:%M``. **Note**: You don't have to give the full "
|
|
608
|
+
"string format to subset time steps ``%Y``, ``%Y-%m`` etc are also"
|
|
609
|
+
" valid."
|
|
610
|
+
),
|
|
611
|
+
),
|
|
612
|
+
bbox: Optional[Tuple[float, float, float, float]] = typer.Option(
|
|
613
|
+
None,
|
|
614
|
+
"-b",
|
|
615
|
+
"--bbox",
|
|
616
|
+
help=(
|
|
617
|
+
"Special search facet to refine/subset search results by spatial "
|
|
618
|
+
"extent. This can be a string representation of a bounding box. "
|
|
619
|
+
"The bounding box has to follow the format ``min_lon max_lon "
|
|
620
|
+
"min_lat max_lat``. Valid strings are ``-10 10 -10 10`` to "
|
|
621
|
+
"``0 5 0 5``."
|
|
622
|
+
),
|
|
623
|
+
),
|
|
624
|
+
bbox_select: SelectMethod = typer.Option(
|
|
625
|
+
"flexible",
|
|
626
|
+
"-bs",
|
|
627
|
+
"--bbox-select",
|
|
628
|
+
help=SelectMethod.get_help("bbox"),
|
|
629
|
+
),
|
|
630
|
+
host: Optional[str] = typer.Option(
|
|
631
|
+
None,
|
|
632
|
+
"--host",
|
|
633
|
+
help=(
|
|
634
|
+
"Set the hostname of the databrowser, if not set (default) "
|
|
635
|
+
"the hostname is read from a config file"
|
|
636
|
+
),
|
|
637
|
+
),
|
|
638
|
+
verbose: int = typer.Option(0, "-v", help="Increase verbosity", count=True),
|
|
639
|
+
multiversion: bool = typer.Option(
|
|
640
|
+
False,
|
|
641
|
+
"--multi-version",
|
|
642
|
+
help="Select all versions and not just the latest version (default).",
|
|
643
|
+
),
|
|
644
|
+
version: Optional[bool] = typer.Option(
|
|
645
|
+
False,
|
|
646
|
+
"-V",
|
|
647
|
+
"--version",
|
|
648
|
+
help="Show version an exit",
|
|
649
|
+
callback=version_callback,
|
|
650
|
+
),
|
|
651
|
+
filename: Optional[Path] = typer.Option(
|
|
652
|
+
None,
|
|
653
|
+
"-o",
|
|
654
|
+
"--filename",
|
|
655
|
+
help=(
|
|
656
|
+
"Path to the file where the static STAC catalogue, "
|
|
657
|
+
"should be written to. If you don't specify or the path "
|
|
658
|
+
"does not exist, the file will be created in the current "
|
|
659
|
+
"working directory. "
|
|
660
|
+
),
|
|
661
|
+
),
|
|
662
|
+
) -> None:
|
|
663
|
+
"""Create a STAC catalogue for climate datasets based on the specified
|
|
664
|
+
Data Reference Syntax (DRS) standard (flavour) and the type of search
|
|
665
|
+
result (uniq_key), which can be either "file" or "uri"."""
|
|
666
|
+
logger.set_verbosity(verbose)
|
|
667
|
+
result = databrowser(
|
|
668
|
+
*(facets or []),
|
|
669
|
+
time=time or "",
|
|
670
|
+
time_select=cast(Literal["file", "flexible", "strict"], time_select),
|
|
671
|
+
bbox=bbox or None,
|
|
672
|
+
bbox_select=cast(Literal["file", "flexible", "strict"], bbox_select),
|
|
673
|
+
flavour=cast(
|
|
674
|
+
Literal["freva", "cmip6", "cmip5", "cordex", "nextgems", "user"],
|
|
675
|
+
flavour.value,
|
|
676
|
+
),
|
|
677
|
+
uniq_key=cast(Literal["uri", "file"], uniq_key.value),
|
|
678
|
+
host=host,
|
|
679
|
+
fail_on_error=False,
|
|
680
|
+
multiversion=multiversion,
|
|
681
|
+
stream_zarr=False,
|
|
682
|
+
**(parse_cli_args(search_keys or [])),
|
|
683
|
+
)
|
|
684
|
+
print(result.stac_catalogue(filename=filename))
|
|
685
|
+
|
|
686
|
+
|
|
467
687
|
@databrowser_app.command(
|
|
468
688
|
name="data-count", help="Count the databrowser search results"
|
|
469
689
|
)
|
|
@@ -500,11 +720,11 @@ def count_values(
|
|
|
500
720
|
"of climate datasets to query."
|
|
501
721
|
),
|
|
502
722
|
),
|
|
503
|
-
time_select:
|
|
723
|
+
time_select: SelectMethod = typer.Option(
|
|
504
724
|
"flexible",
|
|
505
725
|
"-ts",
|
|
506
726
|
"--time-select",
|
|
507
|
-
help=
|
|
727
|
+
help=SelectMethod.get_help("time"),
|
|
508
728
|
),
|
|
509
729
|
time: Optional[str] = typer.Option(
|
|
510
730
|
None,
|
|
@@ -520,6 +740,24 @@ def count_values(
|
|
|
520
740
|
" valid."
|
|
521
741
|
),
|
|
522
742
|
),
|
|
743
|
+
bbox: Optional[Tuple[float, float, float, float]] = typer.Option(
|
|
744
|
+
None,
|
|
745
|
+
"-b",
|
|
746
|
+
"--bbox",
|
|
747
|
+
help=(
|
|
748
|
+
"Special search facet to refine/subset search results by spatial "
|
|
749
|
+
"extent. This can be a string representation of a bounding box. "
|
|
750
|
+
"The bounding box has to follow the format ``min_lon max_lon "
|
|
751
|
+
"min_lat max_lat``. Valid strings are ``-10 10 -10 10`` to "
|
|
752
|
+
"``0 5 0 5``."
|
|
753
|
+
),
|
|
754
|
+
),
|
|
755
|
+
bbox_select: SelectMethod = typer.Option(
|
|
756
|
+
"flexible",
|
|
757
|
+
"-bs",
|
|
758
|
+
"--bbox-select",
|
|
759
|
+
help=SelectMethod.get_help("bbox"),
|
|
760
|
+
),
|
|
523
761
|
extended_search: bool = typer.Option(
|
|
524
762
|
False,
|
|
525
763
|
"-e",
|
|
@@ -564,12 +802,16 @@ def count_values(
|
|
|
564
802
|
result: Union[int, Dict[str, Dict[str, int]]] = 0
|
|
565
803
|
search_kws = parse_cli_args(search_keys or [])
|
|
566
804
|
time = cast(str, time or search_kws.pop("time", ""))
|
|
805
|
+
bbox = cast(Optional[Tuple[float, float, float, float]],
|
|
806
|
+
bbox or search_kws.pop("bbox", None))
|
|
567
807
|
facets = facets or []
|
|
568
808
|
if detail:
|
|
569
809
|
result = databrowser.count_values(
|
|
570
810
|
*facets,
|
|
571
811
|
time=time or "",
|
|
572
812
|
time_select=cast(Literal["file", "flexible", "strict"], time_select),
|
|
813
|
+
bbox=bbox or None,
|
|
814
|
+
bbox_select=cast(Literal["file", "flexible", "strict"], bbox_select),
|
|
573
815
|
flavour=cast(
|
|
574
816
|
Literal["freva", "cmip6", "cmip5", "cordex", "nextgems", "user"],
|
|
575
817
|
flavour.value,
|
|
@@ -585,9 +827,9 @@ def count_values(
|
|
|
585
827
|
databrowser(
|
|
586
828
|
*facets,
|
|
587
829
|
time=time or "",
|
|
588
|
-
time_select=cast(
|
|
589
|
-
|
|
590
|
-
),
|
|
830
|
+
time_select=cast(Literal["file", "flexible", "strict"], time_select),
|
|
831
|
+
bbox=bbox or None,
|
|
832
|
+
bbox_select=cast(Literal["file", "flexible", "strict"], bbox_select),
|
|
591
833
|
flavour=cast(
|
|
592
834
|
Literal[
|
|
593
835
|
"freva", "cmip6", "cmip5", "cordex", "nextgems", "user"
|
|
@@ -63,8 +63,10 @@ class databrowser:
|
|
|
63
63
|
timestamp. The timestamps has to follow ISO-8601. Valid strings are
|
|
64
64
|
``%Y-%m-%dT%H:%M to %Y-%m-%dT%H:%M`` for time ranges or
|
|
65
65
|
``%Y-%m-%dT%H:%M`` for single time stamps.
|
|
66
|
-
|
|
67
|
-
|
|
66
|
+
|
|
67
|
+
.. note:: You don't have to give the full string format to subset time
|
|
68
|
+
steps ``%Y``, ``%Y-%m`` etc are also valid.
|
|
69
|
+
|
|
68
70
|
time_select: str, default: flexible
|
|
69
71
|
Operator that specifies how the time period is selected. Choose from
|
|
70
72
|
flexible (default), strict or file. ``strict`` returns only those files
|
|
@@ -74,11 +76,25 @@ class databrowser:
|
|
|
74
76
|
``flexible`` returns those files that have either start or end period
|
|
75
77
|
covered. ``file`` will only return files where the entire time
|
|
76
78
|
period is contained within `one single` file.
|
|
79
|
+
bbox: str, default: ""
|
|
80
|
+
Special search facet to refine/subset search results by spatial extent.
|
|
81
|
+
This can be a list representation of a bounding box or a WKT polygon.
|
|
82
|
+
Valid lists are ``min_lon max_lon min_lat max_lat`` for bounding
|
|
83
|
+
boxes and Well-Known Text (WKT) format for polygons.
|
|
84
|
+
|
|
85
|
+
bbox_select: str, default: flexible
|
|
86
|
+
Operator that specifies how the spatial extent is selected. Choose from
|
|
87
|
+
flexible (default), strict or file. ``strict`` returns only those files
|
|
88
|
+
that fully contain the query extent. The bbox search ``-10 10 -10 10``
|
|
89
|
+
will not select files covering only ``0 5 0 5`` with the ``strict``
|
|
90
|
+
method. ``flexible`` will select those files as it returns files that
|
|
91
|
+
have any overlap with the query extent. ``file`` will only return files
|
|
92
|
+
where the entire spatial extent is contained by the query geometry.
|
|
77
93
|
uniq_key: str, default: file
|
|
78
94
|
Chose if the solr search query should return paths to files or
|
|
79
95
|
uris, uris will have the file path along with protocol of the storage
|
|
80
|
-
system.
|
|
81
|
-
|
|
96
|
+
system. URIs are useful when working with libraries like fsspec, which
|
|
97
|
+
require protocol information.
|
|
82
98
|
host: str, default: None
|
|
83
99
|
Override the host name of the databrowser server. This is usually the
|
|
84
100
|
url where the freva web site can be found. Such as www.freva.dkrz.de.
|
|
@@ -214,6 +230,8 @@ class databrowser:
|
|
|
214
230
|
time: Optional[str] = None,
|
|
215
231
|
host: Optional[str] = None,
|
|
216
232
|
time_select: Literal["flexible", "strict", "file"] = "flexible",
|
|
233
|
+
bbox: Optional[Tuple[float, float, float, float]] = None,
|
|
234
|
+
bbox_select: Literal["flexible", "strict", "file"] = "flexible",
|
|
217
235
|
stream_zarr: bool = False,
|
|
218
236
|
multiversion: bool = False,
|
|
219
237
|
fail_on_error: bool = False,
|
|
@@ -238,6 +256,10 @@ class databrowser:
|
|
|
238
256
|
if time:
|
|
239
257
|
self._params["time"] = time
|
|
240
258
|
self._params["time_select"] = time_select
|
|
259
|
+
if bbox:
|
|
260
|
+
bbox_str = ",".join(map(str, bbox))
|
|
261
|
+
self._params["bbox"] = bbox_str
|
|
262
|
+
self._params["bbox_select"] = bbox_select
|
|
241
263
|
if facets:
|
|
242
264
|
self._add_search_keyword_args_from_facet(facets, facet_search)
|
|
243
265
|
|
|
@@ -399,6 +421,84 @@ class databrowser:
|
|
|
399
421
|
intake.open_esm_datastore(temp_f.name),
|
|
400
422
|
)
|
|
401
423
|
|
|
424
|
+
def stac_catalogue(
|
|
425
|
+
self,
|
|
426
|
+
filename: Optional[Union[str, Path]] = None,
|
|
427
|
+
**kwargs: Any,
|
|
428
|
+
) -> str:
|
|
429
|
+
"""Create a static STAC catalogue from
|
|
430
|
+
the search.
|
|
431
|
+
|
|
432
|
+
Parameters
|
|
433
|
+
~~~~~~~~~~
|
|
434
|
+
filename: str, default: None
|
|
435
|
+
The filename of the STAC catalogue. If not given
|
|
436
|
+
or doesn't exist the STAC catalogue will be saved
|
|
437
|
+
to the current working directory.
|
|
438
|
+
**kwargs: Any
|
|
439
|
+
Additional keyword arguments to be passed to the request.
|
|
440
|
+
|
|
441
|
+
Returns
|
|
442
|
+
~~~~~~~
|
|
443
|
+
BinaryIO
|
|
444
|
+
A zip file stream
|
|
445
|
+
|
|
446
|
+
Raises
|
|
447
|
+
~~~~~~
|
|
448
|
+
ValueError: If stac-catalogue creation failed.
|
|
449
|
+
|
|
450
|
+
Example
|
|
451
|
+
~~~~~~~
|
|
452
|
+
Let's create a static STAC catalogue:
|
|
453
|
+
|
|
454
|
+
.. execute_code::
|
|
455
|
+
|
|
456
|
+
from tempfile import mktemp
|
|
457
|
+
temp_path = mktemp(suffix=".zip")
|
|
458
|
+
|
|
459
|
+
from freva_client import databrowser
|
|
460
|
+
db = databrowser(dataset="cmip6-hsm")
|
|
461
|
+
db.stac_catalogue(filename=temp_path)
|
|
462
|
+
print(f"STAC catalog saved to: {temp_path}")
|
|
463
|
+
|
|
464
|
+
"""
|
|
465
|
+
|
|
466
|
+
kwargs.update({"stream": True})
|
|
467
|
+
stac_url = self._cfg.stac_url
|
|
468
|
+
pprint("[b][green]Downloading the STAC catalog started ...[green][b]")
|
|
469
|
+
result = self._request("GET", stac_url, **kwargs)
|
|
470
|
+
if result is None or result.status_code == 404:
|
|
471
|
+
raise ValueError( # pragma: no cover
|
|
472
|
+
"No STAC catalog found. Please check if you have any search results."
|
|
473
|
+
)
|
|
474
|
+
default_filename = (
|
|
475
|
+
result.headers.get("Content-Disposition", "")
|
|
476
|
+
.split("filename=")[-1]
|
|
477
|
+
.strip('"')
|
|
478
|
+
)
|
|
479
|
+
|
|
480
|
+
if filename is None:
|
|
481
|
+
save_path = Path.cwd() / default_filename
|
|
482
|
+
else:
|
|
483
|
+
save_path = Path(cast(str, filename))
|
|
484
|
+
if save_path.is_dir() and save_path.exists():
|
|
485
|
+
save_path = save_path / default_filename
|
|
486
|
+
|
|
487
|
+
save_path.parent.mkdir(parents=True, exist_ok=True)
|
|
488
|
+
|
|
489
|
+
total_size = 0
|
|
490
|
+
with open(save_path, "wb") as f:
|
|
491
|
+
for chunk in result.iter_content(chunk_size=8192):
|
|
492
|
+
if chunk:
|
|
493
|
+
f.write(chunk)
|
|
494
|
+
total_size += len(chunk)
|
|
495
|
+
|
|
496
|
+
return (
|
|
497
|
+
f"STAC catalog saved to: {save_path} "
|
|
498
|
+
f"(size: {total_size / 1024 / 1024:.2f} MB). "
|
|
499
|
+
f"Or simply download from: {result.url}"
|
|
500
|
+
)
|
|
501
|
+
|
|
402
502
|
@classmethod
|
|
403
503
|
def count_values(
|
|
404
504
|
cls,
|
|
@@ -409,6 +509,8 @@ class databrowser:
|
|
|
409
509
|
time: Optional[str] = None,
|
|
410
510
|
host: Optional[str] = None,
|
|
411
511
|
time_select: Literal["flexible", "strict", "file"] = "flexible",
|
|
512
|
+
bbox: Optional[Tuple[float, float, float, float]] = None,
|
|
513
|
+
bbox_select: Literal["flexible", "strict", "file"] = "flexible",
|
|
412
514
|
multiversion: bool = False,
|
|
413
515
|
fail_on_error: bool = False,
|
|
414
516
|
extended_search: bool = False,
|
|
@@ -432,8 +534,11 @@ class databrowser:
|
|
|
432
534
|
This can be a string representation of a time range or a single
|
|
433
535
|
timestamp. The timestamp has to follow ISO-8601. Valid strings are
|
|
434
536
|
``%Y-%m-%dT%H:%M`` to ``%Y-%m-%dT%H:%M`` for time ranges and
|
|
435
|
-
``%Y-%m-%dT%H:%M``.
|
|
436
|
-
|
|
537
|
+
``%Y-%m-%dT%H:%M``.
|
|
538
|
+
|
|
539
|
+
.. note:: You don't have to give the full string format to subset time
|
|
540
|
+
steps ``%Y``, ``%Y-%m`` etc are also valid.
|
|
541
|
+
|
|
437
542
|
time_select: str, default: flexible
|
|
438
543
|
Operator that specifies how the time period is selected. Choose from
|
|
439
544
|
flexible (default), strict or file. ``strict`` returns only those files
|
|
@@ -443,6 +548,20 @@ class databrowser:
|
|
|
443
548
|
``flexible`` returns those files that have either start or end period
|
|
444
549
|
covered. ``file`` will only return files where the entire time
|
|
445
550
|
period is contained within `one single` file.
|
|
551
|
+
bbox: str, default: ""
|
|
552
|
+
Special search facet to refine/subset search results by spatial extent.
|
|
553
|
+
This can be a list representation of a bounding box or a WKT polygon.
|
|
554
|
+
Valid lists are ``min_lon max_lon min_lat max_lat`` for bounding
|
|
555
|
+
boxes and Well-Known Text (WKT) format for polygons.
|
|
556
|
+
|
|
557
|
+
bbox_select: str, default: flexible
|
|
558
|
+
Operator that specifies how the spatial extent is selected. Choose from
|
|
559
|
+
flexible (default), strict or file. ``strict`` returns only those files
|
|
560
|
+
that fully contain the query extent. The bbox search ``-10 10 -10 10``
|
|
561
|
+
will not select files covering only ``0 5 0 5`` with the ``strict``
|
|
562
|
+
method. ``flexible`` will select those files as it returns files that
|
|
563
|
+
have any overlap with the query extent. ``file`` will only return files
|
|
564
|
+
where the entire spatial extent is contained by the query geometry.
|
|
446
565
|
extended_search: bool, default: False
|
|
447
566
|
Retrieve information on additional search keys.
|
|
448
567
|
host: str, default: None
|
|
@@ -494,6 +613,8 @@ class databrowser:
|
|
|
494
613
|
flavour=flavour,
|
|
495
614
|
time=time,
|
|
496
615
|
time_select=time_select,
|
|
616
|
+
bbox=bbox,
|
|
617
|
+
bbox_select=bbox_select,
|
|
497
618
|
host=host,
|
|
498
619
|
multiversion=multiversion,
|
|
499
620
|
fail_on_error=fail_on_error,
|
|
@@ -545,6 +666,8 @@ class databrowser:
|
|
|
545
666
|
time: Optional[str] = None,
|
|
546
667
|
host: Optional[str] = None,
|
|
547
668
|
time_select: Literal["flexible", "strict", "file"] = "flexible",
|
|
669
|
+
bbox: Optional[Tuple[float, float, float, float]] = None,
|
|
670
|
+
bbox_select: Literal["flexible", "strict", "file"] = "flexible",
|
|
548
671
|
multiversion: bool = False,
|
|
549
672
|
fail_on_error: bool = False,
|
|
550
673
|
extended_search: bool = False,
|
|
@@ -571,8 +694,11 @@ class databrowser:
|
|
|
571
694
|
This can be a string representation of a time range or a single
|
|
572
695
|
timestamp. The timestamp has to follow ISO-8601. Valid strings are
|
|
573
696
|
``%Y-%m-%dT%H:%M`` to ``%Y-%m-%dT%H:%M`` for time ranges and
|
|
574
|
-
``%Y-%m-%dT%H:%M``.
|
|
575
|
-
|
|
697
|
+
``%Y-%m-%dT%H:%M``.
|
|
698
|
+
|
|
699
|
+
.. note:: You don't have to give the full string format to subset time
|
|
700
|
+
steps ``%Y``, ``%Y-%m`` etc are also valid.
|
|
701
|
+
|
|
576
702
|
time_select: str, default: flexible
|
|
577
703
|
Operator that specifies how the time period is selected. Choose from
|
|
578
704
|
flexible (default), strict or file. ``strict`` returns only those files
|
|
@@ -582,6 +708,20 @@ class databrowser:
|
|
|
582
708
|
``flexible`` returns those files that have either start or end period
|
|
583
709
|
covered. ``file`` will only return files where the entire time
|
|
584
710
|
period is contained within *one single* file.
|
|
711
|
+
bbox: str, default: ""
|
|
712
|
+
Special search facet to refine/subset search results by spatial extent.
|
|
713
|
+
This can be a list representation of a bounding box or a WKT polygon.
|
|
714
|
+
Valid lists are ``min_lon max_lon min_lat max_lat`` for bounding
|
|
715
|
+
boxes and Well-Known Text (WKT) format for polygons.
|
|
716
|
+
|
|
717
|
+
bbox_select: str, default: flexible
|
|
718
|
+
Operator that specifies how the spatial extent is selected. Choose from
|
|
719
|
+
flexible (default), strict or file. ``strict`` returns only those files
|
|
720
|
+
that fully contain the query extent. The bbox search ``-10 10 -10 10``
|
|
721
|
+
will not select files covering only ``0 5 0 5`` with the ``strict``
|
|
722
|
+
method. ``flexible`` will select those files as it returns files that
|
|
723
|
+
have any overlap with the query extent. ``file`` will only return files
|
|
724
|
+
where the entire spatial extent is contained by the query geometry.
|
|
585
725
|
extended_search: bool, default: False
|
|
586
726
|
Retrieve information on additional search keys.
|
|
587
727
|
multiversion: bool, default: False
|
|
@@ -650,12 +790,30 @@ class databrowser:
|
|
|
650
790
|
from freva_client import databrowser
|
|
651
791
|
print(databrowser.metadata_search("reana*", realm="ocean", flavour="cmip6"))
|
|
652
792
|
|
|
793
|
+
In datasets with multiple versions only the `latest` version (i.e.
|
|
794
|
+
`highest` version number) is returned by default. Querying a specific
|
|
795
|
+
version from a multi versioned datasets requires the ``multiversion``
|
|
796
|
+
flag in combination with the ``version`` special attribute:
|
|
797
|
+
|
|
798
|
+
.. execute_code::
|
|
799
|
+
|
|
800
|
+
from freva_client import databrowser
|
|
801
|
+
res = databrowser.metadata_search(dataset="cmip6-fs",
|
|
802
|
+
model="access-cm2", version="v20191108", extended_search=True,
|
|
803
|
+
multiversion=True)
|
|
804
|
+
print(res)
|
|
805
|
+
|
|
806
|
+
If no particular ``version`` is requested, information of all versions
|
|
807
|
+
will be returned.
|
|
808
|
+
|
|
653
809
|
"""
|
|
654
810
|
this = cls(
|
|
655
811
|
*facets,
|
|
656
812
|
flavour=flavour,
|
|
657
813
|
time=time,
|
|
658
814
|
time_select=time_select,
|
|
815
|
+
bbox=bbox,
|
|
816
|
+
bbox_select=bbox_select,
|
|
659
817
|
host=host,
|
|
660
818
|
multiversion=multiversion,
|
|
661
819
|
fail_on_error=fail_on_error,
|
|
@@ -148,6 +148,11 @@ class Config:
|
|
|
148
148
|
"""Define the url for creating intake catalogues."""
|
|
149
149
|
return f"{self.databrowser_url}/intake-catalogue/{self.flavour}/{self.uniq_key}"
|
|
150
150
|
|
|
151
|
+
@property
|
|
152
|
+
def stac_url(self) -> str:
|
|
153
|
+
"""Define the url for creating stac catalogue."""
|
|
154
|
+
return f"{self.databrowser_url}/stac-catalogue/{self.flavour}/{self.uniq_key}"
|
|
155
|
+
|
|
151
156
|
@property
|
|
152
157
|
def metadata_url(self) -> str:
|
|
153
158
|
"""Define the endpoint for the metadata search."""
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|