freva-client 2408.0.0.dev2__tar.gz → 2410.0.0b1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of freva-client might be problematic. Click here for more details.
- {freva_client-2408.0.0.dev2 → freva_client-2410.0.0b1}/PKG-INFO +1 -1
- {freva_client-2408.0.0.dev2 → freva_client-2410.0.0b1}/src/freva_client/__init__.py +1 -1
- {freva_client-2408.0.0.dev2 → freva_client-2410.0.0b1}/src/freva_client/auth.py +22 -17
- {freva_client-2408.0.0.dev2 → freva_client-2410.0.0b1}/src/freva_client/cli/auth_cli.py +1 -3
- {freva_client-2408.0.0.dev2 → freva_client-2410.0.0b1}/src/freva_client/cli/cli_app.py +4 -0
- {freva_client-2408.0.0.dev2 → freva_client-2410.0.0b1}/src/freva_client/cli/databrowser_cli.py +114 -27
- {freva_client-2408.0.0.dev2 → freva_client-2410.0.0b1}/src/freva_client/query.py +165 -43
- {freva_client-2408.0.0.dev2 → freva_client-2410.0.0b1}/src/freva_client/utils/databrowser_utils.py +10 -12
- {freva_client-2408.0.0.dev2 → freva_client-2410.0.0b1}/MANIFEST.in +0 -0
- {freva_client-2408.0.0.dev2 → freva_client-2410.0.0b1}/README.md +0 -0
- {freva_client-2408.0.0.dev2 → freva_client-2410.0.0b1}/assets/share/freva/freva.toml +0 -0
- {freva_client-2408.0.0.dev2 → freva_client-2410.0.0b1}/pyproject.toml +0 -0
- {freva_client-2408.0.0.dev2 → freva_client-2410.0.0b1}/src/freva_client/__main__.py +0 -0
- {freva_client-2408.0.0.dev2 → freva_client-2410.0.0b1}/src/freva_client/cli/__init__.py +0 -0
- {freva_client-2408.0.0.dev2 → freva_client-2410.0.0b1}/src/freva_client/cli/cli_parser.py +0 -0
- {freva_client-2408.0.0.dev2 → freva_client-2410.0.0b1}/src/freva_client/cli/cli_utils.py +0 -0
- {freva_client-2408.0.0.dev2 → freva_client-2410.0.0b1}/src/freva_client/py.typed +0 -0
- {freva_client-2408.0.0.dev2 → freva_client-2410.0.0b1}/src/freva_client/utils/__init__.py +0 -0
- {freva_client-2408.0.0.dev2 → freva_client-2410.0.0b1}/src/freva_client/utils/logger.py +0 -0
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
"""Module that handles the authentication at the rest service."""
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
import datetime
|
|
4
4
|
from getpass import getpass, getuser
|
|
5
|
-
from typing import Optional, TypedDict
|
|
5
|
+
from typing import Optional, TypedDict, Union
|
|
6
6
|
|
|
7
7
|
from authlib.integrations.requests_client import OAuth2Session
|
|
8
8
|
|
|
@@ -14,9 +14,10 @@ Token = TypedDict(
|
|
|
14
14
|
{
|
|
15
15
|
"access_token": str,
|
|
16
16
|
"token_type": str,
|
|
17
|
-
"expires":
|
|
17
|
+
"expires": int,
|
|
18
18
|
"refresh_token": str,
|
|
19
|
-
"refresh_expires":
|
|
19
|
+
"refresh_expires": int,
|
|
20
|
+
"scope": str,
|
|
20
21
|
},
|
|
21
22
|
)
|
|
22
23
|
|
|
@@ -36,13 +37,13 @@ class Auth:
|
|
|
36
37
|
self._auth_cls = OAuth2Session()
|
|
37
38
|
|
|
38
39
|
@property
|
|
39
|
-
def token_expiration_time(self) -> datetime:
|
|
40
|
+
def token_expiration_time(self) -> datetime.datetime:
|
|
40
41
|
"""Get the expiration time of an access token."""
|
|
41
42
|
if self._auth_token is None:
|
|
42
43
|
exp = 0.0
|
|
43
44
|
else:
|
|
44
45
|
exp = self._auth_token["expires"]
|
|
45
|
-
return datetime.fromtimestamp(exp)
|
|
46
|
+
return datetime.datetime.fromtimestamp(exp, datetime.timezone.utc)
|
|
46
47
|
|
|
47
48
|
def set_token(
|
|
48
49
|
self,
|
|
@@ -50,19 +51,21 @@ class Auth:
|
|
|
50
51
|
refresh_token: Optional[str] = None,
|
|
51
52
|
expires_in: int = 10,
|
|
52
53
|
refresh_expires_in: int = 10,
|
|
53
|
-
expires: Optional[float] = None,
|
|
54
|
-
refresh_expires: Optional[float] = None,
|
|
54
|
+
expires: Optional[Union[float, int]] = None,
|
|
55
|
+
refresh_expires: Optional[Union[float, int]] = None,
|
|
55
56
|
token_type: str = "Bearer",
|
|
57
|
+
scope: str = "profile email address",
|
|
56
58
|
) -> Token:
|
|
57
59
|
"""Override the existing auth token."""
|
|
58
|
-
now = datetime.now().timestamp()
|
|
60
|
+
now = datetime.datetime.now(datetime.timezone.utc).timestamp()
|
|
59
61
|
|
|
60
62
|
self._auth_token = Token(
|
|
61
63
|
access_token=access_token or "",
|
|
62
64
|
refresh_token=refresh_token or "",
|
|
63
65
|
token_type=token_type,
|
|
64
|
-
expires=expires or now + expires_in,
|
|
65
|
-
refresh_expires=refresh_expires or now + refresh_expires_in,
|
|
66
|
+
expires=int(expires or now + expires_in),
|
|
67
|
+
refresh_expires=int(refresh_expires or now + refresh_expires_in),
|
|
68
|
+
scope=scope,
|
|
66
69
|
)
|
|
67
70
|
return self._auth_token
|
|
68
71
|
|
|
@@ -75,9 +78,10 @@ class Auth:
|
|
|
75
78
|
return self.set_token(
|
|
76
79
|
access_token=auth["access_token"],
|
|
77
80
|
token_type=auth["token_type"],
|
|
78
|
-
|
|
81
|
+
expires=auth["expires"],
|
|
79
82
|
refresh_token=auth["refresh_token"],
|
|
80
|
-
|
|
83
|
+
refresh_expires=auth["refresh_expires"],
|
|
84
|
+
scope=auth["scope"],
|
|
81
85
|
)
|
|
82
86
|
except KeyError:
|
|
83
87
|
logger.warning("Failed to refresh token: %s", auth.get("detail", ""))
|
|
@@ -94,7 +98,7 @@ class Auth:
|
|
|
94
98
|
"""
|
|
95
99
|
if not self._auth_token:
|
|
96
100
|
raise ValueError("You must authenticate first.")
|
|
97
|
-
now = datetime.now().timestamp()
|
|
101
|
+
now = datetime.datetime.now(datetime.timezone.utc).timestamp()
|
|
98
102
|
if now > self._auth_token["refresh_expires"]:
|
|
99
103
|
raise ValueError("Refresh token has expired.")
|
|
100
104
|
if now > self._auth_token["expires"] and auth_url:
|
|
@@ -111,9 +115,10 @@ class Auth:
|
|
|
111
115
|
return self.set_token(
|
|
112
116
|
access_token=auth["access_token"],
|
|
113
117
|
token_type=auth["token_type"],
|
|
114
|
-
|
|
118
|
+
expires=auth["expires"],
|
|
115
119
|
refresh_token=auth["refresh_token"],
|
|
116
|
-
|
|
120
|
+
refresh_expires=auth["refresh_expires"],
|
|
121
|
+
scope=auth["scope"],
|
|
117
122
|
)
|
|
118
123
|
except KeyError:
|
|
119
124
|
logger.error("Failed to authenticate: %s", auth.get("detail", ""))
|
|
@@ -141,7 +146,7 @@ class Auth:
|
|
|
141
146
|
username = username or getuser()
|
|
142
147
|
if self._auth_token is None or force:
|
|
143
148
|
return self._login_with_password(cfg.auth_url, username)
|
|
144
|
-
if self.token_expiration_time < datetime.now():
|
|
149
|
+
if self.token_expiration_time < datetime.datetime.now(datetime.timezone.utc):
|
|
145
150
|
self._refresh(cfg.auth_url, self._auth_token["refresh_token"], username)
|
|
146
151
|
return self._auth_token
|
|
147
152
|
|
|
@@ -49,9 +49,7 @@ def authenticate_cli(
|
|
|
49
49
|
"-f",
|
|
50
50
|
help="Force token recreation, even if current token is still valid.",
|
|
51
51
|
),
|
|
52
|
-
verbose: int = typer.Option(
|
|
53
|
-
0, "-v", help="Increase verbosity", count=True
|
|
54
|
-
),
|
|
52
|
+
verbose: int = typer.Option(0, "-v", help="Increase verbosity", count=True),
|
|
55
53
|
version: Optional[bool] = typer.Option(
|
|
56
54
|
False,
|
|
57
55
|
"-V",
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"""Freva the Free Evaluation System command line interface."""
|
|
2
2
|
|
|
3
|
+
import os
|
|
3
4
|
from typing import Optional
|
|
4
5
|
|
|
5
6
|
import typer
|
|
@@ -9,6 +10,9 @@ from .auth_cli import authenticate_cli
|
|
|
9
10
|
from .cli_utils import APP_NAME, version_callback
|
|
10
11
|
from .databrowser_cli import databrowser_app
|
|
11
12
|
|
|
13
|
+
if os.getenv("FREVA_NO_RICH_PANELS", "0") == "1":
|
|
14
|
+
typer.core.rich = None # type: ignore
|
|
15
|
+
|
|
12
16
|
app = typer.Typer(
|
|
13
17
|
name=APP_NAME,
|
|
14
18
|
help=__doc__,
|
{freva_client-2408.0.0.dev2 → freva_client-2410.0.0b1}/src/freva_client/cli/databrowser_cli.py
RENAMED
|
@@ -163,9 +163,7 @@ def metadata_search(
|
|
|
163
163
|
parse_json: bool = typer.Option(
|
|
164
164
|
False, "-j", "--json", help="Parse output in json format."
|
|
165
165
|
),
|
|
166
|
-
verbose: int = typer.Option(
|
|
167
|
-
0, "-v", help="Increase verbosity", count=True
|
|
168
|
-
),
|
|
166
|
+
verbose: int = typer.Option(0, "-v", help="Increase verbosity", count=True),
|
|
169
167
|
version: Optional[bool] = typer.Option(
|
|
170
168
|
False,
|
|
171
169
|
"-V",
|
|
@@ -187,9 +185,7 @@ def metadata_search(
|
|
|
187
185
|
result = databrowser.metadata_search(
|
|
188
186
|
*(facets or []),
|
|
189
187
|
time=time or "",
|
|
190
|
-
time_select=cast(
|
|
191
|
-
Literal["file", "flexible", "strict"], time_select.value
|
|
192
|
-
),
|
|
188
|
+
time_select=cast(Literal["file", "flexible", "strict"], time_select.value),
|
|
193
189
|
flavour=cast(
|
|
194
190
|
Literal["freva", "cmip6", "cmip5", "cordex", "nextgems"],
|
|
195
191
|
flavour.value,
|
|
@@ -253,9 +249,7 @@ def data_search(
|
|
|
253
249
|
"--time-select",
|
|
254
250
|
help=TimeSelect.get_help(),
|
|
255
251
|
),
|
|
256
|
-
zarr: bool = typer.Option(
|
|
257
|
-
False, "--zarr", help="Create zarr stream files."
|
|
258
|
-
),
|
|
252
|
+
zarr: bool = typer.Option(False, "--zarr", help="Create zarr stream files."),
|
|
259
253
|
access_token: Optional[str] = typer.Option(
|
|
260
254
|
None,
|
|
261
255
|
"--access-token",
|
|
@@ -289,9 +283,7 @@ def data_search(
|
|
|
289
283
|
"the hostname is read from a config file"
|
|
290
284
|
),
|
|
291
285
|
),
|
|
292
|
-
verbose: int = typer.Option(
|
|
293
|
-
0, "-v", help="Increase verbosity", count=True
|
|
294
|
-
),
|
|
286
|
+
verbose: int = typer.Option(0, "-v", help="Increase verbosity", count=True),
|
|
295
287
|
multiversion: bool = typer.Option(
|
|
296
288
|
False,
|
|
297
289
|
"--multi-version",
|
|
@@ -425,9 +417,7 @@ def intake_catalogue(
|
|
|
425
417
|
"the hostname is read from a config file"
|
|
426
418
|
),
|
|
427
419
|
),
|
|
428
|
-
verbose: int = typer.Option(
|
|
429
|
-
0, "-v", help="Increase verbosity", count=True
|
|
430
|
-
),
|
|
420
|
+
verbose: int = typer.Option(0, "-v", help="Increase verbosity", count=True),
|
|
431
421
|
multiversion: bool = typer.Option(
|
|
432
422
|
False,
|
|
433
423
|
"--multi-version",
|
|
@@ -469,9 +459,7 @@ def intake_catalogue(
|
|
|
469
459
|
print(Path(temp_f.name).read_text())
|
|
470
460
|
|
|
471
461
|
|
|
472
|
-
@databrowser_app.command(
|
|
473
|
-
name="data-count", help="Count the databrowser search results"
|
|
474
|
-
)
|
|
462
|
+
@databrowser_app.command(name="data-count", help="Count the databrowser search results")
|
|
475
463
|
@exception_handler
|
|
476
464
|
def count_values(
|
|
477
465
|
search_keys: Optional[List[str]] = typer.Argument(
|
|
@@ -547,9 +535,7 @@ def count_values(
|
|
|
547
535
|
parse_json: bool = typer.Option(
|
|
548
536
|
False, "-j", "--json", help="Parse output in json format."
|
|
549
537
|
),
|
|
550
|
-
verbose: int = typer.Option(
|
|
551
|
-
0, "-v", help="Increase verbosity", count=True
|
|
552
|
-
),
|
|
538
|
+
verbose: int = typer.Option(0, "-v", help="Increase verbosity", count=True),
|
|
553
539
|
version: Optional[bool] = typer.Option(
|
|
554
540
|
False,
|
|
555
541
|
"-V",
|
|
@@ -576,9 +562,7 @@ def count_values(
|
|
|
576
562
|
result = databrowser.count_values(
|
|
577
563
|
*facets,
|
|
578
564
|
time=time or "",
|
|
579
|
-
time_select=cast(
|
|
580
|
-
Literal["file", "flexible", "strict"], time_select
|
|
581
|
-
),
|
|
565
|
+
time_select=cast(Literal["file", "flexible", "strict"], time_select),
|
|
582
566
|
flavour=cast(
|
|
583
567
|
Literal["freva", "cmip6", "cmip5", "cordex", "nextgems"],
|
|
584
568
|
flavour.value,
|
|
@@ -594,9 +578,7 @@ def count_values(
|
|
|
594
578
|
databrowser(
|
|
595
579
|
*facets,
|
|
596
580
|
time=time or "",
|
|
597
|
-
time_select=cast(
|
|
598
|
-
Literal["file", "flexible", "strict"], time_select
|
|
599
|
-
),
|
|
581
|
+
time_select=cast(Literal["file", "flexible", "strict"], time_select),
|
|
600
582
|
flavour=cast(
|
|
601
583
|
Literal["freva", "cmip6", "cmip5", "cordex", "nextgems"],
|
|
602
584
|
flavour.value,
|
|
@@ -620,3 +602,108 @@ def count_values(
|
|
|
620
602
|
print(f"{key}: {', '.join(counts)}")
|
|
621
603
|
else:
|
|
622
604
|
print(result)
|
|
605
|
+
|
|
606
|
+
|
|
607
|
+
user_data_app = typer.Typer(help="Add or delete user data.")
|
|
608
|
+
databrowser_app.add_typer(user_data_app, name="user-data")
|
|
609
|
+
|
|
610
|
+
|
|
611
|
+
@user_data_app.command(name="add", help="Add user data into the databrowser.")
|
|
612
|
+
@exception_handler
|
|
613
|
+
def user_data_add(
|
|
614
|
+
username: str = typer.Argument(..., help="Username of the data owner"),
|
|
615
|
+
paths: List[str] = typer.Option(
|
|
616
|
+
...,
|
|
617
|
+
"--path",
|
|
618
|
+
"-p",
|
|
619
|
+
help="Paths to the user's data to be added.",
|
|
620
|
+
),
|
|
621
|
+
facets: Optional[List[str]] = typer.Option(
|
|
622
|
+
None,
|
|
623
|
+
"--facet",
|
|
624
|
+
"-f",
|
|
625
|
+
help="Facet key-value pairs for metadata in the format key=value.",
|
|
626
|
+
),
|
|
627
|
+
host: Optional[str] = typer.Option(
|
|
628
|
+
None,
|
|
629
|
+
"--host",
|
|
630
|
+
help=(
|
|
631
|
+
"Set the hostname of the databrowser. If not set (default), "
|
|
632
|
+
"the hostname is read from a config file."
|
|
633
|
+
),
|
|
634
|
+
),
|
|
635
|
+
access_token: Optional[str] = typer.Option(
|
|
636
|
+
None,
|
|
637
|
+
"--access-token",
|
|
638
|
+
help="Access token for authentication when adding user data.",
|
|
639
|
+
),
|
|
640
|
+
verbose: int = typer.Option(0, "-v", help="Increase verbosity", count=True),
|
|
641
|
+
) -> None:
|
|
642
|
+
"""Add user data into the databrowser."""
|
|
643
|
+
logger.set_verbosity(verbose)
|
|
644
|
+
logger.debug("Checking if the user has the right to add data")
|
|
645
|
+
result = databrowser(host=host)
|
|
646
|
+
_auth(result._cfg.auth_url, access_token)
|
|
647
|
+
|
|
648
|
+
facet_dict = {}
|
|
649
|
+
if facets:
|
|
650
|
+
for facet in facets:
|
|
651
|
+
if "=" not in facet:
|
|
652
|
+
logger.error(
|
|
653
|
+
f"Invalid facet format: {facet}. Expected format: key=value."
|
|
654
|
+
)
|
|
655
|
+
raise typer.Exit(code=1)
|
|
656
|
+
key, value = facet.split("=", 1)
|
|
657
|
+
facet_dict[key] = value
|
|
658
|
+
|
|
659
|
+
logger.debug(
|
|
660
|
+
f"Adding user data for {username} with paths {paths} and facets {facet_dict}"
|
|
661
|
+
)
|
|
662
|
+
result.add_user_data(username=username, paths=paths, facets=facet_dict)
|
|
663
|
+
logger.info("User data started crawling. Check the Databrowser to see the updates.")
|
|
664
|
+
|
|
665
|
+
|
|
666
|
+
@user_data_app.command(name="delete", help="Delete user data from the databrowser.")
|
|
667
|
+
@exception_handler
|
|
668
|
+
def user_data_remove(
|
|
669
|
+
username: str = typer.Argument(..., help="Username of the data owner"),
|
|
670
|
+
search_keys: List[str] = typer.Option(
|
|
671
|
+
None,
|
|
672
|
+
"--search-key",
|
|
673
|
+
"-s",
|
|
674
|
+
help="Search keys for the data to be deleted in the format key=value.",
|
|
675
|
+
),
|
|
676
|
+
host: Optional[str] = typer.Option(
|
|
677
|
+
None,
|
|
678
|
+
"--host",
|
|
679
|
+
help=(
|
|
680
|
+
"Set the hostname of the databrowser. If not set (default), "
|
|
681
|
+
"the hostname is read from a config file."
|
|
682
|
+
),
|
|
683
|
+
),
|
|
684
|
+
access_token: Optional[str] = typer.Option(
|
|
685
|
+
None,
|
|
686
|
+
"--access-token",
|
|
687
|
+
help="Access token for authentication when deleting user data.",
|
|
688
|
+
),
|
|
689
|
+
verbose: int = typer.Option(0, "-v", help="Increase verbosity", count=True),
|
|
690
|
+
) -> None:
|
|
691
|
+
"""Delete user data from the databrowser."""
|
|
692
|
+
logger.set_verbosity(verbose)
|
|
693
|
+
logger.debug("Checking if the user has the right to delete data")
|
|
694
|
+
result = databrowser(host=host)
|
|
695
|
+
_auth(result._cfg.auth_url, access_token)
|
|
696
|
+
|
|
697
|
+
search_key_dict = {}
|
|
698
|
+
if search_keys:
|
|
699
|
+
for search_key in search_keys:
|
|
700
|
+
if "=" not in search_key:
|
|
701
|
+
logger.error(
|
|
702
|
+
f"Invalid search key format: {search_key}. "
|
|
703
|
+
"Expected format: key=value."
|
|
704
|
+
)
|
|
705
|
+
raise typer.Exit(code=1)
|
|
706
|
+
key, value = search_key.split("=", 1)
|
|
707
|
+
search_key_dict[key] = value
|
|
708
|
+
result.delete_user_data(username=username, search_keys=search_key_dict)
|
|
709
|
+
logger.info("User data deleted successfully.")
|
|
@@ -206,9 +206,7 @@ class databrowser:
|
|
|
206
206
|
self,
|
|
207
207
|
*facets: str,
|
|
208
208
|
uniq_key: Literal["file", "uri"] = "file",
|
|
209
|
-
flavour: Literal[
|
|
210
|
-
"freva", "cmip6", "cmip5", "cordex", "nextgems"
|
|
211
|
-
] = "freva",
|
|
209
|
+
flavour: Literal["freva", "cmip6", "cmip5", "cordex", "nextgems"] = "freva",
|
|
212
210
|
time: Optional[str] = None,
|
|
213
211
|
host: Optional[str] = None,
|
|
214
212
|
time_select: Literal["flexible", "strict", "file"] = "flexible",
|
|
@@ -243,8 +241,7 @@ class databrowser:
|
|
|
243
241
|
self, facets: Tuple[str, ...], search_kw: Dict[str, List[str]]
|
|
244
242
|
) -> None:
|
|
245
243
|
metadata = {
|
|
246
|
-
k: v[::2]
|
|
247
|
-
for (k, v) in self._facet_search(extended_search=True).items()
|
|
244
|
+
k: v[::2] for (k, v) in self._facet_search(extended_search=True).items()
|
|
248
245
|
}
|
|
249
246
|
primary_key = list(metadata.keys() or ["project"])[0]
|
|
250
247
|
num_facets = 0
|
|
@@ -267,9 +264,7 @@ class databrowser:
|
|
|
267
264
|
headers = {}
|
|
268
265
|
if self._stream_zarr:
|
|
269
266
|
query_url = self._cfg.zarr_loader_url
|
|
270
|
-
token = self._auth.check_authentication(
|
|
271
|
-
auth_url=self._cfg.auth_url
|
|
272
|
-
)
|
|
267
|
+
token = self._auth.check_authentication(auth_url=self._cfg.auth_url)
|
|
273
268
|
headers = {"Authorization": f"Bearer {token['access_token']}"}
|
|
274
269
|
result = self._get(query_url, headers=headers, stream=True)
|
|
275
270
|
if result is not None:
|
|
@@ -277,9 +272,7 @@ class databrowser:
|
|
|
277
272
|
for res in result.iter_lines():
|
|
278
273
|
yield res.decode("utf-8")
|
|
279
274
|
except KeyboardInterrupt:
|
|
280
|
-
pprint(
|
|
281
|
-
"[red][b]User interrupt: Exit[/red][/b]", file=sys.stderr
|
|
282
|
-
)
|
|
275
|
+
pprint("[red][b]User interrupt: Exit[/red][/b]", file=sys.stderr)
|
|
283
276
|
|
|
284
277
|
def __repr__(self) -> str:
|
|
285
278
|
params = ", ".join(
|
|
@@ -306,9 +299,7 @@ class databrowser:
|
|
|
306
299
|
|
|
307
300
|
# Create a table-like structure for available flavors and search facets
|
|
308
301
|
style = 'style="text-align: left"'
|
|
309
|
-
facet_heading =
|
|
310
|
-
f"Available search facets for <em>{self._flavour}</em> flavour"
|
|
311
|
-
)
|
|
302
|
+
facet_heading = f"Available search facets for <em>{self._flavour}</em> flavour"
|
|
312
303
|
html_repr = (
|
|
313
304
|
"<table>"
|
|
314
305
|
f"<tr><th colspan='2' {style}>{self.__class__.__name__}"
|
|
@@ -347,13 +338,9 @@ class databrowser:
|
|
|
347
338
|
kwargs: Dict[str, Any] = {"stream": True}
|
|
348
339
|
url = self._cfg.intake_url
|
|
349
340
|
if self._stream_zarr:
|
|
350
|
-
token = self._auth.check_authentication(
|
|
351
|
-
auth_url=self._cfg.auth_url
|
|
352
|
-
)
|
|
341
|
+
token = self._auth.check_authentication(auth_url=self._cfg.auth_url)
|
|
353
342
|
url = self._cfg.zarr_loader_url
|
|
354
|
-
kwargs["headers"] = {
|
|
355
|
-
"Authorization": f"Bearer {token['access_token']}"
|
|
356
|
-
}
|
|
343
|
+
kwargs["headers"] = {"Authorization": f"Bearer {token['access_token']}"}
|
|
357
344
|
kwargs["params"] = {"catalogue-type": "intake"}
|
|
358
345
|
result = self._get(url, **kwargs)
|
|
359
346
|
if result is None:
|
|
@@ -365,9 +352,7 @@ class databrowser:
|
|
|
365
352
|
for content in result.iter_content(decode_unicode=False):
|
|
366
353
|
stream.write(content)
|
|
367
354
|
except Exception as error:
|
|
368
|
-
raise ValueError(
|
|
369
|
-
f"Couldn't write catalogue content: {error}"
|
|
370
|
-
) from None
|
|
355
|
+
raise ValueError(f"Couldn't write catalogue content: {error}") from None
|
|
371
356
|
|
|
372
357
|
def intake_catalogue(self) -> intake_esm.core.esm_datastore:
|
|
373
358
|
"""Create an intake esm catalogue object from the search.
|
|
@@ -404,9 +389,7 @@ class databrowser:
|
|
|
404
389
|
def count_values(
|
|
405
390
|
cls,
|
|
406
391
|
*facets: str,
|
|
407
|
-
flavour: Literal[
|
|
408
|
-
"freva", "cmip6", "cmip5", "cordex", "nextgems"
|
|
409
|
-
] = "freva",
|
|
392
|
+
flavour: Literal["freva", "cmip6", "cmip5", "cordex", "nextgems"] = "freva",
|
|
410
393
|
time: Optional[str] = None,
|
|
411
394
|
host: Optional[str] = None,
|
|
412
395
|
time_select: Literal["flexible", "strict", "file"] = "flexible",
|
|
@@ -504,9 +487,7 @@ class databrowser:
|
|
|
504
487
|
result = this._facet_search(extended_search=extended_search)
|
|
505
488
|
counts = {}
|
|
506
489
|
for facet, value_counts in result.items():
|
|
507
|
-
counts[facet] = dict(
|
|
508
|
-
zip(value_counts[::2], map(int, value_counts[1::2]))
|
|
509
|
-
)
|
|
490
|
+
counts[facet] = dict(zip(value_counts[::2], map(int, value_counts[1::2])))
|
|
510
491
|
return counts
|
|
511
492
|
|
|
512
493
|
@cached_property
|
|
@@ -531,17 +512,14 @@ class databrowser:
|
|
|
531
512
|
|
|
532
513
|
"""
|
|
533
514
|
return {
|
|
534
|
-
k: v[::2]
|
|
535
|
-
for (k, v) in self._facet_search(extended_search=True).items()
|
|
515
|
+
k: v[::2] for (k, v) in self._facet_search(extended_search=True).items()
|
|
536
516
|
}
|
|
537
517
|
|
|
538
518
|
@classmethod
|
|
539
519
|
def metadata_search(
|
|
540
520
|
cls,
|
|
541
521
|
*facets: str,
|
|
542
|
-
flavour: Literal[
|
|
543
|
-
"freva", "cmip6", "cmip5", "cordex", "nextgems"
|
|
544
|
-
] = "freva",
|
|
522
|
+
flavour: Literal["freva", "cmip6", "cmip5", "cordex", "nextgems"] = "freva",
|
|
545
523
|
time: Optional[str] = None,
|
|
546
524
|
host: Optional[str] = None,
|
|
547
525
|
time_select: Literal["flexible", "strict", "file"] = "flexible",
|
|
@@ -664,9 +642,7 @@ class databrowser:
|
|
|
664
642
|
)
|
|
665
643
|
return {
|
|
666
644
|
k: v[::2]
|
|
667
|
-
for (k, v) in this._facet_search(
|
|
668
|
-
extended_search=extended_search
|
|
669
|
-
).items()
|
|
645
|
+
for (k, v) in this._facet_search(extended_search=extended_search).items()
|
|
670
646
|
}
|
|
671
647
|
|
|
672
648
|
@classmethod
|
|
@@ -733,17 +709,111 @@ class databrowser:
|
|
|
733
709
|
constraints = data["primary_facets"]
|
|
734
710
|
return {f: v for f, v in data["facets"].items() if f in constraints}
|
|
735
711
|
|
|
736
|
-
def
|
|
737
|
-
self,
|
|
738
|
-
) ->
|
|
712
|
+
def add_user_data(
|
|
713
|
+
self, username: str, paths: List[str], facets: Dict[str, str]
|
|
714
|
+
) -> None:
|
|
715
|
+
"""Add user data to the databrowser.
|
|
716
|
+
|
|
717
|
+
Via this functionality, user would be able to add data to the databrowser.
|
|
718
|
+
It accepts file paths and metadata facets to categorize and store the user's
|
|
719
|
+
data.
|
|
720
|
+
|
|
721
|
+
Parameters
|
|
722
|
+
~~~~~~~~~~
|
|
723
|
+
username: str
|
|
724
|
+
The username of user.
|
|
725
|
+
paths: list[str]
|
|
726
|
+
A list of paths to the data files that should be uploaded or cataloged.
|
|
727
|
+
facets: dict[str, str]
|
|
728
|
+
A dictionary containing metadata facets (key-value pairs) to describe the
|
|
729
|
+
data.
|
|
730
|
+
|
|
731
|
+
Returns
|
|
732
|
+
~~~~~~~~
|
|
733
|
+
None
|
|
734
|
+
If the operation is successful, no return value is provided.
|
|
735
|
+
|
|
736
|
+
Raises
|
|
737
|
+
~~~~~~~
|
|
738
|
+
ValueError
|
|
739
|
+
If the operation fails to add the user data.
|
|
740
|
+
|
|
741
|
+
Example
|
|
742
|
+
~~~~~~~
|
|
743
|
+
.. execute_code::
|
|
744
|
+
|
|
745
|
+
from freva_client import authenticate, databrowser
|
|
746
|
+
token_info = authenticate(username="janedoe")
|
|
747
|
+
db = databrowser()
|
|
748
|
+
db.add_user_data(
|
|
749
|
+
"janedoe",
|
|
750
|
+
["."],
|
|
751
|
+
{"project": "cmip5", "experiment": "something"}
|
|
752
|
+
)
|
|
753
|
+
"""
|
|
754
|
+
url = f"{self._cfg.userdata_url}/{username}"
|
|
755
|
+
token = self._auth.check_authentication(auth_url=self._cfg.auth_url)
|
|
756
|
+
headers = {"Authorization": f"Bearer {token['access_token']}"}
|
|
757
|
+
params = {"paths": paths}
|
|
758
|
+
if "username" in facets:
|
|
759
|
+
del facets["username"]
|
|
760
|
+
data = facets
|
|
761
|
+
result = self._put(url, data=data, headers=headers, params=params)
|
|
762
|
+
|
|
763
|
+
if result is None:
|
|
764
|
+
raise ValueError("Failed to add user data")
|
|
765
|
+
|
|
766
|
+
def delete_user_data(self, username: str, search_keys: Dict[str, str]) -> None:
|
|
767
|
+
"""
|
|
768
|
+
Delete user data from the databrowser.
|
|
769
|
+
|
|
770
|
+
Uing this, user would be able to delete the user's data from the databrowser
|
|
771
|
+
based on the provided search keys.
|
|
772
|
+
|
|
773
|
+
Parameters
|
|
774
|
+
~~~~~~~~~~
|
|
775
|
+
username: str
|
|
776
|
+
The username associated with the data to be deleted.
|
|
777
|
+
search_keys: dict[str, str]
|
|
778
|
+
A dictionary containing the search keys to identify the data to be deleted.
|
|
779
|
+
|
|
780
|
+
Returns
|
|
781
|
+
~~~~~~~~
|
|
782
|
+
None
|
|
783
|
+
If the operation is successful, no return value is provided.
|
|
784
|
+
|
|
785
|
+
Raises
|
|
786
|
+
~~~~~~~
|
|
787
|
+
ValueError
|
|
788
|
+
If the operation fails to delete the user data.
|
|
789
|
+
|
|
790
|
+
Example
|
|
791
|
+
~~~~~~~
|
|
792
|
+
.. execute_code::
|
|
793
|
+
|
|
794
|
+
from freva_client import databrowser, authenticate
|
|
795
|
+
token_info = authenticate(username="janedoe")
|
|
796
|
+
db = databrowser()
|
|
797
|
+
db.delete_user_data(
|
|
798
|
+
"janedoe",
|
|
799
|
+
{"project": "cmip5", "experiment": "something"}
|
|
800
|
+
)
|
|
801
|
+
"""
|
|
802
|
+
url = f"{self._cfg.userdata_url}/{username}"
|
|
803
|
+
token = self._auth.check_authentication(auth_url=self._cfg.auth_url)
|
|
804
|
+
headers = {"Authorization": f"Bearer {token['access_token']}"}
|
|
805
|
+
data = search_keys
|
|
806
|
+
result = self._delete(url, headers=headers, json=data)
|
|
807
|
+
if result is None:
|
|
808
|
+
raise ValueError("Failed to delete user data")
|
|
809
|
+
|
|
810
|
+
def _get(self, url: str, **kwargs: Any) -> Optional[requests.models.Response]:
|
|
739
811
|
"""Apply the get method to the databrowser."""
|
|
740
812
|
logger.debug("Searching %s with parameters: %s", url, self._params)
|
|
741
813
|
params = kwargs.pop("params", {})
|
|
742
814
|
kwargs.setdefault("timeout", 30)
|
|
743
815
|
try:
|
|
744
|
-
res = requests.get(
|
|
745
|
-
url, params={**self._params, **params}, **kwargs
|
|
746
|
-
)
|
|
816
|
+
res = requests.get(url, params={**self._params, **params}, **kwargs)
|
|
747
817
|
res.raise_for_status()
|
|
748
818
|
return res
|
|
749
819
|
except KeyboardInterrupt:
|
|
@@ -757,3 +827,55 @@ class databrowser:
|
|
|
757
827
|
raise ValueError(msg) from None
|
|
758
828
|
logger.warning(msg)
|
|
759
829
|
return None
|
|
830
|
+
|
|
831
|
+
def _put(
|
|
832
|
+
self, url: str, data: Dict[str, Any], **kwargs: Any
|
|
833
|
+
) -> Optional[requests.models.Response]:
|
|
834
|
+
"""Apply the PUT method to the databrowser."""
|
|
835
|
+
logger.debug(
|
|
836
|
+
"PUT request to %s with data: %s and parameters: %s",
|
|
837
|
+
url,
|
|
838
|
+
data,
|
|
839
|
+
self._params,
|
|
840
|
+
)
|
|
841
|
+
kwargs.setdefault("timeout", 30)
|
|
842
|
+
params = kwargs.pop("params", {})
|
|
843
|
+
try:
|
|
844
|
+
res = requests.put(
|
|
845
|
+
url, json=data, params={**self._params, **params}, **kwargs
|
|
846
|
+
)
|
|
847
|
+
res.raise_for_status()
|
|
848
|
+
return res
|
|
849
|
+
except KeyboardInterrupt:
|
|
850
|
+
pprint("[red][b]User interrupt: Exit[/red][/b]", file=sys.stderr)
|
|
851
|
+
|
|
852
|
+
except (
|
|
853
|
+
requests.exceptions.ConnectionError,
|
|
854
|
+
requests.exceptions.HTTPError,
|
|
855
|
+
) as error:
|
|
856
|
+
msg = f"adding user data request failed with {error}"
|
|
857
|
+
if self._fail_on_error:
|
|
858
|
+
raise ValueError(msg) from None
|
|
859
|
+
logger.warning(msg)
|
|
860
|
+
return None
|
|
861
|
+
|
|
862
|
+
def _delete(self, url: str, **kwargs: Any) -> Optional[requests.models.Response]:
|
|
863
|
+
"""Apply the DELETE method to the databrowser."""
|
|
864
|
+
logger.debug("DELETE request to %s with parameters: %s", url, self._params)
|
|
865
|
+
params = kwargs.pop("params", {})
|
|
866
|
+
kwargs.setdefault("timeout", 30)
|
|
867
|
+
try:
|
|
868
|
+
res = requests.delete(url, params={**self._params, **params}, **kwargs)
|
|
869
|
+
res.raise_for_status()
|
|
870
|
+
return res
|
|
871
|
+
except KeyboardInterrupt:
|
|
872
|
+
pprint("[red][b]User interrupt: Exit[/red][/b]", file=sys.stderr)
|
|
873
|
+
except (
|
|
874
|
+
requests.exceptions.ConnectionError,
|
|
875
|
+
requests.exceptions.HTTPError,
|
|
876
|
+
) as error:
|
|
877
|
+
msg = f"DELETE request failed with {error}"
|
|
878
|
+
if self._fail_on_error:
|
|
879
|
+
raise ValueError(msg) from None
|
|
880
|
+
logger.warning(msg)
|
|
881
|
+
return None
|
{freva_client-2408.0.0.dev2 → freva_client-2410.0.0b1}/src/freva_client/utils/databrowser_utils.py
RENAMED
|
@@ -57,9 +57,7 @@ class Config:
|
|
|
57
57
|
host = f"{host}:{port}"
|
|
58
58
|
return f"{scheme}://{host}"
|
|
59
59
|
|
|
60
|
-
def _read_config(
|
|
61
|
-
self, path: Path, file_type: Literal["toml", "ini"]
|
|
62
|
-
) -> str:
|
|
60
|
+
def _read_config(self, path: Path, file_type: Literal["toml", "ini"]) -> str:
|
|
63
61
|
"""Read the configuration."""
|
|
64
62
|
data_types = {"toml": self._read_toml, "ini": self._read_ini}
|
|
65
63
|
try:
|
|
@@ -72,11 +70,9 @@ class Config:
|
|
|
72
70
|
def overview(self) -> Dict[str, Any]:
|
|
73
71
|
"""Get an overview of the all databrowser flavours and search keys."""
|
|
74
72
|
try:
|
|
75
|
-
res = requests.get(f"{self.databrowser_url}/overview", timeout=
|
|
73
|
+
res = requests.get(f"{self.databrowser_url}/overview", timeout=15)
|
|
76
74
|
except requests.exceptions.ConnectionError:
|
|
77
|
-
raise ValueError(
|
|
78
|
-
f"Could not connect to {self.databrowser_url}"
|
|
79
|
-
) from None
|
|
75
|
+
raise ValueError(f"Could not connect to {self.databrowser_url}") from None
|
|
80
76
|
return cast(Dict[str, Any], res.json())
|
|
81
77
|
|
|
82
78
|
def _get_databrowser_host_from_config(self) -> str:
|
|
@@ -91,9 +87,7 @@ class Config:
|
|
|
91
87
|
Path(appdirs.user_config_dir("freva")) / "freva.toml": "toml",
|
|
92
88
|
Path(self.get_dirs(user=True)) / "freva.toml": "toml",
|
|
93
89
|
freva_config: "toml",
|
|
94
|
-
Path(
|
|
95
|
-
os.environ.get("EVALUATION_SYSTEM_CONFIG_FILE") or eval_conf
|
|
96
|
-
): "ini",
|
|
90
|
+
Path(os.environ.get("EVALUATION_SYSTEM_CONFIG_FILE") or eval_conf): "ini",
|
|
97
91
|
}
|
|
98
92
|
for config_path, config_type in paths.items():
|
|
99
93
|
if config_path.is_file():
|
|
@@ -136,8 +130,7 @@ class Config:
|
|
|
136
130
|
def metadata_url(self) -> str:
|
|
137
131
|
"""Define the endpoint for the metadata search."""
|
|
138
132
|
return (
|
|
139
|
-
f"{self.databrowser_url}/metadata_search/"
|
|
140
|
-
f"{self.flavour}/{self.uniq_key}"
|
|
133
|
+
f"{self.databrowser_url}/metadata_search/" f"{self.flavour}/{self.uniq_key}"
|
|
141
134
|
)
|
|
142
135
|
|
|
143
136
|
@staticmethod
|
|
@@ -177,3 +170,8 @@ class Config:
|
|
|
177
170
|
# The default scheme is 'posix_prefix' or 'nt', and should work for e.g.
|
|
178
171
|
# installing into a virtualenv
|
|
179
172
|
return Path(sysconfig.get_path("data")) / "share" / "freva"
|
|
173
|
+
|
|
174
|
+
@property
|
|
175
|
+
def userdata_url(self) -> str:
|
|
176
|
+
"""Define the url for adding and deleting user-data."""
|
|
177
|
+
return f"{self.databrowser_url}/userdata"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|