esgpull 0.9.2__py3-none-any.whl → 0.9.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- esgpull/cli/__init__.py +2 -2
- esgpull/cli/index_nodes.py +94 -0
- esgpull/cli/search.py +1 -0
- esgpull/cli/update.py +1 -0
- esgpull/config.py +211 -295
- esgpull/constants.py +1 -0
- esgpull/context.py +32 -3
- esgpull/esgpull.py +0 -5
- esgpull/migrations/versions/0.9.3_update_tables.py +28 -0
- esgpull/plugin.py +3 -2
- esgpull/processor.py +0 -4
- esgpull/tui.py +5 -5
- {esgpull-0.9.2.dist-info → esgpull-0.9.3.dist-info}/METADATA +3 -2
- {esgpull-0.9.2.dist-info → esgpull-0.9.3.dist-info}/RECORD +17 -17
- esgpull/auth.py +0 -181
- esgpull/cli/login.py +0 -56
- {esgpull-0.9.2.dist-info → esgpull-0.9.3.dist-info}/WHEEL +0 -0
- {esgpull-0.9.2.dist-info → esgpull-0.9.3.dist-info}/entry_points.txt +0 -0
- {esgpull-0.9.2.dist-info → esgpull-0.9.3.dist-info}/licenses/LICENSE +0 -0
esgpull/context.py
CHANGED
|
@@ -22,7 +22,11 @@ from esgpull.tui import logger
|
|
|
22
22
|
from esgpull.utils import format_date_iso, sync
|
|
23
23
|
|
|
24
24
|
# workaround for notebooks with running event loop
|
|
25
|
-
|
|
25
|
+
try:
|
|
26
|
+
asyncio.get_running_loop()
|
|
27
|
+
except RuntimeError:
|
|
28
|
+
pass
|
|
29
|
+
else:
|
|
26
30
|
import nest_asyncio
|
|
27
31
|
|
|
28
32
|
nest_asyncio.apply()
|
|
@@ -63,6 +67,16 @@ class IndexNode:
|
|
|
63
67
|
return result
|
|
64
68
|
|
|
65
69
|
|
|
70
|
+
def quote_str(s: str) -> str:
|
|
71
|
+
if "*" in s:
|
|
72
|
+
# don't quote when `*` is present, quotes enforce exact match
|
|
73
|
+
return s
|
|
74
|
+
elif not s.startswith('"') and not s.endswith('"'):
|
|
75
|
+
return f'"{s}"'
|
|
76
|
+
else:
|
|
77
|
+
return s
|
|
78
|
+
|
|
79
|
+
|
|
66
80
|
@dataclass
|
|
67
81
|
class Result:
|
|
68
82
|
query: Query
|
|
@@ -119,14 +133,17 @@ class Result:
|
|
|
119
133
|
# query["end"] = format_date_iso(str(facets.pop("end")))
|
|
120
134
|
solr_terms: list[str] = []
|
|
121
135
|
for name, values in self.query.selection.items():
|
|
122
|
-
|
|
136
|
+
if index.is_bridge():
|
|
137
|
+
value_term = " ".join(quote_str(v) for v in values)
|
|
138
|
+
else:
|
|
139
|
+
value_term = " ".join(values)
|
|
123
140
|
if name == "query": # freetext case
|
|
124
141
|
solr_terms.append(value_term)
|
|
125
142
|
else:
|
|
126
143
|
if len(values) > 1:
|
|
127
144
|
value_term = f"({value_term})"
|
|
128
145
|
if name.startswith("!"):
|
|
129
|
-
solr_terms.append(f"
|
|
146
|
+
solr_terms.append(f"NOT ({name[1:]}:{value_term})")
|
|
130
147
|
else:
|
|
131
148
|
solr_terms.append(f"{name}:{value_term}")
|
|
132
149
|
if solr_terms:
|
|
@@ -507,6 +524,8 @@ class Context:
|
|
|
507
524
|
result.process()
|
|
508
525
|
if result.processed:
|
|
509
526
|
hits.append(result.data)
|
|
527
|
+
else:
|
|
528
|
+
hits.append(0)
|
|
510
529
|
return hits
|
|
511
530
|
|
|
512
531
|
async def _hints(self, *results: ResultHints) -> list[HintsDict]:
|
|
@@ -756,3 +775,13 @@ class Context:
|
|
|
756
775
|
date_to=date_to,
|
|
757
776
|
keep_duplicates=keep_duplicates,
|
|
758
777
|
)
|
|
778
|
+
|
|
779
|
+
def probe(self, index_node: str | None = None) -> None:
|
|
780
|
+
noraise = self.noraise
|
|
781
|
+
self.noraise = False
|
|
782
|
+
_ = self.hits(
|
|
783
|
+
Query(),
|
|
784
|
+
file=True,
|
|
785
|
+
index_node=index_node or self.config.api.index_node,
|
|
786
|
+
)
|
|
787
|
+
self.noraise = noraise
|
esgpull/esgpull.py
CHANGED
|
@@ -22,7 +22,6 @@ from rich.progress import (
|
|
|
22
22
|
TransferSpeedColumn,
|
|
23
23
|
)
|
|
24
24
|
|
|
25
|
-
from esgpull.auth import Auth, Credentials
|
|
26
25
|
from esgpull.config import Config
|
|
27
26
|
from esgpull.context import Context
|
|
28
27
|
from esgpull.database import Database
|
|
@@ -64,7 +63,6 @@ class Esgpull:
|
|
|
64
63
|
path: Path
|
|
65
64
|
config: Config
|
|
66
65
|
ui: UI
|
|
67
|
-
auth: Auth
|
|
68
66
|
db: Database
|
|
69
67
|
context: Context
|
|
70
68
|
fs: Filesystem
|
|
@@ -120,8 +118,6 @@ class Esgpull:
|
|
|
120
118
|
verbosity=verbosity,
|
|
121
119
|
record=record,
|
|
122
120
|
)
|
|
123
|
-
credentials = Credentials.from_config(self.config)
|
|
124
|
-
self.auth = Auth.from_config(self.config, credentials)
|
|
125
121
|
self.context = Context(self.config, noraise=True)
|
|
126
122
|
if load_db:
|
|
127
123
|
self.db = Database.from_config(self.config)
|
|
@@ -431,7 +427,6 @@ class Esgpull:
|
|
|
431
427
|
start_callbacks[file.sha] = [callback]
|
|
432
428
|
processor = Processor(
|
|
433
429
|
config=self.config,
|
|
434
|
-
auth=self.auth,
|
|
435
430
|
fs=self.fs,
|
|
436
431
|
files=queue,
|
|
437
432
|
start_callbacks=start_callbacks,
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""update tables
|
|
2
|
+
|
|
3
|
+
Revision ID: 0.9.3
|
|
4
|
+
Revises: 0.9.2
|
|
5
|
+
Create Date: 2025-11-18 15:40:35.122823
|
|
6
|
+
|
|
7
|
+
"""
|
|
8
|
+
from alembic import op
|
|
9
|
+
import sqlalchemy as sa
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
# revision identifiers, used by Alembic.
|
|
13
|
+
revision = '0.9.3'
|
|
14
|
+
down_revision = '0.9.2'
|
|
15
|
+
branch_labels = None
|
|
16
|
+
depends_on = None
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def upgrade() -> None:
|
|
20
|
+
# ### commands auto generated by Alembic - please adjust! ###
|
|
21
|
+
pass
|
|
22
|
+
# ### end Alembic commands ###
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def downgrade() -> None:
|
|
26
|
+
# ### commands auto generated by Alembic - please adjust! ###
|
|
27
|
+
pass
|
|
28
|
+
# ### end Alembic commands ###
|
esgpull/plugin.py
CHANGED
|
@@ -12,9 +12,9 @@ from typing import Any, Callable, Literal
|
|
|
12
12
|
|
|
13
13
|
import tomlkit
|
|
14
14
|
from packaging import version
|
|
15
|
+
from pydantic import TypeAdapter
|
|
15
16
|
|
|
16
17
|
import esgpull.models
|
|
17
|
-
from esgpull.config import cast_value
|
|
18
18
|
from esgpull.tui import logger
|
|
19
19
|
from esgpull.version import __version__
|
|
20
20
|
|
|
@@ -473,7 +473,8 @@ class PluginManager:
|
|
|
473
473
|
# Update the value in both places
|
|
474
474
|
if key in self.config.plugins[plugin_name]:
|
|
475
475
|
old_value = self.config.plugins[plugin_name][key]
|
|
476
|
-
|
|
476
|
+
ta = TypeAdapter(type(old_value))
|
|
477
|
+
new_value = ta.validate_python(value)
|
|
477
478
|
self.config.plugins[plugin_name][key] = new_value
|
|
478
479
|
# Also update in _raw to keep in sync
|
|
479
480
|
self.config._raw["plugins"][plugin_name][key] = new_value
|
esgpull/processor.py
CHANGED
|
@@ -7,7 +7,6 @@ from typing import TypeAlias
|
|
|
7
7
|
from aiostream.stream import merge
|
|
8
8
|
from httpx import AsyncClient, HTTPError
|
|
9
9
|
|
|
10
|
-
from esgpull.auth import Auth
|
|
11
10
|
from esgpull.config import Config
|
|
12
11
|
from esgpull.download import DownloadCtx, Simple
|
|
13
12
|
from esgpull.exceptions import DownloadSizeError
|
|
@@ -122,13 +121,11 @@ class Processor:
|
|
|
122
121
|
def __init__(
|
|
123
122
|
self,
|
|
124
123
|
config: Config,
|
|
125
|
-
auth: Auth,
|
|
126
124
|
fs: Filesystem,
|
|
127
125
|
files: list[File],
|
|
128
126
|
start_callbacks: dict[str, list[Callback]],
|
|
129
127
|
) -> None:
|
|
130
128
|
self.config = config
|
|
131
|
-
self.auth = auth
|
|
132
129
|
self.fs = fs
|
|
133
130
|
self.files = list(filter(self.should_download, files))
|
|
134
131
|
self.tasks: list[Task] = []
|
|
@@ -161,7 +158,6 @@ class Processor:
|
|
|
161
158
|
semaphore = asyncio.Semaphore(self.config.download.max_concurrent)
|
|
162
159
|
async with AsyncClient(
|
|
163
160
|
follow_redirects=True,
|
|
164
|
-
cert=self.auth.cert,
|
|
165
161
|
verify=self.ssl_context,
|
|
166
162
|
timeout=self.config.download.http_timeout,
|
|
167
163
|
) as client:
|
esgpull/tui.py
CHANGED
|
@@ -28,7 +28,7 @@ from tomlkit import dumps as tomlkit_dumps
|
|
|
28
28
|
from yaml import dump as yaml_dump
|
|
29
29
|
|
|
30
30
|
from esgpull.config import Config
|
|
31
|
-
from esgpull.constants import ESGPULL_DEBUG
|
|
31
|
+
from esgpull.constants import ESGPULL_DEBUG, ESGPULL_DEBUG_LOCALS
|
|
32
32
|
|
|
33
33
|
logger = logging.getLogger("esgpull")
|
|
34
34
|
logging.root.setLevel(logging.DEBUG)
|
|
@@ -180,7 +180,7 @@ class UI:
|
|
|
180
180
|
yield
|
|
181
181
|
except (click.exceptions.Exit, click.exceptions.Abort):
|
|
182
182
|
if temp_path is not None:
|
|
183
|
-
atexit.register(temp_path.unlink)
|
|
183
|
+
atexit.register(lambda: temp_path.unlink(missing_ok=True))
|
|
184
184
|
raise
|
|
185
185
|
except click.exceptions.ClickException:
|
|
186
186
|
raise
|
|
@@ -205,10 +205,10 @@ class UI:
|
|
|
205
205
|
f"See [yellow]{temp_path}[/] for error log.",
|
|
206
206
|
err=True,
|
|
207
207
|
)
|
|
208
|
-
if ESGPULL_DEBUG:
|
|
208
|
+
if ESGPULL_DEBUG or ESGPULL_DEBUG_LOCALS:
|
|
209
209
|
from rich.traceback import install
|
|
210
210
|
|
|
211
|
-
install()
|
|
211
|
+
install(show_locals=ESGPULL_DEBUG_LOCALS)
|
|
212
212
|
raise
|
|
213
213
|
elif onraise is not None:
|
|
214
214
|
raise onraise
|
|
@@ -218,7 +218,7 @@ class UI:
|
|
|
218
218
|
raise
|
|
219
219
|
else:
|
|
220
220
|
if temp_path is not None:
|
|
221
|
-
atexit.register(temp_path.unlink)
|
|
221
|
+
atexit.register(lambda: temp_path.unlink(missing_ok=True))
|
|
222
222
|
finally:
|
|
223
223
|
logging.root.removeHandler(handler)
|
|
224
224
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: esgpull
|
|
3
|
-
Version: 0.9.
|
|
3
|
+
Version: 0.9.3
|
|
4
4
|
Summary: ESGF data discovery, download, replication tool
|
|
5
5
|
Project-URL: Repository, https://github.com/ESGF/esgf-download
|
|
6
6
|
Project-URL: Documentation, https://esgf.github.io/esgf-download/
|
|
@@ -21,10 +21,11 @@ Requires-Dist: cattrs>=22.2.0
|
|
|
21
21
|
Requires-Dist: click-params>=0.4.0
|
|
22
22
|
Requires-Dist: click>=8.1.3
|
|
23
23
|
Requires-Dist: httpx>=0.23.0
|
|
24
|
-
Requires-Dist: myproxyclient>=2.1.0
|
|
25
24
|
Requires-Dist: nest-asyncio>=1.5.6
|
|
26
25
|
Requires-Dist: packaging>=25.0
|
|
27
26
|
Requires-Dist: platformdirs>=2.6.2
|
|
27
|
+
Requires-Dist: pydantic-settings>=2.10.1
|
|
28
|
+
Requires-Dist: pydantic>=2.11.7
|
|
28
29
|
Requires-Dist: pyopenssl>=22.1.0
|
|
29
30
|
Requires-Dist: pyparsing>=3.0.9
|
|
30
31
|
Requires-Dist: pyyaml>=6.0
|
|
@@ -1,23 +1,22 @@
|
|
|
1
1
|
esgpull/__init__.py,sha256=XItFDIMNmFUNNcKtUgXdfmGwUIWt4AAv0a4mZkfj5P8,240
|
|
2
|
-
esgpull/
|
|
3
|
-
esgpull/
|
|
4
|
-
esgpull/
|
|
5
|
-
esgpull/context.py,sha256=iyHAJ9OXU5_0SYcXNlmJI_M6x3vphJjnnAcsfB4MxQ8,24383
|
|
2
|
+
esgpull/config.py,sha256=APA3IqYyz_TY2FAm_TNSLsFIkOxmdGnbFLnd2H2YXIs,12042
|
|
3
|
+
esgpull/constants.py,sha256=VlgRJDth5aDjyTopJ2W2JU83353XcRf18RIFAZt7yck,1263
|
|
4
|
+
esgpull/context.py,sha256=JEnIQGF1owl56n0azF8K3GNIU8TgeMu31NSfTqVAGQg,25123
|
|
6
5
|
esgpull/database.py,sha256=1wGeNbJp0gOLo5Q1N53JfiwVbZ8nfvZVkKlvEaJwOpU,6716
|
|
7
6
|
esgpull/download.py,sha256=aR2c_SOuZtgX7tI2a9_N4Mn86ABq1k7Mxq_BdojFrP4,5600
|
|
8
|
-
esgpull/esgpull.py,sha256
|
|
7
|
+
esgpull/esgpull.py,sha256=mqvrXnyCzTSU3j_u5pYidpLs2j86TEFDPX6MhCnNfMs,19458
|
|
9
8
|
esgpull/exceptions.py,sha256=wgLyhyIITdusNucPjnnURJX1Jxv1VVIr9PzJV_77qhg,3275
|
|
10
9
|
esgpull/fs.py,sha256=sc7Af2E3yh3V9KVuSPSXFBuFtlQ3L99UZmS1ZJuiBeM,7280
|
|
11
10
|
esgpull/graph.py,sha256=Yl2VuF8PNn0R5xRyEK58Q1Xlx8B1PfhbTwt1JftFDro,15929
|
|
12
11
|
esgpull/install_config.py,sha256=hzYpcHMtPMOK9fYcvVH-Hn_8zYsbs3yXlYgMumXo1zE,5598
|
|
13
|
-
esgpull/plugin.py,sha256=
|
|
14
|
-
esgpull/processor.py,sha256=
|
|
12
|
+
esgpull/plugin.py,sha256=t6iejis65UREQ71gnj1-gVFhlrGt1z1n3SpJ6KiP57E,18904
|
|
13
|
+
esgpull/processor.py,sha256=ubhlKMMOY_ieZEcN9zcDPlmZxQDXqb2QVmeCwWo7zOE,5376
|
|
15
14
|
esgpull/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
16
15
|
esgpull/result.py,sha256=f64l9gPFpFWgctmHVYrNWJvuXXB1sxpxXzJEIssLXxc,1020
|
|
17
|
-
esgpull/tui.py,sha256=
|
|
16
|
+
esgpull/tui.py,sha256=XJs8AUm28YLWl3ephpxY8jHFvG7HrIr08u3RdYvABdE,11338
|
|
18
17
|
esgpull/utils.py,sha256=pLMQfY0p2oNIFyCZhHBD74BOAJtmt9QV6dTJ79VsfF8,1213
|
|
19
18
|
esgpull/version.py,sha256=IHT4mKrIr8eV-C3HtmIVD85iGVH25n2ohoff31kaJ1A,93
|
|
20
|
-
esgpull/cli/__init__.py,sha256=
|
|
19
|
+
esgpull/cli/__init__.py,sha256=W0q_eyhvtPgyX5cXKSDDLUUrskGLzgNoJ9VCY9fJ6pY,1701
|
|
21
20
|
esgpull/cli/add.py,sha256=h8xFy5ORzY9O4SPPo4VCtTybeJJ5GcA18w4TZeL0DqU,3507
|
|
22
21
|
esgpull/cli/autoremove.py,sha256=g76_qnc3q84zSO7W0JsbWtGN4AfWBXTkQJO6gPCs2Pw,1336
|
|
23
22
|
esgpull/cli/config.py,sha256=g7Me173B_SWtdnQcitcTDnj-06-1SW0f8Z8SOcjMAcQ,3683
|
|
@@ -26,17 +25,17 @@ esgpull/cli/decorators.py,sha256=X5Ja6HlB_AgpTohpbF2EjtT5sA0IdPxX20g-j1K7raM,672
|
|
|
26
25
|
esgpull/cli/download.py,sha256=3_Fm8JJYBWKs63oQ1OLaHJCv9ccbeY2gIW8SDasaYNE,2356
|
|
27
26
|
esgpull/cli/facet.py,sha256=V1u-DxNkhswwSt0qpXvuHrCI_tE8jAJGEe6_fMhYbaM,584
|
|
28
27
|
esgpull/cli/get.py,sha256=2WXL01Ri0P_2Rf1xrp9bnsrrxir6whxkAC0SnohjFpg,678
|
|
28
|
+
esgpull/cli/index_nodes.py,sha256=v6DySuWsxtBkWIaKnUVUqJdjAb7J5OIpfgP3B0Ut-ls,2627
|
|
29
29
|
esgpull/cli/install.py,sha256=fd8nKwIFvOivgn_gOGn7XIk1BB9LXnhQB47KuIIy5AU,2880
|
|
30
|
-
esgpull/cli/login.py,sha256=FZ63SsB4fCDixwf7gMUR697Dk89W1OvpgeadKE4IqEU,2153
|
|
31
30
|
esgpull/cli/plugins.py,sha256=lM8eQIH4H_bIy49rC2flmTstpr9ILDBYXp1YYYMv7qw,13366
|
|
32
31
|
esgpull/cli/remove.py,sha256=9fqE8NJdr1mHypu_N-TBJy_yl3elUBSfzEARxxTkqKg,2662
|
|
33
32
|
esgpull/cli/retry.py,sha256=UVpAjW_N7l6RTJ-T5qXojwcXPzzjT7sDKb_wBdvavrg,1310
|
|
34
|
-
esgpull/cli/search.py,sha256=
|
|
33
|
+
esgpull/cli/search.py,sha256=E64uvlmgFGyNAvtBk9dlFbNeulNfAHUCQta8PHBxSPo,6968
|
|
35
34
|
esgpull/cli/self.py,sha256=psgFcgkDyemquZEpoWp2cyjgampCgDzRc1QBvzqGs24,7941
|
|
36
35
|
esgpull/cli/show.py,sha256=B-h7bKMrwgjnTHio2du8IPOLlKCaan56RQKAtzlQzcw,2822
|
|
37
36
|
esgpull/cli/status.py,sha256=HEyj6QFABblADtYf1PWmSzghKX3fs33x9p5vpSqA514,2521
|
|
38
37
|
esgpull/cli/track.py,sha256=Q9ZvvV5FFGzp6wQZflAd_OFmqhAWgl1JFBad2dCbEF0,3089
|
|
39
|
-
esgpull/cli/update.py,sha256=
|
|
38
|
+
esgpull/cli/update.py,sha256=TiVKXBxye2ZW627zoNyMQtCPhkdiLiTDngRjaVUqGO4,9460
|
|
40
39
|
esgpull/cli/utils.py,sha256=dE5dIH6tWmhItarLDrNldiUuuX5qUjnVpnu4KkE6V1g,7199
|
|
41
40
|
esgpull/migrations/README,sha256=heMzebYwlGhnE8_4CWJ4LS74WoEZjBy-S-mIJRxAEKI,39
|
|
42
41
|
esgpull/migrations/env.py,sha256=am2HhFrlIZNlXCaA5Ye7yKbIJ2MRSO5UFmUwB8l9fyE,2285
|
|
@@ -71,6 +70,7 @@ esgpull/migrations/versions/0.8.0_update_tables.py,sha256=5rr3guWipnnuciFuviUxZU
|
|
|
71
70
|
esgpull/migrations/versions/0.9.0_update_tables.py,sha256=nXfPiyuseD5BXvu59zkeSTzb6EA7txpBudclU-MIyU0,555
|
|
72
71
|
esgpull/migrations/versions/0.9.1_update_tables.py,sha256=ITewU2qgdCwhorsl2d_t7ENt_6KecG3z5xfRr_LwrtY,541
|
|
73
72
|
esgpull/migrations/versions/0.9.2_update_tables.py,sha256=aux--HPA_jUtvk7Zb2ZRXRhnRLT-whqZ0lLWt8Apyac,541
|
|
73
|
+
esgpull/migrations/versions/0.9.3_update_tables.py,sha256=-k1CneOa7OTO6xPZ8uR0ccy-4B8TU3kM12-oYKFEOlA,541
|
|
74
74
|
esgpull/migrations/versions/14c72daea083_query_add_column_updated_at.py,sha256=MKqz0tfwGwRkgP4QDd-cpUmXCVr4tM_wlC2BfxqJ1_w,1031
|
|
75
75
|
esgpull/migrations/versions/c7c8541fa741_query_add_column_added_at.py,sha256=Al_o7fDmoRqc9vBCQgtgrNbSPIOBxdMZ5T-ztakqVeY,1033
|
|
76
76
|
esgpull/migrations/versions/d14f179e553c_file_add_composite_index_dataset_id_.py,sha256=0vJvttugWmgKns4g-K4i3EU6eid2Z_K2e3H6Ktevf7c,860
|
|
@@ -87,8 +87,8 @@ esgpull/models/sql.py,sha256=K8Nre5HKFPjkRzUUW6p6Qk7aG8upbw8C3pnmCFlg7d8,8942
|
|
|
87
87
|
esgpull/models/synda_file.py,sha256=6o5unPhzVJGnbpA2MxcS0r-hrBwocHYVnLrqjSGtmuk,2387
|
|
88
88
|
esgpull/models/tag.py,sha256=5CQDB9rAeCqog63ec9LPFN46HOFNkHPy-maY4gkBQ3E,461
|
|
89
89
|
esgpull/models/utils.py,sha256=exwlIlIKYjhhfUE82w1kU_HeSQOSY97PTvpkhW0udMA,1631
|
|
90
|
-
esgpull-0.9.
|
|
91
|
-
esgpull-0.9.
|
|
92
|
-
esgpull-0.9.
|
|
93
|
-
esgpull-0.9.
|
|
94
|
-
esgpull-0.9.
|
|
90
|
+
esgpull-0.9.3.dist-info/METADATA,sha256=0wlDuo4TCJskBSO9v4FRuoLTAm7531jOPJDQAQJWBP4,3818
|
|
91
|
+
esgpull-0.9.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
92
|
+
esgpull-0.9.3.dist-info/entry_points.txt,sha256=vyh7HvFrCp4iyMrTkDoSF3weaYrlNj2OJe0Fq5q4QB4,45
|
|
93
|
+
esgpull-0.9.3.dist-info/licenses/LICENSE,sha256=lUqGPGWDHHxjkUDuYgjLLY2XQXXn_EHU7fnrQWHGugc,1540
|
|
94
|
+
esgpull-0.9.3.dist-info/RECORD,,
|
esgpull/auth.py
DELETED
|
@@ -1,181 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from enum import Enum, unique
|
|
4
|
-
from pathlib import Path
|
|
5
|
-
from shutil import rmtree
|
|
6
|
-
from typing import Any
|
|
7
|
-
from urllib.parse import (
|
|
8
|
-
ParseResult,
|
|
9
|
-
ParseResultBytes,
|
|
10
|
-
urljoin,
|
|
11
|
-
urlparse,
|
|
12
|
-
urlunparse,
|
|
13
|
-
)
|
|
14
|
-
from xml.etree import ElementTree
|
|
15
|
-
|
|
16
|
-
import httpx
|
|
17
|
-
import tomlkit
|
|
18
|
-
from attrs import Factory, define, field
|
|
19
|
-
from myproxy.client import MyProxyClient
|
|
20
|
-
from OpenSSL import crypto
|
|
21
|
-
|
|
22
|
-
from esgpull.config import Config
|
|
23
|
-
from esgpull.constants import PROVIDERS
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
class Secret:
|
|
27
|
-
def __init__(self, value: str | None = None) -> None:
|
|
28
|
-
self._value = value
|
|
29
|
-
|
|
30
|
-
def get_value(self) -> str | None:
|
|
31
|
-
return self._value
|
|
32
|
-
|
|
33
|
-
def __str__(self) -> str:
|
|
34
|
-
if self.get_value() is None:
|
|
35
|
-
return str(None)
|
|
36
|
-
else:
|
|
37
|
-
return "*" * 10
|
|
38
|
-
|
|
39
|
-
def __repr__(self) -> str:
|
|
40
|
-
return str(self)
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
@define
|
|
44
|
-
class Credentials:
|
|
45
|
-
provider: str | None = None
|
|
46
|
-
user: str | None = None
|
|
47
|
-
password: Secret = field(default=None, converter=Secret)
|
|
48
|
-
|
|
49
|
-
@staticmethod
|
|
50
|
-
def from_config(config: Config) -> Credentials:
|
|
51
|
-
path = config.paths.auth / config.credentials.filename
|
|
52
|
-
return Credentials.from_path(path)
|
|
53
|
-
|
|
54
|
-
@staticmethod
|
|
55
|
-
def from_path(path: Path) -> Credentials:
|
|
56
|
-
if path.is_file():
|
|
57
|
-
with path.open() as fh:
|
|
58
|
-
doc = tomlkit.load(fh)
|
|
59
|
-
return Credentials(**doc)
|
|
60
|
-
else:
|
|
61
|
-
return Credentials()
|
|
62
|
-
|
|
63
|
-
def write(self, path: Path) -> None:
|
|
64
|
-
if path.is_file():
|
|
65
|
-
raise FileExistsError(path)
|
|
66
|
-
with path.open("w") as f:
|
|
67
|
-
cred_dict = dict(
|
|
68
|
-
provider=self.provider,
|
|
69
|
-
user=self.user,
|
|
70
|
-
password=self.password.get_value(),
|
|
71
|
-
)
|
|
72
|
-
tomlkit.dump(cred_dict, f)
|
|
73
|
-
|
|
74
|
-
def parse_openid(self) -> ParseResult | ParseResultBytes | Any:
|
|
75
|
-
if self.provider not in PROVIDERS:
|
|
76
|
-
raise ValueError(f"unknown provider: {self.provider}")
|
|
77
|
-
ns = {"x": "xri://$xrd*($v*2.0)"}
|
|
78
|
-
provider = urlunparse(
|
|
79
|
-
[
|
|
80
|
-
"https",
|
|
81
|
-
self.provider,
|
|
82
|
-
urljoin(PROVIDERS[self.provider], self.user),
|
|
83
|
-
"",
|
|
84
|
-
"",
|
|
85
|
-
"",
|
|
86
|
-
]
|
|
87
|
-
)
|
|
88
|
-
resp = httpx.get(str(provider))
|
|
89
|
-
resp.raise_for_status()
|
|
90
|
-
root = ElementTree.fromstring(resp.text)
|
|
91
|
-
services = root.findall(".//x:Service", namespaces=ns)
|
|
92
|
-
for service in services:
|
|
93
|
-
t = service.find("x:Type", namespaces=ns)
|
|
94
|
-
if t is None:
|
|
95
|
-
continue
|
|
96
|
-
elif t.text == "urn:esg:security:myproxy-service":
|
|
97
|
-
url = service.find("x:URI", namespaces=ns)
|
|
98
|
-
if url is not None:
|
|
99
|
-
return urlparse(url.text)
|
|
100
|
-
raise ValueError("did not found host/port")
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
@unique
|
|
104
|
-
class AuthStatus(Enum):
|
|
105
|
-
Valid = ("valid", "green")
|
|
106
|
-
Expired = ("expired", "orange")
|
|
107
|
-
Missing = ("missing", "red")
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
@define
|
|
111
|
-
class Auth:
|
|
112
|
-
cert_dir: Path
|
|
113
|
-
cert_file: Path
|
|
114
|
-
credentials: Credentials = Factory(Credentials)
|
|
115
|
-
__status: AuthStatus | None = field(init=False, default=None, repr=False)
|
|
116
|
-
|
|
117
|
-
Valid = AuthStatus.Valid
|
|
118
|
-
Expired = AuthStatus.Expired
|
|
119
|
-
Missing = AuthStatus.Missing
|
|
120
|
-
|
|
121
|
-
@staticmethod
|
|
122
|
-
def from_config(
|
|
123
|
-
config: Config, credentials: Credentials = Credentials()
|
|
124
|
-
) -> Auth:
|
|
125
|
-
return Auth.from_path(config.paths.auth, credentials)
|
|
126
|
-
|
|
127
|
-
@staticmethod
|
|
128
|
-
def from_path(
|
|
129
|
-
path: Path, credentials: Credentials = Credentials()
|
|
130
|
-
) -> Auth:
|
|
131
|
-
cert_dir = path / "certificates"
|
|
132
|
-
cert_file = path / "credentials.pem"
|
|
133
|
-
return Auth(cert_dir, cert_file, credentials)
|
|
134
|
-
|
|
135
|
-
@property
|
|
136
|
-
def cert(self) -> str | None:
|
|
137
|
-
if self.status == AuthStatus.Valid:
|
|
138
|
-
return str(self.cert_file)
|
|
139
|
-
else:
|
|
140
|
-
return None
|
|
141
|
-
|
|
142
|
-
@property
|
|
143
|
-
def status(self) -> AuthStatus:
|
|
144
|
-
if self.__status is None:
|
|
145
|
-
self.__status = self._get_status()
|
|
146
|
-
return self.__status
|
|
147
|
-
|
|
148
|
-
def _get_status(self) -> AuthStatus:
|
|
149
|
-
if not self.cert_file.exists():
|
|
150
|
-
return AuthStatus.Missing
|
|
151
|
-
with self.cert_file.open("rb") as f:
|
|
152
|
-
content = f.read()
|
|
153
|
-
filetype = crypto.FILETYPE_PEM
|
|
154
|
-
pem = crypto.load_certificate(filetype, content)
|
|
155
|
-
if pem.has_expired():
|
|
156
|
-
return AuthStatus.Expired
|
|
157
|
-
return AuthStatus.Valid
|
|
158
|
-
|
|
159
|
-
# TODO: review this
|
|
160
|
-
def renew(self) -> None:
|
|
161
|
-
if self.cert_dir.is_dir():
|
|
162
|
-
rmtree(self.cert_dir)
|
|
163
|
-
self.cert_file.unlink(missing_ok=True)
|
|
164
|
-
openid = self.credentials.parse_openid()
|
|
165
|
-
client = MyProxyClient(
|
|
166
|
-
hostname=openid.hostname,
|
|
167
|
-
port=openid.port,
|
|
168
|
-
caCertDir=str(self.cert_dir),
|
|
169
|
-
proxyCertLifetime=12 * 60 * 60,
|
|
170
|
-
)
|
|
171
|
-
creds = client.logon(
|
|
172
|
-
self.credentials.user,
|
|
173
|
-
self.credentials.password.get_value(),
|
|
174
|
-
bootstrap=True,
|
|
175
|
-
updateTrustRoots=True,
|
|
176
|
-
authnGetTrustRootsCall=False,
|
|
177
|
-
)
|
|
178
|
-
with self.cert_file.open("wb") as file:
|
|
179
|
-
for cred in creds:
|
|
180
|
-
file.write(cred)
|
|
181
|
-
self.__status = None
|
esgpull/cli/login.py
DELETED
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
import click
|
|
2
|
-
from click.exceptions import Abort
|
|
3
|
-
|
|
4
|
-
from esgpull.auth import Auth, AuthStatus, Credentials
|
|
5
|
-
from esgpull.cli.decorators import opts
|
|
6
|
-
from esgpull.cli.utils import init_esgpull
|
|
7
|
-
from esgpull.constants import PROVIDERS
|
|
8
|
-
from esgpull.tui import Verbosity
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
@click.command()
|
|
12
|
-
@opts.verbosity
|
|
13
|
-
@opts.force
|
|
14
|
-
def login(verbosity: Verbosity, force: bool):
|
|
15
|
-
"""
|
|
16
|
-
OpenID authentication and certificates renewal
|
|
17
|
-
|
|
18
|
-
The first call to `login` is a prompt asking for provider/username/password.
|
|
19
|
-
|
|
20
|
-
Subsequent calls check whether the login certificates are valid, renewing them if needed.
|
|
21
|
-
Renewal can be forced using the `--force` flag.
|
|
22
|
-
"""
|
|
23
|
-
esg = init_esgpull(verbosity)
|
|
24
|
-
with esg.ui.logging("login", onraise=Abort):
|
|
25
|
-
cred_file = esg.config.paths.auth / esg.config.credentials.filename
|
|
26
|
-
if not cred_file.is_file():
|
|
27
|
-
esg.ui.print("No credentials found.")
|
|
28
|
-
choices = []
|
|
29
|
-
providers = list(PROVIDERS)
|
|
30
|
-
for i, provider in enumerate(providers):
|
|
31
|
-
choices.append(str(i))
|
|
32
|
-
esg.ui.print(f" [{i}] [i green]{provider}[/]")
|
|
33
|
-
provider_idx = esg.ui.choice(
|
|
34
|
-
"Select a provider",
|
|
35
|
-
choices=choices,
|
|
36
|
-
show_choices=False,
|
|
37
|
-
)
|
|
38
|
-
provider = providers[int(provider_idx)]
|
|
39
|
-
user = esg.ui.prompt("User")
|
|
40
|
-
password = esg.ui.prompt("Password", password=True)
|
|
41
|
-
credentials = Credentials(provider, user, password)
|
|
42
|
-
credentials.write(cred_file)
|
|
43
|
-
esg.auth = Auth.from_config(esg.config, credentials)
|
|
44
|
-
renew = force
|
|
45
|
-
status = esg.auth.status
|
|
46
|
-
status_name = status.value[0]
|
|
47
|
-
status_color = status.value[1]
|
|
48
|
-
esg.ui.print(f"Certificates are [{status_color}]{status_name}[/].")
|
|
49
|
-
if esg.auth.status == AuthStatus.Expired:
|
|
50
|
-
renew = renew or esg.ui.ask("Renew?")
|
|
51
|
-
elif esg.auth.status == AuthStatus.Missing:
|
|
52
|
-
renew = True
|
|
53
|
-
if renew:
|
|
54
|
-
with esg.ui.spinner("Renewing certificates"):
|
|
55
|
-
esg.auth.renew()
|
|
56
|
-
esg.ui.print(":+1: Renewed successfully")
|
|
File without changes
|
|
File without changes
|
|
File without changes
|