esgpull 0.9.0__py3-none-any.whl → 0.9.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- esgpull/cli/plugins.py +1 -1
- esgpull/cli/remove.py +9 -3
- esgpull/cli/search.py +21 -8
- esgpull/cli/self.py +1 -1
- esgpull/cli/update.py +38 -38
- esgpull/config.py +1 -1
- esgpull/context.py +38 -9
- esgpull/database.py +12 -4
- esgpull/fs.py +9 -20
- esgpull/migrations/versions/0.9.1_update_tables.py +28 -0
- esgpull/migrations/versions/0.9.2_update_tables.py +28 -0
- esgpull/models/sql.py +8 -0
- esgpull/plugin.py +22 -19
- esgpull/utils.py +0 -17
- {esgpull-0.9.0.dist-info → esgpull-0.9.2.dist-info}/METADATA +18 -3
- {esgpull-0.9.0.dist-info → esgpull-0.9.2.dist-info}/RECORD +19 -17
- {esgpull-0.9.0.dist-info → esgpull-0.9.2.dist-info}/WHEEL +0 -0
- {esgpull-0.9.0.dist-info → esgpull-0.9.2.dist-info}/entry_points.txt +0 -0
- {esgpull-0.9.0.dist-info → esgpull-0.9.2.dist-info}/licenses/LICENSE +0 -0
esgpull/cli/plugins.py
CHANGED
|
@@ -337,7 +337,7 @@ from datetime import datetime
|
|
|
337
337
|
from pathlib import Path
|
|
338
338
|
from logging import Logger
|
|
339
339
|
|
|
340
|
-
from esgpull.models import File, Query
|
|
340
|
+
from esgpull.models import Dataset, File, Query
|
|
341
341
|
from esgpull.plugin import Event, on
|
|
342
342
|
|
|
343
343
|
# Specify version compatibility (optional)
|
esgpull/cli/remove.py
CHANGED
|
@@ -15,11 +15,13 @@ from esgpull.utils import format_size
|
|
|
15
15
|
@args.query_id
|
|
16
16
|
@opts.tag
|
|
17
17
|
@opts.children
|
|
18
|
+
@opts.yes
|
|
18
19
|
@opts.verbosity
|
|
19
20
|
def remove(
|
|
20
21
|
query_id: str | None,
|
|
21
22
|
tag: str | None,
|
|
22
23
|
children: bool,
|
|
24
|
+
yes: bool,
|
|
23
25
|
verbosity: Verbosity,
|
|
24
26
|
) -> None:
|
|
25
27
|
"""
|
|
@@ -49,7 +51,7 @@ def remove(
|
|
|
49
51
|
graph.add(*queries)
|
|
50
52
|
esg.ui.print(graph)
|
|
51
53
|
msg = f"Remove {nb} quer{ies}?"
|
|
52
|
-
if not esg.ui.ask(msg, default=True):
|
|
54
|
+
if not yes and not esg.ui.ask(msg, default=True):
|
|
53
55
|
raise Abort
|
|
54
56
|
for query in queries:
|
|
55
57
|
if query.has_files:
|
|
@@ -59,14 +61,18 @@ def remove(
|
|
|
59
61
|
f":stop_sign: {query.rich_name} is linked"
|
|
60
62
|
f" to {nb} downloaded files ({format_size(size)})."
|
|
61
63
|
)
|
|
62
|
-
if not esg.ui.ask(
|
|
64
|
+
if not yes and not esg.ui.ask(
|
|
65
|
+
"Delete anyway?", default=False
|
|
66
|
+
):
|
|
63
67
|
raise Abort
|
|
64
68
|
if not children and esg.graph.get_children(query.sha):
|
|
65
69
|
esg.ui.print(
|
|
66
70
|
":stop_sign: Some queries block"
|
|
67
71
|
f" removal of {query.rich_name}."
|
|
68
72
|
)
|
|
69
|
-
if esg.ui.ask(
|
|
73
|
+
if not yes and esg.ui.ask(
|
|
74
|
+
"Show blocking queries?", default=False
|
|
75
|
+
):
|
|
70
76
|
esg.ui.print(esg.graph.subgraph(query, children=True))
|
|
71
77
|
raise Exit(1)
|
|
72
78
|
esg.db.delete(*queries)
|
esgpull/cli/search.py
CHANGED
|
@@ -7,6 +7,7 @@ from click.exceptions import Abort, Exit
|
|
|
7
7
|
|
|
8
8
|
from esgpull.cli.decorators import args, groups, opts
|
|
9
9
|
from esgpull.cli.utils import filter_keys, init_esgpull, parse_query, totable
|
|
10
|
+
from esgpull.context import IndexNode
|
|
10
11
|
from esgpull.exceptions import PageIndexError
|
|
11
12
|
from esgpull.graph import Graph
|
|
12
13
|
from esgpull.models import Query
|
|
@@ -135,14 +136,26 @@ def search(
|
|
|
135
136
|
esg.ui.raise_maybe_record(Exit(0))
|
|
136
137
|
if facets_hints:
|
|
137
138
|
not_distrib_query = query << Query(options=dict(distrib=False))
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
139
|
+
index = IndexNode(esg.config.api.index_node)
|
|
140
|
+
if index.is_bridge():
|
|
141
|
+
first_file_result = esg.context.search_as_queries(
|
|
142
|
+
not_distrib_query,
|
|
143
|
+
file=True,
|
|
144
|
+
max_hits=1,
|
|
145
|
+
date_from=date_from,
|
|
146
|
+
date_to=date_to,
|
|
147
|
+
)
|
|
148
|
+
first_file = first_file_result[0].selection.asdict()
|
|
149
|
+
esg.ui.print(list(first_file), json=True)
|
|
150
|
+
else:
|
|
151
|
+
facet_counts = esg.context.hints(
|
|
152
|
+
not_distrib_query,
|
|
153
|
+
file=file,
|
|
154
|
+
facets=["*"],
|
|
155
|
+
date_from=date_from,
|
|
156
|
+
date_to=date_to,
|
|
157
|
+
)
|
|
158
|
+
esg.ui.print(list(facet_counts[0]), json=True)
|
|
146
159
|
esg.ui.raise_maybe_record(Exit(0))
|
|
147
160
|
if hints is not None:
|
|
148
161
|
facet_counts = esg.context.hints(
|
esgpull/cli/self.py
CHANGED
|
@@ -199,7 +199,7 @@ def delete():
|
|
|
199
199
|
TempUI.print(f"Deleting {path} from config...")
|
|
200
200
|
TempUI.print("To remove all files from this install, run:\n")
|
|
201
201
|
config = Config.load(path=path)
|
|
202
|
-
for p in config.paths:
|
|
202
|
+
for p in config.paths.values():
|
|
203
203
|
if not p.is_relative_to(path):
|
|
204
204
|
TempUI.print(f"$ rm -rf {p}")
|
|
205
205
|
TempUI.print(f"$ rm -rf {path}")
|
esgpull/cli/update.py
CHANGED
|
@@ -11,7 +11,7 @@ from esgpull.cli.utils import get_queries, init_esgpull, valid_name_tag
|
|
|
11
11
|
from esgpull.context import HintsDict, ResultSearch
|
|
12
12
|
from esgpull.exceptions import UnsetOptionsError
|
|
13
13
|
from esgpull.models import Dataset, File, FileStatus, Query
|
|
14
|
-
from esgpull.tui import Verbosity
|
|
14
|
+
from esgpull.tui import Verbosity
|
|
15
15
|
from esgpull.utils import format_size
|
|
16
16
|
|
|
17
17
|
|
|
@@ -177,26 +177,37 @@ def update(
|
|
|
177
177
|
for qf, qf_files in zip(qfs, files):
|
|
178
178
|
qf.files = qf_files
|
|
179
179
|
for qf in qfs:
|
|
180
|
-
new_files = [f for f in qf.files if f not in esg.db]
|
|
181
|
-
new_datasets = [d for d in qf.datasets if d not in esg.db]
|
|
182
|
-
nb_datasets = len(new_datasets)
|
|
183
|
-
nb_files = len(new_files)
|
|
184
180
|
if not qf.query.tracked:
|
|
185
181
|
esg.db.add(qf.query)
|
|
186
182
|
continue
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
183
|
+
with esg.db.commit_context():
|
|
184
|
+
unregistered_datasets = [
|
|
185
|
+
f for f in qf.datasets if f not in esg.db
|
|
186
|
+
]
|
|
187
|
+
if len(unregistered_datasets) > 0:
|
|
188
|
+
esg.ui.print(
|
|
189
|
+
f"Adding {len(unregistered_datasets)} new datasets to database."
|
|
190
|
+
)
|
|
191
|
+
esg.db.session.add_all(unregistered_datasets)
|
|
192
|
+
files_from_db = [
|
|
193
|
+
esg.db.get(File, f.sha) for f in qf.files if f in esg.db
|
|
194
|
+
]
|
|
195
|
+
registered_files = [f for f in files_from_db if f is not None]
|
|
196
|
+
unregistered_files = [f for f in qf.files if f not in esg.db]
|
|
197
|
+
if len(unregistered_files) > 0:
|
|
198
|
+
esg.ui.print(
|
|
199
|
+
f"Adding {len(unregistered_files)} new files to database."
|
|
200
|
+
)
|
|
201
|
+
esg.db.session.add_all(unregistered_files)
|
|
202
|
+
files = registered_files + unregistered_files
|
|
203
|
+
not_done_files = [
|
|
204
|
+
file for file in files if file.status != FileStatus.Done
|
|
205
|
+
]
|
|
206
|
+
download_size = sum(file.size for file in not_done_files)
|
|
195
207
|
msg = (
|
|
196
|
-
f"\n{qf.query.rich_name}: {
|
|
197
|
-
f" files
|
|
198
|
-
f"
|
|
199
|
-
f"\nUpdate the database{queue_msg}?"
|
|
208
|
+
f"\n{qf.query.rich_name}: {len(not_done_files)} "
|
|
209
|
+
f" files ({format_size(download_size)}) to download."
|
|
210
|
+
f"\nLink to query and send to download queue?"
|
|
200
211
|
)
|
|
201
212
|
if yes:
|
|
202
213
|
choice = "y"
|
|
@@ -212,30 +223,19 @@ def update(
|
|
|
212
223
|
if choice == "y":
|
|
213
224
|
legacy = esg.legacy_query
|
|
214
225
|
has_legacy = legacy.state.persistent
|
|
226
|
+
applied_changes = False
|
|
215
227
|
with esg.db.commit_context():
|
|
216
|
-
for dataset in esg.ui.track(
|
|
217
|
-
new_datasets,
|
|
218
|
-
description=f"{qf.query.rich_name} (datasets)",
|
|
219
|
-
):
|
|
220
|
-
esg.db.session.add(dataset)
|
|
221
228
|
for file in esg.ui.track(
|
|
222
|
-
|
|
223
|
-
description=f"{qf.query.rich_name}
|
|
229
|
+
files,
|
|
230
|
+
description=f"{qf.query.rich_name}",
|
|
224
231
|
):
|
|
225
|
-
|
|
226
|
-
if file_db is None:
|
|
227
|
-
if esg.db.has_file_id(file):
|
|
228
|
-
logger.error(
|
|
229
|
-
"File id already exists in database, "
|
|
230
|
-
"there might be an error with its checksum"
|
|
231
|
-
f"\n{file}"
|
|
232
|
-
)
|
|
233
|
-
continue
|
|
232
|
+
if file.status != FileStatus.Done:
|
|
234
233
|
file.status = FileStatus.Queued
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
234
|
+
if has_legacy and legacy in file.queries:
|
|
235
|
+
_ = esg.db.unlink(query=legacy, file=file)
|
|
236
|
+
changed = esg.db.link(query=qf.query, file=file)
|
|
237
|
+
applied_changes = applied_changes or changed
|
|
238
|
+
if applied_changes:
|
|
239
|
+
qf.query.updated_at = datetime.now(timezone.utc)
|
|
240
240
|
esg.db.session.add(qf.query)
|
|
241
241
|
esg.ui.raise_maybe_record(Exit(0))
|
esgpull/config.py
CHANGED
esgpull/context.py
CHANGED
|
@@ -7,6 +7,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Sequence
|
|
|
7
7
|
from dataclasses import dataclass, field
|
|
8
8
|
from datetime import datetime
|
|
9
9
|
from typing import Any, TypeAlias, TypeVar
|
|
10
|
+
from urllib.parse import urlparse
|
|
10
11
|
|
|
11
12
|
if sys.version_info < (3, 11):
|
|
12
13
|
from exceptiongroup import BaseExceptionGroup
|
|
@@ -18,7 +19,7 @@ from esgpull.config import Config
|
|
|
18
19
|
from esgpull.exceptions import SolrUnstableQueryError
|
|
19
20
|
from esgpull.models import DatasetRecord, File, Query
|
|
20
21
|
from esgpull.tui import logger
|
|
21
|
-
from esgpull.utils import format_date_iso,
|
|
22
|
+
from esgpull.utils import format_date_iso, sync
|
|
22
23
|
|
|
23
24
|
# workaround for notebooks with running event loop
|
|
24
25
|
if asyncio.get_event_loop().is_running():
|
|
@@ -39,6 +40,29 @@ DangerousFacets = {
|
|
|
39
40
|
}
|
|
40
41
|
|
|
41
42
|
|
|
43
|
+
@dataclass
|
|
44
|
+
class IndexNode:
|
|
45
|
+
value: str
|
|
46
|
+
|
|
47
|
+
def is_bridge(self) -> bool:
|
|
48
|
+
return "esgf-1-5-bridge" in self.value
|
|
49
|
+
|
|
50
|
+
@property
|
|
51
|
+
def url(self) -> str:
|
|
52
|
+
parsed = urlparse(self.value)
|
|
53
|
+
result: str
|
|
54
|
+
match (parsed.scheme, parsed.netloc, parsed.path, self.is_bridge()):
|
|
55
|
+
case ("", "", path, True):
|
|
56
|
+
result = "https://" + parsed.path
|
|
57
|
+
case ("", "", path, False):
|
|
58
|
+
result = "https://" + parsed.path + "/esg-search/search"
|
|
59
|
+
case _:
|
|
60
|
+
result = self.value
|
|
61
|
+
if "." not in result:
|
|
62
|
+
raise ValueError(self.value)
|
|
63
|
+
return result
|
|
64
|
+
|
|
65
|
+
|
|
42
66
|
@dataclass
|
|
43
67
|
class Result:
|
|
44
68
|
query: Query
|
|
@@ -70,12 +94,12 @@ class Result:
|
|
|
70
94
|
"format": "application/solr+json",
|
|
71
95
|
# "from": self.since,
|
|
72
96
|
}
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
97
|
+
index = IndexNode(value=index_url or index_node)
|
|
98
|
+
if not index.is_bridge():
|
|
99
|
+
if fields_param is not None:
|
|
100
|
+
params["fields"] = ",".join(fields_param)
|
|
101
|
+
else:
|
|
102
|
+
params["fields"] = "instance_id"
|
|
79
103
|
if date_from is not None:
|
|
80
104
|
params["from"] = format_date_iso(date_from)
|
|
81
105
|
if date_to is not None:
|
|
@@ -101,15 +125,20 @@ class Result:
|
|
|
101
125
|
else:
|
|
102
126
|
if len(values) > 1:
|
|
103
127
|
value_term = f"({value_term})"
|
|
104
|
-
|
|
128
|
+
if name.startswith("!"):
|
|
129
|
+
solr_terms.append(f"(NOT {name[1:]}:{value_term})")
|
|
130
|
+
else:
|
|
131
|
+
solr_terms.append(f"{name}:{value_term}")
|
|
105
132
|
if solr_terms:
|
|
106
133
|
params["query"] = " AND ".join(solr_terms)
|
|
107
134
|
for name, option in self.query.options.items(use_default=True):
|
|
108
135
|
if option.is_bool():
|
|
109
136
|
params[name] = option.name
|
|
137
|
+
if index.is_bridge():
|
|
138
|
+
_ = params.pop("retracted", None) # not supported in bridge API
|
|
110
139
|
if params.get("distrib") == "true" and facets_star:
|
|
111
140
|
raise SolrUnstableQueryError(pretty_repr(self.query))
|
|
112
|
-
self.request = Request("GET",
|
|
141
|
+
self.request = Request("GET", index.url, params=params)
|
|
113
142
|
|
|
114
143
|
def to(self, subtype: type[RT]) -> RT:
|
|
115
144
|
result: RT = subtype(self.query, self.file)
|
esgpull/database.py
CHANGED
|
@@ -147,11 +147,19 @@ class Database:
|
|
|
147
147
|
for item in items:
|
|
148
148
|
make_transient(item)
|
|
149
149
|
|
|
150
|
-
def link(self, query: Query, file: File):
|
|
151
|
-
self.session.
|
|
150
|
+
def link(self, query: Query, file: File) -> bool:
|
|
151
|
+
if not self.session.scalar(sql.query_file.is_linked(query, file)):
|
|
152
|
+
self.session.execute(sql.query_file.link(query, file))
|
|
153
|
+
return True
|
|
154
|
+
else:
|
|
155
|
+
return False
|
|
152
156
|
|
|
153
|
-
def unlink(self, query: Query, file: File):
|
|
154
|
-
self.session.
|
|
157
|
+
def unlink(self, query: Query, file: File) -> bool:
|
|
158
|
+
if self.session.scalar(sql.query_file.is_linked(query, file)):
|
|
159
|
+
self.session.execute(sql.query_file.unlink(query, file))
|
|
160
|
+
return True
|
|
161
|
+
else:
|
|
162
|
+
return False
|
|
155
163
|
|
|
156
164
|
def __contains__(self, item: Base | BaseNoSHA) -> bool:
|
|
157
165
|
mapper = inspect(item.__class__)
|
esgpull/fs.py
CHANGED
|
@@ -10,7 +10,7 @@ from shutil import copyfile
|
|
|
10
10
|
import aiofiles
|
|
11
11
|
from aiofiles.threadpool.binary import AsyncBufferedIOBase
|
|
12
12
|
|
|
13
|
-
from esgpull.config import Config
|
|
13
|
+
from esgpull.config import Config, Paths
|
|
14
14
|
from esgpull.models import File
|
|
15
15
|
from esgpull.result import Err, Ok, Result
|
|
16
16
|
from esgpull.tui import logger
|
|
@@ -63,45 +63,34 @@ class Digest:
|
|
|
63
63
|
|
|
64
64
|
@dataclass
|
|
65
65
|
class Filesystem:
|
|
66
|
-
|
|
67
|
-
data: Path
|
|
68
|
-
db: Path
|
|
69
|
-
log: Path
|
|
70
|
-
tmp: Path
|
|
66
|
+
paths: Paths
|
|
71
67
|
disable_checksum: bool = False
|
|
72
68
|
install: InitVar[bool] = True
|
|
73
69
|
|
|
74
70
|
@staticmethod
|
|
75
71
|
def from_config(config: Config, install: bool = False) -> Filesystem:
|
|
76
72
|
return Filesystem(
|
|
77
|
-
|
|
78
|
-
data=config.paths.data,
|
|
79
|
-
db=config.paths.db,
|
|
80
|
-
log=config.paths.log,
|
|
81
|
-
tmp=config.paths.tmp,
|
|
73
|
+
paths=config.paths,
|
|
82
74
|
disable_checksum=config.download.disable_checksum,
|
|
83
75
|
install=install,
|
|
84
76
|
)
|
|
85
77
|
|
|
86
78
|
def __post_init__(self, install: bool = True) -> None:
|
|
87
79
|
if install:
|
|
88
|
-
self.
|
|
89
|
-
|
|
90
|
-
self.db.mkdir(parents=True, exist_ok=True)
|
|
91
|
-
self.log.mkdir(parents=True, exist_ok=True)
|
|
92
|
-
self.tmp.mkdir(parents=True, exist_ok=True)
|
|
80
|
+
for path in self.paths.values():
|
|
81
|
+
path.mkdir(parents=True, exist_ok=True)
|
|
93
82
|
|
|
94
83
|
def __getitem__(self, file: File) -> FilePath:
|
|
95
84
|
if not isinstance(file, File):
|
|
96
85
|
raise TypeError(file)
|
|
97
86
|
return FilePath(
|
|
98
|
-
drs=self.data / file.local_path / file.filename,
|
|
99
|
-
tmp=self.tmp / f"{file.sha}.part",
|
|
87
|
+
drs=self.paths.data / file.local_path / file.filename,
|
|
88
|
+
tmp=self.paths.tmp / f"{file.sha}.part",
|
|
100
89
|
)
|
|
101
90
|
|
|
102
91
|
def glob_netcdf(self) -> Iterator[Path]:
|
|
103
|
-
for path in self.data.glob("**/*.nc"):
|
|
104
|
-
yield path.relative_to(self.data)
|
|
92
|
+
for path in self.paths.data.glob("**/*.nc"):
|
|
93
|
+
yield path.relative_to(self.paths.data)
|
|
105
94
|
|
|
106
95
|
def open(self, file: File) -> FileObject:
|
|
107
96
|
return FileObject(self[file])
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""update tables
|
|
2
|
+
|
|
3
|
+
Revision ID: 0.9.1
|
|
4
|
+
Revises: 0.9.0
|
|
5
|
+
Create Date: 2025-08-08 10:38:14.204594
|
|
6
|
+
|
|
7
|
+
"""
|
|
8
|
+
from alembic import op
|
|
9
|
+
import sqlalchemy as sa
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
# revision identifiers, used by Alembic.
|
|
13
|
+
revision = '0.9.1'
|
|
14
|
+
down_revision = '0.9.0'
|
|
15
|
+
branch_labels = None
|
|
16
|
+
depends_on = None
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def upgrade() -> None:
|
|
20
|
+
# ### commands auto generated by Alembic - please adjust! ###
|
|
21
|
+
pass
|
|
22
|
+
# ### end Alembic commands ###
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def downgrade() -> None:
|
|
26
|
+
# ### commands auto generated by Alembic - please adjust! ###
|
|
27
|
+
pass
|
|
28
|
+
# ### end Alembic commands ###
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""update tables
|
|
2
|
+
|
|
3
|
+
Revision ID: 0.9.2
|
|
4
|
+
Revises: 0.9.1
|
|
5
|
+
Create Date: 2025-09-04 16:56:29.263007
|
|
6
|
+
|
|
7
|
+
"""
|
|
8
|
+
from alembic import op
|
|
9
|
+
import sqlalchemy as sa
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
# revision identifiers, used by Alembic.
|
|
13
|
+
revision = '0.9.2'
|
|
14
|
+
down_revision = '0.9.1'
|
|
15
|
+
branch_labels = None
|
|
16
|
+
depends_on = None
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def upgrade() -> None:
|
|
20
|
+
# ### commands auto generated by Alembic - please adjust! ###
|
|
21
|
+
pass
|
|
22
|
+
# ### end Alembic commands ###
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def downgrade() -> None:
|
|
26
|
+
# ### commands auto generated by Alembic - please adjust! ###
|
|
27
|
+
pass
|
|
28
|
+
# ### end Alembic commands ###
|
esgpull/models/sql.py
CHANGED
|
@@ -301,3 +301,11 @@ class query_file:
|
|
|
301
301
|
.where(query_file_proxy.c.query_sha == query.sha)
|
|
302
302
|
.where(query_file_proxy.c.file_sha == file.sha)
|
|
303
303
|
)
|
|
304
|
+
|
|
305
|
+
@staticmethod
|
|
306
|
+
def is_linked(query: Query, file: File) -> sa.Select[tuple[bool]]:
|
|
307
|
+
return sa.select(
|
|
308
|
+
sa.exists()
|
|
309
|
+
.where(query_file_proxy.c.query_sha == query.sha)
|
|
310
|
+
.where(query_file_proxy.c.file_sha == file.sha)
|
|
311
|
+
)
|
esgpull/plugin.py
CHANGED
|
@@ -151,7 +151,11 @@ class PluginConfig:
|
|
|
151
151
|
enabled: set[str] = field(default_factory=set)
|
|
152
152
|
disabled: set[str] = field(default_factory=set)
|
|
153
153
|
plugins: dict[str, dict[str, Any]] = field(default_factory=dict)
|
|
154
|
-
_raw:
|
|
154
|
+
_raw: tomlkit.TOMLDocument = field(default_factory=tomlkit.TOMLDocument)
|
|
155
|
+
|
|
156
|
+
def __post_init__(self):
|
|
157
|
+
if "plugins" not in self._raw:
|
|
158
|
+
self._raw["plugins"] = {}
|
|
155
159
|
|
|
156
160
|
|
|
157
161
|
class PluginManager:
|
|
@@ -199,12 +203,13 @@ class PluginManager:
|
|
|
199
203
|
|
|
200
204
|
try:
|
|
201
205
|
with open(self.config_path, "r") as f:
|
|
202
|
-
raw = tomlkit.
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
206
|
+
raw = tomlkit.load(f)
|
|
207
|
+
unwrap = raw.unwrap()
|
|
208
|
+
self.config.enabled = set(unwrap.get("enabled", []))
|
|
209
|
+
self.config.disabled = set(unwrap.get("disabled", []))
|
|
210
|
+
self.config.plugins = unwrap.get("plugins", {})
|
|
211
|
+
# Store the raw plugin configuration to preserve what's on disk
|
|
212
|
+
self.config._raw = raw
|
|
208
213
|
except Exception as e:
|
|
209
214
|
logger.error(f"Failed to load plugin config: {e}")
|
|
210
215
|
|
|
@@ -219,18 +224,16 @@ class PluginManager:
|
|
|
219
224
|
return
|
|
220
225
|
|
|
221
226
|
try:
|
|
222
|
-
doc =
|
|
227
|
+
doc = self.config._raw
|
|
223
228
|
doc["enabled"] = list(self.config.enabled)
|
|
224
229
|
doc["disabled"] = list(self.config.disabled)
|
|
225
230
|
|
|
226
231
|
# For plugins section, handle differently based on generate_full_config flag
|
|
227
232
|
if generate_full_config:
|
|
228
233
|
doc["plugins"] = self.config.plugins
|
|
229
|
-
else:
|
|
230
|
-
doc["plugins"] = self.config._raw
|
|
231
234
|
|
|
232
235
|
with open(self.config_path, "w") as f:
|
|
233
|
-
|
|
236
|
+
tomlkit.dump(doc, f)
|
|
234
237
|
except Exception as e:
|
|
235
238
|
logger.error(f"Failed to save plugin config: {e}")
|
|
236
239
|
|
|
@@ -464,8 +467,8 @@ class PluginManager:
|
|
|
464
467
|
# Make sure plugin exists in both plugins and _raw dicts
|
|
465
468
|
if plugin_name not in self.config.plugins:
|
|
466
469
|
self.config.plugins[plugin_name] = {}
|
|
467
|
-
if plugin_name not in self.config._raw:
|
|
468
|
-
self.config._raw[plugin_name] = {}
|
|
470
|
+
if plugin_name not in self.config._raw["plugins"]:
|
|
471
|
+
self.config._raw["plugins"][plugin_name] = {}
|
|
469
472
|
|
|
470
473
|
# Update the value in both places
|
|
471
474
|
if key in self.config.plugins[plugin_name]:
|
|
@@ -473,7 +476,7 @@ class PluginManager:
|
|
|
473
476
|
new_value = cast_value(old_value, value, key)
|
|
474
477
|
self.config.plugins[plugin_name][key] = new_value
|
|
475
478
|
# Also update in _raw to keep in sync
|
|
476
|
-
self.config._raw[plugin_name][key] = new_value
|
|
479
|
+
self.config._raw["plugins"][plugin_name][key] = new_value
|
|
477
480
|
else:
|
|
478
481
|
raise KeyError(key, self.config.plugins[plugin_name])
|
|
479
482
|
|
|
@@ -497,8 +500,8 @@ class PluginManager:
|
|
|
497
500
|
# Make sure the plugin exists in both configs
|
|
498
501
|
if plugin_name not in self.config.plugins:
|
|
499
502
|
self.config.plugins[plugin_name] = {}
|
|
500
|
-
if plugin_name not in self.config._raw:
|
|
501
|
-
self.config._raw[plugin_name] = {}
|
|
503
|
+
if plugin_name not in self.config._raw["plugins"]:
|
|
504
|
+
self.config._raw["plugins"][plugin_name] = {}
|
|
502
505
|
|
|
503
506
|
# Remove the key from both configs
|
|
504
507
|
if key in self.config.plugins[plugin_name]:
|
|
@@ -508,10 +511,10 @@ class PluginManager:
|
|
|
508
511
|
|
|
509
512
|
# Also remove from _raw if it exists
|
|
510
513
|
if (
|
|
511
|
-
plugin_name in self.config._raw
|
|
512
|
-
and key in self.config._raw[plugin_name]
|
|
514
|
+
plugin_name in self.config._raw["plugins"]
|
|
515
|
+
and key in self.config._raw["plugins"][plugin_name]
|
|
513
516
|
):
|
|
514
|
-
self.config._raw[plugin_name].pop(key)
|
|
517
|
+
self.config._raw["plugins"][plugin_name].pop(key)
|
|
515
518
|
|
|
516
519
|
self.write_config()
|
|
517
520
|
|
esgpull/utils.py
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import asyncio
|
|
2
2
|
import datetime
|
|
3
3
|
from typing import Callable, Coroutine, TypeVar
|
|
4
|
-
from urllib.parse import urlparse
|
|
5
4
|
|
|
6
5
|
from rich.filesize import _to_str
|
|
7
6
|
|
|
@@ -52,19 +51,3 @@ def format_date_iso(
|
|
|
52
51
|
date: str | datetime.datetime, fmt: str = "%Y-%m-%d"
|
|
53
52
|
) -> str:
|
|
54
53
|
return parse_date(date, fmt).replace(microsecond=0).isoformat() + "Z"
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
def url2index(url: str) -> str:
|
|
58
|
-
parsed = urlparse(url)
|
|
59
|
-
if parsed.netloc == "":
|
|
60
|
-
return parsed.path
|
|
61
|
-
else:
|
|
62
|
-
return parsed.netloc
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
def index2url(index: str) -> str:
|
|
66
|
-
url = "https://" + url2index(index)
|
|
67
|
-
if "esgf-1-5-bridge" in index:
|
|
68
|
-
return url
|
|
69
|
-
else:
|
|
70
|
-
return url + "/esg-search/search"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: esgpull
|
|
3
|
-
Version: 0.9.
|
|
3
|
+
Version: 0.9.2
|
|
4
4
|
Summary: ESGF data discovery, download, replication tool
|
|
5
5
|
Project-URL: Repository, https://github.com/ESGF/esgf-download
|
|
6
6
|
Project-URL: Documentation, https://esgf.github.io/esgf-download/
|
|
@@ -63,14 +63,29 @@ for dataset in datasets:
|
|
|
63
63
|
|
|
64
64
|
## Installation
|
|
65
65
|
|
|
66
|
-
|
|
66
|
+
`esgpull` is distributed via PyPI:
|
|
67
67
|
|
|
68
68
|
```shell
|
|
69
69
|
pip install esgpull
|
|
70
|
+
esgpull --help
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
For isolated installation, [`uv`](https://github.com/astral-sh/uv) or
|
|
74
|
+
[`pipx`](https://github.com/pypa/pipx) are recommended:
|
|
75
|
+
|
|
76
|
+
```shell
|
|
77
|
+
# with uv
|
|
78
|
+
uv tool install esgpull
|
|
79
|
+
esgpull --help
|
|
80
|
+
|
|
81
|
+
# alternatively, uvx enables running without explicit installation (comes with uv)
|
|
82
|
+
uvx esgpull --help
|
|
70
83
|
```
|
|
71
84
|
|
|
72
85
|
```shell
|
|
73
|
-
|
|
86
|
+
# with pipx
|
|
87
|
+
pipx install esgpull
|
|
88
|
+
esgpull --help
|
|
74
89
|
```
|
|
75
90
|
|
|
76
91
|
## Usage
|
|
@@ -1,21 +1,21 @@
|
|
|
1
1
|
esgpull/__init__.py,sha256=XItFDIMNmFUNNcKtUgXdfmGwUIWt4AAv0a4mZkfj5P8,240
|
|
2
2
|
esgpull/auth.py,sha256=QZ-l1ySLMP0fvuwYHRLv9FZYp1gqfju_eGaTMDByUxw,5205
|
|
3
|
-
esgpull/config.py,sha256=
|
|
3
|
+
esgpull/config.py,sha256=9atYqxc3PiJKY1hfoEWZxf5Ba6U9SzMUs4QvcWWGFy0,13423
|
|
4
4
|
esgpull/constants.py,sha256=WjG7xzMZNckOv5GhRehBtI7hoSwwLZvwkvEq5RG-dv4,1189
|
|
5
|
-
esgpull/context.py,sha256=
|
|
6
|
-
esgpull/database.py,sha256=
|
|
5
|
+
esgpull/context.py,sha256=iyHAJ9OXU5_0SYcXNlmJI_M6x3vphJjnnAcsfB4MxQ8,24383
|
|
6
|
+
esgpull/database.py,sha256=1wGeNbJp0gOLo5Q1N53JfiwVbZ8nfvZVkKlvEaJwOpU,6716
|
|
7
7
|
esgpull/download.py,sha256=aR2c_SOuZtgX7tI2a9_N4Mn86ABq1k7Mxq_BdojFrP4,5600
|
|
8
8
|
esgpull/esgpull.py,sha256=-OwaWvwmkur_mdVNKKxdxlASeK93tmOceV__jGRMwqo,19666
|
|
9
9
|
esgpull/exceptions.py,sha256=wgLyhyIITdusNucPjnnURJX1Jxv1VVIr9PzJV_77qhg,3275
|
|
10
|
-
esgpull/fs.py,sha256=
|
|
10
|
+
esgpull/fs.py,sha256=sc7Af2E3yh3V9KVuSPSXFBuFtlQ3L99UZmS1ZJuiBeM,7280
|
|
11
11
|
esgpull/graph.py,sha256=Yl2VuF8PNn0R5xRyEK58Q1Xlx8B1PfhbTwt1JftFDro,15929
|
|
12
12
|
esgpull/install_config.py,sha256=hzYpcHMtPMOK9fYcvVH-Hn_8zYsbs3yXlYgMumXo1zE,5598
|
|
13
|
-
esgpull/plugin.py,sha256=
|
|
13
|
+
esgpull/plugin.py,sha256=do6b4duPyAhNnYXWmFiVAlgNKj2FJcEjAMPqT8JqxtA,18871
|
|
14
14
|
esgpull/processor.py,sha256=WLf4NFO_dp27E0GhZAiekiRD-gT6uyFJJnFZKbFV5vU,5484
|
|
15
15
|
esgpull/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
16
16
|
esgpull/result.py,sha256=f64l9gPFpFWgctmHVYrNWJvuXXB1sxpxXzJEIssLXxc,1020
|
|
17
17
|
esgpull/tui.py,sha256=jpfDK_g_taNPEu0FeJ4MM1vtNJfOJY5x3dkUxRkoPEA,11210
|
|
18
|
-
esgpull/utils.py,sha256=
|
|
18
|
+
esgpull/utils.py,sha256=pLMQfY0p2oNIFyCZhHBD74BOAJtmt9QV6dTJ79VsfF8,1213
|
|
19
19
|
esgpull/version.py,sha256=IHT4mKrIr8eV-C3HtmIVD85iGVH25n2ohoff31kaJ1A,93
|
|
20
20
|
esgpull/cli/__init__.py,sha256=Q-U7opBTHxF5yUyyv15Cj-4Y9MaDMJ4i5FxNL-EmHfs,1683
|
|
21
21
|
esgpull/cli/add.py,sha256=h8xFy5ORzY9O4SPPo4VCtTybeJJ5GcA18w4TZeL0DqU,3507
|
|
@@ -28,15 +28,15 @@ esgpull/cli/facet.py,sha256=V1u-DxNkhswwSt0qpXvuHrCI_tE8jAJGEe6_fMhYbaM,584
|
|
|
28
28
|
esgpull/cli/get.py,sha256=2WXL01Ri0P_2Rf1xrp9bnsrrxir6whxkAC0SnohjFpg,678
|
|
29
29
|
esgpull/cli/install.py,sha256=fd8nKwIFvOivgn_gOGn7XIk1BB9LXnhQB47KuIIy5AU,2880
|
|
30
30
|
esgpull/cli/login.py,sha256=FZ63SsB4fCDixwf7gMUR697Dk89W1OvpgeadKE4IqEU,2153
|
|
31
|
-
esgpull/cli/plugins.py,sha256=
|
|
32
|
-
esgpull/cli/remove.py,sha256=
|
|
31
|
+
esgpull/cli/plugins.py,sha256=lM8eQIH4H_bIy49rC2flmTstpr9ILDBYXp1YYYMv7qw,13366
|
|
32
|
+
esgpull/cli/remove.py,sha256=9fqE8NJdr1mHypu_N-TBJy_yl3elUBSfzEARxxTkqKg,2662
|
|
33
33
|
esgpull/cli/retry.py,sha256=UVpAjW_N7l6RTJ-T5qXojwcXPzzjT7sDKb_wBdvavrg,1310
|
|
34
|
-
esgpull/cli/search.py,sha256=
|
|
35
|
-
esgpull/cli/self.py,sha256=
|
|
34
|
+
esgpull/cli/search.py,sha256=J41dRi0mS-XsJyWsDzXsv04yWHPwhVG2VOEcdPnaak8,6940
|
|
35
|
+
esgpull/cli/self.py,sha256=psgFcgkDyemquZEpoWp2cyjgampCgDzRc1QBvzqGs24,7941
|
|
36
36
|
esgpull/cli/show.py,sha256=B-h7bKMrwgjnTHio2du8IPOLlKCaan56RQKAtzlQzcw,2822
|
|
37
37
|
esgpull/cli/status.py,sha256=HEyj6QFABblADtYf1PWmSzghKX3fs33x9p5vpSqA514,2521
|
|
38
38
|
esgpull/cli/track.py,sha256=Q9ZvvV5FFGzp6wQZflAd_OFmqhAWgl1JFBad2dCbEF0,3089
|
|
39
|
-
esgpull/cli/update.py,sha256=
|
|
39
|
+
esgpull/cli/update.py,sha256=3JBf1HqglazUYZ63uHtm5y3ZAuS2XLGcwrsVYBkrmb0,9432
|
|
40
40
|
esgpull/cli/utils.py,sha256=dE5dIH6tWmhItarLDrNldiUuuX5qUjnVpnu4KkE6V1g,7199
|
|
41
41
|
esgpull/migrations/README,sha256=heMzebYwlGhnE8_4CWJ4LS74WoEZjBy-S-mIJRxAEKI,39
|
|
42
42
|
esgpull/migrations/env.py,sha256=am2HhFrlIZNlXCaA5Ye7yKbIJ2MRSO5UFmUwB8l9fyE,2285
|
|
@@ -69,6 +69,8 @@ esgpull/migrations/versions/0.7.2_update_tables.py,sha256=cBWEhfYQlhGpE55NOY9eD2
|
|
|
69
69
|
esgpull/migrations/versions/0.7.3_update_tables.py,sha256=4pNkM7VaGQqkLuIoLKuM0ftefLjDBVFRaV8Nkq5OL-Y,541
|
|
70
70
|
esgpull/migrations/versions/0.8.0_update_tables.py,sha256=5rr3guWipnnuciFuviUxZUasdPw6x06WZCmY5nNCn5k,555
|
|
71
71
|
esgpull/migrations/versions/0.9.0_update_tables.py,sha256=nXfPiyuseD5BXvu59zkeSTzb6EA7txpBudclU-MIyU0,555
|
|
72
|
+
esgpull/migrations/versions/0.9.1_update_tables.py,sha256=ITewU2qgdCwhorsl2d_t7ENt_6KecG3z5xfRr_LwrtY,541
|
|
73
|
+
esgpull/migrations/versions/0.9.2_update_tables.py,sha256=aux--HPA_jUtvk7Zb2ZRXRhnRLT-whqZ0lLWt8Apyac,541
|
|
72
74
|
esgpull/migrations/versions/14c72daea083_query_add_column_updated_at.py,sha256=MKqz0tfwGwRkgP4QDd-cpUmXCVr4tM_wlC2BfxqJ1_w,1031
|
|
73
75
|
esgpull/migrations/versions/c7c8541fa741_query_add_column_added_at.py,sha256=Al_o7fDmoRqc9vBCQgtgrNbSPIOBxdMZ5T-ztakqVeY,1033
|
|
74
76
|
esgpull/migrations/versions/d14f179e553c_file_add_composite_index_dataset_id_.py,sha256=0vJvttugWmgKns4g-K4i3EU6eid2Z_K2e3H6Ktevf7c,860
|
|
@@ -81,12 +83,12 @@ esgpull/models/file.py,sha256=-8PPYtq7BWp-O_QtCDbkLdhTGTPhI1F1nodQacMnYGA,1517
|
|
|
81
83
|
esgpull/models/options.py,sha256=BUWf3IthlSvroNn05eC3MtGRq7wHIYlr1dFI1Wg9acM,4744
|
|
82
84
|
esgpull/models/query.py,sha256=le0BLncg7A8mG-DjaLjv1_rAPq6uJZzYJKajtgBIWlE,19659
|
|
83
85
|
esgpull/models/selection.py,sha256=QCX_2eoCJcYB1ULll-J7UV5lCpkweis92FANQY8pH0o,5895
|
|
84
|
-
esgpull/models/sql.py,sha256
|
|
86
|
+
esgpull/models/sql.py,sha256=K8Nre5HKFPjkRzUUW6p6Qk7aG8upbw8C3pnmCFlg7d8,8942
|
|
85
87
|
esgpull/models/synda_file.py,sha256=6o5unPhzVJGnbpA2MxcS0r-hrBwocHYVnLrqjSGtmuk,2387
|
|
86
88
|
esgpull/models/tag.py,sha256=5CQDB9rAeCqog63ec9LPFN46HOFNkHPy-maY4gkBQ3E,461
|
|
87
89
|
esgpull/models/utils.py,sha256=exwlIlIKYjhhfUE82w1kU_HeSQOSY97PTvpkhW0udMA,1631
|
|
88
|
-
esgpull-0.9.
|
|
89
|
-
esgpull-0.9.
|
|
90
|
-
esgpull-0.9.
|
|
91
|
-
esgpull-0.9.
|
|
92
|
-
esgpull-0.9.
|
|
90
|
+
esgpull-0.9.2.dist-info/METADATA,sha256=58JNkrLxhc7gH87SOzgLz6VTIZX5ylTAGfbYFMcv0Jg,3781
|
|
91
|
+
esgpull-0.9.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
92
|
+
esgpull-0.9.2.dist-info/entry_points.txt,sha256=vyh7HvFrCp4iyMrTkDoSF3weaYrlNj2OJe0Fq5q4QB4,45
|
|
93
|
+
esgpull-0.9.2.dist-info/licenses/LICENSE,sha256=lUqGPGWDHHxjkUDuYgjLLY2XQXXn_EHU7fnrQWHGugc,1540
|
|
94
|
+
esgpull-0.9.2.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|