esgpull 0.6.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- esgpull/__init__.py +12 -0
- esgpull/auth.py +181 -0
- esgpull/cli/__init__.py +73 -0
- esgpull/cli/add.py +103 -0
- esgpull/cli/autoremove.py +38 -0
- esgpull/cli/config.py +116 -0
- esgpull/cli/convert.py +285 -0
- esgpull/cli/decorators.py +342 -0
- esgpull/cli/download.py +74 -0
- esgpull/cli/facet.py +23 -0
- esgpull/cli/get.py +28 -0
- esgpull/cli/install.py +85 -0
- esgpull/cli/link.py +105 -0
- esgpull/cli/login.py +56 -0
- esgpull/cli/remove.py +73 -0
- esgpull/cli/retry.py +43 -0
- esgpull/cli/search.py +201 -0
- esgpull/cli/self.py +238 -0
- esgpull/cli/show.py +66 -0
- esgpull/cli/status.py +67 -0
- esgpull/cli/track.py +87 -0
- esgpull/cli/update.py +184 -0
- esgpull/cli/utils.py +247 -0
- esgpull/config.py +410 -0
- esgpull/constants.py +56 -0
- esgpull/context.py +724 -0
- esgpull/database.py +161 -0
- esgpull/download.py +162 -0
- esgpull/esgpull.py +447 -0
- esgpull/exceptions.py +167 -0
- esgpull/fs.py +253 -0
- esgpull/graph.py +460 -0
- esgpull/install_config.py +185 -0
- esgpull/migrations/README +1 -0
- esgpull/migrations/env.py +82 -0
- esgpull/migrations/script.py.mako +24 -0
- esgpull/migrations/versions/0.3.0_update_tables.py +170 -0
- esgpull/migrations/versions/0.3.1_update_tables.py +25 -0
- esgpull/migrations/versions/0.3.2_update_tables.py +26 -0
- esgpull/migrations/versions/0.3.3_update_tables.py +25 -0
- esgpull/migrations/versions/0.3.4_update_tables.py +25 -0
- esgpull/migrations/versions/0.3.5_update_tables.py +25 -0
- esgpull/migrations/versions/0.3.6_update_tables.py +26 -0
- esgpull/migrations/versions/0.3.7_update_tables.py +26 -0
- esgpull/migrations/versions/0.3.8_update_tables.py +26 -0
- esgpull/migrations/versions/0.4.0_update_tables.py +25 -0
- esgpull/migrations/versions/0.5.0_update_tables.py +26 -0
- esgpull/migrations/versions/0.5.1_update_tables.py +26 -0
- esgpull/migrations/versions/0.5.2_update_tables.py +25 -0
- esgpull/migrations/versions/0.5.3_update_tables.py +26 -0
- esgpull/migrations/versions/0.5.4_update_tables.py +25 -0
- esgpull/migrations/versions/0.5.5_update_tables.py +25 -0
- esgpull/migrations/versions/0.6.0_update_tables.py +25 -0
- esgpull/migrations/versions/0.6.1_update_tables.py +25 -0
- esgpull/migrations/versions/0.6.2_update_tables.py +25 -0
- esgpull/migrations/versions/0.6.3_update_tables.py +25 -0
- esgpull/models/__init__.py +31 -0
- esgpull/models/base.py +50 -0
- esgpull/models/dataset.py +34 -0
- esgpull/models/facet.py +18 -0
- esgpull/models/file.py +65 -0
- esgpull/models/options.py +164 -0
- esgpull/models/query.py +481 -0
- esgpull/models/selection.py +201 -0
- esgpull/models/sql.py +258 -0
- esgpull/models/synda_file.py +85 -0
- esgpull/models/tag.py +19 -0
- esgpull/models/utils.py +54 -0
- esgpull/presets.py +13 -0
- esgpull/processor.py +172 -0
- esgpull/py.typed +0 -0
- esgpull/result.py +53 -0
- esgpull/tui.py +346 -0
- esgpull/utils.py +54 -0
- esgpull/version.py +1 -0
- esgpull-0.6.3.dist-info/METADATA +110 -0
- esgpull-0.6.3.dist-info/RECORD +80 -0
- esgpull-0.6.3.dist-info/WHEEL +4 -0
- esgpull-0.6.3.dist-info/entry_points.txt +3 -0
- esgpull-0.6.3.dist-info/licenses/LICENSE +28 -0
esgpull/tui.py
ADDED
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import atexit
|
|
4
|
+
import logging
|
|
5
|
+
import sys
|
|
6
|
+
from collections.abc import Iterable, Mapping
|
|
7
|
+
from contextlib import contextmanager
|
|
8
|
+
from datetime import datetime
|
|
9
|
+
from enum import IntEnum
|
|
10
|
+
from json import dumps as json_dumps
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any, TypeVar
|
|
13
|
+
|
|
14
|
+
if sys.version_info < (3, 11):
|
|
15
|
+
from exceptiongroup import BaseExceptionGroup
|
|
16
|
+
|
|
17
|
+
import click.exceptions
|
|
18
|
+
from attrs import define, field
|
|
19
|
+
from rich.console import Console, Group, RenderableType
|
|
20
|
+
from rich.live import Live
|
|
21
|
+
from rich.logging import RichHandler
|
|
22
|
+
from rich.progress import Progress, ProgressColumn, track
|
|
23
|
+
from rich.prompt import Confirm, Prompt
|
|
24
|
+
from rich.status import Status
|
|
25
|
+
from rich.syntax import Syntax
|
|
26
|
+
from rich.text import Text
|
|
27
|
+
from tomlkit import dumps as tomlkit_dumps
|
|
28
|
+
from yaml import dump as yaml_dump
|
|
29
|
+
|
|
30
|
+
from esgpull.config import Config
|
|
31
|
+
|
|
32
|
+
logger = logging.getLogger("esgpull")
|
|
33
|
+
logging.root.setLevel(logging.DEBUG)
|
|
34
|
+
logging.root.addHandler(logging.NullHandler())
|
|
35
|
+
|
|
36
|
+
# sqlalchemy is very verbose
|
|
37
|
+
logging.getLogger("sqlalchemy.engine").setLevel(logging.WARNING)
|
|
38
|
+
|
|
39
|
+
_console = Console(highlight=True)
|
|
40
|
+
_err_console = Console(stderr=True)
|
|
41
|
+
_record_console = Console(highlight=True, record=True)
|
|
42
|
+
|
|
43
|
+
T = TypeVar("T")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class Verbosity(IntEnum):
|
|
47
|
+
Normal = 0
|
|
48
|
+
Errors = 1
|
|
49
|
+
Detail = 2
|
|
50
|
+
Debug = 3
|
|
51
|
+
|
|
52
|
+
def get_level(self) -> int:
|
|
53
|
+
levels = [
|
|
54
|
+
logging.WARNING,
|
|
55
|
+
logging.WARNING,
|
|
56
|
+
logging.INFO,
|
|
57
|
+
logging.DEBUG,
|
|
58
|
+
]
|
|
59
|
+
return levels[self]
|
|
60
|
+
|
|
61
|
+
def render(self) -> Text:
|
|
62
|
+
return Text(self.name.upper(), style=f"logging.level.{self.name}")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class DummyConsole:
|
|
66
|
+
def print(self, msg: str) -> None:
|
|
67
|
+
pass
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class DummyLive:
|
|
71
|
+
def __enter__(self) -> DummyLive:
|
|
72
|
+
return self
|
|
73
|
+
|
|
74
|
+
def __exit__(self, *args):
|
|
75
|
+
...
|
|
76
|
+
|
|
77
|
+
@property
|
|
78
|
+
def console(self) -> DummyConsole:
|
|
79
|
+
return DummyConsole()
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def yaml_syntax(data: Mapping[str, Any]) -> Syntax:
|
|
83
|
+
return Syntax(yaml_dump(data, sort_keys=False), "yaml", theme="ansi_dark")
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def toml_syntax(data: Mapping[str, Any]) -> Syntax:
|
|
87
|
+
return Syntax(tomlkit_dumps(data), "toml", theme="ansi_dark")
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
LOG_FORMAT = "[%(asctime)s] %(levelname)-10s%(name)s\n%(message)s\n"
|
|
91
|
+
LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
|
|
92
|
+
FILE_DATE_FORMAT = "%Y-%m-%d_%H-%M-%S"
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
@define
|
|
96
|
+
class UI:
|
|
97
|
+
path: Path = field(converter=Path)
|
|
98
|
+
verbosity: Verbosity = Verbosity.Normal
|
|
99
|
+
logfile: bool = True
|
|
100
|
+
record: bool = False
|
|
101
|
+
default_onraise: type[Exception] | Exception | None = None
|
|
102
|
+
# max_size: int = 1 << 30
|
|
103
|
+
|
|
104
|
+
@property
|
|
105
|
+
def console(self) -> Console:
|
|
106
|
+
if self.record:
|
|
107
|
+
return _record_console
|
|
108
|
+
else:
|
|
109
|
+
return _console
|
|
110
|
+
|
|
111
|
+
@property
|
|
112
|
+
def err_console(self) -> Console:
|
|
113
|
+
if self.record:
|
|
114
|
+
return _record_console
|
|
115
|
+
else:
|
|
116
|
+
return _err_console
|
|
117
|
+
|
|
118
|
+
@staticmethod
|
|
119
|
+
def from_config(
|
|
120
|
+
config: Config,
|
|
121
|
+
verbosity: Verbosity = Verbosity.Normal,
|
|
122
|
+
record: bool = False,
|
|
123
|
+
) -> UI:
|
|
124
|
+
return UI(config.paths.log, verbosity=verbosity, record=record)
|
|
125
|
+
|
|
126
|
+
@contextmanager
|
|
127
|
+
def logging(
|
|
128
|
+
self,
|
|
129
|
+
modulename: str = "",
|
|
130
|
+
onraise: type[Exception] | Exception | None = None,
|
|
131
|
+
record: bool | None = None,
|
|
132
|
+
):
|
|
133
|
+
if self.verbosity > Verbosity.Normal:
|
|
134
|
+
logger.setLevel(logging.INFO)
|
|
135
|
+
if record is not None:
|
|
136
|
+
self.record = record
|
|
137
|
+
handler: logging.Handler
|
|
138
|
+
temp_path: Path | None = None
|
|
139
|
+
fmt = LOG_FORMAT
|
|
140
|
+
datefmt = LOG_DATE_FORMAT
|
|
141
|
+
if self.verbosity >= Verbosity.Errors:
|
|
142
|
+
if self.err_console.is_terminal or self.err_console.is_jupyter:
|
|
143
|
+
handler = RichHandler(
|
|
144
|
+
console=self.err_console,
|
|
145
|
+
show_path=False,
|
|
146
|
+
markup=True,
|
|
147
|
+
)
|
|
148
|
+
fmt = "[yellow]· %(name)s ·[/]\n%(message)s"
|
|
149
|
+
datefmt = "[%X]"
|
|
150
|
+
else:
|
|
151
|
+
handler = logging.StreamHandler()
|
|
152
|
+
handler.setLevel(self.verbosity.get_level())
|
|
153
|
+
elif self.logfile:
|
|
154
|
+
date = datetime.utcnow().strftime(FILE_DATE_FORMAT)
|
|
155
|
+
filename = "-".join(["esgpull", modulename, date]) + ".log"
|
|
156
|
+
temp_path = self.path / filename
|
|
157
|
+
handler = logging.FileHandler(temp_path)
|
|
158
|
+
handler.setLevel(logging.DEBUG)
|
|
159
|
+
else:
|
|
160
|
+
handler = logging.NullHandler()
|
|
161
|
+
handler.setFormatter(logging.Formatter(fmt=fmt, datefmt=datefmt))
|
|
162
|
+
logging.root.addHandler(handler)
|
|
163
|
+
try:
|
|
164
|
+
yield
|
|
165
|
+
except (click.exceptions.Exit, click.exceptions.Abort):
|
|
166
|
+
if temp_path is not None:
|
|
167
|
+
atexit.register(temp_path.unlink)
|
|
168
|
+
raise
|
|
169
|
+
except click.exceptions.ClickException:
|
|
170
|
+
raise
|
|
171
|
+
except BaseException as exc:
|
|
172
|
+
tb = exc.__traceback__
|
|
173
|
+
while True:
|
|
174
|
+
if tb is None:
|
|
175
|
+
break
|
|
176
|
+
elif tb.tb_next is None:
|
|
177
|
+
break
|
|
178
|
+
tb = tb.tb_next
|
|
179
|
+
if tb is None:
|
|
180
|
+
f_locals = {}
|
|
181
|
+
else:
|
|
182
|
+
f_locals = tb.tb_frame.f_locals
|
|
183
|
+
locals_text = self.render(f_locals, highlight=False)
|
|
184
|
+
logging.root.debug(f"Locals:\n{locals_text}")
|
|
185
|
+
logging.root.exception("")
|
|
186
|
+
self.print(f"[red]{type(exc).__name__}[/]: {exc}", err=True)
|
|
187
|
+
if self.verbosity < Verbosity.Errors and self.logfile:
|
|
188
|
+
self.print(
|
|
189
|
+
f"See [yellow]{temp_path}[/] for error log.",
|
|
190
|
+
err=True,
|
|
191
|
+
)
|
|
192
|
+
if onraise is not None:
|
|
193
|
+
raise onraise
|
|
194
|
+
elif self.default_onraise is not None:
|
|
195
|
+
raise self.default_onraise
|
|
196
|
+
else:
|
|
197
|
+
raise
|
|
198
|
+
else:
|
|
199
|
+
if temp_path is not None:
|
|
200
|
+
atexit.register(temp_path.unlink)
|
|
201
|
+
finally:
|
|
202
|
+
logging.root.removeHandler(handler)
|
|
203
|
+
|
|
204
|
+
def print(
|
|
205
|
+
self,
|
|
206
|
+
msg: Any,
|
|
207
|
+
err: bool = False,
|
|
208
|
+
json: bool = False,
|
|
209
|
+
yaml: bool = False,
|
|
210
|
+
toml: bool = False,
|
|
211
|
+
verbosity: Verbosity = Verbosity.Normal,
|
|
212
|
+
**kwargs: Any,
|
|
213
|
+
) -> None:
|
|
214
|
+
if self.verbosity >= verbosity:
|
|
215
|
+
console = self.err_console if err else self.console
|
|
216
|
+
if json:
|
|
217
|
+
console.print_json(json_dumps(msg), **kwargs)
|
|
218
|
+
elif yaml:
|
|
219
|
+
console.print(yaml_syntax(msg), **kwargs)
|
|
220
|
+
elif toml:
|
|
221
|
+
console.print(toml_syntax(msg), **kwargs)
|
|
222
|
+
else:
|
|
223
|
+
if not console.is_interactive:
|
|
224
|
+
kwargs.setdefault("crop", False)
|
|
225
|
+
kwargs.setdefault("overflow", "ignore")
|
|
226
|
+
console.print(msg, **kwargs)
|
|
227
|
+
|
|
228
|
+
def render(
|
|
229
|
+
self,
|
|
230
|
+
msg: Any,
|
|
231
|
+
json: bool = False,
|
|
232
|
+
yaml: bool = False,
|
|
233
|
+
toml: bool = False,
|
|
234
|
+
**kwargs: Any,
|
|
235
|
+
) -> str:
|
|
236
|
+
with self.console.capture() as capture:
|
|
237
|
+
if json:
|
|
238
|
+
self.console.print_json(json_dumps(msg), **kwargs)
|
|
239
|
+
elif yaml:
|
|
240
|
+
self.console.print(yaml_syntax(msg), **kwargs)
|
|
241
|
+
elif toml:
|
|
242
|
+
self.console.print(toml_syntax(msg), **kwargs)
|
|
243
|
+
else:
|
|
244
|
+
self.console.print(msg, **kwargs)
|
|
245
|
+
return capture.get()
|
|
246
|
+
|
|
247
|
+
def live(
|
|
248
|
+
self,
|
|
249
|
+
first: RenderableType,
|
|
250
|
+
*rest: RenderableType,
|
|
251
|
+
disable: bool = False,
|
|
252
|
+
) -> Live | DummyLive:
|
|
253
|
+
if disable:
|
|
254
|
+
return DummyLive()
|
|
255
|
+
if not rest:
|
|
256
|
+
renderables = first
|
|
257
|
+
else:
|
|
258
|
+
renderables = Group(first, *rest)
|
|
259
|
+
# use _console to avoid recording the progress bar
|
|
260
|
+
return Live(renderables, console=_console)
|
|
261
|
+
|
|
262
|
+
def track(self, iterable: Iterable[T]) -> Iterable[T]:
|
|
263
|
+
# use _console to avoid recording the progress bar
|
|
264
|
+
return track(iterable, console=_console)
|
|
265
|
+
|
|
266
|
+
def make_progress(
|
|
267
|
+
self,
|
|
268
|
+
*columns: str | ProgressColumn,
|
|
269
|
+
**kwargs: Any,
|
|
270
|
+
) -> Progress:
|
|
271
|
+
# use _console to avoid recording the progress bar
|
|
272
|
+
return Progress(*columns, console=_console, **kwargs)
|
|
273
|
+
|
|
274
|
+
def spinner(self, msg: str) -> Status:
|
|
275
|
+
# use _console to avoid recording the spinner
|
|
276
|
+
return _console.status(msg, spinner="earth")
|
|
277
|
+
|
|
278
|
+
def ask(self, msg: str, default: bool | None = None) -> bool:
|
|
279
|
+
if default is not None:
|
|
280
|
+
return Confirm.ask(msg, default=default, console=self.console)
|
|
281
|
+
else:
|
|
282
|
+
return Confirm.ask(msg, console=self.console)
|
|
283
|
+
|
|
284
|
+
def choice(
|
|
285
|
+
self,
|
|
286
|
+
msg: str,
|
|
287
|
+
choices: list[str],
|
|
288
|
+
default: str | None = None,
|
|
289
|
+
show_choices: bool = True,
|
|
290
|
+
) -> str:
|
|
291
|
+
if default is not None:
|
|
292
|
+
return Prompt.ask(
|
|
293
|
+
msg,
|
|
294
|
+
choices=choices,
|
|
295
|
+
default=default,
|
|
296
|
+
show_choices=show_choices,
|
|
297
|
+
console=self.console,
|
|
298
|
+
)
|
|
299
|
+
else:
|
|
300
|
+
return Prompt.ask(
|
|
301
|
+
msg,
|
|
302
|
+
choices=choices,
|
|
303
|
+
show_choices=show_choices,
|
|
304
|
+
console=self.console,
|
|
305
|
+
)
|
|
306
|
+
|
|
307
|
+
def prompt(
|
|
308
|
+
self, msg: str, default: str | None = None, password: bool = False
|
|
309
|
+
) -> str:
|
|
310
|
+
if default is not None:
|
|
311
|
+
return Prompt.ask(
|
|
312
|
+
msg,
|
|
313
|
+
default=default,
|
|
314
|
+
password=password,
|
|
315
|
+
console=self.console,
|
|
316
|
+
)
|
|
317
|
+
else:
|
|
318
|
+
return Prompt.ask(msg, password=password, console=self.console)
|
|
319
|
+
|
|
320
|
+
def rule(self, msg: str):
|
|
321
|
+
self.console.rule(msg)
|
|
322
|
+
|
|
323
|
+
def export_svg(self) -> Path:
|
|
324
|
+
date = datetime.utcnow().strftime(FILE_DATE_FORMAT)
|
|
325
|
+
filename = "-".join(["record", date]) + ".svg"
|
|
326
|
+
output_path = self.path / filename
|
|
327
|
+
with output_path.open("w") as f:
|
|
328
|
+
f.write(_record_console.export_svg())
|
|
329
|
+
return output_path
|
|
330
|
+
|
|
331
|
+
def raise_maybe_record(
|
|
332
|
+
self,
|
|
333
|
+
exc: type[Exception] | Exception | BaseExceptionGroup,
|
|
334
|
+
) -> None:
|
|
335
|
+
if self.record:
|
|
336
|
+
output_path = self.export_svg()
|
|
337
|
+
self.print(f":+1: Console output exported to {output_path}")
|
|
338
|
+
raise exc
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
TempUI = UI(
|
|
342
|
+
"/tmp",
|
|
343
|
+
Verbosity.Errors,
|
|
344
|
+
logfile=False,
|
|
345
|
+
default_onraise=click.exceptions.Exit(1),
|
|
346
|
+
)
|
esgpull/utils.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import datetime
|
|
3
|
+
from typing import Callable, Coroutine, TypeVar
|
|
4
|
+
from urllib.parse import urlparse
|
|
5
|
+
|
|
6
|
+
from rich.filesize import _to_str
|
|
7
|
+
|
|
8
|
+
T = TypeVar("T")
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def sync(
|
|
12
|
+
coro: Coroutine[None, None, T],
|
|
13
|
+
before_cb: Callable | None = None,
|
|
14
|
+
after_cb: Callable | None = None,
|
|
15
|
+
) -> T:
|
|
16
|
+
if before_cb is not None:
|
|
17
|
+
before_cb()
|
|
18
|
+
result = asyncio.run(coro)
|
|
19
|
+
if after_cb is not None:
|
|
20
|
+
after_cb()
|
|
21
|
+
return result
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def format_size(size: int) -> str:
|
|
25
|
+
return _to_str(
|
|
26
|
+
size,
|
|
27
|
+
("kiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB"),
|
|
28
|
+
1024,
|
|
29
|
+
precision=1,
|
|
30
|
+
separator=" ",
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def format_date(date: str | datetime.datetime, fmt: str = "%Y-%m-%d") -> str:
|
|
35
|
+
match date:
|
|
36
|
+
case datetime.datetime():
|
|
37
|
+
...
|
|
38
|
+
case str():
|
|
39
|
+
date = datetime.datetime.strptime(date, fmt)
|
|
40
|
+
case _:
|
|
41
|
+
raise ValueError(date)
|
|
42
|
+
return date.replace(microsecond=0).isoformat() + "Z"
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def url2index(url: str) -> str:
|
|
46
|
+
parsed = urlparse(url)
|
|
47
|
+
if parsed.netloc == "":
|
|
48
|
+
return parsed.path
|
|
49
|
+
else:
|
|
50
|
+
return parsed.netloc
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def index2url(index: str) -> str:
|
|
54
|
+
return "https://" + url2index(index) + "/esg-search/search"
|
esgpull/version.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.6.3"
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: esgpull
|
|
3
|
+
Version: 0.6.3
|
|
4
|
+
Summary: ESGF data discovery, download, replication tool
|
|
5
|
+
Author-Email: Sven Rodriguez <srodriguez@ipsl.fr>
|
|
6
|
+
License: BSD 3-Clause License
|
|
7
|
+
|
|
8
|
+
Copyright (c) 2023, Institut Pierre-Simon Laplace (IPSL) and contributors
|
|
9
|
+
|
|
10
|
+
Redistribution and use in source and binary forms, with or without
|
|
11
|
+
modification, are permitted provided that the following conditions are met:
|
|
12
|
+
|
|
13
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
|
14
|
+
list of conditions and the following disclaimer.
|
|
15
|
+
|
|
16
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
17
|
+
this list of conditions and the following disclaimer in the documentation
|
|
18
|
+
and/or other materials provided with the distribution.
|
|
19
|
+
|
|
20
|
+
3. Neither the name of the copyright holder nor the names of its
|
|
21
|
+
contributors may be used to endorse or promote products derived from
|
|
22
|
+
this software without specific prior written permission.
|
|
23
|
+
|
|
24
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
25
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
26
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
27
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
28
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
29
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
30
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
31
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
32
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
33
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
34
|
+
Project-URL: Repository, https://github.com/ESGF/esgf-download
|
|
35
|
+
Project-URL: Documentation, https://esgf.github.io/esgf-download/
|
|
36
|
+
Requires-Python: >=3.10
|
|
37
|
+
Requires-Dist: MyProxyClient>=2.1.0
|
|
38
|
+
Requires-Dist: aiofiles>=22.1.0
|
|
39
|
+
Requires-Dist: alembic>=1.8.1
|
|
40
|
+
Requires-Dist: click>=8.1.3
|
|
41
|
+
Requires-Dist: click-params>=0.4.0
|
|
42
|
+
Requires-Dist: httpx>=0.23.0
|
|
43
|
+
Requires-Dist: nest-asyncio>=1.5.6
|
|
44
|
+
Requires-Dist: pyOpenSSL>=22.1.0
|
|
45
|
+
Requires-Dist: pyyaml>=6.0
|
|
46
|
+
Requires-Dist: tomlkit>=0.11.5
|
|
47
|
+
Requires-Dist: rich>=12.6.0
|
|
48
|
+
Requires-Dist: sqlalchemy>=2.0.0b2
|
|
49
|
+
Requires-Dist: setuptools>=65.4.1
|
|
50
|
+
Requires-Dist: aiostream>=0.4.5
|
|
51
|
+
Requires-Dist: attrs>=22.1.0
|
|
52
|
+
Requires-Dist: cattrs>=22.2.0
|
|
53
|
+
Requires-Dist: platformdirs>=2.6.2
|
|
54
|
+
Requires-Dist: pyparsing>=3.0.9
|
|
55
|
+
Description-Content-Type: text/markdown
|
|
56
|
+
|
|
57
|
+
[](https://pdm.fming.dev)
|
|
58
|
+
|
|
59
|
+
# esgpull - ESGF data management utility
|
|
60
|
+
|
|
61
|
+
`esgpull` is a tool that simplifies usage of the [ESGF Search API](https://esgf.github.io/esg-search/ESGF_Search_RESTful_API.html) for data discovery, and manages procedures related to downloading and storing files from ESGF.
|
|
62
|
+
|
|
63
|
+
```py
|
|
64
|
+
from esgpull import Esgpull, Query
|
|
65
|
+
|
|
66
|
+
query = Query()
|
|
67
|
+
query.selection.project = "CMIP6"
|
|
68
|
+
query.options.distrib = True # default=False
|
|
69
|
+
esg = Esgpull()
|
|
70
|
+
nb_datasets = esg.context.hits(query, file=False)[0]
|
|
71
|
+
nb_files = esg.context.hits(query, file=True)[0]
|
|
72
|
+
datasets = esg.context.datasets(query, max_hits=5)
|
|
73
|
+
print(f"Number of CMIP6 datasets: {nb_datasets}")
|
|
74
|
+
print(f"Number of CMIP6 files: {nb_files}")
|
|
75
|
+
for dataset in datasets:
|
|
76
|
+
print(dataset)
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
## Features
|
|
80
|
+
|
|
81
|
+
- Command-line interface
|
|
82
|
+
- HTTP download (async multi-file)
|
|
83
|
+
|
|
84
|
+
## Usage
|
|
85
|
+
|
|
86
|
+
```console
|
|
87
|
+
Usage: esgpull [OPTIONS] COMMAND [ARGS]...
|
|
88
|
+
|
|
89
|
+
esgpull is a management utility for files and datasets from ESGF.
|
|
90
|
+
|
|
91
|
+
Options:
|
|
92
|
+
-V, --version Show the version and exit.
|
|
93
|
+
-h, --help Show this message and exit.
|
|
94
|
+
|
|
95
|
+
Commands:
|
|
96
|
+
add Add queries to the database
|
|
97
|
+
config View/modify config
|
|
98
|
+
convert Convert synda selection files to esgpull queries
|
|
99
|
+
download Asynchronously download files linked to queries
|
|
100
|
+
login OpenID authentication and certificates renewal
|
|
101
|
+
remove Remove queries from the database
|
|
102
|
+
retry Re-queue failed and cancelled downloads
|
|
103
|
+
search Search datasets and files on ESGF
|
|
104
|
+
self Manage esgpull installations / import synda database
|
|
105
|
+
show View query tree
|
|
106
|
+
status View file queue status
|
|
107
|
+
track Track queries
|
|
108
|
+
untrack Untrack queries
|
|
109
|
+
update Fetch files, link files <-> queries, send files to download...
|
|
110
|
+
```
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
esgpull-0.6.3.dist-info/METADATA,sha256=8hz93j4mPbtVd1MXfDNuQKs98fJAHoJVeIfSa1-pqgI,4473
|
|
2
|
+
esgpull-0.6.3.dist-info/WHEEL,sha256=vnE8JVcI2Wz7GRKorsPArnBdnW2SWKWGow5gu5tHlRU,90
|
|
3
|
+
esgpull-0.6.3.dist-info/entry_points.txt,sha256=nfKsESeZyCiVD6dDCTCturf-vRnI-6GCuwqv9jxXCq8,46
|
|
4
|
+
esgpull-0.6.3.dist-info/licenses/LICENSE,sha256=lUqGPGWDHHxjkUDuYgjLLY2XQXXn_EHU7fnrQWHGugc,1540
|
|
5
|
+
esgpull/__init__.py,sha256=XItFDIMNmFUNNcKtUgXdfmGwUIWt4AAv0a4mZkfj5P8,240
|
|
6
|
+
esgpull/auth.py,sha256=QZ-l1ySLMP0fvuwYHRLv9FZYp1gqfju_eGaTMDByUxw,5205
|
|
7
|
+
esgpull/cli/__init__.py,sha256=boqxa_ku266IcZ6GY-7fVA2T9GLku1e01TXe3aqZmR4,1630
|
|
8
|
+
esgpull/cli/add.py,sha256=c-tYR8h5OlSiavSiDzpUa5gIhBPzIPV9P7M2UvHOptc,3111
|
|
9
|
+
esgpull/cli/autoremove.py,sha256=g76_qnc3q84zSO7W0JsbWtGN4AfWBXTkQJO6gPCs2Pw,1336
|
|
10
|
+
esgpull/cli/config.py,sha256=JbSa2NvXxccbOpEyvJd4vSlBnwb_NClx5sCJp_WY7-k,4048
|
|
11
|
+
esgpull/cli/convert.py,sha256=Nonu49aijY3OFq2MNCqs7EQ08NXFiT0ehvNxoGp4NCE,9847
|
|
12
|
+
esgpull/cli/decorators.py,sha256=a8lhbhe9ppaNj_kSwPfdeVkq9pEwchiGb52ZbG8EYWI,6475
|
|
13
|
+
esgpull/cli/download.py,sha256=3_Fm8JJYBWKs63oQ1OLaHJCv9ccbeY2gIW8SDasaYNE,2356
|
|
14
|
+
esgpull/cli/facet.py,sha256=V1u-DxNkhswwSt0qpXvuHrCI_tE8jAJGEe6_fMhYbaM,584
|
|
15
|
+
esgpull/cli/get.py,sha256=2WXL01Ri0P_2Rf1xrp9bnsrrxir6whxkAC0SnohjFpg,678
|
|
16
|
+
esgpull/cli/install.py,sha256=fd8nKwIFvOivgn_gOGn7XIk1BB9LXnhQB47KuIIy5AU,2880
|
|
17
|
+
esgpull/cli/link.py,sha256=hZ3p3AySsOUIsiA7lIdxEctw3A7ocCJgbjWcNx2dRVU,3372
|
|
18
|
+
esgpull/cli/login.py,sha256=FZ63SsB4fCDixwf7gMUR697Dk89W1OvpgeadKE4IqEU,2153
|
|
19
|
+
esgpull/cli/remove.py,sha256=p26hPhsgAgmAF4IsvjAI3jtzlIUq3k8Rxb0GTKYoEQM,2517
|
|
20
|
+
esgpull/cli/retry.py,sha256=UVpAjW_N7l6RTJ-T5qXojwcXPzzjT7sDKb_wBdvavrg,1310
|
|
21
|
+
esgpull/cli/search.py,sha256=6-c-W78b45-wo9GIVIzq8fYkqIY5NE7QvILsUb3M8Uk,6232
|
|
22
|
+
esgpull/cli/self.py,sha256=7nEEsK5W_Pth8IOSmvJHRlfwPPgXldhHAQK9yqf01S8,7932
|
|
23
|
+
esgpull/cli/show.py,sha256=4O1TvH31x9Fg-SI0a8dtblMZuJHJ30eBXOV9bwEl9Lk,1897
|
|
24
|
+
esgpull/cli/status.py,sha256=jgGBbmvcCjw21QTDNXdKAUo664p6junRg5kMwQ7My2I,2470
|
|
25
|
+
esgpull/cli/track.py,sha256=Q9ZvvV5FFGzp6wQZflAd_OFmqhAWgl1JFBad2dCbEF0,3089
|
|
26
|
+
esgpull/cli/update.py,sha256=HvdVkF-ZmJLAL29D757oDAolwXvP_ehOtD0TnGkmZ24,6828
|
|
27
|
+
esgpull/cli/utils.py,sha256=0SAJpeE13pHxt_5lfKn9tnjutUxeXqQ8xZWyJaeW-sU,7121
|
|
28
|
+
esgpull/config.py,sha256=WelBOrkaTPldk4UasFJvku5DRh7062kO6CPHNWSV3Wc,11529
|
|
29
|
+
esgpull/constants.py,sha256=3eq5Yow519p3YLAKjNdwOt1Ooei62FE9HykqCiyhZYE,1074
|
|
30
|
+
esgpull/context.py,sha256=oSymljW0hZLGhCJxo7lh96Eq3SI_KOT_jFMPHckIrm0,23203
|
|
31
|
+
esgpull/database.py,sha256=4JONNJpt7JzeohgBtRO2hpQUUFtV_wTncU20HjWPGVA,5431
|
|
32
|
+
esgpull/download.py,sha256=3YcGLrffzAmuiV49tYsW-7PuVYcSFtbsZnH2Q26khm0,5491
|
|
33
|
+
esgpull/esgpull.py,sha256=nmRZA72erSav2EfJJUNljkZzVKogKPRWuYHyNiCB7NI,16098
|
|
34
|
+
esgpull/exceptions.py,sha256=uK5ApBZJPwLlR7hYpvwQdApVtugn7_-N-MMmR8HpEgE,3215
|
|
35
|
+
esgpull/fs.py,sha256=O52QD7DVJImYYf9wWECMNWYrGt8EcHQ3z9TzroK6VAY,7623
|
|
36
|
+
esgpull/graph.py,sha256=CeaRcIAWMpVAJ-o6v6b3ffzy-ter-8HZ3tmxoN-Z-FM,15925
|
|
37
|
+
esgpull/install_config.py,sha256=mGZLaYEDaXlmX9obl_9d7jTyweT8UcP3XSo5RZRkkpk,5295
|
|
38
|
+
esgpull/migrations/README,sha256=heMzebYwlGhnE8_4CWJ4LS74WoEZjBy-S-mIJRxAEKI,39
|
|
39
|
+
esgpull/migrations/env.py,sha256=qv0RBnOOTx-f8rnemoTIm48JasLxNzbczq5wUBRQchg,2190
|
|
40
|
+
esgpull/migrations/script.py.mako,sha256=HNlf26BI1xvQKjiUojnj15BPrVUfVVr81IOgliJf83c,510
|
|
41
|
+
esgpull/migrations/versions/0.3.0_update_tables.py,sha256=r0rGX_cQSNX74JM5tZ8iqgstsAdfJH5rO3gGPkOxpok,5540
|
|
42
|
+
esgpull/migrations/versions/0.3.1_update_tables.py,sha256=Fm7o-ZVvuyFcyReXXLhXQJQ-feCrbY2c0wnKwYnj_5k,493
|
|
43
|
+
esgpull/migrations/versions/0.3.2_update_tables.py,sha256=pWHUwizBzodeslLWSjzuShQnuTnw_uGIhXcx7oQ7P3E,494
|
|
44
|
+
esgpull/migrations/versions/0.3.3_update_tables.py,sha256=JB1a19rGi-2f2Me79hiz3qtL-W_FDvrXlo5LQL7QjE8,493
|
|
45
|
+
esgpull/migrations/versions/0.3.4_update_tables.py,sha256=e8KSwKbgUeFLV9s968MYB8e-ydCDu_JuSXi_3TpS1xw,493
|
|
46
|
+
esgpull/migrations/versions/0.3.5_update_tables.py,sha256=SWesKJ4u15fL6EQ8roeNZYdP7dRm_uPvghSgX73hS7M,493
|
|
47
|
+
esgpull/migrations/versions/0.3.6_update_tables.py,sha256=7QdPYeQrj-PynPaif7TLyuhX0nCAV5-L4CaMFuooDAc,494
|
|
48
|
+
esgpull/migrations/versions/0.3.7_update_tables.py,sha256=eiqVov0DW3KpUSNL8cyPRDZWEDqeyD_POtp0I-VOgEw,494
|
|
49
|
+
esgpull/migrations/versions/0.3.8_update_tables.py,sha256=sHPHb-G2Lqw2k03RFiIHqyOpbflWAwY1wS8--5Y0Ms4,494
|
|
50
|
+
esgpull/migrations/versions/0.4.0_update_tables.py,sha256=EmsQoQ3bZ_k1vyLFG2ESVjhItqRBSAgDLBsixmEpaIg,493
|
|
51
|
+
esgpull/migrations/versions/0.5.0_update_tables.py,sha256=vinHUQ4EhiLnW71zOSVJ_ynt9u5oCKWW1ixZC2lILq0,494
|
|
52
|
+
esgpull/migrations/versions/0.5.1_update_tables.py,sha256=2U-8Ga3nx_U3jS0OEe_mbTf1uC0-I02uvZm4ggtzwc8,494
|
|
53
|
+
esgpull/migrations/versions/0.5.2_update_tables.py,sha256=mij7nSEYuZ91TuWLyqgQ11Qe8XjiOkDEP4uxlM602kg,493
|
|
54
|
+
esgpull/migrations/versions/0.5.3_update_tables.py,sha256=GrxEmlcqOdckqcdkpZJJfO-N1rhFRn6LxBo9D40rGUk,494
|
|
55
|
+
esgpull/migrations/versions/0.5.4_update_tables.py,sha256=5bWKhk3Ij6Rpc35L7U5kfPbuEoE_ODzaeEmOq8M2Mqs,493
|
|
56
|
+
esgpull/migrations/versions/0.5.5_update_tables.py,sha256=nGBB0rIsJtZqnpjbNVrx7kClXO7k6LMn_SZGLDmXzW8,493
|
|
57
|
+
esgpull/migrations/versions/0.6.0_update_tables.py,sha256=10Ioi84QwEWIB9Mqz-ItEGAj5iInmJz8T5RYI49xZ4A,493
|
|
58
|
+
esgpull/migrations/versions/0.6.1_update_tables.py,sha256=L4mkKlZUz4ftINwNeVNsPhk32QYMeoTie4B-GXVeKXw,493
|
|
59
|
+
esgpull/migrations/versions/0.6.2_update_tables.py,sha256=LFUVgDhxSll9FpvNFm-idD5t-SI3R30IO7cTzvShLVI,493
|
|
60
|
+
esgpull/migrations/versions/0.6.3_update_tables.py,sha256=ut4M0b90OWVB9pKSGTJ4Bl5cucXFJzRD4-WA_ego0WY,493
|
|
61
|
+
esgpull/models/__init__.py,sha256=rfa1yGLVDahFrnhq_8TPGzr7_oeBOFKNVD9EF0slUtY,725
|
|
62
|
+
esgpull/models/base.py,sha256=3nbR2lYMHWfovWz4EiAJ2bIvKpMadRvYZDdMRQDvN7M,1237
|
|
63
|
+
esgpull/models/dataset.py,sha256=1fOIVYIWKK5BivqvBpjfxrNpy9VfUHZng9Yc6ipPK1Q,905
|
|
64
|
+
esgpull/models/facet.py,sha256=COMgFjsxQcgb4uGMLy5JDRFWeMSHO-QDdG-cWpwvYqQ,459
|
|
65
|
+
esgpull/models/file.py,sha256=-8PPYtq7BWp-O_QtCDbkLdhTGTPhI1F1nodQacMnYGA,1517
|
|
66
|
+
esgpull/models/options.py,sha256=yzADDeDgtMymFdKG31JMl8sds-LnWDmjtaCpQ6EUKQ0,4745
|
|
67
|
+
esgpull/models/query.py,sha256=D-D0q1oz1L6FoshXgJDflg6iyR9W85dxnP8vKTRvdww,16719
|
|
68
|
+
esgpull/models/selection.py,sha256=5LlKhCM7FK3WieGuI2xLw5b_GvSlAGrmu67NtgIDd6c,5803
|
|
69
|
+
esgpull/models/sql.py,sha256=T2whCvz_MFb0tY5Ij5xcFlZ4TxrS0JT1rRU_gd2qVsg,7041
|
|
70
|
+
esgpull/models/synda_file.py,sha256=6o5unPhzVJGnbpA2MxcS0r-hrBwocHYVnLrqjSGtmuk,2387
|
|
71
|
+
esgpull/models/tag.py,sha256=5CQDB9rAeCqog63ec9LPFN46HOFNkHPy-maY4gkBQ3E,461
|
|
72
|
+
esgpull/models/utils.py,sha256=exwlIlIKYjhhfUE82w1kU_HeSQOSY97PTvpkhW0udMA,1631
|
|
73
|
+
esgpull/presets.py,sha256=hMFY2QfcLY9icuer5SK_M7Oqbj_GSEGCYQiwhNhFqLA,336
|
|
74
|
+
esgpull/processor.py,sha256=je2GS8eokqskpQx_qRh62BDr7lg8QfpjfM0jW9uJDDU,5444
|
|
75
|
+
esgpull/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
76
|
+
esgpull/result.py,sha256=f64l9gPFpFWgctmHVYrNWJvuXXB1sxpxXzJEIssLXxc,1020
|
|
77
|
+
esgpull/tui.py,sha256=MG3VULh8uTrRspw_kFuhunmHqscO9Xd7PS7MG9WyvMY,10395
|
|
78
|
+
esgpull/utils.py,sha256=eKipKCqj15dzCAqs1o7x5O1XClWPmDEM-qSvNwGZmXY,1233
|
|
79
|
+
esgpull/version.py,sha256=zYiFHqR7JwbvdK9dvKrh-RTNfUqjHUwC4CTcFAPVYLc,22
|
|
80
|
+
esgpull-0.6.3.dist-info/RECORD,,
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
BSD 3-Clause License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2023, Institut Pierre-Simon Laplace (IPSL) and contributors
|
|
4
|
+
|
|
5
|
+
Redistribution and use in source and binary forms, with or without
|
|
6
|
+
modification, are permitted provided that the following conditions are met:
|
|
7
|
+
|
|
8
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
|
9
|
+
list of conditions and the following disclaimer.
|
|
10
|
+
|
|
11
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
12
|
+
this list of conditions and the following disclaimer in the documentation
|
|
13
|
+
and/or other materials provided with the distribution.
|
|
14
|
+
|
|
15
|
+
3. Neither the name of the copyright holder nor the names of its
|
|
16
|
+
contributors may be used to endorse or promote products derived from
|
|
17
|
+
this software without specific prior written permission.
|
|
18
|
+
|
|
19
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
20
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
21
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
22
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
23
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
24
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
25
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
26
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
27
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
28
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|