earthscope-cli 1.2.0__tar.gz → 1.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {earthscope_cli-1.2.0/src/earthscope_cli.egg-info → earthscope_cli-1.3.0}/PKG-INFO +2 -2
- {earthscope_cli-1.2.0 → earthscope_cli-1.3.0}/pyproject.toml +3 -3
- earthscope_cli-1.3.0/src/earthscope_cli/__init__.py +1 -0
- {earthscope_cli-1.2.0 → earthscope_cli-1.3.0}/src/earthscope_cli/dropoff/render.py +18 -4
- {earthscope_cli-1.2.0 → earthscope_cli-1.3.0}/src/earthscope_cli/dropoff/typer.py +34 -21
- {earthscope_cli-1.2.0 → earthscope_cli-1.3.0}/src/earthscope_cli/dropoff/util.py +2 -70
- {earthscope_cli-1.2.0 → earthscope_cli-1.3.0/src/earthscope_cli.egg-info}/PKG-INFO +2 -2
- {earthscope_cli-1.2.0 → earthscope_cli-1.3.0}/src/earthscope_cli.egg-info/SOURCES.txt +1 -0
- {earthscope_cli-1.2.0 → earthscope_cli-1.3.0}/src/earthscope_cli.egg-info/requires.txt +1 -1
- {earthscope_cli-1.2.0 → earthscope_cli-1.3.0}/tests/test_dropoff.py +17 -263
- earthscope_cli-1.3.0/tests/test_startup_imports.py +88 -0
- earthscope_cli-1.2.0/src/earthscope_cli/__init__.py +0 -1
- {earthscope_cli-1.2.0 → earthscope_cli-1.3.0}/LICENSE +0 -0
- {earthscope_cli-1.2.0 → earthscope_cli-1.3.0}/README.md +0 -0
- {earthscope_cli-1.2.0 → earthscope_cli-1.3.0}/setup.cfg +0 -0
- {earthscope_cli-1.2.0 → earthscope_cli-1.3.0}/setup.py +0 -0
- {earthscope_cli-1.2.0 → earthscope_cli-1.3.0}/src/earthscope_cli/common.py +0 -0
- {earthscope_cli-1.2.0 → earthscope_cli-1.3.0}/src/earthscope_cli/debug.py +0 -0
- {earthscope_cli-1.2.0 → earthscope_cli-1.3.0}/src/earthscope_cli/login.py +0 -0
- {earthscope_cli-1.2.0 → earthscope_cli-1.3.0}/src/earthscope_cli/main.py +0 -0
- {earthscope_cli-1.2.0 → earthscope_cli-1.3.0}/src/earthscope_cli/user.py +0 -0
- {earthscope_cli-1.2.0 → earthscope_cli-1.3.0}/src/earthscope_cli/util.py +0 -0
- {earthscope_cli-1.2.0 → earthscope_cli-1.3.0}/src/earthscope_cli.egg-info/dependency_links.txt +0 -0
- {earthscope_cli-1.2.0 → earthscope_cli-1.3.0}/src/earthscope_cli.egg-info/entry_points.txt +0 -0
- {earthscope_cli-1.2.0 → earthscope_cli-1.3.0}/src/earthscope_cli.egg-info/top_level.txt +0 -0
- {earthscope_cli-1.2.0 → earthscope_cli-1.3.0}/tests/test_cli.py +0 -0
- {earthscope_cli-1.2.0 → earthscope_cli-1.3.0}/tests/test_login.py +0 -0
- {earthscope_cli-1.2.0 → earthscope_cli-1.3.0}/tests/test_user.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: earthscope-cli
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.3.0
|
|
4
4
|
Summary: A CLI for the EarthScope API
|
|
5
5
|
Author-email: EarthScope <data-help@earthscope.org>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -15,7 +15,7 @@ Description-Content-Type: text/markdown
|
|
|
15
15
|
License-File: LICENSE
|
|
16
16
|
Requires-Dist: typer<0.24.0
|
|
17
17
|
Requires-Dist: typer-di>=0.1.3
|
|
18
|
-
Requires-Dist: earthscope-sdk>=1.
|
|
18
|
+
Requires-Dist: earthscope-sdk>=1.10.0
|
|
19
19
|
Requires-Dist: rich>=13.8.0
|
|
20
20
|
Provides-Extra: dev
|
|
21
21
|
Requires-Dist: black; extra == "dev"
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "earthscope-cli"
|
|
7
|
-
version = "1.
|
|
7
|
+
version = "1.3.0"
|
|
8
8
|
description = "A CLI for the EarthScope API"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
authors = [{ name = "EarthScope", email = "data-help@earthscope.org" }]
|
|
@@ -20,7 +20,7 @@ classifiers = [
|
|
|
20
20
|
dependencies = [
|
|
21
21
|
"typer<0.24.0",
|
|
22
22
|
"typer-di>=0.1.3",
|
|
23
|
-
"earthscope-sdk>=1.
|
|
23
|
+
"earthscope-sdk>=1.10.0",
|
|
24
24
|
"rich>=13.8.0",
|
|
25
25
|
]
|
|
26
26
|
requires-python = ">=3.10"
|
|
@@ -46,7 +46,7 @@ Changelog = "https://docs.earthscope.org/projects/CLI/page/changelog.html"
|
|
|
46
46
|
es = "earthscope_cli.main:app"
|
|
47
47
|
|
|
48
48
|
[tool.bumpver]
|
|
49
|
-
current_version = "1.
|
|
49
|
+
current_version = "1.3.0"
|
|
50
50
|
version_pattern = "MAJOR.MINOR.PATCH[PYTAGNUM]"
|
|
51
51
|
commit_message = "chore: bump version {old_version} -> {new_version}"
|
|
52
52
|
commit = true
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "1.3.0"
|
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
import threading
|
|
2
2
|
from typing import Dict, List, Optional
|
|
3
3
|
|
|
4
|
-
from earthscope_sdk.client.dropoff.
|
|
5
|
-
|
|
4
|
+
from earthscope_sdk.client.dropoff.models import (
|
|
5
|
+
DropoffObject,
|
|
6
|
+
DropoffSummary,
|
|
7
|
+
UploadStatus,
|
|
8
|
+
)
|
|
6
9
|
from rich import print
|
|
7
10
|
from rich.console import Group
|
|
8
11
|
from rich.live import Live
|
|
@@ -40,6 +43,10 @@ def _colorize_status(status: str) -> Text:
|
|
|
40
43
|
return Text("ACCEPTED", style="green")
|
|
41
44
|
if s == "FAILED":
|
|
42
45
|
return Text("FAILED", style="red")
|
|
46
|
+
if s == "COPYING":
|
|
47
|
+
return Text("COPYING", style="yellow")
|
|
48
|
+
if s == "LOADING":
|
|
49
|
+
return Text("LOADING", style="yellow")
|
|
43
50
|
if s == "INTERNAL_ERROR":
|
|
44
51
|
return Text("INTERNAL_ERROR", style="red")
|
|
45
52
|
return Text(f"{status or '-'}", style="dim")
|
|
@@ -373,7 +380,11 @@ class ConcurrentUploadProgressDisplay:
|
|
|
373
380
|
|
|
374
381
|
if self.live:
|
|
375
382
|
# Update final status
|
|
376
|
-
if
|
|
383
|
+
if (
|
|
384
|
+
exc_type is None
|
|
385
|
+
and self.overall_task is not None
|
|
386
|
+
and self.num_valid > 0
|
|
387
|
+
):
|
|
377
388
|
self.overall_progress.update(
|
|
378
389
|
self.overall_task,
|
|
379
390
|
description=f"[bold green]✓ Completed {self.num_valid} uploads",
|
|
@@ -526,7 +537,10 @@ class ConcurrentUploadProgressDisplay:
|
|
|
526
537
|
self.overall_progress.advance(self.overall_task, 1)
|
|
527
538
|
|
|
528
539
|
# If all files are complete, stop the timer to freeze throughput
|
|
529
|
-
if
|
|
540
|
+
if (
|
|
541
|
+
self.validation_complete
|
|
542
|
+
and len(self.completed_files) == self.num_valid
|
|
543
|
+
):
|
|
530
544
|
self.overall_progress.stop_task(self.overall_task)
|
|
531
545
|
|
|
532
546
|
# For large uploads, hide completed progress bars after a delay
|
|
@@ -4,22 +4,14 @@ from typing import Annotated, Iterable
|
|
|
4
4
|
|
|
5
5
|
import typer
|
|
6
6
|
from earthscope_sdk import EarthScopeClient
|
|
7
|
-
from earthscope_sdk.client.dropoff.
|
|
8
|
-
from earthscope_sdk.client.dropoff.validation import Validator
|
|
7
|
+
from earthscope_sdk.client.dropoff.enums import DropoffCategory
|
|
9
8
|
from rich import print
|
|
10
9
|
from typer_di import Depends, TyperDI
|
|
11
10
|
|
|
12
11
|
from earthscope_cli.common import get_sdk_refreshed
|
|
13
|
-
from earthscope_cli.dropoff.render import (
|
|
14
|
-
ConcurrentUploadProgressDisplay,
|
|
15
|
-
UploadSummary,
|
|
16
|
-
print_dropoff_objects_table,
|
|
17
|
-
print_dropoff_summary,
|
|
18
|
-
)
|
|
19
12
|
from earthscope_cli.dropoff.util import (
|
|
20
13
|
DropoffOptions,
|
|
21
14
|
FilterOptions,
|
|
22
|
-
collect_files,
|
|
23
15
|
get_category,
|
|
24
16
|
get_dropoff_options,
|
|
25
17
|
get_filter_options,
|
|
@@ -52,6 +44,9 @@ def get_object_history(
|
|
|
52
44
|
es dropoff get-object-history -c miniseed -k demo/SAML.IU.2025.303
|
|
53
45
|
es dropoff get-object-history -c miniseed -k demo/SAML.IU.2025.303 --offset 100 --limit 100
|
|
54
46
|
"""
|
|
47
|
+
# lazy import
|
|
48
|
+
from earthscope_cli.dropoff.render import print_dropoff_objects_table
|
|
49
|
+
|
|
55
50
|
result = sdk.dropoff.get_object_history(
|
|
56
51
|
category=options.category.value,
|
|
57
52
|
key=key,
|
|
@@ -90,6 +85,9 @@ def get_summary(
|
|
|
90
85
|
es dropoff get-summary -c miniseed --prefix demo/ --submitted-after 2025-01-01T00:00:00Z
|
|
91
86
|
es dropoff get-summary -c miniseed --prefix demo/ --submitted-before 2025-12-31T23:59:59Z
|
|
92
87
|
"""
|
|
88
|
+
# lazy import
|
|
89
|
+
from earthscope_cli.dropoff.render import print_dropoff_summary
|
|
90
|
+
|
|
93
91
|
result = sdk.dropoff.get_summary(
|
|
94
92
|
category=options.category.value,
|
|
95
93
|
prefix=prefix,
|
|
@@ -120,6 +118,9 @@ def list_objects(
|
|
|
120
118
|
es dropoff list-objects -c miniseed --submitted-after 2025-01-01T00:00:00Z
|
|
121
119
|
es dropoff list-objects -c miniseed --submitted-before 2025-12-31T23:59:59Z
|
|
122
120
|
"""
|
|
121
|
+
# lazy import
|
|
122
|
+
from earthscope_cli.dropoff.render import print_dropoff_objects_table
|
|
123
|
+
|
|
123
124
|
result = sdk.dropoff.list_objects(
|
|
124
125
|
category=options.category.value,
|
|
125
126
|
prefix=prefix,
|
|
@@ -249,6 +250,18 @@ def upload(
|
|
|
249
250
|
Multiple files:
|
|
250
251
|
es dropoff upload -c miniseed -d dest/prefix/ -s file1.mseed -s file2.mseed
|
|
251
252
|
"""
|
|
253
|
+
# lazy import
|
|
254
|
+
from earthscope_sdk.client.dropoff.files import (
|
|
255
|
+
UploadCandidate,
|
|
256
|
+
collect_files,
|
|
257
|
+
validate_files,
|
|
258
|
+
)
|
|
259
|
+
|
|
260
|
+
from earthscope_cli.dropoff.render import (
|
|
261
|
+
ConcurrentUploadProgressDisplay,
|
|
262
|
+
UploadSummary,
|
|
263
|
+
)
|
|
264
|
+
|
|
252
265
|
is_prefix = destination.endswith("/")
|
|
253
266
|
|
|
254
267
|
if not is_prefix and (len(source) > 1 or any(s.is_dir() for s in source)):
|
|
@@ -272,20 +285,20 @@ def upload(
|
|
|
272
285
|
# via asyncio.to_thread, so validation runs off the event loop and uploads
|
|
273
286
|
# can proceed concurrently with validation of remaining files.
|
|
274
287
|
validation_errors: list[str] = []
|
|
275
|
-
|
|
276
|
-
def _validated_files() -> Iterable[tuple[Path, str]]:
|
|
277
|
-
for p, k in collected:
|
|
278
|
-
try:
|
|
279
|
-
Validator(p, category=category).validate_all()
|
|
280
|
-
display.validation_passed()
|
|
281
|
-
yield (p, k)
|
|
282
|
-
except Exception as e:
|
|
283
|
-
validation_errors.append(str(e))
|
|
284
|
-
display.validation_failed()
|
|
285
|
-
display.finalize_validation()
|
|
286
|
-
|
|
287
288
|
display = ConcurrentUploadProgressDisplay(num_collected=total_collected)
|
|
288
289
|
|
|
290
|
+
def _validated_files() -> Iterable[UploadCandidate]:
|
|
291
|
+
try:
|
|
292
|
+
for item in validate_files(collected, category):
|
|
293
|
+
if item.ok:
|
|
294
|
+
display.validation_passed()
|
|
295
|
+
yield item
|
|
296
|
+
else:
|
|
297
|
+
validation_errors.append(str(item.error))
|
|
298
|
+
display.validation_failed()
|
|
299
|
+
finally:
|
|
300
|
+
display.finalize_validation()
|
|
301
|
+
|
|
289
302
|
try:
|
|
290
303
|
with display:
|
|
291
304
|
sdk.dropoff.put_objects(
|
|
@@ -1,13 +1,11 @@
|
|
|
1
1
|
import enum
|
|
2
2
|
from dataclasses import dataclass
|
|
3
3
|
from datetime import datetime
|
|
4
|
-
from
|
|
5
|
-
from typing import Annotated, Iterable, Optional
|
|
4
|
+
from typing import Annotated, Optional
|
|
6
5
|
|
|
7
6
|
import click
|
|
8
7
|
import typer
|
|
9
|
-
from earthscope_sdk.client.dropoff.
|
|
10
|
-
from rich import print
|
|
8
|
+
from earthscope_sdk.client.dropoff.enums import DropoffCategory
|
|
11
9
|
from typer_di import Depends
|
|
12
10
|
|
|
13
11
|
from earthscope_cli.common import get_sdk
|
|
@@ -31,72 +29,6 @@ class DropoffOptions(PaginationOptions):
|
|
|
31
29
|
category: DropoffCategory
|
|
32
30
|
|
|
33
31
|
|
|
34
|
-
def build_key(file_path: Path, source: Path, destination: str) -> str:
|
|
35
|
-
"""
|
|
36
|
-
Build the S3 object key based on source/destination semantics.
|
|
37
|
-
|
|
38
|
-
- destination ends with '/': it's a prefix, append relative filename
|
|
39
|
-
- destination does not end with '/': it's the exact key name
|
|
40
|
-
"""
|
|
41
|
-
if destination.endswith("/"):
|
|
42
|
-
if source.is_dir():
|
|
43
|
-
rel = file_path.relative_to(source)
|
|
44
|
-
return destination + rel.as_posix()
|
|
45
|
-
else:
|
|
46
|
-
return destination + file_path.name
|
|
47
|
-
else:
|
|
48
|
-
return destination
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
def collect_files(
|
|
52
|
-
sources: list[Path],
|
|
53
|
-
destination: str,
|
|
54
|
-
recursive: bool = False,
|
|
55
|
-
) -> Iterable[tuple[Path, str]]:
|
|
56
|
-
"""
|
|
57
|
-
Collect all files and their keys from a list of source paths.
|
|
58
|
-
|
|
59
|
-
Args:
|
|
60
|
-
sources: List of local file or directory paths to collect files from.
|
|
61
|
-
destination: Remote object key or prefix. Trailing '/' indicates a prefix.
|
|
62
|
-
recursive: If True, recursively traverse source directories.
|
|
63
|
-
|
|
64
|
-
Returns:
|
|
65
|
-
Iterable of tuples containing (file, key) pairs. When two distinct files
|
|
66
|
-
would be uploaded to the same destination key, the later one is skipped
|
|
67
|
-
with a warning.
|
|
68
|
-
"""
|
|
69
|
-
|
|
70
|
-
def _gen():
|
|
71
|
-
for src in sources:
|
|
72
|
-
if src.is_file():
|
|
73
|
-
yield (src, src, build_key(src, src, destination))
|
|
74
|
-
|
|
75
|
-
elif src.is_dir():
|
|
76
|
-
it = src.rglob("*") if recursive else src.iterdir()
|
|
77
|
-
yield from (
|
|
78
|
-
(src, f, build_key(f, src, destination))
|
|
79
|
-
for f in it
|
|
80
|
-
if f.is_file()
|
|
81
|
-
)
|
|
82
|
-
|
|
83
|
-
seen: dict[str, tuple[Path, Path]] = {}
|
|
84
|
-
for src, file_path, key in _gen():
|
|
85
|
-
existing = seen.get(key)
|
|
86
|
-
if existing is None:
|
|
87
|
-
seen[key] = (src, file_path)
|
|
88
|
-
yield (file_path, key)
|
|
89
|
-
elif existing[1] == file_path:
|
|
90
|
-
continue
|
|
91
|
-
else:
|
|
92
|
-
existing_src, existing_file = existing
|
|
93
|
-
print(
|
|
94
|
-
f"[yellow]Skipping '{file_path}' from source '{src}': destination "
|
|
95
|
-
f"'{key}' already produced by source '{existing_src}' "
|
|
96
|
-
f"(from '{existing_file}')."
|
|
97
|
-
)
|
|
98
|
-
|
|
99
|
-
|
|
100
32
|
def get_category(
|
|
101
33
|
sdk=Depends(get_sdk),
|
|
102
34
|
category: Annotated[
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: earthscope-cli
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.3.0
|
|
4
4
|
Summary: A CLI for the EarthScope API
|
|
5
5
|
Author-email: EarthScope <data-help@earthscope.org>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -15,7 +15,7 @@ Description-Content-Type: text/markdown
|
|
|
15
15
|
License-File: LICENSE
|
|
16
16
|
Requires-Dist: typer<0.24.0
|
|
17
17
|
Requires-Dist: typer-di>=0.1.3
|
|
18
|
-
Requires-Dist: earthscope-sdk>=1.
|
|
18
|
+
Requires-Dist: earthscope-sdk>=1.10.0
|
|
19
19
|
Requires-Dist: rich>=13.8.0
|
|
20
20
|
Provides-Extra: dev
|
|
21
21
|
Requires-Dist: black; extra == "dev"
|
|
@@ -7,7 +7,6 @@ verifying that arguments are correctly passed to the SDK.
|
|
|
7
7
|
The bulk of the upload logic is tested in the SDK tests, so these tests
|
|
8
8
|
focus on:
|
|
9
9
|
- CLI argument parsing and validation
|
|
10
|
-
- File collection and key generation
|
|
11
10
|
- Progress display and rendering
|
|
12
11
|
- Error handling and user feedback
|
|
13
12
|
"""
|
|
@@ -24,11 +23,7 @@ from earthscope_sdk.client.dropoff.models import (
|
|
|
24
23
|
)
|
|
25
24
|
|
|
26
25
|
from earthscope_cli.dropoff.typer import app
|
|
27
|
-
from earthscope_cli.dropoff.util import
|
|
28
|
-
build_key,
|
|
29
|
-
collect_files,
|
|
30
|
-
safe_str,
|
|
31
|
-
)
|
|
26
|
+
from earthscope_cli.dropoff.util import safe_str
|
|
32
27
|
from tests.conftest import runner
|
|
33
28
|
|
|
34
29
|
|
|
@@ -87,7 +82,7 @@ class TestUploadCommand:
|
|
|
87
82
|
mock_validator = MagicMock()
|
|
88
83
|
mock_validator.validate_all = Mock()
|
|
89
84
|
monkeypatch.setattr(
|
|
90
|
-
"
|
|
85
|
+
"earthscope_sdk.client.dropoff.files.Validator",
|
|
91
86
|
Mock(return_value=mock_validator),
|
|
92
87
|
)
|
|
93
88
|
|
|
@@ -112,9 +107,10 @@ class TestUploadCommand:
|
|
|
112
107
|
|
|
113
108
|
# Verify files were collected correctly
|
|
114
109
|
assert len(captured_objects) == 1
|
|
115
|
-
|
|
116
|
-
assert
|
|
117
|
-
assert key == "dest/prefix/test.mseed"
|
|
110
|
+
item = captured_objects[0]
|
|
111
|
+
assert item.path == test_file
|
|
112
|
+
assert item.key == "dest/prefix/test.mseed"
|
|
113
|
+
assert item.ok
|
|
118
114
|
|
|
119
115
|
def test_upload_single_file_exact_key(self, tmp_path, monkeypatch):
|
|
120
116
|
"""Test uploading a single file with exact key destination (no trailing /)."""
|
|
@@ -128,7 +124,7 @@ class TestUploadCommand:
|
|
|
128
124
|
mock_validator = MagicMock()
|
|
129
125
|
mock_validator.validate_all = Mock()
|
|
130
126
|
monkeypatch.setattr(
|
|
131
|
-
"
|
|
127
|
+
"earthscope_sdk.client.dropoff.files.Validator",
|
|
132
128
|
Mock(return_value=mock_validator),
|
|
133
129
|
)
|
|
134
130
|
|
|
@@ -142,8 +138,8 @@ class TestUploadCommand:
|
|
|
142
138
|
_, kwargs = mock_sdk.dropoff.put_objects.call_args
|
|
143
139
|
files = list(kwargs["objects"])
|
|
144
140
|
assert len(files) == 1
|
|
145
|
-
|
|
146
|
-
assert
|
|
141
|
+
assert files[0].key == "exact/keyname"
|
|
142
|
+
assert files[0].ok
|
|
147
143
|
|
|
148
144
|
def test_upload_directory_recursively(self, tmp_path, monkeypatch):
|
|
149
145
|
"""Test uploading a directory recursively with -r flag."""
|
|
@@ -169,7 +165,7 @@ class TestUploadCommand:
|
|
|
169
165
|
mock_validator = MagicMock()
|
|
170
166
|
mock_validator.validate_all = Mock()
|
|
171
167
|
monkeypatch.setattr(
|
|
172
|
-
"
|
|
168
|
+
"earthscope_sdk.client.dropoff.files.Validator",
|
|
173
169
|
Mock(return_value=mock_validator),
|
|
174
170
|
)
|
|
175
171
|
|
|
@@ -189,7 +185,7 @@ class TestUploadCommand:
|
|
|
189
185
|
assert len(captured_objects) == 3
|
|
190
186
|
|
|
191
187
|
# Verify keys include directory structure
|
|
192
|
-
keys = {
|
|
188
|
+
keys = {item.key for item in captured_objects}
|
|
193
189
|
assert keys == {
|
|
194
190
|
"dest/file1.mseed",
|
|
195
191
|
"dest/file2.mseed",
|
|
@@ -211,7 +207,7 @@ class TestUploadCommand:
|
|
|
211
207
|
mock_validator = MagicMock()
|
|
212
208
|
mock_validator.validate_all = Mock()
|
|
213
209
|
monkeypatch.setattr(
|
|
214
|
-
"
|
|
210
|
+
"earthscope_sdk.client.dropoff.files.Validator",
|
|
215
211
|
Mock(return_value=mock_validator),
|
|
216
212
|
)
|
|
217
213
|
|
|
@@ -227,7 +223,7 @@ class TestUploadCommand:
|
|
|
227
223
|
files = list(kwargs["objects"])
|
|
228
224
|
# Only top-level file, not the one in subdir
|
|
229
225
|
assert len(files) == 1
|
|
230
|
-
keys = {
|
|
226
|
+
keys = {item.key for item in files}
|
|
231
227
|
assert keys == {"dest/file1.mseed"}
|
|
232
228
|
|
|
233
229
|
def test_upload_multiple_sources_to_prefix(self, tmp_path, monkeypatch):
|
|
@@ -254,7 +250,7 @@ class TestUploadCommand:
|
|
|
254
250
|
mock_validator = MagicMock()
|
|
255
251
|
mock_validator.validate_all = Mock()
|
|
256
252
|
monkeypatch.setattr(
|
|
257
|
-
"
|
|
253
|
+
"earthscope_sdk.client.dropoff.files.Validator",
|
|
258
254
|
Mock(return_value=mock_validator),
|
|
259
255
|
)
|
|
260
256
|
|
|
@@ -281,7 +277,7 @@ class TestUploadCommand:
|
|
|
281
277
|
assert len(captured_objects) == 2
|
|
282
278
|
|
|
283
279
|
# Verify keys include directory structure
|
|
284
|
-
keys = {
|
|
280
|
+
keys = {item.key for item in captured_objects}
|
|
285
281
|
assert keys == {
|
|
286
282
|
"dest/test1.mseed",
|
|
287
283
|
"dest/test2.mseed",
|
|
@@ -351,7 +347,7 @@ class TestUploadCommand:
|
|
|
351
347
|
mock_validator = MagicMock()
|
|
352
348
|
mock_validator.validate_all = Mock()
|
|
353
349
|
monkeypatch.setattr(
|
|
354
|
-
"
|
|
350
|
+
"earthscope_sdk.client.dropoff.files.Validator",
|
|
355
351
|
Mock(return_value=mock_validator),
|
|
356
352
|
)
|
|
357
353
|
|
|
@@ -406,7 +402,7 @@ class TestUploadCommand:
|
|
|
406
402
|
side_effect=ValueError("Invalid file format")
|
|
407
403
|
)
|
|
408
404
|
monkeypatch.setattr(
|
|
409
|
-
"
|
|
405
|
+
"earthscope_sdk.client.dropoff.files.Validator",
|
|
410
406
|
Mock(return_value=mock_validator),
|
|
411
407
|
)
|
|
412
408
|
|
|
@@ -980,248 +976,6 @@ class TestGetObjectHistoryCommand:
|
|
|
980
976
|
assert "legacy error" in normalized
|
|
981
977
|
|
|
982
978
|
|
|
983
|
-
class TestCollectFiles:
|
|
984
|
-
"""Test the collect_files utility function."""
|
|
985
|
-
|
|
986
|
-
@pytest.mark.parametrize(
|
|
987
|
-
"description,paths_to_create,sources_to_collect,destination,recursive,expected_file_count,expected_keys",
|
|
988
|
-
[
|
|
989
|
-
# Single file + prefix destination
|
|
990
|
-
(
|
|
991
|
-
"single file to prefix destination",
|
|
992
|
-
["demo/test.mseed"],
|
|
993
|
-
["demo/test.mseed"],
|
|
994
|
-
"dest/",
|
|
995
|
-
False,
|
|
996
|
-
1,
|
|
997
|
-
["dest/test.mseed"],
|
|
998
|
-
),
|
|
999
|
-
# Single file + exact key destination
|
|
1000
|
-
(
|
|
1001
|
-
"single file to exact key destination",
|
|
1002
|
-
["demo/test.mseed"],
|
|
1003
|
-
["demo/test.mseed"],
|
|
1004
|
-
"exact/keyname",
|
|
1005
|
-
False,
|
|
1006
|
-
1,
|
|
1007
|
-
["exact/keyname"],
|
|
1008
|
-
),
|
|
1009
|
-
# Multiple files + prefix destination
|
|
1010
|
-
(
|
|
1011
|
-
"multiple files to prefix destination",
|
|
1012
|
-
["project1/a.mseed", "project2/b.mseed"],
|
|
1013
|
-
["project1/a.mseed", "project2/b.mseed"],
|
|
1014
|
-
"dest/",
|
|
1015
|
-
False,
|
|
1016
|
-
2,
|
|
1017
|
-
["dest/a.mseed", "dest/b.mseed"],
|
|
1018
|
-
),
|
|
1019
|
-
# Directory + prefix + recursive
|
|
1020
|
-
(
|
|
1021
|
-
"directory recursive to prefix destination",
|
|
1022
|
-
[
|
|
1023
|
-
"demo/file1.mseed",
|
|
1024
|
-
"demo/sub1/file2.mseed",
|
|
1025
|
-
"demo/sub1/sub2/file3.mseed",
|
|
1026
|
-
],
|
|
1027
|
-
["demo"],
|
|
1028
|
-
"dest/",
|
|
1029
|
-
True,
|
|
1030
|
-
3,
|
|
1031
|
-
[
|
|
1032
|
-
"dest/file1.mseed",
|
|
1033
|
-
"dest/sub1/file2.mseed",
|
|
1034
|
-
"dest/sub1/sub2/file3.mseed",
|
|
1035
|
-
],
|
|
1036
|
-
),
|
|
1037
|
-
# Directory + prefix + non-recursive
|
|
1038
|
-
(
|
|
1039
|
-
"directory non-recursive to prefix destination",
|
|
1040
|
-
[
|
|
1041
|
-
"demo/file1.mseed",
|
|
1042
|
-
"demo/sub1/file2.mseed",
|
|
1043
|
-
],
|
|
1044
|
-
["demo"],
|
|
1045
|
-
"dest/",
|
|
1046
|
-
False,
|
|
1047
|
-
1,
|
|
1048
|
-
["dest/file1.mseed"],
|
|
1049
|
-
),
|
|
1050
|
-
# Flat directory + prefix
|
|
1051
|
-
(
|
|
1052
|
-
"flat directory to prefix destination",
|
|
1053
|
-
["demo/file1.mseed", "demo/file2.mseed"],
|
|
1054
|
-
["demo"],
|
|
1055
|
-
"output/",
|
|
1056
|
-
False,
|
|
1057
|
-
2,
|
|
1058
|
-
["output/file1.mseed", "output/file2.mseed"],
|
|
1059
|
-
),
|
|
1060
|
-
],
|
|
1061
|
-
)
|
|
1062
|
-
def test_collect_files_scenarios(
|
|
1063
|
-
self,
|
|
1064
|
-
tmp_path,
|
|
1065
|
-
description,
|
|
1066
|
-
paths_to_create,
|
|
1067
|
-
sources_to_collect,
|
|
1068
|
-
destination,
|
|
1069
|
-
recursive,
|
|
1070
|
-
expected_file_count,
|
|
1071
|
-
expected_keys,
|
|
1072
|
-
):
|
|
1073
|
-
"""Test file collection for various source/destination scenarios."""
|
|
1074
|
-
# Create all files
|
|
1075
|
-
for path in paths_to_create:
|
|
1076
|
-
full_path = tmp_path / path
|
|
1077
|
-
full_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1078
|
-
full_path.write_bytes(b"test data")
|
|
1079
|
-
|
|
1080
|
-
# Build the source paths (resolve to full paths)
|
|
1081
|
-
sources = [tmp_path / p for p in sources_to_collect]
|
|
1082
|
-
|
|
1083
|
-
# Collect files
|
|
1084
|
-
collected = list(collect_files(sources, destination, recursive))
|
|
1085
|
-
|
|
1086
|
-
# Verify count
|
|
1087
|
-
assert len(collected) == expected_file_count, (
|
|
1088
|
-
f"\n{description}\n"
|
|
1089
|
-
f"Expected {expected_file_count} files, got {len(collected)}"
|
|
1090
|
-
)
|
|
1091
|
-
|
|
1092
|
-
# Verify keys match expected patterns
|
|
1093
|
-
collected_keys = {key for _, key in collected}
|
|
1094
|
-
expected_keys_set = set(expected_keys)
|
|
1095
|
-
assert collected_keys == expected_keys_set, (
|
|
1096
|
-
f"\n{description}\n"
|
|
1097
|
-
f"Expected keys: {expected_keys_set}\n"
|
|
1098
|
-
f"Got keys: {collected_keys}"
|
|
1099
|
-
)
|
|
1100
|
-
|
|
1101
|
-
def test_collect_deduplicates_files(self, tmp_path):
|
|
1102
|
-
"""Test that duplicate files are not collected twice."""
|
|
1103
|
-
test_file = tmp_path / "demo" / "test.mseed"
|
|
1104
|
-
test_file.parent.mkdir()
|
|
1105
|
-
test_file.write_bytes(b"test data")
|
|
1106
|
-
|
|
1107
|
-
files = list(collect_files([test_file, test_file], "dest/"))
|
|
1108
|
-
|
|
1109
|
-
assert len(files) == 1
|
|
1110
|
-
assert files[0][0] == test_file
|
|
1111
|
-
|
|
1112
|
-
def test_collect_empty_directory(self, tmp_path):
|
|
1113
|
-
"""Test collecting from an empty directory returns no files."""
|
|
1114
|
-
empty_dir = tmp_path / "empty"
|
|
1115
|
-
empty_dir.mkdir()
|
|
1116
|
-
|
|
1117
|
-
files = list(collect_files([empty_dir], "dest/"))
|
|
1118
|
-
assert len(files) == 0
|
|
1119
|
-
|
|
1120
|
-
def test_collect_mixed_files_and_directory(self, tmp_path):
|
|
1121
|
-
"""Test uploading individual files and a directory to a prefix."""
|
|
1122
|
-
project1 = tmp_path / "project1"
|
|
1123
|
-
project1.mkdir()
|
|
1124
|
-
(project1 / "one.mseed").write_bytes(b"project1 data")
|
|
1125
|
-
|
|
1126
|
-
project2 = tmp_path / "project2"
|
|
1127
|
-
project2.mkdir()
|
|
1128
|
-
(project2 / "two.mseed").write_bytes(b"project2 data")
|
|
1129
|
-
(project2 / "extra.mseed").write_bytes(b"extra data")
|
|
1130
|
-
|
|
1131
|
-
sources = [
|
|
1132
|
-
project1 / "one.mseed",
|
|
1133
|
-
project2,
|
|
1134
|
-
]
|
|
1135
|
-
|
|
1136
|
-
collected = list(collect_files(sources, "upload/", recursive=False))
|
|
1137
|
-
|
|
1138
|
-
assert len(collected) == 3
|
|
1139
|
-
|
|
1140
|
-
keys = {key for _, key in collected}
|
|
1141
|
-
assert keys == {
|
|
1142
|
-
"upload/one.mseed",
|
|
1143
|
-
"upload/two.mseed",
|
|
1144
|
-
"upload/extra.mseed",
|
|
1145
|
-
}
|
|
1146
|
-
|
|
1147
|
-
def test_collect_skips_duplicate_destination_between_files(self, tmp_path, capsys):
|
|
1148
|
-
"""Two distinct files producing the same key: keep the first, skip the rest."""
|
|
1149
|
-
p1 = tmp_path / "project1" / "data.mseed"
|
|
1150
|
-
p1.parent.mkdir()
|
|
1151
|
-
p1.write_bytes(b"a")
|
|
1152
|
-
|
|
1153
|
-
p2 = tmp_path / "project2" / "data.mseed"
|
|
1154
|
-
p2.parent.mkdir()
|
|
1155
|
-
p2.write_bytes(b"b")
|
|
1156
|
-
|
|
1157
|
-
collected = list(collect_files([p1, p2], "dest/"))
|
|
1158
|
-
|
|
1159
|
-
assert collected == [(p1, "dest/data.mseed")]
|
|
1160
|
-
out = capsys.readouterr().out
|
|
1161
|
-
assert "Skipping" in out
|
|
1162
|
-
assert "dest/data.mseed" in out
|
|
1163
|
-
|
|
1164
|
-
def test_collect_skips_duplicate_destination_between_file_and_dir(
|
|
1165
|
-
self, tmp_path, capsys
|
|
1166
|
-
):
|
|
1167
|
-
"""File + directory collision: skip the colliding directory entry but
|
|
1168
|
-
still include sibling directory entries."""
|
|
1169
|
-
lone = tmp_path / "lone" / "results.mseed"
|
|
1170
|
-
lone.parent.mkdir()
|
|
1171
|
-
lone.write_bytes(b"a")
|
|
1172
|
-
|
|
1173
|
-
other = tmp_path / "other"
|
|
1174
|
-
other.mkdir()
|
|
1175
|
-
(other / "results.mseed").write_bytes(b"b")
|
|
1176
|
-
(other / "extra.mseed").write_bytes(b"c")
|
|
1177
|
-
|
|
1178
|
-
collected = list(collect_files([lone, other], "upload/"))
|
|
1179
|
-
|
|
1180
|
-
kept = {k: p for p, k in collected}
|
|
1181
|
-
assert set(kept) == {"upload/results.mseed", "upload/extra.mseed"}
|
|
1182
|
-
assert kept["upload/results.mseed"] == lone
|
|
1183
|
-
|
|
1184
|
-
out = capsys.readouterr().out
|
|
1185
|
-
assert "Skipping" in out
|
|
1186
|
-
assert "upload/results.mseed" in out
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
class TestBuildKey:
|
|
1190
|
-
"""Test the build_key utility function."""
|
|
1191
|
-
|
|
1192
|
-
def test_file_with_exact_destination(self, tmp_path):
|
|
1193
|
-
"""Test: file source + exact key destination."""
|
|
1194
|
-
f = tmp_path / "data.mseed"
|
|
1195
|
-
|
|
1196
|
-
key = build_key(f, f, "destination/keyname")
|
|
1197
|
-
assert key == "destination/keyname"
|
|
1198
|
-
|
|
1199
|
-
def test_file_with_prefix_destination(self, tmp_path):
|
|
1200
|
-
"""Test: file source + prefix destination (trailing /)."""
|
|
1201
|
-
f = tmp_path / "data.mseed"
|
|
1202
|
-
|
|
1203
|
-
key = build_key(f, f, "destination/prefix/")
|
|
1204
|
-
assert key == "destination/prefix/data.mseed"
|
|
1205
|
-
|
|
1206
|
-
def test_dir_file_with_prefix_destination(self, tmp_path):
|
|
1207
|
-
"""Test: file from directory source + prefix destination."""
|
|
1208
|
-
d = tmp_path / "mydir"
|
|
1209
|
-
f = d / "data.mseed"
|
|
1210
|
-
|
|
1211
|
-
key = build_key(f, d, "destination/prefix/")
|
|
1212
|
-
assert key == "destination/prefix/data.mseed"
|
|
1213
|
-
|
|
1214
|
-
def test_nested_dir_file_with_prefix_destination(self, tmp_path):
|
|
1215
|
-
"""Test: deeply nested file from directory source + prefix destination."""
|
|
1216
|
-
d = tmp_path / "mydir"
|
|
1217
|
-
nested = d / "sub1" / "sub2"
|
|
1218
|
-
nested.mkdir(parents=True)
|
|
1219
|
-
f = nested / "data.mseed"
|
|
1220
|
-
|
|
1221
|
-
key = build_key(f, d, "destination/prefix/")
|
|
1222
|
-
assert key == "destination/prefix/sub1/sub2/data.mseed"
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
979
|
class TestSafeStr:
|
|
1226
980
|
"""Test the safe_str utility function."""
|
|
1227
981
|
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Guards on what `es` loads before it does anything.
|
|
3
|
+
|
|
4
|
+
Startup cost is dominated by the import graph, and it regresses quietly: a
|
|
5
|
+
convenience re-export imported from the wrong module drags a whole dependency
|
|
6
|
+
tree in behind it.
|
|
7
|
+
|
|
8
|
+
This runs in a fresh interpreter -- under pytest most of this is already
|
|
9
|
+
imported, so an in-process check would pass regardless.
|
|
10
|
+
|
|
11
|
+
The SDK guards its own eager import path in its suite; this covers what the
|
|
12
|
+
CLI adds on top.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
import subprocess
|
|
16
|
+
import sys
|
|
17
|
+
|
|
18
|
+
import pytest
|
|
19
|
+
|
|
20
|
+
# Pulled in only by paths that actually talk to S3, parse data, run the event
|
|
21
|
+
# loop, or draw an upload display -- never merely by importing the CLI.
|
|
22
|
+
#
|
|
23
|
+
# rich's progress machinery is the subtle one: `--help` loads rich.console and
|
|
24
|
+
# rich.table when it renders, so the win here is on every *other* invocation.
|
|
25
|
+
DEFERRED = frozenset(
|
|
26
|
+
{
|
|
27
|
+
"aioboto3",
|
|
28
|
+
"aiobotocore",
|
|
29
|
+
"aiohttp",
|
|
30
|
+
"asyncio",
|
|
31
|
+
"boto3",
|
|
32
|
+
"botocore",
|
|
33
|
+
"httpx",
|
|
34
|
+
"lxml",
|
|
35
|
+
"pydantic",
|
|
36
|
+
"pymseed",
|
|
37
|
+
"rich.live",
|
|
38
|
+
"rich.progress",
|
|
39
|
+
"rich.table",
|
|
40
|
+
"ssl",
|
|
41
|
+
}
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _modules_after(statement: str) -> set:
|
|
46
|
+
"""
|
|
47
|
+
Module names present in a fresh, isolated interpreter after `statement`.
|
|
48
|
+
|
|
49
|
+
`-I` ignores PYTHONPATH and user site-packages, keeping whatever the
|
|
50
|
+
surrounding environment injects out of the measurement. Running the suite
|
|
51
|
+
under VS Code's debugger is the case that matters: it attaches to
|
|
52
|
+
subprocesses by default, and the debugger imports `ssl`.
|
|
53
|
+
"""
|
|
54
|
+
result = subprocess.run(
|
|
55
|
+
[
|
|
56
|
+
sys.executable,
|
|
57
|
+
"-I",
|
|
58
|
+
"-c",
|
|
59
|
+
f"{statement}\nimport sys\nprint('\\n'.join(sys.modules))",
|
|
60
|
+
],
|
|
61
|
+
capture_output=True,
|
|
62
|
+
text=True,
|
|
63
|
+
check=True,
|
|
64
|
+
)
|
|
65
|
+
return set(result.stdout.split())
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _leaked_by(statement: str) -> list:
|
|
69
|
+
"""Which deferred modules `statement` pulls into a bare interpreter"""
|
|
70
|
+
preloaded = sorted(DEFERRED & _modules_after(""))
|
|
71
|
+
if preloaded:
|
|
72
|
+
pytest.skip(
|
|
73
|
+
f"this environment preloads {preloaded} into a bare interpreter, so "
|
|
74
|
+
"the guard cannot attribute imports to the CLI. Run the suite "
|
|
75
|
+
"without a debugger attached."
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
return sorted(DEFERRED & _modules_after(statement))
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def test_cli_startup_stays_cheap():
|
|
82
|
+
leaked = _leaked_by("import earthscope_cli.main")
|
|
83
|
+
|
|
84
|
+
assert not leaked, (
|
|
85
|
+
f"importing the CLI now pulls in {leaked}, which it did not before. "
|
|
86
|
+
"Something on the startup path imports one of these at module scope -- "
|
|
87
|
+
"often a symbol imported from a heavy module that also re-exports it."
|
|
88
|
+
)
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "1.2.0"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{earthscope_cli-1.2.0 → earthscope_cli-1.3.0}/src/earthscope_cli.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|