hyperstudy 0.2.2__py3-none-any.whl → 0.2.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hyperstudy/__init__.py +1 -1
- hyperstudy/_downloads.py +31 -5
- hyperstudy/_http.py +10 -3
- hyperstudy/_pagination.py +3 -2
- hyperstudy/client.py +36 -10
- {hyperstudy-0.2.2.dist-info → hyperstudy-0.2.3.dist-info}/METADATA +1 -1
- hyperstudy-0.2.3.dist-info/RECORD +14 -0
- hyperstudy-0.2.2.dist-info/RECORD +0 -14
- {hyperstudy-0.2.2.dist-info → hyperstudy-0.2.3.dist-info}/WHEEL +0 -0
- {hyperstudy-0.2.2.dist-info → hyperstudy-0.2.3.dist-info}/licenses/LICENSE +0 -0
hyperstudy/__init__.py
CHANGED
hyperstudy/_downloads.py
CHANGED
|
@@ -38,13 +38,39 @@ def build_filename(recording: dict[str, Any]) -> str:
|
|
|
38
38
|
|
|
39
39
|
|
|
40
40
|
def download_file(url: str, dest: Path, timeout: int = 300) -> int:
|
|
41
|
-
"""Stream-download *url* to *dest* and return bytes written.
|
|
41
|
+
"""Stream-download *url* to *dest* and return bytes written.
|
|
42
|
+
|
|
43
|
+
Verifies ``Content-Length`` (when the server provides it) against bytes
|
|
44
|
+
actually written. On any error — HTTP, mid-stream disconnect, or length
|
|
45
|
+
mismatch — the partial file on disk is removed so ``skip_existing``
|
|
46
|
+
logic won't mistake it for a complete download on retry.
|
|
47
|
+
"""
|
|
42
48
|
resp = requests.get(url, stream=True, timeout=timeout)
|
|
43
49
|
resp.raise_for_status()
|
|
44
50
|
|
|
51
|
+
expected: int | None
|
|
52
|
+
if "Content-Length" in resp.headers:
|
|
53
|
+
try:
|
|
54
|
+
expected = int(resp.headers["Content-Length"])
|
|
55
|
+
except (TypeError, ValueError):
|
|
56
|
+
expected = None
|
|
57
|
+
else:
|
|
58
|
+
expected = None
|
|
59
|
+
|
|
45
60
|
written = 0
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
61
|
+
# BaseException so Ctrl-C also cleans up, not just exceptions.
|
|
62
|
+
try:
|
|
63
|
+
with open(dest, "wb") as fh:
|
|
64
|
+
for chunk in resp.iter_content(chunk_size=_CHUNK_SIZE):
|
|
65
|
+
fh.write(chunk)
|
|
66
|
+
written += len(chunk)
|
|
67
|
+
|
|
68
|
+
if expected is not None and written != expected:
|
|
69
|
+
raise IOError(
|
|
70
|
+
f"Truncated download: wrote {written} of {expected} bytes for {dest.name}"
|
|
71
|
+
)
|
|
72
|
+
except BaseException:
|
|
73
|
+
dest.unlink(missing_ok=True)
|
|
74
|
+
raise
|
|
75
|
+
|
|
50
76
|
return written
|
hyperstudy/_http.py
CHANGED
|
@@ -63,8 +63,14 @@ class HttpTransport:
|
|
|
63
63
|
# Public helpers
|
|
64
64
|
# ------------------------------------------------------------------
|
|
65
65
|
|
|
66
|
-
def get(
|
|
67
|
-
|
|
66
|
+
def get(
|
|
67
|
+
self,
|
|
68
|
+
path: str,
|
|
69
|
+
params: dict[str, Any] | None = None,
|
|
70
|
+
*,
|
|
71
|
+
timeout: int | float | None = None,
|
|
72
|
+
) -> dict:
|
|
73
|
+
return self._request("GET", path, params=params, timeout=timeout)
|
|
68
74
|
|
|
69
75
|
def post(self, path: str, json: dict[str, Any] | None = None) -> dict:
|
|
70
76
|
return self._request("POST", path, json=json)
|
|
@@ -81,7 +87,8 @@ class HttpTransport:
|
|
|
81
87
|
|
|
82
88
|
def _request(self, method: str, path: str, **kwargs) -> dict:
|
|
83
89
|
url = f"{self.base_url}/{path.lstrip('/')}"
|
|
84
|
-
kwargs.
|
|
90
|
+
if kwargs.get("timeout") is None:
|
|
91
|
+
kwargs["timeout"] = self.timeout
|
|
85
92
|
|
|
86
93
|
resp = self._session.request(method, url, **kwargs)
|
|
87
94
|
return self._handle_response(resp)
|
hyperstudy/_pagination.py
CHANGED
|
@@ -16,6 +16,7 @@ def fetch_all_pages(
|
|
|
16
16
|
*,
|
|
17
17
|
page_size: int = 1000,
|
|
18
18
|
progress: bool = True,
|
|
19
|
+
timeout: int | float | None = None,
|
|
19
20
|
) -> tuple[list[dict], dict]:
|
|
20
21
|
"""Fetch every page of a paginated endpoint.
|
|
21
22
|
|
|
@@ -28,7 +29,7 @@ def fetch_all_pages(
|
|
|
28
29
|
params.setdefault("offset", 0)
|
|
29
30
|
|
|
30
31
|
# First request to learn total
|
|
31
|
-
body = transport.get(path, params=params)
|
|
32
|
+
body = transport.get(path, params=params, timeout=timeout)
|
|
32
33
|
metadata = body.get("metadata", {})
|
|
33
34
|
pagination = metadata.get("pagination", {})
|
|
34
35
|
total = pagination.get("total", 0)
|
|
@@ -43,7 +44,7 @@ def fetch_all_pages(
|
|
|
43
44
|
while has_more:
|
|
44
45
|
params["offset"] = pagination.get("nextOffset", params["offset"] + page_size)
|
|
45
46
|
|
|
46
|
-
body = transport.get(path, params=params)
|
|
47
|
+
body = transport.get(path, params=params, timeout=timeout)
|
|
47
48
|
metadata = body.get("metadata", {})
|
|
48
49
|
pagination = metadata.get("pagination", {})
|
|
49
50
|
|
hyperstudy/client.py
CHANGED
|
@@ -13,6 +13,7 @@ from ._downloads import build_filename, download_file, get_download_url
|
|
|
13
13
|
from ._http import HttpTransport
|
|
14
14
|
from ._pagination import fetch_all_pages
|
|
15
15
|
from ._types import Scope
|
|
16
|
+
from .exceptions import HyperStudyError
|
|
16
17
|
from .experiments import ExperimentMixin
|
|
17
18
|
|
|
18
19
|
|
|
@@ -533,13 +534,24 @@ class HyperStudy(ExperimentMixin):
|
|
|
533
534
|
pandas DataFrame with recording metadata plus ``local_path``
|
|
534
535
|
and ``download_status`` columns.
|
|
535
536
|
"""
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
537
|
+
try:
|
|
538
|
+
# Signing per-recording GCS URLs scales with count; 30s default is too short.
|
|
539
|
+
recordings = self._fetch_data(
|
|
540
|
+
"recordings",
|
|
541
|
+
scope_id,
|
|
542
|
+
scope=scope,
|
|
543
|
+
deployment_id=deployment_id,
|
|
544
|
+
room_id=room_id,
|
|
545
|
+
output="dict",
|
|
546
|
+
timeout=300,
|
|
547
|
+
)
|
|
548
|
+
except Exception as exc:
|
|
549
|
+
raise HyperStudyError(
|
|
550
|
+
f"Failed to list recordings for {scope}={scope_id!r}: {exc}",
|
|
551
|
+
code=getattr(exc, "code", "LIST_RECORDINGS_FAILED"),
|
|
552
|
+
status_code=getattr(exc, "status_code", None),
|
|
553
|
+
details=getattr(exc, "details", None),
|
|
554
|
+
) from exc
|
|
543
555
|
|
|
544
556
|
if recording_type:
|
|
545
557
|
recordings = [
|
|
@@ -566,7 +578,9 @@ class HyperStudy(ExperimentMixin):
|
|
|
566
578
|
|
|
567
579
|
if skip_existing and dest.exists():
|
|
568
580
|
expected_size = rec.get("fileSize")
|
|
569
|
-
|
|
581
|
+
# Require positive size match; missing size → re-download to
|
|
582
|
+
# avoid trusting a truncated file from a prior failed run.
|
|
583
|
+
if expected_size is not None and dest.stat().st_size == expected_size:
|
|
570
584
|
local_paths.append(str(dest.resolve()))
|
|
571
585
|
statuses.append("skipped")
|
|
572
586
|
continue
|
|
@@ -582,6 +596,13 @@ class HyperStudy(ExperimentMixin):
|
|
|
582
596
|
f"Failed to download recording {rec.get('recordingId')}: {exc}"
|
|
583
597
|
)
|
|
584
598
|
|
|
599
|
+
failed_count = sum(1 for s in statuses if s == "failed")
|
|
600
|
+
if failed_count:
|
|
601
|
+
warnings.warn(
|
|
602
|
+
f"{failed_count}/{len(recordings)} recordings failed to download; "
|
|
603
|
+
"see the 'download_status' column for per-file results."
|
|
604
|
+
)
|
|
605
|
+
|
|
585
606
|
df = to_pandas(recordings)
|
|
586
607
|
if not df.empty:
|
|
587
608
|
df["local_path"] = local_paths
|
|
@@ -649,6 +670,7 @@ class HyperStudy(ExperimentMixin):
|
|
|
649
670
|
offset: int = 0,
|
|
650
671
|
output: str = "pandas",
|
|
651
672
|
progress: bool = True,
|
|
673
|
+
timeout: int | float | None = None,
|
|
652
674
|
**extra_params,
|
|
653
675
|
):
|
|
654
676
|
"""Generic data-fetching logic shared by all ``get_*`` methods."""
|
|
@@ -675,11 +697,15 @@ class HyperStudy(ExperimentMixin):
|
|
|
675
697
|
# Single page or all pages
|
|
676
698
|
if limit is not None:
|
|
677
699
|
params["limit"] = limit
|
|
678
|
-
body = self._transport.get(path, params=params)
|
|
700
|
+
body = self._transport.get(path, params=params, timeout=timeout)
|
|
679
701
|
data = body.get("data", [])
|
|
680
702
|
else:
|
|
681
703
|
data, _ = fetch_all_pages(
|
|
682
|
-
self._transport,
|
|
704
|
+
self._transport,
|
|
705
|
+
path,
|
|
706
|
+
params=params,
|
|
707
|
+
progress=progress,
|
|
708
|
+
timeout=timeout,
|
|
683
709
|
)
|
|
684
710
|
|
|
685
711
|
return self._convert_output(data, output)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
hyperstudy/__init__.py,sha256=-FMsxfDASUokTGR0-iES4eb-SUEjTuThGMqPIg-Imyw,733
|
|
2
|
+
hyperstudy/_dataframe.py,sha256=8pjuMG8phMv8vwdY7sgZ-aF_6_o0UMgP2XXNp5muZ0s,3620
|
|
3
|
+
hyperstudy/_display.py,sha256=5svr4NFOUJkVif57-xJzrXNOSdA5O9tKesu8UhE_jYo,1896
|
|
4
|
+
hyperstudy/_downloads.py,sha256=MKUjzz9UDfo0V-n22UCdLEH2-tQxBeJMN_JoJ2spIT8,2459
|
|
5
|
+
hyperstudy/_http.py,sha256=_RS0ZaRAcKvFQ2NF9II7PpcojE-lTYT6CslaJNlgfPQ,4569
|
|
6
|
+
hyperstudy/_pagination.py,sha256=2NbvNGW_jFQkYquSTi3SNkAmGgsbKop7J9LkGaZWwWA,1812
|
|
7
|
+
hyperstudy/_types.py,sha256=22JTOlM68eGJ4hg4OH-oT-hmARG2CgbhnERJ5hGxHk8,614
|
|
8
|
+
hyperstudy/client.py,sha256=raAEBNxRsFPr260JWOFlOZNwmFk2SB80L2bsMoojUt0,24376
|
|
9
|
+
hyperstudy/exceptions.py,sha256=z_1Ngiq-RU-1yyDI_JqvQ5CvYm2jQ_lyJ9TUbqUKKv8,1529
|
|
10
|
+
hyperstudy/experiments.py,sha256=BHNf-ctKUA9uzy8Kya11nIyHPrsmOslqC_IJ5t4LxKA,4372
|
|
11
|
+
hyperstudy-0.2.3.dist-info/METADATA,sha256=l1DluLkNI41NjQxrCEUAMeWMfi8jZFhXOUdC9RPb_f0,4833
|
|
12
|
+
hyperstudy-0.2.3.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
|
|
13
|
+
hyperstudy-0.2.3.dist-info/licenses/LICENSE,sha256=WJdwRhBIOiiQ6lyK4l95u5IJYuuT-XxzwYRraS31NME,1067
|
|
14
|
+
hyperstudy-0.2.3.dist-info/RECORD,,
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
hyperstudy/__init__.py,sha256=E8l_qKgFt0TPaZycaGM9L09ThfDSB30c4RqIS4IgolQ,733
|
|
2
|
-
hyperstudy/_dataframe.py,sha256=8pjuMG8phMv8vwdY7sgZ-aF_6_o0UMgP2XXNp5muZ0s,3620
|
|
3
|
-
hyperstudy/_display.py,sha256=5svr4NFOUJkVif57-xJzrXNOSdA5O9tKesu8UhE_jYo,1896
|
|
4
|
-
hyperstudy/_downloads.py,sha256=nqIig09P84XbYLBlI5uPMWtFHISGt9H4qhsf5415kdg,1563
|
|
5
|
-
hyperstudy/_http.py,sha256=wMVAhgxNcp4-zcrc4lTHU_GHGSqsFiyMFWsrLP5EGFM,4430
|
|
6
|
-
hyperstudy/_pagination.py,sha256=Fdr2nW84r6NtXiYQs5jyJEUOjFvXl-RclpDIi-SPDlM,1738
|
|
7
|
-
hyperstudy/_types.py,sha256=22JTOlM68eGJ4hg4OH-oT-hmARG2CgbhnERJ5hGxHk8,614
|
|
8
|
-
hyperstudy/client.py,sha256=MTw81lv1sP16wDMIpF4QaJ6fsNFJmNy62PaAcUAL69k,23202
|
|
9
|
-
hyperstudy/exceptions.py,sha256=z_1Ngiq-RU-1yyDI_JqvQ5CvYm2jQ_lyJ9TUbqUKKv8,1529
|
|
10
|
-
hyperstudy/experiments.py,sha256=BHNf-ctKUA9uzy8Kya11nIyHPrsmOslqC_IJ5t4LxKA,4372
|
|
11
|
-
hyperstudy-0.2.2.dist-info/METADATA,sha256=RSn3To1gXSS0SHnz-QXrtQc1uamcHNkPeDUuckeGSUI,4833
|
|
12
|
-
hyperstudy-0.2.2.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
|
|
13
|
-
hyperstudy-0.2.2.dist-info/licenses/LICENSE,sha256=WJdwRhBIOiiQ6lyK4l95u5IJYuuT-XxzwYRraS31NME,1067
|
|
14
|
-
hyperstudy-0.2.2.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|