gi-ingest 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gi_ingest-0.2.0/.gitignore +8 -0
- gi_ingest-0.2.0/PKG-INFO +156 -0
- gi_ingest-0.2.0/README.md +139 -0
- gi_ingest-0.2.0/gi_ingest/__init__.py +10 -0
- gi_ingest-0.2.0/gi_ingest/api.py +145 -0
- gi_ingest-0.2.0/gi_ingest/config.py +129 -0
- gi_ingest-0.2.0/gi_ingest/identity.py +74 -0
- gi_ingest-0.2.0/gi_ingest/ledger.py +509 -0
- gi_ingest-0.2.0/gi_ingest/main.py +930 -0
- gi_ingest-0.2.0/gi_ingest/manifest.py +71 -0
- gi_ingest-0.2.0/gi_ingest/s3.py +253 -0
- gi_ingest-0.2.0/gi_ingest/scanner.py +520 -0
- gi_ingest-0.2.0/gi_ingest/stager.py +137 -0
- gi_ingest-0.2.0/gi_ingest/version_gate.py +81 -0
- gi_ingest-0.2.0/pyproject.toml +45 -0
- gi_ingest-0.2.0/tests/test_card_discovery.py +127 -0
- gi_ingest-0.2.0/tests/test_card_identity.py +314 -0
- gi_ingest-0.2.0/tests/test_config_migration.py +119 -0
- gi_ingest-0.2.0/tests/test_ego_layout.py +593 -0
- gi_ingest-0.2.0/tests/test_identity.py +87 -0
- gi_ingest-0.2.0/tests/test_stage_and_manifest.py +230 -0
- gi_ingest-0.2.0/tests/test_upload.py +203 -0
- gi_ingest-0.2.0/tests/test_upload_resume.py +256 -0
- gi_ingest-0.2.0/tests/test_version_gate.py +81 -0
- gi_ingest-0.2.0/uv.lock +806 -0
gi_ingest-0.2.0/PKG-INFO
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: gi-ingest
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Offload robotics recordings from TF cards and deliver them to GILabs
|
|
5
|
+
License: Proprietary
|
|
6
|
+
Requires-Python: >=3.11
|
|
7
|
+
Requires-Dist: boto3>=1.34
|
|
8
|
+
Requires-Dist: httpx>=0.27
|
|
9
|
+
Requires-Dist: rich>=13.7
|
|
10
|
+
Requires-Dist: typer>=0.12
|
|
11
|
+
Provides-Extra: dev
|
|
12
|
+
Requires-Dist: moto[s3]>=5.0; extra == 'dev'
|
|
13
|
+
Requires-Dist: pytest-asyncio>=0.24; extra == 'dev'
|
|
14
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
15
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
|
|
18
|
+
# gi-ingest
|
|
19
|
+
|
|
20
|
+
Offload robotics recordings from TF cards and deliver them to GILabs.
|
|
21
|
+
|
|
22
|
+
## Install
|
|
23
|
+
|
|
24
|
+
**Not on PyPI yet** — there is no release workflow for this package (
|
|
25
|
+
`.github/workflows/cli-release.yml` publishes `gidata` only). Install from a
|
|
26
|
+
checkout:
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
git clone https://github.com/General-Intelligence-Labs/gilabs-data-center.git
|
|
30
|
+
uv tool install ./gilabs-data-center/vendor-cli # or: pipx install ./…/vendor-cli
|
|
31
|
+
gi-ingest --help
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Needs Python ≥ 3.11. Both commands put `gi-ingest` on your PATH in its own
|
|
35
|
+
isolated environment; `uv tool install --force …` upgrades in place after a
|
|
36
|
+
`git pull`.
|
|
37
|
+
|
|
38
|
+
Once the package is published this becomes `uv tool install gi-ingest` (or
|
|
39
|
+
`pipx install gi-ingest`) with no path — until then that command fails with
|
|
40
|
+
"No matching distribution found", which means it is not published, not that
|
|
41
|
+
anything is wrong with your setup.
|
|
42
|
+
|
|
43
|
+
Then authenticate:
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
gi-ingest login --api-key gik_...
|
|
47
|
+
gi-ingest doctor # verifies connectivity, clock skew, disk, credentials
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Get your token from the vendor portal at
|
|
51
|
+
[vendor.gilabs.xyz](https://vendor.gilabs.xyz) → **Upload tokens**.
|
|
52
|
+
|
|
53
|
+
## The three phases
|
|
54
|
+
|
|
55
|
+
They are three commands because they run on two different timescales, and fusing
|
|
56
|
+
them turns a 30-minute attended task into an all-night one.
|
|
57
|
+
|
|
58
|
+
| Phase | When | What it does |
|
|
59
|
+
|---|---|---|
|
|
60
|
+
| `stage` | attended, minutes per card | Copies cards to a local disk and verifies each copy |
|
|
61
|
+
| `upload` | unattended, hours | Pushes everything staged as **one delivery**, overnight |
|
|
62
|
+
| `reclaim` | attended, next morning | Wipes only the cards the server confirmed |
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
# 1. Drain every card. Swap cards as each finishes; nothing is wiped yet.
|
|
66
|
+
gi-ingest stage --device umi_gripper
|
|
67
|
+
gi-ingest stage --watch # or let it auto-copy on insert
|
|
68
|
+
|
|
69
|
+
# 2. Leave this running. One delivery, resumable, safe to nohup.
|
|
70
|
+
gi-ingest upload --max-bandwidth 50M
|
|
71
|
+
|
|
72
|
+
# 3. Next morning, with the cards back in the reader.
|
|
73
|
+
gi-ingest reclaim
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
### Why the wipe is a separate step
|
|
77
|
+
|
|
78
|
+
`upload` finishes hours after the cards were pulled, so it cannot wipe them —
|
|
79
|
+
they are back in the rig. `stage` writes a `.staged.json` marker and leaves the
|
|
80
|
+
card intact; `reclaim` deletes part files only for episodes the server reports as
|
|
81
|
+
delivered, and writes a tombstone so a re-scan skips them.
|
|
82
|
+
|
|
83
|
+
**Between `stage` and `reclaim`, the staging disk holds the only copy.** Size it
|
|
84
|
+
for a full collection round; `gi-ingest doctor` warns under 50 GiB free.
|
|
85
|
+
|
|
86
|
+
### Choosing where cards are copied to
|
|
87
|
+
|
|
88
|
+
By default cards land in `~/gilabs-staging`. That is rarely where you want a
|
|
89
|
+
multi-terabyte collection round, so it is settable two ways:
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
# Persist it — do this once per machine.
|
|
93
|
+
gi-ingest config set staging-dir /Volumes/BigDisk/gilabs
|
|
94
|
+
gi-ingest config show
|
|
95
|
+
|
|
96
|
+
# Or override for a single run, without changing the stored setting.
|
|
97
|
+
gi-ingest stage --staging /Volumes/OtherDisk/gilabs
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
`config set` also accepts `ledger-path`, `api-url` and `project-id`. Commands
|
|
101
|
+
that read the ledger take `--ledger` for a one-off override, which is how you
|
|
102
|
+
keep two independent offload runs from sharing state on one machine.
|
|
103
|
+
|
|
104
|
+
## Other commands
|
|
105
|
+
|
|
106
|
+
```bash
|
|
107
|
+
gi-ingest whoami # vendor, scopes, and where you may deliver
|
|
108
|
+
gi-ingest destinations # the org/project list, with ids for --project
|
|
109
|
+
gi-ingest doctor # token, connectivity, disk — run before a night
|
|
110
|
+
gi-ingest queue # what is staged / in flight / delivered
|
|
111
|
+
gi-ingest status # deliveries and their QC verdicts
|
|
112
|
+
gi-ingest status --batch btch_...
|
|
113
|
+
gi-ingest retry # re-queue anything that failed
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
## How a delivery works
|
|
117
|
+
|
|
118
|
+
1. `upload` opens a **batch** and receives AWS credentials scoped to that batch's
|
|
119
|
+
folder alone — they cannot read or write anything else, including your own
|
|
120
|
+
other deliveries.
|
|
121
|
+
2. Files go straight to S3. Credentials expire hourly and are re-minted
|
|
122
|
+
automatically, so a multi-hour run and a laptop that sleeps both just work.
|
|
123
|
+
3. `upload` **seals** the batch with a manifest listing every episode, part, size
|
|
124
|
+
and SHA-256. GILabs verifies every declared file is present at its declared
|
|
125
|
+
size before ingesting anything, so a half-finished upload can never be
|
|
126
|
+
processed as if it were complete.
|
|
127
|
+
|
|
128
|
+
Interrupted? Re-run `upload`. It re-uses the same open delivery and skips files
|
|
129
|
+
already sent — a restart costs a listing, not the bytes.
|
|
130
|
+
|
|
131
|
+
## Choosing a destination
|
|
132
|
+
|
|
133
|
+
Every delivery is filed against one project. With a single granted destination,
|
|
134
|
+
or one marked default, `upload` needs no flag. With several and no default it
|
|
135
|
+
will refuse to guess:
|
|
136
|
+
|
|
137
|
+
```bash
|
|
138
|
+
gi-ingest destinations
|
|
139
|
+
gi-ingest upload --project proj_abc123
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
## Duplicates
|
|
143
|
+
|
|
144
|
+
`stage` asks GILabs which episodes it already has before copying, so a card a
|
|
145
|
+
colleague already delivered costs one request instead of its bytes. If you are
|
|
146
|
+
offline it stages anyway and the duplicate is caught server-side at seal.
|
|
147
|
+
|
|
148
|
+
## Local development
|
|
149
|
+
|
|
150
|
+
```bash
|
|
151
|
+
uv venv && uv pip install -e ".[dev]"
|
|
152
|
+
uv run pytest
|
|
153
|
+
uv run ruff check gi_ingest
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
Design doc: [`docs/product-plans/vendor-data-ingestion.md`](../docs/product-plans/vendor-data-ingestion.md).
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
# gi-ingest
|
|
2
|
+
|
|
3
|
+
Offload robotics recordings from TF cards and deliver them to GILabs.
|
|
4
|
+
|
|
5
|
+
## Install
|
|
6
|
+
|
|
7
|
+
**Not on PyPI yet** — there is no release workflow for this package (
|
|
8
|
+
`.github/workflows/cli-release.yml` publishes `gidata` only). Install from a
|
|
9
|
+
checkout:
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
git clone https://github.com/General-Intelligence-Labs/gilabs-data-center.git
|
|
13
|
+
uv tool install ./gilabs-data-center/vendor-cli # or: pipx install ./…/vendor-cli
|
|
14
|
+
gi-ingest --help
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
Needs Python ≥ 3.11. Both commands put `gi-ingest` on your PATH in its own
|
|
18
|
+
isolated environment; `uv tool install --force …` upgrades in place after a
|
|
19
|
+
`git pull`.
|
|
20
|
+
|
|
21
|
+
Once the package is published this becomes `uv tool install gi-ingest` (or
|
|
22
|
+
`pipx install gi-ingest`) with no path — until then that command fails with
|
|
23
|
+
"No matching distribution found", which means it is not published, not that
|
|
24
|
+
anything is wrong with your setup.
|
|
25
|
+
|
|
26
|
+
Then authenticate:
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
gi-ingest login --api-key gik_...
|
|
30
|
+
gi-ingest doctor # verifies connectivity, clock skew, disk, credentials
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Get your token from the vendor portal at
|
|
34
|
+
[vendor.gilabs.xyz](https://vendor.gilabs.xyz) → **Upload tokens**.
|
|
35
|
+
|
|
36
|
+
## The three phases
|
|
37
|
+
|
|
38
|
+
They are three commands because they run on two different timescales, and fusing
|
|
39
|
+
them turns a 30-minute attended task into an all-night one.
|
|
40
|
+
|
|
41
|
+
| Phase | When | What it does |
|
|
42
|
+
|---|---|---|
|
|
43
|
+
| `stage` | attended, minutes per card | Copies cards to a local disk and verifies each copy |
|
|
44
|
+
| `upload` | unattended, hours | Pushes everything staged as **one delivery**, overnight |
|
|
45
|
+
| `reclaim` | attended, next morning | Wipes only the cards the server confirmed |
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
# 1. Drain every card. Swap cards as each finishes; nothing is wiped yet.
|
|
49
|
+
gi-ingest stage --device umi_gripper
|
|
50
|
+
gi-ingest stage --watch # or let it auto-copy on insert
|
|
51
|
+
|
|
52
|
+
# 2. Leave this running. One delivery, resumable, safe to nohup.
|
|
53
|
+
gi-ingest upload --max-bandwidth 50M
|
|
54
|
+
|
|
55
|
+
# 3. Next morning, with the cards back in the reader.
|
|
56
|
+
gi-ingest reclaim
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
### Why the wipe is a separate step
|
|
60
|
+
|
|
61
|
+
`upload` finishes hours after the cards were pulled, so it cannot wipe them —
|
|
62
|
+
they are back in the rig. `stage` writes a `.staged.json` marker and leaves the
|
|
63
|
+
card intact; `reclaim` deletes part files only for episodes the server reports as
|
|
64
|
+
delivered, and writes a tombstone so a re-scan skips them.
|
|
65
|
+
|
|
66
|
+
**Between `stage` and `reclaim`, the staging disk holds the only copy.** Size it
|
|
67
|
+
for a full collection round; `gi-ingest doctor` warns under 50 GiB free.
|
|
68
|
+
|
|
69
|
+
### Choosing where cards are copied to
|
|
70
|
+
|
|
71
|
+
By default cards land in `~/gilabs-staging`. That is rarely where you want a
|
|
72
|
+
multi-terabyte collection round, so it is settable two ways:
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
# Persist it — do this once per machine.
|
|
76
|
+
gi-ingest config set staging-dir /Volumes/BigDisk/gilabs
|
|
77
|
+
gi-ingest config show
|
|
78
|
+
|
|
79
|
+
# Or override for a single run, without changing the stored setting.
|
|
80
|
+
gi-ingest stage --staging /Volumes/OtherDisk/gilabs
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
`config set` also accepts `ledger-path`, `api-url` and `project-id`. Commands
|
|
84
|
+
that read the ledger take `--ledger` for a one-off override, which is how you
|
|
85
|
+
keep two independent offload runs from sharing state on one machine.
|
|
86
|
+
|
|
87
|
+
## Other commands
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
gi-ingest whoami # vendor, scopes, and where you may deliver
|
|
91
|
+
gi-ingest destinations # the org/project list, with ids for --project
|
|
92
|
+
gi-ingest doctor # token, connectivity, disk — run before a night
|
|
93
|
+
gi-ingest queue # what is staged / in flight / delivered
|
|
94
|
+
gi-ingest status # deliveries and their QC verdicts
|
|
95
|
+
gi-ingest status --batch btch_...
|
|
96
|
+
gi-ingest retry # re-queue anything that failed
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
## How a delivery works
|
|
100
|
+
|
|
101
|
+
1. `upload` opens a **batch** and receives AWS credentials scoped to that batch's
|
|
102
|
+
folder alone — they cannot read or write anything else, including your own
|
|
103
|
+
other deliveries.
|
|
104
|
+
2. Files go straight to S3. Credentials expire hourly and are re-minted
|
|
105
|
+
automatically, so a multi-hour run and a laptop that sleeps both just work.
|
|
106
|
+
3. `upload` **seals** the batch with a manifest listing every episode, part, size
|
|
107
|
+
and SHA-256. GILabs verifies every declared file is present at its declared
|
|
108
|
+
size before ingesting anything, so a half-finished upload can never be
|
|
109
|
+
processed as if it were complete.
|
|
110
|
+
|
|
111
|
+
Interrupted? Re-run `upload`. It re-uses the same open delivery and skips files
|
|
112
|
+
already sent — a restart costs a listing, not the bytes.
|
|
113
|
+
|
|
114
|
+
## Choosing a destination
|
|
115
|
+
|
|
116
|
+
Every delivery is filed against one project. With a single granted destination,
|
|
117
|
+
or one marked default, `upload` needs no flag. With several and no default it
|
|
118
|
+
will refuse to guess:
|
|
119
|
+
|
|
120
|
+
```bash
|
|
121
|
+
gi-ingest destinations
|
|
122
|
+
gi-ingest upload --project proj_abc123
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
## Duplicates
|
|
126
|
+
|
|
127
|
+
`stage` asks GILabs which episodes it already has before copying, so a card a
|
|
128
|
+
colleague already delivered costs one request instead of its bytes. If you are
|
|
129
|
+
offline it stages anyway and the duplicate is caught server-side at seal.
|
|
130
|
+
|
|
131
|
+
## Local development
|
|
132
|
+
|
|
133
|
+
```bash
|
|
134
|
+
uv venv && uv pip install -e ".[dev]"
|
|
135
|
+
uv run pytest
|
|
136
|
+
uv run ruff check gi_ingest
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
Design doc: [`docs/product-plans/vendor-data-ingestion.md`](../docs/product-plans/vendor-data-ingestion.md).
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""The vendor offload CLI."""
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
4
|
+
|
|
5
|
+
try:
|
|
6
|
+
__version__ = version("gi-ingest")
|
|
7
|
+
except PackageNotFoundError: # running from a source checkout, not installed
|
|
8
|
+
__version__ = "0.0.0+dev"
|
|
9
|
+
|
|
10
|
+
__all__ = ["__version__"]
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Client for the GILabs vendor ingest API.
|
|
3
|
+
|
|
4
|
+
Authenticates with a ``gik_`` key on ``X-API-Key``. There is no organization
|
|
5
|
+
header: the key is bound to one vendor, so there is no tenant to select.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import contextlib
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
import httpx
|
|
15
|
+
|
|
16
|
+
from gi_ingest import __version__
|
|
17
|
+
|
|
18
|
+
DEFAULT_API_URL = "https://gilabs-data-center-production.up.railway.app"
|
|
19
|
+
|
|
20
|
+
# Generous: seal validates a whole manifest against a bucket listing, which on a
|
|
21
|
+
# large delivery is not instant. Everything else is fast.
|
|
22
|
+
_TIMEOUT = httpx.Timeout(60.0, connect=15.0)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class ApiError(RuntimeError):
|
|
26
|
+
def __init__(self, status: int, detail: str) -> None:
|
|
27
|
+
super().__init__(f"[{status}] {detail}")
|
|
28
|
+
self.status = status
|
|
29
|
+
self.detail = detail
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass(frozen=True)
|
|
33
|
+
class ScopedCredentials:
|
|
34
|
+
access_key_id: str
|
|
35
|
+
secret_access_key: str
|
|
36
|
+
session_token: str
|
|
37
|
+
expiration: str
|
|
38
|
+
region: str
|
|
39
|
+
bucket: str
|
|
40
|
+
prefix: str
|
|
41
|
+
s3_uri: str
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class IngestClient:
|
|
45
|
+
def __init__(self, base_url: str, api_key: str) -> None:
|
|
46
|
+
self.base = base_url.rstrip("/")
|
|
47
|
+
self._client = httpx.Client(
|
|
48
|
+
headers={
|
|
49
|
+
"X-API-Key": api_key,
|
|
50
|
+
"Content-Type": "application/json",
|
|
51
|
+
# Both, because they answer different questions: the agent
|
|
52
|
+
# string is what shows up in a proxy log, and the explicit
|
|
53
|
+
# header is what a server can compare without parsing.
|
|
54
|
+
"User-Agent": f"gi-ingest/{__version__}",
|
|
55
|
+
"X-Client-Version": __version__,
|
|
56
|
+
},
|
|
57
|
+
timeout=_TIMEOUT,
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
def close(self) -> None:
|
|
61
|
+
self._client.close()
|
|
62
|
+
|
|
63
|
+
def __enter__(self) -> IngestClient:
|
|
64
|
+
return self
|
|
65
|
+
|
|
66
|
+
def __exit__(self, *_exc: object) -> None:
|
|
67
|
+
self.close()
|
|
68
|
+
|
|
69
|
+
def _request(self, method: str, path: str, **kwargs: Any) -> Any:
|
|
70
|
+
try:
|
|
71
|
+
r = self._client.request(method, f"{self.base}/api/v1{path}", **kwargs)
|
|
72
|
+
except httpx.HTTPError as exc:
|
|
73
|
+
raise ApiError(0, f"network error: {exc}") from exc
|
|
74
|
+
if r.status_code >= 400:
|
|
75
|
+
detail = r.text
|
|
76
|
+
with contextlib.suppress(Exception): # non-JSON error body
|
|
77
|
+
detail = r.json().get("detail", detail)
|
|
78
|
+
raise ApiError(r.status_code, str(detail))
|
|
79
|
+
if r.status_code == 204 or not r.content:
|
|
80
|
+
return None
|
|
81
|
+
return r.json()
|
|
82
|
+
|
|
83
|
+
# ---- identity + destinations ----
|
|
84
|
+
|
|
85
|
+
def whoami(self) -> dict:
|
|
86
|
+
return self._request("GET", "/ingest/whoami")
|
|
87
|
+
|
|
88
|
+
def destinations(self) -> list[dict]:
|
|
89
|
+
return self._request("GET", "/ingest/destinations")["destinations"]
|
|
90
|
+
|
|
91
|
+
def check_episodes(self, episode_ids: list[str]) -> dict[str, dict]:
|
|
92
|
+
"""Which of these episodes the platform already has.
|
|
93
|
+
|
|
94
|
+
Called at stage time, before anything is uploaded: on a bandwidth-bound
|
|
95
|
+
overnight run, transferring a card a colleague already delivered is the
|
|
96
|
+
expensive mistake. Returns a map keyed by episode id.
|
|
97
|
+
"""
|
|
98
|
+
if not episode_ids:
|
|
99
|
+
return {}
|
|
100
|
+
out: dict[str, dict] = {}
|
|
101
|
+
# The endpoint caps a request at 1000 ids.
|
|
102
|
+
for i in range(0, len(episode_ids), 1000):
|
|
103
|
+
chunk = episode_ids[i : i + 1000]
|
|
104
|
+
body = {"episodes": [{"episode_id": e} for e in chunk]}
|
|
105
|
+
resp = self._request("POST", "/ingest/episodes/check", json=body)
|
|
106
|
+
for result in resp["results"]:
|
|
107
|
+
out[result["episode_id"]] = result
|
|
108
|
+
return out
|
|
109
|
+
|
|
110
|
+
# ---- batches ----
|
|
111
|
+
|
|
112
|
+
def open_batch(self, *, project_id: str | None, device_kind: str | None) -> dict:
|
|
113
|
+
body: dict[str, Any] = {}
|
|
114
|
+
if project_id:
|
|
115
|
+
body["project_id"] = project_id
|
|
116
|
+
if device_kind:
|
|
117
|
+
body["device_kind"] = device_kind
|
|
118
|
+
return self._request("POST", "/ingest/batches", json=body)
|
|
119
|
+
|
|
120
|
+
def refresh_credentials(self, batch_id: str) -> ScopedCredentials:
|
|
121
|
+
raw = self._request("POST", f"/ingest/batches/{batch_id}/credentials")
|
|
122
|
+
return ScopedCredentials(**{k: raw[k] for k in ScopedCredentials.__annotations__})
|
|
123
|
+
|
|
124
|
+
def seal_batch(self, batch_id: str, manifest: dict) -> dict:
|
|
125
|
+
return self._request(
|
|
126
|
+
"POST", f"/ingest/batches/{batch_id}/seal", json={"manifest": manifest}
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
def get_batch(self, batch_id: str) -> dict:
|
|
130
|
+
return self._request("GET", f"/ingest/batches/{batch_id}")
|
|
131
|
+
|
|
132
|
+
def list_batches(self, *, status: str | None = None, limit: int = 50) -> list[dict]:
|
|
133
|
+
params: dict[str, Any] = {"limit": limit}
|
|
134
|
+
if status:
|
|
135
|
+
params["status"] = status
|
|
136
|
+
return self._request("GET", "/ingest/batches", params=params)["batches"]
|
|
137
|
+
|
|
138
|
+
def abandon_batch(self, batch_id: str) -> dict:
|
|
139
|
+
return self._request("DELETE", f"/ingest/batches/{batch_id}")
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def credentials_from_open(payload: dict) -> ScopedCredentials:
|
|
143
|
+
"""Pull the credentials out of an open-batch response."""
|
|
144
|
+
raw = payload["credentials"]
|
|
145
|
+
return ScopedCredentials(**{k: raw[k] for k in ScopedCredentials.__annotations__})
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
"""Credential and configuration storage: ``~/.config/gi-ingest/``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import stat
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from gi_ingest.api import DEFAULT_API_URL
|
|
12
|
+
|
|
13
|
+
CONFIG_DIR = Path.home() / ".config" / "gi-ingest"
|
|
14
|
+
CONFIG_FILE = CONFIG_DIR / "config.json"
|
|
15
|
+
|
|
16
|
+
DEFAULT_STAGING = Path.home() / "gilabs-staging"
|
|
17
|
+
DEFAULT_LEDGER = Path.home() / ".local" / "state" / "gi-ingest" / "ledger.db"
|
|
18
|
+
|
|
19
|
+
# Where the tool kept these before it was renamed from gilabs-ingest.
|
|
20
|
+
_LEGACY_CONFIG_DIR = Path.home() / ".config" / "gilabs-ingest"
|
|
21
|
+
_LEGACY_STATE_DIR = Path.home() / ".local" / "state" / "gilabs-ingest"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def adopt_legacy_paths() -> None:
|
|
25
|
+
"""Move an older install's config and ledger to the new names.
|
|
26
|
+
|
|
27
|
+
The rename would otherwise orphan both, and the ledger is not a cache: it
|
|
28
|
+
holds which episodes are staged, which are in flight, and which the server
|
|
29
|
+
has confirmed. Losing it mid-round means re-staging cards that were already
|
|
30
|
+
copied, and cards that can no longer be reclaimed because nothing remembers
|
|
31
|
+
they were delivered.
|
|
32
|
+
|
|
33
|
+
That matters more than usual right now, because the version floor makes
|
|
34
|
+
upgrading mandatory rather than optional — an upgrade that silently drops
|
|
35
|
+
a manager's queue would be a bad trade for a shorter name.
|
|
36
|
+
|
|
37
|
+
Moves rather than copies, so there is one source of truth afterwards, and
|
|
38
|
+
only when the new location does not already exist. Best-effort: a failure
|
|
39
|
+
here leaves the old files untouched and costs a `login`, not data.
|
|
40
|
+
"""
|
|
41
|
+
pairs = (
|
|
42
|
+
(_LEGACY_CONFIG_DIR, CONFIG_DIR),
|
|
43
|
+
(_LEGACY_STATE_DIR, DEFAULT_LEDGER.parent),
|
|
44
|
+
)
|
|
45
|
+
for legacy, current in pairs:
|
|
46
|
+
if not legacy.is_dir() or current.exists():
|
|
47
|
+
continue
|
|
48
|
+
try:
|
|
49
|
+
current.parent.mkdir(parents=True, exist_ok=True)
|
|
50
|
+
legacy.rename(current)
|
|
51
|
+
except OSError:
|
|
52
|
+
# A different filesystem, a permission, a race with another run.
|
|
53
|
+
# The old path stays put and the user re-runs `login`.
|
|
54
|
+
pass
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@dataclass
|
|
58
|
+
class Config:
|
|
59
|
+
api_url: str = DEFAULT_API_URL
|
|
60
|
+
api_key: str | None = None
|
|
61
|
+
staging_dir: Path = DEFAULT_STAGING
|
|
62
|
+
ledger_path: Path = DEFAULT_LEDGER
|
|
63
|
+
project_id: str | None = None
|
|
64
|
+
|
|
65
|
+
@property
|
|
66
|
+
def is_authenticated(self) -> bool:
|
|
67
|
+
return bool(self.api_key)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _follow_moved_state(path: Path) -> Path:
|
|
71
|
+
"""Repoint a stored path that :func:`adopt_legacy_paths` moved out from under.
|
|
72
|
+
|
|
73
|
+
``login`` writes the ledger path into config.json, so moving the directory
|
|
74
|
+
is only half the job — the recorded path still names the old location, and
|
|
75
|
+
the CLI would helpfully create a brand new empty ledger there. That is the
|
|
76
|
+
precise failure the move exists to prevent: a manager mid-round would find
|
|
77
|
+
nothing staged, nothing in flight, and no card reclaimable.
|
|
78
|
+
|
|
79
|
+
Only rewrites paths that sit under the legacy directory. Anything a manager
|
|
80
|
+
pointed somewhere deliberate is left where they put it.
|
|
81
|
+
"""
|
|
82
|
+
try:
|
|
83
|
+
relative = path.relative_to(_LEGACY_STATE_DIR)
|
|
84
|
+
except ValueError:
|
|
85
|
+
return path
|
|
86
|
+
return DEFAULT_LEDGER.parent / relative
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def load() -> Config:
|
|
90
|
+
adopt_legacy_paths()
|
|
91
|
+
if not CONFIG_FILE.exists():
|
|
92
|
+
return Config()
|
|
93
|
+
try:
|
|
94
|
+
raw = json.loads(CONFIG_FILE.read_text())
|
|
95
|
+
except (OSError, json.JSONDecodeError):
|
|
96
|
+
# A corrupt config reads as "not configured" rather than a traceback —
|
|
97
|
+
# the fix is `gi-ingest login`, and that is what the caller is told.
|
|
98
|
+
return Config()
|
|
99
|
+
return Config(
|
|
100
|
+
api_url=raw.get("api_url") or DEFAULT_API_URL,
|
|
101
|
+
api_key=raw.get("api_key"),
|
|
102
|
+
staging_dir=Path(raw["staging_dir"]) if raw.get("staging_dir") else DEFAULT_STAGING,
|
|
103
|
+
ledger_path=_follow_moved_state(
|
|
104
|
+
Path(raw["ledger_path"]) if raw.get("ledger_path") else DEFAULT_LEDGER
|
|
105
|
+
),
|
|
106
|
+
project_id=raw.get("project_id"),
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def save(config: Config) -> None:
|
|
111
|
+
"""Persist config, 0600. The API key lives here in plaintext.
|
|
112
|
+
|
|
113
|
+
Same posture as the AWS CLI and `gidata`: a file only the user can read.
|
|
114
|
+
Worth knowing rather than assuming — the key is a bearer credential for the
|
|
115
|
+
vendor's whole upload surface, so a shared laptop should use a per-machine
|
|
116
|
+
key that can be revoked on its own.
|
|
117
|
+
"""
|
|
118
|
+
CONFIG_DIR.mkdir(parents=True, exist_ok=True)
|
|
119
|
+
payload = {
|
|
120
|
+
"api_url": config.api_url,
|
|
121
|
+
"api_key": config.api_key,
|
|
122
|
+
"staging_dir": str(config.staging_dir),
|
|
123
|
+
"ledger_path": str(config.ledger_path),
|
|
124
|
+
"project_id": config.project_id,
|
|
125
|
+
}
|
|
126
|
+
tmp = CONFIG_FILE.with_suffix(".tmp")
|
|
127
|
+
tmp.write_text(json.dumps(payload, indent=2) + "\n")
|
|
128
|
+
os.chmod(tmp, stat.S_IRUSR | stat.S_IWUSR)
|
|
129
|
+
tmp.replace(CONFIG_FILE)
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Episode identity — computed client-side so a dedup check costs a request, not an
|
|
3
|
+
upload.
|
|
4
|
+
|
|
5
|
+
``derive_episode_id`` must stay byte-identical to the worker's version in
|
|
6
|
+
data-processing-services (``worker/ego/preprocess/concat.py``). It is the uuid5
|
|
7
|
+
that becomes ``episodes.id`` for pipeline episodes, so a drift here would make
|
|
8
|
+
every dedup check miss and every re-upload look novel — silently, and only
|
|
9
|
+
visible as duplicated storage.
|
|
10
|
+
|
|
11
|
+
There is no import to share: the worker lives in another repository and this
|
|
12
|
+
package ships to vendors. The contract is pinned by ``tests/test_identity.py``,
|
|
13
|
+
which asserts the exact uuid5 for a known triple.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import re
|
|
19
|
+
import uuid
|
|
20
|
+
|
|
21
|
+
# NAMESPACE_OID, matching the worker. Not an arbitrary choice — changing it
|
|
22
|
+
# renames every episode.
|
|
23
|
+
_UUID_NAMESPACE = uuid.NAMESPACE_OID
|
|
24
|
+
|
|
25
|
+
_DATE_IN_TEXT_RE = re.compile(r"(\d{8})")
|
|
26
|
+
# ISO-8601 dates are the common shape in session metadata (`recorded_at`), and
|
|
27
|
+
# their hyphens defeat the 8-consecutive-digit match the worker uses on compact
|
|
28
|
+
# S3 keys. Matched first and compacted, so a metadata timestamp and a path date
|
|
29
|
+
# resolve to the same value instead of the 00000000 sentinel.
|
|
30
|
+
_ISO_DATE_RE = re.compile(r"(\d{4})-(\d{2})-(\d{2})")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def normalize_device(raw: str) -> str:
|
|
34
|
+
"""Lowercase, hyphens to underscores — the worker's normalization."""
|
|
35
|
+
return raw.lower().replace("-", "_")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def derive_episode_id(date: str, device: str, session: str) -> str:
|
|
39
|
+
"""The deterministic episode id the pipeline will assign.
|
|
40
|
+
|
|
41
|
+
Mirrors ``worker/ego/preprocess/concat.py::derive_episode_id``. Keep the
|
|
42
|
+
format string exactly as-is.
|
|
43
|
+
"""
|
|
44
|
+
name = f"gilabs-ingest:{normalize_device(device)}:{session}:{date}"
|
|
45
|
+
return str(uuid.uuid5(_UUID_NAMESPACE, name))
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def extract_date(*values: str | None) -> str:
|
|
49
|
+
"""First 8-digit run found across ``values``, else the worker's ``00000000``.
|
|
50
|
+
|
|
51
|
+
The worker scans the S3 key for a date; the CLI has better sources (session
|
|
52
|
+
metadata, then the session directory name), but falls back to the same
|
|
53
|
+
sentinel so an undated session lands on the same id either way.
|
|
54
|
+
"""
|
|
55
|
+
for value in values:
|
|
56
|
+
if not value:
|
|
57
|
+
continue
|
|
58
|
+
iso = _ISO_DATE_RE.search(value)
|
|
59
|
+
if iso:
|
|
60
|
+
return f"{iso.group(1)}{iso.group(2)}{iso.group(3)}"
|
|
61
|
+
compact = _DATE_IN_TEXT_RE.search(value)
|
|
62
|
+
if compact:
|
|
63
|
+
return compact.group(1)
|
|
64
|
+
return "00000000"
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def episode_key(date: str, device: str, session: str) -> str:
|
|
68
|
+
"""Path segment for this episode inside the batch prefix.
|
|
69
|
+
|
|
70
|
+
Human-readable on purpose: this is what a vendor sees in an error message
|
|
71
|
+
and what an engineer greps for in the bucket. Uniqueness within a batch is
|
|
72
|
+
enforced by the server at seal, not by this string.
|
|
73
|
+
"""
|
|
74
|
+
return f"{date}/{normalize_device(device)}/{session}"
|