kdev-cli 0.1.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
kdev/__init__.py ADDED
@@ -0,0 +1,7 @@
1
+ from importlib.metadata import PackageNotFoundError, version
2
+
3
+ try:
4
+ # The distribution's name, not the module's: PyPI does not allow "kdev".
5
+ __version__ = version("kdev-cli")
6
+ except PackageNotFoundError: # running from a source tree, not an install
7
+ __version__ = "0.0.0+source"
kdev/__main__.py ADDED
@@ -0,0 +1,3 @@
1
+ from .cli import main
2
+
3
+ main()
kdev/api.py ADDED
@@ -0,0 +1,408 @@
1
+ """Kaggle's JSON-over-HTTP RPC API.
2
+
3
+ The official `kaggle` CLI does not expose kernel *sessions*, but the API under
4
+ it does. Every method is `POST {BASE}/{service}/{Method}` with a camelCase JSON
5
+ body and an OAuth Bearer token (see auth.py). Verified against kagglesdk 0.1.37.
6
+
7
+ Run any command with `kdev -v` to log each call, its status and its timing.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import json
13
+ import logging
14
+ import time
15
+ from collections.abc import Iterator
16
+ from dataclasses import dataclass
17
+ from typing import Any
18
+
19
+ import httpx
20
+
21
+ BASE = "https://api.kaggle.com/v1"
22
+ KERNELS = "kernels.KernelsApiService"
23
+ IAM = "security.IamService"
24
+ GROUPS = "users.GroupApiService"
25
+
26
+ #: KaggleResourceType.KAGGLE_RESOURCE_TYPE_KERNELS
27
+ RESOURCE_KERNEL = "KAGGLE_RESOURCE_TYPE_KERNELS"
28
+ #: CanonicalRole values, as the enum serialises by name.
29
+ ROLE_VIEWER, ROLE_EDITOR, ROLE_ADMIN = (
30
+ "CANONICAL_ROLE_VIEWER",
31
+ "CANONICAL_ROLE_EDITOR",
32
+ "CANONICAL_ROLE_ADMIN",
33
+ )
34
+
35
+ # machine_shape values accepted by CreateKernelSession/SaveKernel.
36
+ # None == CPU-only. Only the first four are on a normal account; the gated ones
37
+ # 403 unless yours has them, and are listed so `--gpu` can pass them through.
38
+ SHAPES = {
39
+ "none": None,
40
+ "t4": "NvidiaTeslaT4",
41
+ "p100": "NvidiaTeslaP100",
42
+ "tpu": "Tpu1VmV38",
43
+ "l4": "NvidiaL4",
44
+ "a100": "NvidiaTeslaA100",
45
+ "h100": "NvidiaTeslaH100",
46
+ }
47
+
48
+ #: Shapes every account has. `--gpu` accepts the rest, the picker does not
49
+ #: offer them, and Kaggle answers 403 if the account is not entitled.
50
+ COMMON_SHAPES = ("none", "t4", "p100", "tpu")
51
+
52
+
53
+ log = logging.getLogger("kdev.api")
54
+
55
+
56
+ class KaggleError(RuntimeError):
57
+ """A Kaggle API call failed. `status` is the HTTP status, 0 if none."""
58
+
59
+ def __init__(self, message: str, status: int = 0) -> None:
60
+ super().__init__(message)
61
+ self.status = status
62
+
63
+
64
+ @dataclass(frozen=True)
65
+ class Creds:
66
+ """How one account authenticates: its username and a current OAuth token.
67
+
68
+ Kaggle rejects the legacy username+key on the endpoints kdev needs, so a
69
+ token is the only path.
70
+ """
71
+
72
+ username: str
73
+ token: str = ""
74
+
75
+ @property
76
+ def headers(self) -> dict[str, str]:
77
+ return {"Authorization": f"Bearer {self.token}"} if self.token else {}
78
+
79
+
80
+ #: Status codes worth a second attempt. Anything else is a real answer.
81
+ RETRY_STATUS = frozenset({429, 500, 502, 503, 504})
82
+
83
+
84
+ def call(
85
+ creds: Creds,
86
+ method: str,
87
+ payload: dict[str, Any] | None = None,
88
+ service: str = KERNELS,
89
+ attempts: int = 3,
90
+ ) -> dict:
91
+ url = f"{BASE}/{service}/{method}"
92
+ for attempt in range(attempts):
93
+ last = attempt == attempts - 1
94
+ started = time.monotonic()
95
+ try:
96
+ r = httpx.post(url, json=payload or {}, headers=creds.headers, timeout=120.0)
97
+ except httpx.HTTPError as e:
98
+ log.debug("%s as %s: %s (attempt %d)", method, creds.username, e, attempt + 1)
99
+ if last:
100
+ raise KaggleError(f"{method}: could not reach Kaggle ({e})") from e
101
+ time.sleep(2**attempt)
102
+ continue
103
+ log.debug(
104
+ "%s as %s -> %d in %.2fs",
105
+ method,
106
+ creds.username,
107
+ r.status_code,
108
+ time.monotonic() - started,
109
+ )
110
+ if r.status_code in RETRY_STATUS and not last:
111
+ time.sleep(2**attempt)
112
+ continue
113
+ break
114
+ if r.status_code == 401:
115
+ raise KaggleError(f"{method}: {creds.username or 'this account'} is not signed in", 401)
116
+ if r.status_code >= 400:
117
+ raise KaggleError(f"{method}: {_message(r)}", r.status_code)
118
+ try:
119
+ body = r.json() if r.content else {}
120
+ except ValueError as e:
121
+ raise KaggleError(f"{method}: Kaggle sent something that is not JSON", r.status_code) from e
122
+ # The API returns 200 with an `error` field for validation failures.
123
+ if isinstance(body, dict) and body.get("error"):
124
+ raise KaggleError(f"{method}: {body['error']}")
125
+ return body
126
+
127
+
128
+ def _message(r: httpx.Response) -> str:
129
+ """Kaggle's own error message, not the JSON envelope around it."""
130
+ try:
131
+ err = (r.json() or {}).get("error")
132
+ except ValueError:
133
+ err = None
134
+ if isinstance(err, dict) and err.get("message"):
135
+ return f"{err['message']} (HTTP {r.status_code})"
136
+ return f"HTTP {r.status_code}: {r.text[:200]}"
137
+
138
+
139
+ def save_kernel(
140
+ creds: Creds,
141
+ *,
142
+ slug: str,
143
+ title: str,
144
+ source: str,
145
+ machine_shape: str | None,
146
+ timeout_seconds: int,
147
+ run: bool = True,
148
+ kernel_type: str = "script",
149
+ ) -> dict:
150
+ """Create-or-update the notebook and run it top-to-bottom (Save & Run All)."""
151
+ payload: dict[str, Any] = {
152
+ "slug": slug,
153
+ "newTitle": title,
154
+ "text": source,
155
+ "language": "python",
156
+ "kernelType": kernel_type,
157
+ "isPrivate": True,
158
+ "enableInternet": True,
159
+ "kernelExecutionType": "SAVE_AND_RUN_ALL" if run else "QUICK_SAVE",
160
+ "sessionTimeoutSeconds": timeout_seconds,
161
+ "kernelDataSources": [],
162
+ "datasetDataSources": [],
163
+ }
164
+ if machine_shape:
165
+ payload["machineShape"] = machine_shape
166
+ return call(creds, "SaveKernel", payload)
167
+
168
+
169
+ #: The most ListKernelSessionOutput accepts per page (it says so on a 400).
170
+ OUTPUT_PAGE = 500
171
+
172
+
173
+ def session_output(creds: Creds, slug: str, version: str = "") -> list[dict]:
174
+ """Every file a saved version left in /kaggle/working, with signed URLs.
175
+
176
+ `version` is a label like "v7"; empty means the latest. Older versions stay
177
+ readable, which is what lets a restore build on the last good one when the
178
+ newest was cut short. A notebook cannot mount its own output (Kaggle
179
+ rejects the data source), so this listing is the only way back.
180
+ """
181
+ user, _, kslug = slug.partition("/")
182
+ out = []
183
+ token = None
184
+ while True:
185
+ payload = {"userName": user, "kernelSlug": kslug, "pageSize": OUTPUT_PAGE}
186
+ if version:
187
+ payload["versionLabel"] = version
188
+ if token:
189
+ payload["pageToken"] = token
190
+ resp = call(creds, "ListKernelSessionOutput", payload)
191
+ for f in resp.get("files") or []:
192
+ name = f.get("fileName")
193
+ if name and f.get("url"):
194
+ out.append({"name": name, "url": f["url"]})
195
+ token = resp.get("nextPageToken")
196
+ if not token:
197
+ return out
198
+
199
+
200
+ #: The status of a notebook that has never been run. Kaggle answers that with
201
+ #: HTTP 404 "No runs found for this kernel" rather than a state -- measured on a
202
+ #: notebook freshly created in the Kaggle editor.
203
+ NEVER_RUN = "NEVER_RUN"
204
+
205
+
206
+ def session_status(creds: Creds, slug: str) -> dict:
207
+ """{"status": ..., "failureMessage": ...} -- the second is why it stopped.
208
+
209
+ A 404 means there is no session to report, which is an answer, not a
210
+ failure: every caller only asks "is something running?". Whether the
211
+ notebook itself exists is checked where it matters, by GetKernel.
212
+ """
213
+ user, _, kslug = slug.partition("/")
214
+ try:
215
+ return call(creds, "GetKernelSessionStatus", {"userName": user, "kernelSlug": kslug})
216
+ except KaggleError as e:
217
+ if e.status == 404:
218
+ return {"status": NEVER_RUN}
219
+ raise
220
+
221
+
222
+ #: Session states that mean the box is, or is about to be, alive.
223
+ LIVE_STATES = frozenset({"RUNNING", "QUEUED"})
224
+ #: Stopped, but its files are still being saved. Measured after a cancel: the
225
+ #: version listed 0, then 29, then all 54 files while in this state, and was
226
+ #: complete only once it moved on. (A crash stays RUNNING until it is saved.)
227
+ SAVING = "CANCEL_REQUESTED"
228
+
229
+
230
+ def cancel_session(creds: Creds, kernel_session_id: int) -> dict:
231
+ return call(creds, "CancelKernelSession", {"kernelSessionId": kernel_session_id})
232
+
233
+
234
+ def quota(creds: Creds) -> dict:
235
+ return call(creds, "GetAcceleratorQuotaStatistics", {})
236
+
237
+
238
+ def stream_logs(
239
+ creds: Creds, slug: str, wait_seconds: int = 300, idle: float | None = None
240
+ ) -> Iterator[str]:
241
+ """Yield log lines from a running session.
242
+
243
+ While the session lives the API returns SSE; once it has terminated it
244
+ returns the persisted JSON log instead, so we branch on content-type
245
+ exactly as the SDK docstring instructs. `idle` ends the stream once no line
246
+ has come for that many seconds: the log so far, without following it.
247
+ """
248
+ user, _, kslug = slug.partition("/")
249
+ payload = {
250
+ "userName": user,
251
+ "kernelSlug": kslug,
252
+ "waitForLogsUrlSeconds": min(wait_seconds, 300),
253
+ }
254
+ with httpx.stream(
255
+ "POST",
256
+ f"{BASE}/{KERNELS}/GetKernelSessionLogsStream",
257
+ json=payload,
258
+ headers=creds.headers,
259
+ timeout=httpx.Timeout(30.0, read=idle),
260
+ ) as r:
261
+ if r.status_code >= 400:
262
+ r.read()
263
+ raise KaggleError(f"logs: HTTP {r.status_code}: {r.text[:300]}", r.status_code)
264
+ if "text/event-stream" in r.headers.get("content-type", ""):
265
+ try:
266
+ for line in r.iter_lines():
267
+ if line.startswith("data:"):
268
+ data = line[5:].strip()
269
+ if data == "END_OF_LOG":
270
+ return
271
+ yield _log_text(data)
272
+ except httpx.ReadTimeout:
273
+ if idle is None:
274
+ raise
275
+ else:
276
+ for entry in json.loads(r.read() or b"[]"):
277
+ yield _log_text(entry)
278
+
279
+
280
+ def _log_text(entry: Any) -> str:
281
+ """Log entries arrive as JSON objects with a `data` field, or bare strings."""
282
+ if isinstance(entry, str):
283
+ try:
284
+ entry = json.loads(entry)
285
+ except json.JSONDecodeError:
286
+ return entry
287
+ if isinstance(entry, dict):
288
+ return str(entry.get("data", entry.get("text", entry)))
289
+ return str(entry)
290
+
291
+
292
+ # --- sharing ------------------------------------------------------------------
293
+
294
+
295
+ def get_policy(creds: Creds, kernel_id: int) -> dict:
296
+ return call(
297
+ creds,
298
+ "GetIamPolicy",
299
+ {"resourceId": {"type": RESOURCE_KERNEL, "id": kernel_id}},
300
+ service=IAM,
301
+ )
302
+
303
+
304
+ def share_with_group(
305
+ creds: Creds, kernel_id: int, group_slug: str, role: str = ROLE_EDITOR
306
+ ) -> dict:
307
+ """Grant a Kaggle group a role on a notebook, preserving other bindings.
308
+
309
+ Read-modify-write rather than a blind overwrite: SetIamPolicy replaces the
310
+ whole policy, so sending only our binding would drop every collaborator
311
+ already on the notebook.
312
+ """
313
+ policy = get_policy(creds, kernel_id)
314
+ bindings = [dict(b) for b in policy.get("bindings") or []]
315
+
316
+ def is_our_group(member: dict) -> bool:
317
+ return (member.get("group") or {}).get("slug") == group_slug
318
+
319
+ for b in bindings:
320
+ b["members"] = [m for m in (b.get("members") or []) if not is_our_group(m)]
321
+ bindings = [b for b in bindings if b.get("members")]
322
+
323
+ target = next((b for b in bindings if b.get("role") == role), None)
324
+ if target is None:
325
+ target = {"role": role, "members": []}
326
+ bindings.append(target)
327
+ target["members"].append({"group": {"slug": group_slug}})
328
+
329
+ new_policy = {"bindings": bindings}
330
+ if policy.get("owner"):
331
+ new_policy["owner"] = policy["owner"]
332
+ return call(
333
+ creds,
334
+ "SetIamPolicy",
335
+ {"resourceId": {"type": RESOURCE_KERNEL, "id": kernel_id}, "policy": new_policy},
336
+ service=IAM,
337
+ )
338
+
339
+
340
+ def group_role(policy: dict, group_slug: str) -> str:
341
+ for g in policy_groups(policy):
342
+ if g["slug"] == group_slug:
343
+ return g["role"]
344
+ return ""
345
+
346
+
347
+ def group_members(creds: Creds, group_slug: str) -> list[dict]:
348
+ resp = call(
349
+ creds,
350
+ "ListUserManagedGroupMemberships",
351
+ {"groupSlug": group_slug, "pageSize": 100},
352
+ service=GROUPS,
353
+ )
354
+ return resp.get("memberships") or []
355
+
356
+
357
+ def list_shared_kernels(creds: Creds) -> list[dict]:
358
+ """Notebooks shared with this account -- how a teammate finds the box."""
359
+ resp = call(creds, "ListKernels", {"group": "COLLABORATION", "pageSize": 100})
360
+ return resp.get("kernels") or []
361
+
362
+
363
+ def get_kernel(creds: Creds, slug: str) -> dict:
364
+ """The kernel's metadata. Note the response key is `metadata`, not `kernel`."""
365
+ user, _, kslug = slug.partition("/")
366
+ resp = call(creds, "GetKernel", {"userName": user, "kernelSlug": kslug})
367
+ return resp.get("metadata") or {}
368
+
369
+
370
+ def kernel_source(creds: Creds, slug: str) -> str:
371
+ """The notebook's current source, for checking before overwriting it."""
372
+ user, _, kslug = slug.partition("/")
373
+ resp = call(creds, "GetKernel", {"userName": user, "kernelSlug": kslug})
374
+ return (resp.get("blob") or {}).get("source", "") or ""
375
+
376
+
377
+ def kernel_id(creds: Creds, slug: str) -> int:
378
+ try:
379
+ return int(get_kernel(creds, slug).get("id") or 0)
380
+ except (KaggleError, TypeError, ValueError):
381
+ return 0
382
+
383
+
384
+ def policy_groups(policy: dict) -> list[dict]:
385
+ """Groups a notebook is shared with.
386
+
387
+ The identifying fields live on the nested `avatar`, not on the group
388
+ principal itself -- `group.slug` comes back null while `group.avatar.slug`
389
+ is populated.
390
+ """
391
+ out = []
392
+ for b in policy.get("bindings") or []:
393
+ for m in b.get("members") or []:
394
+ g = m.get("group")
395
+ if not g:
396
+ continue
397
+ av = g.get("avatar") or {}
398
+ out.append(
399
+ {
400
+ "id": g.get("id") or av.get("id"),
401
+ "slug": g.get("slug") or av.get("slug") or "",
402
+ "name": av.get("name") or "",
403
+ "members": av.get("memberCount") or 0,
404
+ "owner": (av.get("owner") or {}).get("userName", ""),
405
+ "role": b.get("role", ""),
406
+ }
407
+ )
408
+ return out
kdev/auth.py ADDED
@@ -0,0 +1,115 @@
1
+ """Kaggle OAuth, one credential set per profile.
2
+
3
+ Kaggle deprecated username+key auth for most of its API: the official CLI now
4
+ refuses it outright and the endpoints kdev needs (ListKernels, quota, session
5
+ status) answer 401. Only a Bearer access token works.
6
+
7
+ Tokens expire after 12 hours, but the credentials file also carries a
8
+ long-lived refresh token, and kagglesdk refreshes transparently. kdev keeps one
9
+ credentials file per profile so several teammates' accounts can coexist -- the
10
+ official CLI only ever has one, at ~/.kaggle/credentials.json.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import json
16
+ import shutil
17
+ import subprocess
18
+ import sys
19
+ from pathlib import Path
20
+
21
+ from . import config as _config
22
+
23
+ KAGGLE_CREDENTIALS = Path.home() / ".kaggle" / "credentials.json"
24
+
25
+
26
+ class AuthError(RuntimeError):
27
+ pass
28
+
29
+
30
+ def creds_dir() -> Path:
31
+ return _config.CONFIG_DIR / "creds"
32
+
33
+
34
+ def creds_path(profile: str) -> Path:
35
+ return creds_dir() / f"{profile}.json"
36
+
37
+
38
+ def _kaggle_cli() -> list[str]:
39
+ """The official CLI drives the browser flow; we only keep the result."""
40
+ exe = shutil.which("kaggle")
41
+ if exe:
42
+ return [exe]
43
+ return [sys.executable, "-m", "kaggle"]
44
+
45
+
46
+ def login(profile: str) -> str:
47
+ """Run the browser OAuth flow and file the result under this profile.
48
+
49
+ Returns the authenticated username. The CLI always writes to the one shared
50
+ path, so the credentials are moved into the profile's own slot immediately
51
+ -- otherwise adding a second account would silently overwrite the first.
52
+ """
53
+ before = KAGGLE_CREDENTIALS.read_text() if KAGGLE_CREDENTIALS.exists() else None
54
+ # The CLI prints a long "you could also use an API token" block before it
55
+ # gets to the browser flow. Capture it; only surface it if login fails, so
56
+ # the user is not asked to choose an auth method kdev has already chosen.
57
+ try:
58
+ proc = subprocess.run(
59
+ [*_kaggle_cli(), "auth", "login", "--force"],
60
+ capture_output=True,
61
+ text=True,
62
+ timeout=300,
63
+ )
64
+ except subprocess.TimeoutExpired as e:
65
+ raise AuthError("Timed out waiting for the browser sign-in.") from e
66
+ except OSError as e:
67
+ raise AuthError(f"Could not run the kaggle CLI: {e}") from e
68
+ if proc.returncode != 0:
69
+ detail = (proc.stderr or proc.stdout or "").strip()[-400:]
70
+ raise AuthError(f"Kaggle sign-in did not complete.\n{detail}")
71
+ if not KAGGLE_CREDENTIALS.exists():
72
+ raise AuthError("Login reported success but wrote no credentials.")
73
+
74
+ data = json.loads(KAGGLE_CREDENTIALS.read_text())
75
+ if not data.get("refresh_token"):
76
+ raise AuthError("Credentials are missing a refresh token.")
77
+
78
+ dest = creds_path(profile)
79
+ dest.parent.mkdir(parents=True, exist_ok=True)
80
+ dest.write_text(json.dumps(data, indent=2))
81
+ dest.chmod(0o600)
82
+
83
+ # Put the shared file back so kdev does not disturb the user's own CLI login.
84
+ if before is not None:
85
+ KAGGLE_CREDENTIALS.write_text(before)
86
+
87
+ return data.get("username") or ""
88
+
89
+
90
+ def access_token(profile: str) -> str:
91
+ """A valid access token for this profile, refreshing it if it has expired."""
92
+ path = creds_path(profile)
93
+ if not path.exists():
94
+ raise AuthError(f"{profile} is not signed in. Run: kdev account add {profile}")
95
+ try:
96
+ from kagglesdk import KaggleClient
97
+ from kagglesdk.kaggle_creds import KaggleCredentials
98
+ except ImportError as e: # pragma: no cover - dependency is declared
99
+ raise AuthError(f"kagglesdk is not installed: {e}") from e
100
+
101
+ creds = KaggleCredentials.load(KaggleClient(), file_path=str(path))
102
+ if creds is None:
103
+ raise AuthError(f"Could not read credentials for {profile!r}.")
104
+ token = creds.get_access_token()
105
+ if not token:
106
+ raise AuthError(f"Could not refresh {profile}'s sign-in. Run: kdev account add {profile}")
107
+ return token
108
+
109
+
110
+ def signed_in(profile: str) -> bool:
111
+ return creds_path(profile).exists()
112
+
113
+
114
+ def forget(profile: str) -> None:
115
+ creds_path(profile).unlink(missing_ok=True)