fastcolab 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
fastcolab/__init__.py ADDED
@@ -0,0 +1,3 @@
1
+ __version__ = "0.0.1"
2
+
3
+ from .core import *
fastcolab/_modidx.py ADDED
@@ -0,0 +1,111 @@
1
+ # Autogenerated by nbdev
2
+
3
+ d = { 'settings': { 'branch': 'main',
4
+ 'doc_baseurl': '/fastcolab',
5
+ 'doc_host': 'https://AnswerDotAI.github.io',
6
+ 'git_url': 'https://github.com/AnswerDotAI/fastcolab',
7
+ 'lib_path': 'fastcolab'},
8
+ 'syms': { 'fastcolab.core': { 'fastcolab.core.Colab': ('core.html#colab', 'fastcolab/core.py'),
9
+ 'fastcolab.core.Colab.__init__': ('core.html#colab.__init__', 'fastcolab/core.py'),
10
+ 'fastcolab.core.Colab.__repr__': ('core.html#colab.__repr__', 'fastcolab/core.py'),
11
+ 'fastcolab.core.Colab.connect': ('core.html#colab.connect', 'fastcolab/core.py'),
12
+ 'fastcolab.core.Colab.delete': ('core.html#colab.delete', 'fastcolab/core.py'),
13
+ 'fastcolab.core.Colab.get': ('core.html#colab.get', 'fastcolab/core.py'),
14
+ 'fastcolab.core.Colab.runtimes': ('core.html#colab.runtimes', 'fastcolab/core.py'),
15
+ 'fastcolab.core.Colab.specs': ('core.html#colab.specs', 'fastcolab/core.py'),
16
+ 'fastcolab.core.Colab.start': ('core.html#colab.start', 'fastcolab/core.py'),
17
+ 'fastcolab.core.Colab.subscription': ('core.html#colab.subscription', 'fastcolab/core.py'),
18
+ 'fastcolab.core.ColabJob': ('core.html#colabjob', 'fastcolab/core.py'),
19
+ 'fastcolab.core.ColabJob.__init__': ('core.html#colabjob.__init__', 'fastcolab/core.py'),
20
+ 'fastcolab.core.ColabJob.__repr__': ('core.html#colabjob.__repr__', 'fastcolab/core.py'),
21
+ 'fastcolab.core.ColabJob.status': ('core.html#colabjob.status', 'fastcolab/core.py'),
22
+ 'fastcolab.core.ColabJob.tail': ('core.html#colabjob.tail', 'fastcolab/core.py'),
23
+ 'fastcolab.core.ColabJob.wait': ('core.html#colabjob.wait', 'fastcolab/core.py'),
24
+ 'fastcolab.core.ColabRuntime': ('core.html#colabruntime', 'fastcolab/core.py'),
25
+ 'fastcolab.core.ColabRuntime.__aenter__': ('core.html#colabruntime.__aenter__', 'fastcolab/core.py'),
26
+ 'fastcolab.core.ColabRuntime.__aexit__': ('core.html#colabruntime.__aexit__', 'fastcolab/core.py'),
27
+ 'fastcolab.core.ColabRuntime.__init__': ('core.html#colabruntime.__init__', 'fastcolab/core.py'),
28
+ 'fastcolab.core.ColabRuntime.__repr__': ('core.html#colabruntime.__repr__', 'fastcolab/core.py'),
29
+ 'fastcolab.core.ColabRuntime._repr_markdown_': ( 'core.html#colabruntime._repr_markdown_',
30
+ 'fastcolab/core.py'),
31
+ 'fastcolab.core.ColabRuntime.conn': ('core.html#colabruntime.conn', 'fastcolab/core.py'),
32
+ 'fastcolab.core.ColabRuntime.download': ('core.html#colabruntime.download', 'fastcolab/core.py'),
33
+ 'fastcolab.core.ColabRuntime.exec': ('core.html#colabruntime.exec', 'fastcolab/core.py'),
34
+ 'fastcolab.core.ColabRuntime.expire_time': ('core.html#colabruntime.expire_time', 'fastcolab/core.py'),
35
+ 'fastcolab.core.ColabRuntime.files': ('core.html#colabruntime.files', 'fastcolab/core.py'),
36
+ 'fastcolab.core.ColabRuntime.from_drive': ('core.html#colabruntime.from_drive', 'fastcolab/core.py'),
37
+ 'fastcolab.core.ColabRuntime.headers': ('core.html#colabruntime.headers', 'fastcolab/core.py'),
38
+ 'fastcolab.core.ColabRuntime.id': ('core.html#colabruntime.id', 'fastcolab/core.py'),
39
+ 'fastcolab.core.ColabRuntime.kernel': ('core.html#colabruntime.kernel', 'fastcolab/core.py'),
40
+ 'fastcolab.core.ColabRuntime.launch': ('core.html#colabruntime.launch', 'fastcolab/core.py'),
41
+ 'fastcolab.core.ColabRuntime.name': ('core.html#colabruntime.name', 'fastcolab/core.py'),
42
+ 'fastcolab.core.ColabRuntime.refresh': ('core.html#colabruntime.refresh', 'fastcolab/core.py'),
43
+ 'fastcolab.core.ColabRuntime.spec': ('core.html#colabruntime.spec', 'fastcolab/core.py'),
44
+ 'fastcolab.core.ColabRuntime.stop': ('core.html#colabruntime.stop', 'fastcolab/core.py'),
45
+ 'fastcolab.core.ColabRuntime.tail': ('core.html#colabruntime.tail', 'fastcolab/core.py'),
46
+ 'fastcolab.core.ColabRuntime.to_drive': ('core.html#colabruntime.to_drive', 'fastcolab/core.py'),
47
+ 'fastcolab.core.ColabRuntime.token': ('core.html#colabruntime.token', 'fastcolab/core.py'),
48
+ 'fastcolab.core.ColabRuntime.upload': ('core.html#colabruntime.upload', 'fastcolab/core.py'),
49
+ 'fastcolab.core.ColabRuntime.url': ('core.html#colabruntime.url', 'fastcolab/core.py'),
50
+ 'fastcolab.core._clean_path': ('core.html#_clean_path', 'fastcolab/core.py'),
51
+ 'fastcolab.core._drive_folder_id': ('core.html#_drive_folder_id', 'fastcolab/core.py'),
52
+ 'fastcolab.core._ensure_parent_dirs': ('core.html#_ensure_parent_dirs', 'fastcolab/core.py'),
53
+ 'fastcolab.core._esc_q': ('core.html#_esc_q', 'fastcolab/core.py'),
54
+ 'fastcolab.core._job_wrapper': ('core.html#_job_wrapper', 'fastcolab/core.py'),
55
+ 'fastcolab.core._rt_name': ('core.html#_rt_name', 'fastcolab/core.py'),
56
+ 'fastcolab.core._wait_op': ('core.html#_wait_op', 'fastcolab/core.py'),
57
+ 'fastcolab.core.mk_spec': ('core.html#mk_spec', 'fastcolab/core.py'),
58
+ 'fastcolab.core.outs_text': ('core.html#outs_text', 'fastcolab/core.py')},
59
+ 'fastcolab.envs': { 'fastcolab.envs.ColabRuntime.activate_venv': ('envs.html#colabruntime.activate_venv', 'fastcolab/envs.py'),
60
+ 'fastcolab.envs.ColabRuntime.create_venv': ('envs.html#colabruntime.create_venv', 'fastcolab/envs.py'),
61
+ 'fastcolab.envs.ColabRuntime.fresh_kernel': ('envs.html#colabruntime.fresh_kernel', 'fastcolab/envs.py'),
62
+ 'fastcolab.envs.ColabRuntime.install': ('envs.html#colabruntime.install', 'fastcolab/envs.py'),
63
+ 'fastcolab.envs.ColabRuntime.load_venv': ('envs.html#colabruntime.load_venv', 'fastcolab/envs.py'),
64
+ 'fastcolab.envs.ColabRuntime.pack_venv': ('envs.html#colabruntime.pack_venv', 'fastcolab/envs.py'),
65
+ 'fastcolab.envs.ColabRuntime.restart': ('envs.html#colabruntime.restart', 'fastcolab/envs.py'),
66
+ 'fastcolab.envs.ColabRuntime.save_venv': ('envs.html#colabruntime.save_venv', 'fastcolab/envs.py'),
67
+ 'fastcolab.envs.ColabRuntime.unpack_venv': ('envs.html#colabruntime.unpack_venv', 'fastcolab/envs.py'),
68
+ 'fastcolab.envs.ColabRuntime.venv_exec': ('envs.html#colabruntime.venv_exec', 'fastcolab/envs.py'),
69
+ 'fastcolab.envs._read_reqs': ('envs.html#_read_reqs', 'fastcolab/envs.py'),
70
+ 'fastcolab.envs._run_cmd': ('envs.html#_run_cmd', 'fastcolab/envs.py'),
71
+ 'fastcolab.envs._tar_flags': ('envs.html#_tar_flags', 'fastcolab/envs.py'),
72
+ 'fastcolab.envs._uv_install': ('envs.html#_uv_install', 'fastcolab/envs.py'),
73
+ 'fastcolab.envs.compile_reqs': ('envs.html#compile_reqs', 'fastcolab/envs.py')},
74
+ 'fastcolab.extras': { 'fastcolab.extras.Colab.run_script': ('extras.html#colab.run_script', 'fastcolab/extras.py'),
75
+ 'fastcolab.extras.Colab.stop_all': ('extras.html#colab.stop_all', 'fastcolab/extras.py'),
76
+ 'fastcolab.extras.Colab.usage': ('extras.html#colab.usage', 'fastcolab/extras.py'),
77
+ 'fastcolab.extras.ColabRuntime.browser_url': ( 'extras.html#colabruntime.browser_url',
78
+ 'fastcolab/extras.py'),
79
+ 'fastcolab.extras.ColabRuntime.download_dir': ( 'extras.html#colabruntime.download_dir',
80
+ 'fastcolab/extras.py'),
81
+ 'fastcolab.extras.ColabRuntime.endpoint': ('extras.html#colabruntime.endpoint', 'fastcolab/extras.py'),
82
+ 'fastcolab.extras.ColabRuntime.run_nb': ('extras.html#colabruntime.run_nb', 'fastcolab/extras.py'),
83
+ 'fastcolab.extras.ColabRuntime.run_script': ( 'extras.html#colabruntime.run_script',
84
+ 'fastcolab/extras.py'),
85
+ 'fastcolab.extras.ColabRuntime.sh': ('extras.html#colabruntime.sh', 'fastcolab/extras.py'),
86
+ 'fastcolab.extras.ColabRuntime.ssh': ('extras.html#colabruntime.ssh', 'fastcolab/extras.py'),
87
+ 'fastcolab.extras.ColabRuntime.sys_info': ('extras.html#colabruntime.sys_info', 'fastcolab/extras.py'),
88
+ 'fastcolab.extras.ColabRuntime.upload_dir': ( 'extras.html#colabruntime.upload_dir',
89
+ 'fastcolab/extras.py'),
90
+ 'fastcolab.extras.ColabUsage': ('extras.html#colabusage', 'fastcolab/extras.py'),
91
+ 'fastcolab.extras.ColabUsage.__repr__': ('extras.html#colabusage.__repr__', 'fastcolab/extras.py'),
92
+ 'fastcolab.extras.ColabUsage._repr_markdown_': ( 'extras.html#colabusage._repr_markdown_',
93
+ 'fastcolab/extras.py'),
94
+ 'fastcolab.extras.ColabUsage.assignments_count': ( 'extras.html#colabusage.assignments_count',
95
+ 'fastcolab/extras.py'),
96
+ 'fastcolab.extras.ColabUsage.balance': ('extras.html#colabusage.balance', 'fastcolab/extras.py'),
97
+ 'fastcolab.extras.ColabUsage.eligible_gpus': ( 'extras.html#colabusage.eligible_gpus',
98
+ 'fastcolab/extras.py'),
99
+ 'fastcolab.extras.ColabUsage.eligible_tpus': ( 'extras.html#colabusage.eligible_tpus',
100
+ 'fastcolab/extras.py'),
101
+ 'fastcolab.extras.ColabUsage.rate_hourly': ('extras.html#colabusage.rate_hourly', 'fastcolab/extras.py'),
102
+ 'fastcolab.extras._excluded': ('extras.html#_excluded', 'fastcolab/extras.py'),
103
+ 'fastcolab.extras._exec_json': ('extras.html#_exec_json', 'fastcolab/extras.py'),
104
+ 'fastcolab.extras._has_ssh_host': ('extras.html#_has_ssh_host', 'fastcolab/extras.py'),
105
+ 'fastcolab.extras._jwt_payload': ('extras.html#_jwt_payload', 'fastcolab/extras.py'),
106
+ 'fastcolab.extras._pack_local_dir': ('extras.html#_pack_local_dir', 'fastcolab/extras.py'),
107
+ 'fastcolab.extras._script_payload': ('extras.html#_script_payload', 'fastcolab/extras.py'),
108
+ 'fastcolab.extras._ssh_host_block': ('extras.html#_ssh_host_block', 'fastcolab/extras.py'),
109
+ 'fastcolab.extras._strip_shebang': ('extras.html#_strip_shebang', 'fastcolab/extras.py'),
110
+ 'fastcolab.extras._tunnel_get': ('extras.html#_tunnel_get', 'fastcolab/extras.py'),
111
+ 'fastcolab.extras.colab_ssh': ('extras.html#colab_ssh', 'fastcolab/extras.py')}}}
fastcolab/core.py ADDED
@@ -0,0 +1,487 @@
1
+ """Async Python client for Google Colab runtimes, kernels, files, and Drive transfers
2
+
3
+ Docs: https://AnswerDotAI.github.io/fastcolab/core.html.md"""
4
+
5
+ # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/00_core.ipynb.
6
+
7
+ # %% auto #0
8
+ __all__ = ['COLAB_SCOPE', 'DRIVE_SCOPE', 'COLAB_V1BETA_URL', 'COLAB_V1_URL', 'PROXY_HEADER', 'CHUNK_SIZE', 'HW_SPECS', 'mk_spec',
9
+ 'Colab', 'ColabRuntime', 'outs_text', 'ColabJob']
10
+
11
+ # %% ../nbs/00_core.ipynb #6161958e
12
+ import asyncio, base64, time, uuid
13
+ from pathlib import Path
14
+ from urllib.parse import quote
15
+
16
+ from fastcore.utils import *
17
+ from fastgws.core import GWSApi, oauth_creds
18
+ from fasttransport.errors import APIError
19
+ from jupyasyncclient.core import JupyAsyncKernelClient
20
+ from jupyasyncclient.files import JupyAsyncFilesClient
21
+ from jupyasyncclient.http import http_client
22
+
23
+ # %% ../nbs/00_core.ipynb #b34e275b
24
+ COLAB_SCOPE = 'https://www.googleapis.com/auth/colaboratory'
25
+ DRIVE_SCOPE = 'https://www.googleapis.com/auth/drive'
26
+ COLAB_V1BETA_URL = 'https://colaboratory.googleapis.com/$discovery/rest?version=v1beta'
27
+ COLAB_V1_URL = 'https://colaboratory.googleapis.com/$discovery/rest?version=v1'
28
+ PROXY_HEADER = 'X-Colab-Runtime-Proxy-Token'
29
+ CHUNK_SIZE = 32 * 1024 * 1024
30
+
31
+ # %% ../nbs/00_core.ipynb #c4caaa20
32
+ HW_SPECS = dict(cpu=('VARIANT_CPU', 'NONE', True), t4=('VARIANT_GPU', 'T4', True), l4=('VARIANT_GPU', 'L4', False), a100=('VARIANT_GPU', 'A100', True),
33
+ g4=('VARIANT_GPU', 'G4', False), h100=('VARIANT_GPU', 'H100', False), v5e1=('VARIANT_TPU', 'V5E1', False), v6e1=('VARIANT_TPU', 'V6E1', False))
34
+
35
+ def mk_spec(
36
+ hw:str|dict='cpu', # Hardware alias (`'cpu'`, `'t4'`, `'l4'`, `'g4'`, `'h100'`, `'v6e1'`, ...) or raw spec dict
37
+ high_mem:bool=True, # Request `SHAPE_HIGHMEM` when the hardware supports a choice?
38
+ )->dict:
39
+ "Build a Colab `runtimeSpec` dict from a short hardware alias or pass through a raw dict."
40
+ if isinstance(hw, dict): return hw
41
+ key = hw.lower().replace('-', '')
42
+ if key not in HW_SPECS: raise ValueError(f'Unknown hardware {hw!r}; expected one of {sorted(HW_SPECS)}')
43
+ variant, acc, has_hm = HW_SPECS[key]
44
+ return dict(variant=variant, accelerator=acc, shape='SHAPE_HIGHMEM' if (high_mem and has_hm) else 'SHAPE_STANDARD')
45
+
46
+ # %% ../nbs/00_core.ipynb #ce16b93b
47
+ class Colab:
48
+ "Async client for Google Colab runtimes and optional Google Drive transfers."
49
+ def __init__(
50
+ self,
51
+ v1beta, # `GWSApi` client for Colab `v1beta`
52
+ v1, # `GWSApi` client for Colab `v1` operations
53
+ creds=None, # OAuth credentials shared with Drive
54
+ drive=None, # Optional `GWSApi('drive', ...)` client
55
+ ): store_attr()
56
+
57
+ def __repr__(self): return f'Colab(drive={self.drive is not None})'
58
+
59
+ @classmethod
60
+ async def connect(
61
+ cls,
62
+ account:str=None, # Google account email stored by `gclientid`
63
+ creds=None, # Existing `google-auth` credentials (skips `oauth_creds` when given)
64
+ scopes:list=None, # OAuth scopes to request (defaults to Colab + Drive when `drive=True`)
65
+ drive:bool=True, # Also attach a `fastgws` Drive client?
66
+ **kwargs,
67
+ ):
68
+ "Authenticate and load both Colab discovery clients."
69
+ if creds is None:
70
+ sc = scopes or ([COLAB_SCOPE, DRIVE_SCOPE] if drive else [COLAB_SCOPE])
71
+ creds = await oauth_creds(account=account, scopes=sc, **kwargs)
72
+ v1b, v1 = await asyncio.gather(*(GWSApi.from_discovery_url(u, creds=creds) for u in (COLAB_V1BETA_URL, COLAB_V1_URL)))
73
+ return cls(v1b, v1, creds=creds, drive=GWSApi('drive', creds=creds) if drive else None)
74
+
75
+ # %% ../nbs/00_core.ipynb #94417fce
76
+ @patch
77
+ async def subscription(self:Colab)->str:
78
+ "Return the Colab subscription tier string for the authenticated user."
79
+ return (await self.v1beta.v1beta.get_subscription(name='subscription')).get('tier', '')
80
+
81
+ # %% ../nbs/00_core.ipynb #cc3ce649
82
+ @patch
83
+ async def specs(
84
+ self:Colab,
85
+ eligible_only:bool=True, # Filter to specs the current account can allocate?
86
+ )->L:
87
+ "List available Colab runtime specs as `AttrDict` entries."
88
+ res = await self.v1beta.runtimespecs.list()
89
+ items = L(res.get('runtimeSpecs', [])).map(lambda s: AttrDict(eligible=s.get('eligible', False), **s['key']))
90
+ return items.filter(lambda s: s.eligible) if eligible_only else items
91
+
92
+ # %% ../nbs/00_core.ipynb #6d4f3b7e
93
+ def _rt_name(name:str)->str: return name if name.startswith('runtimes/') else f'runtimes/{name}'
94
+ def _clean_path(path:str|Path)->str: return str(path).lstrip('/')
95
+
96
+ class ColabRuntime:
97
+ "Handle to an active Google Colab runtime and its Jupyter proxy."
98
+ def __init__(
99
+ self,
100
+ colab:Colab, # Parent `Colab` client
101
+ data:dict, # Runtime resource dict returned by `v1beta.runtimes`
102
+ owned:bool=False, # Delete the runtime automatically on `async with` exit?
103
+ ):
104
+ self.colab,self.data,self.owned,self._kc = colab,AttrDict(data),owned,None
105
+
106
+ @property
107
+ def name(self)->str: return self.data['name']
108
+ @property
109
+ def id(self)->str: return self.name.split('/')[-1]
110
+ @property
111
+ def spec(self)->dict: return self.data.get('runtimeSpec', {})
112
+ @property
113
+ def conn(self)->dict: return self.data.get('connectionInfo', {})
114
+ @property
115
+ def url(self)->str: return self.conn['url']
116
+ @property
117
+ def token(self)->str: return self.conn['token']
118
+ @property
119
+ def expire_time(self)->str: return self.conn.get('expireTime', '')
120
+ @property
121
+ def headers(self)->dict: return {PROXY_HEADER: self.token}
122
+
123
+ def __repr__(self): return f"ColabRuntime({self.id!r}, {self.spec.get('accelerator')}, {self.spec.get('shape')})"
124
+ def _repr_markdown_(self):
125
+ s = self.spec
126
+ return f"`ColabRuntime({self.id})` ({s.get('variant')}, {s.get('accelerator')}, {s.get('shape')}), expires `{self.expire_time}`"
127
+
128
+ # %% ../nbs/00_core.ipynb #ca44372c
129
+ async def _wait_op(ops, op:dict, timeout:float=180, poll:float=2.0)->dict:
130
+ "Wait for a Colab `v1.operations` resource to finish and return its response payload."
131
+ deadline = time.monotonic() + timeout
132
+ while not op.get('done'):
133
+ rem = deadline - time.monotonic()
134
+ if rem <= 0: raise TimeoutError(f"Colab operation {op.get('name')} timed out after {timeout}s")
135
+ t0 = time.monotonic()
136
+ try: op = await ops.operations.wait(name=op['name'], timeout=f'{max(1, int(min(30, rem)))}s')
137
+ except APIError: op = await ops.operations.get(name=op['name'])
138
+ if not op.get('done') and (time.monotonic() - t0) < poll: await asyncio.sleep(poll)
139
+ if op.get('error'): raise RuntimeError(f"Colab operation failed: {op['error']}")
140
+ return op.get('response', op)
141
+
142
+ # %% ../nbs/00_core.ipynb #beefe1ca
143
+ @patch
144
+ async def start(
145
+ self:Colab,
146
+ hw:str|dict='cpu', # Hardware alias or raw `runtimeSpec` dict
147
+ high_mem:bool=True, # Request `SHAPE_HIGHMEM` when supported?
148
+ runtime_id:str=None, # Optional RFC-1034 runtime ID
149
+ version:str=None, # Optional Colab runtime image version
150
+ timeout:float=180, # Seconds to wait for provisioning
151
+ )->ColabRuntime:
152
+ "Provision a Colab runtime, wait until it is ready, and return a `ColabRuntime`."
153
+ kw = dict(runtime_spec=mk_spec(hw, high_mem=high_mem), request_id=str(uuid.uuid4()))
154
+ if runtime_id: kw['runtime_id'] = runtime_id
155
+ if version: kw['version'] = version
156
+ data = await _wait_op(self.v1, await self.v1beta.runtimes.create(**kw), timeout=timeout)
157
+ if 'connectionInfo' not in data and 'name' in data: data = await self.v1beta.runtimes.get(name=data['name'])
158
+ return ColabRuntime(self, data, owned=True)
159
+
160
+ # %% ../nbs/00_core.ipynb #dddbc614
161
+ @patch
162
+ async def runtimes(self:Colab)->L:
163
+ "List currently active Colab runtimes."
164
+ return L((await self.v1beta.runtimes.list()).get('runtimes', [])).map(lambda d: ColabRuntime(self, d))
165
+
166
+ @patch
167
+ async def get(
168
+ self:Colab,
169
+ name:str, # Full resource name (`runtimes/{id}`) or bare runtime ID
170
+ )->ColabRuntime:
171
+ "Fetch one active runtime with fresh `connectionInfo`."
172
+ return ColabRuntime(self, await self.v1beta.runtimes.get(name=_rt_name(name)))
173
+
174
+ @patch
175
+ async def delete(
176
+ self:Colab,
177
+ name:str, # Full resource name (`runtimes/{id}`) or bare runtime ID
178
+ allow_missing:bool=True, # Succeed silently if the runtime is already gone?
179
+ ):
180
+ "Delete a Colab runtime assignment."
181
+ return await self.v1beta.runtimes.delete(name=_rt_name(name), allow_missing=allow_missing)
182
+
183
+ # %% ../nbs/00_core.ipynb #c334ccd6
184
+ @patch
185
+ async def refresh(self:ColabRuntime)->dict:
186
+ "Refresh `connectionInfo` via `v1beta.runtimes.get` and return updated proxy headers."
187
+ self.data = AttrDict(await self.colab.v1beta.runtimes.get(name=self.name))
188
+ return self.headers
189
+
190
+ # %% ../nbs/00_core.ipynb #b2e41838
191
+ def outs_text(outs:list)->str:
192
+ "Join stdout, stderr, execute_result text, and error tracebacks from nbformat outputs."
193
+ parts = []
194
+ for o in outs:
195
+ t = o.get('output_type')
196
+ if t == 'stream': parts.append(o.get('text', ''))
197
+ elif t in ('execute_result', 'display_data'):
198
+ txt = o.get('data', {}).get('text/plain')
199
+ if txt is not None: parts.append(f'{txt}\n')
200
+ elif t == 'error': parts.append('\n'.join(o.get('traceback', [])) or f"{o.get('ename')}: {o.get('evalue')}\n")
201
+ return ''.join(parts)
202
+
203
+ # %% ../nbs/00_core.ipynb #91de7cf5
204
+ @patch
205
+ async def kernel(
206
+ self:ColabRuntime,
207
+ kernel_id:str=None, # Existing kernel ID to attach to, or `None` to start a new kernel
208
+ timeout:float=60, # Connection timeout in seconds
209
+ )->JupyAsyncKernelClient:
210
+ "Connect a `JupyAsyncKernelClient` to this runtime's Jupyter proxy."
211
+ return await JupyAsyncKernelClient.connect(
212
+ self.url, kernel=kernel_id, headers=self.headers, refresh_headers=self.refresh, timeout=timeout)
213
+
214
+ @patch
215
+ async def exec(
216
+ self:ColabRuntime,
217
+ code:str, # Python source to execute on the runtime's default kernel
218
+ timeout:float=None, # Execution timeout in seconds (`None` waits indefinitely)
219
+ )->list:
220
+ "Execute `code` on `rt`'s persistent kernel and return nbformat outputs."
221
+ if self._kc is None or not self._kc.channels_running: self._kc = await self.kernel()
222
+ return await self._kc.exec_outs(code, timeout=timeout)
223
+
224
+ # %% ../nbs/00_core.ipynb #4eca3914
225
+ @patch
226
+ def files(self:ColabRuntime)->JupyAsyncFilesClient:
227
+ "Return a `JupyAsyncFilesClient` for this runtime's Jupyter contents API."
228
+ return JupyAsyncFilesClient(self.url, headers=self.headers)
229
+
230
+ async def _ensure_parent_dirs(rt:ColabRuntime, remote_path:str):
231
+ parts = _clean_path(remote_path).split('/')[:-1]
232
+ fc, cur = rt.files(), ''
233
+ for p in parts:
234
+ cur = f'{cur}/{p}' if cur else p
235
+ if cur == 'content': continue
236
+ try: await fc.put(cur, type='directory')
237
+ except APIError as e:
238
+ if e.status_code not in (400, 409): raise
239
+
240
+ # %% ../nbs/00_core.ipynb #7e23922e
241
+ @patch
242
+ async def upload(
243
+ self:ColabRuntime,
244
+ src:str|Path|bytes, # Local file path (`Path`) or raw `bytes`/`str` content to upload
245
+ remote_path:str, # Destination path on the runtime (e.g. `'content/data/file.csv'`)
246
+ chunk_size:int=CHUNK_SIZE, # Chunk size in bytes for multi-part uploads (must stay under 50 MB)
247
+ ):
248
+ "Upload bytes, text, or a local file to `remote_path`, creating parent directories and chunking large files."
249
+ rpath = _clean_path(remote_path)
250
+ await _ensure_parent_dirs(self, rpath)
251
+ data = src.read_bytes() if isinstance(src, Path) else (src.encode('utf-8') if isinstance(src, str) else bytes(src))
252
+ fc = self.files()
253
+ if len(data) <= chunk_size: return await fc.put(rpath, type='file', format='base64', content=base64.b64encode(data).decode('ascii'))
254
+ n, res = (len(data) + chunk_size - 1) // chunk_size, None
255
+ for i in range(n):
256
+ piece = base64.b64encode(data[i*chunk_size:(i+1)*chunk_size]).decode('ascii')
257
+ res = await fc.put(rpath, type='file', format='base64', content=piece, chunk=-1 if i == n - 1 else i + 1)
258
+ return res
259
+
260
+ # %% ../nbs/00_core.ipynb #787f84ff
261
+ @patch
262
+ async def download(
263
+ self:ColabRuntime,
264
+ remote_path:str, # Source path on the runtime (e.g. `'content/train.log'`)
265
+ local_path:str|Path=None, # Optional local path to write the downloaded bytes to
266
+ offset:int=0, # Byte offset for HTTP `Range` reads
267
+ )->bytes|Path:
268
+ "Download raw bytes from `/files/{remote_path}`, supporting HTTP `Range` offsets and automatic token refresh."
269
+ rpath = quote(_clean_path(remote_path), safe='/')
270
+ for attempt in range(2):
271
+ hdrs = dict(self.headers, Range=f'bytes={offset}-') if offset > 0 else dict(self.headers)
272
+ async with http_client(self.url, headers=hdrs, timeout=60.0) as hc: resp = await hc.get(f'/files/{rpath}')
273
+ if resp.status_code == 404 and attempt == 0:
274
+ await self.refresh()
275
+ continue
276
+ if resp.status_code == 416:
277
+ data = b''
278
+ break
279
+ if resp.status_code >= 400:
280
+ raise FileNotFoundError(remote_path) if resp.status_code == 404 else RuntimeError(f'GET /files/{rpath} failed: HTTP {resp.status_code}')
281
+ data = resp.content[offset:] if (offset > 0 and resp.status_code == 200) else resp.content
282
+ break
283
+ if local_path is None: return data
284
+ p = Path(local_path)
285
+ p.parent.mkdir(parents=True, exist_ok=True)
286
+ p.write_bytes(data)
287
+ return p
288
+
289
+ # %% ../nbs/00_core.ipynb #b586bbb4
290
+ @patch
291
+ async def tail(
292
+ self:ColabRuntime,
293
+ remote_path:str, # Log file path on the runtime
294
+ offset:int=0, # Byte offset returned by the previous `tail` call
295
+ )->tuple[str, int]:
296
+ "Read newly appended UTF-8 text from `remote_path` starting at `offset`, returning `(text, new_offset)`."
297
+ raw = await self.download(remote_path, offset=offset)
298
+ return raw.decode('utf-8', errors='replace'), offset + len(raw)
299
+
300
+ # %% ../nbs/00_core.ipynb #80b63bae
301
+ _FOLDER_MIME = 'application/vnd.google-apps.folder'
302
+ def _esc_q(s:str)->str: return s.replace('\\', '\\\\').replace("'", "\\'")
303
+
304
+ async def _drive_folder_id(
305
+ drive, # `GWSApi('drive', ...)` client
306
+ folder:str, # Folder name, slash-separated path (e.g. `'datasets/demo'`), or Drive folder ID
307
+ create:bool=False, # Create missing folders along the path?
308
+ )->str:
309
+ "Resolve a Google Drive folder path or name to its Drive file ID."
310
+ parent, parts = 'root', [p for p in folder.strip('/').split('/') if p]
311
+ for name in parts:
312
+ q = f"name = '{_esc_q(name)}' and mimeType = '{_FOLDER_MIME}' and '{parent}' in parents and trashed = false"
313
+ files = (await drive.files.list(q=q, fields='files(id, name)', page_size=10)).get('files', [])
314
+ if files: parent = files[0]['id']
315
+ elif len(parts) == 1 and not create:
316
+ meta = await drive.files.get(file_id=name, fields='id, mimeType, trashed')
317
+ if meta.get('mimeType') == _FOLDER_MIME and not meta.get('trashed'): return meta['id']
318
+ raise FileNotFoundError(f'Drive folder not found: {folder!r}')
319
+ elif create: parent = (await drive.files.create(name=name, mime_type=_FOLDER_MIME, parents=[parent], fields='id'))['id']
320
+ else: raise FileNotFoundError(f'Drive folder not found: {folder!r}')
321
+ return parent
322
+
323
+ # %% ../nbs/00_core.ipynb #db62c4b0
324
+ @patch
325
+ async def from_drive(
326
+ self:ColabRuntime,
327
+ folder:str, # Google Drive folder name, path, or folder ID
328
+ remote_dir:str='content/data', # Destination directory on the Colab runtime
329
+ names:list=None, # Optional subset of filenames to copy (defaults to all non-folder files)
330
+ )->L:
331
+ "Copy files from a Google Drive folder onto this Colab runtime, returning uploaded remote paths."
332
+ drive = self.colab.drive
333
+ if drive is None: raise RuntimeError('Drive client is disabled; pass drive=True to Colab.connect()')
334
+ fid = await _drive_folder_id(drive, folder, create=False)
335
+ q = f"'{fid}' in parents and mimeType != '{_FOLDER_MIME}' and trashed = false"
336
+ entries = (await drive.files.list(q=q, fields='files(id, name, mimeType)', page_size=100)).get('files', [])
337
+ if names is not None: entries = [e for e in entries if e['name'] in set(names)]
338
+ dest, uploaded = _clean_path(remote_dir).rstrip('/'), L()
339
+ for e in entries:
340
+ resp = await drive.files.get(file_id=e['id'], alt='media', raw_=True)
341
+ rpath = f"{dest}/{e['name']}"
342
+ await self.upload(resp.content, rpath)
343
+ uploaded.append(rpath)
344
+ return uploaded
345
+
346
+ # %% ../nbs/00_core.ipynb #9a4b3016
347
+ @patch
348
+ async def to_drive(
349
+ self:ColabRuntime,
350
+ remote_path:str, # File path on the Colab runtime to save
351
+ folder:str, # Destination Google Drive folder name, path, or folder ID
352
+ name:str=None, # Filename in Drive (defaults to the basename of `remote_path`)
353
+ create_folder:bool=True, # Create the destination Drive folder if missing?
354
+ )->dict:
355
+ "Copy a file from this Colab runtime into a Google Drive folder, updating in place if it already exists."
356
+ drive = self.colab.drive
357
+ if drive is None: raise RuntimeError('Drive client is disabled; pass drive=True to Colab.connect()')
358
+ fid = await _drive_folder_id(drive, folder, create=create_folder)
359
+ fname, data = name or Path(remote_path).name, await self.download(remote_path)
360
+ q = f"name = '{_esc_q(fname)}' and '{fid}' in parents and trashed = false"
361
+ existing = (await drive.files.list(q=q, fields='files(id, name)', page_size=5)).get('files', [])
362
+ if existing: return await drive.files.update_media(file_id=existing[0]['id'], media=data, fields='id, name')
363
+ return await drive.files.upload(media=data, name=fname, parents=[fid], fields='id, name')
364
+
365
+ # %% ../nbs/00_core.ipynb #5f032f88
366
+ def _job_wrapper(code:str, log_path:str, exit_path:str)->str:
367
+ return f'''import sys, traceback
368
+ _lf = open({log_path!r}, 'w', buffering=1)
369
+ _o, _e = sys.stdout, sys.stderr
370
+ sys.stdout = sys.stderr = _lf
371
+ _rc = 0
372
+ try: exec({code!r}, {{'__name__': '__main__'}})
373
+ except SystemExit as _ex: _rc = _ex.code if isinstance(_ex.code, int) else (1 if _ex.code else 0)
374
+ except BaseException:
375
+ traceback.print_exc()
376
+ _rc = 1
377
+ finally:
378
+ sys.stdout, sys.stderr = _o, _e
379
+ _lf.close()
380
+ open({exit_path!r}, 'w').write(str(_rc))
381
+ '''
382
+
383
+ class ColabJob:
384
+ "Handle to a detached cell running inside a Colab kernel, polled over HTTP."
385
+ def __init__(
386
+ self,
387
+ rt:ColabRuntime, # Runtime hosting the job
388
+ kernel_id:str, # Remote Jupyter kernel ID executing the job
389
+ log:str, # Log file path on the runtime (without leading slash)
390
+ exit_file:str, # Exit-code file path on the runtime (without leading slash)
391
+ ):
392
+ store_attr()
393
+ self.offset = 0
394
+
395
+ def __repr__(self): return f'ColabJob({self.kernel_id!r}, log={self.log!r})'
396
+
397
+ # %% ../nbs/00_core.ipynb #c759c84f
398
+ @patch
399
+ async def launch(
400
+ self:ColabRuntime,
401
+ code:str, # Python source to execute in a dedicated detached kernel
402
+ log:str='content/job.log', # Remote log file path
403
+ exit_file:str='content/job.exit', # Remote exit-code file path
404
+ )->ColabJob:
405
+ "Submit `code` to a dedicated kernel, close the local WebSocket once busy, and return a `ColabJob`."
406
+ log_rel, exit_rel = _clean_path(log), _clean_path(exit_file)
407
+ await _ensure_parent_dirs(self, log_rel)
408
+ await _ensure_parent_dirs(self, exit_rel)
409
+ fc = self.files()
410
+ for p in (log_rel, exit_rel):
411
+ try: await fc.delete(p)
412
+ except APIError: pass
413
+ kc = await self.kernel()
414
+ gen = kc.run(_job_wrapper(code, '/' + log_rel, '/' + exit_rel))
415
+ try:
416
+ async for msg in gen:
417
+ if msg.get('header', {}).get('msg_type') == 'status' and msg.get('content', {}).get('execution_state') == 'busy': break
418
+ finally:
419
+ await gen.aclose()
420
+ await kc.aclose()
421
+ return ColabJob(self, kc.kernel_id, log_rel, exit_rel)
422
+
423
+ # %% ../nbs/00_core.ipynb #c7792d13
424
+ @patch
425
+ async def tail(self:ColabJob)->str:
426
+ "Return newly written log text since the last `tail` call."
427
+ txt, self.offset = await self.rt.tail(self.log, offset=self.offset)
428
+ return txt
429
+
430
+ @patch
431
+ async def status(self:ColabJob)->tuple[str, int|None]:
432
+ "Return `(state, exit_code)` where `state` is `'running'`, `'done'`, `'failed'`, `'idle'`, or `'gone'`."
433
+ try:
434
+ rc = int((await self.rt.download(self.exit_file)).decode().strip())
435
+ return ('done' if rc == 0 else 'failed'), rc
436
+ except FileNotFoundError: pass
437
+ for attempt in range(2):
438
+ async with http_client(self.rt.url, headers=self.rt.headers, timeout=30.0) as hc: resp = await hc.get(f'/api/kernels/{self.kernel_id}')
439
+ if resp.status_code == 404 and attempt == 0:
440
+ await self.rt.refresh()
441
+ continue
442
+ if resp.status_code == 404: return 'gone', None
443
+ state = resp.json().get('execution_state', '')
444
+ return ('running' if state == 'busy' else (state or 'unknown')), None
445
+ return 'gone', None
446
+
447
+ # %% ../nbs/00_core.ipynb #67d03210
448
+ @patch
449
+ async def wait(
450
+ self:ColabJob,
451
+ poll:float=1.0, # Seconds between status polls
452
+ on_log=None, # Optional callback receiving each newly tailed log string
453
+ )->int:
454
+ "Poll until the job writes its exit file (or the kernel disappears), returning the exit code."
455
+ while True:
456
+ chunk = await self.tail()
457
+ if chunk and on_log: on_log(chunk)
458
+ st, rc = await self.status()
459
+ if rc is not None:
460
+ rem = await self.tail()
461
+ if rem and on_log: on_log(rem)
462
+ return rc
463
+ if st in ('gone', 'idle'): raise RuntimeError(f'Detached job kernel ended in state {st!r} without writing {self.exit_file}')
464
+ await asyncio.sleep(poll)
465
+
466
+ # %% ../nbs/00_core.ipynb #01bce225
467
+ @patch
468
+ async def stop(
469
+ self:ColabRuntime,
470
+ allow_missing:bool=True, # Ignore already-deleted runtimes?
471
+ ):
472
+ "Close any local kernel socket and delete this Colab runtime."
473
+ if self._kc is not None:
474
+ try: await self._kc.aclose()
475
+ except Exception: pass
476
+ self._kc = None
477
+ return await self.colab.delete(self.name, allow_missing=allow_missing)
478
+
479
+ @patch
480
+ async def __aenter__(self:ColabRuntime): return self
481
+
482
+ @patch
483
+ async def __aexit__(self:ColabRuntime, *exc):
484
+ if self.owned: await self.stop()
485
+ elif self._kc is not None:
486
+ await self._kc.aclose()
487
+ self._kc = None