framework-py 0.1.0.dev26__py3-none-any.whl → 0.1.0.dev27__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- framework/__init__.py +3 -0
- framework/cloud.py +4 -1
- framework/snippets.py +728 -0
- {framework_py-0.1.0.dev26.dist-info → framework_py-0.1.0.dev27.dist-info}/METADATA +1 -1
- {framework_py-0.1.0.dev26.dist-info → framework_py-0.1.0.dev27.dist-info}/RECORD +7 -6
- {framework_py-0.1.0.dev26.dist-info → framework_py-0.1.0.dev27.dist-info}/WHEEL +0 -0
- {framework_py-0.1.0.dev26.dist-info → framework_py-0.1.0.dev27.dist-info}/licenses/LICENSE +0 -0
framework/__init__.py
CHANGED
|
@@ -131,6 +131,8 @@ from framework.security import AuditResult, git_security_audit
|
|
|
131
131
|
from framework.settings import settings, settings_read, settings_write
|
|
132
132
|
from framework.skill_pull import pull_skills, push_skills
|
|
133
133
|
from framework.skill_sync import skills_sync
|
|
134
|
+
from framework.snippets import load_ipython_extension as load_ipython_extension
|
|
135
|
+
from framework.snippets import snippet
|
|
134
136
|
from framework.spec.check import spec_check
|
|
135
137
|
from framework.spec.merge import SpecError
|
|
136
138
|
from framework.spec.resolve import ResolvedSpec, spec_resolve
|
|
@@ -274,6 +276,7 @@ __all__ = [
|
|
|
274
276
|
"settings_write",
|
|
275
277
|
"show_table",
|
|
276
278
|
"skills_sync",
|
|
279
|
+
"snippet",
|
|
277
280
|
"spec_check",
|
|
278
281
|
"spec_resolve",
|
|
279
282
|
"status",
|
framework/cloud.py
CHANGED
|
@@ -1269,7 +1269,10 @@ def cloud_publish(
|
|
|
1269
1269
|
if response.is_error:
|
|
1270
1270
|
raise _error(response, "Publish")
|
|
1271
1271
|
result = response.json()
|
|
1272
|
-
if result.get("version") is not None:
|
|
1272
|
+
if result.get("unchanged") and result.get("version") is not None:
|
|
1273
|
+
# Cloud keeps the newest version when nothing changed (code snippets)
|
|
1274
|
+
inform(f"Unchanged, still version {result['version']}: {result.get('url')}")
|
|
1275
|
+
elif result.get("version") is not None:
|
|
1273
1276
|
inform(f"Published version {result['version']}: {result.get('url')}")
|
|
1274
1277
|
else:
|
|
1275
1278
|
inform(f"Published: {result.get('url')}")
|
framework/snippets.py
ADDED
|
@@ -0,0 +1,728 @@
|
|
|
1
|
+
"""Code snippets: a cell's or a block's code and the text it printed.
|
|
2
|
+
|
|
3
|
+
A code snippet is a ``framework.artifact/v1`` document with ``kind: "code_snippet"``.
|
|
4
|
+
Framework Cloud highlights the code, styles ANSI output, and applies the size limits,
|
|
5
|
+
so this module only records what ran and what it printed.
|
|
6
|
+
|
|
7
|
+
In Jupyter, ``%load_ext framework`` adds the ``%%fw_snippet`` cell magic. In a script,
|
|
8
|
+
``with fw.snippet("label"):`` records the block.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import ast
|
|
14
|
+
import json
|
|
15
|
+
import linecache
|
|
16
|
+
import os
|
|
17
|
+
import re
|
|
18
|
+
import subprocess
|
|
19
|
+
import sys
|
|
20
|
+
import tempfile
|
|
21
|
+
import traceback
|
|
22
|
+
from collections.abc import Iterator, Sequence
|
|
23
|
+
from contextlib import contextmanager
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
from typing import Any, TextIO
|
|
26
|
+
|
|
27
|
+
from framework._log import inform
|
|
28
|
+
from framework.errors import FrameworkError
|
|
29
|
+
|
|
30
|
+
SCHEMA = "framework.artifact/v1"
|
|
31
|
+
KIND = "code_snippet"
|
|
32
|
+
LINK_RELS = ("notebook", "model")
|
|
33
|
+
_OPTION_LINE = re.compile(r"^\s*#\|")
|
|
34
|
+
_SLUG = re.compile(r"^[A-Za-z0-9_-]{1,120}$")
|
|
35
|
+
_NOTEBOOK_SUFFIXES = (".ipynb", ".qmd", ".md")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
# ---------------------------------------------------------------------------
|
|
39
|
+
# The document
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _links(links: dict[str, str] | None) -> list[dict[str, str]]:
|
|
43
|
+
"""``display.links`` from ``{"notebook": "https://...", "model": "slug or URL"}``."""
|
|
44
|
+
if not links:
|
|
45
|
+
return []
|
|
46
|
+
if not isinstance(links, dict):
|
|
47
|
+
raise FrameworkError(
|
|
48
|
+
'links must be a dict such as {"notebook": "https://...", "model": "model-slug"}.'
|
|
49
|
+
)
|
|
50
|
+
out: list[dict[str, str]] = []
|
|
51
|
+
for rel, value in links.items():
|
|
52
|
+
if rel not in LINK_RELS:
|
|
53
|
+
raise FrameworkError(
|
|
54
|
+
'links must be named "notebook" or "model", as in '
|
|
55
|
+
'{"notebook": "https://...", "model": "model-slug"}.'
|
|
56
|
+
)
|
|
57
|
+
if not isinstance(value, str) or not value:
|
|
58
|
+
raise FrameworkError("Each link must be one string.")
|
|
59
|
+
if re.fullmatch(r"https://\S+", value):
|
|
60
|
+
out.append({"rel": rel, "url": value})
|
|
61
|
+
elif rel == "model" and _SLUG.fullmatch(value):
|
|
62
|
+
out.append({"rel": rel, "artifact": value})
|
|
63
|
+
elif rel == "model":
|
|
64
|
+
raise FrameworkError(
|
|
65
|
+
"A model link takes an artifact slug in this project or an https URL."
|
|
66
|
+
)
|
|
67
|
+
else:
|
|
68
|
+
raise FrameworkError("A notebook link takes an https URL.")
|
|
69
|
+
return out
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _document(
|
|
73
|
+
blocks: Sequence[dict[str, Any]],
|
|
74
|
+
source: dict[str, Any],
|
|
75
|
+
*,
|
|
76
|
+
output: bool = True,
|
|
77
|
+
folded: bool | None = None,
|
|
78
|
+
links: dict[str, str] | None = None,
|
|
79
|
+
) -> dict[str, Any] | None:
|
|
80
|
+
"""The snippet document, or None when it has no code.
|
|
81
|
+
|
|
82
|
+
Adjacent output blocks join, as a notebook shows them. With ``output=False`` only
|
|
83
|
+
the code is kept.
|
|
84
|
+
"""
|
|
85
|
+
kept = [dict(block) for block in blocks if block.get("text")]
|
|
86
|
+
if not output:
|
|
87
|
+
kept = [block for block in kept if block["type"] == "code"]
|
|
88
|
+
merged: list[dict[str, Any]] = []
|
|
89
|
+
for block in kept:
|
|
90
|
+
if merged and merged[-1]["type"] == block["type"] and block["type"] in ("code", "output"):
|
|
91
|
+
merged[-1]["text"] = merged[-1]["text"] + "\n" + block["text"]
|
|
92
|
+
else:
|
|
93
|
+
merged.append(block)
|
|
94
|
+
if not any(block["type"] == "code" for block in merged):
|
|
95
|
+
return None
|
|
96
|
+
if folded is not None:
|
|
97
|
+
for block in merged:
|
|
98
|
+
if block["type"] == "output":
|
|
99
|
+
block["folded"] = folded
|
|
100
|
+
document: dict[str, Any] = {
|
|
101
|
+
"schema": SCHEMA,
|
|
102
|
+
"kind": KIND,
|
|
103
|
+
"source": {key: value for key, value in source.items() if value is not None},
|
|
104
|
+
"snippet": {"blocks": merged},
|
|
105
|
+
}
|
|
106
|
+
shown = _links(links)
|
|
107
|
+
if shown:
|
|
108
|
+
document["display"] = {"links": shown}
|
|
109
|
+
return document
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _trim(text: str) -> str:
|
|
113
|
+
"""Line endings as ``\\n``, without the trailing newline a print adds."""
|
|
114
|
+
text = text.replace("\r\n", "\n").replace("\r", "\n")
|
|
115
|
+
return text[:-1] if text.endswith("\n") else text
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
class _Recorder:
|
|
119
|
+
"""Blocks in the order they were printed. Text of one kind in a row joins."""
|
|
120
|
+
|
|
121
|
+
def __init__(self) -> None:
|
|
122
|
+
self.blocks: list[dict[str, Any]] = []
|
|
123
|
+
self._open: str | None = None
|
|
124
|
+
# Set while IPython writes a display or a result, which a terminal prints to stdout
|
|
125
|
+
self.muted = False
|
|
126
|
+
|
|
127
|
+
def stream(self, kind: str, text: str) -> None:
|
|
128
|
+
if not text or self.muted:
|
|
129
|
+
return
|
|
130
|
+
if self._open == kind and self.blocks:
|
|
131
|
+
self.blocks[-1]["text"] += text
|
|
132
|
+
else:
|
|
133
|
+
self.blocks.append({"type": kind, "text": text})
|
|
134
|
+
self._open = kind
|
|
135
|
+
|
|
136
|
+
def block(self, kind: str, text: str) -> None:
|
|
137
|
+
self._open = None
|
|
138
|
+
self.blocks.append({"type": kind, "text": text})
|
|
139
|
+
|
|
140
|
+
def finished(self) -> list[dict[str, Any]]:
|
|
141
|
+
return [{**block, "text": _trim(block["text"])} for block in self.blocks]
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
class _Tee:
|
|
145
|
+
"""A text stream that writes through and records what passes."""
|
|
146
|
+
|
|
147
|
+
def __init__(self, target: TextIO, record: Any) -> None:
|
|
148
|
+
self._target = target
|
|
149
|
+
self._record = record
|
|
150
|
+
|
|
151
|
+
def write(self, text: str) -> int:
|
|
152
|
+
self._record(text)
|
|
153
|
+
return self._target.write(text)
|
|
154
|
+
|
|
155
|
+
def __getattr__(self, name: str) -> Any:
|
|
156
|
+
return getattr(self._target, name)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
@contextmanager
|
|
160
|
+
def _teed(recorder: _Recorder) -> Iterator[None]:
|
|
161
|
+
"""Record stdout as output and stderr as messages, still printing both."""
|
|
162
|
+
stdout, stderr = sys.stdout, sys.stderr
|
|
163
|
+
sys.stdout = _Tee(stdout, lambda text: recorder.stream("output", text))
|
|
164
|
+
sys.stderr = _Tee(stderr, lambda text: recorder.stream("message", text))
|
|
165
|
+
try:
|
|
166
|
+
yield
|
|
167
|
+
finally:
|
|
168
|
+
sys.stdout, sys.stderr = stdout, stderr
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
# ---------------------------------------------------------------------------
|
|
172
|
+
# Where the code came from
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _git(directory: Path, *args: str) -> str | None:
|
|
176
|
+
try:
|
|
177
|
+
result = subprocess.run(
|
|
178
|
+
["git", "-C", str(directory), *args],
|
|
179
|
+
capture_output=True,
|
|
180
|
+
encoding="utf-8",
|
|
181
|
+
errors="replace",
|
|
182
|
+
check=False,
|
|
183
|
+
)
|
|
184
|
+
except OSError:
|
|
185
|
+
return None
|
|
186
|
+
value = result.stdout.strip()
|
|
187
|
+
return value if result.returncode == 0 and value else None
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _location(path: Path | None) -> dict[str, Any]:
|
|
191
|
+
"""The file's path from the git repository's top (else the project root), and the commit."""
|
|
192
|
+
if path is None or not path.is_file():
|
|
193
|
+
return {"file": None, "commit": None}
|
|
194
|
+
path = path.resolve()
|
|
195
|
+
top = _git(path.parent, "rev-parse", "--show-toplevel")
|
|
196
|
+
commit = None
|
|
197
|
+
root: Path | None
|
|
198
|
+
if top is not None:
|
|
199
|
+
root = Path(top).resolve()
|
|
200
|
+
head = _git(path.parent, "rev-parse", "HEAD")
|
|
201
|
+
if head is not None and re.fullmatch(r"[0-9a-f]{40}", head):
|
|
202
|
+
commit = head
|
|
203
|
+
else:
|
|
204
|
+
from framework.project import find_settings_file
|
|
205
|
+
|
|
206
|
+
found = find_settings_file(path.parent)
|
|
207
|
+
root = found.parent.resolve() if found is not None else None
|
|
208
|
+
try:
|
|
209
|
+
file = path.relative_to(root).as_posix() if root is not None else path.name
|
|
210
|
+
except ValueError:
|
|
211
|
+
file = path.name
|
|
212
|
+
return {"file": file, "commit": commit}
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _repository(repository: str | None) -> str | None:
|
|
216
|
+
"""The argument, else ``publish.repository`` in settings.yml. Sent only when set."""
|
|
217
|
+
if repository is None:
|
|
218
|
+
try:
|
|
219
|
+
from framework.settings import settings_read
|
|
220
|
+
|
|
221
|
+
publish = settings_read().get("publish")
|
|
222
|
+
repository = publish.get("repository") if isinstance(publish, dict) else None
|
|
223
|
+
except Exception:
|
|
224
|
+
repository = None
|
|
225
|
+
if repository is None:
|
|
226
|
+
return None
|
|
227
|
+
if not isinstance(repository, str) or not repository.startswith("https://"):
|
|
228
|
+
raise FrameworkError(
|
|
229
|
+
"repository must be an https URL, such as https://github.com/owner/repo."
|
|
230
|
+
)
|
|
231
|
+
return repository
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _source(
|
|
235
|
+
path: Path | None,
|
|
236
|
+
label: str,
|
|
237
|
+
lines: tuple[int, int] | None,
|
|
238
|
+
repository: str | None,
|
|
239
|
+
) -> dict[str, Any]:
|
|
240
|
+
location = _location(path)
|
|
241
|
+
return {
|
|
242
|
+
"language": "python",
|
|
243
|
+
"file": location["file"],
|
|
244
|
+
"label": label,
|
|
245
|
+
"lines": list(lines) if lines is not None and location["file"] is not None else None,
|
|
246
|
+
"commit": location["commit"],
|
|
247
|
+
"repository": _repository(repository),
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
# ---------------------------------------------------------------------------
|
|
252
|
+
# Publishing
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def _publish(document: dict[str, Any], label: str, options: dict[str, Any]) -> str | list[str]:
|
|
256
|
+
"""Publish a snippet document as ``code_snippet.json``."""
|
|
257
|
+
from framework.publish_bundle import artifact_slug
|
|
258
|
+
from framework.storage import publish
|
|
259
|
+
|
|
260
|
+
options = dict(options)
|
|
261
|
+
options["slug"] = options.get("slug") or artifact_slug(label)
|
|
262
|
+
options["title"] = options.get("title") or label
|
|
263
|
+
with tempfile.TemporaryDirectory(prefix="fw_snippet_") as temp:
|
|
264
|
+
path = Path(temp) / "code_snippet.json"
|
|
265
|
+
path.write_text(json.dumps(document, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
266
|
+
return publish(path, kind=KIND, **options)
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
_PUBLISH_OPTIONS = (
|
|
270
|
+
"slug",
|
|
271
|
+
"title",
|
|
272
|
+
"description",
|
|
273
|
+
"visibility",
|
|
274
|
+
"password",
|
|
275
|
+
"collection",
|
|
276
|
+
"share_unlisted",
|
|
277
|
+
"to",
|
|
278
|
+
"include_private",
|
|
279
|
+
)
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
# ---------------------------------------------------------------------------
|
|
283
|
+
# Scripts: with fw.snippet("label"):
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
@contextmanager
|
|
287
|
+
def snippet(
|
|
288
|
+
label: str,
|
|
289
|
+
*,
|
|
290
|
+
output: bool = True,
|
|
291
|
+
folded: bool | None = None,
|
|
292
|
+
links: dict[str, str] | None = None,
|
|
293
|
+
repository: str | None = None,
|
|
294
|
+
publish: bool = True,
|
|
295
|
+
slug: str | None = None,
|
|
296
|
+
title: str | None = None,
|
|
297
|
+
description: str | None = None,
|
|
298
|
+
visibility: str | None = None,
|
|
299
|
+
password: str | None = None,
|
|
300
|
+
collection: str | int | None = None,
|
|
301
|
+
share_unlisted: bool = False,
|
|
302
|
+
to: str | None = None,
|
|
303
|
+
include_private: bool = False,
|
|
304
|
+
) -> Iterator[None]:
|
|
305
|
+
"""Run a block and publish its code and printed output as a code snippet.
|
|
306
|
+
|
|
307
|
+
::
|
|
308
|
+
|
|
309
|
+
with fw.snippet("fit-ols"):
|
|
310
|
+
model = smf.ols("score ~ income", data=df).fit()
|
|
311
|
+
print(model.summary())
|
|
312
|
+
|
|
313
|
+
The block runs as usual and its output still prints. When it ends, the block's
|
|
314
|
+
code and what it printed to stdout (as output) and stderr (as messages) are
|
|
315
|
+
published to Framework Cloud as a code snippet. A script shows no expression
|
|
316
|
+
values, so none are recorded. If the block raises, the snippet ends with the
|
|
317
|
+
traceback, is published, and the exception continues.
|
|
318
|
+
|
|
319
|
+
The code comes from the source file, so ``with fw.snippet()`` needs a script
|
|
320
|
+
or a module, not the interactive prompt. In Jupyter, use the ``%%fw_snippet``
|
|
321
|
+
cell magic instead (``%load_ext framework``).
|
|
322
|
+
|
|
323
|
+
The snippet names its source: the file's path from the git repository's top
|
|
324
|
+
(or the project root), the label, the block's lines, and the commit checked
|
|
325
|
+
out. The repository URL is sent only when ``repository`` is given or
|
|
326
|
+
``publish.repository`` is set in settings.yml. Framework Cloud builds a GitHub
|
|
327
|
+
permalink from a GitHub repository, the commit, the file, and the lines.
|
|
328
|
+
|
|
329
|
+
Args:
|
|
330
|
+
label: Names the snippet. The default slug and title come from it.
|
|
331
|
+
output: False publishes the code alone.
|
|
332
|
+
folded: True folds every output block, False unfolds them. Default: Framework
|
|
333
|
+
Cloud folds output longer than 12 lines.
|
|
334
|
+
links: Links shown with the snippet: ``{"notebook": "https://...", "model":
|
|
335
|
+
"model-slug"}``. A model link takes an artifact slug in the same project
|
|
336
|
+
or an https URL.
|
|
337
|
+
repository: Repository URL to name, such as ``"https://github.com/owner/repo"``.
|
|
338
|
+
Default: ``publish.repository`` in settings.yml.
|
|
339
|
+
publish: False records the block without publishing it.
|
|
340
|
+
slug: Artifact slug. Default: from ``label``.
|
|
341
|
+
title: Artifact title. Default: ``label``.
|
|
342
|
+
description: Artifact description.
|
|
343
|
+
visibility: "public", "unlisted", or "private".
|
|
344
|
+
password: Password required to view the snippet.
|
|
345
|
+
collection: Project collection ID or slug.
|
|
346
|
+
share_unlisted: Allow collection readers to see an unlisted snippet.
|
|
347
|
+
to: A configured destination name, or "cloud".
|
|
348
|
+
include_private: Publish without asking about private folders.
|
|
349
|
+
"""
|
|
350
|
+
caller = sys._getframe(2)
|
|
351
|
+
filename = caller.f_code.co_filename
|
|
352
|
+
start = caller.f_lineno
|
|
353
|
+
recorder = _Recorder()
|
|
354
|
+
error: BaseException | None = None
|
|
355
|
+
try:
|
|
356
|
+
with _teed(recorder):
|
|
357
|
+
try:
|
|
358
|
+
yield
|
|
359
|
+
except BaseException as raised:
|
|
360
|
+
error = raised
|
|
361
|
+
raise
|
|
362
|
+
finally:
|
|
363
|
+
code, lines = _block_code(filename, start)
|
|
364
|
+
if error is not None and not isinstance(error, (KeyboardInterrupt, GeneratorExit)):
|
|
365
|
+
text = "".join(traceback.format_exception(type(error), error, error.__traceback__))
|
|
366
|
+
recorder.block("error", text)
|
|
367
|
+
if code is None:
|
|
368
|
+
message = (
|
|
369
|
+
f'fw.snippet("{label}") could not read its code from {filename}. '
|
|
370
|
+
"Use it in a script or module, or %%fw_snippet in Jupyter."
|
|
371
|
+
)
|
|
372
|
+
if error is None:
|
|
373
|
+
raise FrameworkError(message)
|
|
374
|
+
inform(message)
|
|
375
|
+
elif publish:
|
|
376
|
+
path = None if _cell_file(filename) else Path(filename)
|
|
377
|
+
if path is None or not path.is_file():
|
|
378
|
+
path = _notebook_path()
|
|
379
|
+
if path is not None and path.suffix in _NOTEBOOK_SUFFIXES:
|
|
380
|
+
lines = None
|
|
381
|
+
document = _document(
|
|
382
|
+
[{"type": "code", "text": code}, *recorder.finished()],
|
|
383
|
+
_source(path, label, lines, repository),
|
|
384
|
+
output=output,
|
|
385
|
+
folded=folded,
|
|
386
|
+
links=links,
|
|
387
|
+
)
|
|
388
|
+
options = {
|
|
389
|
+
"slug": slug,
|
|
390
|
+
"title": title,
|
|
391
|
+
"description": description,
|
|
392
|
+
"visibility": visibility,
|
|
393
|
+
"password": password,
|
|
394
|
+
"collection": collection,
|
|
395
|
+
"share_unlisted": share_unlisted,
|
|
396
|
+
"to": to,
|
|
397
|
+
"include_private": include_private,
|
|
398
|
+
}
|
|
399
|
+
if document is not None:
|
|
400
|
+
if error is None:
|
|
401
|
+
_publish(document, label, options)
|
|
402
|
+
else:
|
|
403
|
+
try:
|
|
404
|
+
_publish(document, label, options)
|
|
405
|
+
except Exception as failed:
|
|
406
|
+
inform(f"The snippet was not published: {failed}")
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
def _cell_file(filename: str) -> bool:
|
|
410
|
+
"""Whether ``filename`` names a Jupyter cell rather than a file on disk."""
|
|
411
|
+
return filename.startswith("<") or "ipykernel_" in filename
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def _block_code(filename: str, start: int) -> tuple[str | None, tuple[int, int] | None]:
|
|
415
|
+
"""The body of the ``with`` statement on line ``start``, dedented, and its lines."""
|
|
416
|
+
text = "".join(linecache.getlines(filename))
|
|
417
|
+
if not text:
|
|
418
|
+
return None, None
|
|
419
|
+
try:
|
|
420
|
+
tree = ast.parse(text)
|
|
421
|
+
except SyntaxError:
|
|
422
|
+
return None, None
|
|
423
|
+
found: ast.With | ast.AsyncWith | None = None
|
|
424
|
+
for node in ast.walk(tree):
|
|
425
|
+
if (
|
|
426
|
+
isinstance(node, (ast.With, ast.AsyncWith))
|
|
427
|
+
and node.lineno <= start <= (node.body[0].lineno - 1 if node.body else node.lineno)
|
|
428
|
+
and (found is None or node.lineno > found.lineno)
|
|
429
|
+
):
|
|
430
|
+
found = node
|
|
431
|
+
if found is None or not found.body:
|
|
432
|
+
return None, None
|
|
433
|
+
first = found.body[0].lineno
|
|
434
|
+
decorated = getattr(found.body[0], "decorator_list", None)
|
|
435
|
+
if decorated:
|
|
436
|
+
first = min(first, *(item.lineno for item in decorated))
|
|
437
|
+
last = found.end_lineno or found.body[-1].lineno
|
|
438
|
+
lines = text.splitlines()[first - 1 : last]
|
|
439
|
+
return _dedent(lines), (first, last)
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def _dedent(lines: list[str]) -> str:
|
|
443
|
+
indents = [len(line) - len(line.lstrip()) for line in lines if line.strip()]
|
|
444
|
+
cut = min(indents, default=0)
|
|
445
|
+
return "\n".join(line[cut:] for line in lines).strip("\n")
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
# ---------------------------------------------------------------------------
|
|
449
|
+
# Jupyter: %%fw_snippet
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
def _notebook_path() -> Path | None:
|
|
453
|
+
"""The running notebook's file, when the front end says."""
|
|
454
|
+
candidates: list[str] = []
|
|
455
|
+
try:
|
|
456
|
+
from IPython import get_ipython
|
|
457
|
+
|
|
458
|
+
shell = get_ipython()
|
|
459
|
+
except ImportError:
|
|
460
|
+
shell = None
|
|
461
|
+
if shell is not None:
|
|
462
|
+
vscode = shell.user_ns.get("__vsc_ipynb_file__")
|
|
463
|
+
if isinstance(vscode, str):
|
|
464
|
+
candidates.append(vscode)
|
|
465
|
+
session = os.environ.get("JPY_SESSION_NAME")
|
|
466
|
+
if session:
|
|
467
|
+
candidates.append(session)
|
|
468
|
+
for candidate in candidates:
|
|
469
|
+
path = Path(candidate)
|
|
470
|
+
if path.is_file():
|
|
471
|
+
return path
|
|
472
|
+
return None
|
|
473
|
+
|
|
474
|
+
|
|
475
|
+
def _find_marker(label: str) -> list[Path]:
|
|
476
|
+
"""Notebooks in the working folder whose code holds ``%%fw_snippet <label>``."""
|
|
477
|
+
pattern = re.compile(r"%%fw_snippet\s+" + re.escape(label) + r"(?![\w-])")
|
|
478
|
+
found = []
|
|
479
|
+
for path in sorted(Path.cwd().iterdir()):
|
|
480
|
+
if path.suffix not in _NOTEBOOK_SUFFIXES or not path.is_file():
|
|
481
|
+
continue
|
|
482
|
+
try:
|
|
483
|
+
if pattern.search(path.read_text(encoding="utf-8")):
|
|
484
|
+
found.append(path)
|
|
485
|
+
except (OSError, UnicodeError):
|
|
486
|
+
continue
|
|
487
|
+
return found
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
def _cell_lines(path: Path, label: str) -> tuple[int, int] | None:
|
|
491
|
+
"""In a .qmd file, the lines of the code after ``%%fw_snippet <label>``."""
|
|
492
|
+
if path.suffix != ".qmd":
|
|
493
|
+
return None
|
|
494
|
+
text = path.read_text(encoding="utf-8").splitlines()
|
|
495
|
+
pattern = re.compile(r"^\s*%%fw_snippet\s+" + re.escape(label) + r"(?![\w-])")
|
|
496
|
+
starts = [index for index, line in enumerate(text) if pattern.match(line)]
|
|
497
|
+
if len(starts) != 1:
|
|
498
|
+
return None
|
|
499
|
+
first = starts[0] + 1
|
|
500
|
+
last = first
|
|
501
|
+
for index in range(first, len(text)):
|
|
502
|
+
if text[index].lstrip().startswith("```"):
|
|
503
|
+
break
|
|
504
|
+
last = index
|
|
505
|
+
while last > first and not text[last].strip():
|
|
506
|
+
last -= 1
|
|
507
|
+
while first < last and (not text[first].strip() or _OPTION_LINE.match(text[first])):
|
|
508
|
+
first += 1
|
|
509
|
+
return (first + 1, last + 1)
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
def _cell_code(cell: str) -> str:
|
|
513
|
+
"""The cell's code without Quarto option lines."""
|
|
514
|
+
lines = [line for line in cell.splitlines() if not _OPTION_LINE.match(line)]
|
|
515
|
+
return "\n".join(lines).strip("\n")
|
|
516
|
+
|
|
517
|
+
|
|
518
|
+
def _magic_parser() -> Any:
|
|
519
|
+
import argparse
|
|
520
|
+
|
|
521
|
+
parser = argparse.ArgumentParser(prog="%%fw_snippet", add_help=False, exit_on_error=False)
|
|
522
|
+
parser.add_argument("label")
|
|
523
|
+
fold = parser.add_mutually_exclusive_group()
|
|
524
|
+
fold.add_argument("--folded", dest="folded", action="store_const", const=True)
|
|
525
|
+
fold.add_argument("--unfolded", dest="folded", action="store_const", const=False)
|
|
526
|
+
parser.add_argument("--no-output", dest="output", action="store_false")
|
|
527
|
+
parser.add_argument("--link", action="append", default=[])
|
|
528
|
+
parser.add_argument("--repository")
|
|
529
|
+
parser.add_argument("--file")
|
|
530
|
+
parser.add_argument("--no-publish", dest="publish", action="store_false")
|
|
531
|
+
parser.add_argument("--slug")
|
|
532
|
+
parser.add_argument("--title")
|
|
533
|
+
parser.add_argument("--description")
|
|
534
|
+
parser.add_argument("--visibility")
|
|
535
|
+
parser.add_argument("--password")
|
|
536
|
+
parser.add_argument("--collection")
|
|
537
|
+
parser.add_argument("--share-unlisted", action="store_true")
|
|
538
|
+
parser.add_argument("--to")
|
|
539
|
+
parser.add_argument("--include-private", action="store_true")
|
|
540
|
+
return parser
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
def _magic_arguments(line: str) -> Any:
|
|
544
|
+
import argparse
|
|
545
|
+
import shlex
|
|
546
|
+
|
|
547
|
+
try:
|
|
548
|
+
arguments = _magic_parser().parse_args(shlex.split(line))
|
|
549
|
+
except (argparse.ArgumentError, SystemExit, ValueError) as error:
|
|
550
|
+
raise FrameworkError(
|
|
551
|
+
f"%%fw_snippet {line}: {error}. Usage: %%fw_snippet LABEL [--folded | --unfolded] "
|
|
552
|
+
"[--no-output] [--link notebook=URL] [--link model=SLUG] [--repository URL] "
|
|
553
|
+
"[--slug SLUG] [--title TITLE] [--visibility VISIBILITY] [--to DESTINATION] "
|
|
554
|
+
"[--no-publish]"
|
|
555
|
+
) from None
|
|
556
|
+
links: dict[str, str] = {}
|
|
557
|
+
for link in arguments.link:
|
|
558
|
+
rel, separator, value = link.partition("=")
|
|
559
|
+
if not separator:
|
|
560
|
+
raise FrameworkError("--link takes REL=VALUE, as in --link notebook=https://...")
|
|
561
|
+
links[rel] = value
|
|
562
|
+
arguments.links = links
|
|
563
|
+
return arguments
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
def run_cell_snippet(shell: Any, line: str, cell: str) -> dict[str, Any] | None:
|
|
567
|
+
"""Run a cell for ``%%fw_snippet``, show its outputs, and publish them.
|
|
568
|
+
|
|
569
|
+
Returns the snippet document, or None when the cell has no code.
|
|
570
|
+
"""
|
|
571
|
+
arguments = _magic_arguments(line)
|
|
572
|
+
code = _cell_code(cell)
|
|
573
|
+
recorder = _Recorder()
|
|
574
|
+
error: dict[str, Any] = {}
|
|
575
|
+
|
|
576
|
+
publisher = shell.display_pub
|
|
577
|
+
original_publish = publisher.publish
|
|
578
|
+
hook = shell.displayhook
|
|
579
|
+
original_write = hook.write_format_data
|
|
580
|
+
original_quiet = hook.quiet
|
|
581
|
+
original_start = hook.start_displayhook
|
|
582
|
+
original_finish = hook.finish_displayhook
|
|
583
|
+
original_showtraceback = shell._showtraceback
|
|
584
|
+
|
|
585
|
+
def plain(data: dict[str, Any]) -> str | None:
|
|
586
|
+
if any(str(key).startswith("image/") for key in data):
|
|
587
|
+
return None
|
|
588
|
+
text = data.get("text/plain")
|
|
589
|
+
return text if isinstance(text, str) else None
|
|
590
|
+
|
|
591
|
+
def publish_display(data: dict[str, Any], *args: Any, **kwargs: Any) -> Any:
|
|
592
|
+
text = plain(data or {})
|
|
593
|
+
if text is not None:
|
|
594
|
+
recorder.block("output", text)
|
|
595
|
+
muted, recorder.muted = recorder.muted, True
|
|
596
|
+
try:
|
|
597
|
+
return original_publish(data, *args, **kwargs)
|
|
598
|
+
finally:
|
|
599
|
+
recorder.muted = muted
|
|
600
|
+
|
|
601
|
+
def start_result() -> Any:
|
|
602
|
+
recorder.muted = True
|
|
603
|
+
return original_start()
|
|
604
|
+
|
|
605
|
+
def finish_result() -> Any:
|
|
606
|
+
try:
|
|
607
|
+
return original_finish()
|
|
608
|
+
finally:
|
|
609
|
+
recorder.muted = False
|
|
610
|
+
|
|
611
|
+
def write_result(format_dict: dict[str, Any], md_dict: Any = None) -> Any:
|
|
612
|
+
text = plain(format_dict or {})
|
|
613
|
+
if text is not None:
|
|
614
|
+
recorder.block("output", text)
|
|
615
|
+
return original_write(format_dict, md_dict)
|
|
616
|
+
|
|
617
|
+
def quiet() -> bool:
|
|
618
|
+
try:
|
|
619
|
+
return bool(hook.semicolon_at_end_of_expression(cell))
|
|
620
|
+
except Exception:
|
|
621
|
+
return False
|
|
622
|
+
|
|
623
|
+
def record_traceback(etype: Any, evalue: Any, stb: list[str]) -> None:
|
|
624
|
+
error["text"] = "\n".join(stb)
|
|
625
|
+
error["value"] = evalue
|
|
626
|
+
|
|
627
|
+
publisher.publish = publish_display
|
|
628
|
+
hook.write_format_data = write_result
|
|
629
|
+
hook.quiet = quiet
|
|
630
|
+
hook.start_displayhook = start_result
|
|
631
|
+
hook.finish_displayhook = finish_result
|
|
632
|
+
shell._showtraceback = record_traceback
|
|
633
|
+
try:
|
|
634
|
+
with _teed(recorder):
|
|
635
|
+
result = shell.run_cell(cell, store_history=False)
|
|
636
|
+
finally:
|
|
637
|
+
publisher.publish = original_publish
|
|
638
|
+
hook.write_format_data = original_write
|
|
639
|
+
hook.quiet = original_quiet
|
|
640
|
+
hook.start_displayhook = original_start
|
|
641
|
+
hook.finish_displayhook = original_finish
|
|
642
|
+
shell._showtraceback = original_showtraceback
|
|
643
|
+
|
|
644
|
+
raised = result.error_in_exec or result.error_before_exec
|
|
645
|
+
if "text" in error:
|
|
646
|
+
recorder.block("error", error["text"])
|
|
647
|
+
elif raised is not None:
|
|
648
|
+
recorder.block("error", "".join(traceback.format_exception_only(type(raised), raised)))
|
|
649
|
+
|
|
650
|
+
path = Path(arguments.file) if arguments.file else _notebook_path()
|
|
651
|
+
if path is None:
|
|
652
|
+
matches = _find_marker(arguments.label)
|
|
653
|
+
path = matches[0] if len(matches) == 1 else None
|
|
654
|
+
lines = _cell_lines(path, arguments.label) if path is not None else None
|
|
655
|
+
document = _document(
|
|
656
|
+
[{"type": "code", "text": code}, *recorder.finished()],
|
|
657
|
+
_source(path, arguments.label, lines, arguments.repository),
|
|
658
|
+
output=arguments.output,
|
|
659
|
+
folded=arguments.folded,
|
|
660
|
+
links=arguments.links,
|
|
661
|
+
)
|
|
662
|
+
if document is not None and arguments.publish:
|
|
663
|
+
options = {name: getattr(arguments, name) for name in _PUBLISH_OPTIONS}
|
|
664
|
+
if raised is None:
|
|
665
|
+
_publish_in_cell(document, arguments.label, options)
|
|
666
|
+
else:
|
|
667
|
+
try:
|
|
668
|
+
_publish_in_cell(document, arguments.label, options)
|
|
669
|
+
except Exception as failed:
|
|
670
|
+
inform(f"The snippet was not published: {failed}")
|
|
671
|
+
if raised is not None:
|
|
672
|
+
# The traceback was recorded instead of shown, so the cell fails once, here.
|
|
673
|
+
raise raised
|
|
674
|
+
return document
|
|
675
|
+
|
|
676
|
+
|
|
677
|
+
# Marks the publishing status a cell shows, so fw snippet publish leaves it out
|
|
678
|
+
STATUS_METADATA = {"framework": {"role": "snippet-status"}}
|
|
679
|
+
|
|
680
|
+
|
|
681
|
+
def _publish_in_cell(
|
|
682
|
+
document: dict[str, Any], label: str, options: dict[str, Any]
|
|
683
|
+
) -> str | list[str]:
|
|
684
|
+
"""Publish, and show Framework's messages as a display the snippet readers skip."""
|
|
685
|
+
import logging
|
|
686
|
+
|
|
687
|
+
from IPython.display import publish_display_data
|
|
688
|
+
|
|
689
|
+
from framework._log import logger
|
|
690
|
+
|
|
691
|
+
messages: list[str] = []
|
|
692
|
+
|
|
693
|
+
class Collect(logging.Handler):
|
|
694
|
+
def emit(self, record: logging.LogRecord) -> None:
|
|
695
|
+
messages.append(record.getMessage())
|
|
696
|
+
|
|
697
|
+
handlers, level = logger.handlers[:], logger.level
|
|
698
|
+
logger.handlers = [Collect()]
|
|
699
|
+
logger.setLevel(logging.INFO)
|
|
700
|
+
try:
|
|
701
|
+
return _publish(document, label, options)
|
|
702
|
+
finally:
|
|
703
|
+
logger.handlers = handlers
|
|
704
|
+
logger.setLevel(level)
|
|
705
|
+
if messages:
|
|
706
|
+
publish_display_data( # type: ignore[no-untyped-call]
|
|
707
|
+
{"text/plain": "\n".join(messages)}, metadata=STATUS_METADATA
|
|
708
|
+
)
|
|
709
|
+
|
|
710
|
+
|
|
711
|
+
def load_ipython_extension(shell: Any) -> None:
|
|
712
|
+
"""``%load_ext framework``: add the ``%%fw_snippet`` magic and show Framework's messages."""
|
|
713
|
+
from framework._log import show_messages
|
|
714
|
+
|
|
715
|
+
show_messages()
|
|
716
|
+
|
|
717
|
+
def fw_snippet(line: str, cell: str) -> None:
|
|
718
|
+
"""Run the cell and publish its code and printed output as a code snippet.
|
|
719
|
+
|
|
720
|
+
%%fw_snippet LABEL [--folded | --unfolded] [--no-output]
|
|
721
|
+
[--link notebook=URL] [--link model=SLUG] [--repository URL] [--file PATH]
|
|
722
|
+
[--slug SLUG] [--title TITLE] [--description TEXT] [--visibility VISIBILITY]
|
|
723
|
+
[--password PASSWORD] [--collection COLLECTION] [--share-unlisted]
|
|
724
|
+
[--to DESTINATION] [--include-private] [--no-publish]
|
|
725
|
+
"""
|
|
726
|
+
run_cell_snippet(shell, line, cell)
|
|
727
|
+
|
|
728
|
+
shell.register_magic_function(fw_snippet, magic_kind="cell", magic_name="fw_snippet")
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: framework-py
|
|
3
|
-
Version: 0.1.0.
|
|
3
|
+
Version: 0.1.0.dev27
|
|
4
4
|
Summary: Structured data science projects: a data catalog, integrity tracking, caching, outputs, and AI guardrails
|
|
5
5
|
Project-URL: Homepage, https://framework.pub
|
|
6
6
|
Project-URL: Source, https://github.com/frameworktools/framework-py
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
framework/__init__.py,sha256=
|
|
1
|
+
framework/__init__.py,sha256=IZuuoHC2nHIPRRC9k6AkpQZ6HmnySBvh0s9us-wlPbE,6834
|
|
2
2
|
framework/__main__.py,sha256=S0ChmRQnNa9W5_MPXhHnH61Kjr_pNthkeS41eJU1w0U,9093
|
|
3
3
|
framework/_agents_md.py,sha256=PSWPJiCUTRirk81sz-Y33qHnqY16HOhHY8pH77ct1ik,11857
|
|
4
4
|
framework/_artifacts.py,sha256=XjYOBQOQJuvztUyqwWsc3xc1qJ5o4O5XpCW49WN548k,55807
|
|
@@ -26,7 +26,7 @@ framework/ai.py,sha256=hqBNXngw7VyQBcSt7kB20Zpa6N_5WS1hNhDi2zWJw0U,9218
|
|
|
26
26
|
framework/bootstrap.py,sha256=mYlD1BG9KaG9PAIHlw3l6677fJYwQdtih95W9Rqmnho,9687
|
|
27
27
|
framework/cache.py,sha256=z8Zaem5Q7dsAghuX9CgK_j0vhX5dDlRw7fl9f-CF3eA,17440
|
|
28
28
|
framework/catalog.py,sha256=sl4V1judSCVoa2LYLXeQdDhWdsONTC_5s2_fpyPylto,13202
|
|
29
|
-
framework/cloud.py,sha256=
|
|
29
|
+
framework/cloud.py,sha256=y4w5euumzkNXOfC9q8v7FVW5yHEO7QORXC3D2BlirMI,67843
|
|
30
30
|
framework/config_home.py,sha256=iSb0VuENgNOSnsZ_v53xFqjRiGgrB7OzZbHbyDOqKzs,8265
|
|
31
31
|
framework/connections.py,sha256=99JqrNAi2zBpNYyxLV6SYj-NAJjh56pYh9jaMSYOHCU,32742
|
|
32
32
|
framework/console.py,sha256=zQ0LuFDcxe1W1kuc5E1bJ03h8d3cvwFoqDT8dAJQyN0,1267
|
|
@@ -54,6 +54,7 @@ framework/security.py,sha256=-x66bSIIE_xOzkTf5Ojt7xIECKuAIpKVJZ5TtIGrwys,51147
|
|
|
54
54
|
framework/settings.py,sha256=Sb2YyE53aYLWPj423P-pQVCvRpLsiHydgZh2K0luTQQ,23893
|
|
55
55
|
framework/skill_pull.py,sha256=vr_yxe446JhHbtI0hps19AnLN6S7BacSt2oxn7BIlos,21859
|
|
56
56
|
framework/skill_sync.py,sha256=0gqaeVkiNaqGJxmrVrKrYaX8wI3DNFZUsJEgG63xcKc,23060
|
|
57
|
+
framework/snippets.py,sha256=J8j36YqHgG-HhWlhJ9RmJK-qwVGOOSOOd1pNakJyuVE,26481
|
|
57
58
|
framework/storage.py,sha256=Haz8CBXrz2CCqGHOVgHKC3mUUwobnod2SanU759lK7c,56115
|
|
58
59
|
framework/sync_all.py,sha256=tmSnapf-JfaL1_8rMpnr3wqHvSGJamn7c8ja4GQFKHA,34610
|
|
59
60
|
framework/upgrade.py,sha256=WtghpLR65YFH_JQFmo1JlH9lHdwsXQd-ZzzOTmyOL28,65593
|
|
@@ -140,7 +141,7 @@ framework/templates/stubs/notebook-default.qmd,sha256=8Js1S2BcHYeQWmJS-kt31gBvIj
|
|
|
140
141
|
framework/templates/stubs/notebook-minimal.qmd,sha256=2U2k8judffhrh_6m6r6OAbQBCYEtViAjeKRBqBYCJTM,178
|
|
141
142
|
framework/templates/stubs/notebook-revealjs.qmd,sha256=_ztT_gHoudpB5DIjlbvtB5zQ8163DeV8xbWcT-HgJrI,409
|
|
142
143
|
framework/templates/stubs/script-default.py,sha256=vbWqgW0_rHot17mQC3tYb92g_9L-cbTOeL3y99BRrSk,216
|
|
143
|
-
framework_py-0.1.0.
|
|
144
|
-
framework_py-0.1.0.
|
|
145
|
-
framework_py-0.1.0.
|
|
146
|
-
framework_py-0.1.0.
|
|
144
|
+
framework_py-0.1.0.dev27.dist-info/METADATA,sha256=dUHbI-_zRBu2n8F4xgWwa9oUNAZ4Jjp7UAAcO6cCSsU,4682
|
|
145
|
+
framework_py-0.1.0.dev27.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
146
|
+
framework_py-0.1.0.dev27.dist-info/licenses/LICENSE,sha256=giOOgSwALkZimC7uhLZQK8Untez641bLW1-2-Cjvmq0,1072
|
|
147
|
+
framework_py-0.1.0.dev27.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|