framework-py 0.1.0.dev26__py3-none-any.whl → 0.1.0.dev28__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
framework/__init__.py CHANGED
@@ -131,6 +131,8 @@ from framework.security import AuditResult, git_security_audit
131
131
  from framework.settings import settings, settings_read, settings_write
132
132
  from framework.skill_pull import pull_skills, push_skills
133
133
  from framework.skill_sync import skills_sync
134
+ from framework.snippets import load_ipython_extension as load_ipython_extension
135
+ from framework.snippets import snippet
134
136
  from framework.spec.check import spec_check
135
137
  from framework.spec.merge import SpecError
136
138
  from framework.spec.resolve import ResolvedSpec, spec_resolve
@@ -274,6 +276,7 @@ __all__ = [
274
276
  "settings_write",
275
277
  "show_table",
276
278
  "skills_sync",
279
+ "snippet",
277
280
  "spec_check",
278
281
  "spec_resolve",
279
282
  "status",
framework/cloud.py CHANGED
@@ -1269,7 +1269,10 @@ def cloud_publish(
1269
1269
  if response.is_error:
1270
1270
  raise _error(response, "Publish")
1271
1271
  result = response.json()
1272
- if result.get("version") is not None:
1272
+ if result.get("unchanged") and result.get("version") is not None:
1273
+ # Cloud keeps the newest version when nothing changed (code snippets)
1274
+ inform(f"Unchanged, still version {result['version']}: {result.get('url')}")
1275
+ elif result.get("version") is not None:
1273
1276
  inform(f"Published version {result['version']}: {result.get('url')}")
1274
1277
  else:
1275
1278
  inform(f"Published: {result.get('url')}")
framework/data.py CHANGED
@@ -436,13 +436,39 @@ def data_read(
436
436
  return data
437
437
 
438
438
 
439
- def _save_target(path: str, type: str | None) -> tuple[Path, str, str | None]:
440
- """The file to write, its type, and the catalog name (None when ``path`` is a file path).
439
+ def _catalog_entry(name: str) -> DataInfo | None:
440
+ """The catalog entry ``name`` already has, or None."""
441
+ try:
442
+ data = settings_read().get("data")
443
+ except FrameworkError:
444
+ return None
445
+ if catalog_locate(name, data) is None:
446
+ return None
447
+ try:
448
+ return data_info(name)
449
+ except DataNotFoundError:
450
+ # An entry without a path is saved like a new name
451
+ return None
452
+
453
+
454
+ def _delimiter_name(delimiter: str) -> str:
455
+ """``comma``, ``tab``, ``semicolon``, or ``space`` for a delimiter or its character."""
456
+ _separator(delimiter)
457
+ for key, character in DELIMITERS.items():
458
+ if delimiter in (key, character):
459
+ return key
460
+ return delimiter
461
+
462
+
463
+ def _save_target(path: str, type: str | None) -> tuple[Path, str, str | None, DataInfo | None]:
464
+ """The file to write, its type, the catalog name (None when ``path`` is a file path),
465
+ and the catalog entry the name already has (None for a new name or a file path).
441
466
 
442
467
  A relative file path starts at the project root, as the catalog's paths do, so a
443
468
  notebook in ``notebooks/`` writes where a script in the root would. A dot name that
444
469
  ends in an extension, such as ``inputs.final.x.csv``, is the name ``inputs.final.x``
445
- saved as that type.
470
+ saved as that type. A name already in the catalog is saved to its entry's file, in
471
+ its entry's type, as Stata's ``fw_save`` does.
446
472
  """
447
473
  has_slash = "/" in path or "\\" in path
448
474
  extension = re.search(r"\.(csv|parquet|rds)$", path, re.IGNORECASE)
@@ -458,7 +484,7 @@ def _save_target(path: str, type: str | None) -> tuple[Path, str, str | None]:
458
484
  if not path.lower().endswith(f".{type}"):
459
485
  path = f"{path}.{type}"
460
486
  inform(f"Note: Added .{type} extension to path")
461
- return in_root(path, root), type, None
487
+ return in_root(path, root), type, None, None
462
488
 
463
489
  if extension:
464
490
  given = extension.group(1).lower()
@@ -467,6 +493,18 @@ def _save_target(path: str, type: str | None) -> tuple[Path, str, str | None]:
467
493
  type = given
468
494
  path = path[: extension.start()]
469
495
 
496
+ entry = _catalog_entry(path)
497
+ if entry is not None:
498
+ # tsv is delimited text with tabs, which data_save() writes as csv
499
+ entry_type = "csv" if entry.type == "tsv" else entry.type
500
+ if type is not None and type.lower() != entry_type:
501
+ raise FrameworkError(
502
+ f"'{path}' is a {entry.type} file in the data catalog "
503
+ f"({relative_to_root(entry.path, root)}), but type is '{type}'.\n"
504
+ "Leave type out to save it as its entry says, or save it under another name."
505
+ )
506
+ return entry.path, entry_type, path, entry
507
+
470
508
  parts = path.split(".")
471
509
  if len(parts) == 1 or not all(parts):
472
510
  suffix = type or "parquet"
@@ -497,7 +535,7 @@ def _save_target(path: str, type: str | None) -> tuple[Path, str, str | None]:
497
535
  )
498
536
  type = type or "parquet"
499
537
  target = in_root(folder, root) / f"{'_'.join(parts[end:])}.{type}"
500
- return target, type, path
538
+ return target, type, path, None
501
539
 
502
540
 
503
541
  def _in_settings_folder(folder: Path) -> bool:
@@ -598,7 +636,7 @@ def data_save(
598
636
  data: pd.DataFrame,
599
637
  path: str,
600
638
  type: str | None = None,
601
- delimiter: str = "comma",
639
+ delimiter: str | None = None,
602
640
  locked: bool | None = None,
603
641
  force: bool = False,
604
642
  unlock: bool = False,
@@ -617,14 +655,20 @@ def data_save(
617
655
  file path with a ``/``, ending in ``.csv`` or ``.parquet``. A name's leading
618
656
  parts select a folder from ``directories``: the longest run that
619
657
  names a key, such as ``inputs_intermediate``, ``inputs_private_raw``, or a
620
- course's ``data``. The rest becomes the file name. A name is also added to
621
- the data catalog, so ``data_read()`` finds it. A name may end in
658
+ course's ``data``. The rest becomes the file name. A new name is also added
659
+ to the data catalog, so ``data_read()`` finds it. A name already in the
660
+ catalog is saved to its entry's file, in its entry's type and delimiter, and
661
+ the entry is left as it is (its description and other fields are kept). A
662
+ name may end in
622
663
  ``.csv`` or ``.parquet`` to choose the type: ``"inputs.final.x.csv"`` is
623
664
  the name ``inputs.final.x`` saved as CSV. A relative file path starts at the
624
665
  project root, wherever Python runs.
625
- type: ``"parquet"`` or ``"csv"``. Default: from the file extension,
626
- or Parquet for a catalog name.
627
- delimiter: Delimiter for CSV files: comma, tab, semicolon, or space.
666
+ type: ``"parquet"`` or ``"csv"``. Default: the catalog entry's type, then the
667
+ file extension, then Parquet for a new catalog name. A type that differs
668
+ from the catalog entry's is an error.
669
+ delimiter: Delimiter for CSV files: comma, tab, semicolon, or space. Default:
670
+ the catalog entry's delimiter, else comma. A delimiter that differs from the
671
+ catalog entry's is an error.
628
672
  locked: Whether later reads must find the file unchanged. Default: unlocked, as
629
673
  in R, so a pipeline can save the data again. Pass ``locked=True`` to lock data
630
674
  you mean to keep as it is. Left out, a save keeps a lock the data already
@@ -648,14 +692,24 @@ def data_save(
648
692
  """
649
693
  if not isinstance(data, pd.DataFrame):
650
694
  raise FrameworkError("data_save() needs a pandas DataFrame")
651
- target, file_type, name = _save_target(path, type)
695
+ target, file_type, name, entry = _save_target(path, type)
696
+ if entry is not None and entry.delimiter is not None and file_type == "csv":
697
+ if delimiter is not None and _delimiter_name(delimiter) != entry.delimiter:
698
+ raise FrameworkError(
699
+ f"'{name}' is {entry.delimiter}-delimited in the data catalog, but "
700
+ f"delimiter is '{delimiter}'. Leave delimiter out to save it as its entry "
701
+ "says, or save it under another name."
702
+ )
703
+ delimiter = entry.delimiter
704
+ delimiter = _delimiter_name(delimiter or "comma")
652
705
  if file_type == "rds":
653
706
  raise UnsupportedFormatError(
654
707
  "Python does not write .rds files. Save as 'parquet' or 'csv', which both "
655
708
  "languages read."
656
709
  )
657
710
  if file_type not in _SAVE_TYPES:
658
- raise UnsupportedFormatError(f"data_save() writes csv or parquet, not {file_type}")
711
+ where = f" ('{name}' in the data catalog)" if entry is not None else ""
712
+ raise UnsupportedFormatError(f"data_save() writes csv or parquet, not {file_type}{where}")
659
713
 
660
714
  folder = target.parent
661
715
  # A missing folder is created when it is a settings folder or inside one (git
@@ -734,7 +788,21 @@ def data_save(
734
788
 
735
789
  ledger_push_quietly(key, digest, target)
736
790
 
737
- if name is not None:
791
+ if name is not None and entry is not None:
792
+ # The entry already names this file. Only its lock can change, so the
793
+ # catalog's flag agrees with framework.db; everything else is left as it is.
794
+ if entry.locked != locked:
795
+ try:
796
+ _catalog_set(name, {"locked": True} if locked else {}, drop=("locked",))
797
+ inform(f"Data catalog entry {'locked' if locked else 'unlocked'}: {name}")
798
+ except FrameworkError as error:
799
+ warnings.warn(
800
+ f"File saved but could not update the lock in the data catalog for "
801
+ f"'{name}': {error}",
802
+ FrameworkWarning,
803
+ stacklevel=2,
804
+ )
805
+ elif name is not None:
738
806
  try:
739
807
  data_add(target, name=name, type=file_type, delimiter=delimiter, locked=locked)
740
808
  except FrameworkError as error:
@@ -768,11 +836,18 @@ def _derive_name(file_path: Path, config: dict[str, Any], root: Path) -> str:
768
836
  return file_path.stem
769
837
 
770
838
 
771
- def _catalog_set(name: str, entry: dict[str, Any]) -> None:
839
+ def _catalog_set(name: str, entry: dict[str, Any], drop: Sequence[str] = ()) -> bool:
772
840
  """Write one catalog entry into settings.yml or the split file that holds ``data``.
773
841
 
774
- The file is edited in place, so its comments and other entries keep their form.
842
+ An existing entry is updated, not replaced: the fields in ``entry`` are set, the
843
+ fields in ``drop`` that ``entry`` leaves out are removed, and every other field (a
844
+ description, notes, stata_names) is kept. The file is edited in place, so its
845
+ comments and other entries keep their form.
846
+
847
+ Returns:
848
+ Whether the entry already existed.
775
849
  """
850
+ existed = False
776
851
  settings_file = find_settings_file()
777
852
  if settings_file is None:
778
853
  raise SettingsError("Configuration file 'settings.yml' (or legacy config.yml) not found")
@@ -782,9 +857,20 @@ def _catalog_set(name: str, entry: dict[str, Any]) -> None:
782
857
 
783
858
  def set_entry(current: dict[str, Any]) -> None:
784
859
  # An entry already written with dotted keys is updated where it is.
860
+ nonlocal existed
785
861
  existing = catalog_locate(name, current)
786
862
  if existing is not None:
787
- existing[0][existing[1]] = entry
863
+ existed = True
864
+ holder, key = existing
865
+ old = holder[key]
866
+ if not isinstance(old, dict):
867
+ # A bare path entry becomes a mapping
868
+ old = {"path": old} if isinstance(old, str) else {}
869
+ holder[key] = old
870
+ for field in drop:
871
+ if field not in entry:
872
+ old.pop(field, None)
873
+ old.update(entry)
788
874
  return
789
875
  node = current
790
876
  parts = name.split(".")
@@ -810,7 +896,7 @@ def _catalog_set(name: str, entry: dict[str, Any]) -> None:
810
896
  return {"data": content}
811
897
 
812
898
  _yaml.update(in_root(source, settings_file.parent), in_split)
813
- return
899
+ return existed
814
900
 
815
901
  def in_settings(raw: Any) -> Any:
816
902
  raw = raw if isinstance(raw, dict) else {}
@@ -821,6 +907,7 @@ def _catalog_set(name: str, entry: dict[str, Any]) -> None:
821
907
  return raw
822
908
 
823
909
  _yaml.update(settings_file, in_settings)
910
+ return existed
824
911
 
825
912
 
826
913
  def data_add(
@@ -889,7 +976,9 @@ def data_add(
889
976
  if locked:
890
977
  entry["locked"] = True
891
978
  if update_config:
892
- _catalog_set(name, entry)
893
- inform(f"Data spec added to config: {name}")
979
+ # An existing entry keeps its other fields (description, notes); the
980
+ # delimiter and lock follow this file
981
+ existed = _catalog_set(name, entry, drop=("delimiter", "locked"))
982
+ inform(f"Data spec {'updated in' if existed else 'added to'} config: {name}")
894
983
  inform(f'\nYou can now read this data with: data_read("{name}")')
895
984
  return entry
framework/snippets.py ADDED
@@ -0,0 +1,728 @@
1
+ """Code snippets: a cell's or a block's code and the text it printed.
2
+
3
+ A code snippet is a ``framework.artifact/v1`` document with ``kind: "code_snippet"``.
4
+ Framework Cloud highlights the code, styles ANSI output, and applies the size limits,
5
+ so this module only records what ran and what it printed.
6
+
7
+ In Jupyter, ``%load_ext framework`` adds the ``%%fw_snippet`` cell magic. In a script,
8
+ ``with fw.snippet("label"):`` records the block.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import ast
14
+ import json
15
+ import linecache
16
+ import os
17
+ import re
18
+ import subprocess
19
+ import sys
20
+ import tempfile
21
+ import traceback
22
+ from collections.abc import Iterator, Sequence
23
+ from contextlib import contextmanager
24
+ from pathlib import Path
25
+ from typing import Any, TextIO
26
+
27
+ from framework._log import inform
28
+ from framework.errors import FrameworkError
29
+
30
+ SCHEMA = "framework.artifact/v1"
31
+ KIND = "code_snippet"
32
+ LINK_RELS = ("notebook", "model")
33
+ _OPTION_LINE = re.compile(r"^\s*#\|")
34
+ _SLUG = re.compile(r"^[A-Za-z0-9_-]{1,120}$")
35
+ _NOTEBOOK_SUFFIXES = (".ipynb", ".qmd", ".md")
36
+
37
+
38
+ # ---------------------------------------------------------------------------
39
+ # The document
40
+
41
+
42
+ def _links(links: dict[str, str] | None) -> list[dict[str, str]]:
43
+ """``display.links`` from ``{"notebook": "https://...", "model": "slug or URL"}``."""
44
+ if not links:
45
+ return []
46
+ if not isinstance(links, dict):
47
+ raise FrameworkError(
48
+ 'links must be a dict such as {"notebook": "https://...", "model": "model-slug"}.'
49
+ )
50
+ out: list[dict[str, str]] = []
51
+ for rel, value in links.items():
52
+ if rel not in LINK_RELS:
53
+ raise FrameworkError(
54
+ 'links must be named "notebook" or "model", as in '
55
+ '{"notebook": "https://...", "model": "model-slug"}.'
56
+ )
57
+ if not isinstance(value, str) or not value:
58
+ raise FrameworkError("Each link must be one string.")
59
+ if re.fullmatch(r"https://\S+", value):
60
+ out.append({"rel": rel, "url": value})
61
+ elif rel == "model" and _SLUG.fullmatch(value):
62
+ out.append({"rel": rel, "artifact": value})
63
+ elif rel == "model":
64
+ raise FrameworkError(
65
+ "A model link takes an artifact slug in this project or an https URL."
66
+ )
67
+ else:
68
+ raise FrameworkError("A notebook link takes an https URL.")
69
+ return out
70
+
71
+
72
+ def _document(
73
+ blocks: Sequence[dict[str, Any]],
74
+ source: dict[str, Any],
75
+ *,
76
+ output: bool = True,
77
+ folded: bool | None = None,
78
+ links: dict[str, str] | None = None,
79
+ ) -> dict[str, Any] | None:
80
+ """The snippet document, or None when it has no code.
81
+
82
+ Adjacent output blocks join, as a notebook shows them. With ``output=False`` only
83
+ the code is kept.
84
+ """
85
+ kept = [dict(block) for block in blocks if block.get("text")]
86
+ if not output:
87
+ kept = [block for block in kept if block["type"] == "code"]
88
+ merged: list[dict[str, Any]] = []
89
+ for block in kept:
90
+ if merged and merged[-1]["type"] == block["type"] and block["type"] in ("code", "output"):
91
+ merged[-1]["text"] = merged[-1]["text"] + "\n" + block["text"]
92
+ else:
93
+ merged.append(block)
94
+ if not any(block["type"] == "code" for block in merged):
95
+ return None
96
+ if folded is not None:
97
+ for block in merged:
98
+ if block["type"] == "output":
99
+ block["folded"] = folded
100
+ document: dict[str, Any] = {
101
+ "schema": SCHEMA,
102
+ "kind": KIND,
103
+ "source": {key: value for key, value in source.items() if value is not None},
104
+ "snippet": {"blocks": merged},
105
+ }
106
+ shown = _links(links)
107
+ if shown:
108
+ document["display"] = {"links": shown}
109
+ return document
110
+
111
+
112
+ def _trim(text: str) -> str:
113
+ """Line endings as ``\\n``, without the trailing newline a print adds."""
114
+ text = text.replace("\r\n", "\n").replace("\r", "\n")
115
+ return text[:-1] if text.endswith("\n") else text
116
+
117
+
118
+ class _Recorder:
119
+ """Blocks in the order they were printed. Text of one kind in a row joins."""
120
+
121
+ def __init__(self) -> None:
122
+ self.blocks: list[dict[str, Any]] = []
123
+ self._open: str | None = None
124
+ # Set while IPython writes a display or a result, which a terminal prints to stdout
125
+ self.muted = False
126
+
127
+ def stream(self, kind: str, text: str) -> None:
128
+ if not text or self.muted:
129
+ return
130
+ if self._open == kind and self.blocks:
131
+ self.blocks[-1]["text"] += text
132
+ else:
133
+ self.blocks.append({"type": kind, "text": text})
134
+ self._open = kind
135
+
136
+ def block(self, kind: str, text: str) -> None:
137
+ self._open = None
138
+ self.blocks.append({"type": kind, "text": text})
139
+
140
+ def finished(self) -> list[dict[str, Any]]:
141
+ return [{**block, "text": _trim(block["text"])} for block in self.blocks]
142
+
143
+
144
+ class _Tee:
145
+ """A text stream that writes through and records what passes."""
146
+
147
+ def __init__(self, target: TextIO, record: Any) -> None:
148
+ self._target = target
149
+ self._record = record
150
+
151
+ def write(self, text: str) -> int:
152
+ self._record(text)
153
+ return self._target.write(text)
154
+
155
+ def __getattr__(self, name: str) -> Any:
156
+ return getattr(self._target, name)
157
+
158
+
159
+ @contextmanager
160
+ def _teed(recorder: _Recorder) -> Iterator[None]:
161
+ """Record stdout as output and stderr as messages, still printing both."""
162
+ stdout, stderr = sys.stdout, sys.stderr
163
+ sys.stdout = _Tee(stdout, lambda text: recorder.stream("output", text))
164
+ sys.stderr = _Tee(stderr, lambda text: recorder.stream("message", text))
165
+ try:
166
+ yield
167
+ finally:
168
+ sys.stdout, sys.stderr = stdout, stderr
169
+
170
+
171
+ # ---------------------------------------------------------------------------
172
+ # Where the code came from
173
+
174
+
175
+ def _git(directory: Path, *args: str) -> str | None:
176
+ try:
177
+ result = subprocess.run(
178
+ ["git", "-C", str(directory), *args],
179
+ capture_output=True,
180
+ encoding="utf-8",
181
+ errors="replace",
182
+ check=False,
183
+ )
184
+ except OSError:
185
+ return None
186
+ value = result.stdout.strip()
187
+ return value if result.returncode == 0 and value else None
188
+
189
+
190
+ def _location(path: Path | None) -> dict[str, Any]:
191
+ """The file's path from the git repository's top (else the project root), and the commit."""
192
+ if path is None or not path.is_file():
193
+ return {"file": None, "commit": None}
194
+ path = path.resolve()
195
+ top = _git(path.parent, "rev-parse", "--show-toplevel")
196
+ commit = None
197
+ root: Path | None
198
+ if top is not None:
199
+ root = Path(top).resolve()
200
+ head = _git(path.parent, "rev-parse", "HEAD")
201
+ if head is not None and re.fullmatch(r"[0-9a-f]{40}", head):
202
+ commit = head
203
+ else:
204
+ from framework.project import find_settings_file
205
+
206
+ found = find_settings_file(path.parent)
207
+ root = found.parent.resolve() if found is not None else None
208
+ try:
209
+ file = path.relative_to(root).as_posix() if root is not None else path.name
210
+ except ValueError:
211
+ file = path.name
212
+ return {"file": file, "commit": commit}
213
+
214
+
215
+ def _repository(repository: str | None) -> str | None:
216
+ """The argument, else ``publish.repository`` in settings.yml. Sent only when set."""
217
+ if repository is None:
218
+ try:
219
+ from framework.settings import settings_read
220
+
221
+ publish = settings_read().get("publish")
222
+ repository = publish.get("repository") if isinstance(publish, dict) else None
223
+ except Exception:
224
+ repository = None
225
+ if repository is None:
226
+ return None
227
+ if not isinstance(repository, str) or not repository.startswith("https://"):
228
+ raise FrameworkError(
229
+ "repository must be an https URL, such as https://github.com/owner/repo."
230
+ )
231
+ return repository
232
+
233
+
234
+ def _source(
235
+ path: Path | None,
236
+ label: str,
237
+ lines: tuple[int, int] | None,
238
+ repository: str | None,
239
+ ) -> dict[str, Any]:
240
+ location = _location(path)
241
+ return {
242
+ "language": "python",
243
+ "file": location["file"],
244
+ "label": label,
245
+ "lines": list(lines) if lines is not None and location["file"] is not None else None,
246
+ "commit": location["commit"],
247
+ "repository": _repository(repository),
248
+ }
249
+
250
+
251
+ # ---------------------------------------------------------------------------
252
+ # Publishing
253
+
254
+
255
+ def _publish(document: dict[str, Any], label: str, options: dict[str, Any]) -> str | list[str]:
256
+ """Publish a snippet document as ``code_snippet.json``."""
257
+ from framework.publish_bundle import artifact_slug
258
+ from framework.storage import publish
259
+
260
+ options = dict(options)
261
+ options["slug"] = options.get("slug") or artifact_slug(label)
262
+ options["title"] = options.get("title") or label
263
+ with tempfile.TemporaryDirectory(prefix="fw_snippet_") as temp:
264
+ path = Path(temp) / "code_snippet.json"
265
+ path.write_text(json.dumps(document, ensure_ascii=False, indent=2), encoding="utf-8")
266
+ return publish(path, kind=KIND, **options)
267
+
268
+
269
+ _PUBLISH_OPTIONS = (
270
+ "slug",
271
+ "title",
272
+ "description",
273
+ "visibility",
274
+ "password",
275
+ "collection",
276
+ "share_unlisted",
277
+ "to",
278
+ "include_private",
279
+ )
280
+
281
+
282
+ # ---------------------------------------------------------------------------
283
+ # Scripts: with fw.snippet("label"):
284
+
285
+
286
+ @contextmanager
287
+ def snippet(
288
+ label: str,
289
+ *,
290
+ output: bool = True,
291
+ folded: bool | None = None,
292
+ links: dict[str, str] | None = None,
293
+ repository: str | None = None,
294
+ publish: bool = True,
295
+ slug: str | None = None,
296
+ title: str | None = None,
297
+ description: str | None = None,
298
+ visibility: str | None = None,
299
+ password: str | None = None,
300
+ collection: str | int | None = None,
301
+ share_unlisted: bool = False,
302
+ to: str | None = None,
303
+ include_private: bool = False,
304
+ ) -> Iterator[None]:
305
+ """Run a block and publish its code and printed output as a code snippet.
306
+
307
+ ::
308
+
309
+ with fw.snippet("fit-ols"):
310
+ model = smf.ols("score ~ income", data=df).fit()
311
+ print(model.summary())
312
+
313
+ The block runs as usual and its output still prints. When it ends, the block's
314
+ code and what it printed to stdout (as output) and stderr (as messages) are
315
+ published to Framework Cloud as a code snippet. A script shows no expression
316
+ values, so none are recorded. If the block raises, the snippet ends with the
317
+ traceback, is published, and the exception continues.
318
+
319
+ The code comes from the source file, so ``with fw.snippet()`` needs a script
320
+ or a module, not the interactive prompt. In Jupyter, use the ``%%fw_snippet``
321
+ cell magic instead (``%load_ext framework``).
322
+
323
+ The snippet names its source: the file's path from the git repository's top
324
+ (or the project root), the label, the block's lines, and the commit checked
325
+ out. The repository URL is sent only when ``repository`` is given or
326
+ ``publish.repository`` is set in settings.yml. Framework Cloud builds a GitHub
327
+ permalink from a GitHub repository, the commit, the file, and the lines.
328
+
329
+ Args:
330
+ label: Names the snippet. The default slug and title come from it.
331
+ output: False publishes the code alone.
332
+ folded: True folds every output block, False unfolds them. Default: Framework
333
+ Cloud folds output longer than 12 lines.
334
+ links: Links shown with the snippet: ``{"notebook": "https://...", "model":
335
+ "model-slug"}``. A model link takes an artifact slug in the same project
336
+ or an https URL.
337
+ repository: Repository URL to name, such as ``"https://github.com/owner/repo"``.
338
+ Default: ``publish.repository`` in settings.yml.
339
+ publish: False records the block without publishing it.
340
+ slug: Artifact slug. Default: from ``label``.
341
+ title: Artifact title. Default: ``label``.
342
+ description: Artifact description.
343
+ visibility: "public", "unlisted", or "private".
344
+ password: Password required to view the snippet.
345
+ collection: Project collection ID or slug.
346
+ share_unlisted: Allow collection readers to see an unlisted snippet.
347
+ to: A configured destination name, or "cloud".
348
+ include_private: Publish without asking about private folders.
349
+ """
350
+ caller = sys._getframe(2)
351
+ filename = caller.f_code.co_filename
352
+ start = caller.f_lineno
353
+ recorder = _Recorder()
354
+ error: BaseException | None = None
355
+ try:
356
+ with _teed(recorder):
357
+ try:
358
+ yield
359
+ except BaseException as raised:
360
+ error = raised
361
+ raise
362
+ finally:
363
+ code, lines = _block_code(filename, start)
364
+ if error is not None and not isinstance(error, (KeyboardInterrupt, GeneratorExit)):
365
+ text = "".join(traceback.format_exception(type(error), error, error.__traceback__))
366
+ recorder.block("error", text)
367
+ if code is None:
368
+ message = (
369
+ f'fw.snippet("{label}") could not read its code from {filename}. '
370
+ "Use it in a script or module, or %%fw_snippet in Jupyter."
371
+ )
372
+ if error is None:
373
+ raise FrameworkError(message)
374
+ inform(message)
375
+ elif publish:
376
+ path = None if _cell_file(filename) else Path(filename)
377
+ if path is None or not path.is_file():
378
+ path = _notebook_path()
379
+ if path is not None and path.suffix in _NOTEBOOK_SUFFIXES:
380
+ lines = None
381
+ document = _document(
382
+ [{"type": "code", "text": code}, *recorder.finished()],
383
+ _source(path, label, lines, repository),
384
+ output=output,
385
+ folded=folded,
386
+ links=links,
387
+ )
388
+ options = {
389
+ "slug": slug,
390
+ "title": title,
391
+ "description": description,
392
+ "visibility": visibility,
393
+ "password": password,
394
+ "collection": collection,
395
+ "share_unlisted": share_unlisted,
396
+ "to": to,
397
+ "include_private": include_private,
398
+ }
399
+ if document is not None:
400
+ if error is None:
401
+ _publish(document, label, options)
402
+ else:
403
+ try:
404
+ _publish(document, label, options)
405
+ except Exception as failed:
406
+ inform(f"The snippet was not published: {failed}")
407
+
408
+
409
+ def _cell_file(filename: str) -> bool:
410
+ """Whether ``filename`` names a Jupyter cell rather than a file on disk."""
411
+ return filename.startswith("<") or "ipykernel_" in filename
412
+
413
+
414
+ def _block_code(filename: str, start: int) -> tuple[str | None, tuple[int, int] | None]:
415
+ """The body of the ``with`` statement on line ``start``, dedented, and its lines."""
416
+ text = "".join(linecache.getlines(filename))
417
+ if not text:
418
+ return None, None
419
+ try:
420
+ tree = ast.parse(text)
421
+ except SyntaxError:
422
+ return None, None
423
+ found: ast.With | ast.AsyncWith | None = None
424
+ for node in ast.walk(tree):
425
+ if (
426
+ isinstance(node, (ast.With, ast.AsyncWith))
427
+ and node.lineno <= start <= (node.body[0].lineno - 1 if node.body else node.lineno)
428
+ and (found is None or node.lineno > found.lineno)
429
+ ):
430
+ found = node
431
+ if found is None or not found.body:
432
+ return None, None
433
+ first = found.body[0].lineno
434
+ decorated = getattr(found.body[0], "decorator_list", None)
435
+ if decorated:
436
+ first = min(first, *(item.lineno for item in decorated))
437
+ last = found.end_lineno or found.body[-1].lineno
438
+ lines = text.splitlines()[first - 1 : last]
439
+ return _dedent(lines), (first, last)
440
+
441
+
442
+ def _dedent(lines: list[str]) -> str:
443
+ indents = [len(line) - len(line.lstrip()) for line in lines if line.strip()]
444
+ cut = min(indents, default=0)
445
+ return "\n".join(line[cut:] for line in lines).strip("\n")
446
+
447
+
448
+ # ---------------------------------------------------------------------------
449
+ # Jupyter: %%fw_snippet
450
+
451
+
452
+ def _notebook_path() -> Path | None:
453
+ """The running notebook's file, when the front end says."""
454
+ candidates: list[str] = []
455
+ try:
456
+ from IPython import get_ipython
457
+
458
+ shell = get_ipython()
459
+ except ImportError:
460
+ shell = None
461
+ if shell is not None:
462
+ vscode = shell.user_ns.get("__vsc_ipynb_file__")
463
+ if isinstance(vscode, str):
464
+ candidates.append(vscode)
465
+ session = os.environ.get("JPY_SESSION_NAME")
466
+ if session:
467
+ candidates.append(session)
468
+ for candidate in candidates:
469
+ path = Path(candidate)
470
+ if path.is_file():
471
+ return path
472
+ return None
473
+
474
+
475
+ def _find_marker(label: str) -> list[Path]:
476
+ """Notebooks in the working folder whose code holds ``%%fw_snippet <label>``."""
477
+ pattern = re.compile(r"%%fw_snippet\s+" + re.escape(label) + r"(?![\w-])")
478
+ found = []
479
+ for path in sorted(Path.cwd().iterdir()):
480
+ if path.suffix not in _NOTEBOOK_SUFFIXES or not path.is_file():
481
+ continue
482
+ try:
483
+ if pattern.search(path.read_text(encoding="utf-8")):
484
+ found.append(path)
485
+ except (OSError, UnicodeError):
486
+ continue
487
+ return found
488
+
489
+
490
+ def _cell_lines(path: Path, label: str) -> tuple[int, int] | None:
491
+ """In a .qmd file, the lines of the code after ``%%fw_snippet <label>``."""
492
+ if path.suffix != ".qmd":
493
+ return None
494
+ text = path.read_text(encoding="utf-8").splitlines()
495
+ pattern = re.compile(r"^\s*%%fw_snippet\s+" + re.escape(label) + r"(?![\w-])")
496
+ starts = [index for index, line in enumerate(text) if pattern.match(line)]
497
+ if len(starts) != 1:
498
+ return None
499
+ first = starts[0] + 1
500
+ last = first
501
+ for index in range(first, len(text)):
502
+ if text[index].lstrip().startswith("```"):
503
+ break
504
+ last = index
505
+ while last > first and not text[last].strip():
506
+ last -= 1
507
+ while first < last and (not text[first].strip() or _OPTION_LINE.match(text[first])):
508
+ first += 1
509
+ return (first + 1, last + 1)
510
+
511
+
512
+ def _cell_code(cell: str) -> str:
513
+ """The cell's code without Quarto option lines."""
514
+ lines = [line for line in cell.splitlines() if not _OPTION_LINE.match(line)]
515
+ return "\n".join(lines).strip("\n")
516
+
517
+
518
+ def _magic_parser() -> Any:
519
+ import argparse
520
+
521
+ parser = argparse.ArgumentParser(prog="%%fw_snippet", add_help=False, exit_on_error=False)
522
+ parser.add_argument("label")
523
+ fold = parser.add_mutually_exclusive_group()
524
+ fold.add_argument("--folded", dest="folded", action="store_const", const=True)
525
+ fold.add_argument("--unfolded", dest="folded", action="store_const", const=False)
526
+ parser.add_argument("--no-output", dest="output", action="store_false")
527
+ parser.add_argument("--link", action="append", default=[])
528
+ parser.add_argument("--repository")
529
+ parser.add_argument("--file")
530
+ parser.add_argument("--no-publish", dest="publish", action="store_false")
531
+ parser.add_argument("--slug")
532
+ parser.add_argument("--title")
533
+ parser.add_argument("--description")
534
+ parser.add_argument("--visibility")
535
+ parser.add_argument("--password")
536
+ parser.add_argument("--collection")
537
+ parser.add_argument("--share-unlisted", action="store_true")
538
+ parser.add_argument("--to")
539
+ parser.add_argument("--include-private", action="store_true")
540
+ return parser
541
+
542
+
543
+ def _magic_arguments(line: str) -> Any:
544
+ import argparse
545
+ import shlex
546
+
547
+ try:
548
+ arguments = _magic_parser().parse_args(shlex.split(line))
549
+ except (argparse.ArgumentError, SystemExit, ValueError) as error:
550
+ raise FrameworkError(
551
+ f"%%fw_snippet {line}: {error}. Usage: %%fw_snippet LABEL [--folded | --unfolded] "
552
+ "[--no-output] [--link notebook=URL] [--link model=SLUG] [--repository URL] "
553
+ "[--slug SLUG] [--title TITLE] [--visibility VISIBILITY] [--to DESTINATION] "
554
+ "[--no-publish]"
555
+ ) from None
556
+ links: dict[str, str] = {}
557
+ for link in arguments.link:
558
+ rel, separator, value = link.partition("=")
559
+ if not separator:
560
+ raise FrameworkError("--link takes REL=VALUE, as in --link notebook=https://...")
561
+ links[rel] = value
562
+ arguments.links = links
563
+ return arguments
564
+
565
+
566
+ def run_cell_snippet(shell: Any, line: str, cell: str) -> dict[str, Any] | None:
567
+ """Run a cell for ``%%fw_snippet``, show its outputs, and publish them.
568
+
569
+ Returns the snippet document, or None when the cell has no code.
570
+ """
571
+ arguments = _magic_arguments(line)
572
+ code = _cell_code(cell)
573
+ recorder = _Recorder()
574
+ error: dict[str, Any] = {}
575
+
576
+ publisher = shell.display_pub
577
+ original_publish = publisher.publish
578
+ hook = shell.displayhook
579
+ original_write = hook.write_format_data
580
+ original_quiet = hook.quiet
581
+ original_start = hook.start_displayhook
582
+ original_finish = hook.finish_displayhook
583
+ original_showtraceback = shell._showtraceback
584
+
585
+ def plain(data: dict[str, Any]) -> str | None:
586
+ if any(str(key).startswith("image/") for key in data):
587
+ return None
588
+ text = data.get("text/plain")
589
+ return text if isinstance(text, str) else None
590
+
591
+ def publish_display(data: dict[str, Any], *args: Any, **kwargs: Any) -> Any:
592
+ text = plain(data or {})
593
+ if text is not None:
594
+ recorder.block("output", text)
595
+ muted, recorder.muted = recorder.muted, True
596
+ try:
597
+ return original_publish(data, *args, **kwargs)
598
+ finally:
599
+ recorder.muted = muted
600
+
601
+ def start_result() -> Any:
602
+ recorder.muted = True
603
+ return original_start()
604
+
605
+ def finish_result() -> Any:
606
+ try:
607
+ return original_finish()
608
+ finally:
609
+ recorder.muted = False
610
+
611
+ def write_result(format_dict: dict[str, Any], md_dict: Any = None) -> Any:
612
+ text = plain(format_dict or {})
613
+ if text is not None:
614
+ recorder.block("output", text)
615
+ return original_write(format_dict, md_dict)
616
+
617
+ def quiet() -> bool:
618
+ try:
619
+ return bool(hook.semicolon_at_end_of_expression(cell))
620
+ except Exception:
621
+ return False
622
+
623
+ def record_traceback(etype: Any, evalue: Any, stb: list[str]) -> None:
624
+ error["text"] = "\n".join(stb)
625
+ error["value"] = evalue
626
+
627
+ publisher.publish = publish_display
628
+ hook.write_format_data = write_result
629
+ hook.quiet = quiet
630
+ hook.start_displayhook = start_result
631
+ hook.finish_displayhook = finish_result
632
+ shell._showtraceback = record_traceback
633
+ try:
634
+ with _teed(recorder):
635
+ result = shell.run_cell(cell, store_history=False)
636
+ finally:
637
+ publisher.publish = original_publish
638
+ hook.write_format_data = original_write
639
+ hook.quiet = original_quiet
640
+ hook.start_displayhook = original_start
641
+ hook.finish_displayhook = original_finish
642
+ shell._showtraceback = original_showtraceback
643
+
644
+ raised = result.error_in_exec or result.error_before_exec
645
+ if "text" in error:
646
+ recorder.block("error", error["text"])
647
+ elif raised is not None:
648
+ recorder.block("error", "".join(traceback.format_exception_only(type(raised), raised)))
649
+
650
+ path = Path(arguments.file) if arguments.file else _notebook_path()
651
+ if path is None:
652
+ matches = _find_marker(arguments.label)
653
+ path = matches[0] if len(matches) == 1 else None
654
+ lines = _cell_lines(path, arguments.label) if path is not None else None
655
+ document = _document(
656
+ [{"type": "code", "text": code}, *recorder.finished()],
657
+ _source(path, arguments.label, lines, arguments.repository),
658
+ output=arguments.output,
659
+ folded=arguments.folded,
660
+ links=arguments.links,
661
+ )
662
+ if document is not None and arguments.publish:
663
+ options = {name: getattr(arguments, name) for name in _PUBLISH_OPTIONS}
664
+ if raised is None:
665
+ _publish_in_cell(document, arguments.label, options)
666
+ else:
667
+ try:
668
+ _publish_in_cell(document, arguments.label, options)
669
+ except Exception as failed:
670
+ inform(f"The snippet was not published: {failed}")
671
+ if raised is not None:
672
+ # The traceback was recorded instead of shown, so the cell fails once, here.
673
+ raise raised
674
+ return document
675
+
676
+
677
+ # Marks the publishing status a cell shows, so fw snippet publish leaves it out
678
+ STATUS_METADATA = {"framework": {"role": "snippet-status"}}
679
+
680
+
681
+ def _publish_in_cell(
682
+ document: dict[str, Any], label: str, options: dict[str, Any]
683
+ ) -> str | list[str]:
684
+ """Publish, and show Framework's messages as a display the snippet readers skip."""
685
+ import logging
686
+
687
+ from IPython.display import publish_display_data
688
+
689
+ from framework._log import logger
690
+
691
+ messages: list[str] = []
692
+
693
+ class Collect(logging.Handler):
694
+ def emit(self, record: logging.LogRecord) -> None:
695
+ messages.append(record.getMessage())
696
+
697
+ handlers, level = logger.handlers[:], logger.level
698
+ logger.handlers = [Collect()]
699
+ logger.setLevel(logging.INFO)
700
+ try:
701
+ return _publish(document, label, options)
702
+ finally:
703
+ logger.handlers = handlers
704
+ logger.setLevel(level)
705
+ if messages:
706
+ publish_display_data( # type: ignore[no-untyped-call]
707
+ {"text/plain": "\n".join(messages)}, metadata=STATUS_METADATA
708
+ )
709
+
710
+
711
+ def load_ipython_extension(shell: Any) -> None:
712
+ """``%load_ext framework``: add the ``%%fw_snippet`` magic and show Framework's messages."""
713
+ from framework._log import show_messages
714
+
715
+ show_messages()
716
+
717
+ def fw_snippet(line: str, cell: str) -> None:
718
+ """Run the cell and publish its code and printed output as a code snippet.
719
+
720
+ %%fw_snippet LABEL [--folded | --unfolded] [--no-output]
721
+ [--link notebook=URL] [--link model=SLUG] [--repository URL] [--file PATH]
722
+ [--slug SLUG] [--title TITLE] [--description TEXT] [--visibility VISIBILITY]
723
+ [--password PASSWORD] [--collection COLLECTION] [--share-unlisted]
724
+ [--to DESTINATION] [--include-private] [--no-publish]
725
+ """
726
+ run_cell_snippet(shell, line, cell)
727
+
728
+ shell.register_magic_function(fw_snippet, magic_kind="cell", magic_name="fw_snippet")
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: framework-py
3
- Version: 0.1.0.dev26
3
+ Version: 0.1.0.dev28
4
4
  Summary: Structured data science projects: a data catalog, integrity tracking, caching, outputs, and AI guardrails
5
5
  Project-URL: Homepage, https://framework.pub
6
6
  Project-URL: Source, https://github.com/frameworktools/framework-py
@@ -1,4 +1,4 @@
1
- framework/__init__.py,sha256=wu0kbAo9WBq74juy_W5v55CuB2KsabuiS21hZC5iAJU,6700
1
+ framework/__init__.py,sha256=IZuuoHC2nHIPRRC9k6AkpQZ6HmnySBvh0s9us-wlPbE,6834
2
2
  framework/__main__.py,sha256=S0ChmRQnNa9W5_MPXhHnH61Kjr_pNthkeS41eJU1w0U,9093
3
3
  framework/_agents_md.py,sha256=PSWPJiCUTRirk81sz-Y33qHnqY16HOhHY8pH77ct1ik,11857
4
4
  framework/_artifacts.py,sha256=XjYOBQOQJuvztUyqwWsc3xc1qJ5o4O5XpCW49WN548k,55807
@@ -26,12 +26,12 @@ framework/ai.py,sha256=hqBNXngw7VyQBcSt7kB20Zpa6N_5WS1hNhDi2zWJw0U,9218
26
26
  framework/bootstrap.py,sha256=mYlD1BG9KaG9PAIHlw3l6677fJYwQdtih95W9Rqmnho,9687
27
27
  framework/cache.py,sha256=z8Zaem5Q7dsAghuX9CgK_j0vhX5dDlRw7fl9f-CF3eA,17440
28
28
  framework/catalog.py,sha256=sl4V1judSCVoa2LYLXeQdDhWdsONTC_5s2_fpyPylto,13202
29
- framework/cloud.py,sha256=6w1oIfgGPPLLC2ILUvAqPXhIMlU08XJXd-KsP85QDbY,67608
29
+ framework/cloud.py,sha256=y4w5euumzkNXOfC9q8v7FVW5yHEO7QORXC3D2BlirMI,67843
30
30
  framework/config_home.py,sha256=iSb0VuENgNOSnsZ_v53xFqjRiGgrB7OzZbHbyDOqKzs,8265
31
31
  framework/connections.py,sha256=99JqrNAi2zBpNYyxLV6SYj-NAJjh56pYh9jaMSYOHCU,32742
32
32
  framework/console.py,sha256=zQ0LuFDcxe1W1kuc5E1bJ03h8d3cvwFoqDT8dAJQyN0,1267
33
33
  framework/create.py,sha256=3ykMUhMslVsjiE__9Scx5v_6CU2Ye36wWqlp17j_vGw,53030
34
- framework/data.py,sha256=cAOxwmF_nowXIZPl6vW5-ZUyI_z16H1bRJM60gJgeoo,36522
34
+ framework/data.py,sha256=HpiRl5qTk1o9G-XXNDgsWhNOgXlyYegv8-8KDz-HyPA,40853
35
35
  framework/data_registry.py,sha256=2hbXsoY9IF5Zj6lkiTE2RO2c5j2Z2vXKm2iTN7FdsQk,22499
36
36
  framework/db.py,sha256=UYEUCvk4vGytBkmdNTUxRpYPtoFSDHwTpgpfWFw31aE,8522
37
37
  framework/display.py,sha256=0FuKUt67uf5h1WjD4bwzoruc2XAN4y_j3mz_f5FskhE,5334
@@ -54,6 +54,7 @@ framework/security.py,sha256=-x66bSIIE_xOzkTf5Ojt7xIECKuAIpKVJZ5TtIGrwys,51147
54
54
  framework/settings.py,sha256=Sb2YyE53aYLWPj423P-pQVCvRpLsiHydgZh2K0luTQQ,23893
55
55
  framework/skill_pull.py,sha256=vr_yxe446JhHbtI0hps19AnLN6S7BacSt2oxn7BIlos,21859
56
56
  framework/skill_sync.py,sha256=0gqaeVkiNaqGJxmrVrKrYaX8wI3DNFZUsJEgG63xcKc,23060
57
+ framework/snippets.py,sha256=J8j36YqHgG-HhWlhJ9RmJK-qwVGOOSOOd1pNakJyuVE,26481
57
58
  framework/storage.py,sha256=Haz8CBXrz2CCqGHOVgHKC3mUUwobnod2SanU759lK7c,56115
58
59
  framework/sync_all.py,sha256=tmSnapf-JfaL1_8rMpnr3wqHvSGJamn7c8ja4GQFKHA,34610
59
60
  framework/upgrade.py,sha256=WtghpLR65YFH_JQFmo1JlH9lHdwsXQd-ZzzOTmyOL28,65593
@@ -140,7 +141,7 @@ framework/templates/stubs/notebook-default.qmd,sha256=8Js1S2BcHYeQWmJS-kt31gBvIj
140
141
  framework/templates/stubs/notebook-minimal.qmd,sha256=2U2k8judffhrh_6m6r6OAbQBCYEtViAjeKRBqBYCJTM,178
141
142
  framework/templates/stubs/notebook-revealjs.qmd,sha256=_ztT_gHoudpB5DIjlbvtB5zQ8163DeV8xbWcT-HgJrI,409
142
143
  framework/templates/stubs/script-default.py,sha256=vbWqgW0_rHot17mQC3tYb92g_9L-cbTOeL3y99BRrSk,216
143
- framework_py-0.1.0.dev26.dist-info/METADATA,sha256=ZFaJ834HxwvPY1yz8vaGH-bRHdi0gdCtaYgYdY3PMrU,4682
144
- framework_py-0.1.0.dev26.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
145
- framework_py-0.1.0.dev26.dist-info/licenses/LICENSE,sha256=giOOgSwALkZimC7uhLZQK8Untez641bLW1-2-Cjvmq0,1072
146
- framework_py-0.1.0.dev26.dist-info/RECORD,,
144
+ framework_py-0.1.0.dev28.dist-info/METADATA,sha256=cTispVoaGZgGGcA8GR630W4VE96ZRIRwoxs1HjTeIdE,4682
145
+ framework_py-0.1.0.dev28.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
146
+ framework_py-0.1.0.dev28.dist-info/licenses/LICENSE,sha256=giOOgSwALkZimC7uhLZQK8Untez641bLW1-2-Cjvmq0,1072
147
+ framework_py-0.1.0.dev28.dist-info/RECORD,,