framework-py 0.1.0.dev26__py3-none-any.whl → 0.1.0.dev28__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- framework/__init__.py +3 -0
- framework/cloud.py +4 -1
- framework/data.py +109 -20
- framework/snippets.py +728 -0
- {framework_py-0.1.0.dev26.dist-info → framework_py-0.1.0.dev28.dist-info}/METADATA +1 -1
- {framework_py-0.1.0.dev26.dist-info → framework_py-0.1.0.dev28.dist-info}/RECORD +8 -7
- {framework_py-0.1.0.dev26.dist-info → framework_py-0.1.0.dev28.dist-info}/WHEEL +0 -0
- {framework_py-0.1.0.dev26.dist-info → framework_py-0.1.0.dev28.dist-info}/licenses/LICENSE +0 -0
framework/__init__.py
CHANGED
|
@@ -131,6 +131,8 @@ from framework.security import AuditResult, git_security_audit
|
|
|
131
131
|
from framework.settings import settings, settings_read, settings_write
|
|
132
132
|
from framework.skill_pull import pull_skills, push_skills
|
|
133
133
|
from framework.skill_sync import skills_sync
|
|
134
|
+
from framework.snippets import load_ipython_extension as load_ipython_extension
|
|
135
|
+
from framework.snippets import snippet
|
|
134
136
|
from framework.spec.check import spec_check
|
|
135
137
|
from framework.spec.merge import SpecError
|
|
136
138
|
from framework.spec.resolve import ResolvedSpec, spec_resolve
|
|
@@ -274,6 +276,7 @@ __all__ = [
|
|
|
274
276
|
"settings_write",
|
|
275
277
|
"show_table",
|
|
276
278
|
"skills_sync",
|
|
279
|
+
"snippet",
|
|
277
280
|
"spec_check",
|
|
278
281
|
"spec_resolve",
|
|
279
282
|
"status",
|
framework/cloud.py
CHANGED
|
@@ -1269,7 +1269,10 @@ def cloud_publish(
|
|
|
1269
1269
|
if response.is_error:
|
|
1270
1270
|
raise _error(response, "Publish")
|
|
1271
1271
|
result = response.json()
|
|
1272
|
-
if result.get("version") is not None:
|
|
1272
|
+
if result.get("unchanged") and result.get("version") is not None:
|
|
1273
|
+
# Cloud keeps the newest version when nothing changed (code snippets)
|
|
1274
|
+
inform(f"Unchanged, still version {result['version']}: {result.get('url')}")
|
|
1275
|
+
elif result.get("version") is not None:
|
|
1273
1276
|
inform(f"Published version {result['version']}: {result.get('url')}")
|
|
1274
1277
|
else:
|
|
1275
1278
|
inform(f"Published: {result.get('url')}")
|
framework/data.py
CHANGED
|
@@ -436,13 +436,39 @@ def data_read(
|
|
|
436
436
|
return data
|
|
437
437
|
|
|
438
438
|
|
|
439
|
-
def
|
|
440
|
-
"""The
|
|
439
|
+
def _catalog_entry(name: str) -> DataInfo | None:
|
|
440
|
+
"""The catalog entry ``name`` already has, or None."""
|
|
441
|
+
try:
|
|
442
|
+
data = settings_read().get("data")
|
|
443
|
+
except FrameworkError:
|
|
444
|
+
return None
|
|
445
|
+
if catalog_locate(name, data) is None:
|
|
446
|
+
return None
|
|
447
|
+
try:
|
|
448
|
+
return data_info(name)
|
|
449
|
+
except DataNotFoundError:
|
|
450
|
+
# An entry without a path is saved like a new name
|
|
451
|
+
return None
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
def _delimiter_name(delimiter: str) -> str:
|
|
455
|
+
"""``comma``, ``tab``, ``semicolon``, or ``space`` for a delimiter or its character."""
|
|
456
|
+
_separator(delimiter)
|
|
457
|
+
for key, character in DELIMITERS.items():
|
|
458
|
+
if delimiter in (key, character):
|
|
459
|
+
return key
|
|
460
|
+
return delimiter
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def _save_target(path: str, type: str | None) -> tuple[Path, str, str | None, DataInfo | None]:
|
|
464
|
+
"""The file to write, its type, the catalog name (None when ``path`` is a file path),
|
|
465
|
+
and the catalog entry the name already has (None for a new name or a file path).
|
|
441
466
|
|
|
442
467
|
A relative file path starts at the project root, as the catalog's paths do, so a
|
|
443
468
|
notebook in ``notebooks/`` writes where a script in the root would. A dot name that
|
|
444
469
|
ends in an extension, such as ``inputs.final.x.csv``, is the name ``inputs.final.x``
|
|
445
|
-
saved as that type.
|
|
470
|
+
saved as that type. A name already in the catalog is saved to its entry's file, in
|
|
471
|
+
its entry's type, as Stata's ``fw_save`` does.
|
|
446
472
|
"""
|
|
447
473
|
has_slash = "/" in path or "\\" in path
|
|
448
474
|
extension = re.search(r"\.(csv|parquet|rds)$", path, re.IGNORECASE)
|
|
@@ -458,7 +484,7 @@ def _save_target(path: str, type: str | None) -> tuple[Path, str, str | None]:
|
|
|
458
484
|
if not path.lower().endswith(f".{type}"):
|
|
459
485
|
path = f"{path}.{type}"
|
|
460
486
|
inform(f"Note: Added .{type} extension to path")
|
|
461
|
-
return in_root(path, root), type, None
|
|
487
|
+
return in_root(path, root), type, None, None
|
|
462
488
|
|
|
463
489
|
if extension:
|
|
464
490
|
given = extension.group(1).lower()
|
|
@@ -467,6 +493,18 @@ def _save_target(path: str, type: str | None) -> tuple[Path, str, str | None]:
|
|
|
467
493
|
type = given
|
|
468
494
|
path = path[: extension.start()]
|
|
469
495
|
|
|
496
|
+
entry = _catalog_entry(path)
|
|
497
|
+
if entry is not None:
|
|
498
|
+
# tsv is delimited text with tabs, which data_save() writes as csv
|
|
499
|
+
entry_type = "csv" if entry.type == "tsv" else entry.type
|
|
500
|
+
if type is not None and type.lower() != entry_type:
|
|
501
|
+
raise FrameworkError(
|
|
502
|
+
f"'{path}' is a {entry.type} file in the data catalog "
|
|
503
|
+
f"({relative_to_root(entry.path, root)}), but type is '{type}'.\n"
|
|
504
|
+
"Leave type out to save it as its entry says, or save it under another name."
|
|
505
|
+
)
|
|
506
|
+
return entry.path, entry_type, path, entry
|
|
507
|
+
|
|
470
508
|
parts = path.split(".")
|
|
471
509
|
if len(parts) == 1 or not all(parts):
|
|
472
510
|
suffix = type or "parquet"
|
|
@@ -497,7 +535,7 @@ def _save_target(path: str, type: str | None) -> tuple[Path, str, str | None]:
|
|
|
497
535
|
)
|
|
498
536
|
type = type or "parquet"
|
|
499
537
|
target = in_root(folder, root) / f"{'_'.join(parts[end:])}.{type}"
|
|
500
|
-
return target, type, path
|
|
538
|
+
return target, type, path, None
|
|
501
539
|
|
|
502
540
|
|
|
503
541
|
def _in_settings_folder(folder: Path) -> bool:
|
|
@@ -598,7 +636,7 @@ def data_save(
|
|
|
598
636
|
data: pd.DataFrame,
|
|
599
637
|
path: str,
|
|
600
638
|
type: str | None = None,
|
|
601
|
-
delimiter: str =
|
|
639
|
+
delimiter: str | None = None,
|
|
602
640
|
locked: bool | None = None,
|
|
603
641
|
force: bool = False,
|
|
604
642
|
unlock: bool = False,
|
|
@@ -617,14 +655,20 @@ def data_save(
|
|
|
617
655
|
file path with a ``/``, ending in ``.csv`` or ``.parquet``. A name's leading
|
|
618
656
|
parts select a folder from ``directories``: the longest run that
|
|
619
657
|
names a key, such as ``inputs_intermediate``, ``inputs_private_raw``, or a
|
|
620
|
-
course's ``data``. The rest becomes the file name. A name is also added
|
|
621
|
-
the data catalog, so ``data_read()`` finds it. A name
|
|
658
|
+
course's ``data``. The rest becomes the file name. A new name is also added
|
|
659
|
+
to the data catalog, so ``data_read()`` finds it. A name already in the
|
|
660
|
+
catalog is saved to its entry's file, in its entry's type and delimiter, and
|
|
661
|
+
the entry is left as it is (its description and other fields are kept). A
|
|
662
|
+
name may end in
|
|
622
663
|
``.csv`` or ``.parquet`` to choose the type: ``"inputs.final.x.csv"`` is
|
|
623
664
|
the name ``inputs.final.x`` saved as CSV. A relative file path starts at the
|
|
624
665
|
project root, wherever Python runs.
|
|
625
|
-
type: ``"parquet"`` or ``"csv"``. Default:
|
|
626
|
-
|
|
627
|
-
|
|
666
|
+
type: ``"parquet"`` or ``"csv"``. Default: the catalog entry's type, then the
|
|
667
|
+
file extension, then Parquet for a new catalog name. A type that differs
|
|
668
|
+
from the catalog entry's is an error.
|
|
669
|
+
delimiter: Delimiter for CSV files: comma, tab, semicolon, or space. Default:
|
|
670
|
+
the catalog entry's delimiter, else comma. A delimiter that differs from the
|
|
671
|
+
catalog entry's is an error.
|
|
628
672
|
locked: Whether later reads must find the file unchanged. Default: unlocked, as
|
|
629
673
|
in R, so a pipeline can save the data again. Pass ``locked=True`` to lock data
|
|
630
674
|
you mean to keep as it is. Left out, a save keeps a lock the data already
|
|
@@ -648,14 +692,24 @@ def data_save(
|
|
|
648
692
|
"""
|
|
649
693
|
if not isinstance(data, pd.DataFrame):
|
|
650
694
|
raise FrameworkError("data_save() needs a pandas DataFrame")
|
|
651
|
-
target, file_type, name = _save_target(path, type)
|
|
695
|
+
target, file_type, name, entry = _save_target(path, type)
|
|
696
|
+
if entry is not None and entry.delimiter is not None and file_type == "csv":
|
|
697
|
+
if delimiter is not None and _delimiter_name(delimiter) != entry.delimiter:
|
|
698
|
+
raise FrameworkError(
|
|
699
|
+
f"'{name}' is {entry.delimiter}-delimited in the data catalog, but "
|
|
700
|
+
f"delimiter is '{delimiter}'. Leave delimiter out to save it as its entry "
|
|
701
|
+
"says, or save it under another name."
|
|
702
|
+
)
|
|
703
|
+
delimiter = entry.delimiter
|
|
704
|
+
delimiter = _delimiter_name(delimiter or "comma")
|
|
652
705
|
if file_type == "rds":
|
|
653
706
|
raise UnsupportedFormatError(
|
|
654
707
|
"Python does not write .rds files. Save as 'parquet' or 'csv', which both "
|
|
655
708
|
"languages read."
|
|
656
709
|
)
|
|
657
710
|
if file_type not in _SAVE_TYPES:
|
|
658
|
-
|
|
711
|
+
where = f" ('{name}' in the data catalog)" if entry is not None else ""
|
|
712
|
+
raise UnsupportedFormatError(f"data_save() writes csv or parquet, not {file_type}{where}")
|
|
659
713
|
|
|
660
714
|
folder = target.parent
|
|
661
715
|
# A missing folder is created when it is a settings folder or inside one (git
|
|
@@ -734,7 +788,21 @@ def data_save(
|
|
|
734
788
|
|
|
735
789
|
ledger_push_quietly(key, digest, target)
|
|
736
790
|
|
|
737
|
-
if name is not None:
|
|
791
|
+
if name is not None and entry is not None:
|
|
792
|
+
# The entry already names this file. Only its lock can change, so the
|
|
793
|
+
# catalog's flag agrees with framework.db; everything else is left as it is.
|
|
794
|
+
if entry.locked != locked:
|
|
795
|
+
try:
|
|
796
|
+
_catalog_set(name, {"locked": True} if locked else {}, drop=("locked",))
|
|
797
|
+
inform(f"Data catalog entry {'locked' if locked else 'unlocked'}: {name}")
|
|
798
|
+
except FrameworkError as error:
|
|
799
|
+
warnings.warn(
|
|
800
|
+
f"File saved but could not update the lock in the data catalog for "
|
|
801
|
+
f"'{name}': {error}",
|
|
802
|
+
FrameworkWarning,
|
|
803
|
+
stacklevel=2,
|
|
804
|
+
)
|
|
805
|
+
elif name is not None:
|
|
738
806
|
try:
|
|
739
807
|
data_add(target, name=name, type=file_type, delimiter=delimiter, locked=locked)
|
|
740
808
|
except FrameworkError as error:
|
|
@@ -768,11 +836,18 @@ def _derive_name(file_path: Path, config: dict[str, Any], root: Path) -> str:
|
|
|
768
836
|
return file_path.stem
|
|
769
837
|
|
|
770
838
|
|
|
771
|
-
def _catalog_set(name: str, entry: dict[str, Any]) ->
|
|
839
|
+
def _catalog_set(name: str, entry: dict[str, Any], drop: Sequence[str] = ()) -> bool:
|
|
772
840
|
"""Write one catalog entry into settings.yml or the split file that holds ``data``.
|
|
773
841
|
|
|
774
|
-
|
|
842
|
+
An existing entry is updated, not replaced: the fields in ``entry`` are set, the
|
|
843
|
+
fields in ``drop`` that ``entry`` leaves out are removed, and every other field (a
|
|
844
|
+
description, notes, stata_names) is kept. The file is edited in place, so its
|
|
845
|
+
comments and other entries keep their form.
|
|
846
|
+
|
|
847
|
+
Returns:
|
|
848
|
+
Whether the entry already existed.
|
|
775
849
|
"""
|
|
850
|
+
existed = False
|
|
776
851
|
settings_file = find_settings_file()
|
|
777
852
|
if settings_file is None:
|
|
778
853
|
raise SettingsError("Configuration file 'settings.yml' (or legacy config.yml) not found")
|
|
@@ -782,9 +857,20 @@ def _catalog_set(name: str, entry: dict[str, Any]) -> None:
|
|
|
782
857
|
|
|
783
858
|
def set_entry(current: dict[str, Any]) -> None:
|
|
784
859
|
# An entry already written with dotted keys is updated where it is.
|
|
860
|
+
nonlocal existed
|
|
785
861
|
existing = catalog_locate(name, current)
|
|
786
862
|
if existing is not None:
|
|
787
|
-
|
|
863
|
+
existed = True
|
|
864
|
+
holder, key = existing
|
|
865
|
+
old = holder[key]
|
|
866
|
+
if not isinstance(old, dict):
|
|
867
|
+
# A bare path entry becomes a mapping
|
|
868
|
+
old = {"path": old} if isinstance(old, str) else {}
|
|
869
|
+
holder[key] = old
|
|
870
|
+
for field in drop:
|
|
871
|
+
if field not in entry:
|
|
872
|
+
old.pop(field, None)
|
|
873
|
+
old.update(entry)
|
|
788
874
|
return
|
|
789
875
|
node = current
|
|
790
876
|
parts = name.split(".")
|
|
@@ -810,7 +896,7 @@ def _catalog_set(name: str, entry: dict[str, Any]) -> None:
|
|
|
810
896
|
return {"data": content}
|
|
811
897
|
|
|
812
898
|
_yaml.update(in_root(source, settings_file.parent), in_split)
|
|
813
|
-
return
|
|
899
|
+
return existed
|
|
814
900
|
|
|
815
901
|
def in_settings(raw: Any) -> Any:
|
|
816
902
|
raw = raw if isinstance(raw, dict) else {}
|
|
@@ -821,6 +907,7 @@ def _catalog_set(name: str, entry: dict[str, Any]) -> None:
|
|
|
821
907
|
return raw
|
|
822
908
|
|
|
823
909
|
_yaml.update(settings_file, in_settings)
|
|
910
|
+
return existed
|
|
824
911
|
|
|
825
912
|
|
|
826
913
|
def data_add(
|
|
@@ -889,7 +976,9 @@ def data_add(
|
|
|
889
976
|
if locked:
|
|
890
977
|
entry["locked"] = True
|
|
891
978
|
if update_config:
|
|
892
|
-
|
|
893
|
-
|
|
979
|
+
# An existing entry keeps its other fields (description, notes); the
|
|
980
|
+
# delimiter and lock follow this file
|
|
981
|
+
existed = _catalog_set(name, entry, drop=("delimiter", "locked"))
|
|
982
|
+
inform(f"Data spec {'updated in' if existed else 'added to'} config: {name}")
|
|
894
983
|
inform(f'\nYou can now read this data with: data_read("{name}")')
|
|
895
984
|
return entry
|
framework/snippets.py
ADDED
|
@@ -0,0 +1,728 @@
|
|
|
1
|
+
"""Code snippets: a cell's or a block's code and the text it printed.
|
|
2
|
+
|
|
3
|
+
A code snippet is a ``framework.artifact/v1`` document with ``kind: "code_snippet"``.
|
|
4
|
+
Framework Cloud highlights the code, styles ANSI output, and applies the size limits,
|
|
5
|
+
so this module only records what ran and what it printed.
|
|
6
|
+
|
|
7
|
+
In Jupyter, ``%load_ext framework`` adds the ``%%fw_snippet`` cell magic. In a script,
|
|
8
|
+
``with fw.snippet("label"):`` records the block.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import ast
|
|
14
|
+
import json
|
|
15
|
+
import linecache
|
|
16
|
+
import os
|
|
17
|
+
import re
|
|
18
|
+
import subprocess
|
|
19
|
+
import sys
|
|
20
|
+
import tempfile
|
|
21
|
+
import traceback
|
|
22
|
+
from collections.abc import Iterator, Sequence
|
|
23
|
+
from contextlib import contextmanager
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
from typing import Any, TextIO
|
|
26
|
+
|
|
27
|
+
from framework._log import inform
|
|
28
|
+
from framework.errors import FrameworkError
|
|
29
|
+
|
|
30
|
+
SCHEMA = "framework.artifact/v1"
|
|
31
|
+
KIND = "code_snippet"
|
|
32
|
+
LINK_RELS = ("notebook", "model")
|
|
33
|
+
_OPTION_LINE = re.compile(r"^\s*#\|")
|
|
34
|
+
_SLUG = re.compile(r"^[A-Za-z0-9_-]{1,120}$")
|
|
35
|
+
_NOTEBOOK_SUFFIXES = (".ipynb", ".qmd", ".md")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
# ---------------------------------------------------------------------------
|
|
39
|
+
# The document
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _links(links: dict[str, str] | None) -> list[dict[str, str]]:
|
|
43
|
+
"""``display.links`` from ``{"notebook": "https://...", "model": "slug or URL"}``."""
|
|
44
|
+
if not links:
|
|
45
|
+
return []
|
|
46
|
+
if not isinstance(links, dict):
|
|
47
|
+
raise FrameworkError(
|
|
48
|
+
'links must be a dict such as {"notebook": "https://...", "model": "model-slug"}.'
|
|
49
|
+
)
|
|
50
|
+
out: list[dict[str, str]] = []
|
|
51
|
+
for rel, value in links.items():
|
|
52
|
+
if rel not in LINK_RELS:
|
|
53
|
+
raise FrameworkError(
|
|
54
|
+
'links must be named "notebook" or "model", as in '
|
|
55
|
+
'{"notebook": "https://...", "model": "model-slug"}.'
|
|
56
|
+
)
|
|
57
|
+
if not isinstance(value, str) or not value:
|
|
58
|
+
raise FrameworkError("Each link must be one string.")
|
|
59
|
+
if re.fullmatch(r"https://\S+", value):
|
|
60
|
+
out.append({"rel": rel, "url": value})
|
|
61
|
+
elif rel == "model" and _SLUG.fullmatch(value):
|
|
62
|
+
out.append({"rel": rel, "artifact": value})
|
|
63
|
+
elif rel == "model":
|
|
64
|
+
raise FrameworkError(
|
|
65
|
+
"A model link takes an artifact slug in this project or an https URL."
|
|
66
|
+
)
|
|
67
|
+
else:
|
|
68
|
+
raise FrameworkError("A notebook link takes an https URL.")
|
|
69
|
+
return out
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _document(
|
|
73
|
+
blocks: Sequence[dict[str, Any]],
|
|
74
|
+
source: dict[str, Any],
|
|
75
|
+
*,
|
|
76
|
+
output: bool = True,
|
|
77
|
+
folded: bool | None = None,
|
|
78
|
+
links: dict[str, str] | None = None,
|
|
79
|
+
) -> dict[str, Any] | None:
|
|
80
|
+
"""The snippet document, or None when it has no code.
|
|
81
|
+
|
|
82
|
+
Adjacent output blocks join, as a notebook shows them. With ``output=False`` only
|
|
83
|
+
the code is kept.
|
|
84
|
+
"""
|
|
85
|
+
kept = [dict(block) for block in blocks if block.get("text")]
|
|
86
|
+
if not output:
|
|
87
|
+
kept = [block for block in kept if block["type"] == "code"]
|
|
88
|
+
merged: list[dict[str, Any]] = []
|
|
89
|
+
for block in kept:
|
|
90
|
+
if merged and merged[-1]["type"] == block["type"] and block["type"] in ("code", "output"):
|
|
91
|
+
merged[-1]["text"] = merged[-1]["text"] + "\n" + block["text"]
|
|
92
|
+
else:
|
|
93
|
+
merged.append(block)
|
|
94
|
+
if not any(block["type"] == "code" for block in merged):
|
|
95
|
+
return None
|
|
96
|
+
if folded is not None:
|
|
97
|
+
for block in merged:
|
|
98
|
+
if block["type"] == "output":
|
|
99
|
+
block["folded"] = folded
|
|
100
|
+
document: dict[str, Any] = {
|
|
101
|
+
"schema": SCHEMA,
|
|
102
|
+
"kind": KIND,
|
|
103
|
+
"source": {key: value for key, value in source.items() if value is not None},
|
|
104
|
+
"snippet": {"blocks": merged},
|
|
105
|
+
}
|
|
106
|
+
shown = _links(links)
|
|
107
|
+
if shown:
|
|
108
|
+
document["display"] = {"links": shown}
|
|
109
|
+
return document
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _trim(text: str) -> str:
|
|
113
|
+
"""Line endings as ``\\n``, without the trailing newline a print adds."""
|
|
114
|
+
text = text.replace("\r\n", "\n").replace("\r", "\n")
|
|
115
|
+
return text[:-1] if text.endswith("\n") else text
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
class _Recorder:
|
|
119
|
+
"""Blocks in the order they were printed. Text of one kind in a row joins."""
|
|
120
|
+
|
|
121
|
+
def __init__(self) -> None:
|
|
122
|
+
self.blocks: list[dict[str, Any]] = []
|
|
123
|
+
self._open: str | None = None
|
|
124
|
+
# Set while IPython writes a display or a result, which a terminal prints to stdout
|
|
125
|
+
self.muted = False
|
|
126
|
+
|
|
127
|
+
def stream(self, kind: str, text: str) -> None:
|
|
128
|
+
if not text or self.muted:
|
|
129
|
+
return
|
|
130
|
+
if self._open == kind and self.blocks:
|
|
131
|
+
self.blocks[-1]["text"] += text
|
|
132
|
+
else:
|
|
133
|
+
self.blocks.append({"type": kind, "text": text})
|
|
134
|
+
self._open = kind
|
|
135
|
+
|
|
136
|
+
def block(self, kind: str, text: str) -> None:
|
|
137
|
+
self._open = None
|
|
138
|
+
self.blocks.append({"type": kind, "text": text})
|
|
139
|
+
|
|
140
|
+
def finished(self) -> list[dict[str, Any]]:
|
|
141
|
+
return [{**block, "text": _trim(block["text"])} for block in self.blocks]
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
class _Tee:
|
|
145
|
+
"""A text stream that writes through and records what passes."""
|
|
146
|
+
|
|
147
|
+
def __init__(self, target: TextIO, record: Any) -> None:
|
|
148
|
+
self._target = target
|
|
149
|
+
self._record = record
|
|
150
|
+
|
|
151
|
+
def write(self, text: str) -> int:
|
|
152
|
+
self._record(text)
|
|
153
|
+
return self._target.write(text)
|
|
154
|
+
|
|
155
|
+
def __getattr__(self, name: str) -> Any:
|
|
156
|
+
return getattr(self._target, name)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
@contextmanager
|
|
160
|
+
def _teed(recorder: _Recorder) -> Iterator[None]:
|
|
161
|
+
"""Record stdout as output and stderr as messages, still printing both."""
|
|
162
|
+
stdout, stderr = sys.stdout, sys.stderr
|
|
163
|
+
sys.stdout = _Tee(stdout, lambda text: recorder.stream("output", text))
|
|
164
|
+
sys.stderr = _Tee(stderr, lambda text: recorder.stream("message", text))
|
|
165
|
+
try:
|
|
166
|
+
yield
|
|
167
|
+
finally:
|
|
168
|
+
sys.stdout, sys.stderr = stdout, stderr
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
# ---------------------------------------------------------------------------
|
|
172
|
+
# Where the code came from
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _git(directory: Path, *args: str) -> str | None:
|
|
176
|
+
try:
|
|
177
|
+
result = subprocess.run(
|
|
178
|
+
["git", "-C", str(directory), *args],
|
|
179
|
+
capture_output=True,
|
|
180
|
+
encoding="utf-8",
|
|
181
|
+
errors="replace",
|
|
182
|
+
check=False,
|
|
183
|
+
)
|
|
184
|
+
except OSError:
|
|
185
|
+
return None
|
|
186
|
+
value = result.stdout.strip()
|
|
187
|
+
return value if result.returncode == 0 and value else None
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _location(path: Path | None) -> dict[str, Any]:
|
|
191
|
+
"""The file's path from the git repository's top (else the project root), and the commit."""
|
|
192
|
+
if path is None or not path.is_file():
|
|
193
|
+
return {"file": None, "commit": None}
|
|
194
|
+
path = path.resolve()
|
|
195
|
+
top = _git(path.parent, "rev-parse", "--show-toplevel")
|
|
196
|
+
commit = None
|
|
197
|
+
root: Path | None
|
|
198
|
+
if top is not None:
|
|
199
|
+
root = Path(top).resolve()
|
|
200
|
+
head = _git(path.parent, "rev-parse", "HEAD")
|
|
201
|
+
if head is not None and re.fullmatch(r"[0-9a-f]{40}", head):
|
|
202
|
+
commit = head
|
|
203
|
+
else:
|
|
204
|
+
from framework.project import find_settings_file
|
|
205
|
+
|
|
206
|
+
found = find_settings_file(path.parent)
|
|
207
|
+
root = found.parent.resolve() if found is not None else None
|
|
208
|
+
try:
|
|
209
|
+
file = path.relative_to(root).as_posix() if root is not None else path.name
|
|
210
|
+
except ValueError:
|
|
211
|
+
file = path.name
|
|
212
|
+
return {"file": file, "commit": commit}
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _repository(repository: str | None) -> str | None:
|
|
216
|
+
"""The argument, else ``publish.repository`` in settings.yml. Sent only when set."""
|
|
217
|
+
if repository is None:
|
|
218
|
+
try:
|
|
219
|
+
from framework.settings import settings_read
|
|
220
|
+
|
|
221
|
+
publish = settings_read().get("publish")
|
|
222
|
+
repository = publish.get("repository") if isinstance(publish, dict) else None
|
|
223
|
+
except Exception:
|
|
224
|
+
repository = None
|
|
225
|
+
if repository is None:
|
|
226
|
+
return None
|
|
227
|
+
if not isinstance(repository, str) or not repository.startswith("https://"):
|
|
228
|
+
raise FrameworkError(
|
|
229
|
+
"repository must be an https URL, such as https://github.com/owner/repo."
|
|
230
|
+
)
|
|
231
|
+
return repository
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _source(
|
|
235
|
+
path: Path | None,
|
|
236
|
+
label: str,
|
|
237
|
+
lines: tuple[int, int] | None,
|
|
238
|
+
repository: str | None,
|
|
239
|
+
) -> dict[str, Any]:
|
|
240
|
+
location = _location(path)
|
|
241
|
+
return {
|
|
242
|
+
"language": "python",
|
|
243
|
+
"file": location["file"],
|
|
244
|
+
"label": label,
|
|
245
|
+
"lines": list(lines) if lines is not None and location["file"] is not None else None,
|
|
246
|
+
"commit": location["commit"],
|
|
247
|
+
"repository": _repository(repository),
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
# ---------------------------------------------------------------------------
|
|
252
|
+
# Publishing
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def _publish(document: dict[str, Any], label: str, options: dict[str, Any]) -> str | list[str]:
|
|
256
|
+
"""Publish a snippet document as ``code_snippet.json``."""
|
|
257
|
+
from framework.publish_bundle import artifact_slug
|
|
258
|
+
from framework.storage import publish
|
|
259
|
+
|
|
260
|
+
options = dict(options)
|
|
261
|
+
options["slug"] = options.get("slug") or artifact_slug(label)
|
|
262
|
+
options["title"] = options.get("title") or label
|
|
263
|
+
with tempfile.TemporaryDirectory(prefix="fw_snippet_") as temp:
|
|
264
|
+
path = Path(temp) / "code_snippet.json"
|
|
265
|
+
path.write_text(json.dumps(document, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
266
|
+
return publish(path, kind=KIND, **options)
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
_PUBLISH_OPTIONS = (
|
|
270
|
+
"slug",
|
|
271
|
+
"title",
|
|
272
|
+
"description",
|
|
273
|
+
"visibility",
|
|
274
|
+
"password",
|
|
275
|
+
"collection",
|
|
276
|
+
"share_unlisted",
|
|
277
|
+
"to",
|
|
278
|
+
"include_private",
|
|
279
|
+
)
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
# ---------------------------------------------------------------------------
|
|
283
|
+
# Scripts: with fw.snippet("label"):
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
@contextmanager
|
|
287
|
+
def snippet(
|
|
288
|
+
label: str,
|
|
289
|
+
*,
|
|
290
|
+
output: bool = True,
|
|
291
|
+
folded: bool | None = None,
|
|
292
|
+
links: dict[str, str] | None = None,
|
|
293
|
+
repository: str | None = None,
|
|
294
|
+
publish: bool = True,
|
|
295
|
+
slug: str | None = None,
|
|
296
|
+
title: str | None = None,
|
|
297
|
+
description: str | None = None,
|
|
298
|
+
visibility: str | None = None,
|
|
299
|
+
password: str | None = None,
|
|
300
|
+
collection: str | int | None = None,
|
|
301
|
+
share_unlisted: bool = False,
|
|
302
|
+
to: str | None = None,
|
|
303
|
+
include_private: bool = False,
|
|
304
|
+
) -> Iterator[None]:
|
|
305
|
+
"""Run a block and publish its code and printed output as a code snippet.
|
|
306
|
+
|
|
307
|
+
::
|
|
308
|
+
|
|
309
|
+
with fw.snippet("fit-ols"):
|
|
310
|
+
model = smf.ols("score ~ income", data=df).fit()
|
|
311
|
+
print(model.summary())
|
|
312
|
+
|
|
313
|
+
The block runs as usual and its output still prints. When it ends, the block's
|
|
314
|
+
code and what it printed to stdout (as output) and stderr (as messages) are
|
|
315
|
+
published to Framework Cloud as a code snippet. A script shows no expression
|
|
316
|
+
values, so none are recorded. If the block raises, the snippet ends with the
|
|
317
|
+
traceback, is published, and the exception continues.
|
|
318
|
+
|
|
319
|
+
The code comes from the source file, so ``with fw.snippet()`` needs a script
|
|
320
|
+
or a module, not the interactive prompt. In Jupyter, use the ``%%fw_snippet``
|
|
321
|
+
cell magic instead (``%load_ext framework``).
|
|
322
|
+
|
|
323
|
+
The snippet names its source: the file's path from the git repository's top
|
|
324
|
+
(or the project root), the label, the block's lines, and the commit checked
|
|
325
|
+
out. The repository URL is sent only when ``repository`` is given or
|
|
326
|
+
``publish.repository`` is set in settings.yml. Framework Cloud builds a GitHub
|
|
327
|
+
permalink from a GitHub repository, the commit, the file, and the lines.
|
|
328
|
+
|
|
329
|
+
Args:
|
|
330
|
+
label: Names the snippet. The default slug and title come from it.
|
|
331
|
+
output: False publishes the code alone.
|
|
332
|
+
folded: True folds every output block, False unfolds them. Default: Framework
|
|
333
|
+
Cloud folds output longer than 12 lines.
|
|
334
|
+
links: Links shown with the snippet: ``{"notebook": "https://...", "model":
|
|
335
|
+
"model-slug"}``. A model link takes an artifact slug in the same project
|
|
336
|
+
or an https URL.
|
|
337
|
+
repository: Repository URL to name, such as ``"https://github.com/owner/repo"``.
|
|
338
|
+
Default: ``publish.repository`` in settings.yml.
|
|
339
|
+
publish: False records the block without publishing it.
|
|
340
|
+
slug: Artifact slug. Default: from ``label``.
|
|
341
|
+
title: Artifact title. Default: ``label``.
|
|
342
|
+
description: Artifact description.
|
|
343
|
+
visibility: "public", "unlisted", or "private".
|
|
344
|
+
password: Password required to view the snippet.
|
|
345
|
+
collection: Project collection ID or slug.
|
|
346
|
+
share_unlisted: Allow collection readers to see an unlisted snippet.
|
|
347
|
+
to: A configured destination name, or "cloud".
|
|
348
|
+
include_private: Publish without asking about private folders.
|
|
349
|
+
"""
|
|
350
|
+
caller = sys._getframe(2)
|
|
351
|
+
filename = caller.f_code.co_filename
|
|
352
|
+
start = caller.f_lineno
|
|
353
|
+
recorder = _Recorder()
|
|
354
|
+
error: BaseException | None = None
|
|
355
|
+
try:
|
|
356
|
+
with _teed(recorder):
|
|
357
|
+
try:
|
|
358
|
+
yield
|
|
359
|
+
except BaseException as raised:
|
|
360
|
+
error = raised
|
|
361
|
+
raise
|
|
362
|
+
finally:
|
|
363
|
+
code, lines = _block_code(filename, start)
|
|
364
|
+
if error is not None and not isinstance(error, (KeyboardInterrupt, GeneratorExit)):
|
|
365
|
+
text = "".join(traceback.format_exception(type(error), error, error.__traceback__))
|
|
366
|
+
recorder.block("error", text)
|
|
367
|
+
if code is None:
|
|
368
|
+
message = (
|
|
369
|
+
f'fw.snippet("{label}") could not read its code from {filename}. '
|
|
370
|
+
"Use it in a script or module, or %%fw_snippet in Jupyter."
|
|
371
|
+
)
|
|
372
|
+
if error is None:
|
|
373
|
+
raise FrameworkError(message)
|
|
374
|
+
inform(message)
|
|
375
|
+
elif publish:
|
|
376
|
+
path = None if _cell_file(filename) else Path(filename)
|
|
377
|
+
if path is None or not path.is_file():
|
|
378
|
+
path = _notebook_path()
|
|
379
|
+
if path is not None and path.suffix in _NOTEBOOK_SUFFIXES:
|
|
380
|
+
lines = None
|
|
381
|
+
document = _document(
|
|
382
|
+
[{"type": "code", "text": code}, *recorder.finished()],
|
|
383
|
+
_source(path, label, lines, repository),
|
|
384
|
+
output=output,
|
|
385
|
+
folded=folded,
|
|
386
|
+
links=links,
|
|
387
|
+
)
|
|
388
|
+
options = {
|
|
389
|
+
"slug": slug,
|
|
390
|
+
"title": title,
|
|
391
|
+
"description": description,
|
|
392
|
+
"visibility": visibility,
|
|
393
|
+
"password": password,
|
|
394
|
+
"collection": collection,
|
|
395
|
+
"share_unlisted": share_unlisted,
|
|
396
|
+
"to": to,
|
|
397
|
+
"include_private": include_private,
|
|
398
|
+
}
|
|
399
|
+
if document is not None:
|
|
400
|
+
if error is None:
|
|
401
|
+
_publish(document, label, options)
|
|
402
|
+
else:
|
|
403
|
+
try:
|
|
404
|
+
_publish(document, label, options)
|
|
405
|
+
except Exception as failed:
|
|
406
|
+
inform(f"The snippet was not published: {failed}")
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
def _cell_file(filename: str) -> bool:
|
|
410
|
+
"""Whether ``filename`` names a Jupyter cell rather than a file on disk."""
|
|
411
|
+
return filename.startswith("<") or "ipykernel_" in filename
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def _block_code(filename: str, start: int) -> tuple[str | None, tuple[int, int] | None]:
|
|
415
|
+
"""The body of the ``with`` statement on line ``start``, dedented, and its lines."""
|
|
416
|
+
text = "".join(linecache.getlines(filename))
|
|
417
|
+
if not text:
|
|
418
|
+
return None, None
|
|
419
|
+
try:
|
|
420
|
+
tree = ast.parse(text)
|
|
421
|
+
except SyntaxError:
|
|
422
|
+
return None, None
|
|
423
|
+
found: ast.With | ast.AsyncWith | None = None
|
|
424
|
+
for node in ast.walk(tree):
|
|
425
|
+
if (
|
|
426
|
+
isinstance(node, (ast.With, ast.AsyncWith))
|
|
427
|
+
and node.lineno <= start <= (node.body[0].lineno - 1 if node.body else node.lineno)
|
|
428
|
+
and (found is None or node.lineno > found.lineno)
|
|
429
|
+
):
|
|
430
|
+
found = node
|
|
431
|
+
if found is None or not found.body:
|
|
432
|
+
return None, None
|
|
433
|
+
first = found.body[0].lineno
|
|
434
|
+
decorated = getattr(found.body[0], "decorator_list", None)
|
|
435
|
+
if decorated:
|
|
436
|
+
first = min(first, *(item.lineno for item in decorated))
|
|
437
|
+
last = found.end_lineno or found.body[-1].lineno
|
|
438
|
+
lines = text.splitlines()[first - 1 : last]
|
|
439
|
+
return _dedent(lines), (first, last)
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def _dedent(lines: list[str]) -> str:
|
|
443
|
+
indents = [len(line) - len(line.lstrip()) for line in lines if line.strip()]
|
|
444
|
+
cut = min(indents, default=0)
|
|
445
|
+
return "\n".join(line[cut:] for line in lines).strip("\n")
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
# ---------------------------------------------------------------------------
|
|
449
|
+
# Jupyter: %%fw_snippet
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
def _notebook_path() -> Path | None:
|
|
453
|
+
"""The running notebook's file, when the front end says."""
|
|
454
|
+
candidates: list[str] = []
|
|
455
|
+
try:
|
|
456
|
+
from IPython import get_ipython
|
|
457
|
+
|
|
458
|
+
shell = get_ipython()
|
|
459
|
+
except ImportError:
|
|
460
|
+
shell = None
|
|
461
|
+
if shell is not None:
|
|
462
|
+
vscode = shell.user_ns.get("__vsc_ipynb_file__")
|
|
463
|
+
if isinstance(vscode, str):
|
|
464
|
+
candidates.append(vscode)
|
|
465
|
+
session = os.environ.get("JPY_SESSION_NAME")
|
|
466
|
+
if session:
|
|
467
|
+
candidates.append(session)
|
|
468
|
+
for candidate in candidates:
|
|
469
|
+
path = Path(candidate)
|
|
470
|
+
if path.is_file():
|
|
471
|
+
return path
|
|
472
|
+
return None
|
|
473
|
+
|
|
474
|
+
|
|
475
|
+
def _find_marker(label: str) -> list[Path]:
|
|
476
|
+
"""Notebooks in the working folder whose code holds ``%%fw_snippet <label>``."""
|
|
477
|
+
pattern = re.compile(r"%%fw_snippet\s+" + re.escape(label) + r"(?![\w-])")
|
|
478
|
+
found = []
|
|
479
|
+
for path in sorted(Path.cwd().iterdir()):
|
|
480
|
+
if path.suffix not in _NOTEBOOK_SUFFIXES or not path.is_file():
|
|
481
|
+
continue
|
|
482
|
+
try:
|
|
483
|
+
if pattern.search(path.read_text(encoding="utf-8")):
|
|
484
|
+
found.append(path)
|
|
485
|
+
except (OSError, UnicodeError):
|
|
486
|
+
continue
|
|
487
|
+
return found
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
def _cell_lines(path: Path, label: str) -> tuple[int, int] | None:
|
|
491
|
+
"""In a .qmd file, the lines of the code after ``%%fw_snippet <label>``."""
|
|
492
|
+
if path.suffix != ".qmd":
|
|
493
|
+
return None
|
|
494
|
+
text = path.read_text(encoding="utf-8").splitlines()
|
|
495
|
+
pattern = re.compile(r"^\s*%%fw_snippet\s+" + re.escape(label) + r"(?![\w-])")
|
|
496
|
+
starts = [index for index, line in enumerate(text) if pattern.match(line)]
|
|
497
|
+
if len(starts) != 1:
|
|
498
|
+
return None
|
|
499
|
+
first = starts[0] + 1
|
|
500
|
+
last = first
|
|
501
|
+
for index in range(first, len(text)):
|
|
502
|
+
if text[index].lstrip().startswith("```"):
|
|
503
|
+
break
|
|
504
|
+
last = index
|
|
505
|
+
while last > first and not text[last].strip():
|
|
506
|
+
last -= 1
|
|
507
|
+
while first < last and (not text[first].strip() or _OPTION_LINE.match(text[first])):
|
|
508
|
+
first += 1
|
|
509
|
+
return (first + 1, last + 1)
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
def _cell_code(cell: str) -> str:
|
|
513
|
+
"""The cell's code without Quarto option lines."""
|
|
514
|
+
lines = [line for line in cell.splitlines() if not _OPTION_LINE.match(line)]
|
|
515
|
+
return "\n".join(lines).strip("\n")
|
|
516
|
+
|
|
517
|
+
|
|
518
|
+
def _magic_parser() -> Any:
|
|
519
|
+
import argparse
|
|
520
|
+
|
|
521
|
+
parser = argparse.ArgumentParser(prog="%%fw_snippet", add_help=False, exit_on_error=False)
|
|
522
|
+
parser.add_argument("label")
|
|
523
|
+
fold = parser.add_mutually_exclusive_group()
|
|
524
|
+
fold.add_argument("--folded", dest="folded", action="store_const", const=True)
|
|
525
|
+
fold.add_argument("--unfolded", dest="folded", action="store_const", const=False)
|
|
526
|
+
parser.add_argument("--no-output", dest="output", action="store_false")
|
|
527
|
+
parser.add_argument("--link", action="append", default=[])
|
|
528
|
+
parser.add_argument("--repository")
|
|
529
|
+
parser.add_argument("--file")
|
|
530
|
+
parser.add_argument("--no-publish", dest="publish", action="store_false")
|
|
531
|
+
parser.add_argument("--slug")
|
|
532
|
+
parser.add_argument("--title")
|
|
533
|
+
parser.add_argument("--description")
|
|
534
|
+
parser.add_argument("--visibility")
|
|
535
|
+
parser.add_argument("--password")
|
|
536
|
+
parser.add_argument("--collection")
|
|
537
|
+
parser.add_argument("--share-unlisted", action="store_true")
|
|
538
|
+
parser.add_argument("--to")
|
|
539
|
+
parser.add_argument("--include-private", action="store_true")
|
|
540
|
+
return parser
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
def _magic_arguments(line: str) -> Any:
|
|
544
|
+
import argparse
|
|
545
|
+
import shlex
|
|
546
|
+
|
|
547
|
+
try:
|
|
548
|
+
arguments = _magic_parser().parse_args(shlex.split(line))
|
|
549
|
+
except (argparse.ArgumentError, SystemExit, ValueError) as error:
|
|
550
|
+
raise FrameworkError(
|
|
551
|
+
f"%%fw_snippet {line}: {error}. Usage: %%fw_snippet LABEL [--folded | --unfolded] "
|
|
552
|
+
"[--no-output] [--link notebook=URL] [--link model=SLUG] [--repository URL] "
|
|
553
|
+
"[--slug SLUG] [--title TITLE] [--visibility VISIBILITY] [--to DESTINATION] "
|
|
554
|
+
"[--no-publish]"
|
|
555
|
+
) from None
|
|
556
|
+
links: dict[str, str] = {}
|
|
557
|
+
for link in arguments.link:
|
|
558
|
+
rel, separator, value = link.partition("=")
|
|
559
|
+
if not separator:
|
|
560
|
+
raise FrameworkError("--link takes REL=VALUE, as in --link notebook=https://...")
|
|
561
|
+
links[rel] = value
|
|
562
|
+
arguments.links = links
|
|
563
|
+
return arguments
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
def run_cell_snippet(shell: Any, line: str, cell: str) -> dict[str, Any] | None:
|
|
567
|
+
"""Run a cell for ``%%fw_snippet``, show its outputs, and publish them.
|
|
568
|
+
|
|
569
|
+
Returns the snippet document, or None when the cell has no code.
|
|
570
|
+
"""
|
|
571
|
+
arguments = _magic_arguments(line)
|
|
572
|
+
code = _cell_code(cell)
|
|
573
|
+
recorder = _Recorder()
|
|
574
|
+
error: dict[str, Any] = {}
|
|
575
|
+
|
|
576
|
+
publisher = shell.display_pub
|
|
577
|
+
original_publish = publisher.publish
|
|
578
|
+
hook = shell.displayhook
|
|
579
|
+
original_write = hook.write_format_data
|
|
580
|
+
original_quiet = hook.quiet
|
|
581
|
+
original_start = hook.start_displayhook
|
|
582
|
+
original_finish = hook.finish_displayhook
|
|
583
|
+
original_showtraceback = shell._showtraceback
|
|
584
|
+
|
|
585
|
+
def plain(data: dict[str, Any]) -> str | None:
|
|
586
|
+
if any(str(key).startswith("image/") for key in data):
|
|
587
|
+
return None
|
|
588
|
+
text = data.get("text/plain")
|
|
589
|
+
return text if isinstance(text, str) else None
|
|
590
|
+
|
|
591
|
+
def publish_display(data: dict[str, Any], *args: Any, **kwargs: Any) -> Any:
|
|
592
|
+
text = plain(data or {})
|
|
593
|
+
if text is not None:
|
|
594
|
+
recorder.block("output", text)
|
|
595
|
+
muted, recorder.muted = recorder.muted, True
|
|
596
|
+
try:
|
|
597
|
+
return original_publish(data, *args, **kwargs)
|
|
598
|
+
finally:
|
|
599
|
+
recorder.muted = muted
|
|
600
|
+
|
|
601
|
+
def start_result() -> Any:
|
|
602
|
+
recorder.muted = True
|
|
603
|
+
return original_start()
|
|
604
|
+
|
|
605
|
+
def finish_result() -> Any:
|
|
606
|
+
try:
|
|
607
|
+
return original_finish()
|
|
608
|
+
finally:
|
|
609
|
+
recorder.muted = False
|
|
610
|
+
|
|
611
|
+
def write_result(format_dict: dict[str, Any], md_dict: Any = None) -> Any:
|
|
612
|
+
text = plain(format_dict or {})
|
|
613
|
+
if text is not None:
|
|
614
|
+
recorder.block("output", text)
|
|
615
|
+
return original_write(format_dict, md_dict)
|
|
616
|
+
|
|
617
|
+
def quiet() -> bool:
|
|
618
|
+
try:
|
|
619
|
+
return bool(hook.semicolon_at_end_of_expression(cell))
|
|
620
|
+
except Exception:
|
|
621
|
+
return False
|
|
622
|
+
|
|
623
|
+
def record_traceback(etype: Any, evalue: Any, stb: list[str]) -> None:
|
|
624
|
+
error["text"] = "\n".join(stb)
|
|
625
|
+
error["value"] = evalue
|
|
626
|
+
|
|
627
|
+
publisher.publish = publish_display
|
|
628
|
+
hook.write_format_data = write_result
|
|
629
|
+
hook.quiet = quiet
|
|
630
|
+
hook.start_displayhook = start_result
|
|
631
|
+
hook.finish_displayhook = finish_result
|
|
632
|
+
shell._showtraceback = record_traceback
|
|
633
|
+
try:
|
|
634
|
+
with _teed(recorder):
|
|
635
|
+
result = shell.run_cell(cell, store_history=False)
|
|
636
|
+
finally:
|
|
637
|
+
publisher.publish = original_publish
|
|
638
|
+
hook.write_format_data = original_write
|
|
639
|
+
hook.quiet = original_quiet
|
|
640
|
+
hook.start_displayhook = original_start
|
|
641
|
+
hook.finish_displayhook = original_finish
|
|
642
|
+
shell._showtraceback = original_showtraceback
|
|
643
|
+
|
|
644
|
+
raised = result.error_in_exec or result.error_before_exec
|
|
645
|
+
if "text" in error:
|
|
646
|
+
recorder.block("error", error["text"])
|
|
647
|
+
elif raised is not None:
|
|
648
|
+
recorder.block("error", "".join(traceback.format_exception_only(type(raised), raised)))
|
|
649
|
+
|
|
650
|
+
path = Path(arguments.file) if arguments.file else _notebook_path()
|
|
651
|
+
if path is None:
|
|
652
|
+
matches = _find_marker(arguments.label)
|
|
653
|
+
path = matches[0] if len(matches) == 1 else None
|
|
654
|
+
lines = _cell_lines(path, arguments.label) if path is not None else None
|
|
655
|
+
document = _document(
|
|
656
|
+
[{"type": "code", "text": code}, *recorder.finished()],
|
|
657
|
+
_source(path, arguments.label, lines, arguments.repository),
|
|
658
|
+
output=arguments.output,
|
|
659
|
+
folded=arguments.folded,
|
|
660
|
+
links=arguments.links,
|
|
661
|
+
)
|
|
662
|
+
if document is not None and arguments.publish:
|
|
663
|
+
options = {name: getattr(arguments, name) for name in _PUBLISH_OPTIONS}
|
|
664
|
+
if raised is None:
|
|
665
|
+
_publish_in_cell(document, arguments.label, options)
|
|
666
|
+
else:
|
|
667
|
+
try:
|
|
668
|
+
_publish_in_cell(document, arguments.label, options)
|
|
669
|
+
except Exception as failed:
|
|
670
|
+
inform(f"The snippet was not published: {failed}")
|
|
671
|
+
if raised is not None:
|
|
672
|
+
# The traceback was recorded instead of shown, so the cell fails once, here.
|
|
673
|
+
raise raised
|
|
674
|
+
return document
|
|
675
|
+
|
|
676
|
+
|
|
677
|
+
# Marks the publishing status a cell shows, so fw snippet publish leaves it out
|
|
678
|
+
STATUS_METADATA = {"framework": {"role": "snippet-status"}}
|
|
679
|
+
|
|
680
|
+
|
|
681
|
+
def _publish_in_cell(
|
|
682
|
+
document: dict[str, Any], label: str, options: dict[str, Any]
|
|
683
|
+
) -> str | list[str]:
|
|
684
|
+
"""Publish, and show Framework's messages as a display the snippet readers skip."""
|
|
685
|
+
import logging
|
|
686
|
+
|
|
687
|
+
from IPython.display import publish_display_data
|
|
688
|
+
|
|
689
|
+
from framework._log import logger
|
|
690
|
+
|
|
691
|
+
messages: list[str] = []
|
|
692
|
+
|
|
693
|
+
class Collect(logging.Handler):
|
|
694
|
+
def emit(self, record: logging.LogRecord) -> None:
|
|
695
|
+
messages.append(record.getMessage())
|
|
696
|
+
|
|
697
|
+
handlers, level = logger.handlers[:], logger.level
|
|
698
|
+
logger.handlers = [Collect()]
|
|
699
|
+
logger.setLevel(logging.INFO)
|
|
700
|
+
try:
|
|
701
|
+
return _publish(document, label, options)
|
|
702
|
+
finally:
|
|
703
|
+
logger.handlers = handlers
|
|
704
|
+
logger.setLevel(level)
|
|
705
|
+
if messages:
|
|
706
|
+
publish_display_data( # type: ignore[no-untyped-call]
|
|
707
|
+
{"text/plain": "\n".join(messages)}, metadata=STATUS_METADATA
|
|
708
|
+
)
|
|
709
|
+
|
|
710
|
+
|
|
711
|
+
def load_ipython_extension(shell: Any) -> None:
|
|
712
|
+
"""``%load_ext framework``: add the ``%%fw_snippet`` magic and show Framework's messages."""
|
|
713
|
+
from framework._log import show_messages
|
|
714
|
+
|
|
715
|
+
show_messages()
|
|
716
|
+
|
|
717
|
+
def fw_snippet(line: str, cell: str) -> None:
|
|
718
|
+
"""Run the cell and publish its code and printed output as a code snippet.
|
|
719
|
+
|
|
720
|
+
%%fw_snippet LABEL [--folded | --unfolded] [--no-output]
|
|
721
|
+
[--link notebook=URL] [--link model=SLUG] [--repository URL] [--file PATH]
|
|
722
|
+
[--slug SLUG] [--title TITLE] [--description TEXT] [--visibility VISIBILITY]
|
|
723
|
+
[--password PASSWORD] [--collection COLLECTION] [--share-unlisted]
|
|
724
|
+
[--to DESTINATION] [--include-private] [--no-publish]
|
|
725
|
+
"""
|
|
726
|
+
run_cell_snippet(shell, line, cell)
|
|
727
|
+
|
|
728
|
+
shell.register_magic_function(fw_snippet, magic_kind="cell", magic_name="fw_snippet")
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: framework-py
|
|
3
|
-
Version: 0.1.0.
|
|
3
|
+
Version: 0.1.0.dev28
|
|
4
4
|
Summary: Structured data science projects: a data catalog, integrity tracking, caching, outputs, and AI guardrails
|
|
5
5
|
Project-URL: Homepage, https://framework.pub
|
|
6
6
|
Project-URL: Source, https://github.com/frameworktools/framework-py
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
framework/__init__.py,sha256=
|
|
1
|
+
framework/__init__.py,sha256=IZuuoHC2nHIPRRC9k6AkpQZ6HmnySBvh0s9us-wlPbE,6834
|
|
2
2
|
framework/__main__.py,sha256=S0ChmRQnNa9W5_MPXhHnH61Kjr_pNthkeS41eJU1w0U,9093
|
|
3
3
|
framework/_agents_md.py,sha256=PSWPJiCUTRirk81sz-Y33qHnqY16HOhHY8pH77ct1ik,11857
|
|
4
4
|
framework/_artifacts.py,sha256=XjYOBQOQJuvztUyqwWsc3xc1qJ5o4O5XpCW49WN548k,55807
|
|
@@ -26,12 +26,12 @@ framework/ai.py,sha256=hqBNXngw7VyQBcSt7kB20Zpa6N_5WS1hNhDi2zWJw0U,9218
|
|
|
26
26
|
framework/bootstrap.py,sha256=mYlD1BG9KaG9PAIHlw3l6677fJYwQdtih95W9Rqmnho,9687
|
|
27
27
|
framework/cache.py,sha256=z8Zaem5Q7dsAghuX9CgK_j0vhX5dDlRw7fl9f-CF3eA,17440
|
|
28
28
|
framework/catalog.py,sha256=sl4V1judSCVoa2LYLXeQdDhWdsONTC_5s2_fpyPylto,13202
|
|
29
|
-
framework/cloud.py,sha256=
|
|
29
|
+
framework/cloud.py,sha256=y4w5euumzkNXOfC9q8v7FVW5yHEO7QORXC3D2BlirMI,67843
|
|
30
30
|
framework/config_home.py,sha256=iSb0VuENgNOSnsZ_v53xFqjRiGgrB7OzZbHbyDOqKzs,8265
|
|
31
31
|
framework/connections.py,sha256=99JqrNAi2zBpNYyxLV6SYj-NAJjh56pYh9jaMSYOHCU,32742
|
|
32
32
|
framework/console.py,sha256=zQ0LuFDcxe1W1kuc5E1bJ03h8d3cvwFoqDT8dAJQyN0,1267
|
|
33
33
|
framework/create.py,sha256=3ykMUhMslVsjiE__9Scx5v_6CU2Ye36wWqlp17j_vGw,53030
|
|
34
|
-
framework/data.py,sha256=
|
|
34
|
+
framework/data.py,sha256=HpiRl5qTk1o9G-XXNDgsWhNOgXlyYegv8-8KDz-HyPA,40853
|
|
35
35
|
framework/data_registry.py,sha256=2hbXsoY9IF5Zj6lkiTE2RO2c5j2Z2vXKm2iTN7FdsQk,22499
|
|
36
36
|
framework/db.py,sha256=UYEUCvk4vGytBkmdNTUxRpYPtoFSDHwTpgpfWFw31aE,8522
|
|
37
37
|
framework/display.py,sha256=0FuKUt67uf5h1WjD4bwzoruc2XAN4y_j3mz_f5FskhE,5334
|
|
@@ -54,6 +54,7 @@ framework/security.py,sha256=-x66bSIIE_xOzkTf5Ojt7xIECKuAIpKVJZ5TtIGrwys,51147
|
|
|
54
54
|
framework/settings.py,sha256=Sb2YyE53aYLWPj423P-pQVCvRpLsiHydgZh2K0luTQQ,23893
|
|
55
55
|
framework/skill_pull.py,sha256=vr_yxe446JhHbtI0hps19AnLN6S7BacSt2oxn7BIlos,21859
|
|
56
56
|
framework/skill_sync.py,sha256=0gqaeVkiNaqGJxmrVrKrYaX8wI3DNFZUsJEgG63xcKc,23060
|
|
57
|
+
framework/snippets.py,sha256=J8j36YqHgG-HhWlhJ9RmJK-qwVGOOSOOd1pNakJyuVE,26481
|
|
57
58
|
framework/storage.py,sha256=Haz8CBXrz2CCqGHOVgHKC3mUUwobnod2SanU759lK7c,56115
|
|
58
59
|
framework/sync_all.py,sha256=tmSnapf-JfaL1_8rMpnr3wqHvSGJamn7c8ja4GQFKHA,34610
|
|
59
60
|
framework/upgrade.py,sha256=WtghpLR65YFH_JQFmo1JlH9lHdwsXQd-ZzzOTmyOL28,65593
|
|
@@ -140,7 +141,7 @@ framework/templates/stubs/notebook-default.qmd,sha256=8Js1S2BcHYeQWmJS-kt31gBvIj
|
|
|
140
141
|
framework/templates/stubs/notebook-minimal.qmd,sha256=2U2k8judffhrh_6m6r6OAbQBCYEtViAjeKRBqBYCJTM,178
|
|
141
142
|
framework/templates/stubs/notebook-revealjs.qmd,sha256=_ztT_gHoudpB5DIjlbvtB5zQ8163DeV8xbWcT-HgJrI,409
|
|
142
143
|
framework/templates/stubs/script-default.py,sha256=vbWqgW0_rHot17mQC3tYb92g_9L-cbTOeL3y99BRrSk,216
|
|
143
|
-
framework_py-0.1.0.
|
|
144
|
-
framework_py-0.1.0.
|
|
145
|
-
framework_py-0.1.0.
|
|
146
|
-
framework_py-0.1.0.
|
|
144
|
+
framework_py-0.1.0.dev28.dist-info/METADATA,sha256=cTispVoaGZgGGcA8GR630W4VE96ZRIRwoxs1HjTeIdE,4682
|
|
145
|
+
framework_py-0.1.0.dev28.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
146
|
+
framework_py-0.1.0.dev28.dist-info/licenses/LICENSE,sha256=giOOgSwALkZimC7uhLZQK8Untez641bLW1-2-Cjvmq0,1072
|
|
147
|
+
framework_py-0.1.0.dev28.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|