afterpython 0.4.0__py3-none-any.whl → 0.4.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- afterpython/builders/__init__.py +4 -0
- afterpython/builders/build_pdoc.py +7 -0
- afterpython/builders/comments_json.py +107 -0
- afterpython/builders/jupyter_notebook.py +23 -7
- afterpython/builders/marimo_content.py +467 -0
- afterpython/builders/marimo_notebook.py +7 -6
- afterpython/cli/commands/build.py +74 -55
- afterpython/cli/commands/clean.py +6 -0
- afterpython/cli/commands/dev.py +19 -1
- afterpython/cli/commands/init.py +113 -51
- afterpython/cli/main.py +3 -1
- afterpython/const.py +1 -1
- afterpython/templates/afterpython-template.toml +6 -0
- afterpython/templates/ci-workflow-template.yml +10 -10
- afterpython/templates/deploy-workflow-template.yml +2 -2
- afterpython/templates/pdoc/custom.css +37 -0
- afterpython/templates/pdoc/frame.html.jinja2 +15 -1
- afterpython/templates/pdoc/syntax-highlighting.css +155 -0
- afterpython/templates/pdoc/theme.css +44 -0
- afterpython/templates/release-workflow-template.yml +1 -1
- afterpython/templates/ruff-template.toml +1 -1
- afterpython/tools/myst.py +36 -19
- {afterpython-0.4.0.dist-info → afterpython-0.4.2.dist-info}/METADATA +18 -16
- {afterpython-0.4.0.dist-info → afterpython-0.4.2.dist-info}/RECORD +26 -21
- {afterpython-0.4.0.dist-info → afterpython-0.4.2.dist-info}/WHEEL +1 -1
- {afterpython-0.4.0.dist-info → afterpython-0.4.2.dist-info}/entry_points.txt +0 -0
afterpython/builders/__init__.py
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
from afterpython.builders.build_pdoc import build_pdoc
|
|
2
|
+
from afterpython.builders.comments_json import build_comments_json
|
|
2
3
|
from afterpython.builders.content_json import build_content_json
|
|
3
4
|
from afterpython.builders.faq_json import build_faq_json
|
|
4
5
|
from afterpython.builders.index_md import (
|
|
@@ -7,16 +8,19 @@ from afterpython.builders.index_md import (
|
|
|
7
8
|
)
|
|
8
9
|
from afterpython.builders.jupyter_notebook import build_jupyter_notebooks
|
|
9
10
|
from afterpython.builders.llms_txt import build_llms_txt
|
|
11
|
+
from afterpython.builders.marimo_content import build_marimo_content
|
|
10
12
|
from afterpython.builders.marimo_notebook import build_marimo_readme
|
|
11
13
|
from afterpython.builders.markdown import build_markdown
|
|
12
14
|
from afterpython.builders.metadata import build_metadata
|
|
13
15
|
from afterpython.builders.url_md import build_url_md
|
|
14
16
|
|
|
15
17
|
__all__ = (
|
|
18
|
+
"build_comments_json",
|
|
16
19
|
"build_content_json",
|
|
17
20
|
"build_faq_json",
|
|
18
21
|
"build_jupyter_notebooks",
|
|
19
22
|
"build_llms_txt",
|
|
23
|
+
"build_marimo_content",
|
|
20
24
|
"build_marimo_readme",
|
|
21
25
|
"build_markdown",
|
|
22
26
|
"build_metadata",
|
|
@@ -19,9 +19,15 @@ def build_pdoc(base_path: str) -> None:
|
|
|
19
19
|
project logo appears in the top-left of every API page and links
|
|
20
20
|
back to the project website root (matches MyST's UX).
|
|
21
21
|
"""
|
|
22
|
+
from afterpython._io.toml import _from_tomlkit
|
|
23
|
+
from afterpython.tools._afterpython import read_afterpython
|
|
24
|
+
|
|
22
25
|
package_dir = find_package_directory()
|
|
23
26
|
output_dir = ap.paths.build_path / "api_reference"
|
|
24
27
|
template_dir = ap.paths.templates_path / "pdoc"
|
|
28
|
+
# e.g. "google"; unset leaves it to pdoc (a module's __docformat__, else restructuredtext)
|
|
29
|
+
website = _from_tomlkit(read_afterpython().get("website", {}))
|
|
30
|
+
docformat = website.get("api_reference_docformat")
|
|
25
31
|
|
|
26
32
|
click.echo("Building api_reference/...")
|
|
27
33
|
|
|
@@ -45,6 +51,7 @@ def build_pdoc(base_path: str) -> None:
|
|
|
45
51
|
f"{base_path}/",
|
|
46
52
|
"--template-directory",
|
|
47
53
|
str(template_dir),
|
|
54
|
+
*(["--docformat", str(docformat)] if docformat else []),
|
|
48
55
|
str(package_dir),
|
|
49
56
|
],
|
|
50
57
|
env=env,
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import re
|
|
5
|
+
|
|
6
|
+
import click
|
|
7
|
+
|
|
8
|
+
import afterpython as ap
|
|
9
|
+
|
|
10
|
+
GISCUS_API = "https://giscus.app/api/discussions/categories"
|
|
11
|
+
DEFAULT_CATEGORY = "Announcements"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _read_comments_config() -> dict:
|
|
15
|
+
from afterpython._io.toml import _from_tomlkit
|
|
16
|
+
from afterpython.tools._afterpython import read_afterpython
|
|
17
|
+
|
|
18
|
+
website = _from_tomlkit(read_afterpython().get("website", {}))
|
|
19
|
+
return dict(website.get("comments", {}))
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _get_github_repo() -> str | None:
|
|
23
|
+
"""Return "owner/repo" from pyproject.toml's repository URL, or the git remote."""
|
|
24
|
+
from afterpython.tools._git import get_github_url
|
|
25
|
+
from afterpython.tools.pyproject import read_metadata
|
|
26
|
+
|
|
27
|
+
url = str(read_metadata().urls.get("repository", "")) or get_github_url() or ""
|
|
28
|
+
match = re.search(r"github\.com[/:]([^/]+)/([^/]+?)(?:\.git)?/?$", url)
|
|
29
|
+
return f"{match[1]}/{match[2]}" if match else None
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _fetch_giscus_categories(repo: str) -> dict:
|
|
33
|
+
"""Fetch the repo and category IDs giscus needs. Raises click.ClickException on failure."""
|
|
34
|
+
import httpx2
|
|
35
|
+
|
|
36
|
+
try:
|
|
37
|
+
response = httpx2.get(GISCUS_API, params={"repo": repo}, timeout=10.0)
|
|
38
|
+
except httpx2.HTTPError as e:
|
|
39
|
+
raise click.ClickException(f"could not reach giscus.app ({e})") from e
|
|
40
|
+
try:
|
|
41
|
+
data = response.json()
|
|
42
|
+
except ValueError:
|
|
43
|
+
data = {}
|
|
44
|
+
if response.status_code != 200 or "repositoryId" not in data:
|
|
45
|
+
error = data.get("error") or f"HTTP {response.status_code}"
|
|
46
|
+
raise click.ClickException(error)
|
|
47
|
+
return data
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def build_comments_json():
|
|
51
|
+
"""Build comments.json for the giscus comment section on content pages.
|
|
52
|
+
|
|
53
|
+
Reads [website.comments] in afterpython.toml. The website template only
|
|
54
|
+
shows comments when comments.json exists, so a disabled or failed setup
|
|
55
|
+
just leaves it out. Never fails the build.
|
|
56
|
+
"""
|
|
57
|
+
stale = ap.paths.website_path / "static" / "comments.json"
|
|
58
|
+
stale.unlink(missing_ok=True)
|
|
59
|
+
|
|
60
|
+
config = _read_comments_config()
|
|
61
|
+
if not config.get("enabled", False):
|
|
62
|
+
click.echo("Comments are disabled, skip building comments.json")
|
|
63
|
+
return
|
|
64
|
+
|
|
65
|
+
def warn(reason: str):
|
|
66
|
+
click.secho(f"Warning: skip comments, {reason}", fg="yellow")
|
|
67
|
+
|
|
68
|
+
repo = _get_github_repo()
|
|
69
|
+
if not repo:
|
|
70
|
+
warn(
|
|
71
|
+
"no GitHub repository found "
|
|
72
|
+
"(set [project.urls] repository in pyproject.toml)"
|
|
73
|
+
)
|
|
74
|
+
return
|
|
75
|
+
|
|
76
|
+
category = str(config.get("category") or DEFAULT_CATEGORY)
|
|
77
|
+
try:
|
|
78
|
+
data = _fetch_giscus_categories(repo)
|
|
79
|
+
except click.ClickException as e:
|
|
80
|
+
warn(
|
|
81
|
+
f"giscus is not ready for {repo}: {e.message}\n"
|
|
82
|
+
f" 1. Enable Discussions: https://github.com/{repo}/settings\n"
|
|
83
|
+
" 2. Install the giscus app: https://github.com/apps/giscus\n"
|
|
84
|
+
" Or set [website.comments] enabled = false in afterpython.toml"
|
|
85
|
+
)
|
|
86
|
+
return
|
|
87
|
+
|
|
88
|
+
category_id = next(
|
|
89
|
+
(c["id"] for c in data.get("categories", []) if c["name"] == category), None
|
|
90
|
+
)
|
|
91
|
+
if not category_id:
|
|
92
|
+
names = ", ".join(c["name"] for c in data.get("categories", []))
|
|
93
|
+
warn(
|
|
94
|
+
f'Discussions category "{category}" not found in {repo} '
|
|
95
|
+
f"(available: {names or 'none'})"
|
|
96
|
+
)
|
|
97
|
+
return
|
|
98
|
+
|
|
99
|
+
click.echo(f"Building comments.json ({repo}, {category})...")
|
|
100
|
+
comments = {
|
|
101
|
+
"repo": repo,
|
|
102
|
+
"repoId": data["repositoryId"],
|
|
103
|
+
"category": category,
|
|
104
|
+
"categoryId": category_id,
|
|
105
|
+
}
|
|
106
|
+
with open(ap.paths.build_path / "comments.json", "w") as f:
|
|
107
|
+
json.dump(comments, f, indent=2)
|
|
@@ -17,7 +17,7 @@ def _read_notebook(notebook_path: Path, content_path: Path) -> dict | None:
|
|
|
17
17
|
Notebook dict if valid, None if invalid or error occurred
|
|
18
18
|
"""
|
|
19
19
|
try:
|
|
20
|
-
with open(notebook_path) as f:
|
|
20
|
+
with open(notebook_path, encoding="utf-8") as f:
|
|
21
21
|
notebook = json.load(f)
|
|
22
22
|
except json.JSONDecodeError:
|
|
23
23
|
click.echo(f"✗ Skipping {content_path} - invalid or empty JSON")
|
|
@@ -36,8 +36,11 @@ def _read_notebook(notebook_path: Path, content_path: Path) -> dict | None:
|
|
|
36
36
|
|
|
37
37
|
def _write_notebook(notebook: dict, notebook_path: Path):
|
|
38
38
|
"""Write a notebook dict to file."""
|
|
39
|
-
|
|
40
|
-
|
|
39
|
+
# match Jupyter's own formatting (unescaped unicode, trailing newline) so
|
|
40
|
+
# rewriting a committed notebook doesn't produce a noisy diff
|
|
41
|
+
with open(notebook_path, "w", encoding="utf-8") as f:
|
|
42
|
+
json.dump(notebook, f, indent=1, ensure_ascii=False)
|
|
43
|
+
f.write("\n")
|
|
41
44
|
|
|
42
45
|
|
|
43
46
|
def _has_frontmatter(notebook: dict) -> bool:
|
|
@@ -243,8 +246,10 @@ def build_jupyter_notebooks():
|
|
|
243
246
|
|
|
244
247
|
# Iterate over all .ipynb files once
|
|
245
248
|
for notebook_path in path.rglob("*.ipynb"):
|
|
246
|
-
# Skip
|
|
247
|
-
|
|
249
|
+
# Skip MyST output and hidden tool dirs (.ipynb_checkpoints/,
|
|
250
|
+
# .virtual_documents/), which aren't content
|
|
251
|
+
rel_parts = notebook_path.relative_to(path).parts
|
|
252
|
+
if "_build" in rel_parts or any(p.startswith(".") for p in rel_parts):
|
|
248
253
|
continue
|
|
249
254
|
|
|
250
255
|
# Get path relative to afterpython/
|
|
@@ -264,9 +269,20 @@ def build_jupyter_notebooks():
|
|
|
264
269
|
# Track if notebook was modified
|
|
265
270
|
modified = False
|
|
266
271
|
|
|
267
|
-
# Add molab badge if enabled
|
|
272
|
+
# Add molab badge if enabled. A notebook generated from a marimo .py
|
|
273
|
+
# isn't committed, so the badge must open the .py instead.
|
|
268
274
|
if add_molab_badge and github_url:
|
|
269
|
-
|
|
275
|
+
generated_from = (
|
|
276
|
+
notebook.get("metadata", {})
|
|
277
|
+
.get("afterpython", {})
|
|
278
|
+
.get("generated_from")
|
|
279
|
+
)
|
|
280
|
+
molab_path = (
|
|
281
|
+
content_path.with_name(generated_from)
|
|
282
|
+
if generated_from
|
|
283
|
+
else content_path
|
|
284
|
+
)
|
|
285
|
+
_add_molab_badge_to_notebook(notebook, github_url, molab_path)
|
|
270
286
|
modified = True
|
|
271
287
|
|
|
272
288
|
# Add thumbnail to frontmatter
|
|
@@ -0,0 +1,467 @@
|
|
|
1
|
+
"""Convert marimo notebooks in content folders into Jupyter notebooks for MyST.
|
|
2
|
+
|
|
3
|
+
MyST can't read marimo `.py` notebooks, so before MyST runs we export each one
|
|
4
|
+
to an `.ipynb` next to it. `marimo export ipynb --sandbox --include-outputs`
|
|
5
|
+
runs the notebook in its own uv environment, built from the notebook's PEP 723
|
|
6
|
+
header, so the outputs are reproducible no matter where `ap build` runs.
|
|
7
|
+
|
|
8
|
+
Rules
|
|
9
|
+
-----
|
|
10
|
+
- A content folder is either all-marimo or all-Jupyter (`.md` doesn't count).
|
|
11
|
+
Mixing raises. Generated `.ipynb` files (stamped with
|
|
12
|
+
`metadata.afterpython.generated_from`) don't count as Jupyter.
|
|
13
|
+
- In a marimo folder, `*.ipynb` is added to that folder's `.gitignore`: the
|
|
14
|
+
`.py` is the only source, the `.ipynb` is a build artifact.
|
|
15
|
+
- Generated notebooks get `execute: {skip: true}` frontmatter so
|
|
16
|
+
`myst build --execute` never re-runs them on the shared kernel.
|
|
17
|
+
|
|
18
|
+
Flow
|
|
19
|
+
----
|
|
20
|
+
build_marimo_content(use_cache) # called by prebuild()
|
|
21
|
+
└─ for each content folder:
|
|
22
|
+
├─ _scan_folder() → marimo .py / hand-written .ipynb, raises on mixing
|
|
23
|
+
├─ _remove_orphans() → generated .ipynb whose .py is gone
|
|
24
|
+
├─ _sync_gitignore() → add/remove the `*.ipynb` line
|
|
25
|
+
└─ convert_marimo_notebook() → export + _postprocess() per notebook
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import hashlib
|
|
31
|
+
import json
|
|
32
|
+
import os
|
|
33
|
+
import re
|
|
34
|
+
import shutil
|
|
35
|
+
import subprocess
|
|
36
|
+
import tempfile
|
|
37
|
+
import threading
|
|
38
|
+
import time
|
|
39
|
+
from dataclasses import dataclass, field
|
|
40
|
+
from pathlib import Path
|
|
41
|
+
|
|
42
|
+
import click
|
|
43
|
+
from ruamel.yaml import YAML
|
|
44
|
+
from ruamel.yaml.error import YAMLError
|
|
45
|
+
|
|
46
|
+
import afterpython as ap
|
|
47
|
+
from afterpython.builders.jupyter_notebook import _add_fields_to_frontmatter
|
|
48
|
+
from afterpython.builders.marimo_notebook import is_marimo_notebook
|
|
49
|
+
from afterpython.const import CONTENT_TYPES
|
|
50
|
+
|
|
51
|
+
GITIGNORE_MARKER = (
|
|
52
|
+
"# afterpython: .ipynb files here are generated from marimo notebooks (.py)"
|
|
53
|
+
)
|
|
54
|
+
# *.ipynb: generated notebooks; __marimo__/: marimo's session cache, which
|
|
55
|
+
# `marimo export` writes next to each notebook
|
|
56
|
+
GITIGNORE_PATTERNS = ("*.ipynb", "__marimo__/")
|
|
57
|
+
# re-export taking longer than this in `ap dev` prints a "too slow" hint
|
|
58
|
+
SLOW_EXPORT_SECONDS = 10
|
|
59
|
+
# nbformat cell ids: 1-64 chars of [a-zA-Z0-9-_]
|
|
60
|
+
_MAX_CELL_ID_LENGTH = 64
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@dataclass
|
|
64
|
+
class _FolderScan:
|
|
65
|
+
marimo: list[Path] = field(default_factory=list)
|
|
66
|
+
jupyter: list[Path] = field(default_factory=list)
|
|
67
|
+
generated: list[Path] = field(default_factory=list)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _iter_files(content_path: Path):
|
|
71
|
+
"""Content files under a folder, skipping MyST output and hidden tool dirs
|
|
72
|
+
(`.ipynb_checkpoints/`, `.virtual_documents/`), same as url_md."""
|
|
73
|
+
for path in sorted(content_path.rglob("*")):
|
|
74
|
+
rel_parts = path.relative_to(content_path).parts
|
|
75
|
+
if "_build" in rel_parts or any(p.startswith(".") for p in rel_parts):
|
|
76
|
+
continue
|
|
77
|
+
if path.is_file():
|
|
78
|
+
yield path
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _generated_from(notebook_path: Path) -> str | None:
|
|
82
|
+
"""The `.py` filename a generated notebook was exported from, else None."""
|
|
83
|
+
try:
|
|
84
|
+
notebook = json.loads(notebook_path.read_text(encoding="utf-8"))
|
|
85
|
+
except (OSError, ValueError):
|
|
86
|
+
return None
|
|
87
|
+
if not isinstance(notebook, dict):
|
|
88
|
+
return None
|
|
89
|
+
stamp = notebook.get("metadata", {}).get("afterpython", {})
|
|
90
|
+
return stamp.get("generated_from") if isinstance(stamp, dict) else None
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _scan_folder(content_path: Path) -> _FolderScan:
|
|
94
|
+
scan = _FolderScan()
|
|
95
|
+
for path in _iter_files(content_path):
|
|
96
|
+
if path.suffix == ".py" and is_marimo_notebook(path):
|
|
97
|
+
scan.marimo.append(path)
|
|
98
|
+
elif path.suffix == ".ipynb":
|
|
99
|
+
if _generated_from(path):
|
|
100
|
+
scan.generated.append(path)
|
|
101
|
+
else:
|
|
102
|
+
scan.jupyter.append(path)
|
|
103
|
+
return scan
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _check_not_mixed(content_path: Path, scan: _FolderScan):
|
|
107
|
+
if not (scan.marimo and scan.jupyter):
|
|
108
|
+
return
|
|
109
|
+
|
|
110
|
+
def _names(paths: list[Path]) -> str:
|
|
111
|
+
return ", ".join(p.relative_to(content_path).as_posix() for p in paths)
|
|
112
|
+
|
|
113
|
+
example = scan.jupyter[0].relative_to(content_path)
|
|
114
|
+
raise click.ClickException(
|
|
115
|
+
f"{content_path.name}/ mixes marimo and Jupyter notebooks:\n"
|
|
116
|
+
f" marimo: {_names(scan.marimo)}\n"
|
|
117
|
+
f" Jupyter: {_names(scan.jupyter)}\n"
|
|
118
|
+
" A content folder must use one or the other. Convert the .ipynb "
|
|
119
|
+
f"(marimo convert {example} -o {example.with_suffix('.py')}) "
|
|
120
|
+
"or move it to another content folder."
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _remove_orphans(scan: _FolderScan):
|
|
125
|
+
"""Delete generated notebooks whose marimo source no longer exists."""
|
|
126
|
+
marimo = set(scan.marimo)
|
|
127
|
+
for notebook_path in scan.generated:
|
|
128
|
+
source = notebook_path.parent / (_generated_from(notebook_path) or "")
|
|
129
|
+
if source not in marimo:
|
|
130
|
+
notebook_path.unlink()
|
|
131
|
+
click.echo(f"Removed {notebook_path.name} (its marimo source is gone)")
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _sync_gitignore(content_path: Path, has_marimo: bool):
|
|
135
|
+
"""Keep `*.ipynb` (and `__marimo__/`) in the folder's .gitignore exactly
|
|
136
|
+
while it holds marimo notebooks. Only the block afterpython wrote, starting
|
|
137
|
+
at GITIGNORE_MARKER, is ever added or removed."""
|
|
138
|
+
gitignore = content_path / ".gitignore"
|
|
139
|
+
text = gitignore.read_text(encoding="utf-8") if gitignore.exists() else ""
|
|
140
|
+
lines = text.splitlines()
|
|
141
|
+
has_ours = GITIGNORE_MARKER in lines
|
|
142
|
+
|
|
143
|
+
if has_marimo and not has_ours:
|
|
144
|
+
existing = {line.strip() for line in lines}
|
|
145
|
+
missing = [p for p in GITIGNORE_PATTERNS if p not in existing]
|
|
146
|
+
if not missing:
|
|
147
|
+
return # the user already ignores these themselves
|
|
148
|
+
block = "\n".join([GITIGNORE_MARKER, *missing]) + "\n"
|
|
149
|
+
sep = "" if not text or text.endswith("\n") else "\n"
|
|
150
|
+
gitignore.write_text(text + sep + ("\n" if text else "") + block, "utf-8")
|
|
151
|
+
click.echo(
|
|
152
|
+
click.style(
|
|
153
|
+
f"⚠ {content_path.name}/ contains marimo notebooks: "
|
|
154
|
+
f"{', '.join(missing)} added to {content_path.name}/.gitignore\n"
|
|
155
|
+
" (.ipynb files here are generated from .py and not committed)",
|
|
156
|
+
fg="yellow",
|
|
157
|
+
)
|
|
158
|
+
)
|
|
159
|
+
elif not has_marimo and has_ours:
|
|
160
|
+
start = lines.index(GITIGNORE_MARKER)
|
|
161
|
+
end = start + 1
|
|
162
|
+
while end < len(lines) and lines[end].strip() in GITIGNORE_PATTERNS:
|
|
163
|
+
end += 1
|
|
164
|
+
kept = lines[:start] + lines[end:]
|
|
165
|
+
while kept and not kept[-1].strip():
|
|
166
|
+
kept.pop()
|
|
167
|
+
gitignore.write_text("\n".join(kept) + "\n" if kept else "", "utf-8")
|
|
168
|
+
click.echo(
|
|
169
|
+
f"{content_path.name}/ has no marimo notebooks anymore: removed "
|
|
170
|
+
f"afterpython's ignore lines from {content_path.name}/.gitignore"
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _marimo_version() -> str:
|
|
175
|
+
if shutil.which("marimo") is None:
|
|
176
|
+
raise click.ClickException(
|
|
177
|
+
"marimo is required to build marimo notebooks in content folders, "
|
|
178
|
+
'install it with: uv add --dev "afterpython[marimo]"'
|
|
179
|
+
)
|
|
180
|
+
result = subprocess.run(
|
|
181
|
+
["marimo", "--version"], capture_output=True, text=True, check=False
|
|
182
|
+
)
|
|
183
|
+
return result.stdout.strip()
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _source_hash(source: Path) -> str:
|
|
187
|
+
return hashlib.sha256(source.read_bytes()).hexdigest()
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _is_cached(source: Path, output: Path, marimo_version: str) -> bool:
|
|
191
|
+
try:
|
|
192
|
+
notebook = json.loads(output.read_text(encoding="utf-8"))
|
|
193
|
+
stamp = notebook["metadata"]["afterpython"]
|
|
194
|
+
except (OSError, ValueError, KeyError, TypeError):
|
|
195
|
+
return False
|
|
196
|
+
return (
|
|
197
|
+
stamp.get("source_hash") == _source_hash(source)
|
|
198
|
+
and stamp.get("marimo_version") == marimo_version
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _as_lines(source: str | list[str]) -> list[str]:
|
|
203
|
+
text = source if isinstance(source, str) else "".join(source)
|
|
204
|
+
return text.splitlines(keepends=True)
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _parse_frontmatter(cell: dict) -> dict | None:
|
|
208
|
+
"""The YAML mapping if a markdown cell is a `---`-fenced frontmatter block."""
|
|
209
|
+
if cell.get("cell_type") != "markdown":
|
|
210
|
+
return None
|
|
211
|
+
text = "".join(cell.get("source", [])).strip()
|
|
212
|
+
match = re.match(r"^---\s*\n(.*?)\n---\s*(\n|$)", text, re.DOTALL)
|
|
213
|
+
if not match:
|
|
214
|
+
return None
|
|
215
|
+
try:
|
|
216
|
+
data = YAML(typ="safe").load(match.group(1))
|
|
217
|
+
except YAMLError:
|
|
218
|
+
return None
|
|
219
|
+
return data if isinstance(data, dict) and data else None
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _check_frontmatter_first(notebook: dict, source: Path):
|
|
223
|
+
"""MyST only reads frontmatter from the first cell; anywhere else it renders
|
|
224
|
+
as text (the YAML becomes a garbled page title)."""
|
|
225
|
+
for index, cell in enumerate(notebook["cells"]):
|
|
226
|
+
if index > 0 and _parse_frontmatter(cell) is not None:
|
|
227
|
+
raise click.ClickException(
|
|
228
|
+
f"{source}: the frontmatter cell (--- ... ---) must be the first "
|
|
229
|
+
"cell of the notebook, MyST ignores it anywhere else.\n"
|
|
230
|
+
" Move that mo.md(...) cell to the top of the file."
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _project_import_name() -> str | None:
|
|
235
|
+
from afterpython.tools.pyproject import read_pyproject
|
|
236
|
+
|
|
237
|
+
# only the name is needed, so skip read_metadata()'s full validation
|
|
238
|
+
try:
|
|
239
|
+
name = str(read_pyproject()["project"]["name"])
|
|
240
|
+
except Exception:
|
|
241
|
+
return None
|
|
242
|
+
return name.replace("-", "_").replace(".", "_").lower()
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def _check_no_errors(notebook: dict, source: Path):
|
|
246
|
+
errors = [
|
|
247
|
+
output
|
|
248
|
+
for cell in notebook["cells"]
|
|
249
|
+
for output in cell.get("outputs", [])
|
|
250
|
+
if output.get("output_type") == "error"
|
|
251
|
+
]
|
|
252
|
+
if not errors:
|
|
253
|
+
return
|
|
254
|
+
first = errors[0]
|
|
255
|
+
message = (
|
|
256
|
+
f"{source}: {len(errors)} cell(s) raised an error while running in the "
|
|
257
|
+
f"notebook's sandbox, first one:\n"
|
|
258
|
+
f" {first.get('ename')}: {first.get('evalue')}"
|
|
259
|
+
)
|
|
260
|
+
package = _project_import_name()
|
|
261
|
+
if (
|
|
262
|
+
package
|
|
263
|
+
and first.get("ename") == "ModuleNotFoundError"
|
|
264
|
+
and f"'{package}'" in str(first.get("evalue"))
|
|
265
|
+
):
|
|
266
|
+
project_path = os.path.relpath(ap.paths.user_path, source.parent)
|
|
267
|
+
message += (
|
|
268
|
+
"\n The notebook runs in its own environment, so your package must "
|
|
269
|
+
"be declared in its PEP 723 header, e.g.:\n"
|
|
270
|
+
" # /// script\n"
|
|
271
|
+
f' # dependencies = ["marimo", "{package}"]\n'
|
|
272
|
+
" # [tool.uv.sources]\n"
|
|
273
|
+
f' # {package} = {{ path = "{project_path}", editable = true }}\n'
|
|
274
|
+
" # ///"
|
|
275
|
+
)
|
|
276
|
+
raise click.ClickException(message)
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def _cell_id_prefix(source: Path, content_path: Path) -> str:
|
|
280
|
+
rel = source.relative_to(content_path).with_suffix("").as_posix()
|
|
281
|
+
return re.sub(r"[^A-Za-z0-9_-]", "-", rel)
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def _rewrite_cell_ids(notebook: dict, prefix: str):
|
|
285
|
+
"""marimo derives cell ids from cell position, so every notebook repeats the
|
|
286
|
+
same ids and MyST reports duplicate identifiers across the project."""
|
|
287
|
+
for cell in notebook["cells"]:
|
|
288
|
+
cell_id = f"{prefix}-{cell.get('id', '')}"
|
|
289
|
+
if len(cell_id) > _MAX_CELL_ID_LENGTH:
|
|
290
|
+
digest = hashlib.sha1(cell_id.encode()).hexdigest()[:8]
|
|
291
|
+
cell_id = f"{cell_id[: _MAX_CELL_ID_LENGTH - 9]}-{digest}"
|
|
292
|
+
cell["id"] = cell_id
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def _postprocess(
|
|
296
|
+
notebook: dict, source: Path, content_path: Path, marimo_version: str
|
|
297
|
+
) -> dict:
|
|
298
|
+
for cell in notebook["cells"]:
|
|
299
|
+
cell["source"] = _as_lines(cell.get("source", ""))
|
|
300
|
+
_check_no_errors(notebook, source)
|
|
301
|
+
_check_frontmatter_first(notebook, source)
|
|
302
|
+
# outputs come from the sandbox, `myst build --execute` must not redo them
|
|
303
|
+
_add_fields_to_frontmatter(notebook, {"execute": "{skip: true}"})
|
|
304
|
+
_rewrite_cell_ids(notebook, _cell_id_prefix(source, content_path))
|
|
305
|
+
notebook.setdefault("metadata", {})["afterpython"] = {
|
|
306
|
+
"generated_from": source.name,
|
|
307
|
+
"source_hash": _source_hash(source),
|
|
308
|
+
"marimo_version": marimo_version,
|
|
309
|
+
}
|
|
310
|
+
return notebook
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def convert_marimo_notebook(
|
|
314
|
+
source: Path, content_path: Path, marimo_version: str, use_cache: bool = True
|
|
315
|
+
) -> bool:
|
|
316
|
+
"""Export one marimo notebook to `<stem>.ipynb` next to it.
|
|
317
|
+
|
|
318
|
+
Returns True if it was (re-)exported, False if the cached output was reused.
|
|
319
|
+
"""
|
|
320
|
+
output = source.with_suffix(".ipynb")
|
|
321
|
+
if use_cache and _is_cached(source, output, marimo_version):
|
|
322
|
+
return False
|
|
323
|
+
|
|
324
|
+
click.echo(f"Running marimo notebook {source.relative_to(ap.paths.user_path)}...")
|
|
325
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
326
|
+
tmp_output = Path(tmp) / output.name
|
|
327
|
+
result = subprocess.run(
|
|
328
|
+
[
|
|
329
|
+
"marimo",
|
|
330
|
+
"export",
|
|
331
|
+
"ipynb",
|
|
332
|
+
str(source),
|
|
333
|
+
"-o",
|
|
334
|
+
str(tmp_output),
|
|
335
|
+
"--include-outputs",
|
|
336
|
+
"--sandbox",
|
|
337
|
+
# keep the author's cell order, not marimo's dependency order
|
|
338
|
+
"--sort",
|
|
339
|
+
"top-down",
|
|
340
|
+
"--force",
|
|
341
|
+
],
|
|
342
|
+
check=False,
|
|
343
|
+
)
|
|
344
|
+
# marimo (>=0.25) exits 1 when a cell fails but still writes the
|
|
345
|
+
# notebook, so read it whenever it exists: _postprocess reports the
|
|
346
|
+
# actual cell error, which is far more useful than the exit code
|
|
347
|
+
if not tmp_output.exists():
|
|
348
|
+
raise click.ClickException(f"marimo export failed for {source}")
|
|
349
|
+
notebook = json.loads(tmp_output.read_text(encoding="utf-8"))
|
|
350
|
+
|
|
351
|
+
notebook = _postprocess(notebook, source, content_path, marimo_version)
|
|
352
|
+
if result.returncode != 0:
|
|
353
|
+
raise click.ClickException(f"marimo export failed for {source}")
|
|
354
|
+
output.write_text(
|
|
355
|
+
json.dumps(notebook, indent=1, ensure_ascii=False) + "\n", encoding="utf-8"
|
|
356
|
+
)
|
|
357
|
+
return True
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
def _prepare_folder(content_path: Path) -> list[Path]:
|
|
361
|
+
"""Folder-level rules; returns the folder's marimo notebooks."""
|
|
362
|
+
scan = _scan_folder(content_path)
|
|
363
|
+
_check_not_mixed(content_path, scan)
|
|
364
|
+
_remove_orphans(scan)
|
|
365
|
+
_sync_gitignore(content_path, has_marimo=bool(scan.marimo))
|
|
366
|
+
return scan.marimo
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def build_marimo_content(use_cache: bool = True):
|
|
370
|
+
"""Convert every marimo notebook in the content folders to `.ipynb`.
|
|
371
|
+
|
|
372
|
+
Args:
|
|
373
|
+
use_cache: reuse a generated notebook when its `.py` and the marimo
|
|
374
|
+
version are unchanged. `ap build` passes False: a notebook's
|
|
375
|
+
outputs can change without its `.py` changing (e.g. it imports the
|
|
376
|
+
project's own package), so production builds always re-run.
|
|
377
|
+
"""
|
|
378
|
+
marimo_version: str | None = None
|
|
379
|
+
for content_type in sorted(CONTENT_TYPES):
|
|
380
|
+
content_path = ap.paths.afterpython_path / content_type
|
|
381
|
+
if not content_path.exists():
|
|
382
|
+
continue
|
|
383
|
+
notebooks = _prepare_folder(content_path)
|
|
384
|
+
if not notebooks:
|
|
385
|
+
continue
|
|
386
|
+
marimo_version = marimo_version or _marimo_version()
|
|
387
|
+
for source in notebooks:
|
|
388
|
+
convert_marimo_notebook(source, content_path, marimo_version, use_cache)
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def remove_generated_notebooks(content_path: Path) -> int:
|
|
392
|
+
"""Delete the `.ipynb` files afterpython generated in a content folder."""
|
|
393
|
+
if not content_path.exists():
|
|
394
|
+
return 0
|
|
395
|
+
generated = _scan_folder(content_path).generated
|
|
396
|
+
for notebook_path in generated:
|
|
397
|
+
notebook_path.unlink()
|
|
398
|
+
return len(generated)
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def watch_marimo_content(
|
|
402
|
+
content_types: set[str], stop_event: threading.Event, interval: float = 1.0
|
|
403
|
+
):
|
|
404
|
+
"""`ap dev`: re-export marimo notebooks when their `.py` is saved.
|
|
405
|
+
|
|
406
|
+
Best effort: every save re-runs the whole notebook, so errors are printed
|
|
407
|
+
instead of stopping the dev servers, and slow re-runs print a hint.
|
|
408
|
+
"""
|
|
409
|
+
content_paths = [
|
|
410
|
+
ap.paths.afterpython_path / content_type
|
|
411
|
+
for content_type in sorted(content_types)
|
|
412
|
+
]
|
|
413
|
+
|
|
414
|
+
def _snapshot() -> dict[Path, float]:
|
|
415
|
+
mtimes = {}
|
|
416
|
+
for content_path in content_paths:
|
|
417
|
+
if not content_path.exists():
|
|
418
|
+
continue
|
|
419
|
+
for path in content_path.rglob("*.py"):
|
|
420
|
+
rel_parts = path.relative_to(content_path).parts
|
|
421
|
+
if "_build" in rel_parts or any(p.startswith(".") for p in rel_parts):
|
|
422
|
+
continue
|
|
423
|
+
try:
|
|
424
|
+
mtimes[path] = path.stat().st_mtime
|
|
425
|
+
except OSError:
|
|
426
|
+
continue
|
|
427
|
+
return mtimes
|
|
428
|
+
|
|
429
|
+
marimo_version: str | None = None
|
|
430
|
+
seen = _snapshot()
|
|
431
|
+
while not stop_event.wait(interval):
|
|
432
|
+
current = _snapshot()
|
|
433
|
+
changed = {path for path, mtime in current.items() if seen.get(path) != mtime}
|
|
434
|
+
removed = seen.keys() - current.keys()
|
|
435
|
+
seen = current
|
|
436
|
+
if not changed and not removed:
|
|
437
|
+
continue
|
|
438
|
+
for content_path in content_paths:
|
|
439
|
+
if not content_path.exists():
|
|
440
|
+
continue
|
|
441
|
+
if not any(p.is_relative_to(content_path) for p in changed | removed):
|
|
442
|
+
continue
|
|
443
|
+
try:
|
|
444
|
+
notebooks = _prepare_folder(content_path)
|
|
445
|
+
for source in notebooks:
|
|
446
|
+
if source not in changed:
|
|
447
|
+
continue
|
|
448
|
+
marimo_version = marimo_version or _marimo_version()
|
|
449
|
+
start = time.monotonic()
|
|
450
|
+
convert_marimo_notebook(source, content_path, marimo_version)
|
|
451
|
+
elapsed = time.monotonic() - start
|
|
452
|
+
if elapsed > SLOW_EXPORT_SECONDS:
|
|
453
|
+
rel = source.relative_to(ap.paths.afterpython_path)
|
|
454
|
+
click.echo(
|
|
455
|
+
click.style(
|
|
456
|
+
f"⚠ {rel} took {elapsed:.0f}s to re-run after "
|
|
457
|
+
"saving.\n"
|
|
458
|
+
" ap dev re-runs the whole notebook on every "
|
|
459
|
+
"save. If this is too slow, finish editing it in "
|
|
460
|
+
"`marimo edit --sandbox` first, then run ap dev.",
|
|
461
|
+
fg="yellow",
|
|
462
|
+
)
|
|
463
|
+
)
|
|
464
|
+
except click.ClickException as e:
|
|
465
|
+
click.echo(click.style(f"✗ {e.format_message()}", fg="red"))
|
|
466
|
+
except Exception as e: # keep watching: a crash here would go unnoticed
|
|
467
|
+
click.echo(click.style(f"✗ marimo re-export failed: {e!r}", fg="red"))
|