jljb 0.1a1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jljb-0.1a1/LICENSE +37 -0
- jljb-0.1a1/PKG-INFO +87 -0
- jljb-0.1a1/README.md +47 -0
- jljb-0.1a1/pyproject.toml +45 -0
- jljb-0.1a1/src/jljb/__init__.py +4 -0
- jljb-0.1a1/src/jljb/make_jl_dir.py +76 -0
- jljb-0.1a1/src/jljb/process_notebooks.py +412 -0
jljb-0.1a1/LICENSE
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
.. _license:
|
|
2
|
+
|
|
3
|
+
**********************
|
|
4
|
+
Copyright and Licenses
|
|
5
|
+
**********************
|
|
6
|
+
|
|
7
|
+
jljb
|
|
8
|
+
====
|
|
9
|
+
|
|
10
|
+
The jljb package, including all examples, code snippets and attached
|
|
11
|
+
documentation is covered by the 2-clause BSD license.
|
|
12
|
+
|
|
13
|
+
Copyright (c) 2017-19, Matthew Brett
|
|
14
|
+
All rights reserved.
|
|
15
|
+
|
|
16
|
+
Redistribution and use in source and binary forms, with or without
|
|
17
|
+
modification, are permitted provided that the following conditions are
|
|
18
|
+
met:
|
|
19
|
+
|
|
20
|
+
1. Redistributions of source code must retain the above copyright notice,
|
|
21
|
+
this list of conditions and the following disclaimer.
|
|
22
|
+
|
|
23
|
+
2. Redistributions in binary form must reproduce the above copyright
|
|
24
|
+
notice, this list of conditions and the following disclaimer in the
|
|
25
|
+
documentation and/or other materials provided with the distribution.
|
|
26
|
+
|
|
27
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
|
|
28
|
+
IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
|
|
29
|
+
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
|
30
|
+
PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
|
|
31
|
+
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
|
|
32
|
+
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
|
33
|
+
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
|
34
|
+
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
|
35
|
+
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
|
36
|
+
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
|
37
|
+
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
jljb-0.1a1/PKG-INFO
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: jljb
|
|
3
|
+
Version: 0.1a1
|
|
4
|
+
Summary: JupyterLite for Jupyter Book processing
|
|
5
|
+
Home-page: https://github.com/matthew-brett/jljb
|
|
6
|
+
Author: Matthew Brett
|
|
7
|
+
Author-email: matthew.brett@gmail.com
|
|
8
|
+
Requires-Python: >=3.8
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
Classifier: License :: OSI Approved :: BSD License
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Intended Audience :: Science/Research
|
|
16
|
+
Classifier: License :: OSI Approved :: BSD License
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.6
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.7
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
24
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
25
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
26
|
+
Classifier: Topic :: Scientific/Engineering
|
|
27
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
28
|
+
Classifier: Operating System :: POSIX
|
|
29
|
+
Classifier: Operating System :: Unix
|
|
30
|
+
Classifier: Operating System :: MacOS
|
|
31
|
+
License-File: LICENSE
|
|
32
|
+
Requires-Dist: jupytext
|
|
33
|
+
Requires-Dist: jupyterlite-core
|
|
34
|
+
Requires-Dist: jupyterlite-pyodide-kernel
|
|
35
|
+
Requires-Dist: jupyterlab_server
|
|
36
|
+
Requires-Dist: sphinx
|
|
37
|
+
Requires-Dist: myst_parser
|
|
38
|
+
Requires-Dist: linkify-it-py
|
|
39
|
+
|
|
40
|
+
# JupyterLite / JupyterBook notebook processing
|
|
41
|
+
|
|
42
|
+
These are utilities to take text-format notebooks in a Jupyter Book directory,
|
|
43
|
+
and write out a JupyterLite installation, containing `.ipynb` (JSON) notebooks with suitable processing.
|
|
44
|
+
|
|
45
|
+
The module provides the executable `jljb-write-dir`. See the help for that command for more details.
|
|
46
|
+
|
|
47
|
+
That command will:
|
|
48
|
+
|
|
49
|
+
* Look for suitable `_config.yml` configuration in your specified
|
|
50
|
+
configuration directory (defaulting to current directory).
|
|
51
|
+
* It then looks in the configuration for a `jupyterlite` section, and within
|
|
52
|
+
that section looks for these values (default if not specified):
|
|
53
|
+
|
|
54
|
+
| Name | Default |
|
|
55
|
+
| ---------------- | ------- |
|
|
56
|
+
| in_nb_ext | .md |
|
|
57
|
+
| out_nb_ext | .ipynb |
|
|
58
|
+
| in_nb_fmt | myst |
|
|
59
|
+
| remove_remove | True |
|
|
60
|
+
| proc_admonitions | True |
|
|
61
|
+
|
|
62
|
+
* With that information, the script finds text format notebooks with the
|
|
63
|
+
`in_nb_ext` extension, and applies the following processing to write into an
|
|
64
|
+
output folder:
|
|
65
|
+
|
|
66
|
+
* Replaces local kernel with Pyodide kernel in metadata.
|
|
67
|
+
* Filters:
|
|
68
|
+
* Note and admonition markers (if `proc_admonitions` is True)
|
|
69
|
+
* Exercise markers (see
|
|
70
|
+
[sphinx_exercise](https://ebp-sphinx-exercise.readthedocs.io).
|
|
71
|
+
* Solution blocks for exercises.
|
|
72
|
+
* Cells marked with [`remove-cell`
|
|
73
|
+
tag](https://jupyterbook.org/v1/interactive/hiding.html#removing-code-cell-content).
|
|
74
|
+
* Writes notebooks to output directory.
|
|
75
|
+
* Writes JSON jupyterlite file.
|
|
76
|
+
|
|
77
|
+
The typical way to use this module / command is to write your settings into
|
|
78
|
+
the `_config.yml` file as above, and have a `Makefile` target of form:
|
|
79
|
+
|
|
80
|
+
```makefile
|
|
81
|
+
jl:
|
|
82
|
+
# Jupyter-lite files for book build.
|
|
83
|
+
# Install specified requirements for built JupyterLite site.
|
|
84
|
+
$(PIP_INSTALL_CMD) -r jl-build-requirements.txt
|
|
85
|
+
jljb-write-dir $(BUILD_DIR)/interact data images --jl-tmp $(JL_DIR)
|
|
86
|
+
```
|
|
87
|
+
|
jljb-0.1a1/README.md
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# JupyterLite / JupyterBook notebook processing
|
|
2
|
+
|
|
3
|
+
These are utilities to take text-format notebooks in a Jupyter Book directory,
|
|
4
|
+
and write out a JupyterLite installation, containing `.ipynb` (JSON) notebooks with suitable processing.
|
|
5
|
+
|
|
6
|
+
The module provides the executable `jljb-write-dir`. See the help for that command for more details.
|
|
7
|
+
|
|
8
|
+
That command will:
|
|
9
|
+
|
|
10
|
+
* Look for suitable `_config.yml` configuration in your specified
|
|
11
|
+
configuration directory (defaulting to current directory).
|
|
12
|
+
* It then looks in the configuration for a `jupyterlite` section, and within
|
|
13
|
+
that section looks for these values (default if not specified):
|
|
14
|
+
|
|
15
|
+
| Name | Default |
|
|
16
|
+
| ---------------- | ------- |
|
|
17
|
+
| in_nb_ext | .md |
|
|
18
|
+
| out_nb_ext | .ipynb |
|
|
19
|
+
| in_nb_fmt | myst |
|
|
20
|
+
| remove_remove | True |
|
|
21
|
+
| proc_admonitions | True |
|
|
22
|
+
|
|
23
|
+
* With that information, the script finds text format notebooks with the
|
|
24
|
+
`in_nb_ext` extension, and applies the following processing to write into an
|
|
25
|
+
output folder:
|
|
26
|
+
|
|
27
|
+
* Replaces local kernel with Pyodide kernel in metadata.
|
|
28
|
+
* Filters:
|
|
29
|
+
* Note and admonition markers (if `proc_admonitions` is True)
|
|
30
|
+
* Exercise markers (see
|
|
31
|
+
[sphinx_exercise](https://ebp-sphinx-exercise.readthedocs.io).
|
|
32
|
+
* Solution blocks for exercises.
|
|
33
|
+
* Cells marked with [`remove-cell`
|
|
34
|
+
tag](https://jupyterbook.org/v1/interactive/hiding.html#removing-code-cell-content).
|
|
35
|
+
* Writes notebooks to output directory.
|
|
36
|
+
* Writes JSON jupyterlite file.
|
|
37
|
+
|
|
38
|
+
The typical way to use this module / command is to write your settings into
|
|
39
|
+
the `_config.yml` file as above, and have a `Makefile` target of form:
|
|
40
|
+
|
|
41
|
+
```makefile
|
|
42
|
+
jl:
|
|
43
|
+
# Jupyter-lite files for book build.
|
|
44
|
+
# Install specified requirements for built JupyterLite site.
|
|
45
|
+
$(PIP_INSTALL_CMD) -r jl-build-requirements.txt
|
|
46
|
+
jljb-write-dir $(BUILD_DIR)/interact data images --jl-tmp $(JL_DIR)
|
|
47
|
+
```
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["flit_core >=2,<4"]
|
|
3
|
+
build-backend = "flit_core.buildapi"
|
|
4
|
+
|
|
5
|
+
[tool.flit.metadata]
|
|
6
|
+
module = "jljb"
|
|
7
|
+
author = "Matthew Brett"
|
|
8
|
+
author-email = "matthew.brett@gmail.com"
|
|
9
|
+
home-page = "https://github.com/matthew-brett/jljb"
|
|
10
|
+
classifiers = ["License :: OSI Approved :: BSD License",
|
|
11
|
+
"Programming Language :: Python :: 3",
|
|
12
|
+
'Development Status :: 3 - Alpha',
|
|
13
|
+
'Environment :: Console',
|
|
14
|
+
'Intended Audience :: Developers',
|
|
15
|
+
'Intended Audience :: Science/Research',
|
|
16
|
+
'License :: OSI Approved :: BSD License',
|
|
17
|
+
'Programming Language :: Python :: 3',
|
|
18
|
+
'Programming Language :: Python :: 3.6',
|
|
19
|
+
'Programming Language :: Python :: 3.7',
|
|
20
|
+
'Programming Language :: Python :: 3.8',
|
|
21
|
+
'Programming Language :: Python :: 3.9',
|
|
22
|
+
'Programming Language :: Python :: 3.10',
|
|
23
|
+
'Programming Language :: Python :: 3.11',
|
|
24
|
+
'Programming Language :: Python :: 3.12',
|
|
25
|
+
'Programming Language :: Python :: 3.13',
|
|
26
|
+
'Topic :: Scientific/Engineering',
|
|
27
|
+
'Operating System :: Microsoft :: Windows',
|
|
28
|
+
'Operating System :: POSIX',
|
|
29
|
+
'Operating System :: Unix',
|
|
30
|
+
'Operating System :: MacOS']
|
|
31
|
+
description-file="README.md"
|
|
32
|
+
# Check against requirements.txt
|
|
33
|
+
requires = [
|
|
34
|
+
'jupytext',
|
|
35
|
+
'jupyterlite-core',
|
|
36
|
+
'jupyterlite-pyodide-kernel',
|
|
37
|
+
'jupyterlab_server',
|
|
38
|
+
'sphinx',
|
|
39
|
+
'myst_parser',
|
|
40
|
+
'linkify-it-py'
|
|
41
|
+
]
|
|
42
|
+
requires-python=">=3.8"
|
|
43
|
+
|
|
44
|
+
[tool.flit.scripts]
|
|
45
|
+
jljb-write-dir = "jljb.make_jl_dir:main"
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
""" Create JupyterLite directory
|
|
2
|
+
"""
|
|
3
|
+
|
|
4
|
+
from argparse import ArgumentParser, RawDescriptionHelpFormatter
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
import shutil
|
|
7
|
+
from subprocess import check_call
|
|
8
|
+
import sys
|
|
9
|
+
import tempfile
|
|
10
|
+
|
|
11
|
+
from .process_notebooks import write_proc_nbs, load_config
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def mk_jl_dir(out_dir,
|
|
15
|
+
config,
|
|
16
|
+
support_files=(),
|
|
17
|
+
jl_dir=None):
|
|
18
|
+
tmp_jl_dir = jl_dir is None
|
|
19
|
+
if tmp_jl_dir:
|
|
20
|
+
tmp_dir = tempfile.TemporaryDirectory()
|
|
21
|
+
jl_dir = Path(tmp_dir.name)
|
|
22
|
+
else:
|
|
23
|
+
jl_dir = Path(jl_dir)
|
|
24
|
+
if jl_dir.is_dir():
|
|
25
|
+
shutil.rmtree(jl_dir)
|
|
26
|
+
jl_dir.mkdir()
|
|
27
|
+
out_dir = Path(out_dir)
|
|
28
|
+
for support_file in support_files:
|
|
29
|
+
s_pth = Path(support_file)
|
|
30
|
+
if s_pth.is_file():
|
|
31
|
+
shutil.copy2(s_pth, jl_dir)
|
|
32
|
+
elif s_pth.is_dir():
|
|
33
|
+
shutil.copytree(s_pth, jl_dir / s_pth.resolve().name)
|
|
34
|
+
write_proc_nbs(config, jl_dir)
|
|
35
|
+
out_dir.mkdir(parents=True, exist_ok=True)
|
|
36
|
+
cmd = [sys.executable, '-m',
|
|
37
|
+
'jupyter', 'lite', 'build',
|
|
38
|
+
'--contents', str(jl_dir),
|
|
39
|
+
'--output-dir', str(out_dir),
|
|
40
|
+
'--lite-dir', str(jl_dir)]
|
|
41
|
+
check_call(cmd)
|
|
42
|
+
if tmp_jl_dir:
|
|
43
|
+
tmp_dir.cleanup()
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def get_parser():
|
|
47
|
+
parser = ArgumentParser(description=__doc__, # Usage from docstring
|
|
48
|
+
formatter_class=RawDescriptionHelpFormatter)
|
|
49
|
+
parser.add_argument('output_dir',
|
|
50
|
+
help='Output directory for JL distribution')
|
|
51
|
+
parser.add_argument('support_files', nargs='*',
|
|
52
|
+
help='Supporting files / dirs to copy into `out_dir`')
|
|
53
|
+
parser.add_argument('--jl-tmp',
|
|
54
|
+
help='Temporary directory for JL intermediate files')
|
|
55
|
+
parser.add_argument(
|
|
56
|
+
"--config", default="_config.yml",
|
|
57
|
+
help="Jupyter Book YaML configuration file"
|
|
58
|
+
)
|
|
59
|
+
parser.add_argument(
|
|
60
|
+
'--input_dir',
|
|
61
|
+
help=("Input directory containing notebooks "
|
|
62
|
+
"(default is directory of _config.yml file)"))
|
|
63
|
+
return parser
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def main():
|
|
67
|
+
parser = get_parser()
|
|
68
|
+
args = parser.parse_args()
|
|
69
|
+
config = load_config(Path(args.config))
|
|
70
|
+
if args.input_dir:
|
|
71
|
+
config['input_dir'] = Path(args.input_dir)
|
|
72
|
+
mk_jl_dir(args.output_dir, config, args.support_files, jl_dir=args.jl_tmp)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
if __name__ == '__main__':
|
|
76
|
+
main()
|
|
@@ -0,0 +1,412 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Process notebooks
|
|
3
|
+
|
|
4
|
+
* Replace local kernel with Pyodide kernel in metadata.
|
|
5
|
+
* Filter:
|
|
6
|
+
* Note and admonition markers.
|
|
7
|
+
* Exercise markers.
|
|
8
|
+
* Solution blocks.
|
|
9
|
+
* Write notebooks to output directory.
|
|
10
|
+
* Write JSON jupyterlite file.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from argparse import ArgumentParser, RawDescriptionHelpFormatter
|
|
14
|
+
from copy import deepcopy
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
import re
|
|
17
|
+
from urllib.parse import quote as urlquote, urlparse
|
|
18
|
+
|
|
19
|
+
import docutils.core as duc
|
|
20
|
+
import docutils.nodes as dun
|
|
21
|
+
from docutils.utils import Reporter
|
|
22
|
+
from sphinx.util.matching import get_matching_files
|
|
23
|
+
from myst_parser.docutils_ import Parser
|
|
24
|
+
import yaml
|
|
25
|
+
|
|
26
|
+
_END_DIV_RE = re.compile(r"^\s*(:::+|```+|~~~+)\s*$")
|
|
27
|
+
import jupytext
|
|
28
|
+
|
|
29
|
+
_JL_JSON_FMT = r"""\
|
|
30
|
+
{{
|
|
31
|
+
"jupyter-lite-schema-version": 0,
|
|
32
|
+
"jupyter-config-data": {{
|
|
33
|
+
"contentsStorageName": "rss-{language}"
|
|
34
|
+
}}
|
|
35
|
+
}}
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
_DIV_RE = r"\s*(:::+|```+|~~~+)\s*"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
_ADM_HEADER = re.compile(
|
|
42
|
+
rf"""
|
|
43
|
+
^{_DIV_RE}
|
|
44
|
+
\{{\s*(?P<ad_type>\S+)\s*\}}\s*
|
|
45
|
+
(?P<ad_title>.*)\s*$
|
|
46
|
+
""",
|
|
47
|
+
flags=re.VERBOSE,
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
_EX_SOL_MARKER = re.compile(
|
|
52
|
+
rf"""
|
|
53
|
+
(?P<newlines>\n*)
|
|
54
|
+
{_DIV_RE}
|
|
55
|
+
\{{\s*
|
|
56
|
+
(?P<ex_sol>exercise|solution)-
|
|
57
|
+
(?P<st_end>start|end)
|
|
58
|
+
\s*\}}
|
|
59
|
+
\s*
|
|
60
|
+
(?P<suffix>\S+)?\s*
|
|
61
|
+
\n
|
|
62
|
+
(?P<attrs>\s*:\S+: \s* \S+\s*\n)*
|
|
63
|
+
\n*
|
|
64
|
+
\s*(\2)\s*
|
|
65
|
+
\n
|
|
66
|
+
""",
|
|
67
|
+
flags=re.VERBOSE,
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
_SOL_MARKED = re.compile(
|
|
72
|
+
r"""
|
|
73
|
+
\n?
|
|
74
|
+
<!--\sstart-solution\s-->\n
|
|
75
|
+
.*?
|
|
76
|
+
<!--\send-solution\s-->\n?
|
|
77
|
+
""",
|
|
78
|
+
flags=re.VERBOSE | re.MULTILINE | re.DOTALL,
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
_END_DIV_RE = re.compile(rf"^{_DIV_RE}$")
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
# https://myst-parser.readthedocs.io/en/latest/syntax/optional.html#syntax-extensions
|
|
86
|
+
MYST_EXTENSIONS = [
|
|
87
|
+
"amsmath",
|
|
88
|
+
"attrs_inline",
|
|
89
|
+
"colon_fence",
|
|
90
|
+
"deflist",
|
|
91
|
+
"dollarmath",
|
|
92
|
+
"fieldlist",
|
|
93
|
+
"html_admonition",
|
|
94
|
+
"html_image",
|
|
95
|
+
"linkify",
|
|
96
|
+
"replacements",
|
|
97
|
+
"smartquotes",
|
|
98
|
+
"strikethrough",
|
|
99
|
+
"substitution",
|
|
100
|
+
"tasklist",
|
|
101
|
+
]
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
DEF_JUPYTERLITE_CONFIG = {
|
|
105
|
+
"in_nb_ext": ".md",
|
|
106
|
+
"out_nb_ext": ".ipynb",
|
|
107
|
+
"in_nb_fmt": "myst",
|
|
108
|
+
"remove_remove": True,
|
|
109
|
+
"proc_admonitions": True,
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _replace_markers(m):
|
|
114
|
+
st_end = m["st_end"]
|
|
115
|
+
if m["ex_sol"] == "exercise":
|
|
116
|
+
return f"{m['newlines']}**{st_end.capitalize()} of exercise**\n\n"
|
|
117
|
+
return f"\n<!-- {st_end}-solution -->\n"
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def get_admonition_lines(nb_text, nb_path):
|
|
121
|
+
parser = Parser()
|
|
122
|
+
doc = duc.publish_doctree(
|
|
123
|
+
source=nb_text,
|
|
124
|
+
source_path=str(nb_path),
|
|
125
|
+
settings_overrides={
|
|
126
|
+
"myst_enable_extensions": MYST_EXTENSIONS,
|
|
127
|
+
"report_level": Reporter.SEVERE_LEVEL,
|
|
128
|
+
},
|
|
129
|
+
parser=parser,
|
|
130
|
+
)
|
|
131
|
+
lines = nb_text.splitlines()
|
|
132
|
+
n_lines = len(lines)
|
|
133
|
+
admonition_lines = []
|
|
134
|
+
for admonition in doc.findall(dun.Admonition):
|
|
135
|
+
start_line = admonition.line - 1
|
|
136
|
+
# Find first node of subsequent doctree.
|
|
137
|
+
node0 = next(
|
|
138
|
+
admonition.findall(include_self=False, descend=False, ascend=True), None
|
|
139
|
+
)
|
|
140
|
+
# There can be a system_message as next node, in which case the correct
|
|
141
|
+
# line is in the 'line' attribute.
|
|
142
|
+
last_line = node0.get("line", node0.line) - 2 if node0 else n_lines - 1
|
|
143
|
+
for end_line in range(last_line, start_line + 1, -1):
|
|
144
|
+
if _END_DIV_RE.match(lines[end_line]):
|
|
145
|
+
break
|
|
146
|
+
else:
|
|
147
|
+
raise ValueError("Could not find end div")
|
|
148
|
+
admonition_lines.append((start_line, end_line))
|
|
149
|
+
return admonition_lines
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
_ADM_HEADER = re.compile(
|
|
153
|
+
r"""
|
|
154
|
+
^\s*(:::+|```+|~~~+)\s*
|
|
155
|
+
\{\s*(?P<ad_type>\S+)\s*\}\s*
|
|
156
|
+
(?P<ad_title>.*)\s*$
|
|
157
|
+
""",
|
|
158
|
+
flags=re.VERBOSE,
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
_DIR_OPTION = re.compile(r"^\s*:\w+:")
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def process_admonitions(nb_text, nb_path):
|
|
166
|
+
lines = nb_text.splitlines()
|
|
167
|
+
out_lines = []
|
|
168
|
+
start_i = last = 0
|
|
169
|
+
for first, last in get_admonition_lines(nb_text, nb_path):
|
|
170
|
+
m = _ADM_HEADER.match(lines[first])
|
|
171
|
+
if not m:
|
|
172
|
+
raise ValueError(f"Cannot get match from {lines[first]}")
|
|
173
|
+
out_lines += lines[start_i:first]
|
|
174
|
+
start_i = last + 1
|
|
175
|
+
ad_type, ad_title = m["ad_type"], m["ad_title"]
|
|
176
|
+
suffix = f": {ad_title}" if ad_title else ""
|
|
177
|
+
in_i = first + 1
|
|
178
|
+
while _DIR_OPTION.match(lines[in_i]):
|
|
179
|
+
in_i += 1
|
|
180
|
+
adm_txt = "\n".join(lines[in_i:last]).strip("\n")
|
|
181
|
+
out_lines.append(
|
|
182
|
+
f"**Start of {ad_type}{suffix}**\n\n{adm_txt}\n\n**End of {ad_type}**"
|
|
183
|
+
)
|
|
184
|
+
return "\n".join(out_lines + lines[start_i:])
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def process_cells(nb, processors):
|
|
188
|
+
"""Process cells in notebooks.
|
|
189
|
+
|
|
190
|
+
Parameters
|
|
191
|
+
----------
|
|
192
|
+
nb : dict
|
|
193
|
+
processors : sequence
|
|
194
|
+
Sequences of callables, taking a cell as input, and returning a cell as
|
|
195
|
+
output. If None returned, delete this cell.
|
|
196
|
+
|
|
197
|
+
Returns
|
|
198
|
+
-------
|
|
199
|
+
out_nb : dict
|
|
200
|
+
"""
|
|
201
|
+
out_nb = deepcopy(nb)
|
|
202
|
+
out_cells = []
|
|
203
|
+
for cell in out_nb["cells"]:
|
|
204
|
+
for processor in processors:
|
|
205
|
+
cell = processor(cell)
|
|
206
|
+
if cell is None:
|
|
207
|
+
break
|
|
208
|
+
if cell:
|
|
209
|
+
out_cells.append(cell)
|
|
210
|
+
out_nb["cells"] = out_cells
|
|
211
|
+
return out_nb
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
_LABEL = re.compile(r"^\s*\(\s*\S+\s*\)\=\s*\n", flags=re.MULTILINE)
|
|
215
|
+
|
|
216
|
+
_GLUE_DIR = re.compile(
|
|
217
|
+
r"""
|
|
218
|
+
(:::+|```+)\s*
|
|
219
|
+
\{\s*glue:*\s*\}\s+
|
|
220
|
+
(\w+)\n
|
|
221
|
+
(?:\s*:doc: .*?)*
|
|
222
|
+
\n\s*\1\s*\n
|
|
223
|
+
""",
|
|
224
|
+
flags=re.MULTILINE | re.DOTALL | re.VERBOSE,
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
_GLUE_ROLE = re.compile(
|
|
229
|
+
r"""
|
|
230
|
+
\{\s*glue:{0,1}\s*\}\s*`(.*)?`
|
|
231
|
+
""",
|
|
232
|
+
flags=re.MULTILINE | re.DOTALL | re.VERBOSE,
|
|
233
|
+
)
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def label_processor(cell):
|
|
237
|
+
if cell["cell_type"] == "markdown":
|
|
238
|
+
cell["source"] = _LABEL.sub("", cell["source"])
|
|
239
|
+
return cell
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def remove_processor(cell):
|
|
243
|
+
tags = cell.get("metadata", {}).get("tags", {})
|
|
244
|
+
if "remove-cell" in tags:
|
|
245
|
+
return None
|
|
246
|
+
return cell
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
_GLUE_DIR = re.compile(
|
|
250
|
+
r"""
|
|
251
|
+
(:::+|```+)\s*
|
|
252
|
+
\{\s*glue:*\s*\}\s+
|
|
253
|
+
(?P<ref>\w+)\n
|
|
254
|
+
(\s*:doc:\s*(?P<doc>.*?)$){0,1}
|
|
255
|
+
\n\s*\1\s*\n
|
|
256
|
+
""",
|
|
257
|
+
flags=re.MULTILINE | re.DOTALL | re.VERBOSE,
|
|
258
|
+
)
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
_GLUE_ROLE = re.compile(
|
|
262
|
+
r"""
|
|
263
|
+
\{\s*glue:{0,1}\s*\}\s*`(.*?)`
|
|
264
|
+
""",
|
|
265
|
+
flags=re.MULTILINE | re.DOTALL | re.VERBOSE,
|
|
266
|
+
)
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def _glue_replacer(m):
|
|
270
|
+
d = m.groupdict()
|
|
271
|
+
ref, doc = d["ref"], d["doc"]
|
|
272
|
+
doc_msg = f' in "{doc}"' if doc else ""
|
|
273
|
+
return f"(Ref to `{ref}`{doc_msg})\n"
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def glue_processor(cell):
|
|
277
|
+
if cell["cell_type"] != "markdown":
|
|
278
|
+
return cell
|
|
279
|
+
cell["source"] = _GLUE_DIR.sub(_glue_replacer, cell["source"])
|
|
280
|
+
cell["source"] = _GLUE_ROLE.sub(r"(Ref to `\1`)", cell["source"])
|
|
281
|
+
return cell
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def load_process_nb(nb_path, fmt="myst", url=None, proc_admonitions=True):
|
|
285
|
+
"""Load and process notebook
|
|
286
|
+
|
|
287
|
+
Deal with:
|
|
288
|
+
|
|
289
|
+
* Note and admonition markers.
|
|
290
|
+
* Exercise markers.
|
|
291
|
+
* Solution blocks.
|
|
292
|
+
|
|
293
|
+
Parameters
|
|
294
|
+
----------
|
|
295
|
+
nb_path : file-like
|
|
296
|
+
Path to notebook
|
|
297
|
+
fmt : str, optional
|
|
298
|
+
Format of notebook (for Jupytext)
|
|
299
|
+
url : str, optional
|
|
300
|
+
URL for output page.
|
|
301
|
+
proc_admonitions : {True, False}, optional
|
|
302
|
+
If True, process admonition blocks to plain paragraphs.
|
|
303
|
+
|
|
304
|
+
Returns
|
|
305
|
+
-------
|
|
306
|
+
nb : dict
|
|
307
|
+
Notebook as loaded and parsed.
|
|
308
|
+
"""
|
|
309
|
+
link_txt = "corresponding page"
|
|
310
|
+
page_link = f"[{link_txt}]({url})" if url else link_txt
|
|
311
|
+
nb_path = Path(nb_path)
|
|
312
|
+
nb_text = nb_path.read_text()
|
|
313
|
+
nbt1 = _EX_SOL_MARKER.sub(_replace_markers, nb_text)
|
|
314
|
+
nbt2 = _SOL_MARKED.sub(f"\n**See the {page_link} for solution**\n\n", nbt1)
|
|
315
|
+
if proc_admonitions:
|
|
316
|
+
nbt2 = process_admonitions(nbt2, nb_path)
|
|
317
|
+
nb = jupytext.reads(nbt2, fmt={"format_name": fmt, "extension": nb_path.suffix})
|
|
318
|
+
return process_cells(nb, [label_processor, glue_processor])
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def process_notebooks(
|
|
322
|
+
config, output_dir, kernel_name="python", kernel_dname="Python (Pyodide)"
|
|
323
|
+
):
|
|
324
|
+
# Get processing params from jupyterlite config section.
|
|
325
|
+
jl_config = config["jupyterlite"]
|
|
326
|
+
input_dir = Path(config["input_dir"])
|
|
327
|
+
# Use sphinx utility to find not-excluded files.
|
|
328
|
+
for fn in get_matching_files(
|
|
329
|
+
input_dir, exclude_patterns=config["exclude_patterns"]
|
|
330
|
+
):
|
|
331
|
+
rel_path = Path(fn)
|
|
332
|
+
if rel_path.suffix != jl_config["in_nb_ext"]:
|
|
333
|
+
continue
|
|
334
|
+
print(f"Processing {rel_path}")
|
|
335
|
+
nb_url = (
|
|
336
|
+
config["base_path"]
|
|
337
|
+
+ "/"
|
|
338
|
+
+ urlquote(rel_path.with_suffix(".html").as_posix())
|
|
339
|
+
)
|
|
340
|
+
nb = load_process_nb(
|
|
341
|
+
input_dir / rel_path,
|
|
342
|
+
jl_config["in_nb_fmt"],
|
|
343
|
+
nb_url,
|
|
344
|
+
jl_config["proc_admonitions"],
|
|
345
|
+
)
|
|
346
|
+
if jl_config["remove_remove"]:
|
|
347
|
+
nb = process_cells(nb, [remove_processor])
|
|
348
|
+
nb["metadata"]["kernelspec"] = {
|
|
349
|
+
"name": kernel_name,
|
|
350
|
+
"display_name": kernel_dname,
|
|
351
|
+
}
|
|
352
|
+
out_path = (output_dir / rel_path).with_suffix(jl_config["out_nb_ext"])
|
|
353
|
+
out_path.parent.mkdir(exist_ok=True, parents=True)
|
|
354
|
+
jupytext.write(nb, out_path)
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def load_config(config_path):
|
|
358
|
+
config_path = Path(config_path).resolve()
|
|
359
|
+
with config_path.open("rt") as fobj:
|
|
360
|
+
config = yaml.safe_load(fobj)
|
|
361
|
+
# Post-processing.
|
|
362
|
+
config["input_dir"] = Path(
|
|
363
|
+
config.get("repository", {}).get("path_to_book", config_path.parent)
|
|
364
|
+
)
|
|
365
|
+
config["base_path"] = urlparse(config.get("html", {}).get("baseurl", "")).path
|
|
366
|
+
config["exclude_patterns"] = config.get("exclude_patterns", [])
|
|
367
|
+
config["exclude_patterns"].append("_build")
|
|
368
|
+
config["jupyterlite"] = dict(
|
|
369
|
+
DEF_JUPYTERLITE_CONFIG, **config.get("jupyterlite", {})
|
|
370
|
+
)
|
|
371
|
+
return config
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def get_parser():
|
|
375
|
+
parser = ArgumentParser(
|
|
376
|
+
description=__doc__, # Usage from docstring
|
|
377
|
+
formatter_class=RawDescriptionHelpFormatter,
|
|
378
|
+
)
|
|
379
|
+
parser.add_argument(
|
|
380
|
+
"output_dir", help="Directory to which we will output notebooks"
|
|
381
|
+
)
|
|
382
|
+
parser.add_argument(
|
|
383
|
+
"--config", default="_config.yml",
|
|
384
|
+
help="Jupyter Book YaML configuration file"
|
|
385
|
+
)
|
|
386
|
+
parser.add_argument(
|
|
387
|
+
"--input-dir",
|
|
388
|
+
help="Input directory (default is directory of _config.yml file)"
|
|
389
|
+
)
|
|
390
|
+
return parser
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def write_proc_nbs(config, output_dir):
|
|
394
|
+
out_path = Path(output_dir)
|
|
395
|
+
out_path.mkdir(parents=True, exist_ok=True)
|
|
396
|
+
process_notebooks(config, out_path)
|
|
397
|
+
(out_path / "jupyter-lite.json").write_text(
|
|
398
|
+
_JL_JSON_FMT.format(language="python"))
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def main():
|
|
403
|
+
parser = get_parser()
|
|
404
|
+
args = parser.parse_args()
|
|
405
|
+
config = load_config(Path(args.config))
|
|
406
|
+
if args.input_dir:
|
|
407
|
+
config['input_dir'] = Path(args.input_dir)
|
|
408
|
+
write_proc_nbs(config, args.output_dir)
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
if __name__ == "__main__":
|
|
412
|
+
main()
|