simplibs-extractor-html 0.1.0__py3-none-any.whl

simplibs/extractor_html/file_rules/atomic/VCS_DIRS.py

@@ -0,0 +1,21 @@
+from typing import Final
+from simplibs.path import IGNORE_VCS
+from simplibs.path.rules import AnyParentNameIs
+from simplibs.rules import IsIn, Rule
+
+
+VCS_DIRS: Final[Rule] = AnyParentNameIs(IsIn(IGNORE_VCS))
+"""The file lies inside a version-control folder (`.git`, `.svn`, `.hg`, `.bzr`)."""
+
+
+_DESIGN_NOTES = """
+# VCS_DIRS — Not the Project's Own Files
+
+## Purpose
+Part of the strict HTML exclusion. The names are `IGNORE_VCS` of
+`simplibs-path`: version control is the same for every language, so the
+library borrows the list instead of keeping its own.
+
+## Whole Names Only
+`IsIn` compares a folder name as a whole, so `.github` is not `.git`.
+"""

simplibs/extractor_html/file_rules/atomic/WRAPPED_HTML.py

@@ -0,0 +1,22 @@
+from typing import Final
+from simplibs.path.rules import HasSuffix, StemIs
+from simplibs.rules import EndsWith, Rule
+# Outers
+from ...constants import HTML_WRAPPED_STEM_ENDING, HTML_WRAPPED_SUFFIXES
+
+
+WRAPPED_HTML: Final[Rule] = HasSuffix(*HTML_WRAPPED_SUFFIXES) & StemIs(EndsWith(HTML_WRAPPED_STEM_ENDING))
+"""The file is a page wrapped in a template suffix: `index.html.j2`."""
+
+
+_DESIGN_NOTES = """
+# WRAPPED_HTML — A Page Behind a `.j2`
+
+## Purpose
+`.j2` alone could be anything, so the rule asks for both: the wrapper suffix
+and a stem that ends in `.html`. `nginx.conf.j2` fails the second half.
+
+## Why `StemIs`
+`StemIs` judges the name without its last suffix, which for `index.html.j2`
+is `index.html`.
+"""

simplibs/extractor_html/file_rules/atomic/__init__.py

@@ -0,0 +1,26 @@
+from .DEPENDENCY_DIRS import DEPENDENCY_DIRS
+from .DOT_NAME import DOT_NAME
+from .OUTPUT_DIRS import OUTPUT_DIRS
+from .VCS_DIRS import VCS_DIRS
+from .WRAPPED_HTML import WRAPPED_HTML
+
+
+_DESIGN_NOTES = """
+# atomic — The Shared Pieces of the File Rules
+
+Plain imports, no `__all__` (the package root decides what is public).
+
+## Internal Components Registry
+
+| Component         | Type | Description                                                  |
+| :---------------- | :--- | :----------------------------------------------------------- |
+| `DEPENDENCY_DIRS` | Rule | A folder above the file holds pages of other people.         |
+| `DOT_NAME`        | Rule | The name starts with a dot.                                  |
+| `OUTPUT_DIRS`     | Rule | A folder above the file holds generated pages.               |
+| `VCS_DIRS`        | Rule | A folder above the file is a version-control folder.         |
+| `WRAPPED_HTML`    | Rule | The file is `something.html` with a `.j2` wrapper.           |
+
+## Layering & Design
+- Every piece is one rule of one idea; the `html` folder only combines them.
+- The rules about folders use `AnyParentNameIs` of `simplibs-path` and `IsIn` of `simplibs-rules`.
+"""

simplibs/extractor_html/file_rules/html/HTML_BASE.py

@@ -0,0 +1,22 @@
+from typing import Final
+from simplibs.rules import Rule
+# Outers
+from ..atomic import DOT_NAME
+# Inners
+from .HTML_FILES import HTML_FILES
+
+
+HTML_BASE: Final[Rule] = HTML_FILES & ~DOT_NAME
+"""The default rule for pages worth reading: every page and template that is not hidden."""
+
+
+_DESIGN_NOTES = """
+# HTML_BASE — The Default Choice of Pages
+
+## Purpose
+Every HTML profile of this library uses it as `include`.
+
+## No Switches
+Nothing in a page needs a switch: a partial such as `_header.html` or
+`base.html` is a perfectly ordinary source file and is always taken.
+"""

simplibs/extractor_html/file_rules/html/HTML_FILES.py

@@ -0,0 +1,24 @@
+from typing import Final
+from simplibs.path.rules import HasSuffix
+from simplibs.rules import Rule
+# Outers
+from ...constants import HTML_PLAIN_SUFFIXES, HTML_TEMPLATE_SUFFIXES
+from ..atomic import WRAPPED_HTML
+
+
+HTML_FILES: Final[Rule] = HasSuffix(*HTML_PLAIN_SUFFIXES, *HTML_TEMPLATE_SUFFIXES) | WRAPPED_HTML
+"""The file is an HTML page or template (`.html .htm .xhtml .jinja .jinja2 .njk`, or `.html.j2`)."""
+
+
+_DESIGN_NOTES = """
+# HTML_FILES — Pages and Templates
+
+## Purpose
+The core of `HTML_BASE`. A suffix decides for the first two groups; the
+`.j2` wrapper needs the look behind it that `WRAPPED_HTML` makes.
+
+## Django Templates
+They are `.html` files and are taken with the pages. Which steps a file gets
+cannot depend on whether it holds template tags; the steps that know them
+leave a plain page alone.
+"""

simplibs/extractor_html/file_rules/html/HTML_PLAIN_BASE.py

@@ -0,0 +1,24 @@
+from typing import Final
+from simplibs.path.rules import HasSuffix
+from simplibs.rules import Rule
+# Outers
+from ...constants import HTML_PLAIN_SUFFIXES
+from ..atomic import DOT_NAME
+
+
+HTML_PLAIN_BASE: Final[Rule] = HasSuffix(*HTML_PLAIN_SUFFIXES) & ~DOT_NAME
+"""The pages whose suffix is `.html`, `.htm` or `.xhtml`, hidden ones left out: the files a Django project keeps its templates in."""
+
+
+_DESIGN_NOTES = """
+# HTML_PLAIN_BASE — Pages Without the Jinja Suffixes
+
+## Purpose
+The `include` of the Django profiles. A Django template is a `.html` file, and
+a file with a suffix of the Jinja family (`.jinja`, `.j2`) in a Django project
+is not read by Django's engine, so it is not read as a Django template here
+either.
+
+## Not `HTML_BASE`
+`HTML_BASE` also takes the Jinja suffixes, which the Jinja profiles need.
+"""

simplibs/extractor_html/file_rules/html/HTML_STRICT_EXCLUDE.py

@@ -0,0 +1,21 @@
+from typing import Final
+from simplibs.rules import Rule
+# Outers
+from ..atomic import DEPENDENCY_DIRS, OUTPUT_DIRS, VCS_DIRS
+
+
+HTML_STRICT_EXCLUDE: Final[Rule] = VCS_DIRS | DEPENDENCY_DIRS | OUTPUT_DIRS
+"""Everything that is a page but not the project's own source: packages of other people, generated output, version control."""
+
+
+_DESIGN_NOTES = """
+# HTML_STRICT_EXCLUDE — Only the Project's Own Pages
+
+## Purpose
+The `exclude` of the profile `html/src-clean`.
+
+## What Is Not Excluded
+There are no tests, no configuration files and no asset folders in this
+rule: a page is not tested, no tool keeps its settings in one, and
+`templates/` and `static/` are where a Django project keeps its own files.
+"""

simplibs/extractor_html/file_rules/html/__init__.py

@@ -0,0 +1,20 @@
+from .HTML_BASE import HTML_BASE
+from .HTML_FILES import HTML_FILES
+from .HTML_PLAIN_BASE import HTML_PLAIN_BASE
+from .HTML_STRICT_EXCLUDE import HTML_STRICT_EXCLUDE
+
+
+_DESIGN_NOTES = """
+# html — File Rules for Pages and Templates
+
+Plain imports, no `__all__` (the package root decides what is public).
+
+## Internal Components Registry
+
+| Component             | Type | Description                                                         |
+| :-------------------- | :--- | :------------------------------------------------------------------ |
+| `HTML_BASE`           | Rule | The default choice of pages: every one that is not hidden.          |
+| `HTML_FILES`          | Rule | Files with a page or template suffix.                               |
+| `HTML_PLAIN_BASE`     | Rule | The pages with a `.html`, `.htm` or `.xhtml` suffix, not hidden.    |
+| `HTML_STRICT_EXCLUDE` | Rule | Other people's packages, generated output, version control.         |
+"""

simplibs/extractor_html/logic/__init__.py

@@ -0,0 +1,44 @@
+from .apply_to_html_blocks import apply_to_html_blocks
+from .cleanup_html_whitespace import cleanup_html_whitespace
+from .compact_html import compact_html
+from .html_comment_spans import html_comment_spans
+from .html_outline import html_outline
+from .strip_html_comments import strip_html_comments
+from .templates import (
+    remove_template_imports,
+    strip_template_comments,
+    template_comment_spans,
+    template_outline,
+)
+
+
+_DESIGN_NOTES = """
+# logic — The Steps That Read Pages
+
+Plain imports, no `__all__` (the package root decides what is public).
+
+## Internal Components Registry
+
+| Component                  | Type     | Description                                                          |
+| :------------------------- | :------- | :------------------------------------------------------------------- |
+| `apply_to_html_blocks`     | Function | Runs the given steps on the code inside `script` and `style`.        |
+| `cleanup_html_whitespace`  | Function | Trims line ends, reduces blank lines to one; spares `pre` and the like. |
+| `compact_html`             | Function | Drops every blank line; spares `pre` and the like.                   |
+| `html_comment_spans`       | Function | The spans of the HTML comments (by the scanner).                     |
+| `html_outline`             | Function | The landmarks, headings and ids of a page, indented by nesting.      |
+| `remove_template_imports`  | Function | Removes `{% load %}`, `{% import %}` and `{% from %}`.               |
+| `strip_html_comments`      | Function | Removes the HTML comments.                                           |
+| `strip_template_comments`  | Function | Removes the template comments.                                       |
+| `template_comment_spans`   | Function | The spans of the template comments.                                  |
+| `template_outline`         | Function | The skeleton of a template: extends, include, blocks.                |
+| `templates`                | Package  | The steps for Django and Jinja templates.                            |
+| `_helpers`                 | Private  | The scanners and the parts they are made of.                         |
+
+## Layering & Design
+- A page is read by a scanner that knows tags, quoted values and raw text; it is not a parser and cannot fail.
+- A template is read first, by its own scanner, and the page is then read with the template constructs blanked.
+- A file that looks damaged is left as it is and reported by a warning, so the extractor calls it `partial`.
+- Each step is a function of a text and returns a text; the `Action` wrappers come in `actions`.
+- The steps import only `constants` and `annotations`; the ranges they find are `simplibs.text.Span`.
+- The steps for the code inside a page are given as parameters: the library does not depend on the JavaScript or CSS add-on.
+"""

simplibs/extractor_html/logic/_helpers/HtmlBlock.py

@@ -0,0 +1,31 @@
+from dataclasses import dataclass
+from simplibs.text import Span
+
+
+@dataclass(frozen=True, slots=True)
+class HtmlBlock:
+    """The content of a `script` or `style` element.
+
+    Attributes:
+        kind: `"script"` or `"style"`.
+        span: The range of the content, between the end of the start tag and the beginning of the end tag.
+        language: `"javascript"` or `"css"` when the `type` says the content is code, otherwise `None`
+            (JSON, a framework template, a shader...).
+    """
+
+    kind: str
+    span: Span
+    language: str | None
+
+
+_DESIGN_NOTES = """
+# HtmlBlock — A Piece of Another Language Inside a Page
+
+## Purpose
+The block step of the next batch cleans the JavaScript in `<script>` and the
+CSS in `<style>` with the steps of the other add-ons. It needs to know where
+the content is and whether it is code at all.
+
+## Only Non-Blank Content
+An element with nothing in it (`<script src="a.js"></script>`) is not a block.
+"""

simplibs/extractor_html/logic/_helpers/HtmlOutlineBuilder.py

@@ -0,0 +1,189 @@
+from dataclasses import dataclass
+from simplibs.text import Span
+# Outers
+from ...constants import (
+    HTML_ANY_TAG_PATTERN,
+    HTML_HEADING_ELEMENTS,
+    HTML_ID_ATTRIBUTE_PATTERN,
+    HTML_IMPLIED_END,
+    HTML_OUTLINE_ELEMENTS,
+    HTML_OUTLINE_TEXT_LIMIT,
+    HTML_TEXT_ELEMENTS,
+    HTML_VOID_ELEMENTS,
+)
+
+
+@dataclass(slots=True)
+class _Open:
+    """An element that has been opened and not closed yet."""
+
+    name: str
+    listed: bool
+    line: int | None
+    content_start: int
+
+
+class HtmlOutlineBuilder:
+    """Follows the start tags and end tags of a page and writes the outline.
+
+    Args:
+        text: The source.
+        max_depth: The number of levels to list, or `None` for all of them.
+    """
+
+    __slots__ = ("_text", "_max_depth", "_stack", "_depth", "_lines")
+
+    def __init__(self, text: str, max_depth: int | None) -> None:
+        self._text = text
+        self._max_depth = max_depth
+        self._stack: list[_Open] = []
+        self._depth = 0
+        self._lines: list[list] = []
+
+    def start_tag(self, name: str, span: Span) -> None:
+        """A start tag: close what it implicitly closes, list the element if it belongs in the outline, open it."""
+
+        # 1. An end tag that may be left out is implied by the start tag that follows
+        self._close_implied(name)
+
+        # 2. Is the element listed, and under what name
+        raw = self._text[span[0]:span[1]]
+        element_id = self._find_id(raw)
+        listed = name in HTML_OUTLINE_ELEMENTS or name in HTML_TEXT_ELEMENTS or element_id is not None
+
+        # 3. A listed element writes its line now, unless it is deeper than wanted
+        line: int | None = None
+        if listed and (self._max_depth is None or self._depth < self._max_depth):
+            label = name if element_id is None else f"{name}#{element_id}"
+            self._lines.append([self._depth, label, ""])
+            line = len(self._lines) - 1
+
+        # 4. An element that is complete in its tag opens nothing
+        if name in HTML_VOID_ELEMENTS or raw.endswith("/>"):
+            return
+
+        # 5. Anything else stays open until its end tag (or until a later tag implies it)
+        self._stack.append(_Open(name, listed, line, span[1]))
+        if listed:
+            self._depth += 1
+
+    def end_tag(self, name: str, span: Span) -> None:
+        """An end tag: close the nearest open element of that name, and everything opened inside it."""
+
+        # 1. The nearest open element with this name; an end tag with none is ignored
+        for index in range(len(self._stack) - 1, -1, -1):
+            if self._stack[index].name == name:
+                self._pop_to(index, closing_start=span[0])
+                return
+
+    def lines(self) -> list[str]:
+        """The finished lines of the outline."""
+
+        # 1. Indented by depth; the text of a heading in quotes behind its label
+        written = []
+        for depth, label, text in self._lines:
+            written.append("  " * depth + label + (f' "{text}"' if text else ""))
+
+        return written
+
+    def _close_implied(self, name: str) -> None:
+        """Pop the open elements that a start tag of this name closes by itself."""
+
+        # 1. A heading directly inside a heading ends it: `<h1>a<h2>b` has two headings side by side
+        if name in HTML_HEADING_ELEMENTS and self._stack and self._stack[-1].name in HTML_HEADING_ELEMENTS:
+            self._pop_to(len(self._stack) - 1, closing_start=None)
+            return
+
+        # 2. Only elements whose end tag may be left out have a rule
+        rule = HTML_IMPLIED_END.get(name)
+        if rule is None:
+            return
+        closes, bounds = rule
+
+        # 3. From the top down: a bound stops the search, the outermost element that is closed is the target
+        target: int | None = None
+        for index in range(len(self._stack) - 1, -1, -1):
+            open_name = self._stack[index].name
+            if open_name in bounds:
+                break
+            if open_name in closes:
+                target = index
+
+        if target is not None:
+            self._pop_to(target, closing_start=None)
+
+    def _pop_to(self, index: int, *, closing_start: int | None) -> None:
+        """Pop the open elements down to and including the one at `index`; it is the one that an end tag closes."""
+
+        # 1. The elements above it were never closed; their text, if any, is unknown
+        while len(self._stack) > index:
+            entry = self._stack.pop()
+            if entry.listed:
+                self._depth -= 1
+
+            # 2. The element itself gets the text between its tags, when it is a title or a heading
+            is_closed_here = len(self._stack) == index and closing_start is not None
+            if is_closed_here and entry.line is not None and entry.name in HTML_TEXT_ELEMENTS:
+                self._lines[entry.line][2] = self._clean_text(self._text[entry.content_start:closing_start])
+
+    @staticmethod
+    def _find_id(raw: str) -> str | None:
+        """The value of the `id` of a tag, with its blanks collapsed; `None` if there is none."""
+
+        # 1. The first of the three groups that matched
+        match = HTML_ID_ATTRIBUTE_PATTERN.search(raw)
+        if match is None:
+            return None
+        value = " ".join(next(group for group in match.groups() if group is not None).split())
+
+        return value or None
+
+    @staticmethod
+    def _clean_text(raw: str) -> str:
+        """The text of a heading: no tags, blanks collapsed, cut at the limit."""
+
+        # 1. The tags inside it go, and the blanks between words become single spaces
+        text = " ".join(HTML_ANY_TAG_PATTERN.sub("", raw).split())
+
+        # 2. A text that is too long is cut and marked
+        if len(text) > HTML_OUTLINE_TEXT_LIMIT:
+            return text[:HTML_OUTLINE_TEXT_LIMIT - 1].rstrip() + "…"
+
+        return text
+
+
+_DESIGN_NOTES = """
+# HtmlOutlineBuilder — A Little Nesting, Just Enough for the Outline
+
+## Purpose
+The outline shows which parts of a page lie inside which. That needs a stack of
+the elements that are open, not a tree of the whole page, and the builder is
+that stack with the lines it has written.
+
+## What It Lists
+An element is listed when it is one of `HTML_OUTLINE_ELEMENTS`, a title or a
+heading, or has an `id`. Its depth is the number of listed elements around it,
+so a `div` in a `div` in a `section` costs no indentation.
+
+## End Tags That May Be Left Out
+`<li>a<li>b` and `<td>a<td>b` are valid. A start tag with an entry in
+`HTML_IMPLIED_END` closes the open elements it implies, up to the bound of its
+entry.
+
+## Headings
+A heading that follows a heading directly closes it (`<h1>a<h2>b`), because a
+heading cannot contain another.
+
+## The Text of a Heading
+It is what lies between the start tag and the end tag, without tags, with blanks
+collapsed. A heading that is closed only implicitly has no text: the builder
+does not guess where it ended.
+
+## Stray End Tags
+An end tag with no open element of that name is ignored, as a browser does.
+
+## Known Limits
+`p` and the other elements whose end tag a following element implies are not
+followed. A stray `</div>` closes the nearest open `div`, which may not be the
+one the author meant.
+"""

simplibs/extractor_html/logic/_helpers/HtmlScan.py

@@ -0,0 +1,50 @@
+from dataclasses import dataclass
+from simplibs.text import Span
+# Inners
+from .HtmlBlock import HtmlBlock
+from .HtmlTag import HtmlTag
+
+
+@dataclass(frozen=True, slots=True)
+class HtmlScan:
+    """What the scanner found in a page.
+
+    Attributes:
+        comments: The spans of every HTML comment, with the blanks in front of it where they can go.
+        conditional: The comments (a subset of `comments`) that are Internet Explorer conditional comments.
+        template_comments: The spans of the template comments (`{# ... #}`, `{% comment %}`); empty
+            when no engine was given.
+        verbatim: The spans of the template blocks that are plain text (`{% verbatim %}`, `{% raw %}`).
+        blocks: The `script` and `style` elements with content.
+        preserved: The spans where whitespace has a meaning: the content of `pre` and `textarea`.
+        tags: Every start tag and end tag, in the order of the page; empty unless they were asked for.
+        problems: What looked damaged, one sentence each.
+    """
+
+    comments: tuple[Span, ...]
+    conditional: tuple[Span, ...]
+    template_comments: tuple[Span, ...]
+    verbatim: tuple[Span, ...]
+    blocks: tuple[HtmlBlock, ...]
+    preserved: tuple[Span, ...]
+    tags: tuple[HtmlTag, ...]
+    problems: tuple[str, ...]
+
+
+_DESIGN_NOTES = """
+# HtmlScan — The Result of One Pass
+
+## Purpose
+A page is read once and the answers to all the questions of the steps come
+together: where the comments are, which of them IE needs, where the template
+comments and the raw template blocks are, which pieces are code of another
+language, where whitespace matters and, when asked, which tags there are.
+
+## Why a Frozen Record
+The lists belong together and are not changed after the scan.
+
+## `problems`
+The scanner never refuses a text. What it notices on the way (a comment, a tag
+or a `script` that is never closed) it writes down, and the steps turn it into
+a warning, so the extractor can call the file `partial`.
+"""

simplibs/extractor_html/logic/_helpers/HtmlTag.py

@@ -0,0 +1,32 @@
+from dataclasses import dataclass
+from simplibs.text import Span
+
+
+@dataclass(frozen=True, slots=True)
+class HtmlTag:
+    """One start tag or end tag of a page.
+
+    Attributes:
+        name: The name of the element, in lower case (`div`).
+        closing: `True` for an end tag (`</div>`), `False` for a start tag.
+        span: The range of the whole tag, from `<` to `>`.
+    """
+
+    name: str
+    closing: bool
+    span: Span
+
+
+_DESIGN_NOTES = """
+# HtmlTag — A Tag, Seen From Outside
+
+## Purpose
+The outline follows the nesting of the elements, and for that it needs the tags
+in the order of the page: what each is called, whether it opens or closes and
+where it is. Its attributes are read from the text of the tag when they are
+wanted.
+
+## Recorded Only on Request
+Most steps never look at a tag, and a page has a lot of them; the scanner
+records them only when it is asked to (`collect_tags`).
+"""

simplibs/extractor_html/logic/_helpers/Ranges.py

@@ -0,0 +1,48 @@
+from bisect import bisect_left
+from collections.abc import Iterable
+from simplibs.text import Span
+
+
+class Ranges:
+    """A set of ranges of a text that can be asked whether a stretch of the text touches one of them.
+
+    Args:
+        spans: The ranges; they may overlap and may come in any order.
+    """
+
+    __slots__ = ("_spans", "_starts")
+
+    def __init__(self, spans: Iterable[Span]) -> None:
+
+        # 1. Sorted, and ranges that overlap or touch merged into one, so a question costs a binary search
+        merged: list[Span] = []
+        for start, end in sorted(spans):
+            if merged and start <= merged[-1][1]:
+                merged[-1] = (merged[-1][0], max(end, merged[-1][1]))
+            else:
+                merged.append((start, end))
+        self._spans = merged
+        self._starts = [start for start, _ in merged]
+
+    def touches(self, start: int, end: int) -> bool:
+        """Say whether the stretch `start:end` of the text has something in common with one of the ranges."""
+
+        # 1. The last range that begins before the stretch ends has the largest end of all that can reach it
+        index = bisect_left(self._starts, end) - 1
+        return index >= 0 and self._spans[index][1] > start
+
+
+_DESIGN_NOTES = """
+# Ranges — Where Whitespace Must Not Be Touched
+
+## Purpose
+The whitespace steps work line by line and must leave alone every line that
+has a part in a range where whitespace has a meaning (the content of `pre`
+and `textarea`, a `script`, a `style`, a verbatim block). They ask this object
+once for each line.
+
+## Why Merged
+A `{% verbatim %}` inside a `<pre>` gives two ranges, one inside the other.
+Merged, the ranges are sorted and apart, so the last one that begins before a
+stretch ends is the only one that can reach it.
+"""

simplibs/extractor_html/logic/_helpers/TemplateScan.py

@@ -0,0 +1,43 @@
+from dataclasses import dataclass
+from simplibs.text import Span
+# Inners
+from .TemplateTag import TemplateTag
+
+
+@dataclass(frozen=True, slots=True)
+class TemplateScan:
+    """What the scanner found of the template language in a page.
+
+    Attributes:
+        comments: The spans of the template comments, with the blanks in front of them where they can go.
+        verbatim: The spans of the blocks that are plain text for the template engine.
+        tags: The `{% ... %}` tags, in the order of the page.
+        masked: The text of the same length in which every template construct (variable, tag, comment,
+            verbatim block) is replaced by blanks, so that the page can be read as HTML.
+        problems: What looked damaged, one sentence each.
+    """
+
+    comments: tuple[Span, ...]
+    verbatim: tuple[Span, ...]
+    tags: tuple[TemplateTag, ...]
+    masked: str
+    problems: tuple[str, ...]
+
+
+_DESIGN_NOTES = """
+# TemplateScan — The Template Layer of a Page
+
+## Purpose
+`{{ x }}`, `{% if a > b %}` and `{# ... #}` are not HTML, and they may hold
+characters that break the reading of HTML (`>` in a condition, quotes inside
+quotes). So they are read first, by their own rules, and what is left
+(`masked`) is the page as a browser would see it.
+
+## Same Length
+The masked text has the length and the lines of the original, so every
+offset the HTML scan reports is valid in both.
+
+## What the Steps Take From It
+The comment step takes `comments`, the import step takes `tags`, and the HTML
+scan takes `masked`.
+"""

simplibs/extractor_html/logic/_helpers/TemplateTag.py

@@ -0,0 +1,33 @@
+from dataclasses import dataclass
+from simplibs.text import Span
+
+
+@dataclass(frozen=True, slots=True)
+class TemplateTag:
+    """One `{% ... %}` tag of a template.
+
+    Attributes:
+        name: The name of the tag (`block` of `{% block content %}`).
+        span: The range of the whole tag, from `{%` to `%}`.
+        arguments: What stands behind the name, without the blanks and without the whitespace
+            control (`content` of `{% block content %}`; empty for `{% endblock %}`).
+    """
+
+    name: str
+    span: Span
+    arguments: str
+
+
+_DESIGN_NOTES = """
+# TemplateTag — A Tag, Seen From Outside
+
+## Purpose
+The steps that work with the structure of a template (the imports, and later
+the overview of `extends`, `block` and `include`) need three things about a
+tag: what it is called, where it is and what is written in it.
+
+## Which Tags Are Recorded
+Every tag that is not the opening of a block running to an end tag
+(`comment`, `verbatim`, `raw`). What stands inside such a block is not a tag
+of the template: it is a comment or a text.
+"""