simplibs-extractor-html 0.1.0__py3-none-any.whl

simplibs/extractor_html/__init__.py

@@ -0,0 +1,82 @@
+from .annotations import TemplateEngine
+from .logic import (
+    apply_to_html_blocks,
+    cleanup_html_whitespace,
+    compact_html,
+    html_comment_spans,
+    html_outline,
+    remove_template_imports,
+    strip_html_comments,
+    strip_template_comments,
+    template_comment_spans,
+    template_outline,
+)
+from .plugin import register_all
+
+
+__all__ = [
+    # annotations
+    "TemplateEngine",
+    # steps for pages
+    "apply_to_html_blocks",
+    "cleanup_html_whitespace",
+    "compact_html",
+    "html_comment_spans",
+    "html_outline",
+    "strip_html_comments",
+    # steps for templates
+    "remove_template_imports",
+    "strip_template_comments",
+    "template_comment_spans",
+    "template_outline",
+    # add-on of the extractor
+    "register_all",
+]
+
+
+_DESIGN_NOTES = """
+# simplibs-extractor-html — Package Root
+
+## Purpose
+The add-on that teaches `simplibs-extractor` about pages (HTML) and about the
+templates of Django and Jinja: file rules for such projects, the steps that
+clean the source (comments, imports, whitespace, the code inside the page) or
+reduce it to an overview, and thirteen ready profiles. Install it and the
+profiles are there:
+
+    simplibs-extractor html django-clean my_project
+
+## What the Root Exports
+The steps as plain functions (`str -> str`, except the spans), the annotation
+of their `engine` parameter, and `register_all` (the target of the entry point
+`simplibs.extractor.plugins`):
+
+    from simplibs.extractor_html import strip_html_comments, html_outline, TemplateEngine
+
+## What the Root Deliberately Does NOT Export
+Four domain folders with their own `__all__`, imported through their domain:
+
+    from simplibs.extractor_html.actions import StripHtmlComments
+    from simplibs.extractor_html.file_rules import HTML_BASE
+    from simplibs.extractor_html.presets import html_clean_steps
+    from simplibs.extractor_html.profiles import DJANGO_CLEAN
+
+and the internals: `_helpers/` inside `logic`.
+
+## Spans
+The steps find ranges of characters as `simplibs.text.Span`. The type is not
+defined here, it is taken from `simplibs-text`.
+
+## The Other Add-Ons
+The presets that clean the code inside a page use `simplibs-extractor-javascript`
+and `simplibs-extractor-css`. The steps themselves (`apply_to_html_blocks`) take
+the steps for the code as parameters and know nothing of either.
+
+## Layering
+`constants` ← `annotations` ← `file_rules` ← `logic` ← `actions` ← `presets` ← `profiles` ← `plugin`.
+Each layer imports only from the layers on its left; the root imports from the ones it exports.
+
+## `__all__`
+Defined here and in the domain folders (`actions`, `file_rules`, `presets`,
+`profiles`) only; the layers inside them use plain imports.
+"""

simplibs/extractor_html/actions/ApplyToHtmlBlocks.py

@@ -0,0 +1,29 @@
+from simplibs.actions import create_action
+# Outers
+from ..logic import apply_to_html_blocks
+
+
+ApplyToHtmlBlocks = create_action(
+    apply_to_html_blocks,
+    main_param="text",
+    class_name="ApplyToHtmlBlocks",
+)
+"""Action wrapping `apply_to_html_blocks` — runs the given steps on the code inside `script` and `style` of the page flowing in."""
+
+
+_DESIGN_NOTES = """
+# ApplyToHtmlBlocks — Action Over `apply_to_html_blocks`
+
+## Purpose
+The step that gives the code inside a page to the steps of its own language:
+`ApplyToHtmlBlocks(script=JS_CLEAN, style=CSS_PLAIN_CLEAN)`.
+
+## Shape
+Main input: `text`. Options: `script` and `style` (each a step, an action or a
+whole pipeline that turns the text of a block into the new text, or `None`) and
+`engine`. Text in, text out.
+
+## Steps Inside an Action
+An action is callable, so it is a step for this one: the pipelines of the other
+add-ons are passed as they are.
+"""

simplibs/extractor_html/actions/CleanupHtmlWhitespace.py

@@ -0,0 +1,22 @@
+from simplibs.actions import create_action
+# Outers
+from ..logic import cleanup_html_whitespace
+
+
+CleanupHtmlWhitespace = create_action(
+    cleanup_html_whitespace,
+    main_param="text",
+    class_name="CleanupHtmlWhitespace",
+)
+"""Action wrapping `cleanup_html_whitespace` — tidies the whitespace of the page flowing in and spares `pre` and the like."""
+
+
+_DESIGN_NOTES = """
+# CleanupHtmlWhitespace — Action Over `cleanup_html_whitespace`
+
+## Purpose
+The gentle tidying as a link of a pipeline: `CleanupHtmlWhitespace(engine="jinja")`.
+
+## Shape
+Main input: `text`. Option: `engine`. Text in, text out.
+"""

simplibs/extractor_html/actions/CompactHtml.py

@@ -0,0 +1,22 @@
+from simplibs.actions import create_action
+# Outers
+from ..logic import compact_html
+
+
+CompactHtml = create_action(
+    compact_html,
+    main_param="text",
+    class_name="CompactHtml",
+)
+"""Action wrapping `compact_html` — drops the blank lines of the page flowing in and spares `pre` and the like."""
+
+
+_DESIGN_NOTES = """
+# CompactHtml — Action Over `compact_html`
+
+## Purpose
+The thorough tidying as a link of a pipeline: `CompactHtml(engine="django")`.
+
+## Shape
+Main input: `text`. Option: `engine`. Text in, text out.
+"""

simplibs/extractor_html/actions/HtmlOutline.py

@@ -0,0 +1,23 @@
+from simplibs.actions import create_action
+# Outers
+from ..logic import html_outline
+
+
+HtmlOutline = create_action(
+    html_outline,
+    main_param="text",
+    class_name="HtmlOutline",
+)
+"""Action wrapping `html_outline` — turns the page flowing in into the list of its landmarks, headings and ids."""
+
+
+_DESIGN_NOTES = """
+# HtmlOutline — Action Over `html_outline`
+
+## Purpose
+The overview step as a link of a pipeline: `HtmlOutline(engine="django", max_depth=3)`.
+
+## Shape
+Main input: `text`. Options: `engine` and `max_depth`. Text in, text out (one
+element per line, indented by depth).
+"""

simplibs/extractor_html/actions/RemoveTemplateImports.py

@@ -0,0 +1,22 @@
+from simplibs.actions import create_action
+# Outers
+from ..logic import remove_template_imports
+
+
+RemoveTemplateImports = create_action(
+    remove_template_imports,
+    main_param="text",
+    class_name="RemoveTemplateImports",
+)
+"""Action wrapping `remove_template_imports` — takes `{% load %}`, `{% import %}` and `{% from %}` out of the template flowing in."""
+
+
+_DESIGN_NOTES = """
+# RemoveTemplateImports — Action Over `remove_template_imports`
+
+## Purpose
+The import step as a link of a pipeline: `RemoveTemplateImports(engine="jinja")`.
+
+## Shape
+Main input: `text`. Option: `engine`, required. Text in, text out.
+"""

simplibs/extractor_html/actions/StripHtmlComments.py

@@ -0,0 +1,23 @@
+from simplibs.actions import create_action
+# Outers
+from ..logic import strip_html_comments
+
+
+StripHtmlComments = create_action(
+    strip_html_comments,
+    main_param="text",
+    class_name="StripHtmlComments",
+)
+"""Action wrapping `strip_html_comments` — takes the HTML comments out of the page flowing in."""
+
+
+_DESIGN_NOTES = """
+# StripHtmlComments — Action Over `strip_html_comments`
+
+## Purpose
+The comment step as a link of a pipeline: `StripHtmlComments(engine="django", keep_license=True)`.
+
+## Shape
+Main input: `text`. Options: `engine` (`"django"`, `"jinja"` or `None`),
+`keep_conditional` and `keep_license`. Text in, text out.
+"""

simplibs/extractor_html/actions/StripTemplateComments.py

@@ -0,0 +1,27 @@
+from simplibs.actions import create_action
+# Outers
+from ..logic import strip_template_comments
+
+
+StripTemplateComments = create_action(
+    strip_template_comments,
+    main_param="text",
+    class_name="StripTemplateComments",
+)
+"""Action wrapping `strip_template_comments` — takes `{# ... #}` and `{% comment %}` out of the template flowing in."""
+
+
+_DESIGN_NOTES = """
+# StripTemplateComments — Action Over `strip_template_comments`
+
+## Purpose
+The template comment step as a link of a pipeline: `StripTemplateComments(engine="django")`.
+
+## Shape
+Main input: `text`. Option: `engine`, **required** (`"django"` or `"jinja"`):
+building the action without it is a `TypeError`. Text in, text out.
+
+## One Action, Not Two
+Django and Jinja differ only in the option, so there is no class for each of
+them; the profiles build the two pipelines from the same action.
+"""

simplibs/extractor_html/actions/TemplateOutline.py

@@ -0,0 +1,23 @@
+from simplibs.actions import create_action
+# Outers
+from ..logic import template_outline
+
+
+TemplateOutline = create_action(
+    template_outline,
+    main_param="text",
+    class_name="TemplateOutline",
+)
+"""Action wrapping `template_outline` — turns the template flowing in into the list of its `extends`, `include` and blocks."""
+
+
+_DESIGN_NOTES = """
+# TemplateOutline — Action Over `template_outline`
+
+## Purpose
+The skeleton step as a link of a pipeline: `TemplateOutline(engine="django", max_depth=2)`.
+
+## Shape
+Main input: `text`. Options: `engine` (required) and `max_depth`. Text in, text
+out (one tag per line, indented by the depth of its blocks).
+"""

simplibs/extractor_html/actions/__init__.py

@@ -0,0 +1,57 @@
+from .ApplyToHtmlBlocks import ApplyToHtmlBlocks
+from .CleanupHtmlWhitespace import CleanupHtmlWhitespace
+from .CompactHtml import CompactHtml
+from .HtmlOutline import HtmlOutline
+from .RemoveTemplateImports import RemoveTemplateImports
+from .StripHtmlComments import StripHtmlComments
+from .StripTemplateComments import StripTemplateComments
+from .TemplateOutline import TemplateOutline
+
+
+__all__ = [
+    "ApplyToHtmlBlocks",
+    "CleanupHtmlWhitespace",
+    "CompactHtml",
+    "HtmlOutline",
+    "RemoveTemplateImports",
+    "StripHtmlComments",
+    "StripTemplateComments",
+    "TemplateOutline",
+]
+
+
+_DESIGN_NOTES = """
+# simplibs.extractor_html.actions — The Steps as Pipeline Actions
+
+## Purpose
+Every text step of `logic/` as an `Action` of `simplibs-actions`, so it can be
+chained with `>>` and used as the `steps` of an extractor profile.
+
+    from simplibs.extractor_html.actions import StripHtmlComments, CompactHtml
+    steps = StripHtmlComments() >> CompactHtml()
+
+## Domain Folder
+`actions/` has its own `__all__` and is not re-exported from the package root:
+a function and an action are different kinds of thing and stay visibly
+different at the import line.
+
+## Thin Wrappers
+Each action is `create_action(function, main_param="text", ...)`.
+
+## What Has No Action
+`html_comment_spans` and `template_comment_spans` return spans, not a text:
+they are not links of a pipeline that carries text.
+
+## Internal Components Registry
+
+| Component               | Type   | Description                                       |
+| :---------------------- | :----- | :------------------------------------------------ |
+| `ApplyToHtmlBlocks`     | Action | Over `apply_to_html_blocks`.                      |
+| `CleanupHtmlWhitespace` | Action | Over `cleanup_html_whitespace`.                   |
+| `CompactHtml`           | Action | Over `compact_html`.                              |
+| `HtmlOutline`           | Action | Over `html_outline`.                              |
+| `RemoveTemplateImports` | Action | Over `remove_template_imports`.                   |
+| `StripHtmlComments`     | Action | Over `strip_html_comments`.                       |
+| `StripTemplateComments` | Action | Over `strip_template_comments`.                   |
+| `TemplateOutline`       | Action | Over `template_outline`.                          |
+"""

simplibs/extractor_html/annotations/TemplateEngine.py

@@ -0,0 +1,26 @@
+from typing import Literal
+
+
+TemplateEngine = Literal["django", "jinja"]
+"""The template language a page is written in: `"django"` or `"jinja"` (Nunjucks counts as Jinja)."""
+
+
+_DESIGN_NOTES = """
+# TemplateEngine — Which Rules the Curly Braces Follow
+
+## Purpose
+Django and Jinja look alike and differ in details that matter to a scanner:
+
+* Django keeps `{{ ... }}`, `{% ... %}` and `{# ... #}` on one line; Jinja lets
+  them run over several.
+* Django has `{% comment %}...{% endcomment %}` and `{% verbatim %}`; Jinja has
+  `{% raw %}` and no comment tag.
+
+## Why a `Literal`
+A typo (`"jinja2"`, `"Django"`) fails at the signature, and an editor
+completes both values. Nunjucks, which uses the same syntax as Jinja, is read
+as `"jinja"`.
+
+## No "none"
+A page that is not a template needs no engine: the parameter is `None`.
+"""

simplibs/extractor_html/annotations/__init__.py

@@ -0,0 +1,14 @@
+from .TemplateEngine import TemplateEngine
+
+
+_DESIGN_NOTES = """
+# annotations — Types for the Signatures of simplibs-extractor-html
+
+Plain imports, no `__all__` (the package root decides what is public).
+
+## Internal Components Registry
+
+| Component        | Type    | Description                                     |
+| :--------------- | :------ | :---------------------------------------------- |
+| `TemplateEngine` | Literal | `"django"` or `"jinja"`.                        |
+"""

simplibs/extractor_html/constants/HTML_ANY_TAG_PATTERN.py

@@ -0,0 +1,16 @@
+import re
+from typing import Final
+
+
+HTML_ANY_TAG_PATTERN: Final[re.Pattern[str]] = re.compile(r"<[^>]*>")
+"""Anything that looks like a tag: from `<` to the next `>`."""
+
+
+_DESIGN_NOTES = """
+# HTML_ANY_TAG_PATTERN — Text Without Its Markup
+
+## Purpose
+The text of a heading (`<h1>Hello <em>world</em></h1>`) is what remains when
+the tags inside it are taken out. For this, the exact shape of a tag does not
+matter.
+"""

simplibs/extractor_html/constants/HTML_BLOCK_LANGUAGE_BY_TYPE.py

@@ -0,0 +1,41 @@
+from typing import Final
+
+
+HTML_BLOCK_LANGUAGE_BY_TYPE: Final[dict[str, dict[str, str]]] = {
+    "script": {
+        "": "javascript",
+        "module": "javascript",
+        "text/javascript": "javascript",
+        "application/javascript": "javascript",
+        "text/ecmascript": "javascript",
+        "application/ecmascript": "javascript",
+        "text/jsx": "javascript",
+        "text/babel": "javascript",
+    },
+    "style": {
+        "": "css",
+        "text/css": "css",
+    },
+}
+"""For `script` and `style`, which values of `type` mean that the content is code, and in what language."""
+
+
+_DESIGN_NOTES = """
+# HTML_BLOCK_LANGUAGE_BY_TYPE — Code or Something Else
+
+## Purpose
+A `script` may hold JavaScript, JSON, a template for a framework or a shader.
+Only the first kind can be cleaned by the JavaScript steps, so the scanner
+says what a block is, and the block step touches only code.
+
+## The Empty Type
+No `type` at all means JavaScript for a `script` and CSS for a `style`.
+
+## `text/jsx` and `text/babel`
+Pages that compile JSX in the browser put it there. The JavaScript add-on
+reads JSX.
+
+## What Is Not Here
+`application/json`, `application/ld+json`, `importmap`, `text/template` and
+anything unknown: such a block is data, and it stays as it is.
+"""

simplibs/extractor_html/constants/HTML_COMMENT_END_PATTERN.py

@@ -0,0 +1,19 @@
+import re
+from typing import Final
+
+
+HTML_COMMENT_END_PATTERN: Final[re.Pattern[str]] = re.compile(r"--!?>")
+"""How a comment ends: `-->`, or `--!>`, which browsers accept as well."""
+
+
+_DESIGN_NOTES = """
+# HTML_COMMENT_END_PATTERN — The End of a Comment
+
+## Purpose
+A comment ends at `-->`. The HTML specification also closes it at `--!>`
+(an error, but one every browser forgives), so the scanner does the same.
+
+## The Shortest Comments
+`<!-->` and `<!--->` end at once, by the same specification. The scanner
+checks them before it looks for this pattern.
+"""

simplibs/extractor_html/constants/HTML_CONDITIONAL_PATTERN.py

@@ -0,0 +1,23 @@
+import re
+from typing import Final
+
+
+HTML_CONDITIONAL_PATTERN: Final[re.Pattern[str]] = re.compile(
+    r"<!--\s*(?:\[if\b|<!\[endif\])",
+    re.IGNORECASE,
+)
+"""The beginning of an Internet Explorer conditional comment: `<!--[if IE]>` or `<!--<![endif]-->`."""
+
+
+_DESIGN_NOTES = """
+# HTML_CONDITIONAL_PATTERN — Comments That Are Not Comments
+
+## Purpose
+Old Internet Explorer read `<!--[if lt IE 9]> ... <![endif]-->` as code, and
+every other browser as a comment. Such a comment holds real tags (a script
+for IE), so by default the comment step keeps it.
+
+## Both Forms
+The "hidden" form begins `<!--[if`; the "revealed" form ends in
+`<!--<![endif]-->`. Both begin like this.
+"""

simplibs/extractor_html/constants/HTML_DEPENDENCY_DIR_NAMES.py

@@ -0,0 +1,18 @@
+from typing import Final
+
+
+HTML_DEPENDENCY_DIR_NAMES: Final[tuple[str, ...]] = ("node_modules", "bower_components", "vendor")
+"""Names of folders that hold pages written by other people."""
+
+
+_DESIGN_NOTES = """
+# HTML_DEPENDENCY_DIR_NAMES — Other People's Pages
+
+## Purpose
+The names behind `DEPENDENCY_DIRS`. Packages bring their own examples,
+documentation and templates, and none of them is the project's source.
+
+## Own List
+Each add-on decides for its own language what counts as dependencies; the
+library does not borrow the list of another add-on.
+"""

simplibs/extractor_html/constants/HTML_EXTRA_LANGUAGE_BY_SUFFIX.py

@@ -0,0 +1,31 @@
+from typing import Final
+
+
+HTML_EXTRA_LANGUAGE_BY_SUFFIX: Final[dict[str, str]] = {
+    ".htm": "html",
+    ".xhtml": "html",
+    ".jinja": "jinja2",
+    ".jinja2": "jinja2",
+    ".njk": "jinja2",
+    ".j2": "jinja2",
+}
+"""The code-fence language of the suffixes the extractor does not know on its own."""
+
+
+_DESIGN_NOTES = """
+# HTML_EXTRA_LANGUAGE_BY_SUFFIX — Names for the Fence
+
+## Purpose
+The markdown output writes the language after the opening fence, so a viewer
+can colour the code. The extractor knows `.html` itself; the other suffixes are
+what this add-on brings, and `register_all` tells the extractor about them.
+
+## `jinja2`
+The name that the common highlighters know for the syntax of Jinja. Nunjucks
+(`.njk`) is close enough to it to be coloured the same way, and the `.j2`
+wrapper is named for the template language, as a suffix cannot tell which
+language is inside.
+
+## Only What Is Missing
+`.html` is not listed: registering it again would change nothing.
+"""

simplibs/extractor_html/constants/HTML_HEADING_ELEMENTS.py

@@ -0,0 +1,15 @@
+from typing import Final
+
+
+HTML_HEADING_ELEMENTS: Final[tuple[str, ...]] = ("h1", "h2", "h3", "h4", "h5", "h6")
+"""The six heading elements."""
+
+
+_DESIGN_NOTES = """
+# HTML_HEADING_ELEMENTS — The Six Headings
+
+## Purpose
+A heading is closed implicitly by another heading that follows it:
+`<h1>Title<h2>Sub</h2>` has two headings side by side, not one inside the
+other. The outline needs to know which names are headings to follow that rule.
+"""

simplibs/extractor_html/constants/HTML_ID_ATTRIBUTE_PATTERN.py

@@ -0,0 +1,21 @@
+import re
+from typing import Final
+
+
+HTML_ID_ATTRIBUTE_PATTERN: Final[re.Pattern[str]] = re.compile(
+    r"""\sid\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>]+))""",
+    re.IGNORECASE,
+)
+"""The `id` attribute of a tag; its value is in one of the three groups (double quotes, single quotes, none)."""
+
+
+_DESIGN_NOTES = """
+# HTML_ID_ATTRIBUTE_PATTERN — What the Part Is Called
+
+## Purpose
+The outline lists every element that has an `id`.
+
+## The Blank in Front
+A blank must come before `id`, so `data-id="x"` and `aria-labelledby` are not
+mistaken for it.
+"""

simplibs/extractor_html/constants/HTML_IMPLIED_END.py

@@ -0,0 +1,34 @@
+from typing import Final
+
+
+HTML_IMPLIED_END: Final[dict[str, tuple[tuple[str, ...], tuple[str, ...]]]] = {
+    "li": (("li",), ("ul", "ol", "menu")),
+    "dt": (("dt", "dd"), ("dl",)),
+    "dd": (("dt", "dd"), ("dl",)),
+    "td": (("td", "th"), ("tr", "table")),
+    "th": (("td", "th"), ("tr", "table")),
+    "tr": (("tr", "td", "th"), ("table", "thead", "tbody", "tfoot")),
+    "thead": (("thead", "tbody", "tfoot", "tr", "td", "th"), ("table",)),
+    "tbody": (("thead", "tbody", "tfoot", "tr", "td", "th"), ("table",)),
+    "tfoot": (("thead", "tbody", "tfoot", "tr", "td", "th"), ("table",)),
+    "option": (("option",), ("select", "optgroup", "datalist")),
+}
+"""For each element whose end tag may be left out: which open elements its start tag closes, and which ones stop the search."""
+
+
+_DESIGN_NOTES = """
+# HTML_IMPLIED_END — The End Tags a Page May Leave Out
+
+## Purpose
+`<ul><li>a<li>b</ul>` is valid: the second `<li>` ends the first. Without this
+the outline would put `b` inside `a`.
+
+## How to Read an Entry
+`"li": (("li",), ("ul", "ol", "menu"))`: a new `li` closes an open `li`, but
+the search stops at a `ul`, `ol` or `menu`, because an `li` inside a nested
+list belongs to that list and not to the one outside.
+
+## What Is Not Here
+`p` is left out: what closes it depends on the element that follows, and it
+rarely has a heading or an `id` inside.
+"""

simplibs/extractor_html/constants/HTML_LICENSE_START.py

@@ -0,0 +1,14 @@
+from typing import Final
+
+
+HTML_LICENSE_START: Final[str] = "<!--!"
+"""How a comment that has to survive minification begins."""
+
+
+_DESIGN_NOTES = """
+# HTML_LICENSE_START — The Comment That Tools Keep
+
+## Purpose
+By the convention of minifiers, a comment starting `<!--!` is meant to stay:
+a licence, a banner, a copyright line. With `keep_license` the step keeps it.
+"""

simplibs/extractor_html/constants/HTML_LICENSE_TAGS.py

@@ -0,0 +1,14 @@
+from typing import Final
+
+
+HTML_LICENSE_TAGS: Final[tuple[str, ...]] = ("@license", "@preserve")
+"""Words inside a comment that mark it as one to keep."""
+
+
+_DESIGN_NOTES = """
+# HTML_LICENSE_TAGS — The Other Way to Say "Keep This"
+
+## Purpose
+Besides the leading `!`, minifiers keep a comment that contains `@license` or
+`@preserve`. The same two words keep it here.
+"""