simplibs-extractor-html 0.1.0__py3-none-any.whl

simplibs/extractor_html/logic/cleanup_html_whitespace.py

@@ -0,0 +1,54 @@
+from simplibs.validate import validate_call
+# Outers
+from ..annotations import TemplateEngine
+# Inners
+from ._helpers import Ranges, find_protected_spans, scan_checked, tidy_lines
+
+
+@validate_call
+def cleanup_html_whitespace(text: str, *, engine: TemplateEngine | None = None) -> str:
+    """Tidy the whitespace of a page without changing how it looks: trailing blanks go, runs of blank lines become one.
+
+    Args:
+        text: The source.
+        engine: The template language of the page (`"django"` or `"jinja"`); `None` for a page that is only HTML.
+
+    Returns:
+        The page with no blanks at the ends of the lines and no more than one blank line in a row, and none
+        at the start or at the end. The lines that touch the content of `pre`, `textarea`, `script`, `style`
+        or a verbatim block are left exactly as they were. A page that looks damaged comes back unchanged,
+        with a warning.
+    """
+
+    # 1. Read the page; a damaged one is left as it is
+    scan = scan_checked(text, engine=engine)
+    if scan is None:
+        return text
+
+    # 2. Tidy every line that has no part in a protected range
+    return tidy_lines(text, Ranges(find_protected_spans(scan)), keep_blank_line=True)
+
+
+_DESIGN_NOTES = """
+# cleanup_html_whitespace — Tidy, and Leave the `pre` Alone
+
+## Purpose
+The step of the profiles that clean pages. The general cleaning of
+`simplibs-text` does not know HTML: it would trim the lines of a `<pre>` and
+change what the reader sees. This one reads the page first.
+
+## What Is Safe
+Between tags a browser treats a run of blanks and line breaks as one space, so
+the blanks at the end of a line and the extra blank lines between elements
+change nothing on screen.
+
+## What Is Left Alone
+`pre`, `textarea`, `script`, `style` and verbatim blocks (see `find_protected_spans`).
+
+## Idempotent
+A second run finds nothing to tidy.
+
+## Known Limits
+Inline text is judged as the other text: `a  <b>b</b>` keeps its two spaces,
+because the step trims line ends only; it never joins or splits lines.
+"""

simplibs/extractor_html/logic/compact_html.py

@@ -0,0 +1,44 @@
+from simplibs.validate import validate_call
+# Outers
+from ..annotations import TemplateEngine
+# Inners
+from ._helpers import Ranges, find_protected_spans, scan_checked, tidy_lines
+
+
+@validate_call
+def compact_html(text: str, *, engine: TemplateEngine | None = None) -> str:
+    """Remove every blank line of a page and the trailing blanks of its lines, without changing how it looks.
+
+    Args:
+        text: The source.
+        engine: The template language of the page (`"django"` or `"jinja"`); `None` for a page that is only HTML.
+
+    Returns:
+        The page with no blank lines and no blanks at the ends of the lines. The lines that touch the content of
+        `pre`, `textarea`, `script`, `style` or a verbatim block are left exactly as they were. A page that looks
+        damaged comes back unchanged, with a warning.
+    """
+
+    # 1. Read the page; a damaged one is left as it is
+    scan = scan_checked(text, engine=engine)
+    if scan is None:
+        return text
+
+    # 2. Tidy every line that has no part in a protected range, dropping all the blank ones
+    return tidy_lines(text, Ranges(find_protected_spans(scan)), keep_blank_line=False)
+
+
+_DESIGN_NOTES = """
+# compact_html — The Page Without Air
+
+## Purpose
+The step of the profiles that reduce a page to its content: the same as
+`cleanup_html_whitespace`, but no blank line survives. It is the counterpart
+of the `COMPACT` preset of `simplibs-text` for pages.
+
+## Same Care
+The protected ranges are the same as for the gentler step.
+
+## Idempotent
+A second run finds nothing to drop.
+"""

simplibs/extractor_html/logic/html_comment_spans.py

@@ -0,0 +1,54 @@
+from simplibs.text import Span
+from simplibs.validate import validate_call
+# Outers
+from ..annotations import TemplateEngine
+# Inners
+from ._helpers import find_comments
+
+
+@validate_call
+def html_comment_spans(
+    text: str,
+    *,
+    engine: TemplateEngine | None = None,
+    keep_conditional: bool = True,
+    keep_license: bool = False,
+) -> list[Span]:
+    """Find the HTML comments of a page.
+
+    Args:
+        text: The source.
+        engine: The template language of the page (`"django"` or `"jinja"`); `None` for a page that is only HTML.
+        keep_conditional: Leave out the Internet Explorer conditional comments (`<!--[if IE]> ... <![endif]-->`).
+        keep_license: Leave out the comments that tools keep (`<!--! ... -->`, `@license`, `@preserve`).
+
+    Returns:
+        A span for each comment, from the blanks in front of it to its end; an empty list for a page
+        that looks damaged (a comment, a tag or a `script` that is never closed), with a warning.
+    """
+
+    # 1. The comments the scanner found, the ones that are to stay aside
+    return find_comments(
+        text,
+        engine=engine,
+        keep_conditional=keep_conditional,
+        keep_license=keep_license,
+    )
+
+
+_DESIGN_NOTES = """
+# html_comment_spans — Where the Comments Are
+
+## Purpose
+The spans of the comments, for anything that wants to look at them or cut them.
+
+## What Is a Comment
+`<!-- ... -->` outside an attribute, a `script`, a `style`, a `textarea` and a
+`title`. A `<!--` inside any of those is text. Template comments (`{# #}`) are
+not reported here.
+
+## Never Raises
+The scanner has no syntax it refuses: whatever the text is, it gets an answer.
+A page that looks damaged is answered with no spans and a warning, so a step
+that cuts them leaves the page as it is.
+"""

simplibs/extractor_html/logic/html_outline.py

@@ -0,0 +1,66 @@
+from simplibs.types import int_positive
+from simplibs.validate import validate_call
+# Outers
+from ..annotations import TemplateEngine
+# Inners
+from ._helpers import build_html_outline, scan_checked
+
+
+@validate_call
+def html_outline(text: str, *, engine: TemplateEngine | None = None, max_depth: int_positive | None = None) -> str:
+    """Write an overview of a page: its landmarks, its headings and everything that has an `id`, indented by nesting.
+
+    Args:
+        text: The source.
+        engine: The template language of the page (`"django"` or `"jinja"`); `None` for a page that is only HTML.
+        max_depth: How many levels of nesting to list (`1` is the top level only); `None` lists all.
+
+    Returns:
+        One line for each listed element: `header#top`, `h2 "Opening hours"`; an empty string for a page
+        with nothing to list. A page that looks damaged (a comment, a tag or a `script` that is never
+        closed) comes back unchanged, with a warning.
+
+    Example:
+        html_outline('<main><h1>Hi</h1><div id="a"></div></main>')
+        # 'main\\n  h1 "Hi"\\n  div#a\\n'
+    """
+
+    # 1. Read the page with its tags; a damaged one is left as it is
+    scan = scan_checked(text, engine=engine, collect_tags=True)
+    if scan is None:
+        return text
+
+    # 2. The outline
+    return build_html_outline(text, scan, max_depth)
+
+
+_DESIGN_NOTES = """
+# html_outline — What a Page Is Made Of, Without the Markup
+
+## Purpose
+The step of the profile `html/outline`: the parts of a page and what they are
+called, in a fraction of its size. For a reader with a limited window it is the
+map of a page that would not fit.
+
+## What Is Listed
+* the landmarks (`header`, `nav`, `main`, `section`, `article`, `aside`,
+  `footer`, `form`, `table`, `dialog`, `template`),
+* the title and the headings, with their text,
+* every element with an `id`, written `tag#id`.
+
+Depth is the number of listed elements around, so the `div`s between them are
+not counted.
+
+## Not Listed
+Classes, attributes other than the `id`, text outside the headings, scripts and
+styles. They are the content; the outline is the shape.
+
+## Templates
+With an `engine` the constructs are read first, so `<div {% if a > b %}>` is a
+tag. An `id` that contains a template construct is written as it stands
+(`item-{{ i }}`), and so is a heading with `{{ title }}` in it.
+
+## A Damaged Page
+The whole text is returned, as `python_outline` does for a source it cannot
+parse, so the reader sees the page and the report says why it is not an outline.
+"""

simplibs/extractor_html/logic/strip_html_comments.py

@@ -0,0 +1,63 @@
+from simplibs.text import remove_spans
+from simplibs.validate import validate_call
+# Outers
+from ..annotations import TemplateEngine
+# Inners
+from ._helpers import find_comments
+
+
+@validate_call
+def strip_html_comments(
+    text: str,
+    *,
+    engine: TemplateEngine | None = None,
+    keep_conditional: bool = True,
+    keep_license: bool = False,
+) -> str:
+    """Remove the HTML comments of a page.
+
+    Args:
+        text: The source.
+        engine: The template language of the page (`"django"` or `"jinja"`); `None` for a page that is only HTML.
+        keep_conditional: Keep the Internet Explorer conditional comments (`<!--[if IE]> ... <![endif]-->`).
+        keep_license: Keep the comments that tools keep (`<!--! ... -->`, `@license`, `@preserve`).
+
+    Returns:
+        The text without HTML comments; a line that held only a comment disappears. A page that looks
+        damaged (a comment, a tag or a `script` that is never closed) comes back unchanged, with a warning.
+    """
+
+    # 1. Which comments go
+    comments = find_comments(
+        text,
+        engine=engine,
+        keep_conditional=keep_conditional,
+        keep_license=keep_license,
+    )
+
+    # 2. Cut them
+    return remove_spans(text, comments)
+
+
+_DESIGN_NOTES = """
+# strip_html_comments — The Comments Out, The Scripts Untouched
+
+## Purpose
+A step of the profiles that clean pages. It removes only `<!-- ... -->`: a
+`<!--` inside an attribute, a `script`, a `style` or a `textarea` is text and
+stays.
+
+## Options
+`engine` tells the scanner that the page is a template, so `{{ }}`, `{% %}`
+and `{# #}` are read first and cannot confuse the HTML. It does not remove
+template comments; that is a step of its own. `keep_conditional` is on by
+default: an Internet Explorer conditional comment holds real tags, and
+`keep_license` is off by default.
+
+## A Damaged Page
+It is returned as it is, and the warning makes the extractor call the file
+`partial`. The step never cuts on a basis it does not trust.
+
+## Idempotent
+A second run finds nothing and returns the very same text.
+"""

simplibs/extractor_html/logic/templates/__init__.py

@@ -0,0 +1,25 @@
+from .remove_template_imports import remove_template_imports
+from .strip_template_comments import strip_template_comments
+from .template_comment_spans import template_comment_spans
+from .template_outline import template_outline
+
+
+_DESIGN_NOTES = """
+# templates — The Steps for Django and Jinja Templates
+
+Plain imports, no `__all__` (the package root decides what is public).
+
+## Internal Components Registry
+
+| Component                 | Type     | Description                                                      |
+| :------------------------ | :------- | :--------------------------------------------------------------- |
+| `remove_template_imports` | Function | Removes `{% load %}` (Django), `{% import %}` and `{% from %}` (Jinja). |
+| `strip_template_comments` | Function | Removes `{# ... #}` and `{% comment %} ... {% endcomment %}`.    |
+| `template_comment_spans`  | Function | The spans of the template comments.                              |
+| `template_outline`        | Function | The skeleton of a template: extends, include, blocks.            |
+
+## Layering & Design
+- Every step takes the engine as a required keyword: `"django"` or `"jinja"`.
+- The steps read the template constructs only; the HTML around them is not read.
+- A template that looks damaged is left as it is and reported by a warning.
+"""

simplibs/extractor_html/logic/templates/remove_template_imports.py

@@ -0,0 +1,51 @@
+from simplibs.text import remove_spans
+from simplibs.validate import validate_call
+# Outers
+from ...annotations import TemplateEngine
+from .._helpers import find_template_import_spans, scan_templates_checked
+
+
+@validate_call
+def remove_template_imports(text: str, *, engine: TemplateEngine) -> str:
+    """Remove the tags that bring names into a template: `{% load %}` (Django), `{% import %}` and `{% from %}` (Jinja).
+
+    Args:
+        text: The source.
+        engine: `"django"` or `"jinja"`.
+
+    Returns:
+        The text without those tags; a line that held only one disappears. A template that looks damaged
+        (a `{% comment %}` that is never closed) comes back unchanged, with a warning.
+    """
+
+    # 1. Read the constructs; a damaged template is left as it is
+    scan = scan_templates_checked(text, engine=engine)
+    if scan is None:
+        return text
+
+    # 2. Each tag with one of the engine's import names is a range; cut them
+    return remove_spans(text, find_template_import_spans(text, scan, engine))
+
+
+_DESIGN_NOTES = """
+# remove_template_imports — The Page Without Its Imports
+
+## Purpose
+A step of the profile of pages without imports: what a template needs from
+other files is the part of its text that says least about the page itself.
+
+## What Is an Import
+Only the tags of `HTML_TEMPLATE_IMPORT_TAGS`: `load` in Django, `import` and
+`from` in Jinja. `{% extends %}` and `{% include %}` stay, as they say what
+the page is made of.
+
+## By the Scan, Not by a Line Pattern
+Tags are read by the scanner of templates, so a `{% load static %}` inside a
+`{% comment %}` or a `{% verbatim %}` is never cut, and one on the same line
+as other text (`{% load a %}{% load b %}`) is cut alone.
+
+## Removing Is Not Harmless
+A template that uses `{% static 'x' %}` no longer says where `static` comes
+from. That is exactly what the reader of a page without imports gives up, as
+the reader of a program without its `import` lines does.
+"""

simplibs/extractor_html/logic/templates/strip_template_comments.py

@@ -0,0 +1,60 @@
+from simplibs.text import remove_spans
+from simplibs.validate import validate_call
+# Outers
+from ...annotations import TemplateEngine
+from .._helpers import scan_templates_checked
+
+
+@validate_call
+def strip_template_comments(text: str, *, engine: TemplateEngine) -> str:
+    """Remove the comments of a template: `{# ... #}` and `{% comment %} ... {% endcomment %}`.
+
+    Args:
+        text: The source.
+        engine: `"django"` (a `{# #}` stays on one line; `{% comment %}` exists) or `"jinja"` (a `{# #}`
+            may span lines; there is no comment tag).
+
+    Returns:
+        The text without template comments; a line that held only a comment disappears. A template that
+        looks damaged (a `{% comment %}` that is never closed) comes back unchanged, with a warning.
+    """
+
+    # 1. Read the constructs; a damaged template is left as it is
+    scan = scan_templates_checked(text, engine=engine)
+    if scan is None:
+        return text
+
+    # 2. Cut the comments
+    return remove_spans(text, scan.comments)
+
+
+_DESIGN_NOTES = """
+# strip_template_comments — The Notes of the Author
+
+## Purpose
+The step of the profiles that clean templates. An HTML comment is sent to the
+browser, a template comment never leaves the server: this step removes the
+second kind, `strip_html_comments` the first. A profile uses both, in either
+order.
+
+## Why `engine` Is Required
+Whether a `{# ... #}` may span lines, and whether `{% comment %}` exists at all,
+depends on the engine, and a page cannot say which one it is written for.
+Guessing would cut text in one engine that is not a comment in the other.
+
+## No Gap Needed
+A comment between two words (`a{# c #}b`) disappears from the output of the
+engine without a trace, so cutting it changes nothing.
+
+## Whitespace Control
+Jinja's `{#- ... -#}` also trims the whitespace around it in the output.
+Removing the comment keeps that whitespace, so the rendered text can differ
+by a blank; the structure of the template does not.
+
+## A Damaged Template
+It is returned as it is, and the warning makes the extractor call the file
+`partial`.
+
+## Idempotent
+A second run finds nothing and returns the very same text.
+"""

simplibs/extractor_html/logic/templates/template_comment_spans.py

@@ -0,0 +1,49 @@
+from simplibs.text import Span
+from simplibs.validate import validate_call
+# Outers
+from ...annotations import TemplateEngine
+from .._helpers import scan_templates_checked
+
+
+@validate_call
+def template_comment_spans(text: str, *, engine: TemplateEngine) -> list[Span]:
+    """Find the comments of a template: `{# ... #}` and `{% comment %} ... {% endcomment %}`.
+
+    Args:
+        text: The source.
+        engine: `"django"` (a `{# #}` stays on one line; `{% comment %}` exists) or `"jinja"` (a `{# #}`
+            may span lines; there is no comment tag).
+
+    Returns:
+        A span for each comment, from the blanks in front of it to its end; an empty list for a template
+        that looks damaged (a `{% comment %}` that is never closed), with a warning.
+    """
+
+    # 1. Read the constructs; a damaged template is left as it is
+    scan = scan_templates_checked(text, engine=engine)
+    if scan is None:
+        return []
+
+    return list(scan.comments)
+
+
+_DESIGN_NOTES = """
+# template_comment_spans — Where the Template Comments Are
+
+## Purpose
+The spans of the comments that the template engine drops and the browser never
+sees, for anything that wants to look at them or cut them.
+
+## What Is a Comment
+`{# ... #}` in both engines, and in Django `{% comment %} ... {% endcomment %}`
+with an optional note. What stands inside is not read: a `{{ x }}` in a
+comment is text of the comment.
+
+## What Is Not
+A `{# ... #}` inside `{% verbatim %}` or `{% raw %}` is plain text for the
+engine and is not reported. A `{#` whose `#}` is on a later line is, in
+Django, only text.
+
+## Never Raises
+A template that looks damaged is answered with no spans and a warning.
+"""

simplibs/extractor_html/logic/templates/template_outline.py

@@ -0,0 +1,62 @@
+from simplibs.types import int_positive
+from simplibs.validate import validate_call
+# Outers
+from ...annotations import TemplateEngine
+from .._helpers import build_template_outline, scan_templates_checked, warn_problems
+
+
+@validate_call
+def template_outline(text: str, *, engine: TemplateEngine, max_depth: int_positive | None = None) -> str:
+    """Write the skeleton of a template: `extends`, `include`, the blocks, and in Jinja the macros.
+
+    Args:
+        text: The source.
+        engine: `"django"` or `"jinja"`.
+        max_depth: How many levels of blocks to list (`1` is the top level only); `None` lists all.
+
+    Returns:
+        One line for each of those tags, written as it stands in the template and indented by the depth of
+        its blocks; an empty string for a template with none. A template that looks damaged (a
+        `{% comment %}` that is never closed, blocks that do not pair up) comes back unchanged, with a warning.
+
+    Example:
+        template_outline("{% extends 'b.html' %}{% block a %}{% include 'c.html' %}{% endblock %}", engine="django")
+        # "{% extends 'b.html' %}\\n{% block a %}\\n  {% include 'c.html' %}\\n"
+    """
+
+    # 1. Read the constructs; a damaged template is left as it is
+    scan = scan_templates_checked(text, engine=engine)
+    if scan is None:
+        return text
+
+    # 2. The skeleton; blocks that do not pair up mean the nesting cannot be trusted
+    lines, problems = build_template_outline(scan, engine, max_depth)
+    if problems:
+        warn_problems(problems)
+        return text
+
+    return "\n".join(lines) + "\n" if lines else ""
+
+
+_DESIGN_NOTES = """
+# template_outline — What the Template Is Built Of
+
+## Purpose
+The step of the profile for templates: which template this one extends, what it
+includes and which blocks it defines or fills in, in the order and nesting in
+which they stand. A reader who has the outlines of all the templates of a
+project sees how its pages are put together without reading a page.
+
+## Why `engine` Is Required
+A Jinja `macro` is part of the skeleton and a Django one does not exist, and a
+`{# #}` that spans lines is a comment only in Jinja. A page cannot say which
+engine it is written for.
+
+## Tags in Comments
+A tag inside `{% comment %}` or `{% verbatim %}` is not a tag of the template
+and is not listed.
+
+## A Damaged Template
+It is returned as it is, with a warning, so the extractor calls the file
+`partial`.
+"""

simplibs/extractor_html/plugin/__init__.py

@@ -0,0 +1,14 @@
+from .register_all import register_all
+
+
+_DESIGN_NOTES = """
+# plugin — The Meeting Point With the Extractor
+
+Plain imports, no `__all__`.
+
+## Internal Components Registry
+
+| Component      | Type     | Description                                                      |
+| :------------- | :------- | :--------------------------------------------------------------- |
+| `register_all` | Function | Registers the languages and profiles; the target of the entry point. |
+"""

simplibs/extractor_html/plugin/register_all.py

@@ -0,0 +1,60 @@
+from simplibs.extractor import register_language, register_profile
+from simplibs.validate import validate_call
+# Outers
+from ..constants import HTML_EXTRA_LANGUAGE_BY_SUFFIX
+from ..profiles import HTML_PROFILES
+
+
+@validate_call
+def register_all(*, replace_existing: bool = True) -> list[str]:
+    """Register every profile and language of this library in the registries of the extractor.
+
+    The extractor calls it, with no arguments, for every installed add-on
+    (entry-point group `simplibs.extractor.plugins`).
+
+    Args:
+        replace_existing: Replace a profile or a language that already has the key (the default, so calling it twice changes nothing).
+
+    Returns:
+        The keys of the registered profiles (`"html/code-clean"`...).
+
+    Raises:
+        StateError: A profile key is taken and `replace_existing` is `False`.
+        ParamError: A language suffix is taken and `replace_existing` is `False`.
+    """
+
+    # 1. The languages of the other suffixes, so the output fences them properly
+    for suffix, language in HTML_EXTRA_LANGUAGE_BY_SUFFIX.items():
+        register_language(suffix, language, replace_existing=replace_existing)
+
+    # 2. Register each profile
+    keys: list[str] = []
+    for profile in HTML_PROFILES:
+        registered = register_profile(profile, replace_existing=replace_existing)
+        keys.append(registered.key)
+
+    return keys
+
+
+_DESIGN_NOTES = """
+# register_all — What the Entry Point Calls
+
+## Purpose
+`pip install simplibs-extractor-html` is the whole installation: the entry
+point `html = "simplibs.extractor_html:register_all"` makes the extractor find
+this function, and the thirteen profiles appear in `--list-systems`.
+
+## The Rules of an Add-On Function
+It takes no required argument, registers with `replace_existing=True`, only
+registers (no scanning, no network, no printing) and uses only the public API
+of the extractor (`Profile`, `register_profile`, `register_language`).
+
+## Languages
+`.htm`, `.xhtml` and the suffixes of the Jinja family get their fence language
+here. Without it the extractor would write their code blocks with no language
+and a viewer would not colour them.
+
+## Order
+Languages first: a profile that is registered is usable at once, and its
+output should already be fenced correctly.
+"""

simplibs/extractor_html/presets/__init__.py

@@ -0,0 +1,43 @@
+from .html_clean_steps import html_clean_steps
+from .html_no_imports_steps import html_no_imports_steps
+from .html_outline_steps import html_outline_steps
+from .template_structure_steps import template_structure_steps
+
+
+__all__ = [
+    "html_clean_steps",
+    "html_no_imports_steps",
+    "html_outline_steps",
+    "template_structure_steps",
+]
+
+
+_DESIGN_NOTES = """
+# simplibs.extractor_html.presets — Ready-Made Pipeline Steps
+
+## Purpose
+The chains of actions that the profiles use. Each is a function of the template
+engine (`None` for a page that is only HTML), because the chain is the same
+for every engine and only the option of its actions differs:
+
+    from simplibs.extractor_html.presets import html_clean_steps
+    steps = html_clean_steps("django")
+
+## Domain Folder
+`presets/` has its own `__all__` and is not re-exported from the package root.
+It stands to the right of `actions` (it uses them) and to the left of `profiles`.
+
+## The Other Add-Ons
+The presets for the code inside a page come from the add-ons of its languages
+(`JS_CLEAN`, `CSS_PLAIN_CLEAN`...): this library depends on both, and a program
+that wants other steps for the blocks calls `apply_to_html_blocks` itself.
+
+## Internal Components Registry
+
+| Component                 | Type   | Description                                                       |
+| :------------------------ | :----- | :---------------------------------------------------------------- |
+| `html_clean_steps`        | Preset | Comments out, code inside cleaned, whitespace tidied.             |
+| `html_no_imports_steps`   | Preset | The same, without imports and blank lines.                        |
+| `html_outline_steps`      | Preset | The landmarks, headings and ids of a page.                        |
+| `template_structure_steps`| Preset | The `extends`, `include` and blocks of a template.                |
+"""