simplibs-extractor-html 0.1.0__py3-none-any.whl

simplibs/extractor_html/logic/_helpers/__init__.py

@@ -0,0 +1,68 @@
+from .HtmlBlock import HtmlBlock
+from .HtmlOutlineBuilder import HtmlOutlineBuilder
+from .HtmlScan import HtmlScan
+from .HtmlTag import HtmlTag
+from .Ranges import Ranges
+from .TemplateScan import TemplateScan
+from .TemplateTag import TemplateTag
+from .blank_spans import blank_spans
+from .build_html_outline import build_html_outline
+from .build_template_outline import build_template_outline
+from .find_blanks_start import find_blanks_start
+from .find_block_language import find_block_language
+from .find_comments import find_comments
+from .find_protected_spans import find_protected_spans
+from .find_template_import_spans import find_template_import_spans
+from .fit_removal_span import fit_removal_span
+from .is_license_comment import is_license_comment
+from .line_of import line_of
+from .raise_block_result_invalid import raise_block_result_invalid
+from .replace_spans import replace_spans
+from .scan_checked import scan_checked
+from .scan_html import scan_html
+from .scan_templates import scan_templates
+from .scan_templates_checked import scan_templates_checked
+from .skip_html_comment import skip_html_comment
+from .skip_html_tag import skip_html_tag
+from .tidy_lines import tidy_lines
+from .warn_problems import warn_problems
+
+
+_DESIGN_NOTES = """
+# _helpers — The Scanners and Their Parts
+
+Private. Plain imports, no `__all__`.
+
+## Internal Components Registry
+
+| Component                    | Type     | Description                                                          |
+| :--------------------------- | :------- | :------------------------------------------------------------------- |
+| `HtmlBlock`                  | Class    | The content of a `script` or `style` element, and its language.      |
+| `HtmlOutlineBuilder`         | Class    | Follows the tags and writes the outline of a page.                   |
+| `HtmlScan`                   | Class    | Comments, blocks, preserved ranges, tags and problems of one pass.   |
+| `HtmlTag`                    | Class    | One start tag or end tag: name, direction, span.                     |
+| `Ranges`                     | Class    | Ranges that a stretch of the text can be asked to touch.             |
+| `TemplateScan`               | Class    | Template comments, verbatim blocks, tags and the blanked text.       |
+| `TemplateTag`                | Class    | One `{% ... %}` tag: name, span, arguments.                          |
+| `blank_spans`                | Function | Replaces ranges with blanks, keeping every offset.                   |
+| `build_html_outline`         | Function | The outline of a page from the tags of its scan.                     |
+| `build_template_outline`     | Function | The skeleton of a template from its tags.                            |
+| `find_blanks_start`          | Function | Moves back over the blanks in front of a comment.                    |
+| `find_block_language`        | Function | Is the content of a `script` or `style` code, and of what language?  |
+| `find_comments`              | Function | Scans and chooses the HTML comments a step cuts.                     |
+| `find_protected_spans`       | Function | The ranges where the whitespace steps change nothing.                |
+| `find_template_import_spans` | Function | The ranges of the import tags of a template.                         |
+| `fit_removal_span`           | Function | The cut of one comment or tag.                                       |
+| `is_license_comment`         | Function | Is this a comment that tools keep?                                   |
+| `line_of`                    | Function | The line of an offset.                                               |
+| `raise_block_result_invalid` | Function | Reports a block step that did not return a text.                     |
+| `replace_spans`              | Function | Puts a new text in place of each of some ranges.                     |
+| `scan_checked`               | Function | Scans a page; gives `None` and warns when it looks damaged.          |
+| `scan_html`                  | Function | One pass over a page.                                                |
+| `scan_templates`             | Function | One pass over the template constructs of a page.                     |
+| `scan_templates_checked`     | Function | Scans the constructs; gives `None` and warns when they look damaged. |
+| `skip_html_comment`          | Function | The end of a comment.                                                |
+| `skip_html_tag`              | Function | The end of a tag, quoted values included.                            |
+| `tidy_lines`                 | Function | Trims the line ends and reduces the blank lines, sparing protected lines. |
+| `warn_problems`              | Function | Turns sentences about a damaged page into warnings.                  |
+"""

simplibs/extractor_html/logic/_helpers/blank_spans.py

@@ -0,0 +1,40 @@
+from collections.abc import Iterable
+from simplibs.text import Span
+
+
+def blank_spans(text: str, spans: Iterable[Span]) -> str:
+    """Replace the characters of some ranges with spaces, keeping every line break and every offset.
+
+    Args:
+        text: The source.
+        spans: The ranges to blank (the template constructs of a scan).
+
+    Returns:
+        A text of the same length in which the ranges hold only spaces and line breaks.
+    """
+
+    # 1. Copy what is between the ranges, put blanks where the ranges were
+    pieces: list[str] = []
+    previous = 0
+    for start, end in sorted(spans):
+        pieces.append(text[previous:start])
+        pieces.append("".join("\n" if character == "\n" else " " for character in text[start:end]))
+        previous = end
+    pieces.append(text[previous:])
+
+    return "".join(pieces)
+
+
+_DESIGN_NOTES = """
+# blank_spans — Hide Without Moving
+
+## Purpose
+A tag like `<a href="{% url 'x' %}" {% if y %}class="b"{% endif %}>` has
+template constructs in the middle of it. Cutting them out would shift every
+offset; blanking them leaves all offsets where they were, and the scan of the
+HTML reads a tag with a few more blanks in it.
+
+## Line Breaks Stay
+The blanked text has the same lines as the original, so a line number counted
+in it is the line number in the page.
+"""

simplibs/extractor_html/logic/_helpers/build_html_outline.py

@@ -0,0 +1,38 @@
+# Inners
+from .HtmlOutlineBuilder import HtmlOutlineBuilder
+from .HtmlScan import HtmlScan
+
+
+def build_html_outline(text: str, scan: HtmlScan, max_depth: int | None) -> str:
+    """Write the outline of a page from the tags of its scan.
+
+    Args:
+        text: The source.
+        scan: The result of `scan_html` for `text`, made with `collect_tags=True`.
+        max_depth: The number of levels to list, or `None` for all of them.
+
+    Returns:
+        One line for each listed element, indented by depth; an empty string if there is none.
+    """
+
+    # 1. Feed the tags to the builder in the order of the page
+    builder = HtmlOutlineBuilder(text, max_depth)
+    for tag in scan.tags:
+        if tag.closing:
+            builder.end_tag(tag.name, tag.span)
+        else:
+            builder.start_tag(tag.name, tag.span)
+
+    # 2. The lines, each ended by a line break
+    lines = builder.lines()
+    return "\n".join(lines) + "\n" if lines else ""
+
+
+_DESIGN_NOTES = """
+# build_html_outline — From the Tags to the Lines
+
+## Purpose
+The thin layer between the scan and the builder: it knows the order of the tags
+and the format of the result (one line each, a line break at the end, nothing
+for a page with nothing to list).
+"""

simplibs/extractor_html/logic/_helpers/build_template_outline.py

@@ -0,0 +1,75 @@
+# Outers
+from ...annotations import TemplateEngine
+from ...constants import HTML_TEMPLATE_OUTLINE_TAGS
+# Inners
+from .TemplateScan import TemplateScan
+
+
+def build_template_outline(
+    scan: TemplateScan,
+    engine: TemplateEngine,
+    max_depth: int | None,
+) -> tuple[list[str], list[str]]:
+    """Write the skeleton of a template: `extends`, `include` and the blocks, indented by depth.
+
+    Args:
+        scan: The result of `scan_templates`.
+        engine: `"django"` or `"jinja"`.
+        max_depth: The number of levels to list, or `None` for all of them.
+
+    Returns:
+        The lines, and the problems found (blocks that do not pair up).
+    """
+
+    # 1. Which tags open a level, which close it, which stand alone
+    names = HTML_TEMPLATE_OUTLINE_TAGS[engine]
+    lines: list[str] = []
+    depth = 0
+    stray = 0
+
+    for tag in scan.tags:
+
+        # 2. A tag that closes a level
+        if tag.name in names["closes"]:
+            if depth:
+                depth -= 1
+            else:
+                stray += 1
+            continue
+
+        # 3. A tag of the outline is written as the tag it is, with its arguments on one line
+        if tag.name in names["opens"] or tag.name in names["lines"]:
+            if max_depth is None or depth < max_depth:
+                arguments = " ".join(tag.arguments.split())
+                written = f"{{% {tag.name} {arguments} %}}" if arguments else f"{{% {tag.name} %}}"
+                lines.append("  " * depth + written)
+            if tag.name in names["opens"]:
+                depth += 1
+
+    # 4. Levels that do not pair up
+    problems: list[str] = []
+    if depth:
+        problems.append(f"{depth} block(s) of the template are never closed")
+    if stray:
+        problems.append(f"{stray} end tag(s) of the template have no opening one")
+
+    return lines, problems
+
+
+_DESIGN_NOTES = """
+# build_template_outline — What the Template Is Built Of
+
+## Purpose
+`{% extends "base.html" %}` and the blocks around the content say what a
+template is for; the rest of it is detail. The outline writes those tags as
+they stand in the template, so they can be found by searching for them.
+
+## Nesting
+`block` (and in Jinja `macro`) open a level, `endblock` (and `endmacro`) close
+it. A level is counted even when it is deeper than `max_depth`, so the blocks
+that follow are still on the right level.
+
+## Problems
+Blocks that do not pair up mean the nesting cannot be trusted. The step then
+leaves the template as it is, with a warning.
+"""

simplibs/extractor_html/logic/_helpers/find_blanks_start.py

@@ -0,0 +1,29 @@
+def find_blanks_start(text: str, position: int) -> int:
+    """Move back over the spaces and tabs in front of a position.
+
+    Args:
+        text: The source.
+        position: Where a comment starts.
+
+    Returns:
+        The offset of the first blank of the run that ends at `position`, or `position` itself.
+    """
+
+    # 1. Spaces and tabs only: a line break is never swallowed
+    while position > 0 and text[position - 1] in " \t":
+        position -= 1
+
+    return position
+
+
+_DESIGN_NOTES = """
+# find_blanks_start — The Blanks Go With the Comment
+
+## Purpose
+`<p>a</p> <!-- note -->` should become `<p>a</p>`, not `<p>a</p> `. The blanks
+in front of a comment belong to it, so the cut takes them along.
+
+## Only Spaces and Tabs
+The line break in front of a comment is not touched: removing it would pull
+the next line up.
+"""

simplibs/extractor_html/logic/_helpers/find_block_language.py

@@ -0,0 +1,35 @@
+from ...constants import HTML_BLOCK_LANGUAGE_BY_TYPE, HTML_TYPE_ATTRIBUTE_PATTERN
+
+
+def find_block_language(kind: str, tag: str) -> str | None:
+    """Say what the content of a `script` or `style` element is.
+
+    Args:
+        kind: `"script"` or `"style"`.
+        tag: The start tag of the element, with its attributes.
+
+    Returns:
+        `"javascript"` or `"css"` if the `type` of the tag says the content is code; otherwise `None`.
+    """
+
+    # 1. The value of `type`; a tag without it means the default
+    match = HTML_TYPE_ATTRIBUTE_PATTERN.search(tag)
+    value = ""
+    if match is not None:
+        value = next(group for group in match.groups() if group is not None).strip().lower()
+
+    # 2. A known type is code of a language, anything else is data
+    return HTML_BLOCK_LANGUAGE_BY_TYPE[kind].get(value)
+
+
+_DESIGN_NOTES = """
+# find_block_language — Is the Content Code
+
+## Purpose
+`<script type="application/ld+json">` holds JSON and `<script type="text/x-template">`
+a template of a framework; running the JavaScript steps over them would
+damage them. Only the types of `HTML_BLOCK_LANGUAGE_BY_TYPE` are code.
+
+## Case
+Values of `type` are compared in lower case and without the blanks around them.
+"""

simplibs/extractor_html/logic/_helpers/find_comments.py

@@ -0,0 +1,60 @@
+from simplibs.text import Span
+# Outers
+from ...annotations import TemplateEngine
+# Inners
+from .is_license_comment import is_license_comment
+from .scan_checked import scan_checked
+
+
+def find_comments(
+    text: str,
+    *,
+    engine: TemplateEngine | None,
+    keep_conditional: bool,
+    keep_license: bool,
+) -> list[Span]:
+    """Scan a page and choose the HTML comments a step is going to cut.
+
+    Args:
+        text: The source.
+        engine: The template language of the page; `None` for a page that is only HTML.
+        keep_conditional: Leave out the Internet Explorer conditional comments.
+        keep_license: Leave out the comments that tools keep (`<!--! ... -->`, `@license`, `@preserve`).
+
+    Returns:
+        The comments to cut; empty for a page that looks damaged (a warning has been given).
+    """
+
+    # 1. One pass; a damaged page is left alone
+    scan = scan_checked(text, engine=engine)
+    if scan is None:
+        return []
+
+    # 2. What is to stay is told apart by the position and by the text
+    kept: set[Span] = set()
+    if keep_conditional:
+        kept.update(scan.conditional)
+    if keep_license:
+        kept.update(span for span in scan.comments if is_license_comment(text[span[0]:span[1]]))
+
+    # 3. The rest goes
+    return [span for span in scan.comments if span not in kept]
+
+
+_DESIGN_NOTES = """
+# find_comments — The One Place That Chooses the Comments
+
+## Purpose
+`html_comment_spans` and `strip_html_comments` answer the same question,
+"which comments", and differ only in what they do with the answer. The scan,
+the damage check and the exceptions are decided here, once.
+
+## HTML Comments Only
+The template comments of the scan are the business of the template step, so
+this step leaves them as they are, even when an engine is given (the engine is
+needed only to read the page correctly).
+
+## A Damaged Page
+No comments are reported at all, so a step that cuts them cuts nothing and
+returns the text unchanged.
+"""

simplibs/extractor_html/logic/_helpers/find_protected_spans.py

@@ -0,0 +1,48 @@
+from simplibs.text import Span
+# Inners
+from .HtmlScan import HtmlScan
+
+
+def find_protected_spans(scan: HtmlScan) -> list[Span]:
+    """Collect the ranges of a page where the whitespace steps must not change anything.
+
+    Args:
+        scan: The result of `scan_html`.
+
+    Returns:
+        The content of `pre` and `textarea`, the content of every `script` and `style` with content, and the
+        verbatim blocks of a template.
+    """
+
+    # 1. Places where a space is shown to the reader
+    spans = list(scan.preserved)
+
+    # 2. Code of another language: its own steps decide what to do with its whitespace
+    spans.extend(block.span for block in scan.blocks)
+
+    # 3. Text that a template engine passes on as it is
+    spans.extend(scan.verbatim)
+
+    return spans
+
+
+_DESIGN_NOTES = """
+# find_protected_spans — What the Whitespace Steps Leave Alone
+
+## Purpose
+`cleanup_html_whitespace` and `compact_html` tidy the lines of a page. These
+ranges are the exceptions.
+
+## `pre` and `textarea`
+A browser shows the spaces and the line breaks inside them as they are.
+
+## `script` and `style`
+Whether the blank lines of a script may go is a question for the steps of the
+language of the script (`apply_to_html_blocks` gives them the content). A line
+break inside a template string of JavaScript is part of the string, and the
+whitespace steps of HTML cannot know where one is.
+
+## Verbatim Blocks
+`{% verbatim %}` and `{% raw %}` pass their content to the output as it is; it
+is often a piece of code shown in a manual.
+"""

simplibs/extractor_html/logic/_helpers/find_template_import_spans.py

@@ -0,0 +1,37 @@
+from simplibs.text import Span
+# Outers
+from ...annotations import TemplateEngine
+from ...constants import HTML_TEMPLATE_IMPORT_TAGS
+# Inners
+from .TemplateScan import TemplateScan
+from .fit_removal_span import fit_removal_span
+
+
+def find_template_import_spans(text: str, scan: TemplateScan, engine: TemplateEngine) -> list[Span]:
+    """Find the tags that bring names into a template.
+
+    Args:
+        text: The source.
+        scan: The result of `scan_templates` for `text`.
+        engine: `"django"` or `"jinja"`.
+
+    Returns:
+        A span for each import tag, from the blanks in front of it to its end.
+    """
+
+    # 1. The names of the engine's imports, and every recorded tag that has one of them
+    names = HTML_TEMPLATE_IMPORT_TAGS[engine]
+    return [fit_removal_span(text, *tag.span) for tag in scan.tags if tag.name in names]
+
+
+_DESIGN_NOTES = """
+# find_template_import_spans — The Imports of a Template
+
+## Purpose
+`remove_template_imports` cuts these ranges.
+
+## Blanks Go With the Tag
+Like a comment, an import tag takes the blanks in front of it when something
+blank follows, so `{% load a %} {% load b %}` leaves no stray space and a tag
+alone on its line leaves no empty line.
+"""

simplibs/extractor_html/logic/_helpers/fit_removal_span.py

@@ -0,0 +1,40 @@
+from simplibs.text import Span
+# Inners
+from .find_blanks_start import find_blanks_start
+
+
+def fit_removal_span(text: str, start: int, end: int) -> Span:
+    """Decide what to cut for a comment: the comment, and the blanks in front of it when they belong to it.
+
+    Args:
+        text: The source.
+        start: The offset where the comment begins.
+        end: The offset just after the comment (or the end of the text).
+
+    Returns:
+        The span to cut.
+    """
+
+    # 1. What stands behind the comment; the end of the text counts as a line break
+    after = text[end] if end < len(text) else "\n"
+
+    # 2. A comment followed by a blank or a line break takes the blanks in front of it along
+    if after in " \t\r\n":
+        return find_blanks_start(text, start), end
+
+    # 3. Otherwise only the comment goes
+    return start, end
+
+
+_DESIGN_NOTES = """
+# fit_removal_span — The Cut of One Comment
+
+## Purpose
+`<b>a</b> <!-- c -->` loses its trailing blank together with the comment, and
+`<b>a</b><!-- c --><i>b</i>` loses nothing but the comment.
+
+## No Gap Needed
+A comment between two words (`a<!-- c -->b`) is two pieces of text that a
+browser shows as one, without a space. Cutting the comment out changes
+nothing, so unlike in stylesheets no space has to be left behind.
+"""

simplibs/extractor_html/logic/_helpers/is_license_comment.py

@@ -0,0 +1,26 @@
+from ...constants import HTML_LICENSE_START, HTML_LICENSE_TAGS
+
+
+def is_license_comment(comment: str) -> bool:
+    """Say whether a comment is one that tools keep (a licence, a banner).
+
+    Args:
+        comment: The text of the comment, with its `<!--` and `-->` (blanks in front are allowed).
+
+    Returns:
+        `True` if it begins with `<!--!` or contains `@license` or `@preserve`.
+    """
+
+    # 1. The marker at the start, or one of the words anywhere
+    stripped = comment.lstrip()
+    return stripped.startswith(HTML_LICENSE_START) or any(tag in stripped for tag in HTML_LICENSE_TAGS)
+
+
+_DESIGN_NOTES = """
+# is_license_comment — Who Wants to Be Kept
+
+## Purpose
+The only comments a reader of a page may not want to lose are the licence and
+the banner of a template. The convention that marks them is the one
+minifiers already follow.
+"""

simplibs/extractor_html/logic/_helpers/line_of.py

@@ -0,0 +1,23 @@
+def line_of(text: str, offset: int) -> int:
+    """Say on which line an offset lies.
+
+    Args:
+        text: The source.
+        offset: A position in it.
+
+    Returns:
+        The line number, counted from 1.
+    """
+
+    # 1. One more than the number of line breaks in front of the offset
+    return text.count("\n", 0, offset) + 1
+
+
+_DESIGN_NOTES = """
+# line_of — Where Was That
+
+## Purpose
+A warning that says "a comment is not closed" is useless without the line.
+The count is made only when there is a problem to report, so a healthy file
+never pays for it.
+"""

simplibs/extractor_html/logic/_helpers/raise_block_result_invalid.py

@@ -0,0 +1,38 @@
+from typing import Any, NoReturn
+from simplibs.exception.exceptions import ParamError
+
+
+def raise_block_result_invalid(kind: str, result: Any) -> NoReturn:
+    """Report a step for a `script` or `style` block that did not return a text.
+
+    Args:
+        kind: `"script"` or `"style"`.
+        result: What the step returned.
+
+    Raises:
+        ParamError: Always.
+    """
+
+    # 1. Name the step, say what was expected and what came back
+    raise ParamError(
+        label=f"the step for a {kind} block",
+        expected="a function that takes the text of the block and returns a str",
+        value=result,
+        problem="it returned something that is not a str",
+        how_to_fix="Give a step that returns the new text of the block (an action of simplibs-text does).",
+    )
+
+
+_DESIGN_NOTES = """
+# raise_block_result_invalid — The Step Gave Back the Wrong Thing
+
+## Purpose
+The steps of a block are given by the caller: a lambda, a function or a whole
+pipeline. The text would otherwise fail deep in a `join` with a message about
+`str` and `NoneType`, far from the cause. The error names the block and the
+step.
+
+## Why Not a Check of the Argument
+Every function with the right signature is a step for this one; what it
+returns is known only when it is called.
+"""

simplibs/extractor_html/logic/_helpers/replace_spans.py

@@ -0,0 +1,42 @@
+from collections.abc import Sequence
+from simplibs.text import Span
+
+
+def replace_spans(text: str, replacements: Sequence[tuple[Span, str]]) -> str:
+    """Put a new text in place of each of some ranges.
+
+    Args:
+        text: The source.
+        replacements: Pairs of a range and its new text, in the order of the text, without overlaps.
+
+    Returns:
+        The text with the ranges replaced; `text` itself when there is nothing to replace.
+    """
+
+    # 1. Nothing to do
+    if not replacements:
+        return text
+
+    # 2. What lies between the ranges is kept, each range gives way to its new text
+    pieces: list[str] = []
+    previous = 0
+    for (start, end), new_text in replacements:
+        pieces.append(text[previous:start])
+        pieces.append(new_text)
+        previous = end
+    pieces.append(text[previous:])
+
+    return "".join(pieces)
+
+
+_DESIGN_NOTES = """
+# replace_spans — Put Something Else in the Same Place
+
+## Purpose
+`remove_spans` of `simplibs-text` only removes. The block step has to put the
+cleaned content of every `script` and `style` where the original was.
+
+## Order
+The ranges come in the order of the page, as the scanner finds them, so a
+single pass is enough.
+"""

simplibs/extractor_html/logic/_helpers/scan_checked.py

@@ -0,0 +1,45 @@
+# Outers
+from ...annotations import TemplateEngine
+# Inners
+from .HtmlScan import HtmlScan
+from .scan_html import scan_html
+from .warn_problems import warn_problems
+
+
+def scan_checked(text: str, *, engine: TemplateEngine | None, collect_tags: bool = False) -> HtmlScan | None:
+    """Scan a page and say so when it cannot be trusted.
+
+    Args:
+        text: The source.
+        engine: The template language of the page; `None` for a page that is only HTML.
+        collect_tags: Also record every start tag and end tag (for the outline).
+
+    Returns:
+        The scan; `None` if the scanner noticed damage (a warning has been given for each problem).
+    """
+
+    # 1. One pass
+    scan = scan_html(text, engine=engine, collect_tags=collect_tags)
+
+    # 2. A page with damage is not read any further: the step leaves it as it is
+    if scan.problems:
+        warn_problems(scan.problems)
+        return None
+
+    return scan
+
+
+_DESIGN_NOTES = """
+# scan_checked — Trust the Scan Only if It Found Nothing Wrong
+
+## Purpose
+Every step starts the same way: scan, and if the page looks damaged, give up
+on it. The extractor's contract for such a file is that the step returns the
+text unchanged and warns; the file then stays in the output as `partial`.
+This is the one place where that is decided for the whole library.
+
+## Why Not Go On
+A comment that is never closed would swallow the rest of the page, and a tag
+that is never closed would hide whatever follows it. Cutting on such a basis
+loses text silently; leaving the page alone loses nothing.
+"""