simplibs-extractor-html 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. simplibs/extractor_html/__init__.py +82 -0
  2. simplibs/extractor_html/actions/ApplyToHtmlBlocks.py +29 -0
  3. simplibs/extractor_html/actions/CleanupHtmlWhitespace.py +22 -0
  4. simplibs/extractor_html/actions/CompactHtml.py +22 -0
  5. simplibs/extractor_html/actions/HtmlOutline.py +23 -0
  6. simplibs/extractor_html/actions/RemoveTemplateImports.py +22 -0
  7. simplibs/extractor_html/actions/StripHtmlComments.py +23 -0
  8. simplibs/extractor_html/actions/StripTemplateComments.py +27 -0
  9. simplibs/extractor_html/actions/TemplateOutline.py +23 -0
  10. simplibs/extractor_html/actions/__init__.py +57 -0
  11. simplibs/extractor_html/annotations/TemplateEngine.py +26 -0
  12. simplibs/extractor_html/annotations/__init__.py +14 -0
  13. simplibs/extractor_html/constants/HTML_ANY_TAG_PATTERN.py +16 -0
  14. simplibs/extractor_html/constants/HTML_BLOCK_LANGUAGE_BY_TYPE.py +41 -0
  15. simplibs/extractor_html/constants/HTML_COMMENT_END_PATTERN.py +19 -0
  16. simplibs/extractor_html/constants/HTML_CONDITIONAL_PATTERN.py +23 -0
  17. simplibs/extractor_html/constants/HTML_DEPENDENCY_DIR_NAMES.py +18 -0
  18. simplibs/extractor_html/constants/HTML_EXTRA_LANGUAGE_BY_SUFFIX.py +31 -0
  19. simplibs/extractor_html/constants/HTML_HEADING_ELEMENTS.py +15 -0
  20. simplibs/extractor_html/constants/HTML_ID_ATTRIBUTE_PATTERN.py +21 -0
  21. simplibs/extractor_html/constants/HTML_IMPLIED_END.py +34 -0
  22. simplibs/extractor_html/constants/HTML_LICENSE_START.py +14 -0
  23. simplibs/extractor_html/constants/HTML_LICENSE_TAGS.py +14 -0
  24. simplibs/extractor_html/constants/HTML_OUTLINE_ELEMENTS.py +32 -0
  25. simplibs/extractor_html/constants/HTML_OUTLINE_TEXT_LIMIT.py +15 -0
  26. simplibs/extractor_html/constants/HTML_OUTPUT_DIR_NAMES.py +43 -0
  27. simplibs/extractor_html/constants/HTML_PLAIN_SUFFIXES.py +18 -0
  28. simplibs/extractor_html/constants/HTML_RAW_TEXT_END_PATTERNS.py +28 -0
  29. simplibs/extractor_html/constants/HTML_SUFFIXES.py +22 -0
  30. simplibs/extractor_html/constants/HTML_TAG_NAME_PATTERN.py +16 -0
  31. simplibs/extractor_html/constants/HTML_TEMPLATE_BLOCKS.py +38 -0
  32. simplibs/extractor_html/constants/HTML_TEMPLATE_IMPORT_TAGS.py +29 -0
  33. simplibs/extractor_html/constants/HTML_TEMPLATE_OUTLINE_TAGS.py +35 -0
  34. simplibs/extractor_html/constants/HTML_TEMPLATE_SUFFIXES.py +19 -0
  35. simplibs/extractor_html/constants/HTML_TEMPLATE_TAG_NAME_PATTERN.py +16 -0
  36. simplibs/extractor_html/constants/HTML_TEMPLATE_TOKEN_PATTERN.py +16 -0
  37. simplibs/extractor_html/constants/HTML_TEXT_ELEMENTS.py +16 -0
  38. simplibs/extractor_html/constants/HTML_TOKEN_PATTERN.py +26 -0
  39. simplibs/extractor_html/constants/HTML_TYPE_ATTRIBUTE_PATTERN.py +21 -0
  40. simplibs/extractor_html/constants/HTML_VOID_ELEMENTS.py +21 -0
  41. simplibs/extractor_html/constants/HTML_WRAPPED_STEM_ENDING.py +14 -0
  42. simplibs/extractor_html/constants/HTML_WRAPPED_SUFFIXES.py +15 -0
  43. simplibs/extractor_html/constants/__init__.py +77 -0
  44. simplibs/extractor_html/file_rules/__init__.py +56 -0
  45. simplibs/extractor_html/file_rules/atomic/DEPENDENCY_DIRS.py +17 -0
  46. simplibs/extractor_html/file_rules/atomic/DOT_NAME.py +19 -0
  47. simplibs/extractor_html/file_rules/atomic/OUTPUT_DIRS.py +17 -0
  48. simplibs/extractor_html/file_rules/atomic/VCS_DIRS.py +21 -0
  49. simplibs/extractor_html/file_rules/atomic/WRAPPED_HTML.py +22 -0
  50. simplibs/extractor_html/file_rules/atomic/__init__.py +26 -0
  51. simplibs/extractor_html/file_rules/html/HTML_BASE.py +22 -0
  52. simplibs/extractor_html/file_rules/html/HTML_FILES.py +24 -0
  53. simplibs/extractor_html/file_rules/html/HTML_PLAIN_BASE.py +24 -0
  54. simplibs/extractor_html/file_rules/html/HTML_STRICT_EXCLUDE.py +21 -0
  55. simplibs/extractor_html/file_rules/html/__init__.py +20 -0
  56. simplibs/extractor_html/logic/__init__.py +44 -0
  57. simplibs/extractor_html/logic/_helpers/HtmlBlock.py +31 -0
  58. simplibs/extractor_html/logic/_helpers/HtmlOutlineBuilder.py +189 -0
  59. simplibs/extractor_html/logic/_helpers/HtmlScan.py +50 -0
  60. simplibs/extractor_html/logic/_helpers/HtmlTag.py +32 -0
  61. simplibs/extractor_html/logic/_helpers/Ranges.py +48 -0
  62. simplibs/extractor_html/logic/_helpers/TemplateScan.py +43 -0
  63. simplibs/extractor_html/logic/_helpers/TemplateTag.py +33 -0
  64. simplibs/extractor_html/logic/_helpers/__init__.py +68 -0
  65. simplibs/extractor_html/logic/_helpers/blank_spans.py +40 -0
  66. simplibs/extractor_html/logic/_helpers/build_html_outline.py +38 -0
  67. simplibs/extractor_html/logic/_helpers/build_template_outline.py +75 -0
  68. simplibs/extractor_html/logic/_helpers/find_blanks_start.py +29 -0
  69. simplibs/extractor_html/logic/_helpers/find_block_language.py +35 -0
  70. simplibs/extractor_html/logic/_helpers/find_comments.py +60 -0
  71. simplibs/extractor_html/logic/_helpers/find_protected_spans.py +48 -0
  72. simplibs/extractor_html/logic/_helpers/find_template_import_spans.py +37 -0
  73. simplibs/extractor_html/logic/_helpers/fit_removal_span.py +40 -0
  74. simplibs/extractor_html/logic/_helpers/is_license_comment.py +26 -0
  75. simplibs/extractor_html/logic/_helpers/line_of.py +23 -0
  76. simplibs/extractor_html/logic/_helpers/raise_block_result_invalid.py +38 -0
  77. simplibs/extractor_html/logic/_helpers/replace_spans.py +42 -0
  78. simplibs/extractor_html/logic/_helpers/scan_checked.py +45 -0
  79. simplibs/extractor_html/logic/_helpers/scan_html.py +208 -0
  80. simplibs/extractor_html/logic/_helpers/scan_templates.py +128 -0
  81. simplibs/extractor_html/logic/_helpers/scan_templates_checked.py +39 -0
  82. simplibs/extractor_html/logic/_helpers/skip_html_comment.py +41 -0
  83. simplibs/extractor_html/logic/_helpers/skip_html_tag.py +46 -0
  84. simplibs/extractor_html/logic/_helpers/tidy_lines.py +86 -0
  85. simplibs/extractor_html/logic/_helpers/warn_problems.py +32 -0
  86. simplibs/extractor_html/logic/apply_to_html_blocks.py +103 -0
  87. simplibs/extractor_html/logic/cleanup_html_whitespace.py +54 -0
  88. simplibs/extractor_html/logic/compact_html.py +44 -0
  89. simplibs/extractor_html/logic/html_comment_spans.py +54 -0
  90. simplibs/extractor_html/logic/html_outline.py +66 -0
  91. simplibs/extractor_html/logic/strip_html_comments.py +63 -0
  92. simplibs/extractor_html/logic/templates/__init__.py +25 -0
  93. simplibs/extractor_html/logic/templates/remove_template_imports.py +51 -0
  94. simplibs/extractor_html/logic/templates/strip_template_comments.py +60 -0
  95. simplibs/extractor_html/logic/templates/template_comment_spans.py +49 -0
  96. simplibs/extractor_html/logic/templates/template_outline.py +62 -0
  97. simplibs/extractor_html/plugin/__init__.py +14 -0
  98. simplibs/extractor_html/plugin/register_all.py +60 -0
  99. simplibs/extractor_html/presets/__init__.py +43 -0
  100. simplibs/extractor_html/presets/html_clean_steps.py +54 -0
  101. simplibs/extractor_html/presets/html_no_imports_steps.py +50 -0
  102. simplibs/extractor_html/presets/html_outline_steps.py +28 -0
  103. simplibs/extractor_html/presets/template_structure_steps.py +27 -0
  104. simplibs/extractor_html/profiles/DJANGO_CLEAN.py +36 -0
  105. simplibs/extractor_html/profiles/DJANGO_NO_IMPORTS.py +32 -0
  106. simplibs/extractor_html/profiles/DJANGO_OUTLINE.py +32 -0
  107. simplibs/extractor_html/profiles/DJANGO_STRUCTURE.py +32 -0
  108. simplibs/extractor_html/profiles/HTML_CODE_CLEAN.py +32 -0
  109. simplibs/extractor_html/profiles/HTML_CODE_NO_IMPORTS.py +30 -0
  110. simplibs/extractor_html/profiles/HTML_DATA_ALL.py +26 -0
  111. simplibs/extractor_html/profiles/HTML_OUTLINE.py +31 -0
  112. simplibs/extractor_html/profiles/HTML_PROFILES.py +42 -0
  113. simplibs/extractor_html/profiles/HTML_SRC_CLEAN.py +31 -0
  114. simplibs/extractor_html/profiles/JINJA_CLEAN.py +35 -0
  115. simplibs/extractor_html/profiles/JINJA_NO_IMPORTS.py +32 -0
  116. simplibs/extractor_html/profiles/JINJA_OUTLINE.py +30 -0
  117. simplibs/extractor_html/profiles/JINJA_STRUCTURE.py +31 -0
  118. simplibs/extractor_html/profiles/__init__.py +86 -0
  119. simplibs/extractor_html/profiles/steps_for_engine.py +39 -0
  120. simplibs/extractor_html/profiles/steps_for_pages.py +36 -0
  121. simplibs/extractor_html/py.typed +1 -0
  122. simplibs_extractor_html-0.1.0.dist-info/METADATA +504 -0
  123. simplibs_extractor_html-0.1.0.dist-info/RECORD +127 -0
  124. simplibs_extractor_html-0.1.0.dist-info/WHEEL +5 -0
  125. simplibs_extractor_html-0.1.0.dist-info/entry_points.txt +2 -0
  126. simplibs_extractor_html-0.1.0.dist-info/licenses/LICENSE +21 -0
  127. simplibs_extractor_html-0.1.0.dist-info/top_level.txt +1 -0
simplibs/extractor_html/logic/_helpers/scan_html.py ADDED
@@ -0,0 +1,208 @@
1
+ from simplibs.text import Span
2
+ # Outers
3
+ from ...annotations import TemplateEngine
4
+ from ...constants import (
5
+ HTML_CONDITIONAL_PATTERN,
6
+ HTML_RAW_TEXT_END_PATTERNS,
7
+ HTML_TAG_NAME_PATTERN,
8
+ HTML_TOKEN_PATTERN,
9
+ )
10
+ # Inners
11
+ from .HtmlBlock import HtmlBlock
12
+ from .HtmlScan import HtmlScan
13
+ from .HtmlTag import HtmlTag
14
+ from .find_block_language import find_block_language
15
+ from .fit_removal_span import fit_removal_span
16
+ from .line_of import line_of
17
+ from .scan_templates import scan_templates
18
+ from .skip_html_comment import skip_html_comment
19
+ from .skip_html_tag import skip_html_tag
20
+
21
+
22
+ def scan_html(text: str, *, engine: TemplateEngine | None = None, collect_tags: bool = False) -> HtmlScan:
23
+ """Read a page once and find its comments, its blocks of code and the places where whitespace matters.
24
+
25
+ Args:
26
+ text: The source.
27
+ engine: The template language of the page; `None` for a page that is only HTML.
28
+ collect_tags: Also record every start tag and end tag (for the outline).
29
+
30
+ Returns:
31
+ The comments (and which of them are conditional), the template comments, the verbatim
32
+ blocks, the `script` and `style` blocks, the preserved ranges, the tags (if asked) and the
33
+ problems noticed.
34
+ """
35
+
36
+ # 1. A template is read first; the HTML scan then sees the page with the constructs blanked
37
+ templates = scan_templates(text, engine) if engine is not None else None
38
+ source = templates.masked if templates is not None else text
39
+
40
+ # 2. State: where the next stop is looked for, and what has been found
41
+ length = len(source)
42
+ position = 0
43
+ comments: list[Span] = []
44
+ conditional: list[Span] = []
45
+ blocks: list[HtmlBlock] = []
46
+ preserved: list[Span] = []
47
+ tags: list[HtmlTag] = []
48
+ problems: list[str] = []
49
+ pre_start: int | None = None
50
+
51
+ while True:
52
+
53
+ # 3. Jump to the next place where markup may begin; the text in between is not looked at
54
+ found = HTML_TOKEN_PATTERN.search(source, position)
55
+ if found is None:
56
+ break
57
+ start = found.start()
58
+ token = found.group()
59
+
60
+ # 4. A comment
61
+ if token == "<!--":
62
+ end, closed = skip_html_comment(source, start)
63
+ if not closed:
64
+ problems.append(
65
+ f"a comment on line {line_of(source, start)} is not closed; it runs to the end of the file"
66
+ )
67
+ span = fit_removal_span(text, start, end)
68
+ comments.append(span)
69
+ if HTML_CONDITIONAL_PATTERN.match(source, start):
70
+ conditional.append(span)
71
+ position = end
72
+
73
+ # 5. A CDATA section (in svg and math)
74
+ elif token.upper() == "<![CDATA[":
75
+ close = source.find("]]>", start)
76
+ if close == -1:
77
+ problems.append(f"a CDATA section on line {line_of(source, start)} is not closed")
78
+ position = length
79
+ else:
80
+ position = close + 3
81
+
82
+ # 6. A declaration (`<!DOCTYPE html>`) or a processing instruction (`<?xml ... ?>`)
83
+ elif token == "<?" or token.startswith("<!"):
84
+ close = source.find(">", start)
85
+ if close == -1:
86
+ problems.append(f"a declaration on line {line_of(source, start)} is not closed")
87
+ position = length
88
+ else:
89
+ position = close + 1
90
+
91
+ # 7. An end tag; the end of `pre` closes the range where whitespace matters
92
+ elif token.startswith("</"):
93
+ end, closed = skip_html_tag(source, start)
94
+ if not closed:
95
+ problems.append(f"a tag on line {line_of(source, start)} is not closed")
96
+ name = HTML_TAG_NAME_PATTERN.match(source, start).group(1).lower()
97
+ if collect_tags:
98
+ tags.append(HtmlTag(name, True, (start, end)))
99
+ if name == "pre" and pre_start is not None:
100
+ preserved.append((pre_start, start))
101
+ pre_start = None
102
+ position = end
103
+
104
+ # 8. A start tag
105
+ else:
106
+ end, closed = skip_html_tag(source, start)
107
+ position = end
108
+ if not closed:
109
+ problems.append(f"a tag on line {line_of(source, start)} is not closed")
110
+ continue
111
+ name = HTML_TAG_NAME_PATTERN.match(source, start).group(1).lower()
112
+ if collect_tags:
113
+ tags.append(HtmlTag(name, False, (start, end)))
114
+
115
+ # 9. An element with raw text: the content is skipped as a whole, up to its end tag
116
+ end_pattern = HTML_RAW_TEXT_END_PATTERNS.get(name)
117
+ if end_pattern is not None:
118
+ closing = end_pattern.search(source, end)
119
+ content_end = length if closing is None else closing.start()
120
+ if closing is None:
121
+ problems.append(f"a {name} element on line {line_of(source, start)} is not closed")
122
+ if name in ("script", "style") and source[end:content_end].strip():
123
+ language = find_block_language(name, source[start:end])
124
+ blocks.append(HtmlBlock(name, (end, content_end), language))
125
+ elif name == "textarea":
126
+ preserved.append((end, content_end))
127
+ position = content_end
128
+
129
+ # 10. `pre` opens a range where whitespace matters
130
+ elif name == "pre" and pre_start is None:
131
+ pre_start = end
132
+
133
+ # 11. A `pre` that is never closed keeps its whitespace to the end of the page
134
+ if pre_start is not None:
135
+ preserved.append((pre_start, length))
136
+
137
+ # 12. What the template scan found goes into the result as well
138
+ template_comments = templates.comments if templates is not None else ()
139
+ verbatim = templates.verbatim if templates is not None else ()
140
+ template_problems = templates.problems if templates is not None else ()
141
+
142
+ return HtmlScan(
143
+ comments=tuple(comments),
144
+ conditional=tuple(conditional),
145
+ template_comments=tuple(template_comments),
146
+ verbatim=tuple(verbatim),
147
+ blocks=tuple(blocks),
148
+ preserved=tuple(preserved),
149
+ tags=tuple(tags),
150
+ problems=template_problems + tuple(problems),
151
+ )
152
+
153
+
154
+ _DESIGN_NOTES = """
155
+ # scan_html — A Scanner, Not a Parser
156
+
157
+ ## Purpose
158
+ Everything the HTML steps need to know about a page comes from this one pass:
159
+ where the comments are (not the `<!--` inside an attribute or inside a
160
+ script), where the code of another language lies, where whitespace is
161
+ meaningful and, for the outline, which tags there are.
162
+
163
+ ## Not a Parser
164
+ It builds no tree. It follows what decides where a comment begins: tags (to
165
+ skip their attributes), the elements with raw text, and the markup that starts
166
+ with `<!` or `<?`. That is why it cannot fail: there is no syntax it refuses.
167
+ The outline builds the little nesting it needs from the recorded tags.
168
+
169
+ ## It Jumps
170
+ Most of a page is text and tags. The scanner asks the pattern for the next
171
+ stop and jumps there instead of walking character by character.
172
+
173
+ ## Raw Text
174
+ `script` and `style` hold code, `textarea` and `title` hold text; none of them
175
+ holds a comment. Their content is skipped up to the end tag. The content of
176
+ `script` and `style` is reported as a block, with its language, for the step
177
+ that cleans it; the content of `textarea` is reported as preserved. Their
178
+ start and end tags are tags like any other.
179
+
180
+ ## Preserved Whitespace
181
+ The content of `pre` and `textarea` is where a changed space changes the page.
182
+ Comments inside `pre` are comments, so the scan goes on in it; only its range
183
+ is recorded.
184
+
185
+ ## Tags on Request
186
+ `collect_tags` records every start tag and end tag. A page has a lot of them
187
+ and almost no step needs them, so they are collected only for the outline.
188
+
189
+ ## Templates
190
+ With an `engine` the template constructs are read first and blanked, and the
191
+ HTML is read from the blanked text; the offsets are the same in both. Where a
192
+ cut begins and ends is decided on the original text: in the blanked one, the
193
+ blanks that stand for a `{{ x }}` would be taken for blanks that belong to the
194
+ comment, and the comment would cut the construct in front of it as well.
195
+
196
+ ## Problems
197
+ A comment, a tag, a CDATA section, a declaration or a raw element that is never
198
+ closed is written down in `problems`, and the steps do not trust a file that
199
+ has any: they leave it as it is and warn (see `scan_checked`).
200
+
201
+ ## Known Limits
202
+ * A `<` followed by a letter in text that is not HTML (`a<b` in prose, or in a
203
+ template when no engine is given) is read as a tag.
204
+ * `<script>` is read to the first `</script`, as a browser reads it, even when
205
+ that is inside a string of the script.
206
+ * The legacy raw elements `xmp`, `iframe`, `noembed` and `noframes` are read as
207
+ ordinary elements.
208
+ """
simplibs/extractor_html/logic/_helpers/scan_templates.py ADDED
@@ -0,0 +1,128 @@
1
+ from simplibs.text import Span
2
+ # Outers
3
+ from ...constants import HTML_TEMPLATE_BLOCKS, HTML_TEMPLATE_TAG_NAME_PATTERN, HTML_TEMPLATE_TOKEN_PATTERN
4
+ # Inners
5
+ from .TemplateScan import TemplateScan
6
+ from .TemplateTag import TemplateTag
7
+ from .blank_spans import blank_spans
8
+ from .fit_removal_span import fit_removal_span
9
+ from .line_of import line_of
10
+
11
+
12
+ _CLOSERS = {"{#": "#}", "{%": "%}", "{{": "}}"}
13
+
14
+
15
+ def scan_templates(text: str, engine: str) -> TemplateScan:
16
+ """Read the template constructs of a page once.
17
+
18
+ Args:
19
+ text: The source.
20
+ engine: `"django"` (constructs stay on one line) or `"jinja"` (they may span lines).
21
+
22
+ Returns:
23
+ The template comments, the verbatim blocks, the tags, the text with every construct blanked,
24
+ and the problems that were noticed.
25
+ """
26
+
27
+ # 1. State: what the engine allows, and what has been found
28
+ single_line = engine == "django"
29
+ blocks = HTML_TEMPLATE_BLOCKS[engine]
30
+ position = 0
31
+ comments: list[Span] = []
32
+ verbatim: list[Span] = []
33
+ tags: list[TemplateTag] = []
34
+ masked_regions: list[Span] = []
35
+ problems: list[str] = []
36
+
37
+ while True:
38
+
39
+ # 2. Jump to the next opening of a construct
40
+ found = HTML_TEMPLATE_TOKEN_PATTERN.search(text, position)
41
+ if found is None:
42
+ break
43
+ start = found.start()
44
+ token = found.group()
45
+
46
+ # 3. A construct that is not closed (in Django: on the same line) is only text
47
+ close = text.find(_CLOSERS[token], start + 2)
48
+ if close == -1 or (single_line and text.find("\n", start, close) != -1):
49
+ position = start + 2
50
+ continue
51
+ end = close + 2
52
+
53
+ # 4. A comment: removable, and hidden from the HTML scan
54
+ if token == "{#":
55
+ comments.append(fit_removal_span(text, start, end))
56
+
57
+ # 5. A tag: a few open a block that runs to an end tag, the others are recorded
58
+ elif token == "{%":
59
+ name_match = HTML_TEMPLATE_TAG_NAME_PATTERN.match(text, start)
60
+ name = name_match.group(1) if name_match else None
61
+ block = blocks.get(name) if name else None
62
+ if block is not None:
63
+ kind, end_pattern = block
64
+ closing = end_pattern.search(text, end)
65
+ if closing is None:
66
+ problems.append(f"a {name} block on line {line_of(text, start)} is not closed")
67
+ else:
68
+ end = closing.end()
69
+ if kind == "comment":
70
+ comments.append(fit_removal_span(text, start, end))
71
+ else:
72
+ verbatim.append((start, end))
73
+ elif name_match is not None:
74
+ arguments = text[name_match.end():close].strip()
75
+ if arguments.endswith(("-", "+")):
76
+ arguments = arguments[:-1].rstrip()
77
+ tags.append(TemplateTag(name, (start, end), arguments))
78
+
79
+ # 6. Everything found is blanked for the HTML scan; reading goes on behind it
80
+ masked_regions.append((start, end))
81
+ position = end
82
+
83
+ return TemplateScan(
84
+ tuple(comments),
85
+ tuple(verbatim),
86
+ tuple(tags),
87
+ blank_spans(text, masked_regions),
88
+ tuple(problems),
89
+ )
90
+
91
+
92
+ _DESIGN_NOTES = """
93
+ # scan_templates — The Curly Braces First
94
+
95
+ ## Purpose
96
+ Django and Jinja put `{{ }}`, `{% %}` and `{# #}` anywhere: in text, inside a
97
+ tag, inside a quoted value. They are read first, by their own rules, and the
98
+ page is then read as HTML with them blanked out, so `<a {% if a > b %}x{% endif %}>`
99
+ is a tag and `title="{{ name|default:"x" }}"` is one value.
100
+
101
+ ## Django and Jinja
102
+ Django never spans lines with a construct, so a `{#` without its `#}` on the
103
+ same line is only text. Jinja does, and its `{# ... #}` may be long.
104
+
105
+ ## Blocks
106
+ `{% comment %}`, `{% verbatim %}` and `{% raw %}` run to their end tag
107
+ (`HTML_TEMPLATE_BLOCKS`). Whatever HTML or template syntax they contain is not
108
+ read: the whole block is blanked, and the tags inside it are not recorded.
109
+
110
+ ## Tags
111
+ Every other tag is recorded with its name and arguments, so the steps that
112
+ work with the structure of a template need no scan of their own. A tag inside
113
+ an HTML comment is recorded: the engine renders it, whatever the browser
114
+ makes of the comment.
115
+
116
+ ## Only Text When Not Closed
117
+ `{{` without `}}` is plain text. It may be a Vue or Angular expression, a
118
+ piece of documentation or a typing mistake; it is no reason to call the file
119
+ damaged. Only a block that is opened and never closed is.
120
+
121
+ ## Known Limits
122
+ * `{{` in a page that is not a template at all (an Angular page, say) is read
123
+ as a variable if `}}` follows on the same line. That changes nothing for the
124
+ HTML scan: the construct is only blanked.
125
+ * Template constructs inside `<script>` and `<style>` are read too, as their
126
+ comments are to be removed there as well.
127
+ * The comment prefixes of Jinja line statements (`# for x in y`) are not read.
128
+ """
simplibs/extractor_html/logic/_helpers/scan_templates_checked.py ADDED
@@ -0,0 +1,39 @@
1
+ # Outers
2
+ from ...annotations import TemplateEngine
3
+ # Inners
4
+ from .TemplateScan import TemplateScan
5
+ from .scan_templates import scan_templates
6
+ from .warn_problems import warn_problems
7
+
8
+
9
+ def scan_templates_checked(text: str, *, engine: TemplateEngine) -> TemplateScan | None:
10
+ """Scan the template constructs of a page and say so when it cannot be trusted.
11
+
12
+ Args:
13
+ text: The source.
14
+ engine: `"django"` or `"jinja"`.
15
+
16
+ Returns:
17
+ The scan; `None` if the scanner noticed damage (a warning has been given for each problem).
18
+ """
19
+
20
+ # 1. One pass
21
+ scan = scan_templates(text, engine)
22
+
23
+ # 2. A template with damage is not read any further: the step leaves it as it is
24
+ if scan.problems:
25
+ warn_problems(scan.problems)
26
+ return None
27
+
28
+ return scan
29
+
30
+
31
+ _DESIGN_NOTES = """
32
+ # scan_templates_checked — Trust the Scan Only if It Found Nothing Wrong
33
+
34
+ ## Purpose
35
+ The template steps do not need to read the HTML of the page, only its
36
+ constructs, so they use this scan and not `scan_checked`. The rule is the
37
+ same: a template with a block that is never closed is left as it is, with a
38
+ warning, and the extractor calls the file `partial`.
39
+ """
simplibs/extractor_html/logic/_helpers/skip_html_comment.py ADDED
@@ -0,0 +1,41 @@
1
+ from ...constants import HTML_COMMENT_END_PATTERN
2
+
3
+
4
+ def skip_html_comment(text: str, start: int) -> tuple[int, bool]:
5
+ """Find where a comment ends.
6
+
7
+ Args:
8
+ text: The source.
9
+ start: The offset of the opening `<!--`.
10
+
11
+ Returns:
12
+ The offset just after the end of the comment and `True`; for a comment that is never
13
+ closed, the end of the text and `False`.
14
+ """
15
+
16
+ # 1. The two shortest comments end at once: `<!-->` and `<!--->`
17
+ content = start + 4
18
+ if text.startswith(">", content):
19
+ return content + 1, True
20
+ if text.startswith("->", content):
21
+ return content + 2, True
22
+
23
+ # 2. Otherwise the first `-->` (or `--!>`) ends it
24
+ closing = HTML_COMMENT_END_PATTERN.search(text, content)
25
+ if closing is None:
26
+ return len(text), False
27
+
28
+ return closing.end(), True
29
+
30
+
31
+ _DESIGN_NOTES = """
32
+ # skip_html_comment — A Comment Ends at `-->`
33
+
34
+ ## Purpose
35
+ The one rule of an HTML comment, as the specification gives it. Comments do
36
+ not nest, so `<!-- a <!-- b --> c -->` ends at the first `-->`.
37
+
38
+ ## An Unclosed Comment
39
+ By the specification, and so for every browser, it runs to the end of the
40
+ file. The scanner reports it: a page cut in half is usually a damaged file.
41
+ """
simplibs/extractor_html/logic/_helpers/skip_html_tag.py ADDED
@@ -0,0 +1,46 @@
1
+ def skip_html_tag(text: str, start: int) -> tuple[int, bool]:
2
+ """Find where a start tag or an end tag ends.
3
+
4
+ Args:
5
+ text: The source (template constructs already blanked).
6
+ start: The offset of the `<`.
7
+
8
+ Returns:
9
+ The offset just after the closing `>` and `True`; for a tag that is never closed, the end
10
+ of the text and `False`.
11
+ """
12
+
13
+ # 1. Walk to the first `>` that is not inside a quoted value
14
+ length = len(text)
15
+ position = start + 1
16
+ while position < length:
17
+ character = text[position]
18
+ if character == ">":
19
+ return position + 1, True
20
+ if character in "\"'":
21
+ closing = text.find(character, position + 1)
22
+ if closing == -1:
23
+ return length, False
24
+ position = closing + 1
25
+ else:
26
+ position += 1
27
+
28
+ return length, False
29
+
30
+
31
+ _DESIGN_NOTES = """
32
+ # skip_html_tag — A Tag Ends at the First `>` Outside Quotes
33
+
34
+ ## Purpose
35
+ `<a title="a > b" href='x'>` is one tag. A `>` or a `<!--` inside a quoted
36
+ value is a part of the value, so the tag is skipped as a whole and nothing in
37
+ it is taken for a comment.
38
+
39
+ ## Quotes
40
+ Any quote opens a value and ends at the same quote, line breaks included, as
41
+ in HTML. A quote that is never closed makes the tag, and the file, damaged.
42
+
43
+ ## Template Constructs
44
+ They are blanked before the scan, so `title="{{ x|default:"y" }}"` has no
45
+ quote inside quote left.
46
+ """
simplibs/extractor_html/logic/_helpers/tidy_lines.py ADDED
@@ -0,0 +1,86 @@
1
+ # Inners
2
+ from .Ranges import Ranges
3
+
4
+
5
+ def tidy_lines(text: str, protected: Ranges, *, keep_blank_line: bool) -> str:
6
+ """Trim the end of every line and reduce the blank lines, except in the lines that touch a protected range.
7
+
8
+ Args:
9
+ text: The source.
10
+ protected: The ranges that must not change.
11
+ keep_blank_line: `True` to keep one blank line where there were several (in the middle of the
12
+ text); `False` to drop every blank line.
13
+
14
+ Returns:
15
+ The tidied text. A line that touches a protected range is returned exactly as it was.
16
+ """
17
+
18
+ # 1. Lines without their breaks; a final line break is put back at the end
19
+ has_final_break = text.endswith("\n")
20
+ lines = text.split("\n")
21
+ if has_final_break:
22
+ lines.pop()
23
+
24
+ result: list[str] = []
25
+ pending_blank: str | None = None
26
+ offset = 0
27
+
28
+ for line in lines:
29
+
30
+ # 2. Where this line lies in the text, with its break
31
+ start = offset
32
+ offset += len(line) + 1
33
+
34
+ # 3. A line with a protected part is kept as it is; a blank line owed before it is paid
35
+ if protected.touches(start, offset):
36
+ if pending_blank is not None:
37
+ result.append(pending_blank)
38
+ pending_blank = None
39
+ result.append(line)
40
+ continue
41
+
42
+ # 4. The end of the line is trimmed; a carriage return of a Windows line break stays
43
+ carriage = "\r" if line.endswith("\r") else ""
44
+ body = (line[:-1] if carriage else line).rstrip(" \t")
45
+
46
+ # 5. A blank line is dropped, or reduced to one that is written only if text follows
47
+ if not body.strip():
48
+ if keep_blank_line and result and pending_blank is None:
49
+ pending_blank = carriage
50
+ continue
51
+
52
+ # 6. A line with text: the blank line it was owed first
53
+ if pending_blank is not None:
54
+ result.append(pending_blank)
55
+ pending_blank = None
56
+ result.append(body + carriage)
57
+
58
+ # 7. Blank lines at the end are not written (the one still owed is dropped)
59
+ tidied = "\n".join(result)
60
+ return tidied + "\n" if has_final_break and result else tidied
61
+
62
+
63
+ _DESIGN_NOTES = """
64
+ # tidy_lines — The Lines of a Page, Tidied
65
+
66
+ ## Purpose
67
+ The shared work of the two whitespace steps: the blanks at the end of each
68
+ line go, and the blank lines go or are reduced to one.
69
+
70
+ ## The Protected Lines
71
+ A line that has any part in a protected range is returned untouched, and a
72
+ blank line owed in front of it is paid first, so the spacing around a `<pre>`
73
+ is kept as it was reduced to.
74
+
75
+ ## The Edges
76
+ There is no blank line at the start or at the end of the result, and a text
77
+ that ended with a line break still does. A text of blank lines only is empty.
78
+
79
+ ## Windows Line Breaks
80
+ The carriage return in front of a line break is kept: the step trims blanks,
81
+ it does not change the kind of line break of the file.
82
+
83
+ ## Known Limits
84
+ A quoted attribute value or a template tag that spans lines is tidied as any
85
+ other text: trailing blanks inside it go.
86
+ """
simplibs/extractor_html/logic/_helpers/warn_problems.py ADDED
@@ -0,0 +1,32 @@
1
+ import warnings
2
+ from collections.abc import Iterable
3
+
4
+
5
+ def warn_problems(problems: Iterable[str]) -> None:
6
+ """Turn sentences about a damaged file into warnings.
7
+
8
+ Args:
9
+ problems: What looked wrong, one sentence each.
10
+ """
11
+
12
+ # 1. One warning for each, in the order of the file
13
+ for problem in problems:
14
+ warnings.warn(problem, UserWarning, stacklevel=2)
15
+
16
+
17
+ _DESIGN_NOTES = """
18
+ # warn_problems — A File That Is Not Quite Right Is Not an Error
19
+
20
+ ## Purpose
21
+ A step is a plain function of a text and knows nothing about the extractor.
22
+ `warnings.warn` is the one channel it can use without importing anything. The
23
+ extractor collects the warnings of a file, calls the file `partial` and writes
24
+ the sentence into the report, while the file stays in the output.
25
+
26
+ ## Why `UserWarning`
27
+ It is the category the extractor listens to.
28
+
29
+ ## Outside the Extractor
30
+ Called on its own, the step gives the usual Python warning on the error
31
+ stream, once for each place.
32
+ """
simplibs/extractor_html/logic/apply_to_html_blocks.py ADDED
@@ -0,0 +1,103 @@
1
+ from collections.abc import Callable
2
+ from simplibs.validate import validate_call
3
+ # Outers
4
+ from ..annotations import TemplateEngine
5
+ # Inners
6
+ from ._helpers import raise_block_result_invalid, replace_spans, scan_checked
7
+
8
+
9
+ @validate_call
10
+ def apply_to_html_blocks(
11
+ text: str,
12
+ *,
13
+ script: Callable[[str], str] | None = None,
14
+ style: Callable[[str], str] | None = None,
15
+ engine: TemplateEngine | None = None,
16
+ ) -> str:
17
+ """Run a step on the code inside the `script` and `style` elements of a page.
18
+
19
+ Args:
20
+ text: The source.
21
+ script: The step for the content of each `<script>` that holds JavaScript (no `type`, `module`,
22
+ `text/javascript`...). `None` leaves them as they are.
23
+ style: The step for the content of each `<style>` that holds CSS. `None` leaves them as they are.
24
+ engine: The template language of the page (`"django"` or `"jinja"`); `None` for a page that is only HTML.
25
+
26
+ Returns:
27
+ The page with the content of those elements replaced by what the steps returned. A `script` that holds
28
+ JSON or a template of a framework is not touched. A page that looks damaged (a comment, a tag or a
29
+ `script` that is never closed) comes back unchanged, with a warning.
30
+
31
+ Raises:
32
+ ParamError: A step returned something that is not a text.
33
+
34
+ Example:
35
+ apply_to_html_blocks("<style>a{}</style>", style=str.upper)
36
+ # '<style>A{}</style>'
37
+ """
38
+
39
+ # 1. Read the page; a damaged one is left as it is
40
+ scan = scan_checked(text, engine=engine)
41
+ if scan is None:
42
+ return text
43
+
44
+ # 2. Each block of code gets the step of its kind
45
+ steps = {"script": script, "style": style}
46
+ replacements = []
47
+ for block in scan.blocks:
48
+ step = steps[block.kind]
49
+ if step is None or block.language is None:
50
+ continue
51
+
52
+ start, end = block.span
53
+ content = text[start:end]
54
+ result = step(content)
55
+ if not isinstance(result, str):
56
+ raise_block_result_invalid(block.kind, result)
57
+ if result != content:
58
+ replacements.append((block.span, result))
59
+
60
+ # 3. Put the new content where the old one was
61
+ return replace_spans(text, replacements)
62
+
63
+
64
+ _DESIGN_NOTES = """
65
+ # apply_to_html_blocks — Another Language Inside a Page
66
+
67
+ ## Purpose
68
+ A page holds JavaScript in `<script>` and CSS in `<style>`. The steps that
69
+ clean them exist (the JavaScript and CSS add-ons); this one gives them the
70
+ right piece of the page and puts the result back. It knows nothing about what
71
+ the steps do: they are parameters, so the steps of other add-ons are used
72
+ without this library depending on them.
73
+
74
+ ## Steps Are Functions of a Text
75
+ Any `str -> str` is a step: a function, a lambda, an action, a whole pipeline
76
+ (`StripJsComments() >> RemoveJsImports()`).
77
+
78
+ ## Only Code
79
+ A block whose `type` says it is not code (`application/json`, `importmap`, a
80
+ template of a framework) is data, and the step never sees it. A block with
81
+ nothing in it (`<script src="a.js"></script>`) is not offered either.
82
+
83
+ ## The Content Only
84
+ A step gets what stands between `>` and `</script`, and nothing more, so a
85
+ line comment at the end of an inline script cannot reach the end tag.
86
+
87
+ ## Templates
88
+ With an `engine` the page is read with the template constructs blanked, so
89
+ `{{ x }}` or a `{% if %}` inside a start tag does not hide the block. The
90
+ step is given the original content, template tags and all.
91
+
92
+ ## What It Does Not Touch
93
+ The attributes that hold code (`style="..."`, `onclick="..."`) are not blocks
94
+ and stay as they are.
95
+
96
+ ## Errors
97
+ A step that raises is not caught: the exception reaches the caller as it was.
98
+ A step that returns something other than a text is reported by name.
99
+
100
+ ## A Damaged Page
101
+ It is returned as it is, with a warning, so the extractor calls the file
102
+ `partial`.
103
+ """