justhtml 3.0.0__tar.gz → 3.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. {justhtml-3.0.0 → justhtml-3.1.0}/CHANGELOG.md +14 -0
  2. {justhtml-3.0.0 → justhtml-3.1.0}/PKG-INFO +1 -1
  3. {justhtml-3.0.0 → justhtml-3.1.0}/benchmarks/profile.py +34 -4
  4. {justhtml-3.0.0 → justhtml-3.1.0}/docs/api.md +9 -0
  5. {justhtml-3.0.0 → justhtml-3.1.0}/docs/building.md +27 -0
  6. {justhtml-3.0.0 → justhtml-3.1.0}/docs/transforms.md +8 -3
  7. {justhtml-3.0.0 → justhtml-3.1.0}/docs/url-cleaning.md +5 -2
  8. {justhtml-3.0.0 → justhtml-3.1.0}/pyproject.toml +1 -1
  9. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/core/constants.py +51 -0
  10. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/parser/engine.py +28 -9
  11. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/selector/core.py +10 -7
  12. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/serializer/html.py +33 -23
  13. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/transforms/__init__.py +2 -0
  14. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/transforms/compile.py +7 -3
  15. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/transforms/runtime.py +37 -2
  16. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/transforms/spec.py +9 -2
  17. {justhtml-3.0.0 → justhtml-3.1.0}/tests/test_errors.py +19 -0
  18. {justhtml-3.0.0 → justhtml-3.1.0}/tests/test_parser_engine_coverage.py +24 -2
  19. {justhtml-3.0.0 → justhtml-3.1.0}/tests/test_sanitize.py +30 -0
  20. {justhtml-3.0.0 → justhtml-3.1.0}/tests/test_serialize.py +11 -0
  21. {justhtml-3.0.0 → justhtml-3.1.0}/tests/test_transforms.py +85 -0
  22. {justhtml-3.0.0 → justhtml-3.1.0}/.github/copilot-instructions.md +0 -0
  23. {justhtml-3.0.0 → justhtml-3.1.0}/.github/workflows/ci.yml +0 -0
  24. {justhtml-3.0.0 → justhtml-3.1.0}/.github/workflows/publish.yml +0 -0
  25. {justhtml-3.0.0 → justhtml-3.1.0}/.gitignore +0 -0
  26. {justhtml-3.0.0 → justhtml-3.1.0}/.pre-commit-config.yaml +0 -0
  27. {justhtml-3.0.0 → justhtml-3.1.0}/CODE_OF_CONDUCT.md +0 -0
  28. {justhtml-3.0.0 → justhtml-3.1.0}/CONTRIBUTING.md +0 -0
  29. {justhtml-3.0.0 → justhtml-3.1.0}/LICENSE +0 -0
  30. {justhtml-3.0.0 → justhtml-3.1.0}/Makefile +0 -0
  31. {justhtml-3.0.0 → justhtml-3.1.0}/README.md +0 -0
  32. {justhtml-3.0.0 → justhtml-3.1.0}/SECURITY.md +0 -0
  33. {justhtml-3.0.0 → justhtml-3.1.0}/assets/justhtml-readme-explainer.png +0 -0
  34. {justhtml-3.0.0 → justhtml-3.1.0}/benchmarks/chromium_parser_diff.py +0 -0
  35. {justhtml-3.0.0 → justhtml-3.1.0}/benchmarks/correctness.py +0 -0
  36. {justhtml-3.0.0 → justhtml-3.1.0}/benchmarks/fuzz.py +0 -0
  37. {justhtml-3.0.0 → justhtml-3.1.0}/benchmarks/html5lib_engine_diff.py +0 -0
  38. {justhtml-3.0.0 → justhtml-3.1.0}/benchmarks/parser_engine_gate.py +0 -0
  39. {justhtml-3.0.0 → justhtml-3.1.0}/benchmarks/performance.py +0 -0
  40. {justhtml-3.0.0 → justhtml-3.1.0}/docs/_config.yml +0 -0
  41. {justhtml-3.0.0 → justhtml-3.1.0}/docs/_layouts/default.html +0 -0
  42. {justhtml-3.0.0 → justhtml-3.1.0}/docs/assets/search.js +0 -0
  43. {justhtml-3.0.0 → justhtml-3.1.0}/docs/bleach-migration.md +0 -0
  44. {justhtml-3.0.0 → justhtml-3.1.0}/docs/cli.md +0 -0
  45. {justhtml-3.0.0 → justhtml-3.1.0}/docs/comparison.md +0 -0
  46. {justhtml-3.0.0 → justhtml-3.1.0}/docs/correctness.md +0 -0
  47. {justhtml-3.0.0 → justhtml-3.1.0}/docs/encoding.md +0 -0
  48. {justhtml-3.0.0 → justhtml-3.1.0}/docs/errors.md +0 -0
  49. {justhtml-3.0.0 → justhtml-3.1.0}/docs/fragments.md +0 -0
  50. {justhtml-3.0.0 → justhtml-3.1.0}/docs/html-cleaning.md +0 -0
  51. {justhtml-3.0.0 → justhtml-3.1.0}/docs/index.md +0 -0
  52. {justhtml-3.0.0 → justhtml-3.1.0}/docs/linkify.md +0 -0
  53. {justhtml-3.0.0 → justhtml-3.1.0}/docs/migration-examples.md +0 -0
  54. {justhtml-3.0.0 → justhtml-3.1.0}/docs/parser-engine-findings.md +0 -0
  55. {justhtml-3.0.0 → justhtml-3.1.0}/docs/playground/app.js +0 -0
  56. {justhtml-3.0.0 → justhtml-3.1.0}/docs/playground/index.html +0 -0
  57. {justhtml-3.0.0 → justhtml-3.1.0}/docs/playground/py/__init__.py +0 -0
  58. {justhtml-3.0.0 → justhtml-3.1.0}/docs/playground/py/install_latest_justhtml.py +0 -0
  59. {justhtml-3.0.0 → justhtml-3.1.0}/docs/playground/py/render.py +0 -0
  60. {justhtml-3.0.0 → justhtml-3.1.0}/docs/playground/py/use_local_repo.py +0 -0
  61. {justhtml-3.0.0 → justhtml-3.1.0}/docs/quickstart.md +0 -0
  62. {justhtml-3.0.0 → justhtml-3.1.0}/docs/sanitization.md +0 -0
  63. {justhtml-3.0.0 → justhtml-3.1.0}/docs/selectors.md +0 -0
  64. {justhtml-3.0.0 → justhtml-3.1.0}/docs/streaming.md +0 -0
  65. {justhtml-3.0.0 → justhtml-3.1.0}/docs/text.md +0 -0
  66. {justhtml-3.0.0 → justhtml-3.1.0}/docs/unsafe-handling.md +0 -0
  67. {justhtml-3.0.0 → justhtml-3.1.0}/llms.txt +0 -0
  68. {justhtml-3.0.0 → justhtml-3.1.0}/run_tests.py +0 -0
  69. {justhtml-3.0.0 → justhtml-3.1.0}/scripts/release.py +0 -0
  70. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/__init__.py +0 -0
  71. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/__main__.py +0 -0
  72. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/core/__init__.py +0 -0
  73. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/core/doctype.py +0 -0
  74. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/core/entities.py +0 -0
  75. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/core/errors.py +0 -0
  76. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/core/rawtext.py +0 -0
  77. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/core/types.py +0 -0
  78. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/dom/__init__.py +0 -0
  79. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/dom/builder.py +0 -0
  80. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/parser/__init__.py +0 -0
  81. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/parser/context.py +0 -0
  82. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/parser/encoding.py +0 -0
  83. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/parser/options.py +0 -0
  84. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/parser/scanner.py +0 -0
  85. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/parser/stream.py +0 -0
  86. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/py.typed +0 -0
  87. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/sanitizer/__init__.py +0 -0
  88. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/sanitizer/css.py +0 -0
  89. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/sanitizer/dom.py +0 -0
  90. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/sanitizer/policy.py +0 -0
  91. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/sanitizer/policy_defaults.py +0 -0
  92. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/sanitizer/rawtext.py +0 -0
  93. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/sanitizer/url/__init__.py +0 -0
  94. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/sanitizer/url/policy.py +0 -0
  95. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/sanitizer/url/runtime.py +0 -0
  96. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/sanitizer/url/spec.py +0 -0
  97. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/selector/__init__.py +0 -0
  98. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/serializer/__init__.py +0 -0
  99. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/serializer/markdown.py +0 -0
  100. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/transforms/linkify.py +0 -0
  101. {justhtml-3.0.0 → justhtml-3.1.0}/src/justhtml/transforms/linkify_core.py +0 -0
  102. {justhtml-3.0.0 → justhtml-3.1.0}/test-summary.txt +0 -0
  103. {justhtml-3.0.0 → justhtml-3.1.0}/tests/README.md +0 -0
  104. {justhtml-3.0.0 → justhtml-3.1.0}/tests/__init__.py +0 -0
  105. {justhtml-3.0.0 → justhtml-3.1.0}/tests/data/wikipedia.html +0 -0
  106. {justhtml-3.0.0 → justhtml-3.1.0}/tests/harness/__init__.py +0 -0
  107. {justhtml-3.0.0 → justhtml-3.1.0}/tests/harness/encoding.py +0 -0
  108. {justhtml-3.0.0 → justhtml-3.1.0}/tests/harness/regressions.py +0 -0
  109. {justhtml-3.0.0 → justhtml-3.1.0}/tests/harness/reporter.py +0 -0
  110. {justhtml-3.0.0 → justhtml-3.1.0}/tests/harness/serializer.py +0 -0
  111. {justhtml-3.0.0 → justhtml-3.1.0}/tests/harness/tree.py +0 -0
  112. {justhtml-3.0.0 → justhtml-3.1.0}/tests/justhtml-sanitize-tests/cases.json +0 -0
  113. {justhtml-3.0.0 → justhtml-3.1.0}/tests/justhtml-tests/branch_coverage.dat +0 -0
  114. {justhtml-3.0.0 → justhtml-3.1.0}/tests/justhtml-tests/coverage_gaps.test +0 -0
  115. {justhtml-3.0.0 → justhtml-3.1.0}/tests/justhtml-tests/empty_stack_edge_cases.dat +0 -0
  116. {justhtml-3.0.0 → justhtml-3.1.0}/tests/justhtml-tests/entities.test +0 -0
  117. {justhtml-3.0.0 → justhtml-3.1.0}/tests/justhtml-tests/iframe_srcdoc.dat +0 -0
  118. {justhtml-3.0.0 → justhtml-3.1.0}/tests/justhtml-tests/tokenizer_edge_cases.test +0 -0
  119. {justhtml-3.0.0 → justhtml-3.1.0}/tests/justhtml-tests/treebuilder_coverage.dat +0 -0
  120. {justhtml-3.0.0 → justhtml-3.1.0}/tests/justhtml-tests/xml_coercion.dat +0 -0
  121. {justhtml-3.0.0 → justhtml-3.1.0}/tests/justhtml-tests/xml_coercion_coverage.test +0 -0
  122. {justhtml-3.0.0 → justhtml-3.1.0}/tests/linkify-it/LICENSE.txt +0 -0
  123. {justhtml-3.0.0 → justhtml-3.1.0}/tests/linkify-it/README.md +0 -0
  124. {justhtml-3.0.0 → justhtml-3.1.0}/tests/linkify-it/fixtures/links.txt +0 -0
  125. {justhtml-3.0.0 → justhtml-3.1.0}/tests/linkify-it/fixtures/not_links.txt +0 -0
  126. {justhtml-3.0.0 → justhtml-3.1.0}/tests/test_builder.py +0 -0
  127. {justhtml-3.0.0 → justhtml-3.1.0}/tests/test_cli.py +0 -0
  128. {justhtml-3.0.0 → justhtml-3.1.0}/tests/test_docs_examples.py +0 -0
  129. {justhtml-3.0.0 → justhtml-3.1.0}/tests/test_encoding.py +0 -0
  130. {justhtml-3.0.0 → justhtml-3.1.0}/tests/test_entities.py +0 -0
  131. {justhtml-3.0.0 → justhtml-3.1.0}/tests/test_linkify_internals.py +0 -0
  132. {justhtml-3.0.0 → justhtml-3.1.0}/tests/test_linkify_it.py +0 -0
  133. {justhtml-3.0.0 → justhtml-3.1.0}/tests/test_linkify_transform.py +0 -0
  134. {justhtml-3.0.0 → justhtml-3.1.0}/tests/test_node.py +0 -0
  135. {justhtml-3.0.0 → justhtml-3.1.0}/tests/test_parser_regressions.py +0 -0
  136. {justhtml-3.0.0 → justhtml-3.1.0}/tests/test_parser_tree.py +0 -0
  137. {justhtml-3.0.0 → justhtml-3.1.0}/tests/test_playground_local_repo_file_list.py +0 -0
  138. {justhtml-3.0.0 → justhtml-3.1.0}/tests/test_sanitize_integration.py +0 -0
  139. {justhtml-3.0.0 → justhtml-3.1.0}/tests/test_sanitize_transform.py +0 -0
  140. {justhtml-3.0.0 → justhtml-3.1.0}/tests/test_selector.py +0 -0
  141. {justhtml-3.0.0 → justhtml-3.1.0}/tests/test_stream.py +0 -0
  142. {justhtml-3.0.0 → justhtml-3.1.0}/tests/test_transforms_compiler.py +0 -0
  143. {justhtml-3.0.0 → justhtml-3.1.0}/tests/test_transforms_edge_cases.py +0 -0
  144. {justhtml-3.0.0 → justhtml-3.1.0}/tests/test_transforms_sanitize_integration.py +0 -0
  145. {justhtml-3.0.0 → justhtml-3.1.0}/tests/test_wikipedia.py +0 -0
@@ -7,6 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
7
7
 
8
8
  ## [Unreleased]
9
9
 
10
+ ## [3.1.0] - 2026-07-02
11
+
12
+ ### Changed
13
+ - `CollapseWhitespace(...)` now trims leading and trailing whitespace at block-container edges by default. Pass `trim_blocks=False` to keep the previous html5lib-like behavior, or customize `block_tags=(...)` for project-specific block containers.
14
+ - Document that DOM insertion methods adopt existing nodes, and show how to clone nodes before appending when using parsed DOMs as templates.
15
+
16
+ ### Fixed
17
+ - Apply explicit `UrlPolicy.allow_rules` to custom URL attributes in constructor sanitization and `DropUrlAttrs(...)`. Previously, rules such as `("el-custom", "data-url")` were ignored unless the attribute name was one of JustHTML's built-in URL sinks.
18
+ - Restore strict-mode reporting for unexpected `</p>` end tags after implicit paragraph closure.
19
+
20
+ ### Performance
21
+ - Speed up pretty HTML serialization for direct text-only elements, repeated tag/attribute names, and common empty-attribute elements; reduce repeated selector name normalization and hot parser namespace checks; and fix the real-world profiler script when run directly.
22
+ - Speed up parser URL sanitization for common lowercase `http://` and `https://` attributes.
23
+
10
24
  ## [3.0.0] - 2026-06-21
11
25
 
12
26
  JustHTML 3.0.0 ships a new parser core. The old tokenizer and tree-builder are now one integrated system, which makes the parser much faster while keeping browser-style HTML5 recovery and default-safe behavior intact. In practice, this release is mainly about getting materially better performance without asking most users to change how they use the library.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: justhtml
3
- Version: 3.0.0
3
+ Version: 3.1.0
4
4
  Summary: A pure Python HTML5 parser that just works.
5
5
  Project-URL: Homepage, https://github.com/emilstenstrom/justhtml
6
6
  Project-URL: Issues, https://github.com/emilstenstrom/justhtml/issues
@@ -2,9 +2,10 @@
2
2
  """Profile JustHTML on real-world HTML."""
3
3
 
4
4
  import argparse
5
- import cProfile
5
+ import importlib
6
6
  import pathlib
7
7
  import pstats
8
+ import sys
8
9
  import tarfile
9
10
 
10
11
  import zstandard as zstd
@@ -49,15 +50,30 @@ def load_html_files(batch_path: pathlib.Path, dict_bytes: bytes, limit: int = 10
49
50
 
50
51
  def parse_args() -> argparse.Namespace:
51
52
  parser = argparse.ArgumentParser(description=__doc__)
53
+ parser.add_argument(
54
+ "--mode",
55
+ choices=["parse", "compact-html", "pretty-html", "text", "fragment"],
56
+ default="parse",
57
+ help="Pipeline to profile (default: parse)",
58
+ )
52
59
  parser.add_argument(
53
60
  "--to-html",
54
61
  action="store_true",
55
- help="Profile parse + serialization via to_html() (pretty=True by default)",
62
+ help="Deprecated alias for --mode pretty-html",
56
63
  )
57
64
  return parser.parse_args()
58
65
 
59
66
 
67
+ def remove_script_dir_from_import_path() -> None:
68
+ script_dir = str(pathlib.Path(__file__).resolve().parent)
69
+ if sys.path and sys.path[0] == script_dir:
70
+ sys.path.pop(0)
71
+
72
+
60
73
  def main() -> None:
74
+ remove_script_dir_from_import_path()
75
+ cprofile = importlib.import_module("cProfile")
76
+
61
77
  args = parse_args()
62
78
 
63
79
  dict_bytes = load_dict(pathlib.Path("/home/emilstenstrom/Projects/web100k/html.dict"))
@@ -69,12 +85,26 @@ def main() -> None:
69
85
 
70
86
  print(f"Loaded {len(html_files)} files")
71
87
 
72
- profiler = cProfile.Profile()
88
+ mode = "pretty-html" if args.to_html else args.mode
89
+
90
+ profiler = cprofile.Profile()
73
91
  profiler.enable()
74
92
 
75
93
  for _filename, html in html_files:
94
+ if mode == "fragment":
95
+ parser = JustHTML(html, fragment=True)
96
+ _ = parser.root
97
+ continue
98
+
76
99
  parser = JustHTML(html)
77
- _ = parser.to_html() if args.to_html else parser.root
100
+ if mode == "compact-html":
101
+ _ = parser.to_html(pretty=False)
102
+ elif mode == "pretty-html":
103
+ _ = parser.to_html()
104
+ elif mode == "text":
105
+ _ = parser.to_text()
106
+ else:
107
+ _ = parser.root
78
108
 
79
109
  profiler.disable()
80
110
 
@@ -408,10 +408,16 @@ When you build documents with `JustHTML(..., sanitize=True)` (the default), this
408
408
 
409
409
  Append a child node to this node.
410
410
 
411
+ If `node` already has a parent, it is moved from that parent to this node. Use
412
+ `node.clone_node(deep=True)` before appending when you want to keep the source
413
+ DOM unchanged.
414
+
411
415
  #### `insert_before(node, reference_node)`
412
416
 
413
417
  Insert `node` before `reference_node` (or append if `reference_node` is `None`).
414
418
 
419
+ If `node` already has a parent, it is moved from that parent to this node.
420
+
415
421
  #### `remove_child(node)`
416
422
 
417
423
  Remove a direct child node.
@@ -420,6 +426,9 @@ Remove a direct child node.
420
426
 
421
427
  Replace a direct child node with a new node.
422
428
 
429
+ If `new_node` already has a parent, it is moved from that parent before it
430
+ replaces `old_node`.
431
+
423
432
  #### `clone_node(deep=False, override_attrs=None)`
424
433
 
425
434
  Clone this node. If `deep=True`, children are cloned recursively.
@@ -435,6 +435,33 @@ lead = doc.query_one("p.lead")
435
435
  lead.to_html(pretty=False) # => <p class="lead">Hello</p>
436
436
  ```
437
437
 
438
+ ### Clone from template DOMs before appending
439
+
440
+ DOM insertion methods adopt existing nodes. If you append a node that already
441
+ belongs to another tree, it is removed from its old parent and moved into the new
442
+ tree.
443
+
444
+ When you use one parsed DOM as a template for another document, clone the source
445
+ node before appending it:
446
+
447
+ ```python
448
+ from justhtml import JustHTML
449
+ from justhtml.dom.builder import element
450
+
451
+ template = JustHTML('<section><p class="lead">Hello</p></section>', fragment=True, sanitize=False)
452
+ output = JustHTML(element("main"), fragment=True, sanitize=False)
453
+
454
+ lead = template.query_one("p.lead")
455
+ assert lead is not None
456
+ main = output.query_one("main")
457
+ assert main is not None
458
+
459
+ main.append_child(lead.clone_node(deep=True))
460
+
461
+ template.to_html(pretty=False) # => <section><p class="lead">Hello</p></section>
462
+ output.to_html(pretty=False) # => <main><p class="lead">Hello</p></main>
463
+ ```
464
+
438
465
  ## When Not to Use the Builder
439
466
 
440
467
  Don’t use the builder just because it exists.
@@ -268,11 +268,16 @@ Advanced building blocks (useful for policy-driven pipelines):
268
268
 
269
269
  See [`Linkify(...)`](linkify.md) for full documentation and examples.
270
270
 
271
- ### `CollapseWhitespace(skip_tags=(...), enabled=True, callback=None, report=None)`
271
+ ### `CollapseWhitespace(skip_tags=(...), trim_blocks=True, block_tags=(...), enabled=True, callback=None, report=None)`
272
272
 
273
273
  Collapses runs of HTML whitespace characters in text nodes to a single space.
274
+ By default, it also trims leading and trailing whitespace at the edges of block
275
+ containers such as `<p>`, `<div>`, and `<li>`.
274
276
 
275
- This is similar to `html5lib.filters.whitespace.Filter`.
277
+ Pass `trim_blocks=False` for html5lib-like whitespace filtering without edge
278
+ trimming. Use `block_tags=(...)` to define which containers trim their edge
279
+ whitespace; `skip_tags` still wins and preserves whitespace inside matching
280
+ tags.
276
281
 
277
282
  By default it skips `<pre>`, `<textarea>`, `<code>`, `<title>`, `<script>`, and `<style>`.
278
283
 
@@ -280,7 +285,7 @@ By default it skips `<pre>`, `<textarea>`, `<code>`, `<title>`, `<script>`, and
280
285
  from justhtml import CollapseWhitespace, JustHTML
281
286
 
282
287
  doc = JustHTML(
283
- "<p>Hello \n\t world</p><pre>a b</pre>",
288
+ "<p>\n Hello \n\t world\n</p><pre>a b</pre>",
284
289
  fragment=True,
285
290
  transforms=[CollapseWhitespace()],
286
291
  )
@@ -85,7 +85,8 @@ Output:
85
85
 
86
86
  ## How URL cleaning works (in order)
87
87
 
88
- For a URL-like attribute (like `img[src]` or `a[href]`), JustHTML applies these steps:
88
+ For a URL-like attribute (like `img[src]` or `a[href]`) or a custom attribute with an explicit
89
+ `UrlPolicy(allow_rules=...)` entry, JustHTML applies these steps:
89
90
 
90
91
  1. The tag must be allowed by `SanitizationPolicy.allowed_tags`.
91
92
  2. The attribute name must be allowed by `SanitizationPolicy.allowed_attributes`.
@@ -103,7 +104,9 @@ URL behavior is controlled by `UrlPolicy`:
103
104
  - `default_handling`: the default post-validation action for URL-like attributes that match an allow rule and do not set `UrlRule.handling`.
104
105
  - `default_allow_relative`: whether **relative** URLs (like `/path`, `./path`, `../path`, `?q`) are allowed by default.
105
106
 
106
- For URL-like attributes that match an explicit `(tag, attr)` rule in `allow_rules`, validated URLs use `UrlPolicy.default_handling` unless that specific `UrlRule` overrides it with `handling=...`.
107
+ For URL-like attributes, or custom attributes that match an explicit `(tag, attr)` or `("*", attr)` rule in
108
+ `allow_rules`, validated URLs use `UrlPolicy.default_handling` unless that specific `UrlRule` overrides it with
109
+ `handling=...`.
107
110
 
108
111
  Note: URL validation is always enforced by `UrlRule`.
109
112
 
@@ -1,7 +1,7 @@
1
1
  [project]
2
2
  name = "justhtml"
3
3
  authors = [{ name = "Emil Stenström", email = "emil@emilstenstrom.se" }]
4
- version = "3.0.0"
4
+ version = "3.1.0"
5
5
  description = "A pure Python HTML5 parser that just works."
6
6
  readme = "README.md"
7
7
  license = { file = "LICENSE" }
@@ -200,6 +200,57 @@ WHITESPACE_PRESERVING_ELEMENTS: Final[frozenset[str]] = frozenset(
200
200
  }
201
201
  )
202
202
 
203
+ WHITESPACE_TRIMMING_BLOCK_ELEMENTS: Final[frozenset[str]] = frozenset(
204
+ {
205
+ "address",
206
+ "article",
207
+ "aside",
208
+ "blockquote",
209
+ "body",
210
+ "caption",
211
+ "center",
212
+ "dd",
213
+ "details",
214
+ "dialog",
215
+ "dir",
216
+ "div",
217
+ "dl",
218
+ "dt",
219
+ "fieldset",
220
+ "figcaption",
221
+ "figure",
222
+ "footer",
223
+ "form",
224
+ "h1",
225
+ "h2",
226
+ "h3",
227
+ "h4",
228
+ "h5",
229
+ "h6",
230
+ "header",
231
+ "hgroup",
232
+ "html",
233
+ "li",
234
+ "listing",
235
+ "main",
236
+ "menu",
237
+ "nav",
238
+ "ol",
239
+ "p",
240
+ "search",
241
+ "section",
242
+ "summary",
243
+ "table",
244
+ "tbody",
245
+ "td",
246
+ "tfoot",
247
+ "th",
248
+ "thead",
249
+ "tr",
250
+ "ul",
251
+ }
252
+ )
253
+
203
254
  FORMATTING_ELEMENTS = {
204
255
  "a",
205
256
  "b",
@@ -319,12 +319,17 @@ def _compile_tag_actions(
319
319
  preserve_state_attrs = tag in _ACTIVE_FORMATTING_TAGS and not allowed
320
320
  url_attr_kinds: dict[str, UrlSinkKind] = {}
321
321
  url_attr_rules: dict[str, UrlRule] = {}
322
- for attr in allowed_attrs.intersection(_URL_SINK_ATTRS):
323
- rule = url_rules.get((tag, attr))
324
- if rule is None: # pragma: no cover - planner only iterates explicitly configured URL attributes
322
+ for attr in allowed_attrs:
323
+ rule = url_rules.get((tag, attr)) or url_rules.get(("*", attr))
324
+ if attr not in _URL_SINK_ATTRS and rule is None:
325
325
  continue
326
- kind = _url_sink_kind_for_attr(tag=tag, attr=attr, attrs={attr: ""})
327
- if kind is None: # pragma: no cover - URL sink attributes always have a declared sink kind
326
+ if rule is None: # pragma: no cover - URL sink policy validation rejects missing rules
327
+ continue
328
+ if attr in _URL_SINK_ATTRS:
329
+ kind = _url_sink_kind_for_attr(tag=tag, attr=attr, attrs={attr: ""})
330
+ else:
331
+ kind = "url"
332
+ if kind is None: # pragma: no cover - guarded URL sink attrs require matching state attrs
328
333
  continue
329
334
  url_attr_kinds[attr] = kind
330
335
  url_attr_rules[attr] = rule
@@ -972,6 +977,13 @@ class ParseEngine:
972
977
  if tag_end == -1:
973
978
  self._emit_error("eof-in-tag", end - 1)
974
979
  return
980
+ if name in self._p_closing_start_tags:
981
+ for idx in range(len(open_tags) - 1, -1, -1):
982
+ if open_tags[idx] == "p":
983
+ del open_tags[idx:]
984
+ break
985
+ if open_tags[idx] in _P_SCOPE_BOUNDARIES:
986
+ break
975
987
  if name not in self._void_elements and not self._is_self_closing_source_tag(pos + len(name), tag_end):
976
988
  open_tags.append(name)
977
989
  pos = tag_end + 1
@@ -1514,7 +1526,7 @@ class ParseEngine:
1514
1526
  and not self._ignore_lf
1515
1527
  and not self._track_node_locations
1516
1528
  and not self._xml_coercion
1517
- and parent.namespace in {None, "html"}
1529
+ and (parent.namespace is None or parent.namespace == "html")
1518
1530
  and type(parent) is not Template
1519
1531
  and parent is not self._head
1520
1532
  and parent is not self._html
@@ -2330,8 +2342,10 @@ class ParseEngine:
2330
2342
  if name == "image":
2331
2343
  name = "img"
2332
2344
  action = self._tag_actions.get(name)
2333
- if self._foreign_context_seen and self._stack[-1].namespace not in {None, "html", _PARSER_ONLY_NAMESPACE}:
2334
- return self._parse_start_tag(name_start, end)
2345
+ if self._foreign_context_seen:
2346
+ namespace = self._stack[-1].namespace
2347
+ if namespace is not None and namespace != "html" and namespace != _PARSER_ONLY_NAMESPACE:
2348
+ return self._parse_start_tag(name_start, end)
2335
2349
  if not self._initial_mode_done:
2336
2350
  self._mark_initial_content()
2337
2351
 
@@ -5088,6 +5102,12 @@ class ParseEngine:
5088
5102
  if not stripped:
5089
5103
  return None
5090
5104
 
5105
+ allowed_schemes = rule.allowed_schemes
5106
+ if stripped.startswith("https://") and " " not in stripped and stripped.isprintable():
5107
+ return stripped if "https" in allowed_schemes else None
5108
+ if stripped.startswith("http://") and " " not in stripped and stripped.isprintable():
5109
+ return stripped if "http" in allowed_schemes else None
5110
+
5091
5111
  if _URL_CONTROL_CHAR_REGEX.search(stripped):
5092
5112
  return None
5093
5113
 
@@ -5117,7 +5137,6 @@ class ParseEngine:
5117
5137
  stripped if rule.allow_fragment else None
5118
5138
  ) # pragma: no cover - unreachable after parser-state guards
5119
5139
 
5120
- allowed_schemes = rule.allowed_schemes
5121
5140
  if normalized.startswith("//"): # pragma: no branch - opposite edge requires invalid parser state
5122
5141
  resolved = rule.resolve_protocol_relative # pragma: no cover - unreachable after parser-state guards
5123
5142
  if not resolved: # pragma: no cover - unreachable after parser-state guards
@@ -575,6 +575,7 @@ class SelectorParser:
575
575
  self._expect(TokenType.ATTR_START)
576
576
 
577
577
  attr_name = self._expect(TokenType.TAG).value
578
+ attr_name = attr_name.lower() if attr_name else attr_name
578
579
 
579
580
  token = self._peek()
580
581
  if token.type == TokenType.ATTR_END:
@@ -591,6 +592,7 @@ class SelectorParser:
591
592
  """Parse a pseudo-class selector like :first-child or :not(selector)."""
592
593
  self._expect(TokenType.COLON)
593
594
  name = self._expect(TokenType.TAG).value
595
+ name = name.lower() if name else name
594
596
 
595
597
  # Functional pseudo-class
596
598
  if self._peek().type == TokenType.PAREN_OPEN:
@@ -601,7 +603,7 @@ class SelectorParser:
601
603
  self._expect(TokenType.PAREN_CLOSE)
602
604
  parsed_arg = (
603
605
  _parse_selector(arg, limits=self.limits, parse_depth=self.parse_depth + 1)
604
- if (name or "").lower() == "not" and arg
606
+ if name == "not" and arg
605
607
  else None
606
608
  )
607
609
  return SimpleSelector(SimpleSelector.TYPE_PSEUDO, name=name, arg=arg, parsed_arg=parsed_arg)
@@ -773,7 +775,9 @@ class SelectorMatcher:
773
775
 
774
776
  if sel_type == SimpleSelector.TYPE_TAG:
775
777
  # HTML tag names are case-insensitive
776
- return bool(node.name.lower() == (selector.name.lower() if selector.name else ""))
778
+ selector_name = selector.name or ""
779
+ node_name = node.name
780
+ return bool(node_name == selector_name or node_name.lower() == selector_name)
777
781
 
778
782
  if sel_type == SimpleSelector.TYPE_ID:
779
783
  node_id = node.attrs.get("id", "") if node.attrs else ""
@@ -803,7 +807,7 @@ class SelectorMatcher:
803
807
 
804
808
  def _matches_attribute(self, node: Any, selector: SimpleSelector) -> bool:
805
809
  """Match an attribute selector."""
806
- attr_name = (selector.name or "").lower() # Attribute names are case-insensitive in HTML
810
+ attr_name = selector.name or "" # Attribute names are normalized during parsing.
807
811
  attr_value = self._attribute_value(node, attr_name)
808
812
 
809
813
  if attr_value is None:
@@ -893,7 +897,7 @@ class SelectorMatcher:
893
897
 
894
898
  def _matches_pseudo(self, node: Any, selector: SimpleSelector, *, depth: int = 0) -> bool:
895
899
  """Match a pseudo-class selector."""
896
- name = (selector.name or "").lower()
900
+ name = selector.name or ""
897
901
 
898
902
  if name == "first-child":
899
903
  return self._is_first_child(node)
@@ -1067,7 +1071,7 @@ class SelectorMatcher:
1067
1071
  def _compound_tag_name(self, compound: CompoundSelector) -> str | None:
1068
1072
  for simple in compound.selectors:
1069
1073
  if simple.type == SimpleSelector.TYPE_TAG and simple.name:
1070
- return simple.name.lower()
1074
+ return simple.name
1071
1075
  return None
1072
1076
 
1073
1077
  def _element_child_names(self, parent: Any) -> frozenset[str]:
@@ -1376,8 +1380,7 @@ def _selector_allows_non_elements(selector: ParsedSelector | CompoundSelector |
1376
1380
  if isinstance(selector, SimpleSelector):
1377
1381
  if selector.type != SimpleSelector.TYPE_PSEUDO:
1378
1382
  return False
1379
- name = (selector.name or "").lower()
1380
- return name == "comment"
1383
+ return selector.name == "comment"
1381
1384
  return False
1382
1385
 
1383
1386
 
@@ -6,6 +6,7 @@ from __future__ import annotations
6
6
 
7
7
  import re
8
8
  from enum import Enum
9
+ from functools import lru_cache
9
10
  from typing import TYPE_CHECKING, Any
10
11
  from urllib.parse import quote as url_quote
11
12
 
@@ -63,10 +64,12 @@ def _validate_serializable_name(name: str, pattern: re.Pattern[str], kind: str)
63
64
  return name
64
65
 
65
66
 
67
+ @lru_cache(maxsize=4096)
66
68
  def _validate_serializable_tag_name(name: str) -> str:
67
69
  return _validate_serializable_name(name, _SERIALIZABLE_TAG_NAME_RE, "element")
68
70
 
69
71
 
72
+ @lru_cache(maxsize=8192)
70
73
  def _validate_serializable_attr_name(name: str) -> str:
71
74
  return _validate_serializable_name(name, _SERIALIZABLE_ATTR_NAME_RE, "attribute")
72
75
 
@@ -228,31 +231,33 @@ def serialize_start_tag(
228
231
  is_void: bool = False,
229
232
  ) -> str:
230
233
  name = _validate_serializable_tag_name(name)
231
- parts: list[str] = ["<", name]
232
- if attrs:
233
- parts_extend = parts.extend
234
- for raw_key, value in attrs.items():
235
- key = _validate_serializable_attr_name(raw_key)
236
- if minimize_boolean_attributes:
237
- if value is None or value == "" or value == key:
238
- parts_extend((" ", key))
239
- continue
240
- if len(value) == len(key) and value.lower() == key:
241
- parts_extend((" ", key))
242
- continue
234
+ if not attrs:
235
+ return f"<{name} />" if use_trailing_solidus and is_void else f"<{name}>"
243
236
 
244
- if value is None or value == "":
245
- parts_extend((" ", key, '=""'))
237
+ parts: list[str] = ["<", name]
238
+ parts_extend = parts.extend
239
+ for raw_key, value in attrs.items():
240
+ key = _validate_serializable_attr_name(raw_key)
241
+ if minimize_boolean_attributes:
242
+ if value is None or value == "" or value == key:
243
+ parts_extend((" ", key))
246
244
  continue
247
-
248
- if not quote_attr_values and _can_unquote_attr_value(value):
249
- escaped = value.replace("&", "&amp;").replace("<", "&lt;")
250
- parts_extend((" ", key, "=", escaped))
245
+ if len(value) == len(key) and value.lower() == key:
246
+ parts_extend((" ", key))
251
247
  continue
252
248
 
253
- quote = _choose_attr_quote(value, quote_char)
254
- escaped = _escape_attr_value(value, quote)
255
- parts_extend((" ", key, "=", quote, escaped, quote))
249
+ if value is None or value == "":
250
+ parts_extend((" ", key, '=""'))
251
+ continue
252
+
253
+ if not quote_attr_values and _can_unquote_attr_value(value):
254
+ escaped = value.replace("&", "&amp;").replace("<", "&lt;")
255
+ parts_extend((" ", key, "=", escaped))
256
+ continue
257
+
258
+ quote = _choose_attr_quote(value, quote_char)
259
+ escaped = _escape_attr_value(value, quote)
260
+ parts_extend((" ", key, "=", quote, escaped, quote))
256
261
 
257
262
  if use_trailing_solidus and is_void:
258
263
  parts.append(" />")
@@ -723,12 +728,13 @@ def _node_to_html(node: Any, indent: int = 0, indent_size: int = 2, *, in_pre: b
723
728
  continue
724
729
 
725
730
  open_tag = serialize_start_tag(name, current.attrs)
726
- close_tag = serialize_end_tag(name)
727
731
 
728
732
  if name in VOID_ELEMENTS:
729
733
  results.append(f"{prefix}{open_tag}")
730
734
  continue
731
735
 
736
+ close_tag = f"</{name}>"
737
+
732
738
  children: list[Any] = (
733
739
  current.template_content.children
734
740
  if name == "template" and current.namespace in {None, "html"} and current.template_content is not None
@@ -748,7 +754,11 @@ def _node_to_html(node: Any, indent: int = 0, indent_size: int = 2, *, in_pre: b
748
754
  all_text = False
749
755
  break
750
756
  if all_text:
751
- text_content = current.to_text(separator="", strip=False)
757
+ text_content = (
758
+ children[0].data or ""
759
+ if len(children) == 1 and children[0] is not None
760
+ else "".join(child.data or "" for child in children if child is not None)
761
+ )
752
762
  text_content = _collapse_html_whitespace(text_content)
753
763
  results.append(f"{prefix}{open_tag}{_serialize_text_for_parent(text_content, name)}{close_tag}")
754
764
  continue
@@ -233,6 +233,8 @@ TransformSpec = Transform | Stage
233
233
  class _CompiledCollapseWhitespaceTransform:
234
234
  kind: Literal["collapse_whitespace"]
235
235
  skip_tags: frozenset[str]
236
+ trim_blocks: bool
237
+ block_tags: frozenset[str]
236
238
  callback: NodeCallback | None
237
239
  report: ReportCallback | None
238
240
 
@@ -775,11 +775,14 @@ def _compile_drop_url_attrs_transform(
775
775
 
776
776
  for key in attrs:
777
777
  lower_key = key if key.islower() else key.lower()
778
- if lower_key not in url_sink_attrs:
778
+ rule = url_policy.allow_rules.get((tag, lower_key)) or url_policy.allow_rules.get(("*", lower_key))
779
+ if lower_key not in url_sink_attrs and rule is None:
779
780
  continue
780
781
  raw_value = attrs[key]
781
782
 
782
- sink_kind = _url_sink_kind_for_attr(tag=tag, attr=lower_key, attrs=attrs)
783
+ sink_kind = (
784
+ _url_sink_kind_for_attr(tag=tag, attr=lower_key, attrs=attrs) if lower_key in url_sink_attrs else "url"
785
+ )
783
786
  if sink_kind is None:
784
787
  continue
785
788
 
@@ -791,7 +794,6 @@ def _compile_drop_url_attrs_transform(
791
794
  to_drop.append(key)
792
795
  continue
793
796
 
794
- rule = url_policy.allow_rules.get((tag, lower_key))
795
797
  if rule is None:
796
798
  if on_report is not None: # pragma: no cover
797
799
  on_report(f"Unsafe URL in attribute '{lower_key}' (no rule)", node=node)
@@ -968,6 +970,8 @@ def _compile_collapse_whitespace_transform(t: CollapseWhitespace) -> _CompiledCo
968
970
  return _CompiledCollapseWhitespaceTransform(
969
971
  kind="collapse_whitespace",
970
972
  skip_tags=t.skip_tags,
973
+ trim_blocks=t.trim_blocks,
974
+ block_tags=t.block_tags,
971
975
  callback=t.callback,
972
976
  report=t.report,
973
977
  )
@@ -258,6 +258,26 @@ def apply_compiled_transforms(
258
258
  node.parent = None
259
259
  return True
260
260
 
261
+ def _trim_block_edge_text(
262
+ text_data: str,
263
+ *,
264
+ parent: Node,
265
+ children: list[Node],
266
+ child_index: int,
267
+ block_tags: frozenset[str],
268
+ ) -> str:
269
+ parent_name = parent.name
270
+ if parent_name.startswith("#") or parent_name == "!doctype":
271
+ return text_data
272
+ if parent_name.lower() not in block_tags:
273
+ return text_data
274
+
275
+ if child_index == 0:
276
+ text_data = text_data.lstrip(" \t\n\f\r")
277
+ if child_index == len(children) - 1:
278
+ text_data = text_data.rstrip(" \t\n\f\r")
279
+ return text_data
280
+
261
281
  def apply_to_children(
262
282
  parent: Node,
263
283
  *,
@@ -480,12 +500,27 @@ def apply_compiled_transforms(
480
500
  text_data = str(node.data or "")
481
501
  if text_data:
482
502
  collapsed = _collapse_html_space_characters(text_data)
483
- if collapsed != text_data:
503
+ normalized = collapsed
504
+ if t.trim_blocks:
505
+ normalized = _trim_block_edge_text(
506
+ normalized,
507
+ parent=parent,
508
+ children=children,
509
+ child_index=i,
510
+ block_tags=t.block_tags,
511
+ )
512
+ if normalized != text_data:
484
513
  if t.callback is not None:
485
514
  t.callback(node)
486
515
  if t.report is not None:
487
516
  t.report("Collapsed whitespace in text node", node=node)
488
- node.data = collapsed
517
+ if normalized:
518
+ node.data = normalized
519
+ else:
520
+ children.pop(i)
521
+ node.parent = None
522
+ changed = True
523
+ break
489
524
  continue
490
525
 
491
526
  if k == "strip_invisible_unicode":