zimscraperlib 5.4.0__tar.gz → 5.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (148) hide show
  1. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/.gitignore +1 -0
  2. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/.pre-commit-config.yaml +2 -2
  3. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/CHANGELOG.md +44 -0
  4. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/PKG-INFO +32 -30
  5. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/docs/technical_architecture.md +1 -1
  6. zimscraperlib-5.5.0/javascript/.yarnrc.yml +1 -0
  7. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/javascript/package.json +9 -13
  8. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/javascript/src/wombatSetup.js +26 -9
  9. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/javascript/test/wombatSetup.js +23 -0
  10. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/javascript/test/wombatUrlRewriting.js +90 -13
  11. zimscraperlib-5.5.0/javascript/yarn.lock +2626 -0
  12. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/openzim.toml +1 -1
  13. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/pyproject.toml +27 -24
  14. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/rules/rules.yaml +23 -3
  15. zimscraperlib-5.5.0/src/zimscraperlib/__about__.py +1 -0
  16. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/constants.py +1 -0
  17. zimscraperlib-5.5.0/src/zimscraperlib/download.py +400 -0
  18. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/html.py +3 -2
  19. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/i18n.py +1 -1
  20. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/image/illustration.py +2 -1
  21. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/logging.py +1 -1
  22. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/css.py +1 -1
  23. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/html.py +78 -28
  24. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/js.py +161 -45
  25. zimscraperlib-5.5.0/src/zimscraperlib/rewriting/js_ast.py +132 -0
  26. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/rules.py +4 -0
  27. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/rx_replacer.py +30 -16
  28. zimscraperlib-5.5.0/src/zimscraperlib/rewriting/statics/wombat.js +21 -0
  29. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/statics/wombatSetup.js +3 -3
  30. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/url_rewriting.py +151 -31
  31. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/uri.py +1 -1
  32. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/creator.py +7 -4
  33. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/filesystem.py +3 -3
  34. zimscraperlib-5.5.0/src/zimscraperlib/zim/indexing.py +241 -0
  35. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/items.py +20 -5
  36. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/metadata.py +7 -9
  37. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/conftest.py +99 -1
  38. zimscraperlib-5.5.0/tests/download/test_download.py +776 -0
  39. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/html/test_html.py +26 -10
  40. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/i18n/test_i18n.py +19 -12
  41. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/image/test_image.py +12 -3
  42. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/rewriting/test_css_rewriting.py +1 -1
  43. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/rewriting/test_fuzzy_rules.py +62 -0
  44. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/rewriting/test_html_rewriting.py +156 -16
  45. zimscraperlib-5.5.0/tests/rewriting/test_js_globals.py +264 -0
  46. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/rewriting/test_js_rewriting.py +97 -15
  47. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/rewriting/test_rx_replacer.py +8 -0
  48. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/rewriting/test_url_rewriting.py +307 -18
  49. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/video/test_video.py +2 -2
  50. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/zim/test_indexing.py +93 -3
  51. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/zim/test_libkiwix.py +8 -6
  52. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/zim/test_metadata.py +6 -10
  53. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/zim/test_zim_creator.py +80 -5
  54. zimscraperlib-5.4.0/javascript/yarn.lock +0 -1855
  55. zimscraperlib-5.4.0/src/zimscraperlib/__about__.py +0 -1
  56. zimscraperlib-5.4.0/src/zimscraperlib/download.py +0 -248
  57. zimscraperlib-5.4.0/src/zimscraperlib/rewriting/statics/wombat.js +0 -21
  58. zimscraperlib-5.4.0/src/zimscraperlib/zim/indexing.py +0 -130
  59. zimscraperlib-5.4.0/tests/download/test_download.py +0 -346
  60. zimscraperlib-5.4.0/yarn.lock +0 -248
  61. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/.readthedocs.yaml +0 -0
  62. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/LICENSE +0 -0
  63. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/README.md +0 -0
  64. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/contrib/README.md +0 -0
  65. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/contrib/encode_video.py +0 -0
  66. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/docs/assets/openzim.png +0 -0
  67. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/docs/functional_architecture.md +0 -0
  68. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/docs/index.md +0 -0
  69. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/docs/license.md +0 -0
  70. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/docs/scripts/generate_api_nav.py +0 -0
  71. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/docs/software_architecture.md +0 -0
  72. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/javascript/.prettierignore +0 -0
  73. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/javascript/.prettierrc.json +0 -0
  74. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/javascript/eslint.config.js +0 -0
  75. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/javascript/rollup.config.js +0 -0
  76. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/mkdocs.yml +0 -0
  77. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/rules/generate_rules.py +0 -0
  78. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/__init__.py +0 -0
  79. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/filesystem.py +0 -0
  80. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/fix_ogvjs_dist.py +0 -0
  81. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/image/__init__.py +0 -0
  82. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/image/conversion.py +0 -0
  83. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/image/optimization.py +0 -0
  84. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/image/presets.py +0 -0
  85. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/image/probing.py +0 -0
  86. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/image/transformation.py +0 -0
  87. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/image/utils.py +0 -0
  88. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/inputs.py +0 -0
  89. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/misc.py +0 -0
  90. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/py.typed +0 -0
  91. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/__init__.py +0 -0
  92. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/statics/README.md +0 -0
  93. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/statics/__wb_module_decl.js +0 -0
  94. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/templates/head_insert.html +0 -0
  95. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/types.py +0 -0
  96. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/typing.py +0 -0
  97. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/video/__init__.py +0 -0
  98. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/video/config.py +0 -0
  99. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/video/encoding.py +0 -0
  100. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/video/presets.py +0 -0
  101. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/video/probing.py +0 -0
  102. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/__init__.py +0 -0
  103. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/_libkiwix.py +0 -0
  104. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/archive.py +0 -0
  105. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/dedup.py +0 -0
  106. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/providers.py +0 -0
  107. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tasks.py +0 -0
  108. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/DroidSans.ttf +0 -0
  109. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/audio.mp3 +0 -0
  110. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/blue.jpg +0 -0
  111. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/commons.png +0 -0
  112. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/commons48.png +0 -0
  113. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/encrypted.pdf +0 -0
  114. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/encrypted.txt +0 -0
  115. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/mail.gif +0 -0
  116. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/milderm.pdf +0 -0
  117. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/milderm.txt +0 -0
  118. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/ninja.webp +0 -0
  119. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/pluto.jpg +0 -0
  120. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/single_wave_icon.gbr +0 -0
  121. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/square.jpg +0 -0
  122. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/square.png +0 -0
  123. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/star.svg +0 -0
  124. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/video.mkv +0 -0
  125. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/video.mp4 +0 -0
  126. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/video.webm +0 -0
  127. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/filesystem/test_filesystem.py +0 -0
  128. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/html/conftest.py +0 -0
  129. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/image/test_illustration.py +0 -0
  130. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/inputs/test_inputs.py +0 -0
  131. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/logging/conftest.py +0 -0
  132. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/logging/test_logging.py +0 -0
  133. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/misc/test_misc.py +0 -0
  134. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/ogvjs/conftest.py +0 -0
  135. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/ogvjs/test_ogvjs.py +0 -0
  136. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/rewriting/__init__.py +0 -0
  137. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/rewriting/conftest.py +0 -0
  138. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/rewriting/utils.py +0 -0
  139. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/test_constants.py +0 -0
  140. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/types/test_types.py +0 -0
  141. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/typing/test_typing.py +0 -0
  142. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/uri/test_uri.py +0 -0
  143. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/video/conftest.py +0 -0
  144. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/video/test_encoding.py +0 -0
  145. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/zim/conftest.py +0 -0
  146. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/zim/test_archive.py +0 -0
  147. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/zim/test_dedup.py +0 -0
  148. {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/zim/test_fs.py +0 -0
@@ -253,6 +253,7 @@ $RECYCLE.BIN/
253
253
  .vscode
254
254
  src/libzim-stubs
255
255
  javascript/node_modules
256
+ javascript/.yarn/*
256
257
 
257
258
  # rule files are generated by rules/generate_rules.py
258
259
  src/zimscraperlib/rewriting/rules.py
@@ -8,12 +8,12 @@ repos:
8
8
  - id: trailing-whitespace
9
9
  - id: end-of-file-fixer
10
10
  - repo: https://github.com/astral-sh/ruff-pre-commit
11
- rev: v0.15.14
11
+ rev: v0.17.3
12
12
  hooks:
13
13
  - id: ruff-check
14
14
  - id: ruff-format
15
15
  - repo: https://github.com/RobertCraigie/pyright-python
16
- rev: v1.1.409
16
+ rev: v1.1.414
17
17
  hooks:
18
18
  - id: pyright
19
19
  name: pyright (system)
@@ -5,6 +5,50 @@ All notable changes to this project are documented in this file.
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0).
7
7
 
8
+ ## [5.5.0] - 2026-10-29
9
+
10
+ ### Added
11
+
12
+ - Index ePub documents for full-text search, with a new `get_epub_index_data` helper and automatic indexing of ePub items, like PDFs already are (#333)
13
+ - Send a Wikimedia-compliant default `User-Agent` in `stream_file` when downloading from `upload.wikimedia.org` (#214)
14
+ - `save_large_file` accepts arbitrary HTTP headers and sends the default Wikimedia-compliant `User-Agent` for `upload.wikimedia.org`, like `stream_file` (#360)
15
+ - Add fuzzy rule to rewrite URLs of lesbases.anct.gouv.fr (#334)
16
+ - Report progress from `save_large_file` through a new `on_progress` callback and periodic logging, and stream the download to a `.part` file that is only moved into place once complete (#348)
17
+ - New `get_pdf_title` and `get_epub_title` helpers to get a document's title from its own metadata without extracting its full text content, cheaper than `get_pdf_index_data`/`get_epub_index_data` when only the title is needed (openzim/warc2zim#462, #359)
18
+
19
+ ### Changed
20
+
21
+ - Upgrade to wombat 3.11.0 (#338)
22
+ - Upgrade Python dependencies, allowing pymupdf 2, pillow 13, beartype 0.23, urllib3 2.8 and xxhash 4, and Node.JS dependencies (#366, #369)
23
+
24
+ ### Fixed
25
+
26
+ - Rewrite `save_large_file` to stream and resume the download itself instead of running `wget` once, whose `-t` counter is a budget for the whole download: a flaky connection now keeps resuming and the retry budget only counts attempts that make no progress, so long downloads are no longer abandoned mid-way (#348)
27
+ - Carry a script's top-level `const`, `let`, `class` and `function` declarations across the wombat block, so other scripts on the page can still see them, re-declaring classes with `let` so they can be reassigned (#329, #370, webrecorder/wabac.js#380)
28
+ - Keep a rewritten `<script>`'s original `src` in `__wb_orig_src`, so wombat can return it from `getAttribute("src")` and bundlers that identify chunks by it (Next.js/Turbopack, Vite) still work (openzim/warc2zim#473, #337)
29
+ - Keep the original of every rewritten `src`, `href` and `xlink:href` in `__wb_orig_<attr>`, not just a script's `src`, so wombat can return it instead of a broken URL (openzim/warc2zim#413, #338)
30
+ - Keep URL-encoded `#` characters of the path or querystring when rewriting document URIs, instead of dropping everything after them (#341)
31
+ - Find the language of an HTML page in a `content-language` `<meta>` that is not the first `<meta>` of the page (#356)
32
+ - Rewrite the `poster` attribute (e.g. of `<video>`) when rewriting HTML, like `src` and `href` (#339)
33
+ - Do not collapse consecutive slashes `//` into a single `/` when normalizing URLs into ZIM paths, in both Python and JS rewriting (#340)
34
+ - Resolve relative links against a page URL containing consecutive slashes `//` the way browsers do, instead of collapsing them like `urllib.parse.urljoin` does (#346)
35
+ - `Creator.finish()` no longer silently discards errors raised by libzim while finalizing the ZIM, so a failure during finalization is no longer reported as a successful run (openzim/warc2zim#462, #354)
36
+ - Escape `.` and `..` path segments (e.g. folded into the path from a querystring) with a leading `_` when normalizing URLs into ZIM paths, so they are kept as literal entry names instead of being resolved away as navigation, in both Python and JS rewriting (#321)
37
+
38
+ ## [5.4.1] - 2026-07-31
39
+
40
+ ### Fixed
41
+
42
+ - Do not URL-encode `=` and `,` characters when rewriting document URIs, fixing YouTube JS player breakage (#316)
43
+ - Fix sources of memory leaks in JS and CSS rewriting (#322)
44
+
45
+ ### Changed
46
+
47
+ - Upgrade Python and Javascript dependencies (especially wombat 3.10.6) (#319, #328)
48
+ - Backport wabac changes up to wabac.js 2.26.6 (#330)
49
+ - Migrate `javascript/` from Yarn Classic to Yarn Berry (#320)
50
+ - Test bogus Content-Length HTTP header behavior (#318)
51
+
8
52
  ## [5.4.0] - 2026-05-28
9
53
 
10
54
  ### Added
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.4
1
+ Metadata-Version: 2.5
2
2
  Name: zimscraperlib
3
- Version: 5.4.0
3
+ Version: 5.5.0
4
4
  Summary: Collection of python tools to re-use common code across scrapers
5
5
  Project-URL: Donate, https://www.kiwix.org/en/support-us/
6
6
  Project-URL: Homepage, https://github.com/openzim/python-scraperlib
@@ -15,7 +15,7 @@ Classifier: Programming Language :: Python :: 3
15
15
  Classifier: Programming Language :: Python :: 3.14
16
16
  Requires-Python: <3.15,>=3.14
17
17
  Requires-Dist: babel<3.0,>=2.9
18
- Requires-Dist: beartype<0.23,>=0.19
18
+ Requires-Dist: beartype<0.24,>=0.19
19
19
  Requires-Dist: beautifulsoup4<5.0,>=4.9.3
20
20
  Requires-Dist: cairosvg<3.0,>=2.2.0
21
21
  Requires-Dist: colorthief==0.2.1
@@ -23,59 +23,61 @@ Requires-Dist: idna<4.0,>=2.5
23
23
  Requires-Dist: iso639-lang<3.0,>=2.4.0
24
24
  Requires-Dist: libzim<4.0,>=3.8.0
25
25
  Requires-Dist: lxml<7.0,>=4.6.3
26
- Requires-Dist: optimize-images<2.0,>=1.3.6
26
+ Requires-Dist: optimize-images<3.0,>=1.3.6
27
27
  Requires-Dist: piexif==1.1.3
28
- Requires-Dist: pillow<13.0,>=7.0.0
29
- Requires-Dist: pymupdf<2.0,>=1.24.0
28
+ Requires-Dist: pillow<14.0,>=7.0.0
29
+ Requires-Dist: pymupdf<3.0,>=1.24.0
30
30
  Requires-Dist: python-magic<0.5,>=0.4.3
31
31
  Requires-Dist: python-resize-image<1.2,>=1.1.19
32
32
  Requires-Dist: regex>=2020.7.14
33
33
  Requires-Dist: requests<3.0,>=2.25.1
34
+ Requires-Dist: tree-sitter-javascript<1.0,>=0.23
35
+ Requires-Dist: tree-sitter<1.0,>=0.23
34
36
  Requires-Dist: types-xxhash<4.0,>=2.0
35
- Requires-Dist: urllib3<2.8.0,>=1.26.5
36
- Requires-Dist: xxhash<4.0,>=2.0
37
+ Requires-Dist: urllib3<2.9.0,>=1.26.5
38
+ Requires-Dist: xxhash<5.0,>=2.0
37
39
  Requires-Dist: yt-dlp
38
40
  Provides-Extra: dev
39
- Requires-Dist: coverage==7.14.1; extra == 'dev'
41
+ Requires-Dist: coverage==7.17.0; extra == 'dev'
40
42
  Requires-Dist: invoke==3.0.3; extra == 'dev'
41
- Requires-Dist: ipython==9.13.0; extra == 'dev'
43
+ Requires-Dist: ipython==9.17.1; extra == 'dev'
42
44
  Requires-Dist: jinja2==3.1.6; extra == 'dev'
43
45
  Requires-Dist: mkdocs-gen-files==0.6.1; extra == 'dev'
44
46
  Requires-Dist: mkdocs-include-markdown-plugin==7.3.0; extra == 'dev'
45
47
  Requires-Dist: mkdocs-literate-nav==0.6.3; extra == 'dev'
46
- Requires-Dist: mkdocs-material==9.7.6; extra == 'dev'
48
+ Requires-Dist: mkdocs-material==9.8.0; extra == 'dev'
47
49
  Requires-Dist: mkdocs==1.6.1; extra == 'dev'
48
- Requires-Dist: mkdocstrings[python]==1.0.3; extra == 'dev'
49
- Requires-Dist: pre-commit==4.6.0; extra == 'dev'
50
- Requires-Dist: pygments==2.19.2; extra == 'dev'
51
- Requires-Dist: pymdown-extensions==10.21.3; extra == 'dev'
52
- Requires-Dist: pyright==1.1.409; extra == 'dev'
53
- Requires-Dist: pytest-mock==3.15.1; extra == 'dev'
54
- Requires-Dist: pytest==9.0.3; extra == 'dev'
50
+ Requires-Dist: mkdocstrings[python]==1.0.8; extra == 'dev'
51
+ Requires-Dist: pre-commit==4.7.0; extra == 'dev'
52
+ Requires-Dist: pygments==2.21.0; extra == 'dev'
53
+ Requires-Dist: pymdown-extensions==12.3; extra == 'dev'
54
+ Requires-Dist: pyright==1.1.414; extra == 'dev'
55
+ Requires-Dist: pytest-mock==3.17.0; extra == 'dev'
56
+ Requires-Dist: pytest==9.2.0; extra == 'dev'
55
57
  Requires-Dist: pyyaml==6.0.3; extra == 'dev'
56
- Requires-Dist: ruff==0.15.14; extra == 'dev'
58
+ Requires-Dist: ruff==0.17.3; extra == 'dev'
57
59
  Provides-Extra: docs
58
60
  Requires-Dist: mkdocs-gen-files==0.6.1; extra == 'docs'
59
61
  Requires-Dist: mkdocs-include-markdown-plugin==7.3.0; extra == 'docs'
60
62
  Requires-Dist: mkdocs-literate-nav==0.6.3; extra == 'docs'
61
- Requires-Dist: mkdocs-material==9.7.6; extra == 'docs'
63
+ Requires-Dist: mkdocs-material==9.8.0; extra == 'docs'
62
64
  Requires-Dist: mkdocs==1.6.1; extra == 'docs'
63
- Requires-Dist: mkdocstrings[python]==1.0.3; extra == 'docs'
64
- Requires-Dist: pygments==2.19.2; extra == 'docs'
65
- Requires-Dist: pymdown-extensions==10.21.3; extra == 'docs'
66
- Requires-Dist: ruff==0.15.14; extra == 'docs'
65
+ Requires-Dist: mkdocstrings[python]==1.0.8; extra == 'docs'
66
+ Requires-Dist: pygments==2.21.0; extra == 'docs'
67
+ Requires-Dist: pymdown-extensions==12.3; extra == 'docs'
68
+ Requires-Dist: ruff==0.17.3; extra == 'docs'
67
69
  Provides-Extra: qa
68
- Requires-Dist: pyright==1.1.409; extra == 'qa'
69
- Requires-Dist: pytest==9.0.3; extra == 'qa'
70
- Requires-Dist: ruff==0.15.14; extra == 'qa'
70
+ Requires-Dist: pyright==1.1.414; extra == 'qa'
71
+ Requires-Dist: pytest==9.2.0; extra == 'qa'
72
+ Requires-Dist: ruff==0.17.3; extra == 'qa'
71
73
  Provides-Extra: scripts
72
74
  Requires-Dist: invoke==3.0.3; extra == 'scripts'
73
75
  Requires-Dist: jinja2==3.1.6; extra == 'scripts'
74
76
  Requires-Dist: pyyaml==6.0.3; extra == 'scripts'
75
77
  Provides-Extra: test
76
- Requires-Dist: coverage==7.14.1; extra == 'test'
77
- Requires-Dist: pytest-mock==3.15.1; extra == 'test'
78
- Requires-Dist: pytest==9.0.3; extra == 'test'
78
+ Requires-Dist: coverage==7.17.0; extra == 'test'
79
+ Requires-Dist: pytest-mock==3.17.0; extra == 'test'
80
+ Requires-Dist: pytest==9.2.0; extra == 'test'
79
81
  Description-Content-Type: text/markdown
80
82
 
81
83
  # zimscraperlib
@@ -44,7 +44,7 @@ Below is an example case of the rewrite operation on an image URL found in an HT
44
44
  - Document original URL: `https://kiwix.org/a/article/document.html`
45
45
  - Document ZIM path: `kiwix.org/a/article/document.html`
46
46
  - Image original URL: `//xn--exmple-cva.com/a/resource/image.png?foo=bar`
47
- - Image rewritten URL: `../../../ex%C3%A9mple.com/a/resource/image.png%3Ffoo%3Dbar`
47
+ - Image rewritten URL: `../../../ex%C3%A9mple.com/a/resource/image.png%3Ffoo=bar`
48
48
  - Image ZIM Path: `exémple.com/a/resource/image.png?foo=bar`
49
49
 
50
50
  ## JS Rewriting
@@ -0,0 +1 @@
1
+ nodeLinker: node-modules
@@ -1,20 +1,18 @@
1
1
  {
2
2
  "name": "@openzim/wombat-setup",
3
3
  "type": "module",
4
- "version": "5.3.1-dev0",
4
+ "version": "5.5.0",
5
5
  "license": "GPL-3.0-or-later",
6
6
  "author": "openZIM",
7
7
  "devDependencies": {
8
- "@rollup/plugin-commonjs": "29.0.2",
8
+ "@rollup/plugin-commonjs": "29.0.3",
9
9
  "@rollup/plugin-node-resolve": "16.0.3",
10
10
  "@rollup/plugin-strip": "3.0.4",
11
- "@rollup/plugin-terser": "1.0.0",
12
- "ava": "8.0.1",
13
- "babel-eslint": "10.1.0",
14
- "eslint": "10.4.0",
15
- "eslint-config-prettier": "10.1.8",
16
- "prettier": "3.8.3",
17
- "rollup": "4.60.4",
11
+ "@rollup/plugin-terser": "1.0.2",
12
+ "ava": "8.2.0",
13
+ "eslint": "10.13.0",
14
+ "prettier": "3.9.10",
15
+ "rollup": "4.65.1",
18
16
  "rollup-plugin-version-injector": "1.3.3"
19
17
  },
20
18
  "scripts": {
@@ -35,9 +33,6 @@
35
33
  "serial": true,
36
34
  "files": [
37
35
  "test/*.js"
38
- ],
39
- "sources": [
40
- "src/**/*"
41
36
  ]
42
37
  },
43
38
  "dependencies": {
@@ -45,5 +40,6 @@
45
40
  },
46
41
  "engines": {
47
42
  "node": ">=26.0.0 <27.0.0"
48
- }
43
+ },
44
+ "packageManager": "yarn@4.18.1"
49
45
  }
@@ -18,6 +18,20 @@ export function applyFuzzyRules(path) {
18
18
  return path;
19
19
  }
20
20
 
21
+ export function escapeDotSegments(path) {
22
+ // Escape '.' and '..' path segments so they are not resolved as navigation.
23
+ // Per RFC 3986, a '.' or '..' path segment is a dot-segment which any URI consumer
24
+ // removes/resolves while normalizing a reference. A ZIM path must keep such a
25
+ // segment as a literal entry name instead (e.g. one folded in from a querystring),
26
+ // so it is prefixed with a '_', which is not itself special in a ZIM path.
27
+ // Segments which only look like a dot-segment (e.g. 'foo..', '..foo', '...') are
28
+ // not dot-segments and are left untouched.
29
+ return path
30
+ .split('/')
31
+ .map((part) => (part === '.' || part === '..' ? '_' + part : part))
32
+ .join('/');
33
+ }
34
+
21
35
  export function hasAlreadyBeenRewritten(
22
36
  original_absolute_url,
23
37
  orig_url,
@@ -63,12 +77,6 @@ export function hasAlreadyBeenRewritten(
63
77
  return false;
64
78
  }
65
79
 
66
- function removeSubsequentSlashes(value) {
67
- // Remove all successive occurrences of a slash `/` in a given string
68
- // E.g `val//ue` or `val///ue` or `val////ue` (and so on) are transformed into `value`
69
- return value.replace(/\/\/+/g, '/');
70
- }
71
-
72
80
  export function urlRewriteFunction(
73
81
  current_url, // The current (real) url we are on, e.g. http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/index.html
74
82
  orig_host, // The host of the original url, e.g. www.example.com
@@ -186,15 +194,23 @@ export function urlRewriteFunction(
186
194
  : '?' + decodeURIComponent(absolute_url_parts.query).replaceAll('+', ' ');
187
195
 
188
196
  // combine all decoded parts to get the ZIM path
189
- const zimPath =
190
- decoded_host + removeSubsequentSlashes(decoded_path + decoded_query);
197
+ // (consecutive slashes are kept, just like in Python)
198
+ const zimPath = decoded_host + decoded_path + decoded_query;
191
199
 
192
200
  // apply the fuzzy rules to the ZIM path
193
201
  const fuzzifiedPath = applyFuzzyRules(zimPath);
194
202
 
203
+ // escape '.' and '..' path segments so they are not resolved as navigation by the
204
+ // reader once this ZIM path is turned into a URI
205
+ const escapedPath = escapeDotSegments(fuzzifiedPath);
206
+
195
207
  // Reencode everything but '/' (we decode it afterwards for simplicity)
196
208
  const finalUrl =
197
- prefix + encodeURIComponent(fuzzifiedPath).replaceAll('%2F', '/');
209
+ prefix +
210
+ encodeURIComponent(escapedPath)
211
+ .replaceAll('%2F', '/')
212
+ .replaceAll('%3D', '=')
213
+ .replaceAll('%2C', ',');
198
214
 
199
215
  console.debug(
200
216
  'urlRewriten:\n\t- current_url: ' +
@@ -312,6 +328,7 @@ export function getWombatInfo(
312
328
 
313
329
  export default {
314
330
  applyFuzzyRules: applyFuzzyRules,
331
+ escapeDotSegments: escapeDotSegments,
315
332
  urlRewriteFunction: urlRewriteFunction,
316
333
  getWombatInfo: getWombatInfo,
317
334
  };
@@ -8,6 +8,29 @@ test.beforeEach((t) => {
8
8
  t.context.originalScheme = 'https';
9
9
  });
10
10
 
11
+ test('escapeDotSegmentsDot', (t) => {
12
+ t.is(utils.escapeDotSegments('a/./b'), 'a/_./b');
13
+ });
14
+
15
+ test('escapeDotSegmentsDotDot', (t) => {
16
+ t.is(utils.escapeDotSegments('a/../b'), 'a/_../b');
17
+ });
18
+
19
+ test('escapeDotSegmentsTrailingDotDot', (t) => {
20
+ t.is(utils.escapeDotSegments('a/..'), 'a/_..');
21
+ });
22
+
23
+ test('escapeDotSegmentsConsecutive', (t) => {
24
+ t.is(utils.escapeDotSegments('a/../../b'), 'a/_../_../b');
25
+ });
26
+
27
+ test('escapeDotSegmentsLookAlikesUntouched', (t) => {
28
+ t.is(
29
+ utils.escapeDotSegments('a/.foo/foo../..foo/.../b'),
30
+ 'a/.foo/foo../..foo/.../b',
31
+ );
32
+ });
33
+
11
34
  test('nominalWbInfo', (t) => {
12
35
  const path = 'path1/resource1.js';
13
36
  const originalUrl =
@@ -84,6 +84,46 @@ test('simpleContentNoScheme', (t) => {
84
84
  );
85
85
  });
86
86
 
87
+ test('consecutiveSlashesAreKept', (t) => {
88
+ t.is(
89
+ urlRewriteFunction(
90
+ t.context.currentUrl,
91
+ t.context.originalHost,
92
+ t.context.originalScheme,
93
+ t.context.originalUrl,
94
+ t.context.prefix,
95
+ 'https://www.example.com/a///b',
96
+ undefined,
97
+ undefined,
98
+ undefined,
99
+ ),
100
+ 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/a///b',
101
+ );
102
+ });
103
+
104
+ test('relativeLinkOnPageWithConsecutiveSlashesKeepsThem', (t) => {
105
+ // #346: a relative link must be resolved against the page's own URL the way a
106
+ // browser would, keeping consecutive slashes in that URL instead of collapsing
107
+ // them (uri-js's URI.resolve, unlike Python's urllib.parse.urljoin, already does
108
+ // this correctly)
109
+ const originalUrl = 'https://www.example.com/path1//resource1.html';
110
+ const currentUrl = t.context.prefix + 'www.example.com/path1//resource1.html';
111
+ t.is(
112
+ urlRewriteFunction(
113
+ currentUrl,
114
+ t.context.originalHost,
115
+ t.context.originalScheme,
116
+ originalUrl,
117
+ t.context.prefix,
118
+ 'other.html',
119
+ undefined,
120
+ undefined,
121
+ undefined,
122
+ ),
123
+ t.context.prefix + 'www.example.com/path1//other.html',
124
+ );
125
+ });
126
+
87
127
  test('simpleContentEmptyPath', (t) => {
88
128
  t.is(
89
129
  urlRewriteFunction(
@@ -335,7 +375,7 @@ test('contentWithEncodedQuestionMarkAndQueryParam', (t) => {
335
375
  undefined,
336
376
  undefined,
337
377
  ),
338
- 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/cont%3Fnt.txt%3Fquery%3Dvalue',
378
+ 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/cont%3Fnt.txt%3Fquery=value',
339
379
  );
340
380
  });
341
381
 
@@ -454,7 +494,43 @@ test('contentWithSimpleQueryString', (t) => {
454
494
  undefined,
455
495
  undefined,
456
496
  ),
457
- 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery%3Dvalue',
497
+ 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery=value',
498
+ );
499
+ });
500
+
501
+ test('contentWithQueryStringDotDotSegment', (t) => {
502
+ // a `..` dot-segment folded into the path from a querystring must be escaped so
503
+ // it is not resolved as navigation once this ZIM path is turned into a URI
504
+ t.is(
505
+ urlRewriteFunction(
506
+ t.context.currentUrl,
507
+ t.context.originalHost,
508
+ t.context.originalScheme,
509
+ t.context.originalUrl,
510
+ t.context.prefix,
511
+ 'https://www.example.com/javascript/content.txt?css=../../prg',
512
+ undefined,
513
+ undefined,
514
+ undefined,
515
+ ),
516
+ 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fcss=../_../prg',
517
+ );
518
+ });
519
+
520
+ test('contentWithSimplePathWithEqualAndComma', (t) => {
521
+ t.is(
522
+ urlRewriteFunction(
523
+ t.context.currentUrl,
524
+ t.context.originalHost,
525
+ t.context.originalScheme,
526
+ t.context.originalUrl,
527
+ t.context.prefix,
528
+ 'https://www.example.com/javascript/content.txt/query1=value1,param2=value2',
529
+ undefined,
530
+ undefined,
531
+ undefined,
532
+ ),
533
+ 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt/query1=value1,param2=value2',
458
534
  );
459
535
  });
460
536
 
@@ -471,7 +547,7 @@ test('contentWithQueryValueEqualSign', (t) => {
471
547
  undefined,
472
548
  undefined,
473
549
  ),
474
- 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery%3Dval%3Deue',
550
+ 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery=val=eue',
475
551
  );
476
552
  });
477
553
 
@@ -488,7 +564,7 @@ test('contentWithQueryValuePercentSign', (t) => {
488
564
  undefined,
489
565
  undefined,
490
566
  ),
491
- 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery%3Dval%25eue',
567
+ 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery=val%25eue',
492
568
  );
493
569
  });
494
570
 
@@ -505,7 +581,7 @@ test('contentWithQueryParamPercentSign', (t) => {
505
581
  undefined,
506
582
  undefined,
507
583
  ),
508
- 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fque%25ry%3Dvaleue',
584
+ 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fque%25ry=valeue',
509
585
  );
510
586
  });
511
587
 
@@ -522,7 +598,7 @@ test('contentWithQueryParamPlusSign', (t) => {
522
598
  undefined,
523
599
  undefined,
524
600
  ),
525
- 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fparam%3Dval%20ue',
601
+ 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fparam=val%20ue',
526
602
  );
527
603
  });
528
604
 
@@ -811,7 +887,7 @@ test('youtubeFuzzyNotEncoded', (t) => {
811
887
  undefined,
812
888
  undefined,
813
889
  ),
814
- 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/path1/youtube.fuzzy.replayweb.page/get_video_info%3Fvideo_id%3D123ah',
890
+ 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/path1/youtube.fuzzy.replayweb.page/get_video_info%3Fvideo_id=123ah',
815
891
  );
816
892
  });
817
893
 
@@ -828,7 +904,7 @@ test('youtubeFuzzyEncoded', (t) => {
828
904
  undefined,
829
905
  undefined,
830
906
  ),
831
- 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/path1/youtube.fuzzy.replayweb.page/get_video_info%3Fvideo_id%3D12%3D3ah',
907
+ 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/path1/youtube.fuzzy.replayweb.page/get_video_info%3Fvideo_id=12=3ah',
832
908
  );
833
909
  });
834
910
 
@@ -931,7 +1007,7 @@ test('relAlreadyEncoded', (t) => {
931
1007
  undefined,
932
1008
  undefined,
933
1009
  ),
934
- 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery%3Dvalue',
1010
+ 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery=value',
935
1011
  );
936
1012
  });
937
1013
 
@@ -1001,7 +1077,8 @@ test('relAnotherHostAlreadyRewrittenEmptyPath', (t) => {
1001
1077
  });
1002
1078
 
1003
1079
  // this is an edge case where the URL has already been statically rewritten and is located
1004
- // on a different fuzzified domain name => we do not touch it at all
1080
+ // on a different fuzzified domain name => we do not touch it at all (for instance it keeps
1081
+ // %3D even if it should not have them)
1005
1082
  test('relAnotherFuzzifiedHostAlreadyRewritten', (t) => {
1006
1083
  t.is(
1007
1084
  urlRewriteFunction(
@@ -1141,7 +1218,7 @@ test('doubleSlash1', (t) => {
1141
1218
  undefined,
1142
1219
  undefined,
1143
1220
  ),
1144
- 'http://library.kiwix.org/content/myzim_yyyy-mm/example.com/some/path/http%3A/example.com/some/path',
1221
+ 'http://library.kiwix.org/content/myzim_yyyy-mm/example.com/some/path/http%3A//example.com//some/path',
1145
1222
  );
1146
1223
  });
1147
1224
 
@@ -1158,7 +1235,7 @@ test('doubleSlash2', (t) => {
1158
1235
  undefined,
1159
1236
  undefined,
1160
1237
  ),
1161
- 'http://library.kiwix.org/content/myzim_yyyy-mm/example.com/some/pa%3Fth/http%3A/example.com/some/path',
1238
+ 'http://library.kiwix.org/content/myzim_yyyy-mm/example.com/some/pa%3Fth/http%3A//example.com//some/path',
1162
1239
  );
1163
1240
  });
1164
1241
 
@@ -1175,6 +1252,6 @@ test('doubleSlash3', (t) => {
1175
1252
  undefined,
1176
1253
  undefined,
1177
1254
  ),
1178
- 'http://library.kiwix.org/content/myzim_yyyy-mm/example.com/so%3Fme/pa%3Fth/http%3A/example.com/some/path',
1255
+ 'http://library.kiwix.org/content/myzim_yyyy-mm/example.com/so%3Fme/pa%3Fth/http%3A//example.com//some/path',
1179
1256
  );
1180
1257
  });