zimscraperlib 5.4.1__tar.gz → 5.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/.pre-commit-config.yaml +2 -2
  2. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/CHANGELOG.md +30 -0
  3. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/PKG-INFO +31 -29
  4. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/javascript/package.json +7 -9
  5. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/javascript/src/wombatSetup.js +22 -9
  6. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/javascript/test/wombatSetup.js +23 -0
  7. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/javascript/test/wombatUrlRewriting.js +62 -3
  8. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/javascript/yarn.lock +406 -477
  9. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/openzim.toml +1 -1
  10. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/pyproject.toml +26 -19
  11. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/rules/rules.yaml +23 -3
  12. zimscraperlib-5.5.0/src/zimscraperlib/__about__.py +1 -0
  13. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/constants.py +1 -0
  14. zimscraperlib-5.5.0/src/zimscraperlib/download.py +400 -0
  15. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/html.py +3 -2
  16. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/html.py +71 -23
  17. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/js.py +98 -5
  18. zimscraperlib-5.5.0/src/zimscraperlib/rewriting/js_ast.py +132 -0
  19. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/rules.py +4 -0
  20. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/rx_replacer.py +2 -1
  21. zimscraperlib-5.5.0/src/zimscraperlib/rewriting/statics/wombat.js +21 -0
  22. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/statics/wombatSetup.js +3 -3
  23. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/url_rewriting.py +146 -28
  24. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/creator.py +7 -4
  25. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/filesystem.py +3 -3
  26. zimscraperlib-5.5.0/src/zimscraperlib/zim/indexing.py +241 -0
  27. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/items.py +18 -3
  28. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/metadata.py +7 -9
  29. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/conftest.py +99 -1
  30. zimscraperlib-5.5.0/tests/download/test_download.py +776 -0
  31. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/html/test_html.py +8 -0
  32. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/rewriting/test_fuzzy_rules.py +62 -0
  33. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/rewriting/test_html_rewriting.py +150 -11
  34. zimscraperlib-5.5.0/tests/rewriting/test_js_globals.py +264 -0
  35. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/rewriting/test_js_rewriting.py +54 -14
  36. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/rewriting/test_rx_replacer.py +8 -0
  37. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/rewriting/test_url_rewriting.py +270 -3
  38. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/zim/test_indexing.py +92 -3
  39. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/zim/test_metadata.py +6 -10
  40. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/zim/test_zim_creator.py +21 -3
  41. zimscraperlib-5.4.1/src/zimscraperlib/__about__.py +0 -1
  42. zimscraperlib-5.4.1/src/zimscraperlib/download.py +0 -248
  43. zimscraperlib-5.4.1/src/zimscraperlib/rewriting/statics/wombat.js +0 -21
  44. zimscraperlib-5.4.1/src/zimscraperlib/zim/indexing.py +0 -130
  45. zimscraperlib-5.4.1/tests/download/test_download.py +0 -346
  46. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/.gitignore +0 -0
  47. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/.readthedocs.yaml +0 -0
  48. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/LICENSE +0 -0
  49. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/README.md +0 -0
  50. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/contrib/README.md +0 -0
  51. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/contrib/encode_video.py +0 -0
  52. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/docs/assets/openzim.png +0 -0
  53. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/docs/functional_architecture.md +0 -0
  54. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/docs/index.md +0 -0
  55. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/docs/license.md +0 -0
  56. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/docs/scripts/generate_api_nav.py +0 -0
  57. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/docs/software_architecture.md +0 -0
  58. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/docs/technical_architecture.md +0 -0
  59. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/javascript/.prettierignore +0 -0
  60. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/javascript/.prettierrc.json +0 -0
  61. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/javascript/.yarnrc.yml +0 -0
  62. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/javascript/eslint.config.js +0 -0
  63. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/javascript/rollup.config.js +0 -0
  64. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/mkdocs.yml +0 -0
  65. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/rules/generate_rules.py +0 -0
  66. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/__init__.py +0 -0
  67. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/filesystem.py +0 -0
  68. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/fix_ogvjs_dist.py +0 -0
  69. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/i18n.py +0 -0
  70. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/image/__init__.py +0 -0
  71. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/image/conversion.py +0 -0
  72. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/image/illustration.py +0 -0
  73. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/image/optimization.py +0 -0
  74. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/image/presets.py +0 -0
  75. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/image/probing.py +0 -0
  76. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/image/transformation.py +0 -0
  77. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/image/utils.py +0 -0
  78. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/inputs.py +0 -0
  79. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/logging.py +0 -0
  80. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/misc.py +0 -0
  81. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/py.typed +0 -0
  82. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/__init__.py +0 -0
  83. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/css.py +0 -0
  84. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/statics/README.md +0 -0
  85. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/statics/__wb_module_decl.js +0 -0
  86. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/templates/head_insert.html +0 -0
  87. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/types.py +0 -0
  88. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/typing.py +0 -0
  89. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/uri.py +0 -0
  90. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/video/__init__.py +0 -0
  91. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/video/config.py +0 -0
  92. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/video/encoding.py +0 -0
  93. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/video/presets.py +0 -0
  94. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/video/probing.py +0 -0
  95. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/__init__.py +0 -0
  96. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/_libkiwix.py +0 -0
  97. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/archive.py +0 -0
  98. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/dedup.py +0 -0
  99. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/providers.py +0 -0
  100. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tasks.py +0 -0
  101. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/DroidSans.ttf +0 -0
  102. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/audio.mp3 +0 -0
  103. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/blue.jpg +0 -0
  104. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/commons.png +0 -0
  105. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/commons48.png +0 -0
  106. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/encrypted.pdf +0 -0
  107. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/encrypted.txt +0 -0
  108. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/mail.gif +0 -0
  109. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/milderm.pdf +0 -0
  110. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/milderm.txt +0 -0
  111. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/ninja.webp +0 -0
  112. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/pluto.jpg +0 -0
  113. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/single_wave_icon.gbr +0 -0
  114. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/square.jpg +0 -0
  115. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/square.png +0 -0
  116. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/star.svg +0 -0
  117. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/video.mkv +0 -0
  118. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/video.mp4 +0 -0
  119. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/video.webm +0 -0
  120. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/filesystem/test_filesystem.py +0 -0
  121. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/html/conftest.py +0 -0
  122. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/i18n/test_i18n.py +0 -0
  123. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/image/test_illustration.py +0 -0
  124. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/image/test_image.py +0 -0
  125. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/inputs/test_inputs.py +0 -0
  126. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/logging/conftest.py +0 -0
  127. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/logging/test_logging.py +0 -0
  128. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/misc/test_misc.py +0 -0
  129. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/ogvjs/conftest.py +0 -0
  130. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/ogvjs/test_ogvjs.py +0 -0
  131. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/rewriting/__init__.py +0 -0
  132. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/rewriting/conftest.py +0 -0
  133. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/rewriting/test_css_rewriting.py +0 -0
  134. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/rewriting/utils.py +0 -0
  135. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/test_constants.py +0 -0
  136. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/types/test_types.py +0 -0
  137. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/typing/test_typing.py +0 -0
  138. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/uri/test_uri.py +0 -0
  139. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/video/conftest.py +0 -0
  140. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/video/test_encoding.py +0 -0
  141. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/video/test_video.py +0 -0
  142. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/zim/conftest.py +0 -0
  143. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/zim/test_archive.py +0 -0
  144. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/zim/test_dedup.py +0 -0
  145. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/zim/test_fs.py +0 -0
  146. {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/zim/test_libkiwix.py +0 -0
@@ -8,12 +8,12 @@ repos:
8
8
  - id: trailing-whitespace
9
9
  - id: end-of-file-fixer
10
10
  - repo: https://github.com/astral-sh/ruff-pre-commit
11
- rev: v0.16.0
11
+ rev: v0.17.3
12
12
  hooks:
13
13
  - id: ruff-check
14
14
  - id: ruff-format
15
15
  - repo: https://github.com/RobertCraigie/pyright-python
16
- rev: v1.1.411
16
+ rev: v1.1.414
17
17
  hooks:
18
18
  - id: pyright
19
19
  name: pyright (system)
@@ -5,6 +5,36 @@ All notable changes to this project are documented in this file.
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0).
7
7
 
8
+ ## [5.5.0] - 2026-10-29
9
+
10
+ ### Added
11
+
12
+ - Index ePub documents for full-text search, with a new `get_epub_index_data` helper and automatic indexing of ePub items, like PDFs already are (#333)
13
+ - Send a Wikimedia-compliant default `User-Agent` in `stream_file` when downloading from `upload.wikimedia.org` (#214)
14
+ - `save_large_file` accepts arbitrary HTTP headers and sends the default Wikimedia-compliant `User-Agent` for `upload.wikimedia.org`, like `stream_file` (#360)
15
+ - Add fuzzy rule to rewrite URLs of lesbases.anct.gouv.fr (#334)
16
+ - Report progress from `save_large_file` through a new `on_progress` callback and periodic logging, and stream the download to a `.part` file that is only moved into place once complete (#348)
17
+ - New `get_pdf_title` and `get_epub_title` helpers to get a document's title from its own metadata without extracting its full text content, cheaper than `get_pdf_index_data`/`get_epub_index_data` when only the title is needed (openzim/warc2zim#462, #359)
18
+
19
+ ### Changed
20
+
21
+ - Upgrade to wombat 3.11.0 (#338)
22
+ - Upgrade Python dependencies, allowing pymupdf 2, pillow 13, beartype 0.23, urllib3 2.8 and xxhash 4, and Node.JS dependencies (#366, #369)
23
+
24
+ ### Fixed
25
+
26
+ - Rewrite `save_large_file` to stream and resume the download itself instead of running `wget` once, whose `-t` counter is a budget for the whole download: a flaky connection now keeps resuming and the retry budget only counts attempts that make no progress, so long downloads are no longer abandoned mid-way (#348)
27
+ - Carry a script's top-level `const`, `let`, `class` and `function` declarations across the wombat block, so other scripts on the page can still see them, re-declaring classes with `let` so they can be reassigned (#329, #370, webrecorder/wabac.js#380)
28
+ - Keep a rewritten `<script>`'s original `src` in `__wb_orig_src`, so wombat can return it from `getAttribute("src")` and bundlers that identify chunks by it (Next.js/Turbopack, Vite) still work (openzim/warc2zim#473, #337)
29
+ - Keep the original of every rewritten `src`, `href` and `xlink:href` in `__wb_orig_<attr>`, not just a script's `src`, so wombat can return it instead of a broken URL (openzim/warc2zim#413, #338)
30
+ - Keep URL-encoded `#` characters of the path or querystring when rewriting document URIs, instead of dropping everything after them (#341)
31
+ - Find the language of an HTML page in a `content-language` `<meta>` that is not the first `<meta>` of the page (#356)
32
+ - Rewrite the `poster` attribute (e.g. of `<video>`) when rewriting HTML, like `src` and `href` (#339)
33
+ - Do not collapse consecutive slashes `//` into a single `/` when normalizing URLs into ZIM paths, in both Python and JS rewriting (#340)
34
+ - Resolve relative links against a page URL containing consecutive slashes `//` the way browsers do, instead of collapsing them like `urllib.parse.urljoin` does (#346)
35
+ - `Creator.finish()` no longer silently discards errors raised by libzim while finalizing the ZIM, so a failure during finalization is no longer reported as a successful run (openzim/warc2zim#462, #354)
36
+ - Escape `.` and `..` path segments (e.g. folded into the path from a querystring) with a leading `_` when normalizing URLs into ZIM paths, so they are kept as literal entry names instead of being resolved away as navigation, in both Python and JS rewriting (#321)
37
+
8
38
  ## [5.4.1] - 2026-07-31
9
39
 
10
40
  ### Fixed
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.4
1
+ Metadata-Version: 2.5
2
2
  Name: zimscraperlib
3
- Version: 5.4.1
3
+ Version: 5.5.0
4
4
  Summary: Collection of python tools to re-use common code across scrapers
5
5
  Project-URL: Donate, https://www.kiwix.org/en/support-us/
6
6
  Project-URL: Homepage, https://github.com/openzim/python-scraperlib
@@ -15,7 +15,7 @@ Classifier: Programming Language :: Python :: 3
15
15
  Classifier: Programming Language :: Python :: 3.14
16
16
  Requires-Python: <3.15,>=3.14
17
17
  Requires-Dist: babel<3.0,>=2.9
18
- Requires-Dist: beartype<0.23,>=0.19
18
+ Requires-Dist: beartype<0.24,>=0.19
19
19
  Requires-Dist: beautifulsoup4<5.0,>=4.9.3
20
20
  Requires-Dist: cairosvg<3.0,>=2.2.0
21
21
  Requires-Dist: colorthief==0.2.1
@@ -25,57 +25,59 @@ Requires-Dist: libzim<4.0,>=3.8.0
25
25
  Requires-Dist: lxml<7.0,>=4.6.3
26
26
  Requires-Dist: optimize-images<3.0,>=1.3.6
27
27
  Requires-Dist: piexif==1.1.3
28
- Requires-Dist: pillow<13.0,>=7.0.0
29
- Requires-Dist: pymupdf<2.0,>=1.24.0
28
+ Requires-Dist: pillow<14.0,>=7.0.0
29
+ Requires-Dist: pymupdf<3.0,>=1.24.0
30
30
  Requires-Dist: python-magic<0.5,>=0.4.3
31
31
  Requires-Dist: python-resize-image<1.2,>=1.1.19
32
32
  Requires-Dist: regex>=2020.7.14
33
33
  Requires-Dist: requests<3.0,>=2.25.1
34
+ Requires-Dist: tree-sitter-javascript<1.0,>=0.23
35
+ Requires-Dist: tree-sitter<1.0,>=0.23
34
36
  Requires-Dist: types-xxhash<4.0,>=2.0
35
- Requires-Dist: urllib3<2.8.0,>=1.26.5
36
- Requires-Dist: xxhash<4.0,>=2.0
37
+ Requires-Dist: urllib3<2.9.0,>=1.26.5
38
+ Requires-Dist: xxhash<5.0,>=2.0
37
39
  Requires-Dist: yt-dlp
38
40
  Provides-Extra: dev
39
- Requires-Dist: coverage==7.15.2; extra == 'dev'
41
+ Requires-Dist: coverage==7.17.0; extra == 'dev'
40
42
  Requires-Dist: invoke==3.0.3; extra == 'dev'
41
- Requires-Dist: ipython==9.15.0; extra == 'dev'
43
+ Requires-Dist: ipython==9.17.1; extra == 'dev'
42
44
  Requires-Dist: jinja2==3.1.6; extra == 'dev'
43
45
  Requires-Dist: mkdocs-gen-files==0.6.1; extra == 'dev'
44
46
  Requires-Dist: mkdocs-include-markdown-plugin==7.3.0; extra == 'dev'
45
47
  Requires-Dist: mkdocs-literate-nav==0.6.3; extra == 'dev'
46
- Requires-Dist: mkdocs-material==9.7.7; extra == 'dev'
48
+ Requires-Dist: mkdocs-material==9.8.0; extra == 'dev'
47
49
  Requires-Dist: mkdocs==1.6.1; extra == 'dev'
48
- Requires-Dist: mkdocstrings[python]==1.0.6; extra == 'dev'
49
- Requires-Dist: pre-commit==4.6.0; extra == 'dev'
50
- Requires-Dist: pygments==2.19.2; extra == 'dev'
51
- Requires-Dist: pymdown-extensions==11.0.1; extra == 'dev'
52
- Requires-Dist: pyright==1.1.411; extra == 'dev'
53
- Requires-Dist: pytest-mock==3.15.1; extra == 'dev'
54
- Requires-Dist: pytest==9.1.1; extra == 'dev'
50
+ Requires-Dist: mkdocstrings[python]==1.0.8; extra == 'dev'
51
+ Requires-Dist: pre-commit==4.7.0; extra == 'dev'
52
+ Requires-Dist: pygments==2.21.0; extra == 'dev'
53
+ Requires-Dist: pymdown-extensions==12.3; extra == 'dev'
54
+ Requires-Dist: pyright==1.1.414; extra == 'dev'
55
+ Requires-Dist: pytest-mock==3.17.0; extra == 'dev'
56
+ Requires-Dist: pytest==9.2.0; extra == 'dev'
55
57
  Requires-Dist: pyyaml==6.0.3; extra == 'dev'
56
- Requires-Dist: ruff==0.16.0; extra == 'dev'
58
+ Requires-Dist: ruff==0.17.3; extra == 'dev'
57
59
  Provides-Extra: docs
58
60
  Requires-Dist: mkdocs-gen-files==0.6.1; extra == 'docs'
59
61
  Requires-Dist: mkdocs-include-markdown-plugin==7.3.0; extra == 'docs'
60
62
  Requires-Dist: mkdocs-literate-nav==0.6.3; extra == 'docs'
61
- Requires-Dist: mkdocs-material==9.7.7; extra == 'docs'
63
+ Requires-Dist: mkdocs-material==9.8.0; extra == 'docs'
62
64
  Requires-Dist: mkdocs==1.6.1; extra == 'docs'
63
- Requires-Dist: mkdocstrings[python]==1.0.6; extra == 'docs'
64
- Requires-Dist: pygments==2.19.2; extra == 'docs'
65
- Requires-Dist: pymdown-extensions==11.0.1; extra == 'docs'
66
- Requires-Dist: ruff==0.16.0; extra == 'docs'
65
+ Requires-Dist: mkdocstrings[python]==1.0.8; extra == 'docs'
66
+ Requires-Dist: pygments==2.21.0; extra == 'docs'
67
+ Requires-Dist: pymdown-extensions==12.3; extra == 'docs'
68
+ Requires-Dist: ruff==0.17.3; extra == 'docs'
67
69
  Provides-Extra: qa
68
- Requires-Dist: pyright==1.1.411; extra == 'qa'
69
- Requires-Dist: pytest==9.1.1; extra == 'qa'
70
- Requires-Dist: ruff==0.16.0; extra == 'qa'
70
+ Requires-Dist: pyright==1.1.414; extra == 'qa'
71
+ Requires-Dist: pytest==9.2.0; extra == 'qa'
72
+ Requires-Dist: ruff==0.17.3; extra == 'qa'
71
73
  Provides-Extra: scripts
72
74
  Requires-Dist: invoke==3.0.3; extra == 'scripts'
73
75
  Requires-Dist: jinja2==3.1.6; extra == 'scripts'
74
76
  Requires-Dist: pyyaml==6.0.3; extra == 'scripts'
75
77
  Provides-Extra: test
76
- Requires-Dist: coverage==7.15.2; extra == 'test'
77
- Requires-Dist: pytest-mock==3.15.1; extra == 'test'
78
- Requires-Dist: pytest==9.1.1; extra == 'test'
78
+ Requires-Dist: coverage==7.17.0; extra == 'test'
79
+ Requires-Dist: pytest-mock==3.17.0; extra == 'test'
80
+ Requires-Dist: pytest==9.2.0; extra == 'test'
79
81
  Description-Content-Type: text/markdown
80
82
 
81
83
  # zimscraperlib
@@ -1,20 +1,18 @@
1
1
  {
2
2
  "name": "@openzim/wombat-setup",
3
3
  "type": "module",
4
- "version": "5.4.1",
4
+ "version": "5.5.0",
5
5
  "license": "GPL-3.0-or-later",
6
6
  "author": "openZIM",
7
7
  "devDependencies": {
8
8
  "@rollup/plugin-commonjs": "29.0.3",
9
9
  "@rollup/plugin-node-resolve": "16.0.3",
10
10
  "@rollup/plugin-strip": "3.0.4",
11
- "@rollup/plugin-terser": "1.0.0",
12
- "ava": "8.0.1",
13
- "babel-eslint": "10.1.0",
14
- "eslint": "10.7.0",
15
- "eslint-config-prettier": "10.1.8",
16
- "prettier": "3.9.6",
17
- "rollup": "4.62.2",
11
+ "@rollup/plugin-terser": "1.0.2",
12
+ "ava": "8.2.0",
13
+ "eslint": "10.13.0",
14
+ "prettier": "3.9.10",
15
+ "rollup": "4.65.1",
18
16
  "rollup-plugin-version-injector": "1.3.3"
19
17
  },
20
18
  "scripts": {
@@ -43,5 +41,5 @@
43
41
  "engines": {
44
42
  "node": ">=26.0.0 <27.0.0"
45
43
  },
46
- "packageManager": "yarn@4.17.1"
44
+ "packageManager": "yarn@4.18.1"
47
45
  }
@@ -18,6 +18,20 @@ export function applyFuzzyRules(path) {
18
18
  return path;
19
19
  }
20
20
 
21
+ export function escapeDotSegments(path) {
22
+ // Escape '.' and '..' path segments so they are not resolved as navigation.
23
+ // Per RFC 3986, a '.' or '..' path segment is a dot-segment which any URI consumer
24
+ // removes/resolves while normalizing a reference. A ZIM path must keep such a
25
+ // segment as a literal entry name instead (e.g. one folded in from a querystring),
26
+ // so it is prefixed with a '_', which is not itself special in a ZIM path.
27
+ // Segments which only look like a dot-segment (e.g. 'foo..', '..foo', '...') are
28
+ // not dot-segments and are left untouched.
29
+ return path
30
+ .split('/')
31
+ .map((part) => (part === '.' || part === '..' ? '_' + part : part))
32
+ .join('/');
33
+ }
34
+
21
35
  export function hasAlreadyBeenRewritten(
22
36
  original_absolute_url,
23
37
  orig_url,
@@ -63,12 +77,6 @@ export function hasAlreadyBeenRewritten(
63
77
  return false;
64
78
  }
65
79
 
66
- function removeSubsequentSlashes(value) {
67
- // Remove all successive occurrences of a slash `/` in a given string
68
- // E.g `val//ue` or `val///ue` or `val////ue` (and so on) are transformed into `value`
69
- return value.replace(/\/\/+/g, '/');
70
- }
71
-
72
80
  export function urlRewriteFunction(
73
81
  current_url, // The current (real) url we are on, e.g. http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/index.html
74
82
  orig_host, // The host of the original url, e.g. www.example.com
@@ -186,16 +194,20 @@ export function urlRewriteFunction(
186
194
  : '?' + decodeURIComponent(absolute_url_parts.query).replaceAll('+', ' ');
187
195
 
188
196
  // combine all decoded parts to get the ZIM path
189
- const zimPath =
190
- decoded_host + removeSubsequentSlashes(decoded_path + decoded_query);
197
+ // (consecutive slashes are kept, just like in Python)
198
+ const zimPath = decoded_host + decoded_path + decoded_query;
191
199
 
192
200
  // apply the fuzzy rules to the ZIM path
193
201
  const fuzzifiedPath = applyFuzzyRules(zimPath);
194
202
 
203
+ // escape '.' and '..' path segments so they are not resolved as navigation by the
204
+ // reader once this ZIM path is turned into a URI
205
+ const escapedPath = escapeDotSegments(fuzzifiedPath);
206
+
195
207
  // Reencode everything but '/' (we decode it afterwards for simplicity)
196
208
  const finalUrl =
197
209
  prefix +
198
- encodeURIComponent(fuzzifiedPath)
210
+ encodeURIComponent(escapedPath)
199
211
  .replaceAll('%2F', '/')
200
212
  .replaceAll('%3D', '=')
201
213
  .replaceAll('%2C', ',');
@@ -316,6 +328,7 @@ export function getWombatInfo(
316
328
 
317
329
  export default {
318
330
  applyFuzzyRules: applyFuzzyRules,
331
+ escapeDotSegments: escapeDotSegments,
319
332
  urlRewriteFunction: urlRewriteFunction,
320
333
  getWombatInfo: getWombatInfo,
321
334
  };
@@ -8,6 +8,29 @@ test.beforeEach((t) => {
8
8
  t.context.originalScheme = 'https';
9
9
  });
10
10
 
11
+ test('escapeDotSegmentsDot', (t) => {
12
+ t.is(utils.escapeDotSegments('a/./b'), 'a/_./b');
13
+ });
14
+
15
+ test('escapeDotSegmentsDotDot', (t) => {
16
+ t.is(utils.escapeDotSegments('a/../b'), 'a/_../b');
17
+ });
18
+
19
+ test('escapeDotSegmentsTrailingDotDot', (t) => {
20
+ t.is(utils.escapeDotSegments('a/..'), 'a/_..');
21
+ });
22
+
23
+ test('escapeDotSegmentsConsecutive', (t) => {
24
+ t.is(utils.escapeDotSegments('a/../../b'), 'a/_../_../b');
25
+ });
26
+
27
+ test('escapeDotSegmentsLookAlikesUntouched', (t) => {
28
+ t.is(
29
+ utils.escapeDotSegments('a/.foo/foo../..foo/.../b'),
30
+ 'a/.foo/foo../..foo/.../b',
31
+ );
32
+ });
33
+
11
34
  test('nominalWbInfo', (t) => {
12
35
  const path = 'path1/resource1.js';
13
36
  const originalUrl =
@@ -84,6 +84,46 @@ test('simpleContentNoScheme', (t) => {
84
84
  );
85
85
  });
86
86
 
87
+ test('consecutiveSlashesAreKept', (t) => {
88
+ t.is(
89
+ urlRewriteFunction(
90
+ t.context.currentUrl,
91
+ t.context.originalHost,
92
+ t.context.originalScheme,
93
+ t.context.originalUrl,
94
+ t.context.prefix,
95
+ 'https://www.example.com/a///b',
96
+ undefined,
97
+ undefined,
98
+ undefined,
99
+ ),
100
+ 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/a///b',
101
+ );
102
+ });
103
+
104
+ test('relativeLinkOnPageWithConsecutiveSlashesKeepsThem', (t) => {
105
+ // #346: a relative link must be resolved against the page's own URL the way a
106
+ // browser would, keeping consecutive slashes in that URL instead of collapsing
107
+ // them (uri-js's URI.resolve, unlike Python's urllib.parse.urljoin, already does
108
+ // this correctly)
109
+ const originalUrl = 'https://www.example.com/path1//resource1.html';
110
+ const currentUrl = t.context.prefix + 'www.example.com/path1//resource1.html';
111
+ t.is(
112
+ urlRewriteFunction(
113
+ currentUrl,
114
+ t.context.originalHost,
115
+ t.context.originalScheme,
116
+ originalUrl,
117
+ t.context.prefix,
118
+ 'other.html',
119
+ undefined,
120
+ undefined,
121
+ undefined,
122
+ ),
123
+ t.context.prefix + 'www.example.com/path1//other.html',
124
+ );
125
+ });
126
+
87
127
  test('simpleContentEmptyPath', (t) => {
88
128
  t.is(
89
129
  urlRewriteFunction(
@@ -458,6 +498,25 @@ test('contentWithSimpleQueryString', (t) => {
458
498
  );
459
499
  });
460
500
 
501
+ test('contentWithQueryStringDotDotSegment', (t) => {
502
+ // a `..` dot-segment folded into the path from a querystring must be escaped so
503
+ // it is not resolved as navigation once this ZIM path is turned into a URI
504
+ t.is(
505
+ urlRewriteFunction(
506
+ t.context.currentUrl,
507
+ t.context.originalHost,
508
+ t.context.originalScheme,
509
+ t.context.originalUrl,
510
+ t.context.prefix,
511
+ 'https://www.example.com/javascript/content.txt?css=../../prg',
512
+ undefined,
513
+ undefined,
514
+ undefined,
515
+ ),
516
+ 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fcss=../_../prg',
517
+ );
518
+ });
519
+
461
520
  test('contentWithSimplePathWithEqualAndComma', (t) => {
462
521
  t.is(
463
522
  urlRewriteFunction(
@@ -1159,7 +1218,7 @@ test('doubleSlash1', (t) => {
1159
1218
  undefined,
1160
1219
  undefined,
1161
1220
  ),
1162
- 'http://library.kiwix.org/content/myzim_yyyy-mm/example.com/some/path/http%3A/example.com/some/path',
1221
+ 'http://library.kiwix.org/content/myzim_yyyy-mm/example.com/some/path/http%3A//example.com//some/path',
1163
1222
  );
1164
1223
  });
1165
1224
 
@@ -1176,7 +1235,7 @@ test('doubleSlash2', (t) => {
1176
1235
  undefined,
1177
1236
  undefined,
1178
1237
  ),
1179
- 'http://library.kiwix.org/content/myzim_yyyy-mm/example.com/some/pa%3Fth/http%3A/example.com/some/path',
1238
+ 'http://library.kiwix.org/content/myzim_yyyy-mm/example.com/some/pa%3Fth/http%3A//example.com//some/path',
1180
1239
  );
1181
1240
  });
1182
1241
 
@@ -1193,6 +1252,6 @@ test('doubleSlash3', (t) => {
1193
1252
  undefined,
1194
1253
  undefined,
1195
1254
  ),
1196
- 'http://library.kiwix.org/content/myzim_yyyy-mm/example.com/so%3Fme/pa%3Fth/http%3A/example.com/some/path',
1255
+ 'http://library.kiwix.org/content/myzim_yyyy-mm/example.com/so%3Fme/pa%3Fth/http%3A//example.com//some/path',
1197
1256
  );
1198
1257
  });