zimscraperlib 5.1.1__tar.gz → 5.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/.pre-commit-config.yaml +3 -3
  2. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/CHANGELOG.md +17 -0
  3. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/PKG-INFO +33 -31
  4. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/javascript/package.json +1 -1
  5. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/openzim.toml +1 -1
  6. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/pyproject.toml +23 -21
  7. zimscraperlib-5.2.0/src/zimscraperlib/__about__.py +1 -0
  8. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/constants.py +3 -0
  9. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/download.py +3 -1
  10. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/html.py +3 -1
  11. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/i18n.py +11 -0
  12. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/image/conversion.py +1 -1
  13. zimscraperlib-5.2.0/src/zimscraperlib/image/illustration.py +52 -0
  14. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/image/probing.py +7 -7
  15. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/inputs.py +1 -4
  16. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/rewriting/js.py +18 -8
  17. zimscraperlib-5.2.0/src/zimscraperlib/rewriting/statics/wombat.js +21 -0
  18. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/rewriting/statics/wombatSetup.js +1 -1
  19. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/zim/creator.py +2 -0
  20. zimscraperlib-5.2.0/src/zimscraperlib/zim/dedup.py +80 -0
  21. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/zim/items.py +1 -0
  22. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/zim/metadata.py +11 -14
  23. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/conftest.py +4 -5
  24. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/download/test_download.py +1 -1
  25. zimscraperlib-5.2.0/tests/image/test_illustration.py +69 -0
  26. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/rewriting/test_html_rewriting.py +9 -9
  27. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/rewriting/test_js_rewriting.py +95 -0
  28. zimscraperlib-5.2.0/tests/zim/test_dedup.py +107 -0
  29. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/zim/test_indexing.py +1 -0
  30. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/zim/test_zim_creator.py +5 -8
  31. zimscraperlib-5.1.1/src/zimscraperlib/__about__.py +0 -1
  32. zimscraperlib-5.1.1/src/zimscraperlib/rewriting/statics/wombat.js +0 -21
  33. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/.gitignore +0 -0
  34. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/.readthedocs.yaml +0 -0
  35. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/LICENSE +0 -0
  36. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/README.md +0 -0
  37. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/contrib/README.md +0 -0
  38. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/contrib/encode_video.py +0 -0
  39. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/docs/assets/openzim.png +0 -0
  40. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/docs/functional_architecture.md +0 -0
  41. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/docs/index.md +0 -0
  42. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/docs/license.md +0 -0
  43. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/docs/scripts/generate_api_nav.py +0 -0
  44. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/docs/software_architecture.md +0 -0
  45. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/docs/technical_architecture.md +0 -0
  46. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/javascript/.prettierignore +0 -0
  47. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/javascript/.prettierrc.json +0 -0
  48. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/javascript/eslint.config.js +0 -0
  49. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/javascript/rollup.config.js +0 -0
  50. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/javascript/src/wombatSetup.js +0 -0
  51. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/javascript/test/wombatSetup.js +0 -0
  52. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/javascript/test/wombatUrlRewriting.js +0 -0
  53. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/javascript/yarn.lock +0 -0
  54. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/mkdocs.yml +0 -0
  55. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/rules/generate_rules.py +0 -0
  56. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/rules/rules.yaml +0 -0
  57. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/__init__.py +0 -0
  58. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/filesystem.py +0 -0
  59. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/fix_ogvjs_dist.py +0 -0
  60. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/image/__init__.py +0 -0
  61. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/image/optimization.py +0 -0
  62. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/image/presets.py +0 -0
  63. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/image/transformation.py +0 -0
  64. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/image/utils.py +0 -0
  65. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/logging.py +0 -0
  66. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/misc.py +0 -0
  67. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/py.typed +0 -0
  68. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/rewriting/__init__.py +0 -0
  69. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/rewriting/css.py +0 -0
  70. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/rewriting/html.py +0 -0
  71. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/rewriting/rules.py +0 -0
  72. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/rewriting/rx_replacer.py +0 -0
  73. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/rewriting/statics/README.md +0 -0
  74. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/rewriting/statics/__wb_module_decl.js +0 -0
  75. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/rewriting/templates/head_insert.html +0 -0
  76. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/rewriting/url_rewriting.py +0 -0
  77. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/types.py +0 -0
  78. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/typing.py +0 -0
  79. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/uri.py +0 -0
  80. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/video/__init__.py +0 -0
  81. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/video/config.py +0 -0
  82. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/video/encoding.py +0 -0
  83. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/video/presets.py +0 -0
  84. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/video/probing.py +0 -0
  85. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/zim/__init__.py +0 -0
  86. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/zim/_libkiwix.py +0 -0
  87. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/zim/archive.py +0 -0
  88. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/zim/filesystem.py +0 -0
  89. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/zim/indexing.py +0 -0
  90. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/src/zimscraperlib/zim/providers.py +0 -0
  91. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tasks.py +0 -0
  92. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/files/DroidSans.ttf +0 -0
  93. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/files/audio.mp3 +0 -0
  94. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/files/blue.jpg +0 -0
  95. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/files/commons.png +0 -0
  96. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/files/commons48.png +0 -0
  97. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/files/encrypted.pdf +0 -0
  98. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/files/encrypted.txt +0 -0
  99. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/files/mail.gif +0 -0
  100. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/files/milderm.pdf +0 -0
  101. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/files/milderm.txt +0 -0
  102. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/files/ninja.webp +0 -0
  103. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/files/pluto.jpg +0 -0
  104. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/files/single_wave_icon.gbr +0 -0
  105. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/files/square.jpg +0 -0
  106. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/files/square.png +0 -0
  107. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/files/star.svg +0 -0
  108. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/files/video.mkv +0 -0
  109. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/files/video.mp4 +0 -0
  110. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/files/video.webm +0 -0
  111. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/filesystem/test_filesystem.py +0 -0
  112. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/html/conftest.py +0 -0
  113. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/html/test_html.py +0 -0
  114. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/i18n/test_i18n.py +0 -0
  115. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/image/test_image.py +0 -0
  116. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/inputs/test_inputs.py +0 -0
  117. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/logging/conftest.py +0 -0
  118. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/logging/test_logging.py +0 -0
  119. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/misc/test_misc.py +0 -0
  120. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/ogvjs/conftest.py +0 -0
  121. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/ogvjs/test_ogvjs.py +0 -0
  122. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/rewriting/__init__.py +0 -0
  123. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/rewriting/conftest.py +0 -0
  124. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/rewriting/test_css_rewriting.py +0 -0
  125. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/rewriting/test_fuzzy_rules.py +0 -0
  126. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/rewriting/test_rx_replacer.py +0 -0
  127. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/rewriting/test_url_rewriting.py +0 -0
  128. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/rewriting/utils.py +0 -0
  129. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/test_constants.py +0 -0
  130. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/types/test_types.py +0 -0
  131. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/typing/test_typing.py +0 -0
  132. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/uri/test_uri.py +0 -0
  133. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/video/conftest.py +0 -0
  134. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/video/test_encoding.py +0 -0
  135. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/video/test_video.py +0 -0
  136. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/zim/conftest.py +0 -0
  137. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/zim/test_archive.py +0 -0
  138. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/zim/test_fs.py +0 -0
  139. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/zim/test_libkiwix.py +0 -0
  140. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/tests/zim/test_metadata.py +0 -0
  141. {zimscraperlib-5.1.1 → zimscraperlib-5.2.0}/yarn.lock +0 -0
@@ -8,15 +8,15 @@ repos:
8
8
  - id: trailing-whitespace
9
9
  - id: end-of-file-fixer
10
10
  - repo: https://github.com/psf/black
11
- rev: '25.1.0'
11
+ rev: '25.9.0'
12
12
  hooks:
13
13
  - id: black
14
14
  - repo: https://github.com/astral-sh/ruff-pre-commit
15
- rev: v0.9.6
15
+ rev: v0.13.1
16
16
  hooks:
17
17
  - id: ruff
18
18
  - repo: https://github.com/RobertCraigie/pyright-python
19
- rev: v1.1.394
19
+ rev: v1.1.405
20
20
  hooks:
21
21
  - id: pyright
22
22
  name: pyright (system)
@@ -5,6 +5,23 @@ All notable changes to this project are documented in this file.
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0).
7
7
 
8
+ ## [5.2.0] - 2025-10-02
9
+
10
+ ### Added
11
+
12
+ - Add utility to fetch and prepare ZIM illustration (#254)
13
+ - New `zim.dedup.Deduplicator` class to handle automatic deduplication of content before adding to the ZIM (#33)
14
+
15
+ ### Changed
16
+
17
+ - Upgrade dependencies, especially wombat 3.9.1 (#262, #263)
18
+ - Backport changes in wabac.js around JS rewriting rules (#259, #265)
19
+
20
+ ### Fixed
21
+
22
+ - JS rewriting abusively rewrite import function (#255)
23
+ - Test about badly escaped src in HTML is failing (#264)
24
+
8
25
  ## [5.1.1] - 2025-02-17
9
26
 
10
27
  ### Changed
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: zimscraperlib
3
- Version: 5.1.1
3
+ Version: 5.2.0
4
4
  Summary: Collection of python tools to re-use common code across scrapers
5
5
  Project-URL: Donate, https://www.kiwix.org/en/support-us/
6
6
  Project-URL: Homepage, https://github.com/openzim/python-scraperlib
@@ -15,14 +15,14 @@ Classifier: Programming Language :: Python :: 3
15
15
  Classifier: Programming Language :: Python :: 3.13
16
16
  Requires-Python: <3.14,>=3.13
17
17
  Requires-Dist: babel<3.0,>=2.9
18
- Requires-Dist: beartype==0.19.0
18
+ Requires-Dist: beartype<0.22,>=0.19
19
19
  Requires-Dist: beautifulsoup4<5.0,>=4.9.3
20
20
  Requires-Dist: cairosvg<3.0,>=2.2.0
21
21
  Requires-Dist: colorthief==0.2.1
22
22
  Requires-Dist: idna<4.0,>=2.5
23
23
  Requires-Dist: iso639-lang<3.0,>=2.4.0
24
24
  Requires-Dist: libzim<4.0,>=3.4.0
25
- Requires-Dist: lxml<6.0,>=4.6.3
25
+ Requires-Dist: lxml<7.0,>=4.6.3
26
26
  Requires-Dist: optimize-images<2.0,>=1.3.6
27
27
  Requires-Dist: piexif==1.1.3
28
28
  Requires-Dist: pillow<12.0,>=7.0.0
@@ -31,49 +31,51 @@ Requires-Dist: python-magic<0.5,>=0.4.3
31
31
  Requires-Dist: python-resize-image<1.2,>=1.1.19
32
32
  Requires-Dist: regex>=2020.7.14
33
33
  Requires-Dist: requests<3.0,>=2.25.1
34
- Requires-Dist: urllib3<2.4.0,>=1.26.5
34
+ Requires-Dist: types-xxhash<4.0,>=2.0
35
+ Requires-Dist: urllib3<2.6.0,>=1.26.5
36
+ Requires-Dist: xxhash<4.0,>=2.0
35
37
  Requires-Dist: yt-dlp
36
38
  Provides-Extra: check
37
- Requires-Dist: pyright==1.1.394; extra == 'check'
38
- Requires-Dist: pytest==8.3.4; extra == 'check'
39
+ Requires-Dist: pyright==1.1.405; extra == 'check'
40
+ Requires-Dist: pytest==8.4.2; extra == 'check'
39
41
  Provides-Extra: dev
40
- Requires-Dist: black==25.1.0; extra == 'dev'
41
- Requires-Dist: coverage==7.6.12; extra == 'dev'
42
+ Requires-Dist: black==25.9.0; extra == 'dev'
43
+ Requires-Dist: coverage==7.10.7; extra == 'dev'
42
44
  Requires-Dist: invoke==2.2.0; extra == 'dev'
43
- Requires-Dist: ipython==8.32.0; extra == 'dev'
44
- Requires-Dist: jinja2==3.1.5; extra == 'dev'
45
+ Requires-Dist: ipython==9.5.0; extra == 'dev'
46
+ Requires-Dist: jinja2==3.1.6; extra == 'dev'
45
47
  Requires-Dist: mkdocs-gen-files==0.5.0; extra == 'dev'
46
- Requires-Dist: mkdocs-include-markdown-plugin==7.1.4; extra == 'dev'
47
- Requires-Dist: mkdocs-literate-nav==0.6.1; extra == 'dev'
48
- Requires-Dist: mkdocs-material==9.6.4; extra == 'dev'
48
+ Requires-Dist: mkdocs-include-markdown-plugin==7.1.7; extra == 'dev'
49
+ Requires-Dist: mkdocs-literate-nav==0.6.2; extra == 'dev'
50
+ Requires-Dist: mkdocs-material==9.6.20; extra == 'dev'
49
51
  Requires-Dist: mkdocs==1.6.1; extra == 'dev'
50
- Requires-Dist: mkdocstrings[python]==0.28.1; extra == 'dev'
51
- Requires-Dist: pre-commit==4.1.0; extra == 'dev'
52
- Requires-Dist: pymdown-extensions==10.14.3; extra == 'dev'
53
- Requires-Dist: pyright==1.1.394; extra == 'dev'
54
- Requires-Dist: pytest-mock==3.14.0; extra == 'dev'
55
- Requires-Dist: pytest==8.3.4; extra == 'dev'
52
+ Requires-Dist: mkdocstrings[python]==0.30.1; extra == 'dev'
53
+ Requires-Dist: pre-commit==4.3.0; extra == 'dev'
54
+ Requires-Dist: pymdown-extensions==10.16.1; extra == 'dev'
55
+ Requires-Dist: pyright==1.1.405; extra == 'dev'
56
+ Requires-Dist: pytest-mock==3.15.1; extra == 'dev'
57
+ Requires-Dist: pytest==8.4.2; extra == 'dev'
56
58
  Requires-Dist: pyyaml==6.0.2; extra == 'dev'
57
- Requires-Dist: ruff==0.9.6; extra == 'dev'
59
+ Requires-Dist: ruff==0.13.1; extra == 'dev'
58
60
  Provides-Extra: docs
59
61
  Requires-Dist: mkdocs-gen-files==0.5.0; extra == 'docs'
60
- Requires-Dist: mkdocs-include-markdown-plugin==7.1.4; extra == 'docs'
61
- Requires-Dist: mkdocs-literate-nav==0.6.1; extra == 'docs'
62
- Requires-Dist: mkdocs-material==9.6.4; extra == 'docs'
62
+ Requires-Dist: mkdocs-include-markdown-plugin==7.1.7; extra == 'docs'
63
+ Requires-Dist: mkdocs-literate-nav==0.6.2; extra == 'docs'
64
+ Requires-Dist: mkdocs-material==9.6.20; extra == 'docs'
63
65
  Requires-Dist: mkdocs==1.6.1; extra == 'docs'
64
- Requires-Dist: mkdocstrings[python]==0.28.1; extra == 'docs'
65
- Requires-Dist: pymdown-extensions==10.14.3; extra == 'docs'
66
+ Requires-Dist: mkdocstrings[python]==0.30.1; extra == 'docs'
67
+ Requires-Dist: pymdown-extensions==10.16.1; extra == 'docs'
66
68
  Provides-Extra: lint
67
- Requires-Dist: black==25.1.0; extra == 'lint'
68
- Requires-Dist: ruff==0.9.6; extra == 'lint'
69
+ Requires-Dist: black==25.9.0; extra == 'lint'
70
+ Requires-Dist: ruff==0.13.1; extra == 'lint'
69
71
  Provides-Extra: scripts
70
72
  Requires-Dist: invoke==2.2.0; extra == 'scripts'
71
- Requires-Dist: jinja2==3.1.5; extra == 'scripts'
73
+ Requires-Dist: jinja2==3.1.6; extra == 'scripts'
72
74
  Requires-Dist: pyyaml==6.0.2; extra == 'scripts'
73
75
  Provides-Extra: test
74
- Requires-Dist: coverage==7.6.12; extra == 'test'
75
- Requires-Dist: pytest-mock==3.14.0; extra == 'test'
76
- Requires-Dist: pytest==8.3.4; extra == 'test'
76
+ Requires-Dist: coverage==7.10.7; extra == 'test'
77
+ Requires-Dist: pytest-mock==3.15.1; extra == 'test'
78
+ Requires-Dist: pytest==8.4.2; extra == 'test'
77
79
  Description-Content-Type: text/markdown
78
80
 
79
81
  # zimscraperlib
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@openzim/wombat-setup",
3
3
  "type": "module",
4
- "version": "0.1.0-dev.0",
4
+ "version": "5.2.0",
5
5
  "license": "GPL-3.0-or-later",
6
6
  "author": "openZIM",
7
7
  "devDependencies": {
@@ -6,5 +6,5 @@ execute_after=[
6
6
 
7
7
  [files.assets.actions."wombat.js"]
8
8
  action="get_file"
9
- source="https://cdn.jsdelivr.net/npm/@webrecorder/wombat@3.8.8/dist/wombat.js"
9
+ source="https://cdn.jsdelivr.net/npm/@webrecorder/wombat@3.9.1/dist/wombat.js"
10
10
  target_file="wombat.js"
@@ -1,7 +1,7 @@
1
1
  [build-system]
2
2
  # jinja2 is required to generate JS and Python rules at build time
3
3
  # PyYAML is used to parse fuzzy rules and generate Python/JS code
4
- requires = ["hatchling", "hatch-openzim>=0.2", "jinja2==3.1.5", "PyYAML==6.0.2"]
4
+ requires = ["hatchling", "hatch-openzim>=0.2", "jinja2==3.1.6", "PyYAML==6.0.2"]
5
5
  build-backend = "hatchling.build"
6
6
 
7
7
  [project]
@@ -18,7 +18,7 @@ dependencies = [
18
18
  "python-magic>=0.4.3,<0.5",
19
19
  "libzim>=3.4.0,<4.0",
20
20
  "beautifulsoup4>=4.9.3,<5.0",
21
- "lxml>=4.6.3,<6.0",
21
+ "lxml>=4.6.3,<7.0",
22
22
  "optimize-images>=1.3.6,<2.0",
23
23
  # regex has no upper-bound due to "date-based" release numbers, no semver, so their
24
24
  # promise is that they will never (or always) break the API, and the API is very
@@ -26,13 +26,15 @@ dependencies = [
26
26
  "regex>=2020.7.14",
27
27
  "pymupdf>=1.24.0,<2.0",
28
28
  "CairoSVG>=2.2.0,<3.0",
29
- "beartype==0.19.0",
29
+ "beartype>=0.19,<0.22",
30
30
  # youtube-dl should be updated as frequently as possible
31
31
  "yt-dlp",
32
32
  "pillow>=7.0.0,<12.0",
33
- "urllib3>=1.26.5,<2.4.0",
33
+ "urllib3>=1.26.5,<2.6.0",
34
34
  "piexif==1.1.3", # this dep is a nightmare in terms of release management, better pinned just like in optimize-images anyway
35
- "idna>=2.5,<4.0"
35
+ "idna>=2.5,<4.0",
36
+ "xxhash>=2.0,<4.0",
37
+ "types-xxhash>=2.0,<4.0",
36
38
  ]
37
39
  dynamic = ["authors", "classifiers", "keywords", "license", "version", "urls"]
38
40
 
@@ -50,36 +52,36 @@ scripts = [
50
52
  "invoke==2.2.0",
51
53
  # jinja2 is required to generate JS and Python rules at build time
52
54
  # PyYAML is used to parse fuzzy rules and generate Python/JS code
53
- # also update version in build-system above and in build_js.sh
54
- "jinja2==3.1.5",
55
+ # also update version in build-system above
56
+ "jinja2==3.1.6",
55
57
  "PyYAML==6.0.2",
56
58
 
57
59
  ]
58
60
  lint = [
59
- "black==25.1.0",
60
- "ruff==0.9.6",
61
+ "black==25.9.0",
62
+ "ruff==0.13.1",
61
63
  ]
62
64
  check = [
63
- "pyright==1.1.394",
64
- "pytest==8.3.4",
65
+ "pyright==1.1.405",
66
+ "pytest==8.4.2",
65
67
  ]
66
68
  test = [
67
- "pytest==8.3.4",
68
- "pytest-mock==3.14.0",
69
- "coverage==7.6.12",
69
+ "pytest==8.4.2",
70
+ "pytest-mock==3.15.1",
71
+ "coverage==7.10.7",
70
72
  ]
71
73
  docs = [
72
74
  "mkdocs==1.6.1",
73
- "mkdocstrings[python]==0.28.1",
74
- "mkdocs-material==9.6.4",
75
- "pymdown-extensions==10.14.3",
75
+ "mkdocs-include-markdown-plugin==7.1.7",
76
+ "mkdocs-material==9.6.20",
77
+ "mkdocstrings[python]==0.30.1",
78
+ "pymdown-extensions==10.16.1",
76
79
  "mkdocs-gen-files==0.5.0",
77
- "mkdocs-literate-nav==0.6.1",
78
- "mkdocs-include-markdown-plugin==7.1.4",
80
+ "mkdocs-literate-nav==0.6.2",
79
81
  ]
80
82
  dev = [
81
- "ipython==8.32.0",
82
- "pre-commit==4.1.0",
83
+ "ipython==9.5.0",
84
+ "pre-commit==4.3.0",
83
85
  "zimscraperlib[scripts]",
84
86
  "zimscraperlib[lint]",
85
87
  "zimscraperlib[test]",
@@ -0,0 +1 @@
1
+ __version__ = "5.2.0"
@@ -29,3 +29,6 @@ ILLUSTRATIONS_METADATA_RE = re.compile(
29
29
  # default timeout to get responses from upstream when doing web requests ; this is not
30
30
  # the total time it gets to download the whole resource
31
31
  DEFAULT_WEB_REQUESTS_TIMEOUT = 10
32
+
33
+ DEFAULT_ZIM_ILLLUSTRATION_SIZE = 48
34
+ DEFAULT_ZIM_ILLLUSTRATION_SCALE = 1
@@ -34,7 +34,9 @@ class YoutubeDownloader:
34
34
  self.executor.shutdown(wait=True)
35
35
 
36
36
  def _run_youtube_dl(self, url: str, options: dict[str, Any]) -> None:
37
- with youtube_dl.YoutubeDL(options) as ydl:
37
+ with youtube_dl.YoutubeDL(
38
+ options # pyright: ignore[reportArgumentType]
39
+ ) as ydl:
38
40
  ydl.download([url]) # pyright: ignore[reportUnknownMemberType]
39
41
 
40
42
  def download(
@@ -39,7 +39,9 @@ def find_language_in(content: str | BinaryIO | TextIO, mime_type: str) -> str:
39
39
  for key in keylist:
40
40
  node = soup.find(nodename)
41
41
  if node:
42
- if not isinstance(node, element.Tag) or not node.has_attr(key):
42
+ if not isinstance(
43
+ node, element.Tag # pyright:ignore[reportUnnecessaryIsInstance]
44
+ ) or not node.has_attr(key):
43
45
  continue
44
46
  if (
45
47
  nodename == "meta"
@@ -164,6 +164,17 @@ class Language:
164
164
  and self.native == getattr(value, "native", None)
165
165
  )
166
166
 
167
+ def __hash__(self):
168
+ return hash(
169
+ f"{getattr(self, "iso_639_1", None)}$"
170
+ f"{getattr(self, "iso_639_2b", None)}$"
171
+ f"{getattr(self, "iso_639_2t", None)}$"
172
+ f"{getattr(self, "iso_639_3", None)}$"
173
+ f"{getattr(self, "iso_639_5", None)}$"
174
+ f"{getattr(self, "english", None)}$"
175
+ f"{getattr(self, "native", None)}"
176
+ )
177
+
167
178
 
168
179
  def find_language_names(query: str) -> tuple[str, str]:
169
180
  """(native, english) language names for query"""
@@ -2,7 +2,7 @@ import io
2
2
  import pathlib
3
3
  from typing import Any
4
4
 
5
- import cairosvg.svg # pyright: ignore[reportMissingTypeStubs]
5
+ import cairosvg # pyright: ignore[reportMissingTypeStubs]
6
6
  from PIL.Image import open as pilopen
7
7
 
8
8
  from zimscraperlib.constants import ALPHA_NOT_SUPPORTED
@@ -0,0 +1,52 @@
1
+ import io
2
+ import pathlib
3
+
4
+ from zimscraperlib.constants import DEFAULT_ZIM_ILLLUSTRATION_SIZE
5
+ from zimscraperlib.image.conversion import convert_image, convert_svg2png
6
+ from zimscraperlib.image.optimization import optimize_png
7
+ from zimscraperlib.image.probing import format_for
8
+ from zimscraperlib.image.transformation import resize_image
9
+ from zimscraperlib.inputs import handle_user_provided_file
10
+
11
+
12
+ def get_zim_illustration(
13
+ illustration_location: pathlib.Path | str,
14
+ width: int = DEFAULT_ZIM_ILLLUSTRATION_SIZE,
15
+ height: int = DEFAULT_ZIM_ILLLUSTRATION_SIZE,
16
+ resize_method: str = "contain",
17
+ ) -> io.BytesIO:
18
+ """Get ZIM-ready illustration from any image path or URL
19
+
20
+ illustration_location will be downloaded if needed. Image is automatically
21
+ converted to PNG, resized and optimized as needed.
22
+
23
+ Arguments:
24
+ illustration_location: path or URL to an image
25
+ width: target illustration width
26
+ height: target illustration height
27
+ resize_method: method to resize the image ; in general only 'contain' or
28
+ 'cover' make sense, but 'crop', 'width', 'height' and 'thumbnail' can be used
29
+ """
30
+
31
+ illustration_path = handle_user_provided_file(illustration_location)
32
+
33
+ if not illustration_path:
34
+ # given handle_user_provided_file logic, this is not supposed to happen besides
35
+ # when empty string is passed, hence the simple error message
36
+ raise ValueError("Illustration is missing")
37
+
38
+ illustration = io.BytesIO()
39
+ illustration_format = format_for(illustration_path, from_suffix=False)
40
+ if illustration_format == "SVG":
41
+ convert_svg2png(illustration_path, illustration, width, height)
42
+ else:
43
+ if illustration_format != "PNG":
44
+ convert_image(illustration_path, illustration, fmt="PNG")
45
+ else:
46
+ illustration = io.BytesIO(illustration_path.read_bytes())
47
+ resize_image(illustration, width, height, method=resize_method)
48
+
49
+ optimized_illustration = io.BytesIO()
50
+ optimize_png(illustration, optimized_illustration)
51
+
52
+ return optimized_illustration
@@ -5,9 +5,16 @@ import re
5
5
 
6
6
  import colorthief # pyright: ignore[reportMissingTypeStubs]
7
7
  import PIL.Image
8
+ from PIL.Image import EXTENSION as PIL_FMT_EXTENSION
9
+ from PIL.Image import init as init_pil
8
10
 
9
11
  from zimscraperlib.filesystem import get_content_mimetype, get_file_mimetype
10
12
 
13
+ init_pil() # populate the PIL_FMT_EXTENSION dictionary
14
+
15
+ known_extensions = {".svg": "SVG"}
16
+ known_extensions.update(PIL_FMT_EXTENSION)
17
+
11
18
 
12
19
  def get_colors(
13
20
  src: pathlib.Path, *, use_palette: bool | None = True
@@ -82,13 +89,6 @@ def format_for(
82
89
  "Cannot guess image format from file suffix when byte array is passed"
83
90
  )
84
91
 
85
- from PIL.Image import EXTENSION as PIL_FMT_EXTENSION
86
- from PIL.Image import init as init_pil
87
-
88
- init_pil() # populate the PIL_FMT_EXTENSION dictionary
89
-
90
- known_extensions = {".svg": "SVG"}
91
- known_extensions.update(PIL_FMT_EXTENSION)
92
92
  return known_extensions[src.suffix] if src.suffix in known_extensions else None
93
93
 
94
94
 
@@ -2,7 +2,6 @@ import pathlib
2
2
  import shutil
3
3
  import tempfile
4
4
  from collections.abc import Iterable
5
- from typing import TypeVar
6
5
 
7
6
  from zimscraperlib import logger
8
7
  from zimscraperlib.constants import DEFAULT_USER_AGENT
@@ -14,8 +13,6 @@ from zimscraperlib.constants import (
14
13
  )
15
14
  from zimscraperlib.download import stream_file
16
15
 
17
- T = TypeVar("T")
18
-
19
16
 
20
17
  def handle_user_provided_file(
21
18
  source: pathlib.Path | str | None = None,
@@ -136,6 +133,6 @@ def compute_tags(
136
133
  }
137
134
 
138
135
 
139
- def unique_values(items: list[T]) -> list[T]:
136
+ def unique_values[T](items: list[T]) -> list[T]:
140
137
  """Return unique values in input list while preserving list order"""
141
138
  return list(dict.fromkeys(items))
@@ -11,6 +11,10 @@ This modules assumes that:
11
11
  it appropriately
12
12
  - a specific JS file (provided in `statics` folder) for JS modules is included in the
13
13
  ZIM at `_zim_static/__wb_module_decl.js`
14
+
15
+ This code is based on https://github.com/webrecorder/wabac.js/blob/main/src/rewrite/jsrewriter.ts
16
+ Last backport of upstream changes is from Sept 13, 2025
17
+ Commit 6dd2d9ae664cfcd2ea8637d7d6c7ed7a0ca332a0
14
18
  """
15
19
 
16
20
  import re
@@ -29,13 +33,13 @@ from zimscraperlib.rewriting.rx_replacer import (
29
33
  from zimscraperlib.rewriting.url_rewriting import ArticleUrlRewriter, ZimPath
30
34
 
31
35
  # The regex used to rewrite `import ...` in module code.
32
- IMPORT_MATCH_RX = re.compile(
33
- r"""^\s*?import(?:['"\s]*(?:[\w*${}\s,]+from\s*)?['"\s]?['"\s])(?:.*?)['"\s]""",
36
+ IMPORT_EXPORT_MATCH_RX = re.compile(
37
+ r"""(^|;)\s*?(?:im|ex)port(?:['"\s]*(?:[\w*${}\s,]+from\s*)?['"\s]?['"\s])(?:.*?)['"\s]""",
34
38
  )
35
39
 
36
40
  # A sub regex used inside `import ...` rewrite to rewrite http url imported
37
- IMPORT_HTTP_RX = re.compile(
38
- r"""(import(?:['"\s]*(?:[\w*${}\s,]+from\s*)?['"\s]?['"\s]))((?:https?|[./]).*?)(['"\s])""",
41
+ IMPORT_EXPORT_HTTP_RX = re.compile(
42
+ r"""((?:im|ex)port(?:['"\s]*(?:[\w*${}\s,]+from\s*)?['"\s]?['"\s]))((?:https?|[./]).*?)(['"\s])""",
39
43
  )
40
44
 
41
45
  # This list of global variables we want to wrap.
@@ -153,14 +157,19 @@ def create_js_rules() -> list[TransformationRule]:
153
157
 
154
158
  return [
155
159
  # rewriting `eval(...)` - invocation
156
- (re.compile(r"(?:^|\s)\beval\s*\("), replace_prefix_from(eval_str, "eval")),
160
+ (
161
+ re.compile(r"(?<!static)(?<!function)(?<!})(?:^|\s)\beval\s*\("),
162
+ replace_prefix_from(eval_str, "eval"),
163
+ ),
164
+ (re.compile(r"\([\w]+,\s*eval\)\("), m2str(lambda _: f" {eval_str}")),
157
165
  # rewriting `x = eval` - no invocation
158
166
  (re.compile(r"[=]\s*\beval\b(?![(:.$])"), replace("eval", "self.eval")),
167
+ (re.compile(r"var\s+self"), replace("var", "let")),
159
168
  # rewriting `.postMessage` -> `__WB_pmw(self).postMessage`
160
169
  (re.compile(r"\.postMessage\b\("), add_prefix(".__WB_pmw(self)")),
161
170
  # rewriting `location = ` to custom expression `(...).href =` assignement
162
171
  (
163
- re.compile(r"[^$.]?\s?\blocation\b\s*[=]\s*(?![\s\d=])"),
172
+ re.compile(r"(?:^|[^$.+*/%^-])\s?\blocation\b\s*[=]\s*(?![\s\d=>])"),
164
173
  add_suffix_non_prop(check_loc),
165
174
  ),
166
175
  # rewriting `return this`
@@ -186,6 +195,7 @@ def create_js_rules() -> list[TransformationRule]:
186
195
  # As the rule will match first, it will prevent next rule matching `import` to
187
196
  # be apply to `async import`.
188
197
  (re.compile(r"async\s+import\s*\("), m2str(lambda x: x)),
198
+ (re.compile(r"[^$.]\bimport\s*\([^)]*\)\s*\{"), m2str(lambda x: x)),
189
199
  # esm dynamic import, if found, mark as module
190
200
  (
191
201
  re.compile(r"[^$.]\bimport\s*\("),
@@ -309,8 +319,8 @@ class JsRewriter(RxRewriter):
309
319
  f"{match.group(3)}"
310
320
  )
311
321
 
312
- return IMPORT_HTTP_RX.sub(sub_funct, m_object[0])
322
+ return IMPORT_EXPORT_HTTP_RX.sub(sub_funct, m_object[0])
313
323
 
314
324
  return func
315
325
 
316
- return (IMPORT_MATCH_RX, rewrite_import())
326
+ return (IMPORT_EXPORT_MATCH_RX, rewrite_import())