zimscraperlib 5.2.0__tar.gz → 5.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/.pre-commit-config.yaml +5 -8
  2. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/.readthedocs.yaml +1 -1
  3. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/CHANGELOG.md +34 -0
  4. zimscraperlib-5.4.0/PKG-INFO +193 -0
  5. zimscraperlib-5.4.0/README.md +113 -0
  6. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/javascript/package.json +14 -14
  7. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/javascript/src/wombatSetup.js +12 -5
  8. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/javascript/test/wombatSetup.js +2 -2
  9. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/javascript/yarn.lock +694 -1036
  10. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/mkdocs.yml +1 -1
  11. zimscraperlib-5.4.0/openzim.toml +20 -0
  12. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/pyproject.toml +39 -53
  13. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/rules/rules.yaml +2 -2
  14. zimscraperlib-5.4.0/src/zimscraperlib/__about__.py +1 -0
  15. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/download.py +24 -14
  16. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/html.py +3 -4
  17. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/i18n.py +10 -9
  18. zimscraperlib-5.4.0/src/zimscraperlib/image/optimization.py +475 -0
  19. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/image/probing.py +9 -1
  20. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/misc.py +1 -5
  21. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/rewriting/css.py +9 -21
  22. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/rewriting/js.py +57 -6
  23. zimscraperlib-5.4.0/src/zimscraperlib/rewriting/statics/wombat.js +21 -0
  24. zimscraperlib-5.4.0/src/zimscraperlib/rewriting/statics/wombatSetup.js +3 -0
  25. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/video/encoding.py +0 -1
  26. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/zim/creator.py +29 -6
  27. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/zim/indexing.py +9 -4
  28. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/zim/metadata.py +0 -2
  29. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tasks.py +35 -39
  30. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/download/test_download.py +22 -0
  31. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/image/test_image.py +243 -42
  32. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/rewriting/conftest.py +12 -12
  33. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/rewriting/test_html_rewriting.py +1 -3
  34. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/rewriting/test_js_rewriting.py +9 -11
  35. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/rewriting/test_url_rewriting.py +2 -6
  36. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/rewriting/utils.py +0 -1
  37. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/zim/test_dedup.py +1 -1
  38. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/zim/test_fs.py +2 -2
  39. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/zim/test_indexing.py +4 -2
  40. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/zim/test_metadata.py +1 -3
  41. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/zim/test_zim_creator.py +62 -2
  42. zimscraperlib-5.2.0/PKG-INFO +0 -167
  43. zimscraperlib-5.2.0/README.md +0 -87
  44. zimscraperlib-5.2.0/openzim.toml +0 -10
  45. zimscraperlib-5.2.0/src/zimscraperlib/__about__.py +0 -1
  46. zimscraperlib-5.2.0/src/zimscraperlib/image/optimization.py +0 -401
  47. zimscraperlib-5.2.0/src/zimscraperlib/rewriting/statics/wombat.js +0 -21
  48. zimscraperlib-5.2.0/src/zimscraperlib/rewriting/statics/wombatSetup.js +0 -3
  49. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/.gitignore +0 -0
  50. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/LICENSE +0 -0
  51. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/contrib/README.md +0 -0
  52. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/contrib/encode_video.py +0 -0
  53. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/docs/assets/openzim.png +0 -0
  54. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/docs/functional_architecture.md +0 -0
  55. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/docs/index.md +0 -0
  56. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/docs/license.md +0 -0
  57. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/docs/scripts/generate_api_nav.py +0 -0
  58. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/docs/software_architecture.md +0 -0
  59. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/docs/technical_architecture.md +0 -0
  60. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/javascript/.prettierignore +0 -0
  61. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/javascript/.prettierrc.json +0 -0
  62. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/javascript/eslint.config.js +0 -0
  63. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/javascript/rollup.config.js +0 -0
  64. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/javascript/test/wombatUrlRewriting.js +0 -0
  65. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/rules/generate_rules.py +0 -0
  66. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/__init__.py +0 -0
  67. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/constants.py +0 -0
  68. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/filesystem.py +0 -0
  69. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/fix_ogvjs_dist.py +0 -0
  70. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/image/__init__.py +0 -0
  71. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/image/conversion.py +0 -0
  72. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/image/illustration.py +0 -0
  73. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/image/presets.py +0 -0
  74. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/image/transformation.py +0 -0
  75. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/image/utils.py +0 -0
  76. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/inputs.py +0 -0
  77. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/logging.py +0 -0
  78. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/py.typed +0 -0
  79. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/rewriting/__init__.py +0 -0
  80. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/rewriting/html.py +0 -0
  81. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/rewriting/rules.py +0 -0
  82. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/rewriting/rx_replacer.py +0 -0
  83. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/rewriting/statics/README.md +0 -0
  84. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/rewriting/statics/__wb_module_decl.js +0 -0
  85. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/rewriting/templates/head_insert.html +0 -0
  86. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/rewriting/url_rewriting.py +0 -0
  87. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/types.py +0 -0
  88. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/typing.py +0 -0
  89. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/uri.py +0 -0
  90. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/video/__init__.py +0 -0
  91. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/video/config.py +0 -0
  92. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/video/presets.py +0 -0
  93. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/video/probing.py +0 -0
  94. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/zim/__init__.py +0 -0
  95. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/zim/_libkiwix.py +0 -0
  96. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/zim/archive.py +0 -0
  97. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/zim/dedup.py +0 -0
  98. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/zim/filesystem.py +0 -0
  99. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/zim/items.py +0 -0
  100. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/zim/providers.py +0 -0
  101. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/conftest.py +0 -0
  102. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/DroidSans.ttf +0 -0
  103. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/audio.mp3 +0 -0
  104. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/blue.jpg +0 -0
  105. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/commons.png +0 -0
  106. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/commons48.png +0 -0
  107. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/encrypted.pdf +0 -0
  108. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/encrypted.txt +0 -0
  109. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/mail.gif +0 -0
  110. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/milderm.pdf +0 -0
  111. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/milderm.txt +0 -0
  112. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/ninja.webp +0 -0
  113. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/pluto.jpg +0 -0
  114. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/single_wave_icon.gbr +0 -0
  115. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/square.jpg +0 -0
  116. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/square.png +0 -0
  117. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/star.svg +0 -0
  118. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/video.mkv +0 -0
  119. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/video.mp4 +0 -0
  120. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/video.webm +0 -0
  121. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/filesystem/test_filesystem.py +0 -0
  122. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/html/conftest.py +0 -0
  123. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/html/test_html.py +0 -0
  124. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/i18n/test_i18n.py +0 -0
  125. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/image/test_illustration.py +0 -0
  126. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/inputs/test_inputs.py +0 -0
  127. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/logging/conftest.py +0 -0
  128. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/logging/test_logging.py +0 -0
  129. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/misc/test_misc.py +0 -0
  130. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/ogvjs/conftest.py +0 -0
  131. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/ogvjs/test_ogvjs.py +0 -0
  132. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/rewriting/__init__.py +0 -0
  133. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/rewriting/test_css_rewriting.py +0 -0
  134. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/rewriting/test_fuzzy_rules.py +0 -0
  135. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/rewriting/test_rx_replacer.py +0 -0
  136. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/test_constants.py +0 -0
  137. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/types/test_types.py +0 -0
  138. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/typing/test_typing.py +0 -0
  139. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/uri/test_uri.py +0 -0
  140. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/video/conftest.py +0 -0
  141. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/video/test_encoding.py +0 -0
  142. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/video/test_video.py +0 -0
  143. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/zim/conftest.py +0 -0
  144. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/zim/test_archive.py +0 -0
  145. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/zim/test_libkiwix.py +0 -0
  146. {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/yarn.lock +0 -0
@@ -3,20 +3,17 @@
3
3
  exclude: ^tests/files # these are raw test files, no need to mess with them
4
4
  repos:
5
5
  - repo: https://github.com/pre-commit/pre-commit-hooks
6
- rev: v5.0.0
6
+ rev: v6.0.0
7
7
  hooks:
8
8
  - id: trailing-whitespace
9
9
  - id: end-of-file-fixer
10
- - repo: https://github.com/psf/black
11
- rev: '25.9.0'
12
- hooks:
13
- - id: black
14
10
  - repo: https://github.com/astral-sh/ruff-pre-commit
15
- rev: v0.13.1
11
+ rev: v0.15.14
16
12
  hooks:
17
- - id: ruff
13
+ - id: ruff-check
14
+ - id: ruff-format
18
15
  - repo: https://github.com/RobertCraigie/pyright-python
19
- rev: v1.1.405
16
+ rev: v1.1.409
20
17
  hooks:
21
18
  - id: pyright
22
19
  name: pyright (system)
@@ -6,7 +6,7 @@ version: 2
6
6
  build:
7
7
  os: ubuntu-24.04
8
8
  tools:
9
- python: '3.13'
9
+ python: '3.14'
10
10
 
11
11
  # custom commands to run mkdocs build within hatch, as suggested by maintainer in
12
12
  # https://github.com/readthedocs/readthedocs.org/issues/10706
@@ -5,6 +5,40 @@ All notable changes to this project are documented in this file.
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0).
7
7
 
8
+ ## [5.4.0] - 2026-05-28
9
+
10
+ ### Added
11
+
12
+ - Add support for custom number of retries and user-agent in save_large_file (#278)
13
+ - Enhance save_large_file log level (#279)
14
+ - Extend image optimization to support in-memory streams(BytesIO/bytes) and dst_format param (#289)
15
+ - Automatically add \_ftindex:<yes/no> tag at creator start based on indexing configuration (#295)
16
+ - **BEHAVIOR CHANGE**: it is not possible anymore to add "Tags" metadata with Creator.add_metadata method after creator has started ; this is anyway not recommended anymore since 5.x (one should prefer Creator.config_metadata)
17
+
18
+ ### Fixed
19
+
20
+ - Add proper typing @overload to `zimscraperlib.image.optimize_xxx` methods (#273)
21
+ - Backport wabac.JS change around rewriting: detect when JS is in 'strict' mode and avoid using 'arguments' (#286)
22
+ - Fix installation from source, which missed wombatSetup.js (#287)
23
+ - Fix outdated system dependencies in README: remove Pillow build-time deps (bundled in wheels), add missing `libcairo` across all platforms (#152)
24
+ - Improve contribution setup instructions: use `hatch shell`, clarify commands must be run from local clone root (#153)
25
+ - Bring coverage back to 100% (#293)
26
+ - Fix `Creator.config_indexing` docstring to reflect that only the full-text index is toggled; title indexing is always performed by libzim (#294)
27
+ - Fix `format_for` to not modify input BytesIO position (#296)
28
+
29
+ ### Changed
30
+
31
+ - Upgrade to wombat 3.10.5 (#306)
32
+ - Adopt openzim/\_python-bootstrap 2.0.0 (#303)
33
+ - Upgrade Python dependencies, Node.JS runtime and Node.JS dependencies (#306)
34
+
35
+ ## [5.3.0] - 2025-11-14
36
+
37
+ ### Changed
38
+
39
+ - Upgrade to support only Python 3.14 (#266)
40
+ - Upgrade dependencies (#269)
41
+
8
42
  ## [5.2.0] - 2025-10-02
9
43
 
10
44
  ### Added
@@ -0,0 +1,193 @@
1
+ Metadata-Version: 2.4
2
+ Name: zimscraperlib
3
+ Version: 5.4.0
4
+ Summary: Collection of python tools to re-use common code across scrapers
5
+ Project-URL: Donate, https://www.kiwix.org/en/support-us/
6
+ Project-URL: Homepage, https://github.com/openzim/python-scraperlib
7
+ Author-email: openZIM <dev@openzim.org>
8
+ License: GPL-3.0-or-later
9
+ License-File: LICENSE
10
+ Keywords: offline,openzim,zim
11
+ Classifier: Development Status :: 5 - Production/Stable
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.14
16
+ Requires-Python: <3.15,>=3.14
17
+ Requires-Dist: babel<3.0,>=2.9
18
+ Requires-Dist: beartype<0.23,>=0.19
19
+ Requires-Dist: beautifulsoup4<5.0,>=4.9.3
20
+ Requires-Dist: cairosvg<3.0,>=2.2.0
21
+ Requires-Dist: colorthief==0.2.1
22
+ Requires-Dist: idna<4.0,>=2.5
23
+ Requires-Dist: iso639-lang<3.0,>=2.4.0
24
+ Requires-Dist: libzim<4.0,>=3.8.0
25
+ Requires-Dist: lxml<7.0,>=4.6.3
26
+ Requires-Dist: optimize-images<2.0,>=1.3.6
27
+ Requires-Dist: piexif==1.1.3
28
+ Requires-Dist: pillow<13.0,>=7.0.0
29
+ Requires-Dist: pymupdf<2.0,>=1.24.0
30
+ Requires-Dist: python-magic<0.5,>=0.4.3
31
+ Requires-Dist: python-resize-image<1.2,>=1.1.19
32
+ Requires-Dist: regex>=2020.7.14
33
+ Requires-Dist: requests<3.0,>=2.25.1
34
+ Requires-Dist: types-xxhash<4.0,>=2.0
35
+ Requires-Dist: urllib3<2.8.0,>=1.26.5
36
+ Requires-Dist: xxhash<4.0,>=2.0
37
+ Requires-Dist: yt-dlp
38
+ Provides-Extra: dev
39
+ Requires-Dist: coverage==7.14.1; extra == 'dev'
40
+ Requires-Dist: invoke==3.0.3; extra == 'dev'
41
+ Requires-Dist: ipython==9.13.0; extra == 'dev'
42
+ Requires-Dist: jinja2==3.1.6; extra == 'dev'
43
+ Requires-Dist: mkdocs-gen-files==0.6.1; extra == 'dev'
44
+ Requires-Dist: mkdocs-include-markdown-plugin==7.3.0; extra == 'dev'
45
+ Requires-Dist: mkdocs-literate-nav==0.6.3; extra == 'dev'
46
+ Requires-Dist: mkdocs-material==9.7.6; extra == 'dev'
47
+ Requires-Dist: mkdocs==1.6.1; extra == 'dev'
48
+ Requires-Dist: mkdocstrings[python]==1.0.3; extra == 'dev'
49
+ Requires-Dist: pre-commit==4.6.0; extra == 'dev'
50
+ Requires-Dist: pygments==2.19.2; extra == 'dev'
51
+ Requires-Dist: pymdown-extensions==10.21.3; extra == 'dev'
52
+ Requires-Dist: pyright==1.1.409; extra == 'dev'
53
+ Requires-Dist: pytest-mock==3.15.1; extra == 'dev'
54
+ Requires-Dist: pytest==9.0.3; extra == 'dev'
55
+ Requires-Dist: pyyaml==6.0.3; extra == 'dev'
56
+ Requires-Dist: ruff==0.15.14; extra == 'dev'
57
+ Provides-Extra: docs
58
+ Requires-Dist: mkdocs-gen-files==0.6.1; extra == 'docs'
59
+ Requires-Dist: mkdocs-include-markdown-plugin==7.3.0; extra == 'docs'
60
+ Requires-Dist: mkdocs-literate-nav==0.6.3; extra == 'docs'
61
+ Requires-Dist: mkdocs-material==9.7.6; extra == 'docs'
62
+ Requires-Dist: mkdocs==1.6.1; extra == 'docs'
63
+ Requires-Dist: mkdocstrings[python]==1.0.3; extra == 'docs'
64
+ Requires-Dist: pygments==2.19.2; extra == 'docs'
65
+ Requires-Dist: pymdown-extensions==10.21.3; extra == 'docs'
66
+ Requires-Dist: ruff==0.15.14; extra == 'docs'
67
+ Provides-Extra: qa
68
+ Requires-Dist: pyright==1.1.409; extra == 'qa'
69
+ Requires-Dist: pytest==9.0.3; extra == 'qa'
70
+ Requires-Dist: ruff==0.15.14; extra == 'qa'
71
+ Provides-Extra: scripts
72
+ Requires-Dist: invoke==3.0.3; extra == 'scripts'
73
+ Requires-Dist: jinja2==3.1.6; extra == 'scripts'
74
+ Requires-Dist: pyyaml==6.0.3; extra == 'scripts'
75
+ Provides-Extra: test
76
+ Requires-Dist: coverage==7.14.1; extra == 'test'
77
+ Requires-Dist: pytest-mock==3.15.1; extra == 'test'
78
+ Requires-Dist: pytest==9.0.3; extra == 'test'
79
+ Description-Content-Type: text/markdown
80
+
81
+ # zimscraperlib
82
+
83
+ [![QA Status](https://github.com/openzim/python-scraperlib/actions/workflows/QA.yaml/badge.svg?branch=main)](https://github.com/openzim/python-scraperlib/actions/workflows/QA.yaml)
84
+ [![Tests Status](https://github.com/openzim/python-scraperlib/actions/workflows/Tests.yaml/badge.svg?branch=main)](https://github.com/openzim/python-scraperlib/actions/workflows/Tests.yaml)
85
+ [![CodeFactor](https://www.codefactor.io/repository/github/openzim/python-scraperlib/badge)](https://www.codefactor.io/repository/github/openzim/python-scraperlib)
86
+ [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
87
+ [![PyPI version shields.io](https://img.shields.io/pypi/v/zimscraperlib.svg)](https://pypi.org/project/zimscraperlib/)
88
+ [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/zimscraperlib.svg)](https://pypi.org/project/zimscraperlib)
89
+ [![codecov](https://codecov.io/gh/openzim/python-scraperlib/branch/master/graph/badge.svg)](https://codecov.io/gh/openzim/python-scraperlib)
90
+ [![Read the Docs](https://img.shields.io/readthedocs/python-scraperlib)](https://python-scraperlib.readthedocs.io/)
91
+
92
+ Collection of python code to re-use across python-based scrapers
93
+
94
+ # Usage
95
+
96
+ - This library is meant to be installed via PyPI ([`zimscraperlib`](https://pypi.org/project/zimscraperlib/)).
97
+ - Make sure to reference it using a version code as the API is subject to frequent changes.
98
+ - API should remain the same only within the same _minor_ version.
99
+
100
+ Example usage:
101
+
102
+ ```pip
103
+ zimscraperlib>=1.1,<1.2
104
+ ```
105
+
106
+ See documentation at [Read the Docs](https://python-scraperlib.readthedocs.io/) for details.
107
+
108
+ > [!WARNING]
109
+ > While this library brings support for downloading videos with yt-dlp, recent changes in Youtube have forced yt-dlp team
110
+ > to require new dependencies for youtube videos (see https://github.com/yt-dlp/yt-dlp/issues/15012). These dependencies
111
+ > are significantly big and not needed for all other backend supported by yt-dlp (only youtube needs it). These dependencies
112
+ > are hence not included in this library dependencies (yet, see https://github.com/openzim/python-scraperlib/issues/268),
113
+ > you have to install them on your own if you intend to download videos from Youtube.
114
+
115
+ # Dependencies
116
+
117
+ Most dependencies are installed automatically by pip (from PyPI by default). The following system packages may be required depending on which features you use:
118
+
119
+ - **libmagic** — required for file type detection (used in most scrapers)
120
+ - **wget** — required only for `zimscraperlib.download` functions
121
+ - **FFmpeg** — required only for video processing functions
122
+ - **gifsicle** (>=1.92) — required only for GIF optimization
123
+ - **libcairo** — required only for SVG-to-PNG conversion
124
+ - **libzim** — auto-installed via PyPI, not available on Windows
125
+ - **Pillow** — auto-installed via PyPI; pre-built wheels are used by default and no system image libraries are needed. Only if you need to build Pillow from source should you install additional system libraries — see [Pillow's build documentation](https://pillow.readthedocs.io/en/latest/installation/building-from-source.html) for details.
126
+ > **Note:** To run the full test suite, all system dependencies listed above must be installed.
127
+
128
+ ## macOS
129
+
130
+ ```sh
131
+ brew install libmagic wget ffmpeg gifsicle cairo
132
+ ```
133
+
134
+ ## Linux
135
+
136
+ ```sh
137
+ sudo apt install libmagic1 wget ffmpeg gifsicle libcairo2
138
+ ```
139
+
140
+ ## Alpine
141
+
142
+ ```sh
143
+ apk add ffmpeg gifsicle libmagic wget cairo
144
+ ```
145
+
146
+ # Contribution
147
+
148
+ This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing).
149
+
150
+ This project has implemented openZIM's [Python bootstrap, conventions and policies](https://github.com/openzim/_python-bootstrap/docs/Policy.md) **v2.0.0**.
151
+
152
+ All instructions below must be run from the root of your local clone of this repository.
153
+
154
+ If you do not already have it on your system, install [hatch](https://hatch.pypa.io/latest/install/):
155
+
156
+ ```shell
157
+ pip install hatch
158
+ ```
159
+
160
+ Start a hatch shell — this will install all dependencies including dev in an isolated virtual environment:
161
+
162
+ ```shell
163
+ hatch shell
164
+ ```
165
+
166
+ Set up the pre-commit Git hook (runs linters automatically before each commit):
167
+
168
+ ```shell
169
+ pre-commit install
170
+ ```
171
+
172
+ Run tests with coverage:
173
+
174
+ ```shell
175
+ invoke coverage
176
+ ```
177
+
178
+ # Users
179
+
180
+ Non-exhaustive list of scrapers using it (check status when updating API):
181
+
182
+ - [openzim/freecodecamp](https://github.com/openzim/freecodecamp)
183
+ - [openzim/gutenberg](https://github.com/openzim/gutenberg)
184
+ - [openzim/ifixit](https://github.com/openzim/ifixit)
185
+ - [openzim/kolibri](https://github.com/openzim/kolibri)
186
+ - [openzim/nautilus](https://github.com/openzim/nautilus)
187
+ - [openzim/nautilus](https://github.com/openzim/nautilus)
188
+ - [openzim/openedx](https://github.com/openzim/openedx)
189
+ - [openzim/sotoki](https://github.com/openzim/sotoki)
190
+ - [openzim/ted](https://github.com/openzim/ted)
191
+ - [openzim/warc2zim](https://github.com/openzim/warc2zim)
192
+ - [openzim/wikihow](https://github.com/openzim/wikihow)
193
+ - [openzim/youtube](https://github.com/openzim/youtube)
@@ -0,0 +1,113 @@
1
+ # zimscraperlib
2
+
3
+ [![QA Status](https://github.com/openzim/python-scraperlib/actions/workflows/QA.yaml/badge.svg?branch=main)](https://github.com/openzim/python-scraperlib/actions/workflows/QA.yaml)
4
+ [![Tests Status](https://github.com/openzim/python-scraperlib/actions/workflows/Tests.yaml/badge.svg?branch=main)](https://github.com/openzim/python-scraperlib/actions/workflows/Tests.yaml)
5
+ [![CodeFactor](https://www.codefactor.io/repository/github/openzim/python-scraperlib/badge)](https://www.codefactor.io/repository/github/openzim/python-scraperlib)
6
+ [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
7
+ [![PyPI version shields.io](https://img.shields.io/pypi/v/zimscraperlib.svg)](https://pypi.org/project/zimscraperlib/)
8
+ [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/zimscraperlib.svg)](https://pypi.org/project/zimscraperlib)
9
+ [![codecov](https://codecov.io/gh/openzim/python-scraperlib/branch/master/graph/badge.svg)](https://codecov.io/gh/openzim/python-scraperlib)
10
+ [![Read the Docs](https://img.shields.io/readthedocs/python-scraperlib)](https://python-scraperlib.readthedocs.io/)
11
+
12
+ Collection of python code to re-use across python-based scrapers
13
+
14
+ # Usage
15
+
16
+ - This library is meant to be installed via PyPI ([`zimscraperlib`](https://pypi.org/project/zimscraperlib/)).
17
+ - Make sure to reference it using a version code as the API is subject to frequent changes.
18
+ - API should remain the same only within the same _minor_ version.
19
+
20
+ Example usage:
21
+
22
+ ```pip
23
+ zimscraperlib>=1.1,<1.2
24
+ ```
25
+
26
+ See documentation at [Read the Docs](https://python-scraperlib.readthedocs.io/) for details.
27
+
28
+ > [!WARNING]
29
+ > While this library brings support for downloading videos with yt-dlp, recent changes in Youtube have forced yt-dlp team
30
+ > to require new dependencies for youtube videos (see https://github.com/yt-dlp/yt-dlp/issues/15012). These dependencies
31
+ > are significantly big and not needed for all other backend supported by yt-dlp (only youtube needs it). These dependencies
32
+ > are hence not included in this library dependencies (yet, see https://github.com/openzim/python-scraperlib/issues/268),
33
+ > you have to install them on your own if you intend to download videos from Youtube.
34
+
35
+ # Dependencies
36
+
37
+ Most dependencies are installed automatically by pip (from PyPI by default). The following system packages may be required depending on which features you use:
38
+
39
+ - **libmagic** — required for file type detection (used in most scrapers)
40
+ - **wget** — required only for `zimscraperlib.download` functions
41
+ - **FFmpeg** — required only for video processing functions
42
+ - **gifsicle** (>=1.92) — required only for GIF optimization
43
+ - **libcairo** — required only for SVG-to-PNG conversion
44
+ - **libzim** — auto-installed via PyPI, not available on Windows
45
+ - **Pillow** — auto-installed via PyPI; pre-built wheels are used by default and no system image libraries are needed. Only if you need to build Pillow from source should you install additional system libraries — see [Pillow's build documentation](https://pillow.readthedocs.io/en/latest/installation/building-from-source.html) for details.
46
+ > **Note:** To run the full test suite, all system dependencies listed above must be installed.
47
+
48
+ ## macOS
49
+
50
+ ```sh
51
+ brew install libmagic wget ffmpeg gifsicle cairo
52
+ ```
53
+
54
+ ## Linux
55
+
56
+ ```sh
57
+ sudo apt install libmagic1 wget ffmpeg gifsicle libcairo2
58
+ ```
59
+
60
+ ## Alpine
61
+
62
+ ```sh
63
+ apk add ffmpeg gifsicle libmagic wget cairo
64
+ ```
65
+
66
+ # Contribution
67
+
68
+ This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing).
69
+
70
+ This project has implemented openZIM's [Python bootstrap, conventions and policies](https://github.com/openzim/_python-bootstrap/docs/Policy.md) **v2.0.0**.
71
+
72
+ All instructions below must be run from the root of your local clone of this repository.
73
+
74
+ If you do not already have it on your system, install [hatch](https://hatch.pypa.io/latest/install/):
75
+
76
+ ```shell
77
+ pip install hatch
78
+ ```
79
+
80
+ Start a hatch shell — this will install all dependencies including dev in an isolated virtual environment:
81
+
82
+ ```shell
83
+ hatch shell
84
+ ```
85
+
86
+ Set up the pre-commit Git hook (runs linters automatically before each commit):
87
+
88
+ ```shell
89
+ pre-commit install
90
+ ```
91
+
92
+ Run tests with coverage:
93
+
94
+ ```shell
95
+ invoke coverage
96
+ ```
97
+
98
+ # Users
99
+
100
+ Non-exhaustive list of scrapers using it (check status when updating API):
101
+
102
+ - [openzim/freecodecamp](https://github.com/openzim/freecodecamp)
103
+ - [openzim/gutenberg](https://github.com/openzim/gutenberg)
104
+ - [openzim/ifixit](https://github.com/openzim/ifixit)
105
+ - [openzim/kolibri](https://github.com/openzim/kolibri)
106
+ - [openzim/nautilus](https://github.com/openzim/nautilus)
107
+ - [openzim/nautilus](https://github.com/openzim/nautilus)
108
+ - [openzim/openedx](https://github.com/openzim/openedx)
109
+ - [openzim/sotoki](https://github.com/openzim/sotoki)
110
+ - [openzim/ted](https://github.com/openzim/ted)
111
+ - [openzim/warc2zim](https://github.com/openzim/warc2zim)
112
+ - [openzim/wikihow](https://github.com/openzim/wikihow)
113
+ - [openzim/youtube](https://github.com/openzim/youtube)
@@ -1,21 +1,21 @@
1
1
  {
2
2
  "name": "@openzim/wombat-setup",
3
3
  "type": "module",
4
- "version": "5.2.0",
4
+ "version": "5.3.1-dev0",
5
5
  "license": "GPL-3.0-or-later",
6
6
  "author": "openZIM",
7
7
  "devDependencies": {
8
- "@rollup/plugin-commonjs": "28.0.2",
9
- "@rollup/plugin-node-resolve": "16.0.0",
10
- "@rollup/plugin-strip": "^3.0.4",
11
- "@rollup/plugin-terser": "0.4.4",
12
- "ava": "^6.2.0",
13
- "babel-eslint": "^10.1.0",
14
- "eslint": "9.20.1",
15
- "eslint-config-prettier": "10.0.1",
16
- "prettier": "3.5.1",
17
- "rollup": "4.34.7",
18
- "rollup-plugin-version-injector": "^1.3.3"
8
+ "@rollup/plugin-commonjs": "29.0.2",
9
+ "@rollup/plugin-node-resolve": "16.0.3",
10
+ "@rollup/plugin-strip": "3.0.4",
11
+ "@rollup/plugin-terser": "1.0.0",
12
+ "ava": "8.0.1",
13
+ "babel-eslint": "10.1.0",
14
+ "eslint": "10.4.0",
15
+ "eslint-config-prettier": "10.1.8",
16
+ "prettier": "3.8.3",
17
+ "rollup": "4.60.4",
18
+ "rollup-plugin-version-injector": "1.3.3"
19
19
  },
20
20
  "scripts": {
21
21
  "prettier-check": "prettier . --check",
@@ -41,9 +41,9 @@
41
41
  ]
42
42
  },
43
43
  "dependencies": {
44
- "uri-js": "^4.4.1"
44
+ "uri-js": "4.4.1"
45
45
  },
46
46
  "engines": {
47
- "node": ">=20.0.0 <21.0.0"
47
+ "node": ">=26.0.0 <27.0.0"
48
48
  }
49
49
  }
@@ -292,14 +292,21 @@ export function getWombatInfo(
292
292
  // The host of the original url
293
293
  wombat_host: orig_host,
294
294
 
295
- // Extra options ?
296
- wombat_opts: {},
295
+ // We are not running inside a service worker, wombat needs to know about it since
296
+ // some "magic" URLs like blobs are not available
297
+ isSW: false,
297
298
 
298
- // ?
299
- enable_auto_fetch: true,
299
+ // Convert all post request to get request
300
300
  convert_post_to_get: true,
301
+
302
+ // Not used, we are not replaying in a frame
301
303
  target_frame: '___wb_replay_top_frame',
302
- isSW: true,
304
+
305
+ // Not used, we are not running in live mode
306
+ enable_auto_fetch: false,
307
+
308
+ // Extra options, not used
309
+ wombat_opts: {},
303
310
  };
304
311
  }
305
312
 
@@ -21,8 +21,8 @@ test('nominalWbInfo', (t) => {
21
21
  );
22
22
  t.is(wmInfo.coll, '');
23
23
  t.is(wmInfo.convert_post_to_get, true);
24
- t.is(wmInfo.enable_auto_fetch, true);
25
- t.is(wmInfo.isSW, true);
24
+ t.is(wmInfo.enable_auto_fetch, false);
25
+ t.is(wmInfo.isSW, false);
26
26
  t.is(wmInfo.is_framed, false);
27
27
  t.is(wmInfo.is_live, false);
28
28
  t.is(wmInfo.mod, '');