zimscraperlib 5.1.1__tar.gz → 5.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/.pre-commit-config.yaml +4 -4
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/.readthedocs.yaml +1 -1
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/CHANGELOG.md +24 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/PKG-INFO +48 -39
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/README.md +7 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/javascript/package.json +1 -1
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/openzim.toml +1 -1
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/pyproject.toml +31 -35
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/rules/rules.yaml +2 -2
- zimscraperlib-5.3.0/src/zimscraperlib/__about__.py +1 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/constants.py +3 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/download.py +3 -1
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/html.py +3 -1
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/i18n.py +11 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/image/conversion.py +1 -1
- zimscraperlib-5.3.0/src/zimscraperlib/image/illustration.py +52 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/image/probing.py +7 -7
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/inputs.py +1 -4
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/rewriting/js.py +18 -8
- zimscraperlib-5.3.0/src/zimscraperlib/rewriting/statics/wombat.js +21 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/rewriting/statics/wombatSetup.js +1 -1
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/zim/creator.py +2 -0
- zimscraperlib-5.3.0/src/zimscraperlib/zim/dedup.py +80 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/zim/indexing.py +9 -4
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/zim/items.py +1 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/zim/metadata.py +11 -14
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/conftest.py +4 -5
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/download/test_download.py +1 -1
- zimscraperlib-5.3.0/tests/image/test_illustration.py +69 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/rewriting/test_html_rewriting.py +9 -9
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/rewriting/test_js_rewriting.py +101 -0
- zimscraperlib-5.3.0/tests/zim/test_dedup.py +107 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/zim/test_indexing.py +1 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/zim/test_zim_creator.py +5 -8
- zimscraperlib-5.1.1/src/zimscraperlib/__about__.py +0 -1
- zimscraperlib-5.1.1/src/zimscraperlib/rewriting/statics/wombat.js +0 -21
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/.gitignore +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/LICENSE +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/contrib/README.md +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/contrib/encode_video.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/docs/assets/openzim.png +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/docs/functional_architecture.md +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/docs/index.md +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/docs/license.md +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/docs/scripts/generate_api_nav.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/docs/software_architecture.md +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/docs/technical_architecture.md +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/javascript/.prettierignore +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/javascript/.prettierrc.json +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/javascript/eslint.config.js +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/javascript/rollup.config.js +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/javascript/src/wombatSetup.js +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/javascript/test/wombatSetup.js +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/javascript/test/wombatUrlRewriting.js +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/javascript/yarn.lock +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/mkdocs.yml +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/rules/generate_rules.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/__init__.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/filesystem.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/fix_ogvjs_dist.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/image/__init__.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/image/optimization.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/image/presets.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/image/transformation.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/image/utils.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/logging.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/misc.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/py.typed +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/rewriting/__init__.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/rewriting/css.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/rewriting/html.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/rewriting/rules.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/rewriting/rx_replacer.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/rewriting/statics/README.md +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/rewriting/statics/__wb_module_decl.js +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/rewriting/templates/head_insert.html +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/rewriting/url_rewriting.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/types.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/typing.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/uri.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/video/__init__.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/video/config.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/video/encoding.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/video/presets.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/video/probing.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/zim/__init__.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/zim/_libkiwix.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/zim/archive.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/zim/filesystem.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/src/zimscraperlib/zim/providers.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tasks.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/files/DroidSans.ttf +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/files/audio.mp3 +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/files/blue.jpg +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/files/commons.png +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/files/commons48.png +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/files/encrypted.pdf +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/files/encrypted.txt +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/files/mail.gif +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/files/milderm.pdf +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/files/milderm.txt +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/files/ninja.webp +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/files/pluto.jpg +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/files/single_wave_icon.gbr +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/files/square.jpg +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/files/square.png +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/files/star.svg +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/files/video.mkv +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/files/video.mp4 +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/files/video.webm +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/filesystem/test_filesystem.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/html/conftest.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/html/test_html.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/i18n/test_i18n.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/image/test_image.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/inputs/test_inputs.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/logging/conftest.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/logging/test_logging.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/misc/test_misc.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/ogvjs/conftest.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/ogvjs/test_ogvjs.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/rewriting/__init__.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/rewriting/conftest.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/rewriting/test_css_rewriting.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/rewriting/test_fuzzy_rules.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/rewriting/test_rx_replacer.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/rewriting/test_url_rewriting.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/rewriting/utils.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/test_constants.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/types/test_types.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/typing/test_typing.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/uri/test_uri.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/video/conftest.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/video/test_encoding.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/video/test_video.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/zim/conftest.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/zim/test_archive.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/zim/test_fs.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/zim/test_libkiwix.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/tests/zim/test_metadata.py +0 -0
- {zimscraperlib-5.1.1 → zimscraperlib-5.3.0}/yarn.lock +0 -0
|
@@ -3,20 +3,20 @@
|
|
|
3
3
|
exclude: ^tests/files # these are raw test files, no need to mess with them
|
|
4
4
|
repos:
|
|
5
5
|
- repo: https://github.com/pre-commit/pre-commit-hooks
|
|
6
|
-
rev:
|
|
6
|
+
rev: v6.0.0
|
|
7
7
|
hooks:
|
|
8
8
|
- id: trailing-whitespace
|
|
9
9
|
- id: end-of-file-fixer
|
|
10
10
|
- repo: https://github.com/psf/black
|
|
11
|
-
rev: '25.
|
|
11
|
+
rev: '25.11.0'
|
|
12
12
|
hooks:
|
|
13
13
|
- id: black
|
|
14
14
|
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
15
|
-
rev: v0.
|
|
15
|
+
rev: v0.14.5
|
|
16
16
|
hooks:
|
|
17
17
|
- id: ruff
|
|
18
18
|
- repo: https://github.com/RobertCraigie/pyright-python
|
|
19
|
-
rev: v1.1.
|
|
19
|
+
rev: v1.1.407
|
|
20
20
|
hooks:
|
|
21
21
|
- id: pyright
|
|
22
22
|
name: pyright (system)
|
|
@@ -5,6 +5,30 @@ All notable changes to this project are documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0).
|
|
7
7
|
|
|
8
|
+
## [5.3.0] - 2025-11-14
|
|
9
|
+
|
|
10
|
+
### Changed
|
|
11
|
+
|
|
12
|
+
- Upgrade to support only Python 3.14 (#266)
|
|
13
|
+
- Upgrade dependencies (#269)
|
|
14
|
+
|
|
15
|
+
## [5.2.0] - 2025-10-02
|
|
16
|
+
|
|
17
|
+
### Added
|
|
18
|
+
|
|
19
|
+
- Add utility to fetch and prepare ZIM illustration (#254)
|
|
20
|
+
- New `zim.dedup.Deduplicator` class to handle automatic deduplication of content before adding to the ZIM (#33)
|
|
21
|
+
|
|
22
|
+
### Changed
|
|
23
|
+
|
|
24
|
+
- Upgrade dependencies, especially wombat 3.9.1 (#262, #263)
|
|
25
|
+
- Backport changes in wabac.js around JS rewriting rules (#259, #265)
|
|
26
|
+
|
|
27
|
+
### Fixed
|
|
28
|
+
|
|
29
|
+
- JS rewriting abusively rewrite import function (#255)
|
|
30
|
+
- Test about badly escaped src in HTML is failing (#264)
|
|
31
|
+
|
|
8
32
|
## [5.1.1] - 2025-02-17
|
|
9
33
|
|
|
10
34
|
### Changed
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: zimscraperlib
|
|
3
|
-
Version: 5.
|
|
3
|
+
Version: 5.3.0
|
|
4
4
|
Summary: Collection of python tools to re-use common code across scrapers
|
|
5
5
|
Project-URL: Donate, https://www.kiwix.org/en/support-us/
|
|
6
6
|
Project-URL: Homepage, https://github.com/openzim/python-scraperlib
|
|
@@ -12,68 +12,70 @@ Classifier: Development Status :: 5 - Production/Stable
|
|
|
12
12
|
Classifier: Intended Audience :: Developers
|
|
13
13
|
Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
|
|
14
14
|
Classifier: Programming Language :: Python :: 3
|
|
15
|
-
Classifier: Programming Language :: Python :: 3.
|
|
16
|
-
Requires-Python: <3.
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
16
|
+
Requires-Python: <3.15,>=3.14
|
|
17
17
|
Requires-Dist: babel<3.0,>=2.9
|
|
18
|
-
Requires-Dist: beartype
|
|
18
|
+
Requires-Dist: beartype<0.23,>=0.19
|
|
19
19
|
Requires-Dist: beautifulsoup4<5.0,>=4.9.3
|
|
20
20
|
Requires-Dist: cairosvg<3.0,>=2.2.0
|
|
21
21
|
Requires-Dist: colorthief==0.2.1
|
|
22
22
|
Requires-Dist: idna<4.0,>=2.5
|
|
23
23
|
Requires-Dist: iso639-lang<3.0,>=2.4.0
|
|
24
|
-
Requires-Dist: libzim<4.0,>=3.
|
|
25
|
-
Requires-Dist: lxml<
|
|
24
|
+
Requires-Dist: libzim<4.0,>=3.8.0
|
|
25
|
+
Requires-Dist: lxml<7.0,>=4.6.3
|
|
26
26
|
Requires-Dist: optimize-images<2.0,>=1.3.6
|
|
27
27
|
Requires-Dist: piexif==1.1.3
|
|
28
|
-
Requires-Dist: pillow<
|
|
28
|
+
Requires-Dist: pillow<13.0,>=7.0.0
|
|
29
29
|
Requires-Dist: pymupdf<2.0,>=1.24.0
|
|
30
30
|
Requires-Dist: python-magic<0.5,>=0.4.3
|
|
31
31
|
Requires-Dist: python-resize-image<1.2,>=1.1.19
|
|
32
32
|
Requires-Dist: regex>=2020.7.14
|
|
33
33
|
Requires-Dist: requests<3.0,>=2.25.1
|
|
34
|
-
Requires-Dist:
|
|
34
|
+
Requires-Dist: types-xxhash<4.0,>=2.0
|
|
35
|
+
Requires-Dist: urllib3<2.6.0,>=1.26.5
|
|
36
|
+
Requires-Dist: xxhash<4.0,>=2.0
|
|
35
37
|
Requires-Dist: yt-dlp
|
|
36
38
|
Provides-Extra: check
|
|
37
|
-
Requires-Dist: pyright==1.1.
|
|
38
|
-
Requires-Dist: pytest==
|
|
39
|
+
Requires-Dist: pyright==1.1.407; extra == 'check'
|
|
40
|
+
Requires-Dist: pytest==9.0.1; extra == 'check'
|
|
39
41
|
Provides-Extra: dev
|
|
40
|
-
Requires-Dist: black==25.
|
|
41
|
-
Requires-Dist: coverage==7.
|
|
42
|
-
Requires-Dist: invoke==2.2.
|
|
43
|
-
Requires-Dist: ipython==
|
|
44
|
-
Requires-Dist: jinja2==3.1.
|
|
42
|
+
Requires-Dist: black==25.11.0; extra == 'dev'
|
|
43
|
+
Requires-Dist: coverage==7.11.3; extra == 'dev'
|
|
44
|
+
Requires-Dist: invoke==2.2.1; extra == 'dev'
|
|
45
|
+
Requires-Dist: ipython==9.7.0; extra == 'dev'
|
|
46
|
+
Requires-Dist: jinja2==3.1.6; extra == 'dev'
|
|
45
47
|
Requires-Dist: mkdocs-gen-files==0.5.0; extra == 'dev'
|
|
46
|
-
Requires-Dist: mkdocs-include-markdown-plugin==7.
|
|
47
|
-
Requires-Dist: mkdocs-literate-nav==0.6.
|
|
48
|
-
Requires-Dist: mkdocs-material==9.
|
|
48
|
+
Requires-Dist: mkdocs-include-markdown-plugin==7.2.0; extra == 'dev'
|
|
49
|
+
Requires-Dist: mkdocs-literate-nav==0.6.2; extra == 'dev'
|
|
50
|
+
Requires-Dist: mkdocs-material==9.7.0; extra == 'dev'
|
|
49
51
|
Requires-Dist: mkdocs==1.6.1; extra == 'dev'
|
|
50
|
-
Requires-Dist: mkdocstrings[python]==0.
|
|
51
|
-
Requires-Dist: pre-commit==4.
|
|
52
|
-
Requires-Dist: pymdown-extensions==10.
|
|
53
|
-
Requires-Dist: pyright==1.1.
|
|
54
|
-
Requires-Dist: pytest-mock==3.
|
|
55
|
-
Requires-Dist: pytest==
|
|
56
|
-
Requires-Dist: pyyaml==6.0.
|
|
57
|
-
Requires-Dist: ruff==0.
|
|
52
|
+
Requires-Dist: mkdocstrings[python]==0.30.1; extra == 'dev'
|
|
53
|
+
Requires-Dist: pre-commit==4.4.0; extra == 'dev'
|
|
54
|
+
Requires-Dist: pymdown-extensions==10.17.1; extra == 'dev'
|
|
55
|
+
Requires-Dist: pyright==1.1.407; extra == 'dev'
|
|
56
|
+
Requires-Dist: pytest-mock==3.15.1; extra == 'dev'
|
|
57
|
+
Requires-Dist: pytest==9.0.1; extra == 'dev'
|
|
58
|
+
Requires-Dist: pyyaml==6.0.3; extra == 'dev'
|
|
59
|
+
Requires-Dist: ruff==0.14.5; extra == 'dev'
|
|
58
60
|
Provides-Extra: docs
|
|
59
61
|
Requires-Dist: mkdocs-gen-files==0.5.0; extra == 'docs'
|
|
60
|
-
Requires-Dist: mkdocs-include-markdown-plugin==7.
|
|
61
|
-
Requires-Dist: mkdocs-literate-nav==0.6.
|
|
62
|
-
Requires-Dist: mkdocs-material==9.
|
|
62
|
+
Requires-Dist: mkdocs-include-markdown-plugin==7.2.0; extra == 'docs'
|
|
63
|
+
Requires-Dist: mkdocs-literate-nav==0.6.2; extra == 'docs'
|
|
64
|
+
Requires-Dist: mkdocs-material==9.7.0; extra == 'docs'
|
|
63
65
|
Requires-Dist: mkdocs==1.6.1; extra == 'docs'
|
|
64
|
-
Requires-Dist: mkdocstrings[python]==0.
|
|
65
|
-
Requires-Dist: pymdown-extensions==10.
|
|
66
|
+
Requires-Dist: mkdocstrings[python]==0.30.1; extra == 'docs'
|
|
67
|
+
Requires-Dist: pymdown-extensions==10.17.1; extra == 'docs'
|
|
66
68
|
Provides-Extra: lint
|
|
67
|
-
Requires-Dist: black==25.
|
|
68
|
-
Requires-Dist: ruff==0.
|
|
69
|
+
Requires-Dist: black==25.11.0; extra == 'lint'
|
|
70
|
+
Requires-Dist: ruff==0.14.5; extra == 'lint'
|
|
69
71
|
Provides-Extra: scripts
|
|
70
|
-
Requires-Dist: invoke==2.2.
|
|
71
|
-
Requires-Dist: jinja2==3.1.
|
|
72
|
-
Requires-Dist: pyyaml==6.0.
|
|
72
|
+
Requires-Dist: invoke==2.2.1; extra == 'scripts'
|
|
73
|
+
Requires-Dist: jinja2==3.1.6; extra == 'scripts'
|
|
74
|
+
Requires-Dist: pyyaml==6.0.3; extra == 'scripts'
|
|
73
75
|
Provides-Extra: test
|
|
74
|
-
Requires-Dist: coverage==7.
|
|
75
|
-
Requires-Dist: pytest-mock==3.
|
|
76
|
-
Requires-Dist: pytest==
|
|
76
|
+
Requires-Dist: coverage==7.11.3; extra == 'test'
|
|
77
|
+
Requires-Dist: pytest-mock==3.15.1; extra == 'test'
|
|
78
|
+
Requires-Dist: pytest==9.0.1; extra == 'test'
|
|
77
79
|
Description-Content-Type: text/markdown
|
|
78
80
|
|
|
79
81
|
# zimscraperlib
|
|
@@ -102,6 +104,13 @@ zimscraperlib>=1.1,<1.2
|
|
|
102
104
|
|
|
103
105
|
See documentation at [Read the Docs](https://python-scraperlib.readthedocs.io/) for details.
|
|
104
106
|
|
|
107
|
+
> [!WARNING]
|
|
108
|
+
> While this library brings support for downloading videos with yt-dlp, recent changes in Youtube have forced yt-dlp team
|
|
109
|
+
> to require new dependencies for youtube videos (see https://github.com/yt-dlp/yt-dlp/issues/15012). These dependencies
|
|
110
|
+
> are significantly big and not needed for all other backend supported by yt-dlp (only youtube needs it). These dependencies
|
|
111
|
+
> are hence not included in this library dependencies (yet, see https://github.com/openzim/python-scraperlib/issues/268),
|
|
112
|
+
> you have to install them on your own if you intend to download videos from Youtube.
|
|
113
|
+
|
|
105
114
|
# Dependencies
|
|
106
115
|
|
|
107
116
|
- libmagic
|
|
@@ -24,6 +24,13 @@ zimscraperlib>=1.1,<1.2
|
|
|
24
24
|
|
|
25
25
|
See documentation at [Read the Docs](https://python-scraperlib.readthedocs.io/) for details.
|
|
26
26
|
|
|
27
|
+
> [!WARNING]
|
|
28
|
+
> While this library brings support for downloading videos with yt-dlp, recent changes in Youtube have forced yt-dlp team
|
|
29
|
+
> to require new dependencies for youtube videos (see https://github.com/yt-dlp/yt-dlp/issues/15012). These dependencies
|
|
30
|
+
> are significantly big and not needed for all other backend supported by yt-dlp (only youtube needs it). These dependencies
|
|
31
|
+
> are hence not included in this library dependencies (yet, see https://github.com/openzim/python-scraperlib/issues/268),
|
|
32
|
+
> you have to install them on your own if you intend to download videos from Youtube.
|
|
33
|
+
|
|
27
34
|
# Dependencies
|
|
28
35
|
|
|
29
36
|
- libmagic
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
[build-system]
|
|
2
2
|
# jinja2 is required to generate JS and Python rules at build time
|
|
3
3
|
# PyYAML is used to parse fuzzy rules and generate Python/JS code
|
|
4
|
-
requires = ["hatchling", "hatch-openzim>=0.2", "jinja2==3.1.
|
|
4
|
+
requires = ["hatchling", "hatch-openzim>=0.2", "jinja2==3.1.6", "PyYAML==6.0.3"]
|
|
5
5
|
build-backend = "hatchling.build"
|
|
6
6
|
|
|
7
7
|
[project]
|
|
8
8
|
name = "zimscraperlib"
|
|
9
|
-
requires-python = ">=3.
|
|
9
|
+
requires-python = ">=3.14,<3.15"
|
|
10
10
|
description = "Collection of python tools to re-use common code across scrapers"
|
|
11
11
|
readme = "README.md"
|
|
12
12
|
dependencies = [
|
|
@@ -16,9 +16,9 @@ dependencies = [
|
|
|
16
16
|
"python-resize-image>=1.1.19,<1.2",
|
|
17
17
|
"Babel>=2.9,<3.0",
|
|
18
18
|
"python-magic>=0.4.3,<0.5",
|
|
19
|
-
"libzim>=3.
|
|
19
|
+
"libzim>=3.8.0,<4.0",
|
|
20
20
|
"beautifulsoup4>=4.9.3,<5.0",
|
|
21
|
-
"lxml>=4.6.3,<
|
|
21
|
+
"lxml>=4.6.3,<7.0",
|
|
22
22
|
"optimize-images>=1.3.6,<2.0",
|
|
23
23
|
# regex has no upper-bound due to "date-based" release numbers, no semver, so their
|
|
24
24
|
# promise is that they will never (or always) break the API, and the API is very
|
|
@@ -26,13 +26,15 @@ dependencies = [
|
|
|
26
26
|
"regex>=2020.7.14",
|
|
27
27
|
"pymupdf>=1.24.0,<2.0",
|
|
28
28
|
"CairoSVG>=2.2.0,<3.0",
|
|
29
|
-
"beartype
|
|
29
|
+
"beartype>=0.19,<0.23",
|
|
30
30
|
# youtube-dl should be updated as frequently as possible
|
|
31
31
|
"yt-dlp",
|
|
32
|
-
"pillow>=7.0.0,<
|
|
33
|
-
"urllib3>=1.26.5,<2.
|
|
32
|
+
"pillow>=7.0.0,<13.0",
|
|
33
|
+
"urllib3>=1.26.5,<2.6.0",
|
|
34
34
|
"piexif==1.1.3", # this dep is a nightmare in terms of release management, better pinned just like in optimize-images anyway
|
|
35
|
-
"idna>=2.5,<4.0"
|
|
35
|
+
"idna>=2.5,<4.0",
|
|
36
|
+
"xxhash>=2.0,<4.0",
|
|
37
|
+
"types-xxhash>=2.0,<4.0",
|
|
36
38
|
]
|
|
37
39
|
dynamic = ["authors", "classifiers", "keywords", "license", "version", "urls"]
|
|
38
40
|
|
|
@@ -47,39 +49,39 @@ additional-classifiers = [
|
|
|
47
49
|
|
|
48
50
|
[project.optional-dependencies]
|
|
49
51
|
scripts = [
|
|
50
|
-
"invoke==2.2.
|
|
52
|
+
"invoke==2.2.1",
|
|
51
53
|
# jinja2 is required to generate JS and Python rules at build time
|
|
52
54
|
# PyYAML is used to parse fuzzy rules and generate Python/JS code
|
|
53
|
-
# also update version in build-system above
|
|
54
|
-
"jinja2==3.1.
|
|
55
|
-
"PyYAML==6.0.
|
|
55
|
+
# also update version in build-system above
|
|
56
|
+
"jinja2==3.1.6",
|
|
57
|
+
"PyYAML==6.0.3",
|
|
56
58
|
|
|
57
59
|
]
|
|
58
60
|
lint = [
|
|
59
|
-
"black==25.
|
|
60
|
-
"ruff==0.
|
|
61
|
+
"black==25.11.0",
|
|
62
|
+
"ruff==0.14.5",
|
|
61
63
|
]
|
|
62
64
|
check = [
|
|
63
|
-
"pyright==1.1.
|
|
64
|
-
"pytest==
|
|
65
|
+
"pyright==1.1.407",
|
|
66
|
+
"pytest==9.0.1",
|
|
65
67
|
]
|
|
66
68
|
test = [
|
|
67
|
-
"pytest==
|
|
68
|
-
"pytest-mock==3.
|
|
69
|
-
"coverage==7.
|
|
69
|
+
"pytest==9.0.1",
|
|
70
|
+
"pytest-mock==3.15.1",
|
|
71
|
+
"coverage==7.11.3",
|
|
70
72
|
]
|
|
71
73
|
docs = [
|
|
72
74
|
"mkdocs==1.6.1",
|
|
73
|
-
"
|
|
74
|
-
"mkdocs-material==9.
|
|
75
|
-
"
|
|
75
|
+
"mkdocs-include-markdown-plugin==7.2.0",
|
|
76
|
+
"mkdocs-material==9.7.0",
|
|
77
|
+
"mkdocstrings[python]==0.30.1",
|
|
78
|
+
"pymdown-extensions==10.17.1",
|
|
76
79
|
"mkdocs-gen-files==0.5.0",
|
|
77
|
-
"mkdocs-literate-nav==0.6.
|
|
78
|
-
"mkdocs-include-markdown-plugin==7.1.4",
|
|
80
|
+
"mkdocs-literate-nav==0.6.2",
|
|
79
81
|
]
|
|
80
82
|
dev = [
|
|
81
|
-
"ipython==
|
|
82
|
-
"pre-commit==4.
|
|
83
|
+
"ipython==9.7.0",
|
|
84
|
+
"pre-commit==4.4.0",
|
|
83
85
|
"zimscraperlib[scripts]",
|
|
84
86
|
"zimscraperlib[lint]",
|
|
85
87
|
"zimscraperlib[test]",
|
|
@@ -155,10 +157,10 @@ build = "inv docs-build --args '{args}'"
|
|
|
155
157
|
|
|
156
158
|
[tool.black]
|
|
157
159
|
line-length = 88
|
|
158
|
-
target-version = ['
|
|
160
|
+
target-version = ['py314']
|
|
159
161
|
|
|
160
162
|
[tool.ruff]
|
|
161
|
-
target-version = "
|
|
163
|
+
target-version = "py314"
|
|
162
164
|
line-length = 88
|
|
163
165
|
src = ["src", "contrib"]
|
|
164
166
|
|
|
@@ -291,12 +293,6 @@ exclude_lines = [
|
|
|
291
293
|
include = ["contrib", "src", "tests", "tasks.py"]
|
|
292
294
|
exclude = [".env/**", ".venv/**"]
|
|
293
295
|
extraPaths = ["src"]
|
|
294
|
-
pythonVersion = "3.
|
|
296
|
+
pythonVersion = "3.14"
|
|
295
297
|
typeCheckingMode="strict"
|
|
296
298
|
disableBytesTypePromotions = true
|
|
297
|
-
|
|
298
|
-
[[tool.pyright.overrides.files]]
|
|
299
|
-
files = [
|
|
300
|
-
"src/zimscraperlib/rewriting**/*.py",
|
|
301
|
-
"tests/rewriting/**/*.py"
|
|
302
|
-
]
|
|
@@ -6,8 +6,8 @@
|
|
|
6
6
|
# ones) but just rewriting to proper path.
|
|
7
7
|
#
|
|
8
8
|
# This file is in sync with content at commit
|
|
9
|
-
# https://github.com/webrecorder/wabac.js/commit/
|
|
10
|
-
# from October
|
|
9
|
+
# https://github.com/webrecorder/wabac.js/commit/f62756661d06e721bc57ff25199c73ce51227916
|
|
10
|
+
# from October 29, 2025
|
|
11
11
|
#
|
|
12
12
|
# This file should be updated at every release of scraperlib
|
|
13
13
|
#
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "5.3.0"
|
|
@@ -29,3 +29,6 @@ ILLUSTRATIONS_METADATA_RE = re.compile(
|
|
|
29
29
|
# default timeout to get responses from upstream when doing web requests ; this is not
|
|
30
30
|
# the total time it gets to download the whole resource
|
|
31
31
|
DEFAULT_WEB_REQUESTS_TIMEOUT = 10
|
|
32
|
+
|
|
33
|
+
DEFAULT_ZIM_ILLLUSTRATION_SIZE = 48
|
|
34
|
+
DEFAULT_ZIM_ILLLUSTRATION_SCALE = 1
|
|
@@ -34,7 +34,9 @@ class YoutubeDownloader:
|
|
|
34
34
|
self.executor.shutdown(wait=True)
|
|
35
35
|
|
|
36
36
|
def _run_youtube_dl(self, url: str, options: dict[str, Any]) -> None:
|
|
37
|
-
with youtube_dl.YoutubeDL(
|
|
37
|
+
with youtube_dl.YoutubeDL(
|
|
38
|
+
options # pyright: ignore[reportArgumentType]
|
|
39
|
+
) as ydl:
|
|
38
40
|
ydl.download([url]) # pyright: ignore[reportUnknownMemberType]
|
|
39
41
|
|
|
40
42
|
def download(
|
|
@@ -39,7 +39,9 @@ def find_language_in(content: str | BinaryIO | TextIO, mime_type: str) -> str:
|
|
|
39
39
|
for key in keylist:
|
|
40
40
|
node = soup.find(nodename)
|
|
41
41
|
if node:
|
|
42
|
-
if not isinstance(
|
|
42
|
+
if not isinstance(
|
|
43
|
+
node, element.Tag # pyright:ignore[reportUnnecessaryIsInstance]
|
|
44
|
+
) or not node.has_attr(key):
|
|
43
45
|
continue
|
|
44
46
|
if (
|
|
45
47
|
nodename == "meta"
|
|
@@ -164,6 +164,17 @@ class Language:
|
|
|
164
164
|
and self.native == getattr(value, "native", None)
|
|
165
165
|
)
|
|
166
166
|
|
|
167
|
+
def __hash__(self):
|
|
168
|
+
return hash(
|
|
169
|
+
f"{getattr(self, "iso_639_1", None)}$"
|
|
170
|
+
f"{getattr(self, "iso_639_2b", None)}$"
|
|
171
|
+
f"{getattr(self, "iso_639_2t", None)}$"
|
|
172
|
+
f"{getattr(self, "iso_639_3", None)}$"
|
|
173
|
+
f"{getattr(self, "iso_639_5", None)}$"
|
|
174
|
+
f"{getattr(self, "english", None)}$"
|
|
175
|
+
f"{getattr(self, "native", None)}"
|
|
176
|
+
)
|
|
177
|
+
|
|
167
178
|
|
|
168
179
|
def find_language_names(query: str) -> tuple[str, str]:
|
|
169
180
|
"""(native, english) language names for query"""
|
|
@@ -2,7 +2,7 @@ import io
|
|
|
2
2
|
import pathlib
|
|
3
3
|
from typing import Any
|
|
4
4
|
|
|
5
|
-
import cairosvg
|
|
5
|
+
import cairosvg # pyright: ignore[reportMissingTypeStubs]
|
|
6
6
|
from PIL.Image import open as pilopen
|
|
7
7
|
|
|
8
8
|
from zimscraperlib.constants import ALPHA_NOT_SUPPORTED
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import io
|
|
2
|
+
import pathlib
|
|
3
|
+
|
|
4
|
+
from zimscraperlib.constants import DEFAULT_ZIM_ILLLUSTRATION_SIZE
|
|
5
|
+
from zimscraperlib.image.conversion import convert_image, convert_svg2png
|
|
6
|
+
from zimscraperlib.image.optimization import optimize_png
|
|
7
|
+
from zimscraperlib.image.probing import format_for
|
|
8
|
+
from zimscraperlib.image.transformation import resize_image
|
|
9
|
+
from zimscraperlib.inputs import handle_user_provided_file
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def get_zim_illustration(
|
|
13
|
+
illustration_location: pathlib.Path | str,
|
|
14
|
+
width: int = DEFAULT_ZIM_ILLLUSTRATION_SIZE,
|
|
15
|
+
height: int = DEFAULT_ZIM_ILLLUSTRATION_SIZE,
|
|
16
|
+
resize_method: str = "contain",
|
|
17
|
+
) -> io.BytesIO:
|
|
18
|
+
"""Get ZIM-ready illustration from any image path or URL
|
|
19
|
+
|
|
20
|
+
illustration_location will be downloaded if needed. Image is automatically
|
|
21
|
+
converted to PNG, resized and optimized as needed.
|
|
22
|
+
|
|
23
|
+
Arguments:
|
|
24
|
+
illustration_location: path or URL to an image
|
|
25
|
+
width: target illustration width
|
|
26
|
+
height: target illustration height
|
|
27
|
+
resize_method: method to resize the image ; in general only 'contain' or
|
|
28
|
+
'cover' make sense, but 'crop', 'width', 'height' and 'thumbnail' can be used
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
illustration_path = handle_user_provided_file(illustration_location)
|
|
32
|
+
|
|
33
|
+
if not illustration_path:
|
|
34
|
+
# given handle_user_provided_file logic, this is not supposed to happen besides
|
|
35
|
+
# when empty string is passed, hence the simple error message
|
|
36
|
+
raise ValueError("Illustration is missing")
|
|
37
|
+
|
|
38
|
+
illustration = io.BytesIO()
|
|
39
|
+
illustration_format = format_for(illustration_path, from_suffix=False)
|
|
40
|
+
if illustration_format == "SVG":
|
|
41
|
+
convert_svg2png(illustration_path, illustration, width, height)
|
|
42
|
+
else:
|
|
43
|
+
if illustration_format != "PNG":
|
|
44
|
+
convert_image(illustration_path, illustration, fmt="PNG")
|
|
45
|
+
else:
|
|
46
|
+
illustration = io.BytesIO(illustration_path.read_bytes())
|
|
47
|
+
resize_image(illustration, width, height, method=resize_method)
|
|
48
|
+
|
|
49
|
+
optimized_illustration = io.BytesIO()
|
|
50
|
+
optimize_png(illustration, optimized_illustration)
|
|
51
|
+
|
|
52
|
+
return optimized_illustration
|
|
@@ -5,9 +5,16 @@ import re
|
|
|
5
5
|
|
|
6
6
|
import colorthief # pyright: ignore[reportMissingTypeStubs]
|
|
7
7
|
import PIL.Image
|
|
8
|
+
from PIL.Image import EXTENSION as PIL_FMT_EXTENSION
|
|
9
|
+
from PIL.Image import init as init_pil
|
|
8
10
|
|
|
9
11
|
from zimscraperlib.filesystem import get_content_mimetype, get_file_mimetype
|
|
10
12
|
|
|
13
|
+
init_pil() # populate the PIL_FMT_EXTENSION dictionary
|
|
14
|
+
|
|
15
|
+
known_extensions = {".svg": "SVG"}
|
|
16
|
+
known_extensions.update(PIL_FMT_EXTENSION)
|
|
17
|
+
|
|
11
18
|
|
|
12
19
|
def get_colors(
|
|
13
20
|
src: pathlib.Path, *, use_palette: bool | None = True
|
|
@@ -82,13 +89,6 @@ def format_for(
|
|
|
82
89
|
"Cannot guess image format from file suffix when byte array is passed"
|
|
83
90
|
)
|
|
84
91
|
|
|
85
|
-
from PIL.Image import EXTENSION as PIL_FMT_EXTENSION
|
|
86
|
-
from PIL.Image import init as init_pil
|
|
87
|
-
|
|
88
|
-
init_pil() # populate the PIL_FMT_EXTENSION dictionary
|
|
89
|
-
|
|
90
|
-
known_extensions = {".svg": "SVG"}
|
|
91
|
-
known_extensions.update(PIL_FMT_EXTENSION)
|
|
92
92
|
return known_extensions[src.suffix] if src.suffix in known_extensions else None
|
|
93
93
|
|
|
94
94
|
|
|
@@ -2,7 +2,6 @@ import pathlib
|
|
|
2
2
|
import shutil
|
|
3
3
|
import tempfile
|
|
4
4
|
from collections.abc import Iterable
|
|
5
|
-
from typing import TypeVar
|
|
6
5
|
|
|
7
6
|
from zimscraperlib import logger
|
|
8
7
|
from zimscraperlib.constants import DEFAULT_USER_AGENT
|
|
@@ -14,8 +13,6 @@ from zimscraperlib.constants import (
|
|
|
14
13
|
)
|
|
15
14
|
from zimscraperlib.download import stream_file
|
|
16
15
|
|
|
17
|
-
T = TypeVar("T")
|
|
18
|
-
|
|
19
16
|
|
|
20
17
|
def handle_user_provided_file(
|
|
21
18
|
source: pathlib.Path | str | None = None,
|
|
@@ -136,6 +133,6 @@ def compute_tags(
|
|
|
136
133
|
}
|
|
137
134
|
|
|
138
135
|
|
|
139
|
-
def unique_values(items: list[T]) -> list[T]:
|
|
136
|
+
def unique_values[T](items: list[T]) -> list[T]:
|
|
140
137
|
"""Return unique values in input list while preserving list order"""
|
|
141
138
|
return list(dict.fromkeys(items))
|
|
@@ -11,6 +11,10 @@ This modules assumes that:
|
|
|
11
11
|
it appropriately
|
|
12
12
|
- a specific JS file (provided in `statics` folder) for JS modules is included in the
|
|
13
13
|
ZIM at `_zim_static/__wb_module_decl.js`
|
|
14
|
+
|
|
15
|
+
This code is based on https://github.com/webrecorder/wabac.js/blob/main/src/rewrite/jsrewriter.ts
|
|
16
|
+
Last backport of upstream changes is from Oct 12, 2025
|
|
17
|
+
Commit 1849552c3dbcbc065c05afac2dd80061db37b64d
|
|
14
18
|
"""
|
|
15
19
|
|
|
16
20
|
import re
|
|
@@ -29,13 +33,13 @@ from zimscraperlib.rewriting.rx_replacer import (
|
|
|
29
33
|
from zimscraperlib.rewriting.url_rewriting import ArticleUrlRewriter, ZimPath
|
|
30
34
|
|
|
31
35
|
# The regex used to rewrite `import ...` in module code.
|
|
32
|
-
|
|
33
|
-
r"""
|
|
36
|
+
IMPORT_EXPORT_MATCH_RX = re.compile(
|
|
37
|
+
r"""(^|;)\s*?(?:im|ex)port(?:['"\s]*(?:[\w*${}\s,]+from\s*)?['"\s]?['"\s])(?:.*?)['"\s]""",
|
|
34
38
|
)
|
|
35
39
|
|
|
36
40
|
# A sub regex used inside `import ...` rewrite to rewrite http url imported
|
|
37
|
-
|
|
38
|
-
r"""(
|
|
41
|
+
IMPORT_EXPORT_HTTP_RX = re.compile(
|
|
42
|
+
r"""((?:im|ex)port(?:['"\s]*(?:[\w*${}\s,]+from\s*)?['"\s]?['"\s]))((?:https?|[./]).*?)(['"\s])""",
|
|
39
43
|
)
|
|
40
44
|
|
|
41
45
|
# This list of global variables we want to wrap.
|
|
@@ -153,14 +157,19 @@ def create_js_rules() -> list[TransformationRule]:
|
|
|
153
157
|
|
|
154
158
|
return [
|
|
155
159
|
# rewriting `eval(...)` - invocation
|
|
156
|
-
(
|
|
160
|
+
(
|
|
161
|
+
re.compile(r"(?<!static)(?<!function)(?<!})(?:^|\s)\beval\s*\("),
|
|
162
|
+
replace_prefix_from(eval_str, "eval"),
|
|
163
|
+
),
|
|
164
|
+
(re.compile(r"\([\w]+,\s*eval\)\("), m2str(lambda _: f" {eval_str}")),
|
|
157
165
|
# rewriting `x = eval` - no invocation
|
|
158
166
|
(re.compile(r"[=]\s*\beval\b(?![(:.$])"), replace("eval", "self.eval")),
|
|
167
|
+
(re.compile(r"var\s+self"), replace("var", "let")),
|
|
159
168
|
# rewriting `.postMessage` -> `__WB_pmw(self).postMessage`
|
|
160
169
|
(re.compile(r"\.postMessage\b\("), add_prefix(".__WB_pmw(self)")),
|
|
161
170
|
# rewriting `location = ` to custom expression `(...).href =` assignement
|
|
162
171
|
(
|
|
163
|
-
re.compile(r"[
|
|
172
|
+
re.compile(r"(?:^|[^$.+*/%^-])\s?\blocation\b\s*[=]\s*(?![\s\d=>])"),
|
|
164
173
|
add_suffix_non_prop(check_loc),
|
|
165
174
|
),
|
|
166
175
|
# rewriting `return this`
|
|
@@ -186,6 +195,7 @@ def create_js_rules() -> list[TransformationRule]:
|
|
|
186
195
|
# As the rule will match first, it will prevent next rule matching `import` to
|
|
187
196
|
# be apply to `async import`.
|
|
188
197
|
(re.compile(r"async\s+import\s*\("), m2str(lambda x: x)),
|
|
198
|
+
(re.compile(r"[^$.]\bimport\s*\([^)]*\)\s*\{"), m2str(lambda x: x)),
|
|
189
199
|
# esm dynamic import, if found, mark as module
|
|
190
200
|
(
|
|
191
201
|
re.compile(r"[^$.]\bimport\s*\("),
|
|
@@ -309,8 +319,8 @@ class JsRewriter(RxRewriter):
|
|
|
309
319
|
f"{match.group(3)}"
|
|
310
320
|
)
|
|
311
321
|
|
|
312
|
-
return
|
|
322
|
+
return IMPORT_EXPORT_HTTP_RX.sub(sub_funct, m_object[0])
|
|
313
323
|
|
|
314
324
|
return func
|
|
315
325
|
|
|
316
|
-
return (
|
|
326
|
+
return (IMPORT_EXPORT_MATCH_RX, rewrite_import())
|