zimscraperlib 5.4.0__tar.gz → 5.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/.gitignore +1 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/.pre-commit-config.yaml +2 -2
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/CHANGELOG.md +14 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/PKG-INFO +19 -19
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/docs/technical_architecture.md +1 -1
- zimscraperlib-5.4.1/javascript/.yarnrc.yml +1 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/javascript/package.json +7 -9
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/javascript/src/wombatSetup.js +5 -1
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/javascript/test/wombatUrlRewriting.js +28 -10
- zimscraperlib-5.4.1/javascript/yarn.lock +2697 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/openzim.toml +1 -1
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/pyproject.toml +11 -15
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/rules/rules.yaml +2 -2
- zimscraperlib-5.4.1/src/zimscraperlib/__about__.py +1 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/download.py +1 -1
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/i18n.py +1 -1
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/image/illustration.py +2 -1
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/logging.py +1 -1
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/rewriting/css.py +1 -1
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/rewriting/html.py +7 -5
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/rewriting/js.py +67 -44
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/rewriting/rx_replacer.py +28 -15
- zimscraperlib-5.4.1/src/zimscraperlib/rewriting/statics/wombat.js +21 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/rewriting/statics/wombatSetup.js +2 -2
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/rewriting/url_rewriting.py +5 -3
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/uri.py +1 -1
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/zim/items.py +2 -2
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/conftest.py +1 -1
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/html/test_html.py +18 -10
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/i18n/test_i18n.py +19 -12
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/image/test_image.py +12 -3
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/rewriting/test_css_rewriting.py +1 -1
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/rewriting/test_html_rewriting.py +7 -6
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/rewriting/test_js_rewriting.py +43 -1
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/rewriting/test_url_rewriting.py +37 -15
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/video/test_video.py +2 -2
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/zim/test_indexing.py +1 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/zim/test_libkiwix.py +8 -6
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/zim/test_zim_creator.py +59 -2
- zimscraperlib-5.4.0/javascript/yarn.lock +0 -1855
- zimscraperlib-5.4.0/src/zimscraperlib/__about__.py +0 -1
- zimscraperlib-5.4.0/src/zimscraperlib/rewriting/statics/wombat.js +0 -21
- zimscraperlib-5.4.0/yarn.lock +0 -248
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/.readthedocs.yaml +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/LICENSE +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/README.md +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/contrib/README.md +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/contrib/encode_video.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/docs/assets/openzim.png +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/docs/functional_architecture.md +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/docs/index.md +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/docs/license.md +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/docs/scripts/generate_api_nav.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/docs/software_architecture.md +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/javascript/.prettierignore +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/javascript/.prettierrc.json +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/javascript/eslint.config.js +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/javascript/rollup.config.js +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/javascript/test/wombatSetup.js +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/mkdocs.yml +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/rules/generate_rules.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/__init__.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/constants.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/filesystem.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/fix_ogvjs_dist.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/html.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/image/__init__.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/image/conversion.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/image/optimization.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/image/presets.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/image/probing.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/image/transformation.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/image/utils.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/inputs.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/misc.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/py.typed +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/rewriting/__init__.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/rewriting/rules.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/rewriting/statics/README.md +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/rewriting/statics/__wb_module_decl.js +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/rewriting/templates/head_insert.html +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/types.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/typing.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/video/__init__.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/video/config.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/video/encoding.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/video/presets.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/video/probing.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/zim/__init__.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/zim/_libkiwix.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/zim/archive.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/zim/creator.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/zim/dedup.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/zim/filesystem.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/zim/indexing.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/zim/metadata.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/src/zimscraperlib/zim/providers.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tasks.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/download/test_download.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/files/DroidSans.ttf +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/files/audio.mp3 +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/files/blue.jpg +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/files/commons.png +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/files/commons48.png +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/files/encrypted.pdf +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/files/encrypted.txt +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/files/mail.gif +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/files/milderm.pdf +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/files/milderm.txt +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/files/ninja.webp +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/files/pluto.jpg +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/files/single_wave_icon.gbr +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/files/square.jpg +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/files/square.png +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/files/star.svg +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/files/video.mkv +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/files/video.mp4 +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/files/video.webm +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/filesystem/test_filesystem.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/html/conftest.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/image/test_illustration.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/inputs/test_inputs.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/logging/conftest.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/logging/test_logging.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/misc/test_misc.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/ogvjs/conftest.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/ogvjs/test_ogvjs.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/rewriting/__init__.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/rewriting/conftest.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/rewriting/test_fuzzy_rules.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/rewriting/test_rx_replacer.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/rewriting/utils.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/test_constants.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/types/test_types.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/typing/test_typing.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/uri/test_uri.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/video/conftest.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/video/test_encoding.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/zim/conftest.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/zim/test_archive.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/zim/test_dedup.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/zim/test_fs.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.4.1}/tests/zim/test_metadata.py +0 -0
|
@@ -8,12 +8,12 @@ repos:
|
|
|
8
8
|
- id: trailing-whitespace
|
|
9
9
|
- id: end-of-file-fixer
|
|
10
10
|
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
11
|
-
rev: v0.
|
|
11
|
+
rev: v0.16.0
|
|
12
12
|
hooks:
|
|
13
13
|
- id: ruff-check
|
|
14
14
|
- id: ruff-format
|
|
15
15
|
- repo: https://github.com/RobertCraigie/pyright-python
|
|
16
|
-
rev: v1.1.
|
|
16
|
+
rev: v1.1.411
|
|
17
17
|
hooks:
|
|
18
18
|
- id: pyright
|
|
19
19
|
name: pyright (system)
|
|
@@ -5,6 +5,20 @@ All notable changes to this project are documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0).
|
|
7
7
|
|
|
8
|
+
## [5.4.1] - 2026-07-31
|
|
9
|
+
|
|
10
|
+
### Fixed
|
|
11
|
+
|
|
12
|
+
- Do not URL-encode `=` and `,` characters when rewriting document URIs, fixing YouTube JS player breakage (#316)
|
|
13
|
+
- Fix sources of memory leaks in JS and CSS rewriting (#322)
|
|
14
|
+
|
|
15
|
+
### Changed
|
|
16
|
+
|
|
17
|
+
- Upgrade Python and Javascript dependencies (especially wombat 3.10.6) (#319, #328)
|
|
18
|
+
- Backport wabac changes up to wabac.js 2.26.6 (#330)
|
|
19
|
+
- Migrate `javascript/` from Yarn Classic to Yarn Berry (#320)
|
|
20
|
+
- Test bogus Content-Length HTTP header behavior (#318)
|
|
21
|
+
|
|
8
22
|
## [5.4.0] - 2026-05-28
|
|
9
23
|
|
|
10
24
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: zimscraperlib
|
|
3
|
-
Version: 5.4.
|
|
3
|
+
Version: 5.4.1
|
|
4
4
|
Summary: Collection of python tools to re-use common code across scrapers
|
|
5
5
|
Project-URL: Donate, https://www.kiwix.org/en/support-us/
|
|
6
6
|
Project-URL: Homepage, https://github.com/openzim/python-scraperlib
|
|
@@ -23,7 +23,7 @@ Requires-Dist: idna<4.0,>=2.5
|
|
|
23
23
|
Requires-Dist: iso639-lang<3.0,>=2.4.0
|
|
24
24
|
Requires-Dist: libzim<4.0,>=3.8.0
|
|
25
25
|
Requires-Dist: lxml<7.0,>=4.6.3
|
|
26
|
-
Requires-Dist: optimize-images<
|
|
26
|
+
Requires-Dist: optimize-images<3.0,>=1.3.6
|
|
27
27
|
Requires-Dist: piexif==1.1.3
|
|
28
28
|
Requires-Dist: pillow<13.0,>=7.0.0
|
|
29
29
|
Requires-Dist: pymupdf<2.0,>=1.24.0
|
|
@@ -36,46 +36,46 @@ Requires-Dist: urllib3<2.8.0,>=1.26.5
|
|
|
36
36
|
Requires-Dist: xxhash<4.0,>=2.0
|
|
37
37
|
Requires-Dist: yt-dlp
|
|
38
38
|
Provides-Extra: dev
|
|
39
|
-
Requires-Dist: coverage==7.
|
|
39
|
+
Requires-Dist: coverage==7.15.2; extra == 'dev'
|
|
40
40
|
Requires-Dist: invoke==3.0.3; extra == 'dev'
|
|
41
|
-
Requires-Dist: ipython==9.
|
|
41
|
+
Requires-Dist: ipython==9.15.0; extra == 'dev'
|
|
42
42
|
Requires-Dist: jinja2==3.1.6; extra == 'dev'
|
|
43
43
|
Requires-Dist: mkdocs-gen-files==0.6.1; extra == 'dev'
|
|
44
44
|
Requires-Dist: mkdocs-include-markdown-plugin==7.3.0; extra == 'dev'
|
|
45
45
|
Requires-Dist: mkdocs-literate-nav==0.6.3; extra == 'dev'
|
|
46
|
-
Requires-Dist: mkdocs-material==9.7.
|
|
46
|
+
Requires-Dist: mkdocs-material==9.7.7; extra == 'dev'
|
|
47
47
|
Requires-Dist: mkdocs==1.6.1; extra == 'dev'
|
|
48
|
-
Requires-Dist: mkdocstrings[python]==1.0.
|
|
48
|
+
Requires-Dist: mkdocstrings[python]==1.0.6; extra == 'dev'
|
|
49
49
|
Requires-Dist: pre-commit==4.6.0; extra == 'dev'
|
|
50
50
|
Requires-Dist: pygments==2.19.2; extra == 'dev'
|
|
51
|
-
Requires-Dist: pymdown-extensions==
|
|
52
|
-
Requires-Dist: pyright==1.1.
|
|
51
|
+
Requires-Dist: pymdown-extensions==11.0.1; extra == 'dev'
|
|
52
|
+
Requires-Dist: pyright==1.1.411; extra == 'dev'
|
|
53
53
|
Requires-Dist: pytest-mock==3.15.1; extra == 'dev'
|
|
54
|
-
Requires-Dist: pytest==9.
|
|
54
|
+
Requires-Dist: pytest==9.1.1; extra == 'dev'
|
|
55
55
|
Requires-Dist: pyyaml==6.0.3; extra == 'dev'
|
|
56
|
-
Requires-Dist: ruff==0.
|
|
56
|
+
Requires-Dist: ruff==0.16.0; extra == 'dev'
|
|
57
57
|
Provides-Extra: docs
|
|
58
58
|
Requires-Dist: mkdocs-gen-files==0.6.1; extra == 'docs'
|
|
59
59
|
Requires-Dist: mkdocs-include-markdown-plugin==7.3.0; extra == 'docs'
|
|
60
60
|
Requires-Dist: mkdocs-literate-nav==0.6.3; extra == 'docs'
|
|
61
|
-
Requires-Dist: mkdocs-material==9.7.
|
|
61
|
+
Requires-Dist: mkdocs-material==9.7.7; extra == 'docs'
|
|
62
62
|
Requires-Dist: mkdocs==1.6.1; extra == 'docs'
|
|
63
|
-
Requires-Dist: mkdocstrings[python]==1.0.
|
|
63
|
+
Requires-Dist: mkdocstrings[python]==1.0.6; extra == 'docs'
|
|
64
64
|
Requires-Dist: pygments==2.19.2; extra == 'docs'
|
|
65
|
-
Requires-Dist: pymdown-extensions==
|
|
66
|
-
Requires-Dist: ruff==0.
|
|
65
|
+
Requires-Dist: pymdown-extensions==11.0.1; extra == 'docs'
|
|
66
|
+
Requires-Dist: ruff==0.16.0; extra == 'docs'
|
|
67
67
|
Provides-Extra: qa
|
|
68
|
-
Requires-Dist: pyright==1.1.
|
|
69
|
-
Requires-Dist: pytest==9.
|
|
70
|
-
Requires-Dist: ruff==0.
|
|
68
|
+
Requires-Dist: pyright==1.1.411; extra == 'qa'
|
|
69
|
+
Requires-Dist: pytest==9.1.1; extra == 'qa'
|
|
70
|
+
Requires-Dist: ruff==0.16.0; extra == 'qa'
|
|
71
71
|
Provides-Extra: scripts
|
|
72
72
|
Requires-Dist: invoke==3.0.3; extra == 'scripts'
|
|
73
73
|
Requires-Dist: jinja2==3.1.6; extra == 'scripts'
|
|
74
74
|
Requires-Dist: pyyaml==6.0.3; extra == 'scripts'
|
|
75
75
|
Provides-Extra: test
|
|
76
|
-
Requires-Dist: coverage==7.
|
|
76
|
+
Requires-Dist: coverage==7.15.2; extra == 'test'
|
|
77
77
|
Requires-Dist: pytest-mock==3.15.1; extra == 'test'
|
|
78
|
-
Requires-Dist: pytest==9.
|
|
78
|
+
Requires-Dist: pytest==9.1.1; extra == 'test'
|
|
79
79
|
Description-Content-Type: text/markdown
|
|
80
80
|
|
|
81
81
|
# zimscraperlib
|
|
@@ -44,7 +44,7 @@ Below is an example case of the rewrite operation on an image URL found in an HT
|
|
|
44
44
|
- Document original URL: `https://kiwix.org/a/article/document.html`
|
|
45
45
|
- Document ZIM path: `kiwix.org/a/article/document.html`
|
|
46
46
|
- Image original URL: `//xn--exmple-cva.com/a/resource/image.png?foo=bar`
|
|
47
|
-
- Image rewritten URL: `../../../ex%C3%A9mple.com/a/resource/image.png%3Ffoo
|
|
47
|
+
- Image rewritten URL: `../../../ex%C3%A9mple.com/a/resource/image.png%3Ffoo=bar`
|
|
48
48
|
- Image ZIM Path: `exémple.com/a/resource/image.png?foo=bar`
|
|
49
49
|
|
|
50
50
|
## JS Rewriting
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
nodeLinker: node-modules
|
|
@@ -1,20 +1,20 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@openzim/wombat-setup",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "5.
|
|
4
|
+
"version": "5.4.1",
|
|
5
5
|
"license": "GPL-3.0-or-later",
|
|
6
6
|
"author": "openZIM",
|
|
7
7
|
"devDependencies": {
|
|
8
|
-
"@rollup/plugin-commonjs": "29.0.
|
|
8
|
+
"@rollup/plugin-commonjs": "29.0.3",
|
|
9
9
|
"@rollup/plugin-node-resolve": "16.0.3",
|
|
10
10
|
"@rollup/plugin-strip": "3.0.4",
|
|
11
11
|
"@rollup/plugin-terser": "1.0.0",
|
|
12
12
|
"ava": "8.0.1",
|
|
13
13
|
"babel-eslint": "10.1.0",
|
|
14
|
-
"eslint": "10.
|
|
14
|
+
"eslint": "10.7.0",
|
|
15
15
|
"eslint-config-prettier": "10.1.8",
|
|
16
|
-
"prettier": "3.
|
|
17
|
-
"rollup": "4.
|
|
16
|
+
"prettier": "3.9.6",
|
|
17
|
+
"rollup": "4.62.2",
|
|
18
18
|
"rollup-plugin-version-injector": "1.3.3"
|
|
19
19
|
},
|
|
20
20
|
"scripts": {
|
|
@@ -35,9 +35,6 @@
|
|
|
35
35
|
"serial": true,
|
|
36
36
|
"files": [
|
|
37
37
|
"test/*.js"
|
|
38
|
-
],
|
|
39
|
-
"sources": [
|
|
40
|
-
"src/**/*"
|
|
41
38
|
]
|
|
42
39
|
},
|
|
43
40
|
"dependencies": {
|
|
@@ -45,5 +42,6 @@
|
|
|
45
42
|
},
|
|
46
43
|
"engines": {
|
|
47
44
|
"node": ">=26.0.0 <27.0.0"
|
|
48
|
-
}
|
|
45
|
+
},
|
|
46
|
+
"packageManager": "yarn@4.17.1"
|
|
49
47
|
}
|
|
@@ -194,7 +194,11 @@ export function urlRewriteFunction(
|
|
|
194
194
|
|
|
195
195
|
// Reencode everything but '/' (we decode it afterwards for simplicity)
|
|
196
196
|
const finalUrl =
|
|
197
|
-
prefix +
|
|
197
|
+
prefix +
|
|
198
|
+
encodeURIComponent(fuzzifiedPath)
|
|
199
|
+
.replaceAll('%2F', '/')
|
|
200
|
+
.replaceAll('%3D', '=')
|
|
201
|
+
.replaceAll('%2C', ',');
|
|
198
202
|
|
|
199
203
|
console.debug(
|
|
200
204
|
'urlRewriten:\n\t- current_url: ' +
|
|
@@ -335,7 +335,7 @@ test('contentWithEncodedQuestionMarkAndQueryParam', (t) => {
|
|
|
335
335
|
undefined,
|
|
336
336
|
undefined,
|
|
337
337
|
),
|
|
338
|
-
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/cont%3Fnt.txt%3Fquery
|
|
338
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/cont%3Fnt.txt%3Fquery=value',
|
|
339
339
|
);
|
|
340
340
|
});
|
|
341
341
|
|
|
@@ -454,7 +454,24 @@ test('contentWithSimpleQueryString', (t) => {
|
|
|
454
454
|
undefined,
|
|
455
455
|
undefined,
|
|
456
456
|
),
|
|
457
|
-
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery
|
|
457
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery=value',
|
|
458
|
+
);
|
|
459
|
+
});
|
|
460
|
+
|
|
461
|
+
test('contentWithSimplePathWithEqualAndComma', (t) => {
|
|
462
|
+
t.is(
|
|
463
|
+
urlRewriteFunction(
|
|
464
|
+
t.context.currentUrl,
|
|
465
|
+
t.context.originalHost,
|
|
466
|
+
t.context.originalScheme,
|
|
467
|
+
t.context.originalUrl,
|
|
468
|
+
t.context.prefix,
|
|
469
|
+
'https://www.example.com/javascript/content.txt/query1=value1,param2=value2',
|
|
470
|
+
undefined,
|
|
471
|
+
undefined,
|
|
472
|
+
undefined,
|
|
473
|
+
),
|
|
474
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt/query1=value1,param2=value2',
|
|
458
475
|
);
|
|
459
476
|
});
|
|
460
477
|
|
|
@@ -471,7 +488,7 @@ test('contentWithQueryValueEqualSign', (t) => {
|
|
|
471
488
|
undefined,
|
|
472
489
|
undefined,
|
|
473
490
|
),
|
|
474
|
-
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery
|
|
491
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery=val=eue',
|
|
475
492
|
);
|
|
476
493
|
});
|
|
477
494
|
|
|
@@ -488,7 +505,7 @@ test('contentWithQueryValuePercentSign', (t) => {
|
|
|
488
505
|
undefined,
|
|
489
506
|
undefined,
|
|
490
507
|
),
|
|
491
|
-
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery%
|
|
508
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery=val%25eue',
|
|
492
509
|
);
|
|
493
510
|
});
|
|
494
511
|
|
|
@@ -505,7 +522,7 @@ test('contentWithQueryParamPercentSign', (t) => {
|
|
|
505
522
|
undefined,
|
|
506
523
|
undefined,
|
|
507
524
|
),
|
|
508
|
-
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fque%25ry
|
|
525
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fque%25ry=valeue',
|
|
509
526
|
);
|
|
510
527
|
});
|
|
511
528
|
|
|
@@ -522,7 +539,7 @@ test('contentWithQueryParamPlusSign', (t) => {
|
|
|
522
539
|
undefined,
|
|
523
540
|
undefined,
|
|
524
541
|
),
|
|
525
|
-
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fparam%
|
|
542
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fparam=val%20ue',
|
|
526
543
|
);
|
|
527
544
|
});
|
|
528
545
|
|
|
@@ -811,7 +828,7 @@ test('youtubeFuzzyNotEncoded', (t) => {
|
|
|
811
828
|
undefined,
|
|
812
829
|
undefined,
|
|
813
830
|
),
|
|
814
|
-
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/path1/youtube.fuzzy.replayweb.page/get_video_info%3Fvideo_id
|
|
831
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/path1/youtube.fuzzy.replayweb.page/get_video_info%3Fvideo_id=123ah',
|
|
815
832
|
);
|
|
816
833
|
});
|
|
817
834
|
|
|
@@ -828,7 +845,7 @@ test('youtubeFuzzyEncoded', (t) => {
|
|
|
828
845
|
undefined,
|
|
829
846
|
undefined,
|
|
830
847
|
),
|
|
831
|
-
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/path1/youtube.fuzzy.replayweb.page/get_video_info%3Fvideo_id
|
|
848
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/path1/youtube.fuzzy.replayweb.page/get_video_info%3Fvideo_id=12=3ah',
|
|
832
849
|
);
|
|
833
850
|
});
|
|
834
851
|
|
|
@@ -931,7 +948,7 @@ test('relAlreadyEncoded', (t) => {
|
|
|
931
948
|
undefined,
|
|
932
949
|
undefined,
|
|
933
950
|
),
|
|
934
|
-
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery
|
|
951
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery=value',
|
|
935
952
|
);
|
|
936
953
|
});
|
|
937
954
|
|
|
@@ -1001,7 +1018,8 @@ test('relAnotherHostAlreadyRewrittenEmptyPath', (t) => {
|
|
|
1001
1018
|
});
|
|
1002
1019
|
|
|
1003
1020
|
// this is an edge case where the URL has already been statically rewritten and is located
|
|
1004
|
-
// on a different fuzzified domain name => we do not touch it at all
|
|
1021
|
+
// on a different fuzzified domain name => we do not touch it at all (for instance it keeps
|
|
1022
|
+
// %3D even if it should not have them)
|
|
1005
1023
|
test('relAnotherFuzzifiedHostAlreadyRewritten', (t) => {
|
|
1006
1024
|
t.is(
|
|
1007
1025
|
urlRewriteFunction(
|