zimscraperlib 5.3.0__tar.gz → 5.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/.gitignore +1 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/.pre-commit-config.yaml +4 -7
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/CHANGELOG.md +41 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/PKG-INFO +69 -50
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/README.md +37 -18
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/docs/technical_architecture.md +1 -1
- zimscraperlib-5.4.1/javascript/.yarnrc.yml +1 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/javascript/package.json +16 -18
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/javascript/src/wombatSetup.js +17 -6
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/javascript/test/wombatSetup.js +2 -2
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/javascript/test/wombatUrlRewriting.js +28 -10
- zimscraperlib-5.4.1/javascript/yarn.lock +2697 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/mkdocs.yml +1 -1
- zimscraperlib-5.4.1/openzim.toml +20 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/pyproject.toml +31 -43
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/rules/rules.yaml +2 -2
- zimscraperlib-5.4.1/src/zimscraperlib/__about__.py +1 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/download.py +25 -15
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/html.py +3 -4
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/i18n.py +11 -10
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/image/illustration.py +2 -1
- zimscraperlib-5.4.1/src/zimscraperlib/image/optimization.py +475 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/image/probing.py +9 -1
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/logging.py +1 -1
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/misc.py +1 -5
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/rewriting/css.py +10 -22
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/rewriting/html.py +7 -5
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/rewriting/js.py +98 -24
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/rewriting/rx_replacer.py +28 -15
- zimscraperlib-5.4.1/src/zimscraperlib/rewriting/statics/wombat.js +21 -0
- zimscraperlib-5.4.1/src/zimscraperlib/rewriting/statics/wombatSetup.js +3 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/rewriting/url_rewriting.py +5 -3
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/uri.py +1 -1
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/video/encoding.py +0 -1
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/zim/creator.py +29 -6
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/zim/indexing.py +1 -1
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/zim/items.py +2 -2
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/zim/metadata.py +0 -2
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tasks.py +35 -39
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/conftest.py +1 -1
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/download/test_download.py +22 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/html/test_html.py +18 -10
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/i18n/test_i18n.py +19 -12
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/image/test_image.py +255 -45
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/rewriting/conftest.py +12 -12
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/rewriting/test_css_rewriting.py +1 -1
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/rewriting/test_html_rewriting.py +8 -9
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/rewriting/test_js_rewriting.py +46 -12
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/rewriting/test_url_rewriting.py +39 -21
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/rewriting/utils.py +0 -1
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/video/test_video.py +2 -2
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/zim/test_dedup.py +1 -1
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/zim/test_fs.py +2 -2
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/zim/test_indexing.py +5 -2
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/zim/test_libkiwix.py +8 -6
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/zim/test_metadata.py +1 -3
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/zim/test_zim_creator.py +121 -4
- zimscraperlib-5.3.0/javascript/yarn.lock +0 -2197
- zimscraperlib-5.3.0/openzim.toml +0 -10
- zimscraperlib-5.3.0/src/zimscraperlib/__about__.py +0 -1
- zimscraperlib-5.3.0/src/zimscraperlib/image/optimization.py +0 -401
- zimscraperlib-5.3.0/src/zimscraperlib/rewriting/statics/wombat.js +0 -21
- zimscraperlib-5.3.0/src/zimscraperlib/rewriting/statics/wombatSetup.js +0 -3
- zimscraperlib-5.3.0/yarn.lock +0 -248
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/.readthedocs.yaml +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/LICENSE +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/contrib/README.md +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/contrib/encode_video.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/docs/assets/openzim.png +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/docs/functional_architecture.md +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/docs/index.md +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/docs/license.md +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/docs/scripts/generate_api_nav.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/docs/software_architecture.md +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/javascript/.prettierignore +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/javascript/.prettierrc.json +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/javascript/eslint.config.js +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/javascript/rollup.config.js +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/rules/generate_rules.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/__init__.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/constants.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/filesystem.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/fix_ogvjs_dist.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/image/__init__.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/image/conversion.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/image/presets.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/image/transformation.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/image/utils.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/inputs.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/py.typed +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/rewriting/__init__.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/rewriting/rules.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/rewriting/statics/README.md +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/rewriting/statics/__wb_module_decl.js +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/rewriting/templates/head_insert.html +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/types.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/typing.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/video/__init__.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/video/config.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/video/presets.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/video/probing.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/zim/__init__.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/zim/_libkiwix.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/zim/archive.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/zim/dedup.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/zim/filesystem.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/src/zimscraperlib/zim/providers.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/files/DroidSans.ttf +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/files/audio.mp3 +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/files/blue.jpg +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/files/commons.png +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/files/commons48.png +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/files/encrypted.pdf +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/files/encrypted.txt +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/files/mail.gif +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/files/milderm.pdf +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/files/milderm.txt +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/files/ninja.webp +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/files/pluto.jpg +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/files/single_wave_icon.gbr +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/files/square.jpg +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/files/square.png +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/files/star.svg +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/files/video.mkv +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/files/video.mp4 +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/files/video.webm +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/filesystem/test_filesystem.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/html/conftest.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/image/test_illustration.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/inputs/test_inputs.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/logging/conftest.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/logging/test_logging.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/misc/test_misc.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/ogvjs/conftest.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/ogvjs/test_ogvjs.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/rewriting/__init__.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/rewriting/test_fuzzy_rules.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/rewriting/test_rx_replacer.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/test_constants.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/types/test_types.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/typing/test_typing.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/uri/test_uri.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/video/conftest.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/video/test_encoding.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/zim/conftest.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.1}/tests/zim/test_archive.py +0 -0
|
@@ -7,16 +7,13 @@ repos:
|
|
|
7
7
|
hooks:
|
|
8
8
|
- id: trailing-whitespace
|
|
9
9
|
- id: end-of-file-fixer
|
|
10
|
-
- repo: https://github.com/psf/black
|
|
11
|
-
rev: '25.11.0'
|
|
12
|
-
hooks:
|
|
13
|
-
- id: black
|
|
14
10
|
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
15
|
-
rev: v0.
|
|
11
|
+
rev: v0.16.0
|
|
16
12
|
hooks:
|
|
17
|
-
- id: ruff
|
|
13
|
+
- id: ruff-check
|
|
14
|
+
- id: ruff-format
|
|
18
15
|
- repo: https://github.com/RobertCraigie/pyright-python
|
|
19
|
-
rev: v1.1.
|
|
16
|
+
rev: v1.1.411
|
|
20
17
|
hooks:
|
|
21
18
|
- id: pyright
|
|
22
19
|
name: pyright (system)
|
|
@@ -5,6 +5,47 @@ All notable changes to this project are documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0).
|
|
7
7
|
|
|
8
|
+
## [5.4.1] - 2026-07-31
|
|
9
|
+
|
|
10
|
+
### Fixed
|
|
11
|
+
|
|
12
|
+
- Do not URL-encode `=` and `,` characters when rewriting document URIs, fixing YouTube JS player breakage (#316)
|
|
13
|
+
- Fix sources of memory leaks in JS and CSS rewriting (#322)
|
|
14
|
+
|
|
15
|
+
### Changed
|
|
16
|
+
|
|
17
|
+
- Upgrade Python and Javascript dependencies (especially wombat 3.10.6) (#319, #328)
|
|
18
|
+
- Backport wabac changes up to wabac.js 2.26.6 (#330)
|
|
19
|
+
- Migrate `javascript/` from Yarn Classic to Yarn Berry (#320)
|
|
20
|
+
- Test bogus Content-Length HTTP header behavior (#318)
|
|
21
|
+
|
|
22
|
+
## [5.4.0] - 2026-05-28
|
|
23
|
+
|
|
24
|
+
### Added
|
|
25
|
+
|
|
26
|
+
- Add support for custom number of retries and user-agent in save_large_file (#278)
|
|
27
|
+
- Enhance save_large_file log level (#279)
|
|
28
|
+
- Extend image optimization to support in-memory streams(BytesIO/bytes) and dst_format param (#289)
|
|
29
|
+
- Automatically add \_ftindex:<yes/no> tag at creator start based on indexing configuration (#295)
|
|
30
|
+
- **BEHAVIOR CHANGE**: it is not possible anymore to add "Tags" metadata with Creator.add_metadata method after creator has started ; this is anyway not recommended anymore since 5.x (one should prefer Creator.config_metadata)
|
|
31
|
+
|
|
32
|
+
### Fixed
|
|
33
|
+
|
|
34
|
+
- Add proper typing @overload to `zimscraperlib.image.optimize_xxx` methods (#273)
|
|
35
|
+
- Backport wabac.JS change around rewriting: detect when JS is in 'strict' mode and avoid using 'arguments' (#286)
|
|
36
|
+
- Fix installation from source, which missed wombatSetup.js (#287)
|
|
37
|
+
- Fix outdated system dependencies in README: remove Pillow build-time deps (bundled in wheels), add missing `libcairo` across all platforms (#152)
|
|
38
|
+
- Improve contribution setup instructions: use `hatch shell`, clarify commands must be run from local clone root (#153)
|
|
39
|
+
- Bring coverage back to 100% (#293)
|
|
40
|
+
- Fix `Creator.config_indexing` docstring to reflect that only the full-text index is toggled; title indexing is always performed by libzim (#294)
|
|
41
|
+
- Fix `format_for` to not modify input BytesIO position (#296)
|
|
42
|
+
|
|
43
|
+
### Changed
|
|
44
|
+
|
|
45
|
+
- Upgrade to wombat 3.10.5 (#306)
|
|
46
|
+
- Adopt openzim/\_python-bootstrap 2.0.0 (#303)
|
|
47
|
+
- Upgrade Python dependencies, Node.JS runtime and Node.JS dependencies (#306)
|
|
48
|
+
|
|
8
49
|
## [5.3.0] - 2025-11-14
|
|
9
50
|
|
|
10
51
|
### Changed
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: zimscraperlib
|
|
3
|
-
Version: 5.
|
|
3
|
+
Version: 5.4.1
|
|
4
4
|
Summary: Collection of python tools to re-use common code across scrapers
|
|
5
5
|
Project-URL: Donate, https://www.kiwix.org/en/support-us/
|
|
6
6
|
Project-URL: Homepage, https://github.com/openzim/python-scraperlib
|
|
@@ -23,7 +23,7 @@ Requires-Dist: idna<4.0,>=2.5
|
|
|
23
23
|
Requires-Dist: iso639-lang<3.0,>=2.4.0
|
|
24
24
|
Requires-Dist: libzim<4.0,>=3.8.0
|
|
25
25
|
Requires-Dist: lxml<7.0,>=4.6.3
|
|
26
|
-
Requires-Dist: optimize-images<
|
|
26
|
+
Requires-Dist: optimize-images<3.0,>=1.3.6
|
|
27
27
|
Requires-Dist: piexif==1.1.3
|
|
28
28
|
Requires-Dist: pillow<13.0,>=7.0.0
|
|
29
29
|
Requires-Dist: pymupdf<2.0,>=1.24.0
|
|
@@ -32,55 +32,56 @@ Requires-Dist: python-resize-image<1.2,>=1.1.19
|
|
|
32
32
|
Requires-Dist: regex>=2020.7.14
|
|
33
33
|
Requires-Dist: requests<3.0,>=2.25.1
|
|
34
34
|
Requires-Dist: types-xxhash<4.0,>=2.0
|
|
35
|
-
Requires-Dist: urllib3<2.
|
|
35
|
+
Requires-Dist: urllib3<2.8.0,>=1.26.5
|
|
36
36
|
Requires-Dist: xxhash<4.0,>=2.0
|
|
37
37
|
Requires-Dist: yt-dlp
|
|
38
|
-
Provides-Extra: check
|
|
39
|
-
Requires-Dist: pyright==1.1.407; extra == 'check'
|
|
40
|
-
Requires-Dist: pytest==9.0.1; extra == 'check'
|
|
41
38
|
Provides-Extra: dev
|
|
42
|
-
Requires-Dist:
|
|
43
|
-
Requires-Dist:
|
|
44
|
-
Requires-Dist:
|
|
45
|
-
Requires-Dist: ipython==9.7.0; extra == 'dev'
|
|
39
|
+
Requires-Dist: coverage==7.15.2; extra == 'dev'
|
|
40
|
+
Requires-Dist: invoke==3.0.3; extra == 'dev'
|
|
41
|
+
Requires-Dist: ipython==9.15.0; extra == 'dev'
|
|
46
42
|
Requires-Dist: jinja2==3.1.6; extra == 'dev'
|
|
47
|
-
Requires-Dist: mkdocs-gen-files==0.
|
|
48
|
-
Requires-Dist: mkdocs-include-markdown-plugin==7.
|
|
49
|
-
Requires-Dist: mkdocs-literate-nav==0.6.
|
|
50
|
-
Requires-Dist: mkdocs-material==9.7.
|
|
43
|
+
Requires-Dist: mkdocs-gen-files==0.6.1; extra == 'dev'
|
|
44
|
+
Requires-Dist: mkdocs-include-markdown-plugin==7.3.0; extra == 'dev'
|
|
45
|
+
Requires-Dist: mkdocs-literate-nav==0.6.3; extra == 'dev'
|
|
46
|
+
Requires-Dist: mkdocs-material==9.7.7; extra == 'dev'
|
|
51
47
|
Requires-Dist: mkdocs==1.6.1; extra == 'dev'
|
|
52
|
-
Requires-Dist: mkdocstrings[python]==0.
|
|
53
|
-
Requires-Dist: pre-commit==4.
|
|
54
|
-
Requires-Dist:
|
|
55
|
-
Requires-Dist:
|
|
48
|
+
Requires-Dist: mkdocstrings[python]==1.0.6; extra == 'dev'
|
|
49
|
+
Requires-Dist: pre-commit==4.6.0; extra == 'dev'
|
|
50
|
+
Requires-Dist: pygments==2.19.2; extra == 'dev'
|
|
51
|
+
Requires-Dist: pymdown-extensions==11.0.1; extra == 'dev'
|
|
52
|
+
Requires-Dist: pyright==1.1.411; extra == 'dev'
|
|
56
53
|
Requires-Dist: pytest-mock==3.15.1; extra == 'dev'
|
|
57
|
-
Requires-Dist: pytest==9.
|
|
54
|
+
Requires-Dist: pytest==9.1.1; extra == 'dev'
|
|
58
55
|
Requires-Dist: pyyaml==6.0.3; extra == 'dev'
|
|
59
|
-
Requires-Dist: ruff==0.
|
|
56
|
+
Requires-Dist: ruff==0.16.0; extra == 'dev'
|
|
60
57
|
Provides-Extra: docs
|
|
61
|
-
Requires-Dist: mkdocs-gen-files==0.
|
|
62
|
-
Requires-Dist: mkdocs-include-markdown-plugin==7.
|
|
63
|
-
Requires-Dist: mkdocs-literate-nav==0.6.
|
|
64
|
-
Requires-Dist: mkdocs-material==9.7.
|
|
58
|
+
Requires-Dist: mkdocs-gen-files==0.6.1; extra == 'docs'
|
|
59
|
+
Requires-Dist: mkdocs-include-markdown-plugin==7.3.0; extra == 'docs'
|
|
60
|
+
Requires-Dist: mkdocs-literate-nav==0.6.3; extra == 'docs'
|
|
61
|
+
Requires-Dist: mkdocs-material==9.7.7; extra == 'docs'
|
|
65
62
|
Requires-Dist: mkdocs==1.6.1; extra == 'docs'
|
|
66
|
-
Requires-Dist: mkdocstrings[python]==0.
|
|
67
|
-
Requires-Dist:
|
|
68
|
-
|
|
69
|
-
Requires-Dist:
|
|
70
|
-
|
|
63
|
+
Requires-Dist: mkdocstrings[python]==1.0.6; extra == 'docs'
|
|
64
|
+
Requires-Dist: pygments==2.19.2; extra == 'docs'
|
|
65
|
+
Requires-Dist: pymdown-extensions==11.0.1; extra == 'docs'
|
|
66
|
+
Requires-Dist: ruff==0.16.0; extra == 'docs'
|
|
67
|
+
Provides-Extra: qa
|
|
68
|
+
Requires-Dist: pyright==1.1.411; extra == 'qa'
|
|
69
|
+
Requires-Dist: pytest==9.1.1; extra == 'qa'
|
|
70
|
+
Requires-Dist: ruff==0.16.0; extra == 'qa'
|
|
71
71
|
Provides-Extra: scripts
|
|
72
|
-
Requires-Dist: invoke==
|
|
72
|
+
Requires-Dist: invoke==3.0.3; extra == 'scripts'
|
|
73
73
|
Requires-Dist: jinja2==3.1.6; extra == 'scripts'
|
|
74
74
|
Requires-Dist: pyyaml==6.0.3; extra == 'scripts'
|
|
75
75
|
Provides-Extra: test
|
|
76
|
-
Requires-Dist: coverage==7.
|
|
76
|
+
Requires-Dist: coverage==7.15.2; extra == 'test'
|
|
77
77
|
Requires-Dist: pytest-mock==3.15.1; extra == 'test'
|
|
78
|
-
Requires-Dist: pytest==9.
|
|
78
|
+
Requires-Dist: pytest==9.1.1; extra == 'test'
|
|
79
79
|
Description-Content-Type: text/markdown
|
|
80
80
|
|
|
81
81
|
# zimscraperlib
|
|
82
82
|
|
|
83
|
-
[](https://github.com/openzim/python-scraperlib/actions/workflows/QA.yaml)
|
|
84
|
+
[](https://github.com/openzim/python-scraperlib/actions/workflows/Tests.yaml)
|
|
84
85
|
[](https://www.codefactor.io/repository/github/openzim/python-scraperlib)
|
|
85
86
|
[](https://www.gnu.org/licenses/gpl-3.0)
|
|
86
87
|
[](https://pypi.org/project/zimscraperlib/)
|
|
@@ -113,46 +114,64 @@ See documentation at [Read the Docs](https://python-scraperlib.readthedocs.io/)
|
|
|
113
114
|
|
|
114
115
|
# Dependencies
|
|
115
116
|
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
-
|
|
119
|
-
-
|
|
120
|
-
- FFmpeg
|
|
121
|
-
- gifsicle (>=1.92)
|
|
122
|
-
- libcairo
|
|
117
|
+
Most dependencies are installed automatically by pip (from PyPI by default). The following system packages may be required depending on which features you use:
|
|
118
|
+
|
|
119
|
+
- **libmagic** — required for file type detection (used in most scrapers)
|
|
120
|
+
- **wget** — required only for `zimscraperlib.download` functions
|
|
121
|
+
- **FFmpeg** — required only for video processing functions
|
|
122
|
+
- **gifsicle** (>=1.92) — required only for GIF optimization
|
|
123
|
+
- **libcairo** — required only for SVG-to-PNG conversion
|
|
124
|
+
- **libzim** — auto-installed via PyPI, not available on Windows
|
|
125
|
+
- **Pillow** — auto-installed via PyPI; pre-built wheels are used by default and no system image libraries are needed. Only if you need to build Pillow from source should you install additional system libraries — see [Pillow's build documentation](https://pillow.readthedocs.io/en/latest/installation/building-from-source.html) for details.
|
|
126
|
+
> **Note:** To run the full test suite, all system dependencies listed above must be installed.
|
|
123
127
|
|
|
124
128
|
## macOS
|
|
125
129
|
|
|
126
130
|
```sh
|
|
127
|
-
brew install libmagic wget
|
|
131
|
+
brew install libmagic wget ffmpeg gifsicle cairo
|
|
128
132
|
```
|
|
129
133
|
|
|
130
134
|
## Linux
|
|
131
135
|
|
|
132
136
|
```sh
|
|
133
|
-
sudo apt install libmagic1 wget ffmpeg
|
|
134
|
-
libtiff5-dev libjpeg8-dev libopenjp2-7-dev zlib1g-dev \
|
|
135
|
-
libfreetype6-dev liblcms2-dev libwebp-dev tcl8.6-dev tk8.6-dev python3-tk \
|
|
136
|
-
libharfbuzz-dev libfribidi-dev libxcb1-dev gifsicle
|
|
137
|
+
sudo apt install libmagic1 wget ffmpeg gifsicle libcairo2
|
|
137
138
|
```
|
|
138
139
|
|
|
139
140
|
## Alpine
|
|
140
141
|
|
|
141
|
-
```
|
|
142
|
-
apk add ffmpeg gifsicle libmagic wget
|
|
142
|
+
```sh
|
|
143
|
+
apk add ffmpeg gifsicle libmagic wget cairo
|
|
143
144
|
```
|
|
144
145
|
|
|
145
146
|
# Contribution
|
|
146
147
|
|
|
147
148
|
This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing).
|
|
148
149
|
|
|
149
|
-
This project has implemented openZIM's [Python bootstrap, conventions and policies](https://github.com/openzim/_python-bootstrap/docs/Policy.md) **
|
|
150
|
+
This project has implemented openZIM's [Python bootstrap, conventions and policies](https://github.com/openzim/_python-bootstrap/docs/Policy.md) **v2.0.0**.
|
|
151
|
+
|
|
152
|
+
All instructions below must be run from the root of your local clone of this repository.
|
|
153
|
+
|
|
154
|
+
If you do not already have it on your system, install [hatch](https://hatch.pypa.io/latest/install/):
|
|
150
155
|
|
|
151
156
|
```shell
|
|
152
157
|
pip install hatch
|
|
153
|
-
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
Start a hatch shell — this will install all dependencies including dev in an isolated virtual environment:
|
|
161
|
+
|
|
162
|
+
```shell
|
|
163
|
+
hatch shell
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
Set up the pre-commit Git hook (runs linters automatically before each commit):
|
|
167
|
+
|
|
168
|
+
```shell
|
|
154
169
|
pre-commit install
|
|
155
|
-
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
Run tests with coverage:
|
|
173
|
+
|
|
174
|
+
```shell
|
|
156
175
|
invoke coverage
|
|
157
176
|
```
|
|
158
177
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# zimscraperlib
|
|
2
2
|
|
|
3
|
-
[](https://github.com/openzim/python-scraperlib/actions/workflows/QA.yaml)
|
|
4
|
+
[](https://github.com/openzim/python-scraperlib/actions/workflows/Tests.yaml)
|
|
4
5
|
[](https://www.codefactor.io/repository/github/openzim/python-scraperlib)
|
|
5
6
|
[](https://www.gnu.org/licenses/gpl-3.0)
|
|
6
7
|
[](https://pypi.org/project/zimscraperlib/)
|
|
@@ -33,46 +34,64 @@ See documentation at [Read the Docs](https://python-scraperlib.readthedocs.io/)
|
|
|
33
34
|
|
|
34
35
|
# Dependencies
|
|
35
36
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
-
|
|
39
|
-
-
|
|
40
|
-
- FFmpeg
|
|
41
|
-
- gifsicle (>=1.92)
|
|
42
|
-
- libcairo
|
|
37
|
+
Most dependencies are installed automatically by pip (from PyPI by default). The following system packages may be required depending on which features you use:
|
|
38
|
+
|
|
39
|
+
- **libmagic** — required for file type detection (used in most scrapers)
|
|
40
|
+
- **wget** — required only for `zimscraperlib.download` functions
|
|
41
|
+
- **FFmpeg** — required only for video processing functions
|
|
42
|
+
- **gifsicle** (>=1.92) — required only for GIF optimization
|
|
43
|
+
- **libcairo** — required only for SVG-to-PNG conversion
|
|
44
|
+
- **libzim** — auto-installed via PyPI, not available on Windows
|
|
45
|
+
- **Pillow** — auto-installed via PyPI; pre-built wheels are used by default and no system image libraries are needed. Only if you need to build Pillow from source should you install additional system libraries — see [Pillow's build documentation](https://pillow.readthedocs.io/en/latest/installation/building-from-source.html) for details.
|
|
46
|
+
> **Note:** To run the full test suite, all system dependencies listed above must be installed.
|
|
43
47
|
|
|
44
48
|
## macOS
|
|
45
49
|
|
|
46
50
|
```sh
|
|
47
|
-
brew install libmagic wget
|
|
51
|
+
brew install libmagic wget ffmpeg gifsicle cairo
|
|
48
52
|
```
|
|
49
53
|
|
|
50
54
|
## Linux
|
|
51
55
|
|
|
52
56
|
```sh
|
|
53
|
-
sudo apt install libmagic1 wget ffmpeg
|
|
54
|
-
libtiff5-dev libjpeg8-dev libopenjp2-7-dev zlib1g-dev \
|
|
55
|
-
libfreetype6-dev liblcms2-dev libwebp-dev tcl8.6-dev tk8.6-dev python3-tk \
|
|
56
|
-
libharfbuzz-dev libfribidi-dev libxcb1-dev gifsicle
|
|
57
|
+
sudo apt install libmagic1 wget ffmpeg gifsicle libcairo2
|
|
57
58
|
```
|
|
58
59
|
|
|
59
60
|
## Alpine
|
|
60
61
|
|
|
61
|
-
```
|
|
62
|
-
apk add ffmpeg gifsicle libmagic wget
|
|
62
|
+
```sh
|
|
63
|
+
apk add ffmpeg gifsicle libmagic wget cairo
|
|
63
64
|
```
|
|
64
65
|
|
|
65
66
|
# Contribution
|
|
66
67
|
|
|
67
68
|
This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing).
|
|
68
69
|
|
|
69
|
-
This project has implemented openZIM's [Python bootstrap, conventions and policies](https://github.com/openzim/_python-bootstrap/docs/Policy.md) **
|
|
70
|
+
This project has implemented openZIM's [Python bootstrap, conventions and policies](https://github.com/openzim/_python-bootstrap/docs/Policy.md) **v2.0.0**.
|
|
71
|
+
|
|
72
|
+
All instructions below must be run from the root of your local clone of this repository.
|
|
73
|
+
|
|
74
|
+
If you do not already have it on your system, install [hatch](https://hatch.pypa.io/latest/install/):
|
|
70
75
|
|
|
71
76
|
```shell
|
|
72
77
|
pip install hatch
|
|
73
|
-
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Start a hatch shell — this will install all dependencies including dev in an isolated virtual environment:
|
|
81
|
+
|
|
82
|
+
```shell
|
|
83
|
+
hatch shell
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Set up the pre-commit Git hook (runs linters automatically before each commit):
|
|
87
|
+
|
|
88
|
+
```shell
|
|
74
89
|
pre-commit install
|
|
75
|
-
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
Run tests with coverage:
|
|
93
|
+
|
|
94
|
+
```shell
|
|
76
95
|
invoke coverage
|
|
77
96
|
```
|
|
78
97
|
|
|
@@ -44,7 +44,7 @@ Below is an example case of the rewrite operation on an image URL found in an HT
|
|
|
44
44
|
- Document original URL: `https://kiwix.org/a/article/document.html`
|
|
45
45
|
- Document ZIM path: `kiwix.org/a/article/document.html`
|
|
46
46
|
- Image original URL: `//xn--exmple-cva.com/a/resource/image.png?foo=bar`
|
|
47
|
-
- Image rewritten URL: `../../../ex%C3%A9mple.com/a/resource/image.png%3Ffoo
|
|
47
|
+
- Image rewritten URL: `../../../ex%C3%A9mple.com/a/resource/image.png%3Ffoo=bar`
|
|
48
48
|
- Image ZIM Path: `exémple.com/a/resource/image.png?foo=bar`
|
|
49
49
|
|
|
50
50
|
## JS Rewriting
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
nodeLinker: node-modules
|
|
@@ -1,21 +1,21 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@openzim/wombat-setup",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "5.
|
|
4
|
+
"version": "5.4.1",
|
|
5
5
|
"license": "GPL-3.0-or-later",
|
|
6
6
|
"author": "openZIM",
|
|
7
7
|
"devDependencies": {
|
|
8
|
-
"@rollup/plugin-commonjs": "
|
|
9
|
-
"@rollup/plugin-node-resolve": "16.0.
|
|
10
|
-
"@rollup/plugin-strip": "
|
|
11
|
-
"@rollup/plugin-terser": "0.
|
|
12
|
-
"ava": "
|
|
13
|
-
"babel-eslint": "
|
|
14
|
-
"eslint": "
|
|
15
|
-
"eslint-config-prettier": "10.
|
|
16
|
-
"prettier": "3.
|
|
17
|
-
"rollup": "4.
|
|
18
|
-
"rollup-plugin-version-injector": "
|
|
8
|
+
"@rollup/plugin-commonjs": "29.0.3",
|
|
9
|
+
"@rollup/plugin-node-resolve": "16.0.3",
|
|
10
|
+
"@rollup/plugin-strip": "3.0.4",
|
|
11
|
+
"@rollup/plugin-terser": "1.0.0",
|
|
12
|
+
"ava": "8.0.1",
|
|
13
|
+
"babel-eslint": "10.1.0",
|
|
14
|
+
"eslint": "10.7.0",
|
|
15
|
+
"eslint-config-prettier": "10.1.8",
|
|
16
|
+
"prettier": "3.9.6",
|
|
17
|
+
"rollup": "4.62.2",
|
|
18
|
+
"rollup-plugin-version-injector": "1.3.3"
|
|
19
19
|
},
|
|
20
20
|
"scripts": {
|
|
21
21
|
"prettier-check": "prettier . --check",
|
|
@@ -35,15 +35,13 @@
|
|
|
35
35
|
"serial": true,
|
|
36
36
|
"files": [
|
|
37
37
|
"test/*.js"
|
|
38
|
-
],
|
|
39
|
-
"sources": [
|
|
40
|
-
"src/**/*"
|
|
41
38
|
]
|
|
42
39
|
},
|
|
43
40
|
"dependencies": {
|
|
44
|
-
"uri-js": "
|
|
41
|
+
"uri-js": "4.4.1"
|
|
45
42
|
},
|
|
46
43
|
"engines": {
|
|
47
|
-
"node": ">=
|
|
48
|
-
}
|
|
44
|
+
"node": ">=26.0.0 <27.0.0"
|
|
45
|
+
},
|
|
46
|
+
"packageManager": "yarn@4.17.1"
|
|
49
47
|
}
|
|
@@ -194,7 +194,11 @@ export function urlRewriteFunction(
|
|
|
194
194
|
|
|
195
195
|
// Reencode everything but '/' (we decode it afterwards for simplicity)
|
|
196
196
|
const finalUrl =
|
|
197
|
-
prefix +
|
|
197
|
+
prefix +
|
|
198
|
+
encodeURIComponent(fuzzifiedPath)
|
|
199
|
+
.replaceAll('%2F', '/')
|
|
200
|
+
.replaceAll('%3D', '=')
|
|
201
|
+
.replaceAll('%2C', ',');
|
|
198
202
|
|
|
199
203
|
console.debug(
|
|
200
204
|
'urlRewriten:\n\t- current_url: ' +
|
|
@@ -292,14 +296,21 @@ export function getWombatInfo(
|
|
|
292
296
|
// The host of the original url
|
|
293
297
|
wombat_host: orig_host,
|
|
294
298
|
|
|
295
|
-
//
|
|
296
|
-
|
|
299
|
+
// We are not running inside a service worker, wombat needs to know about it since
|
|
300
|
+
// some "magic" URLs like blobs are not available
|
|
301
|
+
isSW: false,
|
|
297
302
|
|
|
298
|
-
//
|
|
299
|
-
enable_auto_fetch: true,
|
|
303
|
+
// Convert all post request to get request
|
|
300
304
|
convert_post_to_get: true,
|
|
305
|
+
|
|
306
|
+
// Not used, we are not replaying in a frame
|
|
301
307
|
target_frame: '___wb_replay_top_frame',
|
|
302
|
-
|
|
308
|
+
|
|
309
|
+
// Not used, we are not running in live mode
|
|
310
|
+
enable_auto_fetch: false,
|
|
311
|
+
|
|
312
|
+
// Extra options, not used
|
|
313
|
+
wombat_opts: {},
|
|
303
314
|
};
|
|
304
315
|
}
|
|
305
316
|
|
|
@@ -21,8 +21,8 @@ test('nominalWbInfo', (t) => {
|
|
|
21
21
|
);
|
|
22
22
|
t.is(wmInfo.coll, '');
|
|
23
23
|
t.is(wmInfo.convert_post_to_get, true);
|
|
24
|
-
t.is(wmInfo.enable_auto_fetch,
|
|
25
|
-
t.is(wmInfo.isSW,
|
|
24
|
+
t.is(wmInfo.enable_auto_fetch, false);
|
|
25
|
+
t.is(wmInfo.isSW, false);
|
|
26
26
|
t.is(wmInfo.is_framed, false);
|
|
27
27
|
t.is(wmInfo.is_live, false);
|
|
28
28
|
t.is(wmInfo.mod, '');
|
|
@@ -335,7 +335,7 @@ test('contentWithEncodedQuestionMarkAndQueryParam', (t) => {
|
|
|
335
335
|
undefined,
|
|
336
336
|
undefined,
|
|
337
337
|
),
|
|
338
|
-
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/cont%3Fnt.txt%3Fquery
|
|
338
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/cont%3Fnt.txt%3Fquery=value',
|
|
339
339
|
);
|
|
340
340
|
});
|
|
341
341
|
|
|
@@ -454,7 +454,24 @@ test('contentWithSimpleQueryString', (t) => {
|
|
|
454
454
|
undefined,
|
|
455
455
|
undefined,
|
|
456
456
|
),
|
|
457
|
-
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery
|
|
457
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery=value',
|
|
458
|
+
);
|
|
459
|
+
});
|
|
460
|
+
|
|
461
|
+
test('contentWithSimplePathWithEqualAndComma', (t) => {
|
|
462
|
+
t.is(
|
|
463
|
+
urlRewriteFunction(
|
|
464
|
+
t.context.currentUrl,
|
|
465
|
+
t.context.originalHost,
|
|
466
|
+
t.context.originalScheme,
|
|
467
|
+
t.context.originalUrl,
|
|
468
|
+
t.context.prefix,
|
|
469
|
+
'https://www.example.com/javascript/content.txt/query1=value1,param2=value2',
|
|
470
|
+
undefined,
|
|
471
|
+
undefined,
|
|
472
|
+
undefined,
|
|
473
|
+
),
|
|
474
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt/query1=value1,param2=value2',
|
|
458
475
|
);
|
|
459
476
|
});
|
|
460
477
|
|
|
@@ -471,7 +488,7 @@ test('contentWithQueryValueEqualSign', (t) => {
|
|
|
471
488
|
undefined,
|
|
472
489
|
undefined,
|
|
473
490
|
),
|
|
474
|
-
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery
|
|
491
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery=val=eue',
|
|
475
492
|
);
|
|
476
493
|
});
|
|
477
494
|
|
|
@@ -488,7 +505,7 @@ test('contentWithQueryValuePercentSign', (t) => {
|
|
|
488
505
|
undefined,
|
|
489
506
|
undefined,
|
|
490
507
|
),
|
|
491
|
-
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery%
|
|
508
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery=val%25eue',
|
|
492
509
|
);
|
|
493
510
|
});
|
|
494
511
|
|
|
@@ -505,7 +522,7 @@ test('contentWithQueryParamPercentSign', (t) => {
|
|
|
505
522
|
undefined,
|
|
506
523
|
undefined,
|
|
507
524
|
),
|
|
508
|
-
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fque%25ry
|
|
525
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fque%25ry=valeue',
|
|
509
526
|
);
|
|
510
527
|
});
|
|
511
528
|
|
|
@@ -522,7 +539,7 @@ test('contentWithQueryParamPlusSign', (t) => {
|
|
|
522
539
|
undefined,
|
|
523
540
|
undefined,
|
|
524
541
|
),
|
|
525
|
-
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fparam%
|
|
542
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fparam=val%20ue',
|
|
526
543
|
);
|
|
527
544
|
});
|
|
528
545
|
|
|
@@ -811,7 +828,7 @@ test('youtubeFuzzyNotEncoded', (t) => {
|
|
|
811
828
|
undefined,
|
|
812
829
|
undefined,
|
|
813
830
|
),
|
|
814
|
-
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/path1/youtube.fuzzy.replayweb.page/get_video_info%3Fvideo_id
|
|
831
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/path1/youtube.fuzzy.replayweb.page/get_video_info%3Fvideo_id=123ah',
|
|
815
832
|
);
|
|
816
833
|
});
|
|
817
834
|
|
|
@@ -828,7 +845,7 @@ test('youtubeFuzzyEncoded', (t) => {
|
|
|
828
845
|
undefined,
|
|
829
846
|
undefined,
|
|
830
847
|
),
|
|
831
|
-
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/path1/youtube.fuzzy.replayweb.page/get_video_info%3Fvideo_id
|
|
848
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/path1/youtube.fuzzy.replayweb.page/get_video_info%3Fvideo_id=12=3ah',
|
|
832
849
|
);
|
|
833
850
|
});
|
|
834
851
|
|
|
@@ -931,7 +948,7 @@ test('relAlreadyEncoded', (t) => {
|
|
|
931
948
|
undefined,
|
|
932
949
|
undefined,
|
|
933
950
|
),
|
|
934
|
-
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery
|
|
951
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery=value',
|
|
935
952
|
);
|
|
936
953
|
});
|
|
937
954
|
|
|
@@ -1001,7 +1018,8 @@ test('relAnotherHostAlreadyRewrittenEmptyPath', (t) => {
|
|
|
1001
1018
|
});
|
|
1002
1019
|
|
|
1003
1020
|
// this is an edge case where the URL has already been statically rewritten and is located
|
|
1004
|
-
// on a different fuzzified domain name => we do not touch it at all
|
|
1021
|
+
// on a different fuzzified domain name => we do not touch it at all (for instance it keeps
|
|
1022
|
+
// %3D even if it should not have them)
|
|
1005
1023
|
test('relAnotherFuzzifiedHostAlreadyRewritten', (t) => {
|
|
1006
1024
|
t.is(
|
|
1007
1025
|
urlRewriteFunction(
|