zimscraperlib 5.4.0__tar.gz → 5.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/.gitignore +1 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/.pre-commit-config.yaml +2 -2
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/CHANGELOG.md +44 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/PKG-INFO +32 -30
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/docs/technical_architecture.md +1 -1
- zimscraperlib-5.5.0/javascript/.yarnrc.yml +1 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/javascript/package.json +9 -13
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/javascript/src/wombatSetup.js +26 -9
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/javascript/test/wombatSetup.js +23 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/javascript/test/wombatUrlRewriting.js +90 -13
- zimscraperlib-5.5.0/javascript/yarn.lock +2626 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/openzim.toml +1 -1
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/pyproject.toml +27 -24
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/rules/rules.yaml +23 -3
- zimscraperlib-5.5.0/src/zimscraperlib/__about__.py +1 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/constants.py +1 -0
- zimscraperlib-5.5.0/src/zimscraperlib/download.py +400 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/html.py +3 -2
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/i18n.py +1 -1
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/image/illustration.py +2 -1
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/logging.py +1 -1
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/css.py +1 -1
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/html.py +78 -28
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/js.py +161 -45
- zimscraperlib-5.5.0/src/zimscraperlib/rewriting/js_ast.py +132 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/rules.py +4 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/rx_replacer.py +30 -16
- zimscraperlib-5.5.0/src/zimscraperlib/rewriting/statics/wombat.js +21 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/statics/wombatSetup.js +3 -3
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/url_rewriting.py +151 -31
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/uri.py +1 -1
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/creator.py +7 -4
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/filesystem.py +3 -3
- zimscraperlib-5.5.0/src/zimscraperlib/zim/indexing.py +241 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/items.py +20 -5
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/metadata.py +7 -9
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/conftest.py +99 -1
- zimscraperlib-5.5.0/tests/download/test_download.py +776 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/html/test_html.py +26 -10
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/i18n/test_i18n.py +19 -12
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/image/test_image.py +12 -3
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/rewriting/test_css_rewriting.py +1 -1
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/rewriting/test_fuzzy_rules.py +62 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/rewriting/test_html_rewriting.py +156 -16
- zimscraperlib-5.5.0/tests/rewriting/test_js_globals.py +264 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/rewriting/test_js_rewriting.py +97 -15
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/rewriting/test_rx_replacer.py +8 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/rewriting/test_url_rewriting.py +307 -18
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/video/test_video.py +2 -2
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/zim/test_indexing.py +93 -3
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/zim/test_libkiwix.py +8 -6
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/zim/test_metadata.py +6 -10
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/zim/test_zim_creator.py +80 -5
- zimscraperlib-5.4.0/javascript/yarn.lock +0 -1855
- zimscraperlib-5.4.0/src/zimscraperlib/__about__.py +0 -1
- zimscraperlib-5.4.0/src/zimscraperlib/download.py +0 -248
- zimscraperlib-5.4.0/src/zimscraperlib/rewriting/statics/wombat.js +0 -21
- zimscraperlib-5.4.0/src/zimscraperlib/zim/indexing.py +0 -130
- zimscraperlib-5.4.0/tests/download/test_download.py +0 -346
- zimscraperlib-5.4.0/yarn.lock +0 -248
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/.readthedocs.yaml +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/LICENSE +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/README.md +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/contrib/README.md +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/contrib/encode_video.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/docs/assets/openzim.png +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/docs/functional_architecture.md +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/docs/index.md +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/docs/license.md +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/docs/scripts/generate_api_nav.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/docs/software_architecture.md +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/javascript/.prettierignore +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/javascript/.prettierrc.json +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/javascript/eslint.config.js +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/javascript/rollup.config.js +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/mkdocs.yml +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/rules/generate_rules.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/__init__.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/filesystem.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/fix_ogvjs_dist.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/image/__init__.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/image/conversion.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/image/optimization.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/image/presets.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/image/probing.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/image/transformation.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/image/utils.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/inputs.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/misc.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/py.typed +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/__init__.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/statics/README.md +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/statics/__wb_module_decl.js +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/templates/head_insert.html +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/types.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/typing.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/video/__init__.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/video/config.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/video/encoding.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/video/presets.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/video/probing.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/__init__.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/_libkiwix.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/archive.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/dedup.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/providers.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tasks.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/DroidSans.ttf +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/audio.mp3 +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/blue.jpg +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/commons.png +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/commons48.png +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/encrypted.pdf +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/encrypted.txt +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/mail.gif +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/milderm.pdf +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/milderm.txt +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/ninja.webp +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/pluto.jpg +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/single_wave_icon.gbr +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/square.jpg +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/square.png +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/star.svg +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/video.mkv +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/video.mp4 +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/files/video.webm +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/filesystem/test_filesystem.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/html/conftest.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/image/test_illustration.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/inputs/test_inputs.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/logging/conftest.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/logging/test_logging.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/misc/test_misc.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/ogvjs/conftest.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/ogvjs/test_ogvjs.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/rewriting/__init__.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/rewriting/conftest.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/rewriting/utils.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/test_constants.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/types/test_types.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/typing/test_typing.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/uri/test_uri.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/video/conftest.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/video/test_encoding.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/zim/conftest.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/zim/test_archive.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/zim/test_dedup.py +0 -0
- {zimscraperlib-5.4.0 → zimscraperlib-5.5.0}/tests/zim/test_fs.py +0 -0
|
@@ -8,12 +8,12 @@ repos:
|
|
|
8
8
|
- id: trailing-whitespace
|
|
9
9
|
- id: end-of-file-fixer
|
|
10
10
|
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
11
|
-
rev: v0.
|
|
11
|
+
rev: v0.17.3
|
|
12
12
|
hooks:
|
|
13
13
|
- id: ruff-check
|
|
14
14
|
- id: ruff-format
|
|
15
15
|
- repo: https://github.com/RobertCraigie/pyright-python
|
|
16
|
-
rev: v1.1.
|
|
16
|
+
rev: v1.1.414
|
|
17
17
|
hooks:
|
|
18
18
|
- id: pyright
|
|
19
19
|
name: pyright (system)
|
|
@@ -5,6 +5,50 @@ All notable changes to this project are documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0).
|
|
7
7
|
|
|
8
|
+
## [5.5.0] - 2026-10-29
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- Index ePub documents for full-text search, with a new `get_epub_index_data` helper and automatic indexing of ePub items, like PDFs already are (#333)
|
|
13
|
+
- Send a Wikimedia-compliant default `User-Agent` in `stream_file` when downloading from `upload.wikimedia.org` (#214)
|
|
14
|
+
- `save_large_file` accepts arbitrary HTTP headers and sends the default Wikimedia-compliant `User-Agent` for `upload.wikimedia.org`, like `stream_file` (#360)
|
|
15
|
+
- Add fuzzy rule to rewrite URLs of lesbases.anct.gouv.fr (#334)
|
|
16
|
+
- Report progress from `save_large_file` through a new `on_progress` callback and periodic logging, and stream the download to a `.part` file that is only moved into place once complete (#348)
|
|
17
|
+
- New `get_pdf_title` and `get_epub_title` helpers to get a document's title from its own metadata without extracting its full text content, cheaper than `get_pdf_index_data`/`get_epub_index_data` when only the title is needed (openzim/warc2zim#462, #359)
|
|
18
|
+
|
|
19
|
+
### Changed
|
|
20
|
+
|
|
21
|
+
- Upgrade to wombat 3.11.0 (#338)
|
|
22
|
+
- Upgrade Python dependencies, allowing pymupdf 2, pillow 13, beartype 0.23, urllib3 2.8 and xxhash 4, and Node.JS dependencies (#366, #369)
|
|
23
|
+
|
|
24
|
+
### Fixed
|
|
25
|
+
|
|
26
|
+
- Rewrite `save_large_file` to stream and resume the download itself instead of running `wget` once, whose `-t` counter is a budget for the whole download: a flaky connection now keeps resuming and the retry budget only counts attempts that make no progress, so long downloads are no longer abandoned mid-way (#348)
|
|
27
|
+
- Carry a script's top-level `const`, `let`, `class` and `function` declarations across the wombat block, so other scripts on the page can still see them, re-declaring classes with `let` so they can be reassigned (#329, #370, webrecorder/wabac.js#380)
|
|
28
|
+
- Keep a rewritten `<script>`'s original `src` in `__wb_orig_src`, so wombat can return it from `getAttribute("src")` and bundlers that identify chunks by it (Next.js/Turbopack, Vite) still work (openzim/warc2zim#473, #337)
|
|
29
|
+
- Keep the original of every rewritten `src`, `href` and `xlink:href` in `__wb_orig_<attr>`, not just a script's `src`, so wombat can return it instead of a broken URL (openzim/warc2zim#413, #338)
|
|
30
|
+
- Keep URL-encoded `#` characters of the path or querystring when rewriting document URIs, instead of dropping everything after them (#341)
|
|
31
|
+
- Find the language of an HTML page in a `content-language` `<meta>` that is not the first `<meta>` of the page (#356)
|
|
32
|
+
- Rewrite the `poster` attribute (e.g. of `<video>`) when rewriting HTML, like `src` and `href` (#339)
|
|
33
|
+
- Do not collapse consecutive slashes `//` into a single `/` when normalizing URLs into ZIM paths, in both Python and JS rewriting (#340)
|
|
34
|
+
- Resolve relative links against a page URL containing consecutive slashes `//` the way browsers do, instead of collapsing them like `urllib.parse.urljoin` does (#346)
|
|
35
|
+
- `Creator.finish()` no longer silently discards errors raised by libzim while finalizing the ZIM, so a failure during finalization is no longer reported as a successful run (openzim/warc2zim#462, #354)
|
|
36
|
+
- Escape `.` and `..` path segments (e.g. folded into the path from a querystring) with a leading `_` when normalizing URLs into ZIM paths, so they are kept as literal entry names instead of being resolved away as navigation, in both Python and JS rewriting (#321)
|
|
37
|
+
|
|
38
|
+
## [5.4.1] - 2026-07-31
|
|
39
|
+
|
|
40
|
+
### Fixed
|
|
41
|
+
|
|
42
|
+
- Do not URL-encode `=` and `,` characters when rewriting document URIs, fixing YouTube JS player breakage (#316)
|
|
43
|
+
- Fix sources of memory leaks in JS and CSS rewriting (#322)
|
|
44
|
+
|
|
45
|
+
### Changed
|
|
46
|
+
|
|
47
|
+
- Upgrade Python and Javascript dependencies (especially wombat 3.10.6) (#319, #328)
|
|
48
|
+
- Backport wabac changes up to wabac.js 2.26.6 (#330)
|
|
49
|
+
- Migrate `javascript/` from Yarn Classic to Yarn Berry (#320)
|
|
50
|
+
- Test bogus Content-Length HTTP header behavior (#318)
|
|
51
|
+
|
|
8
52
|
## [5.4.0] - 2026-05-28
|
|
9
53
|
|
|
10
54
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: zimscraperlib
|
|
3
|
-
Version: 5.
|
|
3
|
+
Version: 5.5.0
|
|
4
4
|
Summary: Collection of python tools to re-use common code across scrapers
|
|
5
5
|
Project-URL: Donate, https://www.kiwix.org/en/support-us/
|
|
6
6
|
Project-URL: Homepage, https://github.com/openzim/python-scraperlib
|
|
@@ -15,7 +15,7 @@ Classifier: Programming Language :: Python :: 3
|
|
|
15
15
|
Classifier: Programming Language :: Python :: 3.14
|
|
16
16
|
Requires-Python: <3.15,>=3.14
|
|
17
17
|
Requires-Dist: babel<3.0,>=2.9
|
|
18
|
-
Requires-Dist: beartype<0.
|
|
18
|
+
Requires-Dist: beartype<0.24,>=0.19
|
|
19
19
|
Requires-Dist: beautifulsoup4<5.0,>=4.9.3
|
|
20
20
|
Requires-Dist: cairosvg<3.0,>=2.2.0
|
|
21
21
|
Requires-Dist: colorthief==0.2.1
|
|
@@ -23,59 +23,61 @@ Requires-Dist: idna<4.0,>=2.5
|
|
|
23
23
|
Requires-Dist: iso639-lang<3.0,>=2.4.0
|
|
24
24
|
Requires-Dist: libzim<4.0,>=3.8.0
|
|
25
25
|
Requires-Dist: lxml<7.0,>=4.6.3
|
|
26
|
-
Requires-Dist: optimize-images<
|
|
26
|
+
Requires-Dist: optimize-images<3.0,>=1.3.6
|
|
27
27
|
Requires-Dist: piexif==1.1.3
|
|
28
|
-
Requires-Dist: pillow<
|
|
29
|
-
Requires-Dist: pymupdf<
|
|
28
|
+
Requires-Dist: pillow<14.0,>=7.0.0
|
|
29
|
+
Requires-Dist: pymupdf<3.0,>=1.24.0
|
|
30
30
|
Requires-Dist: python-magic<0.5,>=0.4.3
|
|
31
31
|
Requires-Dist: python-resize-image<1.2,>=1.1.19
|
|
32
32
|
Requires-Dist: regex>=2020.7.14
|
|
33
33
|
Requires-Dist: requests<3.0,>=2.25.1
|
|
34
|
+
Requires-Dist: tree-sitter-javascript<1.0,>=0.23
|
|
35
|
+
Requires-Dist: tree-sitter<1.0,>=0.23
|
|
34
36
|
Requires-Dist: types-xxhash<4.0,>=2.0
|
|
35
|
-
Requires-Dist: urllib3<2.
|
|
36
|
-
Requires-Dist: xxhash<
|
|
37
|
+
Requires-Dist: urllib3<2.9.0,>=1.26.5
|
|
38
|
+
Requires-Dist: xxhash<5.0,>=2.0
|
|
37
39
|
Requires-Dist: yt-dlp
|
|
38
40
|
Provides-Extra: dev
|
|
39
|
-
Requires-Dist: coverage==7.
|
|
41
|
+
Requires-Dist: coverage==7.17.0; extra == 'dev'
|
|
40
42
|
Requires-Dist: invoke==3.0.3; extra == 'dev'
|
|
41
|
-
Requires-Dist: ipython==9.
|
|
43
|
+
Requires-Dist: ipython==9.17.1; extra == 'dev'
|
|
42
44
|
Requires-Dist: jinja2==3.1.6; extra == 'dev'
|
|
43
45
|
Requires-Dist: mkdocs-gen-files==0.6.1; extra == 'dev'
|
|
44
46
|
Requires-Dist: mkdocs-include-markdown-plugin==7.3.0; extra == 'dev'
|
|
45
47
|
Requires-Dist: mkdocs-literate-nav==0.6.3; extra == 'dev'
|
|
46
|
-
Requires-Dist: mkdocs-material==9.
|
|
48
|
+
Requires-Dist: mkdocs-material==9.8.0; extra == 'dev'
|
|
47
49
|
Requires-Dist: mkdocs==1.6.1; extra == 'dev'
|
|
48
|
-
Requires-Dist: mkdocstrings[python]==1.0.
|
|
49
|
-
Requires-Dist: pre-commit==4.
|
|
50
|
-
Requires-Dist: pygments==2.
|
|
51
|
-
Requires-Dist: pymdown-extensions==
|
|
52
|
-
Requires-Dist: pyright==1.1.
|
|
53
|
-
Requires-Dist: pytest-mock==3.
|
|
54
|
-
Requires-Dist: pytest==9.0
|
|
50
|
+
Requires-Dist: mkdocstrings[python]==1.0.8; extra == 'dev'
|
|
51
|
+
Requires-Dist: pre-commit==4.7.0; extra == 'dev'
|
|
52
|
+
Requires-Dist: pygments==2.21.0; extra == 'dev'
|
|
53
|
+
Requires-Dist: pymdown-extensions==12.3; extra == 'dev'
|
|
54
|
+
Requires-Dist: pyright==1.1.414; extra == 'dev'
|
|
55
|
+
Requires-Dist: pytest-mock==3.17.0; extra == 'dev'
|
|
56
|
+
Requires-Dist: pytest==9.2.0; extra == 'dev'
|
|
55
57
|
Requires-Dist: pyyaml==6.0.3; extra == 'dev'
|
|
56
|
-
Requires-Dist: ruff==0.
|
|
58
|
+
Requires-Dist: ruff==0.17.3; extra == 'dev'
|
|
57
59
|
Provides-Extra: docs
|
|
58
60
|
Requires-Dist: mkdocs-gen-files==0.6.1; extra == 'docs'
|
|
59
61
|
Requires-Dist: mkdocs-include-markdown-plugin==7.3.0; extra == 'docs'
|
|
60
62
|
Requires-Dist: mkdocs-literate-nav==0.6.3; extra == 'docs'
|
|
61
|
-
Requires-Dist: mkdocs-material==9.
|
|
63
|
+
Requires-Dist: mkdocs-material==9.8.0; extra == 'docs'
|
|
62
64
|
Requires-Dist: mkdocs==1.6.1; extra == 'docs'
|
|
63
|
-
Requires-Dist: mkdocstrings[python]==1.0.
|
|
64
|
-
Requires-Dist: pygments==2.
|
|
65
|
-
Requires-Dist: pymdown-extensions==
|
|
66
|
-
Requires-Dist: ruff==0.
|
|
65
|
+
Requires-Dist: mkdocstrings[python]==1.0.8; extra == 'docs'
|
|
66
|
+
Requires-Dist: pygments==2.21.0; extra == 'docs'
|
|
67
|
+
Requires-Dist: pymdown-extensions==12.3; extra == 'docs'
|
|
68
|
+
Requires-Dist: ruff==0.17.3; extra == 'docs'
|
|
67
69
|
Provides-Extra: qa
|
|
68
|
-
Requires-Dist: pyright==1.1.
|
|
69
|
-
Requires-Dist: pytest==9.0
|
|
70
|
-
Requires-Dist: ruff==0.
|
|
70
|
+
Requires-Dist: pyright==1.1.414; extra == 'qa'
|
|
71
|
+
Requires-Dist: pytest==9.2.0; extra == 'qa'
|
|
72
|
+
Requires-Dist: ruff==0.17.3; extra == 'qa'
|
|
71
73
|
Provides-Extra: scripts
|
|
72
74
|
Requires-Dist: invoke==3.0.3; extra == 'scripts'
|
|
73
75
|
Requires-Dist: jinja2==3.1.6; extra == 'scripts'
|
|
74
76
|
Requires-Dist: pyyaml==6.0.3; extra == 'scripts'
|
|
75
77
|
Provides-Extra: test
|
|
76
|
-
Requires-Dist: coverage==7.
|
|
77
|
-
Requires-Dist: pytest-mock==3.
|
|
78
|
-
Requires-Dist: pytest==9.0
|
|
78
|
+
Requires-Dist: coverage==7.17.0; extra == 'test'
|
|
79
|
+
Requires-Dist: pytest-mock==3.17.0; extra == 'test'
|
|
80
|
+
Requires-Dist: pytest==9.2.0; extra == 'test'
|
|
79
81
|
Description-Content-Type: text/markdown
|
|
80
82
|
|
|
81
83
|
# zimscraperlib
|
|
@@ -44,7 +44,7 @@ Below is an example case of the rewrite operation on an image URL found in an HT
|
|
|
44
44
|
- Document original URL: `https://kiwix.org/a/article/document.html`
|
|
45
45
|
- Document ZIM path: `kiwix.org/a/article/document.html`
|
|
46
46
|
- Image original URL: `//xn--exmple-cva.com/a/resource/image.png?foo=bar`
|
|
47
|
-
- Image rewritten URL: `../../../ex%C3%A9mple.com/a/resource/image.png%3Ffoo
|
|
47
|
+
- Image rewritten URL: `../../../ex%C3%A9mple.com/a/resource/image.png%3Ffoo=bar`
|
|
48
48
|
- Image ZIM Path: `exémple.com/a/resource/image.png?foo=bar`
|
|
49
49
|
|
|
50
50
|
## JS Rewriting
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
nodeLinker: node-modules
|
|
@@ -1,20 +1,18 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@openzim/wombat-setup",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "5.
|
|
4
|
+
"version": "5.5.0",
|
|
5
5
|
"license": "GPL-3.0-or-later",
|
|
6
6
|
"author": "openZIM",
|
|
7
7
|
"devDependencies": {
|
|
8
|
-
"@rollup/plugin-commonjs": "29.0.
|
|
8
|
+
"@rollup/plugin-commonjs": "29.0.3",
|
|
9
9
|
"@rollup/plugin-node-resolve": "16.0.3",
|
|
10
10
|
"@rollup/plugin-strip": "3.0.4",
|
|
11
|
-
"@rollup/plugin-terser": "1.0.
|
|
12
|
-
"ava": "8.0
|
|
13
|
-
"
|
|
14
|
-
"
|
|
15
|
-
"
|
|
16
|
-
"prettier": "3.8.3",
|
|
17
|
-
"rollup": "4.60.4",
|
|
11
|
+
"@rollup/plugin-terser": "1.0.2",
|
|
12
|
+
"ava": "8.2.0",
|
|
13
|
+
"eslint": "10.13.0",
|
|
14
|
+
"prettier": "3.9.10",
|
|
15
|
+
"rollup": "4.65.1",
|
|
18
16
|
"rollup-plugin-version-injector": "1.3.3"
|
|
19
17
|
},
|
|
20
18
|
"scripts": {
|
|
@@ -35,9 +33,6 @@
|
|
|
35
33
|
"serial": true,
|
|
36
34
|
"files": [
|
|
37
35
|
"test/*.js"
|
|
38
|
-
],
|
|
39
|
-
"sources": [
|
|
40
|
-
"src/**/*"
|
|
41
36
|
]
|
|
42
37
|
},
|
|
43
38
|
"dependencies": {
|
|
@@ -45,5 +40,6 @@
|
|
|
45
40
|
},
|
|
46
41
|
"engines": {
|
|
47
42
|
"node": ">=26.0.0 <27.0.0"
|
|
48
|
-
}
|
|
43
|
+
},
|
|
44
|
+
"packageManager": "yarn@4.18.1"
|
|
49
45
|
}
|
|
@@ -18,6 +18,20 @@ export function applyFuzzyRules(path) {
|
|
|
18
18
|
return path;
|
|
19
19
|
}
|
|
20
20
|
|
|
21
|
+
export function escapeDotSegments(path) {
|
|
22
|
+
// Escape '.' and '..' path segments so they are not resolved as navigation.
|
|
23
|
+
// Per RFC 3986, a '.' or '..' path segment is a dot-segment which any URI consumer
|
|
24
|
+
// removes/resolves while normalizing a reference. A ZIM path must keep such a
|
|
25
|
+
// segment as a literal entry name instead (e.g. one folded in from a querystring),
|
|
26
|
+
// so it is prefixed with a '_', which is not itself special in a ZIM path.
|
|
27
|
+
// Segments which only look like a dot-segment (e.g. 'foo..', '..foo', '...') are
|
|
28
|
+
// not dot-segments and are left untouched.
|
|
29
|
+
return path
|
|
30
|
+
.split('/')
|
|
31
|
+
.map((part) => (part === '.' || part === '..' ? '_' + part : part))
|
|
32
|
+
.join('/');
|
|
33
|
+
}
|
|
34
|
+
|
|
21
35
|
export function hasAlreadyBeenRewritten(
|
|
22
36
|
original_absolute_url,
|
|
23
37
|
orig_url,
|
|
@@ -63,12 +77,6 @@ export function hasAlreadyBeenRewritten(
|
|
|
63
77
|
return false;
|
|
64
78
|
}
|
|
65
79
|
|
|
66
|
-
function removeSubsequentSlashes(value) {
|
|
67
|
-
// Remove all successive occurrences of a slash `/` in a given string
|
|
68
|
-
// E.g `val//ue` or `val///ue` or `val////ue` (and so on) are transformed into `value`
|
|
69
|
-
return value.replace(/\/\/+/g, '/');
|
|
70
|
-
}
|
|
71
|
-
|
|
72
80
|
export function urlRewriteFunction(
|
|
73
81
|
current_url, // The current (real) url we are on, e.g. http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/index.html
|
|
74
82
|
orig_host, // The host of the original url, e.g. www.example.com
|
|
@@ -186,15 +194,23 @@ export function urlRewriteFunction(
|
|
|
186
194
|
: '?' + decodeURIComponent(absolute_url_parts.query).replaceAll('+', ' ');
|
|
187
195
|
|
|
188
196
|
// combine all decoded parts to get the ZIM path
|
|
189
|
-
|
|
190
|
-
|
|
197
|
+
// (consecutive slashes are kept, just like in Python)
|
|
198
|
+
const zimPath = decoded_host + decoded_path + decoded_query;
|
|
191
199
|
|
|
192
200
|
// apply the fuzzy rules to the ZIM path
|
|
193
201
|
const fuzzifiedPath = applyFuzzyRules(zimPath);
|
|
194
202
|
|
|
203
|
+
// escape '.' and '..' path segments so they are not resolved as navigation by the
|
|
204
|
+
// reader once this ZIM path is turned into a URI
|
|
205
|
+
const escapedPath = escapeDotSegments(fuzzifiedPath);
|
|
206
|
+
|
|
195
207
|
// Reencode everything but '/' (we decode it afterwards for simplicity)
|
|
196
208
|
const finalUrl =
|
|
197
|
-
prefix +
|
|
209
|
+
prefix +
|
|
210
|
+
encodeURIComponent(escapedPath)
|
|
211
|
+
.replaceAll('%2F', '/')
|
|
212
|
+
.replaceAll('%3D', '=')
|
|
213
|
+
.replaceAll('%2C', ',');
|
|
198
214
|
|
|
199
215
|
console.debug(
|
|
200
216
|
'urlRewriten:\n\t- current_url: ' +
|
|
@@ -312,6 +328,7 @@ export function getWombatInfo(
|
|
|
312
328
|
|
|
313
329
|
export default {
|
|
314
330
|
applyFuzzyRules: applyFuzzyRules,
|
|
331
|
+
escapeDotSegments: escapeDotSegments,
|
|
315
332
|
urlRewriteFunction: urlRewriteFunction,
|
|
316
333
|
getWombatInfo: getWombatInfo,
|
|
317
334
|
};
|
|
@@ -8,6 +8,29 @@ test.beforeEach((t) => {
|
|
|
8
8
|
t.context.originalScheme = 'https';
|
|
9
9
|
});
|
|
10
10
|
|
|
11
|
+
test('escapeDotSegmentsDot', (t) => {
|
|
12
|
+
t.is(utils.escapeDotSegments('a/./b'), 'a/_./b');
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
test('escapeDotSegmentsDotDot', (t) => {
|
|
16
|
+
t.is(utils.escapeDotSegments('a/../b'), 'a/_../b');
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
test('escapeDotSegmentsTrailingDotDot', (t) => {
|
|
20
|
+
t.is(utils.escapeDotSegments('a/..'), 'a/_..');
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
test('escapeDotSegmentsConsecutive', (t) => {
|
|
24
|
+
t.is(utils.escapeDotSegments('a/../../b'), 'a/_../_../b');
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
test('escapeDotSegmentsLookAlikesUntouched', (t) => {
|
|
28
|
+
t.is(
|
|
29
|
+
utils.escapeDotSegments('a/.foo/foo../..foo/.../b'),
|
|
30
|
+
'a/.foo/foo../..foo/.../b',
|
|
31
|
+
);
|
|
32
|
+
});
|
|
33
|
+
|
|
11
34
|
test('nominalWbInfo', (t) => {
|
|
12
35
|
const path = 'path1/resource1.js';
|
|
13
36
|
const originalUrl =
|
|
@@ -84,6 +84,46 @@ test('simpleContentNoScheme', (t) => {
|
|
|
84
84
|
);
|
|
85
85
|
});
|
|
86
86
|
|
|
87
|
+
test('consecutiveSlashesAreKept', (t) => {
|
|
88
|
+
t.is(
|
|
89
|
+
urlRewriteFunction(
|
|
90
|
+
t.context.currentUrl,
|
|
91
|
+
t.context.originalHost,
|
|
92
|
+
t.context.originalScheme,
|
|
93
|
+
t.context.originalUrl,
|
|
94
|
+
t.context.prefix,
|
|
95
|
+
'https://www.example.com/a///b',
|
|
96
|
+
undefined,
|
|
97
|
+
undefined,
|
|
98
|
+
undefined,
|
|
99
|
+
),
|
|
100
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/a///b',
|
|
101
|
+
);
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
test('relativeLinkOnPageWithConsecutiveSlashesKeepsThem', (t) => {
|
|
105
|
+
// #346: a relative link must be resolved against the page's own URL the way a
|
|
106
|
+
// browser would, keeping consecutive slashes in that URL instead of collapsing
|
|
107
|
+
// them (uri-js's URI.resolve, unlike Python's urllib.parse.urljoin, already does
|
|
108
|
+
// this correctly)
|
|
109
|
+
const originalUrl = 'https://www.example.com/path1//resource1.html';
|
|
110
|
+
const currentUrl = t.context.prefix + 'www.example.com/path1//resource1.html';
|
|
111
|
+
t.is(
|
|
112
|
+
urlRewriteFunction(
|
|
113
|
+
currentUrl,
|
|
114
|
+
t.context.originalHost,
|
|
115
|
+
t.context.originalScheme,
|
|
116
|
+
originalUrl,
|
|
117
|
+
t.context.prefix,
|
|
118
|
+
'other.html',
|
|
119
|
+
undefined,
|
|
120
|
+
undefined,
|
|
121
|
+
undefined,
|
|
122
|
+
),
|
|
123
|
+
t.context.prefix + 'www.example.com/path1//other.html',
|
|
124
|
+
);
|
|
125
|
+
});
|
|
126
|
+
|
|
87
127
|
test('simpleContentEmptyPath', (t) => {
|
|
88
128
|
t.is(
|
|
89
129
|
urlRewriteFunction(
|
|
@@ -335,7 +375,7 @@ test('contentWithEncodedQuestionMarkAndQueryParam', (t) => {
|
|
|
335
375
|
undefined,
|
|
336
376
|
undefined,
|
|
337
377
|
),
|
|
338
|
-
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/cont%3Fnt.txt%3Fquery
|
|
378
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/cont%3Fnt.txt%3Fquery=value',
|
|
339
379
|
);
|
|
340
380
|
});
|
|
341
381
|
|
|
@@ -454,7 +494,43 @@ test('contentWithSimpleQueryString', (t) => {
|
|
|
454
494
|
undefined,
|
|
455
495
|
undefined,
|
|
456
496
|
),
|
|
457
|
-
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery
|
|
497
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery=value',
|
|
498
|
+
);
|
|
499
|
+
});
|
|
500
|
+
|
|
501
|
+
test('contentWithQueryStringDotDotSegment', (t) => {
|
|
502
|
+
// a `..` dot-segment folded into the path from a querystring must be escaped so
|
|
503
|
+
// it is not resolved as navigation once this ZIM path is turned into a URI
|
|
504
|
+
t.is(
|
|
505
|
+
urlRewriteFunction(
|
|
506
|
+
t.context.currentUrl,
|
|
507
|
+
t.context.originalHost,
|
|
508
|
+
t.context.originalScheme,
|
|
509
|
+
t.context.originalUrl,
|
|
510
|
+
t.context.prefix,
|
|
511
|
+
'https://www.example.com/javascript/content.txt?css=../../prg',
|
|
512
|
+
undefined,
|
|
513
|
+
undefined,
|
|
514
|
+
undefined,
|
|
515
|
+
),
|
|
516
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fcss=../_../prg',
|
|
517
|
+
);
|
|
518
|
+
});
|
|
519
|
+
|
|
520
|
+
test('contentWithSimplePathWithEqualAndComma', (t) => {
|
|
521
|
+
t.is(
|
|
522
|
+
urlRewriteFunction(
|
|
523
|
+
t.context.currentUrl,
|
|
524
|
+
t.context.originalHost,
|
|
525
|
+
t.context.originalScheme,
|
|
526
|
+
t.context.originalUrl,
|
|
527
|
+
t.context.prefix,
|
|
528
|
+
'https://www.example.com/javascript/content.txt/query1=value1,param2=value2',
|
|
529
|
+
undefined,
|
|
530
|
+
undefined,
|
|
531
|
+
undefined,
|
|
532
|
+
),
|
|
533
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt/query1=value1,param2=value2',
|
|
458
534
|
);
|
|
459
535
|
});
|
|
460
536
|
|
|
@@ -471,7 +547,7 @@ test('contentWithQueryValueEqualSign', (t) => {
|
|
|
471
547
|
undefined,
|
|
472
548
|
undefined,
|
|
473
549
|
),
|
|
474
|
-
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery
|
|
550
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery=val=eue',
|
|
475
551
|
);
|
|
476
552
|
});
|
|
477
553
|
|
|
@@ -488,7 +564,7 @@ test('contentWithQueryValuePercentSign', (t) => {
|
|
|
488
564
|
undefined,
|
|
489
565
|
undefined,
|
|
490
566
|
),
|
|
491
|
-
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery%
|
|
567
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery=val%25eue',
|
|
492
568
|
);
|
|
493
569
|
});
|
|
494
570
|
|
|
@@ -505,7 +581,7 @@ test('contentWithQueryParamPercentSign', (t) => {
|
|
|
505
581
|
undefined,
|
|
506
582
|
undefined,
|
|
507
583
|
),
|
|
508
|
-
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fque%25ry
|
|
584
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fque%25ry=valeue',
|
|
509
585
|
);
|
|
510
586
|
});
|
|
511
587
|
|
|
@@ -522,7 +598,7 @@ test('contentWithQueryParamPlusSign', (t) => {
|
|
|
522
598
|
undefined,
|
|
523
599
|
undefined,
|
|
524
600
|
),
|
|
525
|
-
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fparam%
|
|
601
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fparam=val%20ue',
|
|
526
602
|
);
|
|
527
603
|
});
|
|
528
604
|
|
|
@@ -811,7 +887,7 @@ test('youtubeFuzzyNotEncoded', (t) => {
|
|
|
811
887
|
undefined,
|
|
812
888
|
undefined,
|
|
813
889
|
),
|
|
814
|
-
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/path1/youtube.fuzzy.replayweb.page/get_video_info%3Fvideo_id
|
|
890
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/path1/youtube.fuzzy.replayweb.page/get_video_info%3Fvideo_id=123ah',
|
|
815
891
|
);
|
|
816
892
|
});
|
|
817
893
|
|
|
@@ -828,7 +904,7 @@ test('youtubeFuzzyEncoded', (t) => {
|
|
|
828
904
|
undefined,
|
|
829
905
|
undefined,
|
|
830
906
|
),
|
|
831
|
-
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/path1/youtube.fuzzy.replayweb.page/get_video_info%3Fvideo_id
|
|
907
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/path1/youtube.fuzzy.replayweb.page/get_video_info%3Fvideo_id=12=3ah',
|
|
832
908
|
);
|
|
833
909
|
});
|
|
834
910
|
|
|
@@ -931,7 +1007,7 @@ test('relAlreadyEncoded', (t) => {
|
|
|
931
1007
|
undefined,
|
|
932
1008
|
undefined,
|
|
933
1009
|
),
|
|
934
|
-
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery
|
|
1010
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery=value',
|
|
935
1011
|
);
|
|
936
1012
|
});
|
|
937
1013
|
|
|
@@ -1001,7 +1077,8 @@ test('relAnotherHostAlreadyRewrittenEmptyPath', (t) => {
|
|
|
1001
1077
|
});
|
|
1002
1078
|
|
|
1003
1079
|
// this is an edge case where the URL has already been statically rewritten and is located
|
|
1004
|
-
// on a different fuzzified domain name => we do not touch it at all
|
|
1080
|
+
// on a different fuzzified domain name => we do not touch it at all (for instance it keeps
|
|
1081
|
+
// %3D even if it should not have them)
|
|
1005
1082
|
test('relAnotherFuzzifiedHostAlreadyRewritten', (t) => {
|
|
1006
1083
|
t.is(
|
|
1007
1084
|
urlRewriteFunction(
|
|
@@ -1141,7 +1218,7 @@ test('doubleSlash1', (t) => {
|
|
|
1141
1218
|
undefined,
|
|
1142
1219
|
undefined,
|
|
1143
1220
|
),
|
|
1144
|
-
'http://library.kiwix.org/content/myzim_yyyy-mm/example.com/some/path/http%3A
|
|
1221
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/example.com/some/path/http%3A//example.com//some/path',
|
|
1145
1222
|
);
|
|
1146
1223
|
});
|
|
1147
1224
|
|
|
@@ -1158,7 +1235,7 @@ test('doubleSlash2', (t) => {
|
|
|
1158
1235
|
undefined,
|
|
1159
1236
|
undefined,
|
|
1160
1237
|
),
|
|
1161
|
-
'http://library.kiwix.org/content/myzim_yyyy-mm/example.com/some/pa%3Fth/http%3A
|
|
1238
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/example.com/some/pa%3Fth/http%3A//example.com//some/path',
|
|
1162
1239
|
);
|
|
1163
1240
|
});
|
|
1164
1241
|
|
|
@@ -1175,6 +1252,6 @@ test('doubleSlash3', (t) => {
|
|
|
1175
1252
|
undefined,
|
|
1176
1253
|
undefined,
|
|
1177
1254
|
),
|
|
1178
|
-
'http://library.kiwix.org/content/myzim_yyyy-mm/example.com/so%3Fme/pa%3Fth/http%3A
|
|
1255
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/example.com/so%3Fme/pa%3Fth/http%3A//example.com//some/path',
|
|
1179
1256
|
);
|
|
1180
1257
|
});
|