zimscraperlib 5.4.1__tar.gz → 5.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/.pre-commit-config.yaml +2 -2
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/CHANGELOG.md +30 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/PKG-INFO +31 -29
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/javascript/package.json +7 -9
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/javascript/src/wombatSetup.js +22 -9
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/javascript/test/wombatSetup.js +23 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/javascript/test/wombatUrlRewriting.js +62 -3
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/javascript/yarn.lock +406 -477
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/openzim.toml +1 -1
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/pyproject.toml +26 -19
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/rules/rules.yaml +23 -3
- zimscraperlib-5.5.0/src/zimscraperlib/__about__.py +1 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/constants.py +1 -0
- zimscraperlib-5.5.0/src/zimscraperlib/download.py +400 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/html.py +3 -2
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/html.py +71 -23
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/js.py +98 -5
- zimscraperlib-5.5.0/src/zimscraperlib/rewriting/js_ast.py +132 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/rules.py +4 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/rx_replacer.py +2 -1
- zimscraperlib-5.5.0/src/zimscraperlib/rewriting/statics/wombat.js +21 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/statics/wombatSetup.js +3 -3
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/url_rewriting.py +146 -28
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/creator.py +7 -4
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/filesystem.py +3 -3
- zimscraperlib-5.5.0/src/zimscraperlib/zim/indexing.py +241 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/items.py +18 -3
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/metadata.py +7 -9
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/conftest.py +99 -1
- zimscraperlib-5.5.0/tests/download/test_download.py +776 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/html/test_html.py +8 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/rewriting/test_fuzzy_rules.py +62 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/rewriting/test_html_rewriting.py +150 -11
- zimscraperlib-5.5.0/tests/rewriting/test_js_globals.py +264 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/rewriting/test_js_rewriting.py +54 -14
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/rewriting/test_rx_replacer.py +8 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/rewriting/test_url_rewriting.py +270 -3
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/zim/test_indexing.py +92 -3
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/zim/test_metadata.py +6 -10
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/zim/test_zim_creator.py +21 -3
- zimscraperlib-5.4.1/src/zimscraperlib/__about__.py +0 -1
- zimscraperlib-5.4.1/src/zimscraperlib/download.py +0 -248
- zimscraperlib-5.4.1/src/zimscraperlib/rewriting/statics/wombat.js +0 -21
- zimscraperlib-5.4.1/src/zimscraperlib/zim/indexing.py +0 -130
- zimscraperlib-5.4.1/tests/download/test_download.py +0 -346
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/.gitignore +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/.readthedocs.yaml +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/LICENSE +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/README.md +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/contrib/README.md +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/contrib/encode_video.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/docs/assets/openzim.png +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/docs/functional_architecture.md +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/docs/index.md +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/docs/license.md +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/docs/scripts/generate_api_nav.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/docs/software_architecture.md +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/docs/technical_architecture.md +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/javascript/.prettierignore +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/javascript/.prettierrc.json +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/javascript/.yarnrc.yml +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/javascript/eslint.config.js +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/javascript/rollup.config.js +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/mkdocs.yml +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/rules/generate_rules.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/__init__.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/filesystem.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/fix_ogvjs_dist.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/i18n.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/image/__init__.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/image/conversion.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/image/illustration.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/image/optimization.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/image/presets.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/image/probing.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/image/transformation.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/image/utils.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/inputs.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/logging.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/misc.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/py.typed +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/__init__.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/css.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/statics/README.md +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/statics/__wb_module_decl.js +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/rewriting/templates/head_insert.html +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/types.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/typing.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/uri.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/video/__init__.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/video/config.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/video/encoding.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/video/presets.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/video/probing.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/__init__.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/_libkiwix.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/archive.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/dedup.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/src/zimscraperlib/zim/providers.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tasks.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/DroidSans.ttf +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/audio.mp3 +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/blue.jpg +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/commons.png +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/commons48.png +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/encrypted.pdf +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/encrypted.txt +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/mail.gif +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/milderm.pdf +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/milderm.txt +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/ninja.webp +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/pluto.jpg +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/single_wave_icon.gbr +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/square.jpg +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/square.png +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/star.svg +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/video.mkv +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/video.mp4 +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/files/video.webm +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/filesystem/test_filesystem.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/html/conftest.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/i18n/test_i18n.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/image/test_illustration.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/image/test_image.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/inputs/test_inputs.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/logging/conftest.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/logging/test_logging.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/misc/test_misc.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/ogvjs/conftest.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/ogvjs/test_ogvjs.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/rewriting/__init__.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/rewriting/conftest.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/rewriting/test_css_rewriting.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/rewriting/utils.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/test_constants.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/types/test_types.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/typing/test_typing.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/uri/test_uri.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/video/conftest.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/video/test_encoding.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/video/test_video.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/zim/conftest.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/zim/test_archive.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/zim/test_dedup.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/zim/test_fs.py +0 -0
- {zimscraperlib-5.4.1 → zimscraperlib-5.5.0}/tests/zim/test_libkiwix.py +0 -0
|
@@ -8,12 +8,12 @@ repos:
|
|
|
8
8
|
- id: trailing-whitespace
|
|
9
9
|
- id: end-of-file-fixer
|
|
10
10
|
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
11
|
-
rev: v0.
|
|
11
|
+
rev: v0.17.3
|
|
12
12
|
hooks:
|
|
13
13
|
- id: ruff-check
|
|
14
14
|
- id: ruff-format
|
|
15
15
|
- repo: https://github.com/RobertCraigie/pyright-python
|
|
16
|
-
rev: v1.1.
|
|
16
|
+
rev: v1.1.414
|
|
17
17
|
hooks:
|
|
18
18
|
- id: pyright
|
|
19
19
|
name: pyright (system)
|
|
@@ -5,6 +5,36 @@ All notable changes to this project are documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0).
|
|
7
7
|
|
|
8
|
+
## [5.5.0] - 2026-10-29
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- Index ePub documents for full-text search, with a new `get_epub_index_data` helper and automatic indexing of ePub items, like PDFs already are (#333)
|
|
13
|
+
- Send a Wikimedia-compliant default `User-Agent` in `stream_file` when downloading from `upload.wikimedia.org` (#214)
|
|
14
|
+
- `save_large_file` accepts arbitrary HTTP headers and sends the default Wikimedia-compliant `User-Agent` for `upload.wikimedia.org`, like `stream_file` (#360)
|
|
15
|
+
- Add fuzzy rule to rewrite URLs of lesbases.anct.gouv.fr (#334)
|
|
16
|
+
- Report progress from `save_large_file` through a new `on_progress` callback and periodic logging, and stream the download to a `.part` file that is only moved into place once complete (#348)
|
|
17
|
+
- New `get_pdf_title` and `get_epub_title` helpers to get a document's title from its own metadata without extracting its full text content, cheaper than `get_pdf_index_data`/`get_epub_index_data` when only the title is needed (openzim/warc2zim#462, #359)
|
|
18
|
+
|
|
19
|
+
### Changed
|
|
20
|
+
|
|
21
|
+
- Upgrade to wombat 3.11.0 (#338)
|
|
22
|
+
- Upgrade Python dependencies, allowing pymupdf 2, pillow 13, beartype 0.23, urllib3 2.8 and xxhash 4, and Node.JS dependencies (#366, #369)
|
|
23
|
+
|
|
24
|
+
### Fixed
|
|
25
|
+
|
|
26
|
+
- Rewrite `save_large_file` to stream and resume the download itself instead of running `wget` once, whose `-t` counter is a budget for the whole download: a flaky connection now keeps resuming and the retry budget only counts attempts that make no progress, so long downloads are no longer abandoned mid-way (#348)
|
|
27
|
+
- Carry a script's top-level `const`, `let`, `class` and `function` declarations across the wombat block, so other scripts on the page can still see them, re-declaring classes with `let` so they can be reassigned (#329, #370, webrecorder/wabac.js#380)
|
|
28
|
+
- Keep a rewritten `<script>`'s original `src` in `__wb_orig_src`, so wombat can return it from `getAttribute("src")` and bundlers that identify chunks by it (Next.js/Turbopack, Vite) still work (openzim/warc2zim#473, #337)
|
|
29
|
+
- Keep the original of every rewritten `src`, `href` and `xlink:href` in `__wb_orig_<attr>`, not just a script's `src`, so wombat can return it instead of a broken URL (openzim/warc2zim#413, #338)
|
|
30
|
+
- Keep URL-encoded `#` characters of the path or querystring when rewriting document URIs, instead of dropping everything after them (#341)
|
|
31
|
+
- Find the language of an HTML page in a `content-language` `<meta>` that is not the first `<meta>` of the page (#356)
|
|
32
|
+
- Rewrite the `poster` attribute (e.g. of `<video>`) when rewriting HTML, like `src` and `href` (#339)
|
|
33
|
+
- Do not collapse consecutive slashes `//` into a single `/` when normalizing URLs into ZIM paths, in both Python and JS rewriting (#340)
|
|
34
|
+
- Resolve relative links against a page URL containing consecutive slashes `//` the way browsers do, instead of collapsing them like `urllib.parse.urljoin` does (#346)
|
|
35
|
+
- `Creator.finish()` no longer silently discards errors raised by libzim while finalizing the ZIM, so a failure during finalization is no longer reported as a successful run (openzim/warc2zim#462, #354)
|
|
36
|
+
- Escape `.` and `..` path segments (e.g. folded into the path from a querystring) with a leading `_` when normalizing URLs into ZIM paths, so they are kept as literal entry names instead of being resolved away as navigation, in both Python and JS rewriting (#321)
|
|
37
|
+
|
|
8
38
|
## [5.4.1] - 2026-07-31
|
|
9
39
|
|
|
10
40
|
### Fixed
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: zimscraperlib
|
|
3
|
-
Version: 5.
|
|
3
|
+
Version: 5.5.0
|
|
4
4
|
Summary: Collection of python tools to re-use common code across scrapers
|
|
5
5
|
Project-URL: Donate, https://www.kiwix.org/en/support-us/
|
|
6
6
|
Project-URL: Homepage, https://github.com/openzim/python-scraperlib
|
|
@@ -15,7 +15,7 @@ Classifier: Programming Language :: Python :: 3
|
|
|
15
15
|
Classifier: Programming Language :: Python :: 3.14
|
|
16
16
|
Requires-Python: <3.15,>=3.14
|
|
17
17
|
Requires-Dist: babel<3.0,>=2.9
|
|
18
|
-
Requires-Dist: beartype<0.
|
|
18
|
+
Requires-Dist: beartype<0.24,>=0.19
|
|
19
19
|
Requires-Dist: beautifulsoup4<5.0,>=4.9.3
|
|
20
20
|
Requires-Dist: cairosvg<3.0,>=2.2.0
|
|
21
21
|
Requires-Dist: colorthief==0.2.1
|
|
@@ -25,57 +25,59 @@ Requires-Dist: libzim<4.0,>=3.8.0
|
|
|
25
25
|
Requires-Dist: lxml<7.0,>=4.6.3
|
|
26
26
|
Requires-Dist: optimize-images<3.0,>=1.3.6
|
|
27
27
|
Requires-Dist: piexif==1.1.3
|
|
28
|
-
Requires-Dist: pillow<
|
|
29
|
-
Requires-Dist: pymupdf<
|
|
28
|
+
Requires-Dist: pillow<14.0,>=7.0.0
|
|
29
|
+
Requires-Dist: pymupdf<3.0,>=1.24.0
|
|
30
30
|
Requires-Dist: python-magic<0.5,>=0.4.3
|
|
31
31
|
Requires-Dist: python-resize-image<1.2,>=1.1.19
|
|
32
32
|
Requires-Dist: regex>=2020.7.14
|
|
33
33
|
Requires-Dist: requests<3.0,>=2.25.1
|
|
34
|
+
Requires-Dist: tree-sitter-javascript<1.0,>=0.23
|
|
35
|
+
Requires-Dist: tree-sitter<1.0,>=0.23
|
|
34
36
|
Requires-Dist: types-xxhash<4.0,>=2.0
|
|
35
|
-
Requires-Dist: urllib3<2.
|
|
36
|
-
Requires-Dist: xxhash<
|
|
37
|
+
Requires-Dist: urllib3<2.9.0,>=1.26.5
|
|
38
|
+
Requires-Dist: xxhash<5.0,>=2.0
|
|
37
39
|
Requires-Dist: yt-dlp
|
|
38
40
|
Provides-Extra: dev
|
|
39
|
-
Requires-Dist: coverage==7.
|
|
41
|
+
Requires-Dist: coverage==7.17.0; extra == 'dev'
|
|
40
42
|
Requires-Dist: invoke==3.0.3; extra == 'dev'
|
|
41
|
-
Requires-Dist: ipython==9.
|
|
43
|
+
Requires-Dist: ipython==9.17.1; extra == 'dev'
|
|
42
44
|
Requires-Dist: jinja2==3.1.6; extra == 'dev'
|
|
43
45
|
Requires-Dist: mkdocs-gen-files==0.6.1; extra == 'dev'
|
|
44
46
|
Requires-Dist: mkdocs-include-markdown-plugin==7.3.0; extra == 'dev'
|
|
45
47
|
Requires-Dist: mkdocs-literate-nav==0.6.3; extra == 'dev'
|
|
46
|
-
Requires-Dist: mkdocs-material==9.
|
|
48
|
+
Requires-Dist: mkdocs-material==9.8.0; extra == 'dev'
|
|
47
49
|
Requires-Dist: mkdocs==1.6.1; extra == 'dev'
|
|
48
|
-
Requires-Dist: mkdocstrings[python]==1.0.
|
|
49
|
-
Requires-Dist: pre-commit==4.
|
|
50
|
-
Requires-Dist: pygments==2.
|
|
51
|
-
Requires-Dist: pymdown-extensions==
|
|
52
|
-
Requires-Dist: pyright==1.1.
|
|
53
|
-
Requires-Dist: pytest-mock==3.
|
|
54
|
-
Requires-Dist: pytest==9.
|
|
50
|
+
Requires-Dist: mkdocstrings[python]==1.0.8; extra == 'dev'
|
|
51
|
+
Requires-Dist: pre-commit==4.7.0; extra == 'dev'
|
|
52
|
+
Requires-Dist: pygments==2.21.0; extra == 'dev'
|
|
53
|
+
Requires-Dist: pymdown-extensions==12.3; extra == 'dev'
|
|
54
|
+
Requires-Dist: pyright==1.1.414; extra == 'dev'
|
|
55
|
+
Requires-Dist: pytest-mock==3.17.0; extra == 'dev'
|
|
56
|
+
Requires-Dist: pytest==9.2.0; extra == 'dev'
|
|
55
57
|
Requires-Dist: pyyaml==6.0.3; extra == 'dev'
|
|
56
|
-
Requires-Dist: ruff==0.
|
|
58
|
+
Requires-Dist: ruff==0.17.3; extra == 'dev'
|
|
57
59
|
Provides-Extra: docs
|
|
58
60
|
Requires-Dist: mkdocs-gen-files==0.6.1; extra == 'docs'
|
|
59
61
|
Requires-Dist: mkdocs-include-markdown-plugin==7.3.0; extra == 'docs'
|
|
60
62
|
Requires-Dist: mkdocs-literate-nav==0.6.3; extra == 'docs'
|
|
61
|
-
Requires-Dist: mkdocs-material==9.
|
|
63
|
+
Requires-Dist: mkdocs-material==9.8.0; extra == 'docs'
|
|
62
64
|
Requires-Dist: mkdocs==1.6.1; extra == 'docs'
|
|
63
|
-
Requires-Dist: mkdocstrings[python]==1.0.
|
|
64
|
-
Requires-Dist: pygments==2.
|
|
65
|
-
Requires-Dist: pymdown-extensions==
|
|
66
|
-
Requires-Dist: ruff==0.
|
|
65
|
+
Requires-Dist: mkdocstrings[python]==1.0.8; extra == 'docs'
|
|
66
|
+
Requires-Dist: pygments==2.21.0; extra == 'docs'
|
|
67
|
+
Requires-Dist: pymdown-extensions==12.3; extra == 'docs'
|
|
68
|
+
Requires-Dist: ruff==0.17.3; extra == 'docs'
|
|
67
69
|
Provides-Extra: qa
|
|
68
|
-
Requires-Dist: pyright==1.1.
|
|
69
|
-
Requires-Dist: pytest==9.
|
|
70
|
-
Requires-Dist: ruff==0.
|
|
70
|
+
Requires-Dist: pyright==1.1.414; extra == 'qa'
|
|
71
|
+
Requires-Dist: pytest==9.2.0; extra == 'qa'
|
|
72
|
+
Requires-Dist: ruff==0.17.3; extra == 'qa'
|
|
71
73
|
Provides-Extra: scripts
|
|
72
74
|
Requires-Dist: invoke==3.0.3; extra == 'scripts'
|
|
73
75
|
Requires-Dist: jinja2==3.1.6; extra == 'scripts'
|
|
74
76
|
Requires-Dist: pyyaml==6.0.3; extra == 'scripts'
|
|
75
77
|
Provides-Extra: test
|
|
76
|
-
Requires-Dist: coverage==7.
|
|
77
|
-
Requires-Dist: pytest-mock==3.
|
|
78
|
-
Requires-Dist: pytest==9.
|
|
78
|
+
Requires-Dist: coverage==7.17.0; extra == 'test'
|
|
79
|
+
Requires-Dist: pytest-mock==3.17.0; extra == 'test'
|
|
80
|
+
Requires-Dist: pytest==9.2.0; extra == 'test'
|
|
79
81
|
Description-Content-Type: text/markdown
|
|
80
82
|
|
|
81
83
|
# zimscraperlib
|
|
@@ -1,20 +1,18 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@openzim/wombat-setup",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "5.
|
|
4
|
+
"version": "5.5.0",
|
|
5
5
|
"license": "GPL-3.0-or-later",
|
|
6
6
|
"author": "openZIM",
|
|
7
7
|
"devDependencies": {
|
|
8
8
|
"@rollup/plugin-commonjs": "29.0.3",
|
|
9
9
|
"@rollup/plugin-node-resolve": "16.0.3",
|
|
10
10
|
"@rollup/plugin-strip": "3.0.4",
|
|
11
|
-
"@rollup/plugin-terser": "1.0.
|
|
12
|
-
"ava": "8.0
|
|
13
|
-
"
|
|
14
|
-
"
|
|
15
|
-
"
|
|
16
|
-
"prettier": "3.9.6",
|
|
17
|
-
"rollup": "4.62.2",
|
|
11
|
+
"@rollup/plugin-terser": "1.0.2",
|
|
12
|
+
"ava": "8.2.0",
|
|
13
|
+
"eslint": "10.13.0",
|
|
14
|
+
"prettier": "3.9.10",
|
|
15
|
+
"rollup": "4.65.1",
|
|
18
16
|
"rollup-plugin-version-injector": "1.3.3"
|
|
19
17
|
},
|
|
20
18
|
"scripts": {
|
|
@@ -43,5 +41,5 @@
|
|
|
43
41
|
"engines": {
|
|
44
42
|
"node": ">=26.0.0 <27.0.0"
|
|
45
43
|
},
|
|
46
|
-
"packageManager": "yarn@4.
|
|
44
|
+
"packageManager": "yarn@4.18.1"
|
|
47
45
|
}
|
|
@@ -18,6 +18,20 @@ export function applyFuzzyRules(path) {
|
|
|
18
18
|
return path;
|
|
19
19
|
}
|
|
20
20
|
|
|
21
|
+
export function escapeDotSegments(path) {
|
|
22
|
+
// Escape '.' and '..' path segments so they are not resolved as navigation.
|
|
23
|
+
// Per RFC 3986, a '.' or '..' path segment is a dot-segment which any URI consumer
|
|
24
|
+
// removes/resolves while normalizing a reference. A ZIM path must keep such a
|
|
25
|
+
// segment as a literal entry name instead (e.g. one folded in from a querystring),
|
|
26
|
+
// so it is prefixed with a '_', which is not itself special in a ZIM path.
|
|
27
|
+
// Segments which only look like a dot-segment (e.g. 'foo..', '..foo', '...') are
|
|
28
|
+
// not dot-segments and are left untouched.
|
|
29
|
+
return path
|
|
30
|
+
.split('/')
|
|
31
|
+
.map((part) => (part === '.' || part === '..' ? '_' + part : part))
|
|
32
|
+
.join('/');
|
|
33
|
+
}
|
|
34
|
+
|
|
21
35
|
export function hasAlreadyBeenRewritten(
|
|
22
36
|
original_absolute_url,
|
|
23
37
|
orig_url,
|
|
@@ -63,12 +77,6 @@ export function hasAlreadyBeenRewritten(
|
|
|
63
77
|
return false;
|
|
64
78
|
}
|
|
65
79
|
|
|
66
|
-
function removeSubsequentSlashes(value) {
|
|
67
|
-
// Remove all successive occurrences of a slash `/` in a given string
|
|
68
|
-
// E.g `val//ue` or `val///ue` or `val////ue` (and so on) are transformed into `value`
|
|
69
|
-
return value.replace(/\/\/+/g, '/');
|
|
70
|
-
}
|
|
71
|
-
|
|
72
80
|
export function urlRewriteFunction(
|
|
73
81
|
current_url, // The current (real) url we are on, e.g. http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/index.html
|
|
74
82
|
orig_host, // The host of the original url, e.g. www.example.com
|
|
@@ -186,16 +194,20 @@ export function urlRewriteFunction(
|
|
|
186
194
|
: '?' + decodeURIComponent(absolute_url_parts.query).replaceAll('+', ' ');
|
|
187
195
|
|
|
188
196
|
// combine all decoded parts to get the ZIM path
|
|
189
|
-
|
|
190
|
-
|
|
197
|
+
// (consecutive slashes are kept, just like in Python)
|
|
198
|
+
const zimPath = decoded_host + decoded_path + decoded_query;
|
|
191
199
|
|
|
192
200
|
// apply the fuzzy rules to the ZIM path
|
|
193
201
|
const fuzzifiedPath = applyFuzzyRules(zimPath);
|
|
194
202
|
|
|
203
|
+
// escape '.' and '..' path segments so they are not resolved as navigation by the
|
|
204
|
+
// reader once this ZIM path is turned into a URI
|
|
205
|
+
const escapedPath = escapeDotSegments(fuzzifiedPath);
|
|
206
|
+
|
|
195
207
|
// Reencode everything but '/' (we decode it afterwards for simplicity)
|
|
196
208
|
const finalUrl =
|
|
197
209
|
prefix +
|
|
198
|
-
encodeURIComponent(
|
|
210
|
+
encodeURIComponent(escapedPath)
|
|
199
211
|
.replaceAll('%2F', '/')
|
|
200
212
|
.replaceAll('%3D', '=')
|
|
201
213
|
.replaceAll('%2C', ',');
|
|
@@ -316,6 +328,7 @@ export function getWombatInfo(
|
|
|
316
328
|
|
|
317
329
|
export default {
|
|
318
330
|
applyFuzzyRules: applyFuzzyRules,
|
|
331
|
+
escapeDotSegments: escapeDotSegments,
|
|
319
332
|
urlRewriteFunction: urlRewriteFunction,
|
|
320
333
|
getWombatInfo: getWombatInfo,
|
|
321
334
|
};
|
|
@@ -8,6 +8,29 @@ test.beforeEach((t) => {
|
|
|
8
8
|
t.context.originalScheme = 'https';
|
|
9
9
|
});
|
|
10
10
|
|
|
11
|
+
test('escapeDotSegmentsDot', (t) => {
|
|
12
|
+
t.is(utils.escapeDotSegments('a/./b'), 'a/_./b');
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
test('escapeDotSegmentsDotDot', (t) => {
|
|
16
|
+
t.is(utils.escapeDotSegments('a/../b'), 'a/_../b');
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
test('escapeDotSegmentsTrailingDotDot', (t) => {
|
|
20
|
+
t.is(utils.escapeDotSegments('a/..'), 'a/_..');
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
test('escapeDotSegmentsConsecutive', (t) => {
|
|
24
|
+
t.is(utils.escapeDotSegments('a/../../b'), 'a/_../_../b');
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
test('escapeDotSegmentsLookAlikesUntouched', (t) => {
|
|
28
|
+
t.is(
|
|
29
|
+
utils.escapeDotSegments('a/.foo/foo../..foo/.../b'),
|
|
30
|
+
'a/.foo/foo../..foo/.../b',
|
|
31
|
+
);
|
|
32
|
+
});
|
|
33
|
+
|
|
11
34
|
test('nominalWbInfo', (t) => {
|
|
12
35
|
const path = 'path1/resource1.js';
|
|
13
36
|
const originalUrl =
|
|
@@ -84,6 +84,46 @@ test('simpleContentNoScheme', (t) => {
|
|
|
84
84
|
);
|
|
85
85
|
});
|
|
86
86
|
|
|
87
|
+
test('consecutiveSlashesAreKept', (t) => {
|
|
88
|
+
t.is(
|
|
89
|
+
urlRewriteFunction(
|
|
90
|
+
t.context.currentUrl,
|
|
91
|
+
t.context.originalHost,
|
|
92
|
+
t.context.originalScheme,
|
|
93
|
+
t.context.originalUrl,
|
|
94
|
+
t.context.prefix,
|
|
95
|
+
'https://www.example.com/a///b',
|
|
96
|
+
undefined,
|
|
97
|
+
undefined,
|
|
98
|
+
undefined,
|
|
99
|
+
),
|
|
100
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/a///b',
|
|
101
|
+
);
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
test('relativeLinkOnPageWithConsecutiveSlashesKeepsThem', (t) => {
|
|
105
|
+
// #346: a relative link must be resolved against the page's own URL the way a
|
|
106
|
+
// browser would, keeping consecutive slashes in that URL instead of collapsing
|
|
107
|
+
// them (uri-js's URI.resolve, unlike Python's urllib.parse.urljoin, already does
|
|
108
|
+
// this correctly)
|
|
109
|
+
const originalUrl = 'https://www.example.com/path1//resource1.html';
|
|
110
|
+
const currentUrl = t.context.prefix + 'www.example.com/path1//resource1.html';
|
|
111
|
+
t.is(
|
|
112
|
+
urlRewriteFunction(
|
|
113
|
+
currentUrl,
|
|
114
|
+
t.context.originalHost,
|
|
115
|
+
t.context.originalScheme,
|
|
116
|
+
originalUrl,
|
|
117
|
+
t.context.prefix,
|
|
118
|
+
'other.html',
|
|
119
|
+
undefined,
|
|
120
|
+
undefined,
|
|
121
|
+
undefined,
|
|
122
|
+
),
|
|
123
|
+
t.context.prefix + 'www.example.com/path1//other.html',
|
|
124
|
+
);
|
|
125
|
+
});
|
|
126
|
+
|
|
87
127
|
test('simpleContentEmptyPath', (t) => {
|
|
88
128
|
t.is(
|
|
89
129
|
urlRewriteFunction(
|
|
@@ -458,6 +498,25 @@ test('contentWithSimpleQueryString', (t) => {
|
|
|
458
498
|
);
|
|
459
499
|
});
|
|
460
500
|
|
|
501
|
+
test('contentWithQueryStringDotDotSegment', (t) => {
|
|
502
|
+
// a `..` dot-segment folded into the path from a querystring must be escaped so
|
|
503
|
+
// it is not resolved as navigation once this ZIM path is turned into a URI
|
|
504
|
+
t.is(
|
|
505
|
+
urlRewriteFunction(
|
|
506
|
+
t.context.currentUrl,
|
|
507
|
+
t.context.originalHost,
|
|
508
|
+
t.context.originalScheme,
|
|
509
|
+
t.context.originalUrl,
|
|
510
|
+
t.context.prefix,
|
|
511
|
+
'https://www.example.com/javascript/content.txt?css=../../prg',
|
|
512
|
+
undefined,
|
|
513
|
+
undefined,
|
|
514
|
+
undefined,
|
|
515
|
+
),
|
|
516
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fcss=../_../prg',
|
|
517
|
+
);
|
|
518
|
+
});
|
|
519
|
+
|
|
461
520
|
test('contentWithSimplePathWithEqualAndComma', (t) => {
|
|
462
521
|
t.is(
|
|
463
522
|
urlRewriteFunction(
|
|
@@ -1159,7 +1218,7 @@ test('doubleSlash1', (t) => {
|
|
|
1159
1218
|
undefined,
|
|
1160
1219
|
undefined,
|
|
1161
1220
|
),
|
|
1162
|
-
'http://library.kiwix.org/content/myzim_yyyy-mm/example.com/some/path/http%3A
|
|
1221
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/example.com/some/path/http%3A//example.com//some/path',
|
|
1163
1222
|
);
|
|
1164
1223
|
});
|
|
1165
1224
|
|
|
@@ -1176,7 +1235,7 @@ test('doubleSlash2', (t) => {
|
|
|
1176
1235
|
undefined,
|
|
1177
1236
|
undefined,
|
|
1178
1237
|
),
|
|
1179
|
-
'http://library.kiwix.org/content/myzim_yyyy-mm/example.com/some/pa%3Fth/http%3A
|
|
1238
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/example.com/some/pa%3Fth/http%3A//example.com//some/path',
|
|
1180
1239
|
);
|
|
1181
1240
|
});
|
|
1182
1241
|
|
|
@@ -1193,6 +1252,6 @@ test('doubleSlash3', (t) => {
|
|
|
1193
1252
|
undefined,
|
|
1194
1253
|
undefined,
|
|
1195
1254
|
),
|
|
1196
|
-
'http://library.kiwix.org/content/myzim_yyyy-mm/example.com/so%3Fme/pa%3Fth/http%3A
|
|
1255
|
+
'http://library.kiwix.org/content/myzim_yyyy-mm/example.com/so%3Fme/pa%3Fth/http%3A//example.com//some/path',
|
|
1197
1256
|
);
|
|
1198
1257
|
});
|