zimscraperlib 5.3.0__tar.gz → 5.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/.pre-commit-config.yaml +4 -7
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/CHANGELOG.md +27 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/PKG-INFO +68 -49
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/README.md +37 -18
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/javascript/package.json +14 -14
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/javascript/src/wombatSetup.js +12 -5
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/javascript/test/wombatSetup.js +2 -2
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/javascript/yarn.lock +694 -1036
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/mkdocs.yml +1 -1
- zimscraperlib-5.4.0/openzim.toml +20 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/pyproject.toml +30 -38
- zimscraperlib-5.4.0/src/zimscraperlib/__about__.py +1 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/download.py +24 -14
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/html.py +3 -4
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/i18n.py +10 -9
- zimscraperlib-5.4.0/src/zimscraperlib/image/optimization.py +475 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/image/probing.py +9 -1
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/misc.py +1 -5
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/rewriting/css.py +9 -21
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/rewriting/js.py +57 -6
- zimscraperlib-5.4.0/src/zimscraperlib/rewriting/statics/wombat.js +21 -0
- zimscraperlib-5.4.0/src/zimscraperlib/rewriting/statics/wombatSetup.js +3 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/video/encoding.py +0 -1
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/zim/creator.py +29 -6
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/zim/indexing.py +1 -1
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/zim/metadata.py +0 -2
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tasks.py +35 -39
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/download/test_download.py +22 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/image/test_image.py +243 -42
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/rewriting/conftest.py +12 -12
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/rewriting/test_html_rewriting.py +1 -3
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/rewriting/test_js_rewriting.py +3 -11
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/rewriting/test_url_rewriting.py +2 -6
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/rewriting/utils.py +0 -1
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/zim/test_dedup.py +1 -1
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/zim/test_fs.py +2 -2
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/zim/test_indexing.py +4 -2
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/zim/test_metadata.py +1 -3
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/zim/test_zim_creator.py +62 -2
- zimscraperlib-5.3.0/openzim.toml +0 -10
- zimscraperlib-5.3.0/src/zimscraperlib/__about__.py +0 -1
- zimscraperlib-5.3.0/src/zimscraperlib/image/optimization.py +0 -401
- zimscraperlib-5.3.0/src/zimscraperlib/rewriting/statics/wombat.js +0 -21
- zimscraperlib-5.3.0/src/zimscraperlib/rewriting/statics/wombatSetup.js +0 -3
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/.gitignore +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/.readthedocs.yaml +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/LICENSE +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/contrib/README.md +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/contrib/encode_video.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/docs/assets/openzim.png +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/docs/functional_architecture.md +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/docs/index.md +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/docs/license.md +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/docs/scripts/generate_api_nav.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/docs/software_architecture.md +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/docs/technical_architecture.md +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/javascript/.prettierignore +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/javascript/.prettierrc.json +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/javascript/eslint.config.js +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/javascript/rollup.config.js +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/javascript/test/wombatUrlRewriting.js +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/rules/generate_rules.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/rules/rules.yaml +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/__init__.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/constants.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/filesystem.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/fix_ogvjs_dist.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/image/__init__.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/image/conversion.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/image/illustration.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/image/presets.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/image/transformation.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/image/utils.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/inputs.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/logging.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/py.typed +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/rewriting/__init__.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/rewriting/html.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/rewriting/rules.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/rewriting/rx_replacer.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/rewriting/statics/README.md +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/rewriting/statics/__wb_module_decl.js +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/rewriting/templates/head_insert.html +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/rewriting/url_rewriting.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/types.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/typing.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/uri.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/video/__init__.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/video/config.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/video/presets.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/video/probing.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/zim/__init__.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/zim/_libkiwix.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/zim/archive.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/zim/dedup.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/zim/filesystem.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/zim/items.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/src/zimscraperlib/zim/providers.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/conftest.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/files/DroidSans.ttf +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/files/audio.mp3 +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/files/blue.jpg +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/files/commons.png +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/files/commons48.png +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/files/encrypted.pdf +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/files/encrypted.txt +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/files/mail.gif +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/files/milderm.pdf +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/files/milderm.txt +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/files/ninja.webp +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/files/pluto.jpg +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/files/single_wave_icon.gbr +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/files/square.jpg +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/files/square.png +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/files/star.svg +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/files/video.mkv +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/files/video.mp4 +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/files/video.webm +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/filesystem/test_filesystem.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/html/conftest.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/html/test_html.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/i18n/test_i18n.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/image/test_illustration.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/inputs/test_inputs.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/logging/conftest.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/logging/test_logging.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/misc/test_misc.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/ogvjs/conftest.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/ogvjs/test_ogvjs.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/rewriting/__init__.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/rewriting/test_css_rewriting.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/rewriting/test_fuzzy_rules.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/rewriting/test_rx_replacer.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/test_constants.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/types/test_types.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/typing/test_typing.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/uri/test_uri.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/video/conftest.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/video/test_encoding.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/video/test_video.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/zim/conftest.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/zim/test_archive.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/tests/zim/test_libkiwix.py +0 -0
- {zimscraperlib-5.3.0 → zimscraperlib-5.4.0}/yarn.lock +0 -0
|
@@ -7,16 +7,13 @@ repos:
|
|
|
7
7
|
hooks:
|
|
8
8
|
- id: trailing-whitespace
|
|
9
9
|
- id: end-of-file-fixer
|
|
10
|
-
- repo: https://github.com/psf/black
|
|
11
|
-
rev: '25.11.0'
|
|
12
|
-
hooks:
|
|
13
|
-
- id: black
|
|
14
10
|
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
15
|
-
rev: v0.14
|
|
11
|
+
rev: v0.15.14
|
|
16
12
|
hooks:
|
|
17
|
-
- id: ruff
|
|
13
|
+
- id: ruff-check
|
|
14
|
+
- id: ruff-format
|
|
18
15
|
- repo: https://github.com/RobertCraigie/pyright-python
|
|
19
|
-
rev: v1.1.
|
|
16
|
+
rev: v1.1.409
|
|
20
17
|
hooks:
|
|
21
18
|
- id: pyright
|
|
22
19
|
name: pyright (system)
|
|
@@ -5,6 +5,33 @@ All notable changes to this project are documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0).
|
|
7
7
|
|
|
8
|
+
## [5.4.0] - 2026-05-28
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- Add support for custom number of retries and user-agent in save_large_file (#278)
|
|
13
|
+
- Enhance save_large_file log level (#279)
|
|
14
|
+
- Extend image optimization to support in-memory streams(BytesIO/bytes) and dst_format param (#289)
|
|
15
|
+
- Automatically add \_ftindex:<yes/no> tag at creator start based on indexing configuration (#295)
|
|
16
|
+
- **BEHAVIOR CHANGE**: it is not possible anymore to add "Tags" metadata with Creator.add_metadata method after creator has started ; this is anyway not recommended anymore since 5.x (one should prefer Creator.config_metadata)
|
|
17
|
+
|
|
18
|
+
### Fixed
|
|
19
|
+
|
|
20
|
+
- Add proper typing @overload to `zimscraperlib.image.optimize_xxx` methods (#273)
|
|
21
|
+
- Backport wabac.JS change around rewriting: detect when JS is in 'strict' mode and avoid using 'arguments' (#286)
|
|
22
|
+
- Fix installation from source, which missed wombatSetup.js (#287)
|
|
23
|
+
- Fix outdated system dependencies in README: remove Pillow build-time deps (bundled in wheels), add missing `libcairo` across all platforms (#152)
|
|
24
|
+
- Improve contribution setup instructions: use `hatch shell`, clarify commands must be run from local clone root (#153)
|
|
25
|
+
- Bring coverage back to 100% (#293)
|
|
26
|
+
- Fix `Creator.config_indexing` docstring to reflect that only the full-text index is toggled; title indexing is always performed by libzim (#294)
|
|
27
|
+
- Fix `format_for` to not modify input BytesIO position (#296)
|
|
28
|
+
|
|
29
|
+
### Changed
|
|
30
|
+
|
|
31
|
+
- Upgrade to wombat 3.10.5 (#306)
|
|
32
|
+
- Adopt openzim/\_python-bootstrap 2.0.0 (#303)
|
|
33
|
+
- Upgrade Python dependencies, Node.JS runtime and Node.JS dependencies (#306)
|
|
34
|
+
|
|
8
35
|
## [5.3.0] - 2025-11-14
|
|
9
36
|
|
|
10
37
|
### Changed
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: zimscraperlib
|
|
3
|
-
Version: 5.
|
|
3
|
+
Version: 5.4.0
|
|
4
4
|
Summary: Collection of python tools to re-use common code across scrapers
|
|
5
5
|
Project-URL: Donate, https://www.kiwix.org/en/support-us/
|
|
6
6
|
Project-URL: Homepage, https://github.com/openzim/python-scraperlib
|
|
@@ -32,55 +32,56 @@ Requires-Dist: python-resize-image<1.2,>=1.1.19
|
|
|
32
32
|
Requires-Dist: regex>=2020.7.14
|
|
33
33
|
Requires-Dist: requests<3.0,>=2.25.1
|
|
34
34
|
Requires-Dist: types-xxhash<4.0,>=2.0
|
|
35
|
-
Requires-Dist: urllib3<2.
|
|
35
|
+
Requires-Dist: urllib3<2.8.0,>=1.26.5
|
|
36
36
|
Requires-Dist: xxhash<4.0,>=2.0
|
|
37
37
|
Requires-Dist: yt-dlp
|
|
38
|
-
Provides-Extra: check
|
|
39
|
-
Requires-Dist: pyright==1.1.407; extra == 'check'
|
|
40
|
-
Requires-Dist: pytest==9.0.1; extra == 'check'
|
|
41
38
|
Provides-Extra: dev
|
|
42
|
-
Requires-Dist:
|
|
43
|
-
Requires-Dist:
|
|
44
|
-
Requires-Dist:
|
|
45
|
-
Requires-Dist: ipython==9.7.0; extra == 'dev'
|
|
39
|
+
Requires-Dist: coverage==7.14.1; extra == 'dev'
|
|
40
|
+
Requires-Dist: invoke==3.0.3; extra == 'dev'
|
|
41
|
+
Requires-Dist: ipython==9.13.0; extra == 'dev'
|
|
46
42
|
Requires-Dist: jinja2==3.1.6; extra == 'dev'
|
|
47
|
-
Requires-Dist: mkdocs-gen-files==0.
|
|
48
|
-
Requires-Dist: mkdocs-include-markdown-plugin==7.
|
|
49
|
-
Requires-Dist: mkdocs-literate-nav==0.6.
|
|
50
|
-
Requires-Dist: mkdocs-material==9.7.
|
|
43
|
+
Requires-Dist: mkdocs-gen-files==0.6.1; extra == 'dev'
|
|
44
|
+
Requires-Dist: mkdocs-include-markdown-plugin==7.3.0; extra == 'dev'
|
|
45
|
+
Requires-Dist: mkdocs-literate-nav==0.6.3; extra == 'dev'
|
|
46
|
+
Requires-Dist: mkdocs-material==9.7.6; extra == 'dev'
|
|
51
47
|
Requires-Dist: mkdocs==1.6.1; extra == 'dev'
|
|
52
|
-
Requires-Dist: mkdocstrings[python]==0.
|
|
53
|
-
Requires-Dist: pre-commit==4.
|
|
54
|
-
Requires-Dist:
|
|
55
|
-
Requires-Dist:
|
|
48
|
+
Requires-Dist: mkdocstrings[python]==1.0.3; extra == 'dev'
|
|
49
|
+
Requires-Dist: pre-commit==4.6.0; extra == 'dev'
|
|
50
|
+
Requires-Dist: pygments==2.19.2; extra == 'dev'
|
|
51
|
+
Requires-Dist: pymdown-extensions==10.21.3; extra == 'dev'
|
|
52
|
+
Requires-Dist: pyright==1.1.409; extra == 'dev'
|
|
56
53
|
Requires-Dist: pytest-mock==3.15.1; extra == 'dev'
|
|
57
|
-
Requires-Dist: pytest==9.0.
|
|
54
|
+
Requires-Dist: pytest==9.0.3; extra == 'dev'
|
|
58
55
|
Requires-Dist: pyyaml==6.0.3; extra == 'dev'
|
|
59
|
-
Requires-Dist: ruff==0.14
|
|
56
|
+
Requires-Dist: ruff==0.15.14; extra == 'dev'
|
|
60
57
|
Provides-Extra: docs
|
|
61
|
-
Requires-Dist: mkdocs-gen-files==0.
|
|
62
|
-
Requires-Dist: mkdocs-include-markdown-plugin==7.
|
|
63
|
-
Requires-Dist: mkdocs-literate-nav==0.6.
|
|
64
|
-
Requires-Dist: mkdocs-material==9.7.
|
|
58
|
+
Requires-Dist: mkdocs-gen-files==0.6.1; extra == 'docs'
|
|
59
|
+
Requires-Dist: mkdocs-include-markdown-plugin==7.3.0; extra == 'docs'
|
|
60
|
+
Requires-Dist: mkdocs-literate-nav==0.6.3; extra == 'docs'
|
|
61
|
+
Requires-Dist: mkdocs-material==9.7.6; extra == 'docs'
|
|
65
62
|
Requires-Dist: mkdocs==1.6.1; extra == 'docs'
|
|
66
|
-
Requires-Dist: mkdocstrings[python]==0.
|
|
67
|
-
Requires-Dist:
|
|
68
|
-
|
|
69
|
-
Requires-Dist:
|
|
70
|
-
|
|
63
|
+
Requires-Dist: mkdocstrings[python]==1.0.3; extra == 'docs'
|
|
64
|
+
Requires-Dist: pygments==2.19.2; extra == 'docs'
|
|
65
|
+
Requires-Dist: pymdown-extensions==10.21.3; extra == 'docs'
|
|
66
|
+
Requires-Dist: ruff==0.15.14; extra == 'docs'
|
|
67
|
+
Provides-Extra: qa
|
|
68
|
+
Requires-Dist: pyright==1.1.409; extra == 'qa'
|
|
69
|
+
Requires-Dist: pytest==9.0.3; extra == 'qa'
|
|
70
|
+
Requires-Dist: ruff==0.15.14; extra == 'qa'
|
|
71
71
|
Provides-Extra: scripts
|
|
72
|
-
Requires-Dist: invoke==
|
|
72
|
+
Requires-Dist: invoke==3.0.3; extra == 'scripts'
|
|
73
73
|
Requires-Dist: jinja2==3.1.6; extra == 'scripts'
|
|
74
74
|
Requires-Dist: pyyaml==6.0.3; extra == 'scripts'
|
|
75
75
|
Provides-Extra: test
|
|
76
|
-
Requires-Dist: coverage==7.
|
|
76
|
+
Requires-Dist: coverage==7.14.1; extra == 'test'
|
|
77
77
|
Requires-Dist: pytest-mock==3.15.1; extra == 'test'
|
|
78
|
-
Requires-Dist: pytest==9.0.
|
|
78
|
+
Requires-Dist: pytest==9.0.3; extra == 'test'
|
|
79
79
|
Description-Content-Type: text/markdown
|
|
80
80
|
|
|
81
81
|
# zimscraperlib
|
|
82
82
|
|
|
83
|
-
[](https://github.com/openzim/python-scraperlib/actions/workflows/QA.yaml)
|
|
84
|
+
[](https://github.com/openzim/python-scraperlib/actions/workflows/Tests.yaml)
|
|
84
85
|
[](https://www.codefactor.io/repository/github/openzim/python-scraperlib)
|
|
85
86
|
[](https://www.gnu.org/licenses/gpl-3.0)
|
|
86
87
|
[](https://pypi.org/project/zimscraperlib/)
|
|
@@ -113,46 +114,64 @@ See documentation at [Read the Docs](https://python-scraperlib.readthedocs.io/)
|
|
|
113
114
|
|
|
114
115
|
# Dependencies
|
|
115
116
|
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
-
|
|
119
|
-
-
|
|
120
|
-
- FFmpeg
|
|
121
|
-
- gifsicle (>=1.92)
|
|
122
|
-
- libcairo
|
|
117
|
+
Most dependencies are installed automatically by pip (from PyPI by default). The following system packages may be required depending on which features you use:
|
|
118
|
+
|
|
119
|
+
- **libmagic** — required for file type detection (used in most scrapers)
|
|
120
|
+
- **wget** — required only for `zimscraperlib.download` functions
|
|
121
|
+
- **FFmpeg** — required only for video processing functions
|
|
122
|
+
- **gifsicle** (>=1.92) — required only for GIF optimization
|
|
123
|
+
- **libcairo** — required only for SVG-to-PNG conversion
|
|
124
|
+
- **libzim** — auto-installed via PyPI, not available on Windows
|
|
125
|
+
- **Pillow** — auto-installed via PyPI; pre-built wheels are used by default and no system image libraries are needed. Only if you need to build Pillow from source should you install additional system libraries — see [Pillow's build documentation](https://pillow.readthedocs.io/en/latest/installation/building-from-source.html) for details.
|
|
126
|
+
> **Note:** To run the full test suite, all system dependencies listed above must be installed.
|
|
123
127
|
|
|
124
128
|
## macOS
|
|
125
129
|
|
|
126
130
|
```sh
|
|
127
|
-
brew install libmagic wget
|
|
131
|
+
brew install libmagic wget ffmpeg gifsicle cairo
|
|
128
132
|
```
|
|
129
133
|
|
|
130
134
|
## Linux
|
|
131
135
|
|
|
132
136
|
```sh
|
|
133
|
-
sudo apt install libmagic1 wget ffmpeg
|
|
134
|
-
libtiff5-dev libjpeg8-dev libopenjp2-7-dev zlib1g-dev \
|
|
135
|
-
libfreetype6-dev liblcms2-dev libwebp-dev tcl8.6-dev tk8.6-dev python3-tk \
|
|
136
|
-
libharfbuzz-dev libfribidi-dev libxcb1-dev gifsicle
|
|
137
|
+
sudo apt install libmagic1 wget ffmpeg gifsicle libcairo2
|
|
137
138
|
```
|
|
138
139
|
|
|
139
140
|
## Alpine
|
|
140
141
|
|
|
141
|
-
```
|
|
142
|
-
apk add ffmpeg gifsicle libmagic wget
|
|
142
|
+
```sh
|
|
143
|
+
apk add ffmpeg gifsicle libmagic wget cairo
|
|
143
144
|
```
|
|
144
145
|
|
|
145
146
|
# Contribution
|
|
146
147
|
|
|
147
148
|
This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing).
|
|
148
149
|
|
|
149
|
-
This project has implemented openZIM's [Python bootstrap, conventions and policies](https://github.com/openzim/_python-bootstrap/docs/Policy.md) **
|
|
150
|
+
This project has implemented openZIM's [Python bootstrap, conventions and policies](https://github.com/openzim/_python-bootstrap/docs/Policy.md) **v2.0.0**.
|
|
151
|
+
|
|
152
|
+
All instructions below must be run from the root of your local clone of this repository.
|
|
153
|
+
|
|
154
|
+
If you do not already have it on your system, install [hatch](https://hatch.pypa.io/latest/install/):
|
|
150
155
|
|
|
151
156
|
```shell
|
|
152
157
|
pip install hatch
|
|
153
|
-
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
Start a hatch shell — this will install all dependencies including dev in an isolated virtual environment:
|
|
161
|
+
|
|
162
|
+
```shell
|
|
163
|
+
hatch shell
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
Set up the pre-commit Git hook (runs linters automatically before each commit):
|
|
167
|
+
|
|
168
|
+
```shell
|
|
154
169
|
pre-commit install
|
|
155
|
-
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
Run tests with coverage:
|
|
173
|
+
|
|
174
|
+
```shell
|
|
156
175
|
invoke coverage
|
|
157
176
|
```
|
|
158
177
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# zimscraperlib
|
|
2
2
|
|
|
3
|
-
[](https://github.com/openzim/python-scraperlib/actions/workflows/QA.yaml)
|
|
4
|
+
[](https://github.com/openzim/python-scraperlib/actions/workflows/Tests.yaml)
|
|
4
5
|
[](https://www.codefactor.io/repository/github/openzim/python-scraperlib)
|
|
5
6
|
[](https://www.gnu.org/licenses/gpl-3.0)
|
|
6
7
|
[](https://pypi.org/project/zimscraperlib/)
|
|
@@ -33,46 +34,64 @@ See documentation at [Read the Docs](https://python-scraperlib.readthedocs.io/)
|
|
|
33
34
|
|
|
34
35
|
# Dependencies
|
|
35
36
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
-
|
|
39
|
-
-
|
|
40
|
-
- FFmpeg
|
|
41
|
-
- gifsicle (>=1.92)
|
|
42
|
-
- libcairo
|
|
37
|
+
Most dependencies are installed automatically by pip (from PyPI by default). The following system packages may be required depending on which features you use:
|
|
38
|
+
|
|
39
|
+
- **libmagic** — required for file type detection (used in most scrapers)
|
|
40
|
+
- **wget** — required only for `zimscraperlib.download` functions
|
|
41
|
+
- **FFmpeg** — required only for video processing functions
|
|
42
|
+
- **gifsicle** (>=1.92) — required only for GIF optimization
|
|
43
|
+
- **libcairo** — required only for SVG-to-PNG conversion
|
|
44
|
+
- **libzim** — auto-installed via PyPI, not available on Windows
|
|
45
|
+
- **Pillow** — auto-installed via PyPI; pre-built wheels are used by default and no system image libraries are needed. Only if you need to build Pillow from source should you install additional system libraries — see [Pillow's build documentation](https://pillow.readthedocs.io/en/latest/installation/building-from-source.html) for details.
|
|
46
|
+
> **Note:** To run the full test suite, all system dependencies listed above must be installed.
|
|
43
47
|
|
|
44
48
|
## macOS
|
|
45
49
|
|
|
46
50
|
```sh
|
|
47
|
-
brew install libmagic wget
|
|
51
|
+
brew install libmagic wget ffmpeg gifsicle cairo
|
|
48
52
|
```
|
|
49
53
|
|
|
50
54
|
## Linux
|
|
51
55
|
|
|
52
56
|
```sh
|
|
53
|
-
sudo apt install libmagic1 wget ffmpeg
|
|
54
|
-
libtiff5-dev libjpeg8-dev libopenjp2-7-dev zlib1g-dev \
|
|
55
|
-
libfreetype6-dev liblcms2-dev libwebp-dev tcl8.6-dev tk8.6-dev python3-tk \
|
|
56
|
-
libharfbuzz-dev libfribidi-dev libxcb1-dev gifsicle
|
|
57
|
+
sudo apt install libmagic1 wget ffmpeg gifsicle libcairo2
|
|
57
58
|
```
|
|
58
59
|
|
|
59
60
|
## Alpine
|
|
60
61
|
|
|
61
|
-
```
|
|
62
|
-
apk add ffmpeg gifsicle libmagic wget
|
|
62
|
+
```sh
|
|
63
|
+
apk add ffmpeg gifsicle libmagic wget cairo
|
|
63
64
|
```
|
|
64
65
|
|
|
65
66
|
# Contribution
|
|
66
67
|
|
|
67
68
|
This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing).
|
|
68
69
|
|
|
69
|
-
This project has implemented openZIM's [Python bootstrap, conventions and policies](https://github.com/openzim/_python-bootstrap/docs/Policy.md) **
|
|
70
|
+
This project has implemented openZIM's [Python bootstrap, conventions and policies](https://github.com/openzim/_python-bootstrap/docs/Policy.md) **v2.0.0**.
|
|
71
|
+
|
|
72
|
+
All instructions below must be run from the root of your local clone of this repository.
|
|
73
|
+
|
|
74
|
+
If you do not already have it on your system, install [hatch](https://hatch.pypa.io/latest/install/):
|
|
70
75
|
|
|
71
76
|
```shell
|
|
72
77
|
pip install hatch
|
|
73
|
-
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Start a hatch shell — this will install all dependencies including dev in an isolated virtual environment:
|
|
81
|
+
|
|
82
|
+
```shell
|
|
83
|
+
hatch shell
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Set up the pre-commit Git hook (runs linters automatically before each commit):
|
|
87
|
+
|
|
88
|
+
```shell
|
|
74
89
|
pre-commit install
|
|
75
|
-
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
Run tests with coverage:
|
|
93
|
+
|
|
94
|
+
```shell
|
|
76
95
|
invoke coverage
|
|
77
96
|
```
|
|
78
97
|
|
|
@@ -1,21 +1,21 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@openzim/wombat-setup",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "5.3.
|
|
4
|
+
"version": "5.3.1-dev0",
|
|
5
5
|
"license": "GPL-3.0-or-later",
|
|
6
6
|
"author": "openZIM",
|
|
7
7
|
"devDependencies": {
|
|
8
|
-
"@rollup/plugin-commonjs": "
|
|
9
|
-
"@rollup/plugin-node-resolve": "16.0.
|
|
10
|
-
"@rollup/plugin-strip": "
|
|
11
|
-
"@rollup/plugin-terser": "0.
|
|
12
|
-
"ava": "
|
|
13
|
-
"babel-eslint": "
|
|
14
|
-
"eslint": "
|
|
15
|
-
"eslint-config-prettier": "10.
|
|
16
|
-
"prettier": "3.
|
|
17
|
-
"rollup": "4.
|
|
18
|
-
"rollup-plugin-version-injector": "
|
|
8
|
+
"@rollup/plugin-commonjs": "29.0.2",
|
|
9
|
+
"@rollup/plugin-node-resolve": "16.0.3",
|
|
10
|
+
"@rollup/plugin-strip": "3.0.4",
|
|
11
|
+
"@rollup/plugin-terser": "1.0.0",
|
|
12
|
+
"ava": "8.0.1",
|
|
13
|
+
"babel-eslint": "10.1.0",
|
|
14
|
+
"eslint": "10.4.0",
|
|
15
|
+
"eslint-config-prettier": "10.1.8",
|
|
16
|
+
"prettier": "3.8.3",
|
|
17
|
+
"rollup": "4.60.4",
|
|
18
|
+
"rollup-plugin-version-injector": "1.3.3"
|
|
19
19
|
},
|
|
20
20
|
"scripts": {
|
|
21
21
|
"prettier-check": "prettier . --check",
|
|
@@ -41,9 +41,9 @@
|
|
|
41
41
|
]
|
|
42
42
|
},
|
|
43
43
|
"dependencies": {
|
|
44
|
-
"uri-js": "
|
|
44
|
+
"uri-js": "4.4.1"
|
|
45
45
|
},
|
|
46
46
|
"engines": {
|
|
47
|
-
"node": ">=
|
|
47
|
+
"node": ">=26.0.0 <27.0.0"
|
|
48
48
|
}
|
|
49
49
|
}
|
|
@@ -292,14 +292,21 @@ export function getWombatInfo(
|
|
|
292
292
|
// The host of the original url
|
|
293
293
|
wombat_host: orig_host,
|
|
294
294
|
|
|
295
|
-
//
|
|
296
|
-
|
|
295
|
+
// We are not running inside a service worker, wombat needs to know about it since
|
|
296
|
+
// some "magic" URLs like blobs are not available
|
|
297
|
+
isSW: false,
|
|
297
298
|
|
|
298
|
-
//
|
|
299
|
-
enable_auto_fetch: true,
|
|
299
|
+
// Convert all post request to get request
|
|
300
300
|
convert_post_to_get: true,
|
|
301
|
+
|
|
302
|
+
// Not used, we are not replaying in a frame
|
|
301
303
|
target_frame: '___wb_replay_top_frame',
|
|
302
|
-
|
|
304
|
+
|
|
305
|
+
// Not used, we are not running in live mode
|
|
306
|
+
enable_auto_fetch: false,
|
|
307
|
+
|
|
308
|
+
// Extra options, not used
|
|
309
|
+
wombat_opts: {},
|
|
303
310
|
};
|
|
304
311
|
}
|
|
305
312
|
|
|
@@ -21,8 +21,8 @@ test('nominalWbInfo', (t) => {
|
|
|
21
21
|
);
|
|
22
22
|
t.is(wmInfo.coll, '');
|
|
23
23
|
t.is(wmInfo.convert_post_to_get, true);
|
|
24
|
-
t.is(wmInfo.enable_auto_fetch,
|
|
25
|
-
t.is(wmInfo.isSW,
|
|
24
|
+
t.is(wmInfo.enable_auto_fetch, false);
|
|
25
|
+
t.is(wmInfo.isSW, false);
|
|
26
26
|
t.is(wmInfo.is_framed, false);
|
|
27
27
|
t.is(wmInfo.is_live, false);
|
|
28
28
|
t.is(wmInfo.mod, '');
|