zimscraperlib 5.2.0__tar.gz → 5.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/.pre-commit-config.yaml +5 -8
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/.readthedocs.yaml +1 -1
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/CHANGELOG.md +34 -0
- zimscraperlib-5.4.0/PKG-INFO +193 -0
- zimscraperlib-5.4.0/README.md +113 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/javascript/package.json +14 -14
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/javascript/src/wombatSetup.js +12 -5
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/javascript/test/wombatSetup.js +2 -2
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/javascript/yarn.lock +694 -1036
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/mkdocs.yml +1 -1
- zimscraperlib-5.4.0/openzim.toml +20 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/pyproject.toml +39 -53
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/rules/rules.yaml +2 -2
- zimscraperlib-5.4.0/src/zimscraperlib/__about__.py +1 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/download.py +24 -14
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/html.py +3 -4
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/i18n.py +10 -9
- zimscraperlib-5.4.0/src/zimscraperlib/image/optimization.py +475 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/image/probing.py +9 -1
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/misc.py +1 -5
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/rewriting/css.py +9 -21
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/rewriting/js.py +57 -6
- zimscraperlib-5.4.0/src/zimscraperlib/rewriting/statics/wombat.js +21 -0
- zimscraperlib-5.4.0/src/zimscraperlib/rewriting/statics/wombatSetup.js +3 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/video/encoding.py +0 -1
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/zim/creator.py +29 -6
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/zim/indexing.py +9 -4
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/zim/metadata.py +0 -2
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tasks.py +35 -39
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/download/test_download.py +22 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/image/test_image.py +243 -42
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/rewriting/conftest.py +12 -12
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/rewriting/test_html_rewriting.py +1 -3
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/rewriting/test_js_rewriting.py +9 -11
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/rewriting/test_url_rewriting.py +2 -6
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/rewriting/utils.py +0 -1
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/zim/test_dedup.py +1 -1
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/zim/test_fs.py +2 -2
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/zim/test_indexing.py +4 -2
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/zim/test_metadata.py +1 -3
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/zim/test_zim_creator.py +62 -2
- zimscraperlib-5.2.0/PKG-INFO +0 -167
- zimscraperlib-5.2.0/README.md +0 -87
- zimscraperlib-5.2.0/openzim.toml +0 -10
- zimscraperlib-5.2.0/src/zimscraperlib/__about__.py +0 -1
- zimscraperlib-5.2.0/src/zimscraperlib/image/optimization.py +0 -401
- zimscraperlib-5.2.0/src/zimscraperlib/rewriting/statics/wombat.js +0 -21
- zimscraperlib-5.2.0/src/zimscraperlib/rewriting/statics/wombatSetup.js +0 -3
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/.gitignore +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/LICENSE +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/contrib/README.md +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/contrib/encode_video.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/docs/assets/openzim.png +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/docs/functional_architecture.md +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/docs/index.md +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/docs/license.md +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/docs/scripts/generate_api_nav.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/docs/software_architecture.md +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/docs/technical_architecture.md +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/javascript/.prettierignore +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/javascript/.prettierrc.json +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/javascript/eslint.config.js +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/javascript/rollup.config.js +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/javascript/test/wombatUrlRewriting.js +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/rules/generate_rules.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/__init__.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/constants.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/filesystem.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/fix_ogvjs_dist.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/image/__init__.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/image/conversion.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/image/illustration.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/image/presets.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/image/transformation.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/image/utils.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/inputs.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/logging.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/py.typed +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/rewriting/__init__.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/rewriting/html.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/rewriting/rules.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/rewriting/rx_replacer.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/rewriting/statics/README.md +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/rewriting/statics/__wb_module_decl.js +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/rewriting/templates/head_insert.html +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/rewriting/url_rewriting.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/types.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/typing.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/uri.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/video/__init__.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/video/config.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/video/presets.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/video/probing.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/zim/__init__.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/zim/_libkiwix.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/zim/archive.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/zim/dedup.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/zim/filesystem.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/zim/items.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/src/zimscraperlib/zim/providers.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/conftest.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/DroidSans.ttf +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/audio.mp3 +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/blue.jpg +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/commons.png +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/commons48.png +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/encrypted.pdf +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/encrypted.txt +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/mail.gif +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/milderm.pdf +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/milderm.txt +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/ninja.webp +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/pluto.jpg +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/single_wave_icon.gbr +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/square.jpg +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/square.png +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/star.svg +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/video.mkv +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/video.mp4 +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/files/video.webm +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/filesystem/test_filesystem.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/html/conftest.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/html/test_html.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/i18n/test_i18n.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/image/test_illustration.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/inputs/test_inputs.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/logging/conftest.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/logging/test_logging.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/misc/test_misc.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/ogvjs/conftest.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/ogvjs/test_ogvjs.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/rewriting/__init__.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/rewriting/test_css_rewriting.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/rewriting/test_fuzzy_rules.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/rewriting/test_rx_replacer.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/test_constants.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/types/test_types.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/typing/test_typing.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/uri/test_uri.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/video/conftest.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/video/test_encoding.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/video/test_video.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/zim/conftest.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/zim/test_archive.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/tests/zim/test_libkiwix.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.4.0}/yarn.lock +0 -0
|
@@ -3,20 +3,17 @@
|
|
|
3
3
|
exclude: ^tests/files # these are raw test files, no need to mess with them
|
|
4
4
|
repos:
|
|
5
5
|
- repo: https://github.com/pre-commit/pre-commit-hooks
|
|
6
|
-
rev:
|
|
6
|
+
rev: v6.0.0
|
|
7
7
|
hooks:
|
|
8
8
|
- id: trailing-whitespace
|
|
9
9
|
- id: end-of-file-fixer
|
|
10
|
-
- repo: https://github.com/psf/black
|
|
11
|
-
rev: '25.9.0'
|
|
12
|
-
hooks:
|
|
13
|
-
- id: black
|
|
14
10
|
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
15
|
-
rev: v0.
|
|
11
|
+
rev: v0.15.14
|
|
16
12
|
hooks:
|
|
17
|
-
- id: ruff
|
|
13
|
+
- id: ruff-check
|
|
14
|
+
- id: ruff-format
|
|
18
15
|
- repo: https://github.com/RobertCraigie/pyright-python
|
|
19
|
-
rev: v1.1.
|
|
16
|
+
rev: v1.1.409
|
|
20
17
|
hooks:
|
|
21
18
|
- id: pyright
|
|
22
19
|
name: pyright (system)
|
|
@@ -5,6 +5,40 @@ All notable changes to this project are documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0).
|
|
7
7
|
|
|
8
|
+
## [5.4.0] - 2026-05-28
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- Add support for custom number of retries and user-agent in save_large_file (#278)
|
|
13
|
+
- Enhance save_large_file log level (#279)
|
|
14
|
+
- Extend image optimization to support in-memory streams(BytesIO/bytes) and dst_format param (#289)
|
|
15
|
+
- Automatically add \_ftindex:<yes/no> tag at creator start based on indexing configuration (#295)
|
|
16
|
+
- **BEHAVIOR CHANGE**: it is not possible anymore to add "Tags" metadata with Creator.add_metadata method after creator has started ; this is anyway not recommended anymore since 5.x (one should prefer Creator.config_metadata)
|
|
17
|
+
|
|
18
|
+
### Fixed
|
|
19
|
+
|
|
20
|
+
- Add proper typing @overload to `zimscraperlib.image.optimize_xxx` methods (#273)
|
|
21
|
+
- Backport wabac.JS change around rewriting: detect when JS is in 'strict' mode and avoid using 'arguments' (#286)
|
|
22
|
+
- Fix installation from source, which missed wombatSetup.js (#287)
|
|
23
|
+
- Fix outdated system dependencies in README: remove Pillow build-time deps (bundled in wheels), add missing `libcairo` across all platforms (#152)
|
|
24
|
+
- Improve contribution setup instructions: use `hatch shell`, clarify commands must be run from local clone root (#153)
|
|
25
|
+
- Bring coverage back to 100% (#293)
|
|
26
|
+
- Fix `Creator.config_indexing` docstring to reflect that only the full-text index is toggled; title indexing is always performed by libzim (#294)
|
|
27
|
+
- Fix `format_for` to not modify input BytesIO position (#296)
|
|
28
|
+
|
|
29
|
+
### Changed
|
|
30
|
+
|
|
31
|
+
- Upgrade to wombat 3.10.5 (#306)
|
|
32
|
+
- Adopt openzim/\_python-bootstrap 2.0.0 (#303)
|
|
33
|
+
- Upgrade Python dependencies, Node.JS runtime and Node.JS dependencies (#306)
|
|
34
|
+
|
|
35
|
+
## [5.3.0] - 2025-11-14
|
|
36
|
+
|
|
37
|
+
### Changed
|
|
38
|
+
|
|
39
|
+
- Upgrade to support only Python 3.14 (#266)
|
|
40
|
+
- Upgrade dependencies (#269)
|
|
41
|
+
|
|
8
42
|
## [5.2.0] - 2025-10-02
|
|
9
43
|
|
|
10
44
|
### Added
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: zimscraperlib
|
|
3
|
+
Version: 5.4.0
|
|
4
|
+
Summary: Collection of python tools to re-use common code across scrapers
|
|
5
|
+
Project-URL: Donate, https://www.kiwix.org/en/support-us/
|
|
6
|
+
Project-URL: Homepage, https://github.com/openzim/python-scraperlib
|
|
7
|
+
Author-email: openZIM <dev@openzim.org>
|
|
8
|
+
License: GPL-3.0-or-later
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: offline,openzim,zim
|
|
11
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
16
|
+
Requires-Python: <3.15,>=3.14
|
|
17
|
+
Requires-Dist: babel<3.0,>=2.9
|
|
18
|
+
Requires-Dist: beartype<0.23,>=0.19
|
|
19
|
+
Requires-Dist: beautifulsoup4<5.0,>=4.9.3
|
|
20
|
+
Requires-Dist: cairosvg<3.0,>=2.2.0
|
|
21
|
+
Requires-Dist: colorthief==0.2.1
|
|
22
|
+
Requires-Dist: idna<4.0,>=2.5
|
|
23
|
+
Requires-Dist: iso639-lang<3.0,>=2.4.0
|
|
24
|
+
Requires-Dist: libzim<4.0,>=3.8.0
|
|
25
|
+
Requires-Dist: lxml<7.0,>=4.6.3
|
|
26
|
+
Requires-Dist: optimize-images<2.0,>=1.3.6
|
|
27
|
+
Requires-Dist: piexif==1.1.3
|
|
28
|
+
Requires-Dist: pillow<13.0,>=7.0.0
|
|
29
|
+
Requires-Dist: pymupdf<2.0,>=1.24.0
|
|
30
|
+
Requires-Dist: python-magic<0.5,>=0.4.3
|
|
31
|
+
Requires-Dist: python-resize-image<1.2,>=1.1.19
|
|
32
|
+
Requires-Dist: regex>=2020.7.14
|
|
33
|
+
Requires-Dist: requests<3.0,>=2.25.1
|
|
34
|
+
Requires-Dist: types-xxhash<4.0,>=2.0
|
|
35
|
+
Requires-Dist: urllib3<2.8.0,>=1.26.5
|
|
36
|
+
Requires-Dist: xxhash<4.0,>=2.0
|
|
37
|
+
Requires-Dist: yt-dlp
|
|
38
|
+
Provides-Extra: dev
|
|
39
|
+
Requires-Dist: coverage==7.14.1; extra == 'dev'
|
|
40
|
+
Requires-Dist: invoke==3.0.3; extra == 'dev'
|
|
41
|
+
Requires-Dist: ipython==9.13.0; extra == 'dev'
|
|
42
|
+
Requires-Dist: jinja2==3.1.6; extra == 'dev'
|
|
43
|
+
Requires-Dist: mkdocs-gen-files==0.6.1; extra == 'dev'
|
|
44
|
+
Requires-Dist: mkdocs-include-markdown-plugin==7.3.0; extra == 'dev'
|
|
45
|
+
Requires-Dist: mkdocs-literate-nav==0.6.3; extra == 'dev'
|
|
46
|
+
Requires-Dist: mkdocs-material==9.7.6; extra == 'dev'
|
|
47
|
+
Requires-Dist: mkdocs==1.6.1; extra == 'dev'
|
|
48
|
+
Requires-Dist: mkdocstrings[python]==1.0.3; extra == 'dev'
|
|
49
|
+
Requires-Dist: pre-commit==4.6.0; extra == 'dev'
|
|
50
|
+
Requires-Dist: pygments==2.19.2; extra == 'dev'
|
|
51
|
+
Requires-Dist: pymdown-extensions==10.21.3; extra == 'dev'
|
|
52
|
+
Requires-Dist: pyright==1.1.409; extra == 'dev'
|
|
53
|
+
Requires-Dist: pytest-mock==3.15.1; extra == 'dev'
|
|
54
|
+
Requires-Dist: pytest==9.0.3; extra == 'dev'
|
|
55
|
+
Requires-Dist: pyyaml==6.0.3; extra == 'dev'
|
|
56
|
+
Requires-Dist: ruff==0.15.14; extra == 'dev'
|
|
57
|
+
Provides-Extra: docs
|
|
58
|
+
Requires-Dist: mkdocs-gen-files==0.6.1; extra == 'docs'
|
|
59
|
+
Requires-Dist: mkdocs-include-markdown-plugin==7.3.0; extra == 'docs'
|
|
60
|
+
Requires-Dist: mkdocs-literate-nav==0.6.3; extra == 'docs'
|
|
61
|
+
Requires-Dist: mkdocs-material==9.7.6; extra == 'docs'
|
|
62
|
+
Requires-Dist: mkdocs==1.6.1; extra == 'docs'
|
|
63
|
+
Requires-Dist: mkdocstrings[python]==1.0.3; extra == 'docs'
|
|
64
|
+
Requires-Dist: pygments==2.19.2; extra == 'docs'
|
|
65
|
+
Requires-Dist: pymdown-extensions==10.21.3; extra == 'docs'
|
|
66
|
+
Requires-Dist: ruff==0.15.14; extra == 'docs'
|
|
67
|
+
Provides-Extra: qa
|
|
68
|
+
Requires-Dist: pyright==1.1.409; extra == 'qa'
|
|
69
|
+
Requires-Dist: pytest==9.0.3; extra == 'qa'
|
|
70
|
+
Requires-Dist: ruff==0.15.14; extra == 'qa'
|
|
71
|
+
Provides-Extra: scripts
|
|
72
|
+
Requires-Dist: invoke==3.0.3; extra == 'scripts'
|
|
73
|
+
Requires-Dist: jinja2==3.1.6; extra == 'scripts'
|
|
74
|
+
Requires-Dist: pyyaml==6.0.3; extra == 'scripts'
|
|
75
|
+
Provides-Extra: test
|
|
76
|
+
Requires-Dist: coverage==7.14.1; extra == 'test'
|
|
77
|
+
Requires-Dist: pytest-mock==3.15.1; extra == 'test'
|
|
78
|
+
Requires-Dist: pytest==9.0.3; extra == 'test'
|
|
79
|
+
Description-Content-Type: text/markdown
|
|
80
|
+
|
|
81
|
+
# zimscraperlib
|
|
82
|
+
|
|
83
|
+
[](https://github.com/openzim/python-scraperlib/actions/workflows/QA.yaml)
|
|
84
|
+
[](https://github.com/openzim/python-scraperlib/actions/workflows/Tests.yaml)
|
|
85
|
+
[](https://www.codefactor.io/repository/github/openzim/python-scraperlib)
|
|
86
|
+
[](https://www.gnu.org/licenses/gpl-3.0)
|
|
87
|
+
[](https://pypi.org/project/zimscraperlib/)
|
|
88
|
+
[](https://pypi.org/project/zimscraperlib)
|
|
89
|
+
[](https://codecov.io/gh/openzim/python-scraperlib)
|
|
90
|
+
[](https://python-scraperlib.readthedocs.io/)
|
|
91
|
+
|
|
92
|
+
Collection of python code to re-use across python-based scrapers
|
|
93
|
+
|
|
94
|
+
# Usage
|
|
95
|
+
|
|
96
|
+
- This library is meant to be installed via PyPI ([`zimscraperlib`](https://pypi.org/project/zimscraperlib/)).
|
|
97
|
+
- Make sure to reference it using a version code as the API is subject to frequent changes.
|
|
98
|
+
- API should remain the same only within the same _minor_ version.
|
|
99
|
+
|
|
100
|
+
Example usage:
|
|
101
|
+
|
|
102
|
+
```pip
|
|
103
|
+
zimscraperlib>=1.1,<1.2
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
See documentation at [Read the Docs](https://python-scraperlib.readthedocs.io/) for details.
|
|
107
|
+
|
|
108
|
+
> [!WARNING]
|
|
109
|
+
> While this library brings support for downloading videos with yt-dlp, recent changes in Youtube have forced yt-dlp team
|
|
110
|
+
> to require new dependencies for youtube videos (see https://github.com/yt-dlp/yt-dlp/issues/15012). These dependencies
|
|
111
|
+
> are significantly big and not needed for all other backend supported by yt-dlp (only youtube needs it). These dependencies
|
|
112
|
+
> are hence not included in this library dependencies (yet, see https://github.com/openzim/python-scraperlib/issues/268),
|
|
113
|
+
> you have to install them on your own if you intend to download videos from Youtube.
|
|
114
|
+
|
|
115
|
+
# Dependencies
|
|
116
|
+
|
|
117
|
+
Most dependencies are installed automatically by pip (from PyPI by default). The following system packages may be required depending on which features you use:
|
|
118
|
+
|
|
119
|
+
- **libmagic** — required for file type detection (used in most scrapers)
|
|
120
|
+
- **wget** — required only for `zimscraperlib.download` functions
|
|
121
|
+
- **FFmpeg** — required only for video processing functions
|
|
122
|
+
- **gifsicle** (>=1.92) — required only for GIF optimization
|
|
123
|
+
- **libcairo** — required only for SVG-to-PNG conversion
|
|
124
|
+
- **libzim** — auto-installed via PyPI, not available on Windows
|
|
125
|
+
- **Pillow** — auto-installed via PyPI; pre-built wheels are used by default and no system image libraries are needed. Only if you need to build Pillow from source should you install additional system libraries — see [Pillow's build documentation](https://pillow.readthedocs.io/en/latest/installation/building-from-source.html) for details.
|
|
126
|
+
> **Note:** To run the full test suite, all system dependencies listed above must be installed.
|
|
127
|
+
|
|
128
|
+
## macOS
|
|
129
|
+
|
|
130
|
+
```sh
|
|
131
|
+
brew install libmagic wget ffmpeg gifsicle cairo
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
## Linux
|
|
135
|
+
|
|
136
|
+
```sh
|
|
137
|
+
sudo apt install libmagic1 wget ffmpeg gifsicle libcairo2
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
## Alpine
|
|
141
|
+
|
|
142
|
+
```sh
|
|
143
|
+
apk add ffmpeg gifsicle libmagic wget cairo
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
# Contribution
|
|
147
|
+
|
|
148
|
+
This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing).
|
|
149
|
+
|
|
150
|
+
This project has implemented openZIM's [Python bootstrap, conventions and policies](https://github.com/openzim/_python-bootstrap/docs/Policy.md) **v2.0.0**.
|
|
151
|
+
|
|
152
|
+
All instructions below must be run from the root of your local clone of this repository.
|
|
153
|
+
|
|
154
|
+
If you do not already have it on your system, install [hatch](https://hatch.pypa.io/latest/install/):
|
|
155
|
+
|
|
156
|
+
```shell
|
|
157
|
+
pip install hatch
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
Start a hatch shell — this will install all dependencies including dev in an isolated virtual environment:
|
|
161
|
+
|
|
162
|
+
```shell
|
|
163
|
+
hatch shell
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
Set up the pre-commit Git hook (runs linters automatically before each commit):
|
|
167
|
+
|
|
168
|
+
```shell
|
|
169
|
+
pre-commit install
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
Run tests with coverage:
|
|
173
|
+
|
|
174
|
+
```shell
|
|
175
|
+
invoke coverage
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
# Users
|
|
179
|
+
|
|
180
|
+
Non-exhaustive list of scrapers using it (check status when updating API):
|
|
181
|
+
|
|
182
|
+
- [openzim/freecodecamp](https://github.com/openzim/freecodecamp)
|
|
183
|
+
- [openzim/gutenberg](https://github.com/openzim/gutenberg)
|
|
184
|
+
- [openzim/ifixit](https://github.com/openzim/ifixit)
|
|
185
|
+
- [openzim/kolibri](https://github.com/openzim/kolibri)
|
|
186
|
+
- [openzim/nautilus](https://github.com/openzim/nautilus)
|
|
187
|
+
- [openzim/nautilus](https://github.com/openzim/nautilus)
|
|
188
|
+
- [openzim/openedx](https://github.com/openzim/openedx)
|
|
189
|
+
- [openzim/sotoki](https://github.com/openzim/sotoki)
|
|
190
|
+
- [openzim/ted](https://github.com/openzim/ted)
|
|
191
|
+
- [openzim/warc2zim](https://github.com/openzim/warc2zim)
|
|
192
|
+
- [openzim/wikihow](https://github.com/openzim/wikihow)
|
|
193
|
+
- [openzim/youtube](https://github.com/openzim/youtube)
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
# zimscraperlib
|
|
2
|
+
|
|
3
|
+
[](https://github.com/openzim/python-scraperlib/actions/workflows/QA.yaml)
|
|
4
|
+
[](https://github.com/openzim/python-scraperlib/actions/workflows/Tests.yaml)
|
|
5
|
+
[](https://www.codefactor.io/repository/github/openzim/python-scraperlib)
|
|
6
|
+
[](https://www.gnu.org/licenses/gpl-3.0)
|
|
7
|
+
[](https://pypi.org/project/zimscraperlib/)
|
|
8
|
+
[](https://pypi.org/project/zimscraperlib)
|
|
9
|
+
[](https://codecov.io/gh/openzim/python-scraperlib)
|
|
10
|
+
[](https://python-scraperlib.readthedocs.io/)
|
|
11
|
+
|
|
12
|
+
Collection of python code to re-use across python-based scrapers
|
|
13
|
+
|
|
14
|
+
# Usage
|
|
15
|
+
|
|
16
|
+
- This library is meant to be installed via PyPI ([`zimscraperlib`](https://pypi.org/project/zimscraperlib/)).
|
|
17
|
+
- Make sure to reference it using a version code as the API is subject to frequent changes.
|
|
18
|
+
- API should remain the same only within the same _minor_ version.
|
|
19
|
+
|
|
20
|
+
Example usage:
|
|
21
|
+
|
|
22
|
+
```pip
|
|
23
|
+
zimscraperlib>=1.1,<1.2
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
See documentation at [Read the Docs](https://python-scraperlib.readthedocs.io/) for details.
|
|
27
|
+
|
|
28
|
+
> [!WARNING]
|
|
29
|
+
> While this library brings support for downloading videos with yt-dlp, recent changes in Youtube have forced yt-dlp team
|
|
30
|
+
> to require new dependencies for youtube videos (see https://github.com/yt-dlp/yt-dlp/issues/15012). These dependencies
|
|
31
|
+
> are significantly big and not needed for all other backend supported by yt-dlp (only youtube needs it). These dependencies
|
|
32
|
+
> are hence not included in this library dependencies (yet, see https://github.com/openzim/python-scraperlib/issues/268),
|
|
33
|
+
> you have to install them on your own if you intend to download videos from Youtube.
|
|
34
|
+
|
|
35
|
+
# Dependencies
|
|
36
|
+
|
|
37
|
+
Most dependencies are installed automatically by pip (from PyPI by default). The following system packages may be required depending on which features you use:
|
|
38
|
+
|
|
39
|
+
- **libmagic** — required for file type detection (used in most scrapers)
|
|
40
|
+
- **wget** — required only for `zimscraperlib.download` functions
|
|
41
|
+
- **FFmpeg** — required only for video processing functions
|
|
42
|
+
- **gifsicle** (>=1.92) — required only for GIF optimization
|
|
43
|
+
- **libcairo** — required only for SVG-to-PNG conversion
|
|
44
|
+
- **libzim** — auto-installed via PyPI, not available on Windows
|
|
45
|
+
- **Pillow** — auto-installed via PyPI; pre-built wheels are used by default and no system image libraries are needed. Only if you need to build Pillow from source should you install additional system libraries — see [Pillow's build documentation](https://pillow.readthedocs.io/en/latest/installation/building-from-source.html) for details.
|
|
46
|
+
> **Note:** To run the full test suite, all system dependencies listed above must be installed.
|
|
47
|
+
|
|
48
|
+
## macOS
|
|
49
|
+
|
|
50
|
+
```sh
|
|
51
|
+
brew install libmagic wget ffmpeg gifsicle cairo
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## Linux
|
|
55
|
+
|
|
56
|
+
```sh
|
|
57
|
+
sudo apt install libmagic1 wget ffmpeg gifsicle libcairo2
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## Alpine
|
|
61
|
+
|
|
62
|
+
```sh
|
|
63
|
+
apk add ffmpeg gifsicle libmagic wget cairo
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
# Contribution
|
|
67
|
+
|
|
68
|
+
This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing).
|
|
69
|
+
|
|
70
|
+
This project has implemented openZIM's [Python bootstrap, conventions and policies](https://github.com/openzim/_python-bootstrap/docs/Policy.md) **v2.0.0**.
|
|
71
|
+
|
|
72
|
+
All instructions below must be run from the root of your local clone of this repository.
|
|
73
|
+
|
|
74
|
+
If you do not already have it on your system, install [hatch](https://hatch.pypa.io/latest/install/):
|
|
75
|
+
|
|
76
|
+
```shell
|
|
77
|
+
pip install hatch
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Start a hatch shell — this will install all dependencies including dev in an isolated virtual environment:
|
|
81
|
+
|
|
82
|
+
```shell
|
|
83
|
+
hatch shell
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Set up the pre-commit Git hook (runs linters automatically before each commit):
|
|
87
|
+
|
|
88
|
+
```shell
|
|
89
|
+
pre-commit install
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
Run tests with coverage:
|
|
93
|
+
|
|
94
|
+
```shell
|
|
95
|
+
invoke coverage
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
# Users
|
|
99
|
+
|
|
100
|
+
Non-exhaustive list of scrapers using it (check status when updating API):
|
|
101
|
+
|
|
102
|
+
- [openzim/freecodecamp](https://github.com/openzim/freecodecamp)
|
|
103
|
+
- [openzim/gutenberg](https://github.com/openzim/gutenberg)
|
|
104
|
+
- [openzim/ifixit](https://github.com/openzim/ifixit)
|
|
105
|
+
- [openzim/kolibri](https://github.com/openzim/kolibri)
|
|
106
|
+
- [openzim/nautilus](https://github.com/openzim/nautilus)
|
|
107
|
+
- [openzim/nautilus](https://github.com/openzim/nautilus)
|
|
108
|
+
- [openzim/openedx](https://github.com/openzim/openedx)
|
|
109
|
+
- [openzim/sotoki](https://github.com/openzim/sotoki)
|
|
110
|
+
- [openzim/ted](https://github.com/openzim/ted)
|
|
111
|
+
- [openzim/warc2zim](https://github.com/openzim/warc2zim)
|
|
112
|
+
- [openzim/wikihow](https://github.com/openzim/wikihow)
|
|
113
|
+
- [openzim/youtube](https://github.com/openzim/youtube)
|
|
@@ -1,21 +1,21 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@openzim/wombat-setup",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "5.
|
|
4
|
+
"version": "5.3.1-dev0",
|
|
5
5
|
"license": "GPL-3.0-or-later",
|
|
6
6
|
"author": "openZIM",
|
|
7
7
|
"devDependencies": {
|
|
8
|
-
"@rollup/plugin-commonjs": "
|
|
9
|
-
"@rollup/plugin-node-resolve": "16.0.
|
|
10
|
-
"@rollup/plugin-strip": "
|
|
11
|
-
"@rollup/plugin-terser": "0.
|
|
12
|
-
"ava": "
|
|
13
|
-
"babel-eslint": "
|
|
14
|
-
"eslint": "
|
|
15
|
-
"eslint-config-prettier": "10.
|
|
16
|
-
"prettier": "3.
|
|
17
|
-
"rollup": "4.
|
|
18
|
-
"rollup-plugin-version-injector": "
|
|
8
|
+
"@rollup/plugin-commonjs": "29.0.2",
|
|
9
|
+
"@rollup/plugin-node-resolve": "16.0.3",
|
|
10
|
+
"@rollup/plugin-strip": "3.0.4",
|
|
11
|
+
"@rollup/plugin-terser": "1.0.0",
|
|
12
|
+
"ava": "8.0.1",
|
|
13
|
+
"babel-eslint": "10.1.0",
|
|
14
|
+
"eslint": "10.4.0",
|
|
15
|
+
"eslint-config-prettier": "10.1.8",
|
|
16
|
+
"prettier": "3.8.3",
|
|
17
|
+
"rollup": "4.60.4",
|
|
18
|
+
"rollup-plugin-version-injector": "1.3.3"
|
|
19
19
|
},
|
|
20
20
|
"scripts": {
|
|
21
21
|
"prettier-check": "prettier . --check",
|
|
@@ -41,9 +41,9 @@
|
|
|
41
41
|
]
|
|
42
42
|
},
|
|
43
43
|
"dependencies": {
|
|
44
|
-
"uri-js": "
|
|
44
|
+
"uri-js": "4.4.1"
|
|
45
45
|
},
|
|
46
46
|
"engines": {
|
|
47
|
-
"node": ">=
|
|
47
|
+
"node": ">=26.0.0 <27.0.0"
|
|
48
48
|
}
|
|
49
49
|
}
|
|
@@ -292,14 +292,21 @@ export function getWombatInfo(
|
|
|
292
292
|
// The host of the original url
|
|
293
293
|
wombat_host: orig_host,
|
|
294
294
|
|
|
295
|
-
//
|
|
296
|
-
|
|
295
|
+
// We are not running inside a service worker, wombat needs to know about it since
|
|
296
|
+
// some "magic" URLs like blobs are not available
|
|
297
|
+
isSW: false,
|
|
297
298
|
|
|
298
|
-
//
|
|
299
|
-
enable_auto_fetch: true,
|
|
299
|
+
// Convert all post request to get request
|
|
300
300
|
convert_post_to_get: true,
|
|
301
|
+
|
|
302
|
+
// Not used, we are not replaying in a frame
|
|
301
303
|
target_frame: '___wb_replay_top_frame',
|
|
302
|
-
|
|
304
|
+
|
|
305
|
+
// Not used, we are not running in live mode
|
|
306
|
+
enable_auto_fetch: false,
|
|
307
|
+
|
|
308
|
+
// Extra options, not used
|
|
309
|
+
wombat_opts: {},
|
|
303
310
|
};
|
|
304
311
|
}
|
|
305
312
|
|
|
@@ -21,8 +21,8 @@ test('nominalWbInfo', (t) => {
|
|
|
21
21
|
);
|
|
22
22
|
t.is(wmInfo.coll, '');
|
|
23
23
|
t.is(wmInfo.convert_post_to_get, true);
|
|
24
|
-
t.is(wmInfo.enable_auto_fetch,
|
|
25
|
-
t.is(wmInfo.isSW,
|
|
24
|
+
t.is(wmInfo.enable_auto_fetch, false);
|
|
25
|
+
t.is(wmInfo.isSW, false);
|
|
26
26
|
t.is(wmInfo.is_framed, false);
|
|
27
27
|
t.is(wmInfo.is_live, false);
|
|
28
28
|
t.is(wmInfo.mod, '');
|