zimscraperlib 5.2.0__tar.gz → 5.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/.pre-commit-config.yaml +4 -4
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/.readthedocs.yaml +1 -1
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/CHANGELOG.md +7 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/PKG-INFO +36 -29
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/README.md +7 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/javascript/package.json +1 -1
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/pyproject.toml +21 -27
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/rules/rules.yaml +2 -2
- zimscraperlib-5.3.0/src/zimscraperlib/__about__.py +1 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/rewriting/js.py +2 -2
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/rewriting/statics/wombatSetup.js +1 -1
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/zim/indexing.py +9 -4
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/rewriting/test_js_rewriting.py +6 -0
- zimscraperlib-5.2.0/src/zimscraperlib/__about__.py +0 -1
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/.gitignore +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/LICENSE +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/contrib/README.md +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/contrib/encode_video.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/docs/assets/openzim.png +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/docs/functional_architecture.md +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/docs/index.md +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/docs/license.md +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/docs/scripts/generate_api_nav.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/docs/software_architecture.md +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/docs/technical_architecture.md +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/javascript/.prettierignore +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/javascript/.prettierrc.json +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/javascript/eslint.config.js +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/javascript/rollup.config.js +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/javascript/src/wombatSetup.js +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/javascript/test/wombatSetup.js +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/javascript/test/wombatUrlRewriting.js +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/javascript/yarn.lock +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/mkdocs.yml +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/openzim.toml +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/rules/generate_rules.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/__init__.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/constants.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/download.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/filesystem.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/fix_ogvjs_dist.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/html.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/i18n.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/image/__init__.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/image/conversion.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/image/illustration.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/image/optimization.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/image/presets.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/image/probing.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/image/transformation.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/image/utils.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/inputs.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/logging.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/misc.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/py.typed +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/rewriting/__init__.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/rewriting/css.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/rewriting/html.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/rewriting/rules.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/rewriting/rx_replacer.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/rewriting/statics/README.md +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/rewriting/statics/__wb_module_decl.js +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/rewriting/statics/wombat.js +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/rewriting/templates/head_insert.html +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/rewriting/url_rewriting.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/types.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/typing.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/uri.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/video/__init__.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/video/config.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/video/encoding.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/video/presets.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/video/probing.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/zim/__init__.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/zim/_libkiwix.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/zim/archive.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/zim/creator.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/zim/dedup.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/zim/filesystem.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/zim/items.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/zim/metadata.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/zim/providers.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tasks.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/conftest.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/download/test_download.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/files/DroidSans.ttf +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/files/audio.mp3 +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/files/blue.jpg +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/files/commons.png +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/files/commons48.png +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/files/encrypted.pdf +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/files/encrypted.txt +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/files/mail.gif +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/files/milderm.pdf +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/files/milderm.txt +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/files/ninja.webp +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/files/pluto.jpg +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/files/single_wave_icon.gbr +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/files/square.jpg +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/files/square.png +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/files/star.svg +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/files/video.mkv +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/files/video.mp4 +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/files/video.webm +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/filesystem/test_filesystem.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/html/conftest.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/html/test_html.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/i18n/test_i18n.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/image/test_illustration.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/image/test_image.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/inputs/test_inputs.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/logging/conftest.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/logging/test_logging.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/misc/test_misc.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/ogvjs/conftest.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/ogvjs/test_ogvjs.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/rewriting/__init__.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/rewriting/conftest.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/rewriting/test_css_rewriting.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/rewriting/test_fuzzy_rules.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/rewriting/test_html_rewriting.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/rewriting/test_rx_replacer.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/rewriting/test_url_rewriting.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/rewriting/utils.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/test_constants.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/types/test_types.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/typing/test_typing.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/uri/test_uri.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/video/conftest.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/video/test_encoding.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/video/test_video.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/zim/conftest.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/zim/test_archive.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/zim/test_dedup.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/zim/test_fs.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/zim/test_indexing.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/zim/test_libkiwix.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/zim/test_metadata.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/tests/zim/test_zim_creator.py +0 -0
- {zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/yarn.lock +0 -0
|
@@ -3,20 +3,20 @@
|
|
|
3
3
|
exclude: ^tests/files # these are raw test files, no need to mess with them
|
|
4
4
|
repos:
|
|
5
5
|
- repo: https://github.com/pre-commit/pre-commit-hooks
|
|
6
|
-
rev:
|
|
6
|
+
rev: v6.0.0
|
|
7
7
|
hooks:
|
|
8
8
|
- id: trailing-whitespace
|
|
9
9
|
- id: end-of-file-fixer
|
|
10
10
|
- repo: https://github.com/psf/black
|
|
11
|
-
rev: '25.
|
|
11
|
+
rev: '25.11.0'
|
|
12
12
|
hooks:
|
|
13
13
|
- id: black
|
|
14
14
|
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
15
|
-
rev: v0.
|
|
15
|
+
rev: v0.14.5
|
|
16
16
|
hooks:
|
|
17
17
|
- id: ruff
|
|
18
18
|
- repo: https://github.com/RobertCraigie/pyright-python
|
|
19
|
-
rev: v1.1.
|
|
19
|
+
rev: v1.1.407
|
|
20
20
|
hooks:
|
|
21
21
|
- id: pyright
|
|
22
22
|
name: pyright (system)
|
|
@@ -5,6 +5,13 @@ All notable changes to this project are documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0).
|
|
7
7
|
|
|
8
|
+
## [5.3.0] - 2025-11-14
|
|
9
|
+
|
|
10
|
+
### Changed
|
|
11
|
+
|
|
12
|
+
- Upgrade to support only Python 3.14 (#266)
|
|
13
|
+
- Upgrade dependencies (#269)
|
|
14
|
+
|
|
8
15
|
## [5.2.0] - 2025-10-02
|
|
9
16
|
|
|
10
17
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: zimscraperlib
|
|
3
|
-
Version: 5.
|
|
3
|
+
Version: 5.3.0
|
|
4
4
|
Summary: Collection of python tools to re-use common code across scrapers
|
|
5
5
|
Project-URL: Donate, https://www.kiwix.org/en/support-us/
|
|
6
6
|
Project-URL: Homepage, https://github.com/openzim/python-scraperlib
|
|
@@ -12,20 +12,20 @@ Classifier: Development Status :: 5 - Production/Stable
|
|
|
12
12
|
Classifier: Intended Audience :: Developers
|
|
13
13
|
Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
|
|
14
14
|
Classifier: Programming Language :: Python :: 3
|
|
15
|
-
Classifier: Programming Language :: Python :: 3.
|
|
16
|
-
Requires-Python: <3.
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
16
|
+
Requires-Python: <3.15,>=3.14
|
|
17
17
|
Requires-Dist: babel<3.0,>=2.9
|
|
18
|
-
Requires-Dist: beartype<0.
|
|
18
|
+
Requires-Dist: beartype<0.23,>=0.19
|
|
19
19
|
Requires-Dist: beautifulsoup4<5.0,>=4.9.3
|
|
20
20
|
Requires-Dist: cairosvg<3.0,>=2.2.0
|
|
21
21
|
Requires-Dist: colorthief==0.2.1
|
|
22
22
|
Requires-Dist: idna<4.0,>=2.5
|
|
23
23
|
Requires-Dist: iso639-lang<3.0,>=2.4.0
|
|
24
|
-
Requires-Dist: libzim<4.0,>=3.
|
|
24
|
+
Requires-Dist: libzim<4.0,>=3.8.0
|
|
25
25
|
Requires-Dist: lxml<7.0,>=4.6.3
|
|
26
26
|
Requires-Dist: optimize-images<2.0,>=1.3.6
|
|
27
27
|
Requires-Dist: piexif==1.1.3
|
|
28
|
-
Requires-Dist: pillow<
|
|
28
|
+
Requires-Dist: pillow<13.0,>=7.0.0
|
|
29
29
|
Requires-Dist: pymupdf<2.0,>=1.24.0
|
|
30
30
|
Requires-Dist: python-magic<0.5,>=0.4.3
|
|
31
31
|
Requires-Dist: python-resize-image<1.2,>=1.1.19
|
|
@@ -36,46 +36,46 @@ Requires-Dist: urllib3<2.6.0,>=1.26.5
|
|
|
36
36
|
Requires-Dist: xxhash<4.0,>=2.0
|
|
37
37
|
Requires-Dist: yt-dlp
|
|
38
38
|
Provides-Extra: check
|
|
39
|
-
Requires-Dist: pyright==1.1.
|
|
40
|
-
Requires-Dist: pytest==
|
|
39
|
+
Requires-Dist: pyright==1.1.407; extra == 'check'
|
|
40
|
+
Requires-Dist: pytest==9.0.1; extra == 'check'
|
|
41
41
|
Provides-Extra: dev
|
|
42
|
-
Requires-Dist: black==25.
|
|
43
|
-
Requires-Dist: coverage==7.
|
|
44
|
-
Requires-Dist: invoke==2.2.
|
|
45
|
-
Requires-Dist: ipython==9.
|
|
42
|
+
Requires-Dist: black==25.11.0; extra == 'dev'
|
|
43
|
+
Requires-Dist: coverage==7.11.3; extra == 'dev'
|
|
44
|
+
Requires-Dist: invoke==2.2.1; extra == 'dev'
|
|
45
|
+
Requires-Dist: ipython==9.7.0; extra == 'dev'
|
|
46
46
|
Requires-Dist: jinja2==3.1.6; extra == 'dev'
|
|
47
47
|
Requires-Dist: mkdocs-gen-files==0.5.0; extra == 'dev'
|
|
48
|
-
Requires-Dist: mkdocs-include-markdown-plugin==7.
|
|
48
|
+
Requires-Dist: mkdocs-include-markdown-plugin==7.2.0; extra == 'dev'
|
|
49
49
|
Requires-Dist: mkdocs-literate-nav==0.6.2; extra == 'dev'
|
|
50
|
-
Requires-Dist: mkdocs-material==9.
|
|
50
|
+
Requires-Dist: mkdocs-material==9.7.0; extra == 'dev'
|
|
51
51
|
Requires-Dist: mkdocs==1.6.1; extra == 'dev'
|
|
52
52
|
Requires-Dist: mkdocstrings[python]==0.30.1; extra == 'dev'
|
|
53
|
-
Requires-Dist: pre-commit==4.
|
|
54
|
-
Requires-Dist: pymdown-extensions==10.
|
|
55
|
-
Requires-Dist: pyright==1.1.
|
|
53
|
+
Requires-Dist: pre-commit==4.4.0; extra == 'dev'
|
|
54
|
+
Requires-Dist: pymdown-extensions==10.17.1; extra == 'dev'
|
|
55
|
+
Requires-Dist: pyright==1.1.407; extra == 'dev'
|
|
56
56
|
Requires-Dist: pytest-mock==3.15.1; extra == 'dev'
|
|
57
|
-
Requires-Dist: pytest==
|
|
58
|
-
Requires-Dist: pyyaml==6.0.
|
|
59
|
-
Requires-Dist: ruff==0.
|
|
57
|
+
Requires-Dist: pytest==9.0.1; extra == 'dev'
|
|
58
|
+
Requires-Dist: pyyaml==6.0.3; extra == 'dev'
|
|
59
|
+
Requires-Dist: ruff==0.14.5; extra == 'dev'
|
|
60
60
|
Provides-Extra: docs
|
|
61
61
|
Requires-Dist: mkdocs-gen-files==0.5.0; extra == 'docs'
|
|
62
|
-
Requires-Dist: mkdocs-include-markdown-plugin==7.
|
|
62
|
+
Requires-Dist: mkdocs-include-markdown-plugin==7.2.0; extra == 'docs'
|
|
63
63
|
Requires-Dist: mkdocs-literate-nav==0.6.2; extra == 'docs'
|
|
64
|
-
Requires-Dist: mkdocs-material==9.
|
|
64
|
+
Requires-Dist: mkdocs-material==9.7.0; extra == 'docs'
|
|
65
65
|
Requires-Dist: mkdocs==1.6.1; extra == 'docs'
|
|
66
66
|
Requires-Dist: mkdocstrings[python]==0.30.1; extra == 'docs'
|
|
67
|
-
Requires-Dist: pymdown-extensions==10.
|
|
67
|
+
Requires-Dist: pymdown-extensions==10.17.1; extra == 'docs'
|
|
68
68
|
Provides-Extra: lint
|
|
69
|
-
Requires-Dist: black==25.
|
|
70
|
-
Requires-Dist: ruff==0.
|
|
69
|
+
Requires-Dist: black==25.11.0; extra == 'lint'
|
|
70
|
+
Requires-Dist: ruff==0.14.5; extra == 'lint'
|
|
71
71
|
Provides-Extra: scripts
|
|
72
|
-
Requires-Dist: invoke==2.2.
|
|
72
|
+
Requires-Dist: invoke==2.2.1; extra == 'scripts'
|
|
73
73
|
Requires-Dist: jinja2==3.1.6; extra == 'scripts'
|
|
74
|
-
Requires-Dist: pyyaml==6.0.
|
|
74
|
+
Requires-Dist: pyyaml==6.0.3; extra == 'scripts'
|
|
75
75
|
Provides-Extra: test
|
|
76
|
-
Requires-Dist: coverage==7.
|
|
76
|
+
Requires-Dist: coverage==7.11.3; extra == 'test'
|
|
77
77
|
Requires-Dist: pytest-mock==3.15.1; extra == 'test'
|
|
78
|
-
Requires-Dist: pytest==
|
|
78
|
+
Requires-Dist: pytest==9.0.1; extra == 'test'
|
|
79
79
|
Description-Content-Type: text/markdown
|
|
80
80
|
|
|
81
81
|
# zimscraperlib
|
|
@@ -104,6 +104,13 @@ zimscraperlib>=1.1,<1.2
|
|
|
104
104
|
|
|
105
105
|
See documentation at [Read the Docs](https://python-scraperlib.readthedocs.io/) for details.
|
|
106
106
|
|
|
107
|
+
> [!WARNING]
|
|
108
|
+
> While this library brings support for downloading videos with yt-dlp, recent changes in Youtube have forced yt-dlp team
|
|
109
|
+
> to require new dependencies for youtube videos (see https://github.com/yt-dlp/yt-dlp/issues/15012). These dependencies
|
|
110
|
+
> are significantly big and not needed for all other backend supported by yt-dlp (only youtube needs it). These dependencies
|
|
111
|
+
> are hence not included in this library dependencies (yet, see https://github.com/openzim/python-scraperlib/issues/268),
|
|
112
|
+
> you have to install them on your own if you intend to download videos from Youtube.
|
|
113
|
+
|
|
107
114
|
# Dependencies
|
|
108
115
|
|
|
109
116
|
- libmagic
|
|
@@ -24,6 +24,13 @@ zimscraperlib>=1.1,<1.2
|
|
|
24
24
|
|
|
25
25
|
See documentation at [Read the Docs](https://python-scraperlib.readthedocs.io/) for details.
|
|
26
26
|
|
|
27
|
+
> [!WARNING]
|
|
28
|
+
> While this library brings support for downloading videos with yt-dlp, recent changes in Youtube have forced yt-dlp team
|
|
29
|
+
> to require new dependencies for youtube videos (see https://github.com/yt-dlp/yt-dlp/issues/15012). These dependencies
|
|
30
|
+
> are significantly big and not needed for all other backend supported by yt-dlp (only youtube needs it). These dependencies
|
|
31
|
+
> are hence not included in this library dependencies (yet, see https://github.com/openzim/python-scraperlib/issues/268),
|
|
32
|
+
> you have to install them on your own if you intend to download videos from Youtube.
|
|
33
|
+
|
|
27
34
|
# Dependencies
|
|
28
35
|
|
|
29
36
|
- libmagic
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
[build-system]
|
|
2
2
|
# jinja2 is required to generate JS and Python rules at build time
|
|
3
3
|
# PyYAML is used to parse fuzzy rules and generate Python/JS code
|
|
4
|
-
requires = ["hatchling", "hatch-openzim>=0.2", "jinja2==3.1.6", "PyYAML==6.0.
|
|
4
|
+
requires = ["hatchling", "hatch-openzim>=0.2", "jinja2==3.1.6", "PyYAML==6.0.3"]
|
|
5
5
|
build-backend = "hatchling.build"
|
|
6
6
|
|
|
7
7
|
[project]
|
|
8
8
|
name = "zimscraperlib"
|
|
9
|
-
requires-python = ">=3.
|
|
9
|
+
requires-python = ">=3.14,<3.15"
|
|
10
10
|
description = "Collection of python tools to re-use common code across scrapers"
|
|
11
11
|
readme = "README.md"
|
|
12
12
|
dependencies = [
|
|
@@ -16,7 +16,7 @@ dependencies = [
|
|
|
16
16
|
"python-resize-image>=1.1.19,<1.2",
|
|
17
17
|
"Babel>=2.9,<3.0",
|
|
18
18
|
"python-magic>=0.4.3,<0.5",
|
|
19
|
-
"libzim>=3.
|
|
19
|
+
"libzim>=3.8.0,<4.0",
|
|
20
20
|
"beautifulsoup4>=4.9.3,<5.0",
|
|
21
21
|
"lxml>=4.6.3,<7.0",
|
|
22
22
|
"optimize-images>=1.3.6,<2.0",
|
|
@@ -26,10 +26,10 @@ dependencies = [
|
|
|
26
26
|
"regex>=2020.7.14",
|
|
27
27
|
"pymupdf>=1.24.0,<2.0",
|
|
28
28
|
"CairoSVG>=2.2.0,<3.0",
|
|
29
|
-
"beartype>=0.19,<0.
|
|
29
|
+
"beartype>=0.19,<0.23",
|
|
30
30
|
# youtube-dl should be updated as frequently as possible
|
|
31
31
|
"yt-dlp",
|
|
32
|
-
"pillow>=7.0.0,<
|
|
32
|
+
"pillow>=7.0.0,<13.0",
|
|
33
33
|
"urllib3>=1.26.5,<2.6.0",
|
|
34
34
|
"piexif==1.1.3", # this dep is a nightmare in terms of release management, better pinned just like in optimize-images anyway
|
|
35
35
|
"idna>=2.5,<4.0",
|
|
@@ -49,39 +49,39 @@ additional-classifiers = [
|
|
|
49
49
|
|
|
50
50
|
[project.optional-dependencies]
|
|
51
51
|
scripts = [
|
|
52
|
-
"invoke==2.2.
|
|
52
|
+
"invoke==2.2.1",
|
|
53
53
|
# jinja2 is required to generate JS and Python rules at build time
|
|
54
54
|
# PyYAML is used to parse fuzzy rules and generate Python/JS code
|
|
55
55
|
# also update version in build-system above
|
|
56
56
|
"jinja2==3.1.6",
|
|
57
|
-
"PyYAML==6.0.
|
|
57
|
+
"PyYAML==6.0.3",
|
|
58
58
|
|
|
59
59
|
]
|
|
60
60
|
lint = [
|
|
61
|
-
"black==25.
|
|
62
|
-
"ruff==0.
|
|
61
|
+
"black==25.11.0",
|
|
62
|
+
"ruff==0.14.5",
|
|
63
63
|
]
|
|
64
64
|
check = [
|
|
65
|
-
"pyright==1.1.
|
|
66
|
-
"pytest==
|
|
65
|
+
"pyright==1.1.407",
|
|
66
|
+
"pytest==9.0.1",
|
|
67
67
|
]
|
|
68
68
|
test = [
|
|
69
|
-
"pytest==
|
|
69
|
+
"pytest==9.0.1",
|
|
70
70
|
"pytest-mock==3.15.1",
|
|
71
|
-
"coverage==7.
|
|
71
|
+
"coverage==7.11.3",
|
|
72
72
|
]
|
|
73
73
|
docs = [
|
|
74
74
|
"mkdocs==1.6.1",
|
|
75
|
-
"mkdocs-include-markdown-plugin==7.
|
|
76
|
-
"mkdocs-material==9.
|
|
75
|
+
"mkdocs-include-markdown-plugin==7.2.0",
|
|
76
|
+
"mkdocs-material==9.7.0",
|
|
77
77
|
"mkdocstrings[python]==0.30.1",
|
|
78
|
-
"pymdown-extensions==10.
|
|
78
|
+
"pymdown-extensions==10.17.1",
|
|
79
79
|
"mkdocs-gen-files==0.5.0",
|
|
80
80
|
"mkdocs-literate-nav==0.6.2",
|
|
81
81
|
]
|
|
82
82
|
dev = [
|
|
83
|
-
"ipython==9.
|
|
84
|
-
"pre-commit==4.
|
|
83
|
+
"ipython==9.7.0",
|
|
84
|
+
"pre-commit==4.4.0",
|
|
85
85
|
"zimscraperlib[scripts]",
|
|
86
86
|
"zimscraperlib[lint]",
|
|
87
87
|
"zimscraperlib[test]",
|
|
@@ -157,10 +157,10 @@ build = "inv docs-build --args '{args}'"
|
|
|
157
157
|
|
|
158
158
|
[tool.black]
|
|
159
159
|
line-length = 88
|
|
160
|
-
target-version = ['
|
|
160
|
+
target-version = ['py314']
|
|
161
161
|
|
|
162
162
|
[tool.ruff]
|
|
163
|
-
target-version = "
|
|
163
|
+
target-version = "py314"
|
|
164
164
|
line-length = 88
|
|
165
165
|
src = ["src", "contrib"]
|
|
166
166
|
|
|
@@ -293,12 +293,6 @@ exclude_lines = [
|
|
|
293
293
|
include = ["contrib", "src", "tests", "tasks.py"]
|
|
294
294
|
exclude = [".env/**", ".venv/**"]
|
|
295
295
|
extraPaths = ["src"]
|
|
296
|
-
pythonVersion = "3.
|
|
296
|
+
pythonVersion = "3.14"
|
|
297
297
|
typeCheckingMode="strict"
|
|
298
298
|
disableBytesTypePromotions = true
|
|
299
|
-
|
|
300
|
-
[[tool.pyright.overrides.files]]
|
|
301
|
-
files = [
|
|
302
|
-
"src/zimscraperlib/rewriting**/*.py",
|
|
303
|
-
"tests/rewriting/**/*.py"
|
|
304
|
-
]
|
|
@@ -6,8 +6,8 @@
|
|
|
6
6
|
# ones) but just rewriting to proper path.
|
|
7
7
|
#
|
|
8
8
|
# This file is in sync with content at commit
|
|
9
|
-
# https://github.com/webrecorder/wabac.js/commit/
|
|
10
|
-
# from October
|
|
9
|
+
# https://github.com/webrecorder/wabac.js/commit/f62756661d06e721bc57ff25199c73ce51227916
|
|
10
|
+
# from October 29, 2025
|
|
11
11
|
#
|
|
12
12
|
# This file should be updated at every release of scraperlib
|
|
13
13
|
#
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "5.3.0"
|
|
@@ -13,8 +13,8 @@ it appropriately
|
|
|
13
13
|
ZIM at `_zim_static/__wb_module_decl.js`
|
|
14
14
|
|
|
15
15
|
This code is based on https://github.com/webrecorder/wabac.js/blob/main/src/rewrite/jsrewriter.ts
|
|
16
|
-
Last backport of upstream changes is from
|
|
17
|
-
Commit
|
|
16
|
+
Last backport of upstream changes is from Oct 12, 2025
|
|
17
|
+
Commit 1849552c3dbcbc065c05afac2dd80061db37b64d
|
|
18
18
|
"""
|
|
19
19
|
|
|
20
20
|
import re
|
{zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/rewriting/statics/wombatSetup.js
RENAMED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
/* Version: 5.
|
|
1
|
+
/* Version: 5.3.0 - November 14, 2025 12:43:09 */
|
|
2
2
|
var wombatSetup=function(e){const r=[{match:".*googlevideo.com/(videoplayback(?=\\?)).*[?&](id=[^&]+).*",replace:"youtube.fuzzy.replayweb.page/$1?$2"},{match:"(?:www\\.)?youtube(?:-nocookie)?\\.com/(get_video_info\\?).*(video_id=[^&]+).*",replace:"youtube.fuzzy.replayweb.page/$1$2"},{match:"i\\.ytimg\\.com\\/vi\\/(.*?)\\/.*?\\.(\\w*?)(?:\\?.*|$)",replace:"i.ytimg.com.fuzzy.replayweb.page/vi/$1/thumbnail.$2"},{match:"([^?]+)\\?[\\d]+$",replace:"$1"},{match:"(?:www\\.)?youtube(?:-nocookie)?\\.com\\/(youtubei\\/[^?]+).*(videoId[^&]+).*",replace:"youtube.fuzzy.replayweb.page/$1?$2"},{match:"(?:www\\.)?youtube(?:-nocookie)?\\.com/embed/([^?]+).*",replace:"youtube.fuzzy.replayweb.page/embed/$1"},{match:".*(?:gcs-vimeo|vod|vod-progressive|vod-adaptive)\\.akamaized\\.net.*\\/(.+?.mp4)\\?.*range=(.*?)(?:&.*|$)",replace:"vimeo-cdn.fuzzy.replayweb.page/$1?range=$2"},{match:".*(?:gcs-vimeo|vod|vod-progressive)\\.akamaized\\.net.*?\\/([\\d/]+.mp4)$",replace:"vimeo-cdn.fuzzy.replayweb.page/$1"},{match:".*player.vimeo.com\\/(video\\/[\\d]+)\\?.*",replace:"vimeo.fuzzy.replayweb.page/$1"},{match:".*i\\.vimeocdn\\.com\\/(.*)\\?.*",replace:"i.vimeocdn.fuzzy.replayweb.page/$1"},{match:"cheatography\\.com\\/scripts\\/(.*).js.*[?&](v=[^&]+).*",replace:"cheatography.com.fuzzy.replayweb.page/scripts/$1.js?$2"},{match:"blogger.googleusercontent.com\\/img\\/(.*\\.jpg)=.*",replace:"blogger.googleusercontent.com.fuzzy.replayweb.page/img/$1.resized"},{match:"(iranwire\\.com\\/questions\\/detail\\/.*)\\?.*",replace:"$1"}];function t(e){return e&&e.__esModule&&Object.prototype.hasOwnProperty.call(e,"default")?e.default:e}var o,n={exports:{}};
|
|
3
3
|
/** @license URI.js v4.4.1 (c) 2011 Gary Court. License: http://github.com/garycourt/uri-js */var a=(o||(o=1,function(e,r){!function(e){function r(){for(var e=arguments.length,r=Array(e),t=0;t<e;t++)r[t]=arguments[t];if(r.length>1){r[0]=r[0].slice(0,-1);for(var o=r.length-1,n=1;n<o;++n)r[n]=r[n].slice(1,-1);return r[o]=r[o].slice(1),r.join("")}return r[0]}function t(e){return"(?:"+e+")"}function o(e){return void 0===e?"undefined":null===e?"null":Object.prototype.toString.call(e).split(" ").pop().split("]").shift().toLowerCase()}function n(e){return e.toUpperCase()}function a(e){return null!=e?e instanceof Array?e:"number"!=typeof e.length||e.split||e.setInterval||e.call?[e]:Array.prototype.slice.call(e):[]}function i(e,r){var t=e;if(r)for(var o in r)t[o]=r[o];return t}function s(e){var o="[A-Za-z]",n="[0-9]",a=r(n,"[A-Fa-f]"),i=t(t("%[EFef]"+a+"%"+a+a+"%"+a+a)+"|"+t("%[89A-Fa-f]"+a+"%"+a+a)+"|"+t("%"+a+a)),s="[\\!\\$\\&\\'\\(\\)\\*\\+\\,\\;\\=]",c=r("[\\:\\/\\?\\#\\[\\]\\@]",s),u=e?"[\\uE000-\\uF8FF]":"[]",p=r(o,n,"[\\-\\.\\_\\~]",e?"[\\xA0-\\u200D\\u2010-\\u2029\\u202F-\\uD7FF\\uF900-\\uFDCF\\uFDF0-\\uFFEF]":"[]");t(o+r(o,n,"[\\+\\-\\.]")+"*"),t(t(i+"|"+r(p,s,"[\\:]"))+"*");var l=t(t("25[0-5]")+"|"+t("2[0-4]"+n)+"|"+t("1"+n+n)+"|"+t("0?[1-9]"+n)+"|0?0?"+n),h=t(l+"\\."+l+"\\."+l+"\\."+l),f=t(a+"{1,4}"),v=t(t(f+"\\:"+f)+"|"+h),d=t(t(f+"\\:")+"{6}"+v),m=t("\\:\\:"+t(f+"\\:")+"{5}"+v),g=t(t(f)+"?\\:\\:"+t(f+"\\:")+"{4}"+v),y=t(t(t(f+"\\:")+"{0,1}"+f)+"?\\:\\:"+t(f+"\\:")+"{3}"+v),E=t(t(t(f+"\\:")+"{0,2}"+f)+"?\\:\\:"+t(f+"\\:")+"{2}"+v),C=t(t(t(f+"\\:")+"{0,3}"+f)+"?\\:\\:"+f+"\\:"+v),w=t(t(t(f+"\\:")+"{0,4}"+f)+"?\\:\\:"+v),b=t(t(t(f+"\\:")+"{0,5}"+f)+"?\\:\\:"+f),S=t(t(t(f+"\\:")+"{0,6}"+f)+"?\\:\\:"),_=t([d,m,g,y,E,C,w,b,S].join("|")),A=t(t(p+"|"+i)+"+");t("[vV]"+a+"+\\."+r(p,s,"[\\:]")+"+"),t(t(i+"|"+r(p,s))+"*");var D=t(i+"|"+r(p,s,"[\\:\\@]"));return t(t(i+"|"+r(p,s,"[\\@]"))+"+"),t(t(D+"|"+r("[\\/\\?]",u))+"*"),{NOT_SCHEME:new RegExp(r("[^]",o,n,"[\\+\\-\\.]"),"g"),NOT_USERINFO:new RegExp(r("[^\\%\\:]",p,s),"g"),NOT_HOST:new RegExp(r("[^\\%\\[\\]\\:]",p,s),"g"),NOT_PATH:new RegExp(r("[^\\%\\/\\:\\@]",p,s),"g"),NOT_PATH_NOSCHEME:new RegExp(r("[^\\%\\/\\@]",p,s),"g"),NOT_QUERY:new RegExp(r("[^\\%]",p,s,"[\\:\\@\\/\\?]",u),"g"),NOT_FRAGMENT:new RegExp(r("[^\\%]",p,s,"[\\:\\@\\/\\?]"),"g"),ESCAPE:new RegExp(r("[^]",p,s),"g"),UNRESERVED:new RegExp(p,"g"),OTHER_CHARS:new RegExp(r("[^\\%]",p,c),"g"),PCT_ENCODED:new RegExp(i,"g"),IPV4ADDRESS:new RegExp("^("+h+")$"),IPV6ADDRESS:new RegExp("^\\[?("+_+")"+t(t("\\%25|\\%(?!"+a+"{2})")+"("+A+")")+"?\\]?$")}}var c=s(!1),u=s(!0),p=function(){function e(e,r){var t=[],o=!0,n=!1,a=void 0;try{for(var i,s=e[Symbol.iterator]();!(o=(i=s.next()).done)&&(t.push(i.value),!r||t.length!==r);o=!0);}catch(e){n=!0,a=e}finally{try{!o&&s.return&&s.return()}finally{if(n)throw a}}return t}return function(r,t){if(Array.isArray(r))return r;if(Symbol.iterator in Object(r))return e(r,t);throw new TypeError("Invalid attempt to destructure non-iterable instance")}}(),l=function(e){if(Array.isArray(e)){for(var r=0,t=Array(e.length);r<e.length;r++)t[r]=e[r];return t}return Array.from(e)},h=2147483647,f=36,v=1,d=26,m=38,g=700,y=72,E=128,C="-",w=/^xn--/,b=/[^\0-\x7E]/,S=/[\x2E\u3002\uFF0E\uFF61]/g,_={overflow:"Overflow: input needs wider integers to process","not-basic":"Illegal input >= 0x80 (not a basic code point)","invalid-input":"Invalid input"},A=f-v,D=Math.floor,x=String.fromCharCode;function O(e){throw new RangeError(_[e])}function I(e,r){for(var t=[],o=e.length;o--;)t[o]=r(e[o]);return t}function R(e,r){var t=e.split("@"),o="";return t.length>1&&(o=t[0]+"@",e=t[1]),o+I((e=e.replace(S,".")).split("."),r).join(".")}function z(e){for(var r=[],t=0,o=e.length;t<o;){var n=e.charCodeAt(t++);if(n>=55296&&n<=56319&&t<o){var a=e.charCodeAt(t++);56320==(64512&a)?r.push(((1023&n)<<10)+(1023&a)+65536):(r.push(n),t--)}else r.push(n)}return r}var F=function(e){return e-48<10?e-22:e-65<26?e-65:e-97<26?e-97:f},N=function(e,r){return e+22+75*(e<26)-((0!=r)<<5)},T=function(e,r,t){var o=0;for(e=t?D(e/g):e>>1,e+=D(e/r);e>A*d>>1;o+=f)e=D(e/A);return D(o+(A+1)*e/(e+m))},P=function(e){var r=[],t=e.length,o=0,n=E,a=y,i=e.lastIndexOf(C);i<0&&(i=0);for(var s=0;s<i;++s)e.charCodeAt(s)>=128&&O("not-basic"),r.push(e.charCodeAt(s));for(var c=i>0?i+1:0;c<t;){for(var u=o,p=1,l=f;;l+=f){c>=t&&O("invalid-input");var m=F(e.charCodeAt(c++));(m>=f||m>D((h-o)/p))&&O("overflow"),o+=m*p;var g=l<=a?v:l>=a+d?d:l-a;if(m<g)break;var w=f-g;p>D(h/w)&&O("overflow"),p*=w}var b=r.length+1;a=T(o-u,b,0==u),D(o/b)>h-n&&O("overflow"),n+=D(o/b),o%=b,r.splice(o++,0,n)}return String.fromCodePoint.apply(String,r)},$=function(e){var r=[],t=(e=z(e)).length,o=E,n=0,a=y,i=!0,s=!1,c=void 0;try{for(var u,p=e[Symbol.iterator]();!(i=(u=p.next()).done);i=!0){var l=u.value;l<128&&r.push(x(l))}}catch(e){s=!0,c=e}finally{try{!i&&p.return&&p.return()}finally{if(s)throw c}}var m=r.length,g=m;for(m&&r.push(C);g<t;){var w=h,b=!0,S=!1,_=void 0;try{for(var A,I=e[Symbol.iterator]();!(b=(A=I.next()).done);b=!0){var R=A.value;R>=o&&R<w&&(w=R)}}catch(e){S=!0,_=e}finally{try{!b&&I.return&&I.return()}finally{if(S)throw _}}var F=g+1;w-o>D((h-n)/F)&&O("overflow"),n+=(w-o)*F,o=w;var P=!0,$=!1,j=void 0;try{for(var q,U=e[Symbol.iterator]();!(P=(q=U.next()).done);P=!0){var H=q.value;if(H<o&&++n>h&&O("overflow"),H==o){for(var L=n,k=f;;k+=f){var M=k<=a?v:k>=a+d?d:k-a;if(L<M)break;var V=L-M,W=f-M;r.push(x(N(M+V%W,0))),L=D(V/W)}r.push(x(N(L,0))),a=T(n,F,g==m),n=0,++g}}}catch(e){$=!0,j=e}finally{try{!P&&U.return&&U.return()}finally{if($)throw j}}++n,++o}return r.join("")},j=function(e){return R(e,(function(e){return w.test(e)?P(e.slice(4).toLowerCase()):e}))},q=function(e){return R(e,(function(e){return b.test(e)?"xn--"+$(e):e}))},U={version:"2.1.0",ucs2:{decode:z,encode:function(e){return String.fromCodePoint.apply(String,l(e))}},decode:P,encode:$,toASCII:q,toUnicode:j},H={};function L(e){var r=e.charCodeAt(0);return r<16?"%0"+r.toString(16).toUpperCase():r<128?"%"+r.toString(16).toUpperCase():r<2048?"%"+(r>>6|192).toString(16).toUpperCase()+"%"+(63&r|128).toString(16).toUpperCase():"%"+(r>>12|224).toString(16).toUpperCase()+"%"+(r>>6&63|128).toString(16).toUpperCase()+"%"+(63&r|128).toString(16).toUpperCase()}function k(e){for(var r="",t=0,o=e.length;t<o;){var n=parseInt(e.substr(t+1,2),16);if(n<128)r+=String.fromCharCode(n),t+=3;else if(n>=194&&n<224){if(o-t>=6){var a=parseInt(e.substr(t+4,2),16);r+=String.fromCharCode((31&n)<<6|63&a)}else r+=e.substr(t,6);t+=6}else if(n>=224){if(o-t>=9){var i=parseInt(e.substr(t+4,2),16),s=parseInt(e.substr(t+7,2),16);r+=String.fromCharCode((15&n)<<12|(63&i)<<6|63&s)}else r+=e.substr(t,9);t+=9}else r+=e.substr(t,3),t+=3}return r}function M(e,r){function t(e){var t=k(e);return t.match(r.UNRESERVED)?t:e}return e.scheme&&(e.scheme=String(e.scheme).replace(r.PCT_ENCODED,t).toLowerCase().replace(r.NOT_SCHEME,"")),void 0!==e.userinfo&&(e.userinfo=String(e.userinfo).replace(r.PCT_ENCODED,t).replace(r.NOT_USERINFO,L).replace(r.PCT_ENCODED,n)),void 0!==e.host&&(e.host=String(e.host).replace(r.PCT_ENCODED,t).toLowerCase().replace(r.NOT_HOST,L).replace(r.PCT_ENCODED,n)),void 0!==e.path&&(e.path=String(e.path).replace(r.PCT_ENCODED,t).replace(e.scheme?r.NOT_PATH:r.NOT_PATH_NOSCHEME,L).replace(r.PCT_ENCODED,n)),void 0!==e.query&&(e.query=String(e.query).replace(r.PCT_ENCODED,t).replace(r.NOT_QUERY,L).replace(r.PCT_ENCODED,n)),void 0!==e.fragment&&(e.fragment=String(e.fragment).replace(r.PCT_ENCODED,t).replace(r.NOT_FRAGMENT,L).replace(r.PCT_ENCODED,n)),e}function V(e){return e.replace(/^0*(.*)/,"$1")||"0"}function W(e,r){var t=e.match(r.IPV4ADDRESS)||[],o=p(t,2)[1];return o?o.split(".").map(V).join("."):e}function Z(e,r){var t=e.match(r.IPV6ADDRESS)||[],o=p(t,3),n=o[1],a=o[2];if(n){for(var i=n.toLowerCase().split("::").reverse(),s=p(i,2),c=s[0],u=s[1],l=u?u.split(":").map(V):[],h=c.split(":").map(V),f=r.IPV4ADDRESS.test(h[h.length-1]),v=f?7:8,d=h.length-v,m=Array(v),g=0;g<v;++g)m[g]=l[g]||h[d+g]||"";f&&(m[v-1]=W(m[v-1],r));var y=m.reduce((function(e,r,t){if(!r||"0"===r){var o=e[e.length-1];o&&o.index+o.length===t?o.length++:e.push({index:t,length:1})}return e}),[]).sort((function(e,r){return r.length-e.length}))[0],E=void 0;if(y&&y.length>1){var C=m.slice(0,y.index),w=m.slice(y.index+y.length);E=C.join(":")+"::"+w.join(":")}else E=m.join(":");return a&&(E+="%"+a),E}return e}var G=/^(?:([^:\/?#]+):)?(?:\/\/((?:([^\/?#@]*)@)?(\[[^\/?#\]]+\]|[^\/?#:]*)(?:\:(\d*))?))?([^?#]*)(?:\?([^#]*))?(?:#((?:.|\n|\r)*))?/i,Q=void 0==="".match(/(){0}/)[1];function Y(e){var r=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{},t={},o=!1!==r.iri?u:c;"suffix"===r.reference&&(e=(r.scheme?r.scheme+":":"")+"//"+e);var n=e.match(G);if(n){Q?(t.scheme=n[1],t.userinfo=n[3],t.host=n[4],t.port=parseInt(n[5],10),t.path=n[6]||"",t.query=n[7],t.fragment=n[8],isNaN(t.port)&&(t.port=n[5])):(t.scheme=n[1]||void 0,t.userinfo=-1!==e.indexOf("@")?n[3]:void 0,t.host=-1!==e.indexOf("//")?n[4]:void 0,t.port=parseInt(n[5],10),t.path=n[6]||"",t.query=-1!==e.indexOf("?")?n[7]:void 0,t.fragment=-1!==e.indexOf("#")?n[8]:void 0,isNaN(t.port)&&(t.port=e.match(/\/\/(?:.|\n)*\:(?:\/|\?|\#|$)/)?n[4]:void 0)),t.host&&(t.host=Z(W(t.host,o),o)),void 0!==t.scheme||void 0!==t.userinfo||void 0!==t.host||void 0!==t.port||t.path||void 0!==t.query?void 0===t.scheme?t.reference="relative":void 0===t.fragment?t.reference="absolute":t.reference="uri":t.reference="same-document",r.reference&&"suffix"!==r.reference&&r.reference!==t.reference&&(t.error=t.error||"URI is not a "+r.reference+" reference.");var a=H[(r.scheme||t.scheme||"").toLowerCase()];if(r.unicodeSupport||a&&a.unicodeSupport)M(t,o);else{if(t.host&&(r.domainHost||a&&a.domainHost))try{t.host=U.toASCII(t.host.replace(o.PCT_ENCODED,k).toLowerCase())}catch(e){t.error=t.error||"Host's domain name can not be converted to ASCII via punycode: "+e}M(t,c)}a&&a.parse&&a.parse(t,r)}else t.error=t.error||"URI can not be parsed.";return t}function B(e,r){var t=!1!==r.iri?u:c,o=[];return void 0!==e.userinfo&&(o.push(e.userinfo),o.push("@")),void 0!==e.host&&o.push(Z(W(String(e.host),t),t).replace(t.IPV6ADDRESS,(function(e,r,t){return"["+r+(t?"%25"+t:"")+"]"}))),"number"!=typeof e.port&&"string"!=typeof e.port||(o.push(":"),o.push(String(e.port))),o.length?o.join(""):void 0}var J=/^\.\.?\//,K=/^\/\.(\/|$)/,X=/^\/\.\.(\/|$)/,ee=/^\/?(?:.|\n)*?(?=\/|$)/;function re(e){for(var r=[];e.length;)if(e.match(J))e=e.replace(J,"");else if(e.match(K))e=e.replace(K,"/");else if(e.match(X))e=e.replace(X,"/"),r.pop();else if("."===e||".."===e)e="";else{var t=e.match(ee);if(!t)throw new Error("Unexpected dot segment condition");var o=t[0];e=e.slice(o.length),r.push(o)}return r.join("")}function te(e){var r=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{},t=r.iri?u:c,o=[],n=H[(r.scheme||e.scheme||"").toLowerCase()];if(n&&n.serialize&&n.serialize(e,r),e.host)if(t.IPV6ADDRESS.test(e.host));else if(r.domainHost||n&&n.domainHost)try{e.host=r.iri?U.toUnicode(e.host):U.toASCII(e.host.replace(t.PCT_ENCODED,k).toLowerCase())}catch(t){e.error=e.error||"Host's domain name can not be converted to "+(r.iri?"Unicode":"ASCII")+" via punycode: "+t}M(e,t),"suffix"!==r.reference&&e.scheme&&(o.push(e.scheme),o.push(":"));var a=B(e,r);if(void 0!==a&&("suffix"!==r.reference&&o.push("//"),o.push(a),e.path&&"/"!==e.path.charAt(0)&&o.push("/")),void 0!==e.path){var i=e.path;r.absolutePath||n&&n.absolutePath||(i=re(i)),void 0===a&&(i=i.replace(/^\/\//,"/%2F")),o.push(i)}return void 0!==e.query&&(o.push("?"),o.push(e.query)),void 0!==e.fragment&&(o.push("#"),o.push(e.fragment)),o.join("")}function oe(e,r){var t=arguments.length>2&&void 0!==arguments[2]?arguments[2]:{},o={};return arguments[3]||(e=Y(te(e,t),t),r=Y(te(r,t),t)),!(t=t||{}).tolerant&&r.scheme?(o.scheme=r.scheme,o.userinfo=r.userinfo,o.host=r.host,o.port=r.port,o.path=re(r.path||""),o.query=r.query):(void 0!==r.userinfo||void 0!==r.host||void 0!==r.port?(o.userinfo=r.userinfo,o.host=r.host,o.port=r.port,o.path=re(r.path||""),o.query=r.query):(r.path?("/"===r.path.charAt(0)?o.path=re(r.path):(void 0===e.userinfo&&void 0===e.host&&void 0===e.port||e.path?e.path?o.path=e.path.slice(0,e.path.lastIndexOf("/")+1)+r.path:o.path=r.path:o.path="/"+r.path,o.path=re(o.path)),o.query=r.query):(o.path=e.path,void 0!==r.query?o.query=r.query:o.query=e.query),o.userinfo=e.userinfo,o.host=e.host,o.port=e.port),o.scheme=e.scheme),o.fragment=r.fragment,o}function ne(e,r,t){var o=i({scheme:"null"},t);return te(oe(Y(e,o),Y(r,o),o,!0),o)}function ae(e,r){return"string"==typeof e?e=te(Y(e,r),r):"object"===o(e)&&(e=Y(te(e,r),r)),e}function ie(e,r,t){return"string"==typeof e?e=te(Y(e,t),t):"object"===o(e)&&(e=te(e,t)),"string"==typeof r?r=te(Y(r,t),t):"object"===o(r)&&(r=te(r,t)),e===r}function se(e,r){return e&&e.toString().replace(r&&r.iri?u.ESCAPE:c.ESCAPE,L)}function ce(e,r){return e&&e.toString().replace(r&&r.iri?u.PCT_ENCODED:c.PCT_ENCODED,k)}var ue={scheme:"http",domainHost:!0,parse:function(e,r){return e.host||(e.error=e.error||"HTTP URIs must have a host."),e},serialize:function(e,r){var t="https"===String(e.scheme).toLowerCase();return e.port!==(t?443:80)&&""!==e.port||(e.port=void 0),e.path||(e.path="/"),e}},pe={scheme:"https",domainHost:ue.domainHost,parse:ue.parse,serialize:ue.serialize};function le(e){return"boolean"==typeof e.secure?e.secure:"wss"===String(e.scheme).toLowerCase()}var he={scheme:"ws",domainHost:!0,parse:function(e,r){var t=e;return t.secure=le(t),t.resourceName=(t.path||"/")+(t.query?"?"+t.query:""),t.path=void 0,t.query=void 0,t},serialize:function(e,r){if(e.port!==(le(e)?443:80)&&""!==e.port||(e.port=void 0),"boolean"==typeof e.secure&&(e.scheme=e.secure?"wss":"ws",e.secure=void 0),e.resourceName){var t=e.resourceName.split("?"),o=p(t,2),n=o[0],a=o[1];e.path=n&&"/"!==n?n:void 0,e.query=a,e.resourceName=void 0}return e.fragment=void 0,e}},fe={scheme:"wss",domainHost:he.domainHost,parse:he.parse,serialize:he.serialize},ve={},de="[A-Za-z0-9\\-\\.\\_\\~\\xA0-\\u200D\\u2010-\\u2029\\u202F-\\uD7FF\\uF900-\\uFDCF\\uFDF0-\\uFFEF]",me="[0-9A-Fa-f]",ge=t(t("%[EFef]"+me+"%"+me+me+"%"+me+me)+"|"+t("%[89A-Fa-f]"+me+"%"+me+me)+"|"+t("%"+me+me)),ye="[A-Za-z0-9\\!\\$\\%\\'\\*\\+\\-\\^\\_\\`\\{\\|\\}\\~]",Ee=r("[\\!\\$\\%\\'\\(\\)\\*\\+\\,\\-\\.0-9\\<\\>A-Z\\x5E-\\x7E]",'[\\"\\\\]'),Ce="[\\!\\$\\'\\(\\)\\*\\+\\,\\;\\:\\@]",we=new RegExp(de,"g"),be=new RegExp(ge,"g"),Se=new RegExp(r("[^]",ye,"[\\.]",'[\\"]',Ee),"g"),_e=new RegExp(r("[^]",de,Ce),"g"),Ae=_e;function De(e){var r=k(e);return r.match(we)?r:e}var xe={scheme:"mailto",parse:function(e,r){var t=e,o=t.to=t.path?t.path.split(","):[];if(t.path=void 0,t.query){for(var n=!1,a={},i=t.query.split("&"),s=0,c=i.length;s<c;++s){var u=i[s].split("=");switch(u[0]){case"to":for(var p=u[1].split(","),l=0,h=p.length;l<h;++l)o.push(p[l]);break;case"subject":t.subject=ce(u[1],r);break;case"body":t.body=ce(u[1],r);break;default:n=!0,a[ce(u[0],r)]=ce(u[1],r)}}n&&(t.headers=a)}t.query=void 0;for(var f=0,v=o.length;f<v;++f){var d=o[f].split("@");if(d[0]=ce(d[0]),r.unicodeSupport)d[1]=ce(d[1],r).toLowerCase();else try{d[1]=U.toASCII(ce(d[1],r).toLowerCase())}catch(e){t.error=t.error||"Email address's domain name can not be converted to ASCII via punycode: "+e}o[f]=d.join("@")}return t},serialize:function(e,r){var t=e,o=a(e.to);if(o){for(var i=0,s=o.length;i<s;++i){var c=String(o[i]),u=c.lastIndexOf("@"),p=c.slice(0,u).replace(be,De).replace(be,n).replace(Se,L),l=c.slice(u+1);try{l=r.iri?U.toUnicode(l):U.toASCII(ce(l,r).toLowerCase())}catch(e){t.error=t.error||"Email address's domain name can not be converted to "+(r.iri?"Unicode":"ASCII")+" via punycode: "+e}o[i]=p+"@"+l}t.path=o.join(",")}var h=e.headers=e.headers||{};e.subject&&(h.subject=e.subject),e.body&&(h.body=e.body);var f=[];for(var v in h)h[v]!==ve[v]&&f.push(v.replace(be,De).replace(be,n).replace(_e,L)+"="+h[v].replace(be,De).replace(be,n).replace(Ae,L));return f.length&&(t.query=f.join("&")),t}},Oe=/^([^\:]+)\:(.*)/,Ie={scheme:"urn",parse:function(e,r){var t=e.path&&e.path.match(Oe),o=e;if(t){var n=r.scheme||o.scheme||"urn",a=t[1].toLowerCase(),i=t[2],s=n+":"+(r.nid||a),c=H[s];o.nid=a,o.nss=i,o.path=void 0,c&&(o=c.parse(o,r))}else o.error=o.error||"URN can not be parsed.";return o},serialize:function(e,r){var t=r.scheme||e.scheme||"urn",o=e.nid,n=t+":"+(r.nid||o),a=H[n];a&&(e=a.serialize(e,r));var i=e,s=e.nss;return i.path=(o||r.nid)+":"+s,i}},Re=/^[0-9A-Fa-f]{8}(?:\-[0-9A-Fa-f]{4}){3}\-[0-9A-Fa-f]{12}$/,ze={scheme:"urn:uuid",parse:function(e,r){var t=e;return t.uuid=t.nss,t.nss=void 0,r.tolerant||t.uuid&&t.uuid.match(Re)||(t.error=t.error||"UUID is not valid."),t},serialize:function(e,r){var t=e;return t.nss=(e.uuid||"").toLowerCase(),t}};H[ue.scheme]=ue,H[pe.scheme]=pe,H[he.scheme]=he,H[fe.scheme]=fe,H[xe.scheme]=xe,H[Ie.scheme]=Ie,H[ze.scheme]=ze,e.SCHEMES=H,e.pctEncChar=L,e.pctDecChars=k,e.parse=Y,e.removeDotSegments=re,e.serialize=te,e.resolveComponents=oe,e.resolve=ne,e.normalize=ae,e.equal=ie,e.escapeComponent=se,e.unescapeComponent=ce,Object.defineProperty(e,"__esModule",{value:!0})}(r)}(0,n.exports)),n.exports),i=t(a);function s(e){for(const t of r){const r=e.replace(new RegExp(t.match),t.replace);if(r!=e)return r}return e}function c(e,r,t,o){if(void 0===t.scheme&&o.startsWith("../")){const t=o.split("/"),n=i.resolve(r,t.slice(1).join("/")),a=i.resolve(r,t.slice(2).join("/"));if(n==e&&a!=e){if(t.find((e=>".."!==e)).indexOf(".")>-1)return!0}}return!1}function u(e,r,t,o,n,a,u,p,l){if(!a)return a;if(a=String(a),["#","{","*"].includes(a.substring(0,1)))return a;const h=i.parse(a);if(void 0!==h.scheme&&!["http","https"].includes(h.scheme))return a;a.startsWith(n)&&(a=h.scheme+"://"+a.substring(n.length));try{decodeURIComponent(i.parse(a).path)}catch(e){a=encodeURI(a)}const f=i.resolve(o,a);if(c(f,o,h,a))return a;if(void 0===h.scheme&&a.startsWith("../")){const e=a.split("/"),r=i.resolve(o,e.slice(1).join("/")),t=i.resolve(o,e.slice(2).join("/"));if(r==f&&t!=f){if(e.find((e=>".."!==e)).indexOf(".")>-1)return a}}const v=i.parse(f),d=i.serialize(i.parse("http://"+v.host),{iri:!0}),m=d.substring(7,d.length-1),g=v.path&&0!==v.path.length?decodeURIComponent(v.path):"/",y=v.query&&0!==v.query.length?"?"+decodeURIComponent(v.query).replaceAll("+"," "):"";const E=s(m+(g+y).replace(/\/\/+/g,"/"));return n+encodeURIComponent(E).replaceAll("%2F","/")}function p(e,r,t,o,n){return{rewrite_function:(e,r,t,a)=>u(0,0,0,o,n,e),top_url:e,url:o,timestamp:"",request_ts:"",prefix:n,mod:"",is_framed:!1,is_live:!1,coll:"",proxy_magic:"",static_prefix:n+"_zim_static/",wombat_ts:"",wombat_sec:0,wombat_scheme:t,wombat_host:r,wombat_opts:{},enable_auto_fetch:!0,convert_post_to_get:!0,target_frame:"___wb_replay_top_frame",isSW:!0}}var l={applyFuzzyRules:s,urlRewriteFunction:u,getWombatInfo:p};return e.applyFuzzyRules=s,e.default=l,e.getWombatInfo=p,e.hasAlreadyBeenRewritten=c,e.urlRewriteFunction=u,Object.defineProperty(e,"__esModule",{value:!0}),e}({});
|
|
@@ -101,10 +101,15 @@ def get_pdf_index_data(
|
|
|
101
101
|
if parts: # pragma: no branch (always metadata in test PDFs)
|
|
102
102
|
title = " - ".join(parts)
|
|
103
103
|
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
104
|
+
def get_pdf_content(page: pymupdf.Page) -> str:
|
|
105
|
+
text = ( # pyright: ignore[reportUnknownVariableType]
|
|
106
|
+
page.get_text() # pyright: ignore[reportUnknownMemberType]
|
|
107
|
+
)
|
|
108
|
+
if not isinstance(text, str):
|
|
109
|
+
raise Exception("Unexpected text content")
|
|
110
|
+
return text
|
|
111
|
+
|
|
112
|
+
content = "\n".join(get_pdf_content(page) for page in doc)
|
|
108
113
|
|
|
109
114
|
# build list of messages and filter messages which are known to not be relevant
|
|
110
115
|
# in our use-case
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "5.2.0"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/rewriting/statics/__wb_module_decl.js
RENAMED
|
File without changes
|
|
File without changes
|
{zimscraperlib-5.2.0 → zimscraperlib-5.3.0}/src/zimscraperlib/rewriting/templates/head_insert.html
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|