zimscraperlib 3.3.0__tar.gz → 3.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/.pre-commit-config.yaml +3 -3
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/CHANGELOG.md +25 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/PKG-INFO +12 -11
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/README.md +1 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/contrib/encode_video.py +3 -2
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/pyproject.toml +10 -10
- zimscraperlib-3.3.2/src/zimscraperlib/__about__.py +1 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/constants.py +2 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/download.py +8 -8
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/filesystem.py +2 -1
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/fix_ogvjs_dist.py +4 -2
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/i18n.py +8 -6
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/image/optimization.py +3 -2
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/image/presets.py +15 -13
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/image/probing.py +6 -4
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/inputs.py +8 -3
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/logging.py +2 -1
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/types.py +4 -2
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/video/config.py +6 -4
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/video/encoding.py +6 -3
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/video/presets.py +8 -6
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/zim/_libkiwix.py +6 -4
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/zim/archive.py +7 -4
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/zim/creator.py +7 -5
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/zim/filesystem.py +14 -4
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/zim/metadata.py +5 -3
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/conftest.py +5 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/download/test_download.py +4 -2
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/inputs/test_inputs.py +53 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/video/test_encoding.py +21 -3
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/zim/test_metadata.py +4 -1
- zimscraperlib-3.3.0/src/zimscraperlib/__about__.py +0 -1
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/.gitignore +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/LICENSE +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/contrib/README.md +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/__init__.py +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/html.py +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/image/__init__.py +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/image/convertion.py +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/image/transformation.py +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/image/utils.py +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/misc.py +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/uri.py +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/video/__init__.py +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/video/probing.py +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/zim/__init__.py +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/zim/items.py +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/zim/providers.py +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tasks.py +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/files/DroidSans.ttf +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/files/audio.mp3 +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/files/blue.jpg +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/files/commons.png +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/files/commons48.png +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/files/mail.gif +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/files/ninja.webp +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/files/pluto.jpg +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/files/square.jpg +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/files/square.png +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/files/star.svg +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/files/video.mkv +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/files/video.mp4 +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/files/video.webm +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/filesystem/test_filesystem.py +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/html/conftest.py +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/html/test_html.py +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/i18n/locale/fr/LC_MESSAGES/messages.po +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/i18n/locale/pt/LC_MESSAGES/messages.po +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/i18n/test_i18n.py +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/image/test_image.py +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/logging/conftest.py +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/logging/test_logging.py +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/misc/test_misc.py +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/ogvjs/conftest.py +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/ogvjs/test_ogvjs.py +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/types/test_types.py +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/uri/test_uri.py +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/video/conftest.py +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/video/test_video.py +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/zim/conftest.py +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/zim/test_archive.py +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/zim/test_fs.py +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/zim/test_libkiwix.py +0 -0
- {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/zim/test_zim_creator.py +0 -0
|
@@ -7,15 +7,15 @@ repos:
|
|
|
7
7
|
- id: trailing-whitespace
|
|
8
8
|
- id: end-of-file-fixer
|
|
9
9
|
- repo: https://github.com/psf/black
|
|
10
|
-
rev: "24.
|
|
10
|
+
rev: "24.3.0"
|
|
11
11
|
hooks:
|
|
12
12
|
- id: black
|
|
13
13
|
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
14
|
-
rev: v0.
|
|
14
|
+
rev: v0.3.3
|
|
15
15
|
hooks:
|
|
16
16
|
- id: ruff
|
|
17
17
|
- repo: https://github.com/RobertCraigie/pyright-python
|
|
18
|
-
rev: v1.1.
|
|
18
|
+
rev: v1.1.354
|
|
19
19
|
hooks:
|
|
20
20
|
- id: pyright
|
|
21
21
|
name: pyright (system)
|
|
@@ -5,6 +5,31 @@ All notable changes to this project are documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0).
|
|
7
7
|
|
|
8
|
+
## [3.3.2] - 2024-03-25
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- Add support for `disable_metadata_checks` and `ignore_duplicates` arguments in `make_zim_file` function ("zimwritefs-mode")
|
|
13
|
+
|
|
14
|
+
### Changed
|
|
15
|
+
|
|
16
|
+
- Relaxed constraints on Python dependencies
|
|
17
|
+
- Upgraded optional dependencies used for test and QA
|
|
18
|
+
|
|
19
|
+
## [3.3.1] - 2024-02-27
|
|
20
|
+
|
|
21
|
+
### Added
|
|
22
|
+
|
|
23
|
+
- Set a user-agent for `handle_user_provided_file` #103
|
|
24
|
+
|
|
25
|
+
### Changed
|
|
26
|
+
|
|
27
|
+
- Migrate to generic syntax in all std collections #140
|
|
28
|
+
|
|
29
|
+
### Fixed
|
|
30
|
+
|
|
31
|
+
- Do not modify the ffmpeg_args in reencode function #144
|
|
32
|
+
|
|
8
33
|
## [3.3.0] - 2024-02-14
|
|
9
34
|
|
|
10
35
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: zimscraperlib
|
|
3
|
-
Version: 3.3.
|
|
3
|
+
Version: 3.3.2
|
|
4
4
|
Summary: Collection of python tools to re-use common code across scrapers
|
|
5
5
|
Project-URL: Donate, https://www.kiwix.org/en/support-us/
|
|
6
6
|
Project-URL: Homepage, https://github.com/openzim/python-scraperlib
|
|
@@ -17,34 +17,34 @@ Classifier: Programming Language :: Python :: 3.11
|
|
|
17
17
|
Classifier: Programming Language :: Python :: 3.12
|
|
18
18
|
Requires-Python: <3.13,>=3.8
|
|
19
19
|
Requires-Dist: babel<3.0,>=2.9
|
|
20
|
-
Requires-Dist: beautifulsoup4<
|
|
20
|
+
Requires-Dist: beautifulsoup4<5.0,>=4.9.3
|
|
21
21
|
Requires-Dist: colorthief==0.2.1
|
|
22
22
|
Requires-Dist: iso639-lang<3.0,>=2.2.3
|
|
23
23
|
Requires-Dist: libzim<4.0,>=3.4.0
|
|
24
|
-
Requires-Dist: lxml<
|
|
25
|
-
Requires-Dist: optimize-images<
|
|
24
|
+
Requires-Dist: lxml<6.0,>=4.6.3
|
|
25
|
+
Requires-Dist: optimize-images<2.0,>=1.3.6
|
|
26
26
|
Requires-Dist: python-magic<0.5,>=0.4.3
|
|
27
27
|
Requires-Dist: python-resize-image<1.2,>=1.1.19
|
|
28
28
|
Requires-Dist: requests<3.0,>=2.25.1
|
|
29
29
|
Requires-Dist: yt-dlp
|
|
30
30
|
Provides-Extra: check
|
|
31
|
-
Requires-Dist: pyright==1.1.
|
|
31
|
+
Requires-Dist: pyright==1.1.354; extra == 'check'
|
|
32
32
|
Provides-Extra: dev
|
|
33
|
-
Requires-Dist: debugpy==1.8.
|
|
34
|
-
Requires-Dist: pre-commit==3.
|
|
33
|
+
Requires-Dist: debugpy==1.8.1; extra == 'dev'
|
|
34
|
+
Requires-Dist: pre-commit==3.6.2; extra == 'dev'
|
|
35
35
|
Requires-Dist: zimscraperlib[check]; extra == 'dev'
|
|
36
36
|
Requires-Dist: zimscraperlib[lint]; extra == 'dev'
|
|
37
37
|
Requires-Dist: zimscraperlib[scripts]; extra == 'dev'
|
|
38
38
|
Requires-Dist: zimscraperlib[test]; extra == 'dev'
|
|
39
39
|
Provides-Extra: lint
|
|
40
|
-
Requires-Dist: black==24.
|
|
41
|
-
Requires-Dist: ruff==0.
|
|
40
|
+
Requires-Dist: black==24.3.0; extra == 'lint'
|
|
41
|
+
Requires-Dist: ruff==0.3.3; extra == 'lint'
|
|
42
42
|
Provides-Extra: scripts
|
|
43
43
|
Requires-Dist: invoke==2.2.0; extra == 'scripts'
|
|
44
44
|
Provides-Extra: test
|
|
45
|
-
Requires-Dist: coverage==7.4.
|
|
45
|
+
Requires-Dist: coverage==7.4.4; extra == 'test'
|
|
46
46
|
Requires-Dist: pytest-mock==3.12.0; extra == 'test'
|
|
47
|
-
Requires-Dist: pytest==8.
|
|
47
|
+
Requires-Dist: pytest==8.1.1; extra == 'test'
|
|
48
48
|
Description-Content-Type: text/markdown
|
|
49
49
|
|
|
50
50
|
zimscraperlib
|
|
@@ -54,6 +54,7 @@ zimscraperlib
|
|
|
54
54
|
[](https://www.codefactor.io/repository/github/openzim/python-scraperlib)
|
|
55
55
|
[](https://www.gnu.org/licenses/gpl-3.0)
|
|
56
56
|
[](https://pypi.org/project/zimscraperlib/)
|
|
57
|
+
[](https://pypi.org/project/zimscraperlib)
|
|
57
58
|
[](https://codecov.io/gh/openzim/python-scraperlib)
|
|
58
59
|
|
|
59
60
|
Collection of python code to re-use across python-based scrapers
|
|
@@ -5,6 +5,7 @@ zimscraperlib
|
|
|
5
5
|
[](https://www.codefactor.io/repository/github/openzim/python-scraperlib)
|
|
6
6
|
[](https://www.gnu.org/licenses/gpl-3.0)
|
|
7
7
|
[](https://pypi.org/project/zimscraperlib/)
|
|
8
|
+
[](https://pypi.org/project/zimscraperlib)
|
|
8
9
|
[](https://codecov.io/gh/openzim/python-scraperlib)
|
|
9
10
|
|
|
10
11
|
Collection of python code to re-use across python-based scrapers
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
1
3
|
import sys
|
|
2
4
|
from pathlib import Path
|
|
3
|
-
from typing import List
|
|
4
5
|
|
|
5
6
|
from zimscraperlib import logger
|
|
6
7
|
from zimscraperlib.video import presets, reencode
|
|
@@ -25,7 +26,7 @@ def encode_video(src_path: Path, dst_path: Path, preset: str):
|
|
|
25
26
|
logger.error(f"conversion failed:\n{process.stdout}")
|
|
26
27
|
|
|
27
28
|
|
|
28
|
-
def run(args:
|
|
29
|
+
def run(args: list[str] = sys.argv):
|
|
29
30
|
if len(args) < 4: # noqa: PLR2004
|
|
30
31
|
print(f"Usage: {args[0]} <src_path> <dst_path> <preset>") # noqa: T201
|
|
31
32
|
print( # noqa: T201
|
|
@@ -15,9 +15,9 @@ dependencies = [
|
|
|
15
15
|
"Babel>=2.9,<3.0",
|
|
16
16
|
"python-magic>=0.4.3,<0.5",
|
|
17
17
|
"libzim>=3.4.0,<4.0",
|
|
18
|
-
"beautifulsoup4>=4.9.3,<
|
|
19
|
-
"lxml>=4.6.3,<
|
|
20
|
-
"optimize-images>=1.3.6,<
|
|
18
|
+
"beautifulsoup4>=4.9.3,<5.0",
|
|
19
|
+
"lxml>=4.6.3,<6.0",
|
|
20
|
+
"optimize-images>=1.3.6,<2.0",
|
|
21
21
|
# youtube-dl should be updated as frequently as possible
|
|
22
22
|
"yt-dlp"
|
|
23
23
|
]
|
|
@@ -36,20 +36,20 @@ scripts = [
|
|
|
36
36
|
"invoke==2.2.0",
|
|
37
37
|
]
|
|
38
38
|
lint = [
|
|
39
|
-
"black==24.
|
|
40
|
-
"ruff==0.
|
|
39
|
+
"black==24.3.0",
|
|
40
|
+
"ruff==0.3.3",
|
|
41
41
|
]
|
|
42
42
|
check = [
|
|
43
|
-
"pyright==1.1.
|
|
43
|
+
"pyright==1.1.354",
|
|
44
44
|
]
|
|
45
45
|
test = [
|
|
46
|
-
"pytest==8.
|
|
46
|
+
"pytest==8.1.1",
|
|
47
47
|
"pytest-mock==3.12.0",
|
|
48
|
-
"coverage==7.4.
|
|
48
|
+
"coverage==7.4.4",
|
|
49
49
|
]
|
|
50
50
|
dev = [
|
|
51
|
-
"pre-commit==3.
|
|
52
|
-
"debugpy==1.8.
|
|
51
|
+
"pre-commit==3.6.2",
|
|
52
|
+
"debugpy==1.8.1",
|
|
53
53
|
"zimscraperlib[scripts]",
|
|
54
54
|
"zimscraperlib[lint]",
|
|
55
55
|
"zimscraperlib[test]",
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "3.3.2"
|
|
@@ -10,6 +10,8 @@ from zimscraperlib.__about__ import __version__
|
|
|
10
10
|
ROOT_DIR = pathlib.Path(__file__).parent
|
|
11
11
|
NAME = pathlib.Path(__file__).parent.name
|
|
12
12
|
SCRAPER = f"{NAME} {__version__}"
|
|
13
|
+
CONTACT = "dev@openzim.org"
|
|
14
|
+
DEFAULT_USER_AGENT = f"{NAME}/{__version__} ({CONTACT})"
|
|
13
15
|
|
|
14
16
|
UTF8 = "UTF-8"
|
|
15
17
|
|
|
@@ -7,7 +7,7 @@ import io
|
|
|
7
7
|
import pathlib
|
|
8
8
|
import subprocess
|
|
9
9
|
from concurrent.futures import Future, ThreadPoolExecutor
|
|
10
|
-
from typing import ClassVar,
|
|
10
|
+
from typing import ClassVar, Optional, Union
|
|
11
11
|
|
|
12
12
|
import requests
|
|
13
13
|
import yt_dlp as youtube_dl
|
|
@@ -34,14 +34,14 @@ class YoutubeDownloader:
|
|
|
34
34
|
"""shuts down the executor, awaiting completion"""
|
|
35
35
|
self.executor.shutdown(wait=True)
|
|
36
36
|
|
|
37
|
-
def _run_youtube_dl(self, url: str, options:
|
|
37
|
+
def _run_youtube_dl(self, url: str, options: dict) -> None:
|
|
38
38
|
with youtube_dl.YoutubeDL(options) as ydl:
|
|
39
39
|
ydl.download([url])
|
|
40
40
|
|
|
41
41
|
def download(
|
|
42
42
|
self,
|
|
43
43
|
url: str,
|
|
44
|
-
options: Optional[
|
|
44
|
+
options: Optional[dict],
|
|
45
45
|
wait: Optional[bool] = True, # noqa: FBT002
|
|
46
46
|
) -> Union[bool, Future]:
|
|
47
47
|
"""Downloads video using initialized executor.
|
|
@@ -65,8 +65,8 @@ class YoutubeDownloader:
|
|
|
65
65
|
|
|
66
66
|
|
|
67
67
|
class YoutubeConfig(dict):
|
|
68
|
-
options: ClassVar[
|
|
69
|
-
defaults: ClassVar[
|
|
68
|
+
options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {}
|
|
69
|
+
defaults: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
|
|
70
70
|
"writethumbnail": True,
|
|
71
71
|
"write_all_thumbnails": True,
|
|
72
72
|
"writesubtitles": True,
|
|
@@ -109,14 +109,14 @@ class YoutubeConfig(dict):
|
|
|
109
109
|
|
|
110
110
|
|
|
111
111
|
class BestWebm(YoutubeConfig):
|
|
112
|
-
options: ClassVar[
|
|
112
|
+
options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
|
|
113
113
|
"preferredcodec": "webm",
|
|
114
114
|
"format": "best[ext=webm]/bestvideo[ext=webm]+bestaudio[ext=webm]/best",
|
|
115
115
|
}
|
|
116
116
|
|
|
117
117
|
|
|
118
118
|
class BestMp4(YoutubeConfig):
|
|
119
|
-
options: ClassVar[
|
|
119
|
+
options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
|
|
120
120
|
"preferredcodec": "mp4",
|
|
121
121
|
"format": "best[ext=mp4]/bestvideo[ext=mp4]+bestaudio[ext=m4a]/best",
|
|
122
122
|
}
|
|
@@ -179,7 +179,7 @@ def stream_file(
|
|
|
179
179
|
proxies: Optional[dict] = None,
|
|
180
180
|
only_first_block: Optional[bool] = False, # noqa: FBT002
|
|
181
181
|
max_retries: Optional[int] = 5,
|
|
182
|
-
headers: Optional[
|
|
182
|
+
headers: Optional[dict[str, str]] = None,
|
|
183
183
|
session: Optional[requests.Session] = None,
|
|
184
184
|
) -> tuple[int, requests.structures.CaseInsensitiveDict]: # pyright: ignore
|
|
185
185
|
"""Stream data from a URL to either a BytesIO object or a file
|
|
@@ -4,10 +4,12 @@
|
|
|
4
4
|
|
|
5
5
|
""" quick script to fix videojs-ogvjs so that it triggers on webm mimetype """
|
|
6
6
|
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
7
9
|
import logging
|
|
8
10
|
import pathlib
|
|
9
11
|
import sys
|
|
10
|
-
from typing import
|
|
12
|
+
from typing import Union
|
|
11
13
|
|
|
12
14
|
logging.basicConfig(format="%(levelname)s:%(message)s", level=logging.DEBUG)
|
|
13
15
|
logger = logging.getLogger(__name__)
|
|
@@ -33,7 +35,7 @@ def fix_source_dir(source_vendors_path: Union[pathlib.Path, str]):
|
|
|
33
35
|
logger.info("all done.")
|
|
34
36
|
|
|
35
37
|
|
|
36
|
-
def run(args:
|
|
38
|
+
def run(args: list[str] = sys.argv):
|
|
37
39
|
if len(args) < 2: # noqa: PLR2004
|
|
38
40
|
print(f"Usage: {args[0]} <source_vendors_path>") # noqa: T201
|
|
39
41
|
print( # noqa: T201
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
2
|
# vim: ai ts=4 sts=4 et sw=4 nu
|
|
3
3
|
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
4
6
|
import gettext
|
|
5
7
|
import locale
|
|
6
8
|
import pathlib
|
|
7
9
|
import re
|
|
8
|
-
from typing import
|
|
10
|
+
from typing import Optional, Union
|
|
9
11
|
|
|
10
12
|
import babel
|
|
11
13
|
import iso639
|
|
@@ -58,7 +60,7 @@ def setlocale(root_dir: pathlib.Path, locale_name: str):
|
|
|
58
60
|
return Locale.setup(root_dir / "locale", locale_name)
|
|
59
61
|
|
|
60
62
|
|
|
61
|
-
def get_iso_lang_data(lang: str) ->
|
|
63
|
+
def get_iso_lang_data(lang: str) -> tuple[dict, Union[dict, None]]:
|
|
62
64
|
"""ISO-639-x languages details for lang. Raises NotFound
|
|
63
65
|
|
|
64
66
|
Included keys: iso-639-1, iso-639-2b, iso-639-2t, iso-639-3, iso-639-5
|
|
@@ -112,8 +114,8 @@ def get_iso_lang_data(lang: str) -> Tuple[Dict, Union[Dict, None]]:
|
|
|
112
114
|
|
|
113
115
|
|
|
114
116
|
def find_language_names(
|
|
115
|
-
query: str, lang_data: Optional[
|
|
116
|
-
) ->
|
|
117
|
+
query: str, lang_data: Optional[dict] = None
|
|
118
|
+
) -> tuple[str, str]:
|
|
117
119
|
"""(native, english) language names for lang with help from language_details dict
|
|
118
120
|
|
|
119
121
|
Falls back to English name if available or query if not"""
|
|
@@ -140,7 +142,7 @@ def find_language_names(
|
|
|
140
142
|
return default, default
|
|
141
143
|
|
|
142
144
|
|
|
143
|
-
def update_with_macro(lang_data:
|
|
145
|
+
def update_with_macro(lang_data: dict, macro_data: dict):
|
|
144
146
|
"""update empty keys from lang_data with ones of macro_data"""
|
|
145
147
|
if macro_data:
|
|
146
148
|
for key, value in macro_data.items():
|
|
@@ -151,7 +153,7 @@ def update_with_macro(lang_data: Dict, macro_data: Dict):
|
|
|
151
153
|
|
|
152
154
|
def get_language_details(
|
|
153
155
|
query: str, failsafe: Optional[bool] = False # noqa: FBT002
|
|
154
|
-
) ->
|
|
156
|
+
) -> dict:
|
|
155
157
|
"""language details dict from query.
|
|
156
158
|
|
|
157
159
|
Raises NotFound or return `und` language details if failsafe
|
|
@@ -22,12 +22,13 @@
|
|
|
22
22
|
can still run on default settings which give
|
|
23
23
|
a bit less size than the original images but maintain a high quality. """
|
|
24
24
|
|
|
25
|
+
from __future__ import annotations
|
|
25
26
|
|
|
26
27
|
import io
|
|
27
28
|
import os
|
|
28
29
|
import pathlib
|
|
29
30
|
import subprocess
|
|
30
|
-
from typing import Optional,
|
|
31
|
+
from typing import Optional, Union
|
|
31
32
|
|
|
32
33
|
import piexif
|
|
33
34
|
from optimize_images.img_aux_processing import do_reduce_colors, rebuild_palette
|
|
@@ -57,7 +58,7 @@ def optimize_png(
|
|
|
57
58
|
max_colors: Optional[int] = 256,
|
|
58
59
|
fast_mode: Optional[bool] = True, # noqa: FBT002
|
|
59
60
|
remove_transparency: Optional[bool] = False, # noqa: FBT002
|
|
60
|
-
background_color: Optional[
|
|
61
|
+
background_color: Optional[tuple[int, int, int]] = (255, 255, 255),
|
|
61
62
|
**options, # noqa: ARG001
|
|
62
63
|
) -> Union[pathlib.Path, io.BytesIO]:
|
|
63
64
|
"""method to optimize PNG files using a pure python external optimizer
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
2
|
# vim: ai ts=4 sts=4 et sw=4 nu
|
|
3
3
|
|
|
4
|
-
from
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from typing import ClassVar, Optional, Union
|
|
5
7
|
|
|
6
8
|
""" presets for ImageOptimizer in zimscraperlib.image.optimization module """
|
|
7
9
|
|
|
@@ -20,7 +22,7 @@ class WebpLow:
|
|
|
20
22
|
ext = "webp"
|
|
21
23
|
mimetype = f"{preset_type}/webp"
|
|
22
24
|
|
|
23
|
-
options: ClassVar[
|
|
25
|
+
options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
|
|
24
26
|
"lossless": False,
|
|
25
27
|
"quality": 40,
|
|
26
28
|
"method": 6,
|
|
@@ -39,7 +41,7 @@ class WebpMedium:
|
|
|
39
41
|
ext = "webp"
|
|
40
42
|
mimetype = f"{preset_type}/webp"
|
|
41
43
|
|
|
42
|
-
options: ClassVar[
|
|
44
|
+
options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
|
|
43
45
|
"lossless": False,
|
|
44
46
|
"quality": 50,
|
|
45
47
|
"method": 6,
|
|
@@ -58,7 +60,7 @@ class WebpHigh:
|
|
|
58
60
|
ext = "webp"
|
|
59
61
|
mimetype = f"{preset_type}/webp"
|
|
60
62
|
|
|
61
|
-
options: ClassVar[
|
|
63
|
+
options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
|
|
62
64
|
"lossless": False,
|
|
63
65
|
"quality": 90,
|
|
64
66
|
"method": 6,
|
|
@@ -79,7 +81,7 @@ class GifLow:
|
|
|
79
81
|
ext = "gif"
|
|
80
82
|
mimetype = f"{preset_type}/gif"
|
|
81
83
|
|
|
82
|
-
options: ClassVar[
|
|
84
|
+
options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
|
|
83
85
|
"optimize_level": 3,
|
|
84
86
|
"max_colors": 256,
|
|
85
87
|
"lossiness": 80,
|
|
@@ -102,7 +104,7 @@ class GifMedium:
|
|
|
102
104
|
ext = "gif"
|
|
103
105
|
mimetype = f"{preset_type}/gif"
|
|
104
106
|
|
|
105
|
-
options: ClassVar[
|
|
107
|
+
options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
|
|
106
108
|
"optimize_level": 3,
|
|
107
109
|
"lossiness": 20,
|
|
108
110
|
"no_extensions": True,
|
|
@@ -124,7 +126,7 @@ class GifHigh:
|
|
|
124
126
|
ext = "gif"
|
|
125
127
|
mimetype = f"{preset_type}/gif"
|
|
126
128
|
|
|
127
|
-
options: ClassVar[
|
|
129
|
+
options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
|
|
128
130
|
"optimize_level": 2,
|
|
129
131
|
"lossiness": None,
|
|
130
132
|
"no_extensions": True,
|
|
@@ -143,7 +145,7 @@ class PngLow:
|
|
|
143
145
|
ext = "png"
|
|
144
146
|
mimetype = f"{preset_type}/png"
|
|
145
147
|
|
|
146
|
-
options: ClassVar[
|
|
148
|
+
options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
|
|
147
149
|
"reduce_colors": True,
|
|
148
150
|
"remove_transparency": False,
|
|
149
151
|
"max_colors": 256,
|
|
@@ -162,7 +164,7 @@ class PngMedium:
|
|
|
162
164
|
ext = "png"
|
|
163
165
|
mimetype = f"{preset_type}/png"
|
|
164
166
|
|
|
165
|
-
options: ClassVar[
|
|
167
|
+
options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
|
|
166
168
|
"reduce_colors": False,
|
|
167
169
|
"remove_transparency": False,
|
|
168
170
|
"fast_mode": False,
|
|
@@ -180,7 +182,7 @@ class PngHigh:
|
|
|
180
182
|
ext = "png"
|
|
181
183
|
mimetype = f"{preset_type}/png"
|
|
182
184
|
|
|
183
|
-
options: ClassVar[
|
|
185
|
+
options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
|
|
184
186
|
"reduce_colors": False,
|
|
185
187
|
"remove_transparency": False,
|
|
186
188
|
"fast_mode": True,
|
|
@@ -199,7 +201,7 @@ class JpegLow:
|
|
|
199
201
|
ext = "png"
|
|
200
202
|
mimetype = f"{preset_type}/png"
|
|
201
203
|
|
|
202
|
-
options: ClassVar[
|
|
204
|
+
options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
|
|
203
205
|
"quality": 45,
|
|
204
206
|
"keep_exif": False,
|
|
205
207
|
"fast_mode": True,
|
|
@@ -218,7 +220,7 @@ class JpegMedium:
|
|
|
218
220
|
ext = "jpg"
|
|
219
221
|
mimetype = f"{preset_type}/jpeg"
|
|
220
222
|
|
|
221
|
-
options: ClassVar[
|
|
223
|
+
options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
|
|
222
224
|
"quality": 65,
|
|
223
225
|
"keep_exif": False,
|
|
224
226
|
"fast_mode": True,
|
|
@@ -237,7 +239,7 @@ class JpegHigh:
|
|
|
237
239
|
ext = "jpg"
|
|
238
240
|
mimetype = f"{preset_type}/jpeg"
|
|
239
241
|
|
|
240
|
-
options: ClassVar[
|
|
242
|
+
options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
|
|
241
243
|
"quality": 80,
|
|
242
244
|
"keep_exif": True,
|
|
243
245
|
"fast_mode": True,
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
2
|
# vim: ai ts=4 sts=4 et sw=4 nu
|
|
3
3
|
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
4
6
|
import colorsys
|
|
5
7
|
import io
|
|
6
8
|
import pathlib
|
|
7
9
|
import re
|
|
8
|
-
from typing import Optional,
|
|
10
|
+
from typing import Optional, Union
|
|
9
11
|
|
|
10
12
|
import colorthief
|
|
11
13
|
import PIL.Image
|
|
@@ -13,14 +15,14 @@ import PIL.Image
|
|
|
13
15
|
|
|
14
16
|
def get_colors(
|
|
15
17
|
src: pathlib.Path, use_palette: Optional[bool] = True # noqa: FBT002
|
|
16
|
-
) ->
|
|
18
|
+
) -> tuple[str, str]:
|
|
17
19
|
"""(main, secondary) HTML color codes from an image path"""
|
|
18
20
|
|
|
19
21
|
def rgb_to_hex(r: int, g: int, b: int) -> str:
|
|
20
22
|
"""hexadecimal HTML-friendly color code for RGB tuple"""
|
|
21
23
|
return "#{}{}{}".format(*[str(hex(x)[2:]).zfill(2) for x in (r, g, b)]).upper()
|
|
22
24
|
|
|
23
|
-
def solarize(r: int, g: int, b: int) ->
|
|
25
|
+
def solarize(r: int, g: int, b: int) -> tuple[int, int, int]:
|
|
24
26
|
# calculate solarized color for main
|
|
25
27
|
h, l, s = colorsys.rgb_to_hls( # noqa: E741
|
|
26
28
|
float(r) / 256, float(g) / 256, float(b) / 256
|
|
@@ -71,7 +73,7 @@ def format_for(
|
|
|
71
73
|
def is_valid_image(
|
|
72
74
|
image: Union[pathlib.Path, io.IOBase, bytes],
|
|
73
75
|
imformat: str,
|
|
74
|
-
size: Optional[
|
|
76
|
+
size: Optional[tuple[int, int]] = None,
|
|
75
77
|
) -> bool:
|
|
76
78
|
"""whether image is a valid imformat (PNG) image, optionnaly of requested size"""
|
|
77
79
|
if isinstance(image, bytes):
|
|
@@ -1,12 +1,15 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
2
|
# vim: ai ts=4 sts=4 et sw=4 nu
|
|
3
3
|
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
4
6
|
import pathlib
|
|
5
7
|
import shutil
|
|
6
8
|
import tempfile
|
|
7
|
-
from typing import Optional,
|
|
9
|
+
from typing import Optional, Union
|
|
8
10
|
|
|
9
11
|
from zimscraperlib import logger
|
|
12
|
+
from zimscraperlib.constants import DEFAULT_USER_AGENT
|
|
10
13
|
from zimscraperlib.constants import (
|
|
11
14
|
MAXIMUM_DESCRIPTION_METADATA_LENGTH as MAX_DESC_LENGTH,
|
|
12
15
|
)
|
|
@@ -21,6 +24,7 @@ def handle_user_provided_file(
|
|
|
21
24
|
dest: Optional[pathlib.Path] = None,
|
|
22
25
|
in_dir: Optional[pathlib.Path] = None,
|
|
23
26
|
nocopy: bool = False, # noqa: FBT001, FBT002
|
|
27
|
+
user_agent: Optional[str] = DEFAULT_USER_AGENT,
|
|
24
28
|
) -> Union[pathlib.Path, None]:
|
|
25
29
|
"""path to downloaded or copied a user provided file (URL or path)
|
|
26
30
|
|
|
@@ -42,7 +46,8 @@ def handle_user_provided_file(
|
|
|
42
46
|
|
|
43
47
|
if str(source).startswith("http"):
|
|
44
48
|
logger.debug(f"download {source} -> {dest}")
|
|
45
|
-
|
|
49
|
+
headers = {"User-Agent": user_agent} if user_agent else None
|
|
50
|
+
stream_file(url=str(source), fpath=dest, headers=headers)
|
|
46
51
|
else:
|
|
47
52
|
source = pathlib.Path(source).expanduser().resolve()
|
|
48
53
|
if not source.exists():
|
|
@@ -60,7 +65,7 @@ def compute_descriptions(
|
|
|
60
65
|
default_description: str,
|
|
61
66
|
user_description: Optional[str],
|
|
62
67
|
user_long_description: Optional[str],
|
|
63
|
-
) ->
|
|
68
|
+
) -> tuple[str, Optional[str]]:
|
|
64
69
|
"""Computes short and long descriptions compliant with ZIM standard.
|
|
65
70
|
|
|
66
71
|
Based on provided parameters, the function computes a short and a long description
|
|
@@ -5,8 +5,9 @@ import io
|
|
|
5
5
|
import logging
|
|
6
6
|
import pathlib
|
|
7
7
|
import sys
|
|
8
|
+
from collections.abc import Iterable
|
|
8
9
|
from logging.handlers import RotatingFileHandler
|
|
9
|
-
from typing import
|
|
10
|
+
from typing import Optional
|
|
10
11
|
|
|
11
12
|
from zimscraperlib.constants import NAME
|
|
12
13
|
|
|
@@ -15,13 +15,15 @@
|
|
|
15
15
|
Should your scraper need additional mapping, use mimetypes.add_type() and it will
|
|
16
16
|
be automatically used. """
|
|
17
17
|
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
18
20
|
import mimetypes
|
|
19
21
|
import pathlib
|
|
20
|
-
from typing import
|
|
22
|
+
from typing import Optional, Union
|
|
21
23
|
|
|
22
24
|
ARTICLE_MIME: str = "text/html"
|
|
23
25
|
FALLBACK_MIME: str = "application/octet-stream"
|
|
24
|
-
FONT_MIMES:
|
|
26
|
+
FONT_MIMES: list[str] = [
|
|
25
27
|
"font/ttf",
|
|
26
28
|
"application/font-ttf",
|
|
27
29
|
"font/sfnt",
|
|
@@ -1,18 +1,20 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
2
|
# vim: ai ts=4 sts=4 et sw=4 nu
|
|
3
3
|
|
|
4
|
-
from
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from typing import ClassVar, Optional, Union
|
|
5
7
|
|
|
6
8
|
|
|
7
9
|
class Config(dict):
|
|
8
10
|
VERSION = 1
|
|
9
11
|
ext = "dat"
|
|
10
12
|
mimetype = "application/data"
|
|
11
|
-
options: ClassVar[
|
|
12
|
-
defaults: ClassVar[
|
|
13
|
+
options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {}
|
|
14
|
+
defaults: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
|
|
13
15
|
"-max_muxing_queue_size": "9999"
|
|
14
16
|
}
|
|
15
|
-
mapping: ClassVar[
|
|
17
|
+
mapping: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
|
|
16
18
|
"video_codec": "-codec:v",
|
|
17
19
|
"audio_codec": "-codec:a",
|
|
18
20
|
"max_video_bitrate": "-maxrate",
|
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
2
|
# vim: ai ts=4 sts=4 et sw=4 nu
|
|
3
3
|
|
|
4
|
+
from __future__ import annotations
|
|
4
5
|
|
|
5
6
|
import pathlib
|
|
6
7
|
import shutil
|
|
7
8
|
import subprocess
|
|
8
9
|
import tempfile
|
|
9
|
-
from
|
|
10
|
+
from copy import deepcopy
|
|
11
|
+
from typing import Optional
|
|
10
12
|
|
|
11
13
|
from zimscraperlib import logger
|
|
12
14
|
from zimscraperlib.logging import nicer_args_join
|
|
@@ -15,9 +17,10 @@ from zimscraperlib.logging import nicer_args_join
|
|
|
15
17
|
def _build_ffmpeg_args(
|
|
16
18
|
src_path: pathlib.Path,
|
|
17
19
|
tmp_path: pathlib.Path,
|
|
18
|
-
ffmpeg_args:
|
|
20
|
+
ffmpeg_args: list[str],
|
|
19
21
|
threads: Optional[int],
|
|
20
|
-
) ->
|
|
22
|
+
) -> list[str]:
|
|
23
|
+
ffmpeg_args = deepcopy(ffmpeg_args)
|
|
21
24
|
if threads:
|
|
22
25
|
if "-threads" in ffmpeg_args:
|
|
23
26
|
raise AttributeError("Cannot set the number of threads, already set")
|