zimscraperlib 3.3.0__tar.gz → 3.3.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/.pre-commit-config.yaml +3 -3
  2. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/CHANGELOG.md +25 -0
  3. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/PKG-INFO +12 -11
  4. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/README.md +1 -0
  5. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/contrib/encode_video.py +3 -2
  6. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/pyproject.toml +10 -10
  7. zimscraperlib-3.3.2/src/zimscraperlib/__about__.py +1 -0
  8. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/constants.py +2 -0
  9. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/download.py +8 -8
  10. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/filesystem.py +2 -1
  11. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/fix_ogvjs_dist.py +4 -2
  12. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/i18n.py +8 -6
  13. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/image/optimization.py +3 -2
  14. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/image/presets.py +15 -13
  15. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/image/probing.py +6 -4
  16. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/inputs.py +8 -3
  17. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/logging.py +2 -1
  18. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/types.py +4 -2
  19. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/video/config.py +6 -4
  20. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/video/encoding.py +6 -3
  21. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/video/presets.py +8 -6
  22. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/zim/_libkiwix.py +6 -4
  23. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/zim/archive.py +7 -4
  24. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/zim/creator.py +7 -5
  25. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/zim/filesystem.py +14 -4
  26. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/zim/metadata.py +5 -3
  27. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/conftest.py +5 -0
  28. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/download/test_download.py +4 -2
  29. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/inputs/test_inputs.py +53 -0
  30. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/video/test_encoding.py +21 -3
  31. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/zim/test_metadata.py +4 -1
  32. zimscraperlib-3.3.0/src/zimscraperlib/__about__.py +0 -1
  33. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/.gitignore +0 -0
  34. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/LICENSE +0 -0
  35. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/contrib/README.md +0 -0
  36. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/__init__.py +0 -0
  37. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/html.py +0 -0
  38. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/image/__init__.py +0 -0
  39. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/image/convertion.py +0 -0
  40. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/image/transformation.py +0 -0
  41. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/image/utils.py +0 -0
  42. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/misc.py +0 -0
  43. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/uri.py +0 -0
  44. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/video/__init__.py +0 -0
  45. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/video/probing.py +0 -0
  46. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/zim/__init__.py +0 -0
  47. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/zim/items.py +0 -0
  48. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/src/zimscraperlib/zim/providers.py +0 -0
  49. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tasks.py +0 -0
  50. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/files/DroidSans.ttf +0 -0
  51. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/files/audio.mp3 +0 -0
  52. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/files/blue.jpg +0 -0
  53. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/files/commons.png +0 -0
  54. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/files/commons48.png +0 -0
  55. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/files/mail.gif +0 -0
  56. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/files/ninja.webp +0 -0
  57. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/files/pluto.jpg +0 -0
  58. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/files/square.jpg +0 -0
  59. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/files/square.png +0 -0
  60. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/files/star.svg +0 -0
  61. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/files/video.mkv +0 -0
  62. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/files/video.mp4 +0 -0
  63. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/files/video.webm +0 -0
  64. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/filesystem/test_filesystem.py +0 -0
  65. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/html/conftest.py +0 -0
  66. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/html/test_html.py +0 -0
  67. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/i18n/locale/fr/LC_MESSAGES/messages.po +0 -0
  68. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/i18n/locale/pt/LC_MESSAGES/messages.po +0 -0
  69. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/i18n/test_i18n.py +0 -0
  70. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/image/test_image.py +0 -0
  71. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/logging/conftest.py +0 -0
  72. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/logging/test_logging.py +0 -0
  73. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/misc/test_misc.py +0 -0
  74. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/ogvjs/conftest.py +0 -0
  75. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/ogvjs/test_ogvjs.py +0 -0
  76. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/types/test_types.py +0 -0
  77. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/uri/test_uri.py +0 -0
  78. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/video/conftest.py +0 -0
  79. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/video/test_video.py +0 -0
  80. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/zim/conftest.py +0 -0
  81. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/zim/test_archive.py +0 -0
  82. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/zim/test_fs.py +0 -0
  83. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/zim/test_libkiwix.py +0 -0
  84. {zimscraperlib-3.3.0 → zimscraperlib-3.3.2}/tests/zim/test_zim_creator.py +0 -0
@@ -7,15 +7,15 @@ repos:
7
7
  - id: trailing-whitespace
8
8
  - id: end-of-file-fixer
9
9
  - repo: https://github.com/psf/black
10
- rev: "24.1.1"
10
+ rev: "24.3.0"
11
11
  hooks:
12
12
  - id: black
13
13
  - repo: https://github.com/astral-sh/ruff-pre-commit
14
- rev: v0.2.1
14
+ rev: v0.3.3
15
15
  hooks:
16
16
  - id: ruff
17
17
  - repo: https://github.com/RobertCraigie/pyright-python
18
- rev: v1.1.350
18
+ rev: v1.1.354
19
19
  hooks:
20
20
  - id: pyright
21
21
  name: pyright (system)
@@ -5,6 +5,31 @@ All notable changes to this project are documented in this file.
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0).
7
7
 
8
+ ## [3.3.2] - 2024-03-25
9
+
10
+ ### Added
11
+
12
+ - Add support for `disable_metadata_checks` and `ignore_duplicates` arguments in `make_zim_file` function ("zimwritefs-mode")
13
+
14
+ ### Changed
15
+
16
+ - Relaxed constraints on Python dependencies
17
+ - Upgraded optional dependencies used for test and QA
18
+
19
+ ## [3.3.1] - 2024-02-27
20
+
21
+ ### Added
22
+
23
+ - Set a user-agent for `handle_user_provided_file` #103
24
+
25
+ ### Changed
26
+
27
+ - Migrate to generic syntax in all std collections #140
28
+
29
+ ### Fixed
30
+
31
+ - Do not modify the ffmpeg_args in reencode function #144
32
+
8
33
  ## [3.3.0] - 2024-02-14
9
34
 
10
35
  ### Added
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: zimscraperlib
3
- Version: 3.3.0
3
+ Version: 3.3.2
4
4
  Summary: Collection of python tools to re-use common code across scrapers
5
5
  Project-URL: Donate, https://www.kiwix.org/en/support-us/
6
6
  Project-URL: Homepage, https://github.com/openzim/python-scraperlib
@@ -17,34 +17,34 @@ Classifier: Programming Language :: Python :: 3.11
17
17
  Classifier: Programming Language :: Python :: 3.12
18
18
  Requires-Python: <3.13,>=3.8
19
19
  Requires-Dist: babel<3.0,>=2.9
20
- Requires-Dist: beautifulsoup4<4.10,>=4.9.3
20
+ Requires-Dist: beautifulsoup4<5.0,>=4.9.3
21
21
  Requires-Dist: colorthief==0.2.1
22
22
  Requires-Dist: iso639-lang<3.0,>=2.2.3
23
23
  Requires-Dist: libzim<4.0,>=3.4.0
24
- Requires-Dist: lxml<4.10,>=4.6.3
25
- Requires-Dist: optimize-images<1.6,>=1.3.6
24
+ Requires-Dist: lxml<6.0,>=4.6.3
25
+ Requires-Dist: optimize-images<2.0,>=1.3.6
26
26
  Requires-Dist: python-magic<0.5,>=0.4.3
27
27
  Requires-Dist: python-resize-image<1.2,>=1.1.19
28
28
  Requires-Dist: requests<3.0,>=2.25.1
29
29
  Requires-Dist: yt-dlp
30
30
  Provides-Extra: check
31
- Requires-Dist: pyright==1.1.350; extra == 'check'
31
+ Requires-Dist: pyright==1.1.354; extra == 'check'
32
32
  Provides-Extra: dev
33
- Requires-Dist: debugpy==1.8.0; extra == 'dev'
34
- Requires-Dist: pre-commit==3.5.0; extra == 'dev'
33
+ Requires-Dist: debugpy==1.8.1; extra == 'dev'
34
+ Requires-Dist: pre-commit==3.6.2; extra == 'dev'
35
35
  Requires-Dist: zimscraperlib[check]; extra == 'dev'
36
36
  Requires-Dist: zimscraperlib[lint]; extra == 'dev'
37
37
  Requires-Dist: zimscraperlib[scripts]; extra == 'dev'
38
38
  Requires-Dist: zimscraperlib[test]; extra == 'dev'
39
39
  Provides-Extra: lint
40
- Requires-Dist: black==24.1.1; extra == 'lint'
41
- Requires-Dist: ruff==0.2.1; extra == 'lint'
40
+ Requires-Dist: black==24.3.0; extra == 'lint'
41
+ Requires-Dist: ruff==0.3.3; extra == 'lint'
42
42
  Provides-Extra: scripts
43
43
  Requires-Dist: invoke==2.2.0; extra == 'scripts'
44
44
  Provides-Extra: test
45
- Requires-Dist: coverage==7.4.1; extra == 'test'
45
+ Requires-Dist: coverage==7.4.4; extra == 'test'
46
46
  Requires-Dist: pytest-mock==3.12.0; extra == 'test'
47
- Requires-Dist: pytest==8.0.0; extra == 'test'
47
+ Requires-Dist: pytest==8.1.1; extra == 'test'
48
48
  Description-Content-Type: text/markdown
49
49
 
50
50
  zimscraperlib
@@ -54,6 +54,7 @@ zimscraperlib
54
54
  [![CodeFactor](https://www.codefactor.io/repository/github/openzim/python-scraperlib/badge)](https://www.codefactor.io/repository/github/openzim/python-scraperlib)
55
55
  [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
56
56
  [![PyPI version shields.io](https://img.shields.io/pypi/v/zimscraperlib.svg)](https://pypi.org/project/zimscraperlib/)
57
+ [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/zimscraperlib.svg)](https://pypi.org/project/zimscraperlib)
57
58
  [![codecov](https://codecov.io/gh/openzim/python-scraperlib/branch/master/graph/badge.svg)](https://codecov.io/gh/openzim/python-scraperlib)
58
59
 
59
60
  Collection of python code to re-use across python-based scrapers
@@ -5,6 +5,7 @@ zimscraperlib
5
5
  [![CodeFactor](https://www.codefactor.io/repository/github/openzim/python-scraperlib/badge)](https://www.codefactor.io/repository/github/openzim/python-scraperlib)
6
6
  [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
7
7
  [![PyPI version shields.io](https://img.shields.io/pypi/v/zimscraperlib.svg)](https://pypi.org/project/zimscraperlib/)
8
+ [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/zimscraperlib.svg)](https://pypi.org/project/zimscraperlib)
8
9
  [![codecov](https://codecov.io/gh/openzim/python-scraperlib/branch/master/graph/badge.svg)](https://codecov.io/gh/openzim/python-scraperlib)
9
10
 
10
11
  Collection of python code to re-use across python-based scrapers
@@ -1,6 +1,7 @@
1
+ from __future__ import annotations
2
+
1
3
  import sys
2
4
  from pathlib import Path
3
- from typing import List
4
5
 
5
6
  from zimscraperlib import logger
6
7
  from zimscraperlib.video import presets, reencode
@@ -25,7 +26,7 @@ def encode_video(src_path: Path, dst_path: Path, preset: str):
25
26
  logger.error(f"conversion failed:\n{process.stdout}")
26
27
 
27
28
 
28
- def run(args: List[str] = sys.argv):
29
+ def run(args: list[str] = sys.argv):
29
30
  if len(args) < 4: # noqa: PLR2004
30
31
  print(f"Usage: {args[0]} <src_path> <dst_path> <preset>") # noqa: T201
31
32
  print( # noqa: T201
@@ -15,9 +15,9 @@ dependencies = [
15
15
  "Babel>=2.9,<3.0",
16
16
  "python-magic>=0.4.3,<0.5",
17
17
  "libzim>=3.4.0,<4.0",
18
- "beautifulsoup4>=4.9.3,<4.10", # upgrade to 4.10 and later to be done
19
- "lxml>=4.6.3,<4.10", # upgrade to 4.10 and later to be done
20
- "optimize-images>=1.3.6,<1.6",
18
+ "beautifulsoup4>=4.9.3,<5.0",
19
+ "lxml>=4.6.3,<6.0",
20
+ "optimize-images>=1.3.6,<2.0",
21
21
  # youtube-dl should be updated as frequently as possible
22
22
  "yt-dlp"
23
23
  ]
@@ -36,20 +36,20 @@ scripts = [
36
36
  "invoke==2.2.0",
37
37
  ]
38
38
  lint = [
39
- "black==24.1.1",
40
- "ruff==0.2.1",
39
+ "black==24.3.0",
40
+ "ruff==0.3.3",
41
41
  ]
42
42
  check = [
43
- "pyright==1.1.350",
43
+ "pyright==1.1.354",
44
44
  ]
45
45
  test = [
46
- "pytest==8.0.0",
46
+ "pytest==8.1.1",
47
47
  "pytest-mock==3.12.0",
48
- "coverage==7.4.1",
48
+ "coverage==7.4.4",
49
49
  ]
50
50
  dev = [
51
- "pre-commit==3.5.0",
52
- "debugpy==1.8.0",
51
+ "pre-commit==3.6.2",
52
+ "debugpy==1.8.1",
53
53
  "zimscraperlib[scripts]",
54
54
  "zimscraperlib[lint]",
55
55
  "zimscraperlib[test]",
@@ -0,0 +1 @@
1
+ __version__ = "3.3.2"
@@ -10,6 +10,8 @@ from zimscraperlib.__about__ import __version__
10
10
  ROOT_DIR = pathlib.Path(__file__).parent
11
11
  NAME = pathlib.Path(__file__).parent.name
12
12
  SCRAPER = f"{NAME} {__version__}"
13
+ CONTACT = "dev@openzim.org"
14
+ DEFAULT_USER_AGENT = f"{NAME}/{__version__} ({CONTACT})"
13
15
 
14
16
  UTF8 = "UTF-8"
15
17
 
@@ -7,7 +7,7 @@ import io
7
7
  import pathlib
8
8
  import subprocess
9
9
  from concurrent.futures import Future, ThreadPoolExecutor
10
- from typing import ClassVar, Dict, Optional, Union
10
+ from typing import ClassVar, Optional, Union
11
11
 
12
12
  import requests
13
13
  import yt_dlp as youtube_dl
@@ -34,14 +34,14 @@ class YoutubeDownloader:
34
34
  """shuts down the executor, awaiting completion"""
35
35
  self.executor.shutdown(wait=True)
36
36
 
37
- def _run_youtube_dl(self, url: str, options: Dict) -> None:
37
+ def _run_youtube_dl(self, url: str, options: dict) -> None:
38
38
  with youtube_dl.YoutubeDL(options) as ydl:
39
39
  ydl.download([url])
40
40
 
41
41
  def download(
42
42
  self,
43
43
  url: str,
44
- options: Optional[Dict],
44
+ options: Optional[dict],
45
45
  wait: Optional[bool] = True, # noqa: FBT002
46
46
  ) -> Union[bool, Future]:
47
47
  """Downloads video using initialized executor.
@@ -65,8 +65,8 @@ class YoutubeDownloader:
65
65
 
66
66
 
67
67
  class YoutubeConfig(dict):
68
- options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = {}
69
- defaults: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = {
68
+ options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {}
69
+ defaults: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
70
70
  "writethumbnail": True,
71
71
  "write_all_thumbnails": True,
72
72
  "writesubtitles": True,
@@ -109,14 +109,14 @@ class YoutubeConfig(dict):
109
109
 
110
110
 
111
111
  class BestWebm(YoutubeConfig):
112
- options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = {
112
+ options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
113
113
  "preferredcodec": "webm",
114
114
  "format": "best[ext=webm]/bestvideo[ext=webm]+bestaudio[ext=webm]/best",
115
115
  }
116
116
 
117
117
 
118
118
  class BestMp4(YoutubeConfig):
119
- options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = {
119
+ options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
120
120
  "preferredcodec": "mp4",
121
121
  "format": "best[ext=mp4]/bestvideo[ext=mp4]+bestaudio[ext=m4a]/best",
122
122
  }
@@ -179,7 +179,7 @@ def stream_file(
179
179
  proxies: Optional[dict] = None,
180
180
  only_first_block: Optional[bool] = False, # noqa: FBT002
181
181
  max_retries: Optional[int] = 5,
182
- headers: Optional[Dict[str, str]] = None,
182
+ headers: Optional[dict[str, str]] = None,
183
183
  session: Optional[requests.Session] = None,
184
184
  ) -> tuple[int, requests.structures.CaseInsensitiveDict]: # pyright: ignore
185
185
  """Stream data from a URL to either a BytesIO object or a file
@@ -7,7 +7,8 @@
7
7
 
8
8
  import os
9
9
  import pathlib
10
- from typing import Any, Callable, Optional, Union
10
+ from collections.abc import Callable
11
+ from typing import Any, Optional, Union
11
12
 
12
13
  import magic
13
14
 
@@ -4,10 +4,12 @@
4
4
 
5
5
  """ quick script to fix videojs-ogvjs so that it triggers on webm mimetype """
6
6
 
7
+ from __future__ import annotations
8
+
7
9
  import logging
8
10
  import pathlib
9
11
  import sys
10
- from typing import List, Union
12
+ from typing import Union
11
13
 
12
14
  logging.basicConfig(format="%(levelname)s:%(message)s", level=logging.DEBUG)
13
15
  logger = logging.getLogger(__name__)
@@ -33,7 +35,7 @@ def fix_source_dir(source_vendors_path: Union[pathlib.Path, str]):
33
35
  logger.info("all done.")
34
36
 
35
37
 
36
- def run(args: List[str] = sys.argv):
38
+ def run(args: list[str] = sys.argv):
37
39
  if len(args) < 2: # noqa: PLR2004
38
40
  print(f"Usage: {args[0]} <source_vendors_path>") # noqa: T201
39
41
  print( # noqa: T201
@@ -1,11 +1,13 @@
1
1
  #!/usr/bin/env python3
2
2
  # vim: ai ts=4 sts=4 et sw=4 nu
3
3
 
4
+ from __future__ import annotations
5
+
4
6
  import gettext
5
7
  import locale
6
8
  import pathlib
7
9
  import re
8
- from typing import Dict, Optional, Tuple, Union
10
+ from typing import Optional, Union
9
11
 
10
12
  import babel
11
13
  import iso639
@@ -58,7 +60,7 @@ def setlocale(root_dir: pathlib.Path, locale_name: str):
58
60
  return Locale.setup(root_dir / "locale", locale_name)
59
61
 
60
62
 
61
- def get_iso_lang_data(lang: str) -> Tuple[Dict, Union[Dict, None]]:
63
+ def get_iso_lang_data(lang: str) -> tuple[dict, Union[dict, None]]:
62
64
  """ISO-639-x languages details for lang. Raises NotFound
63
65
 
64
66
  Included keys: iso-639-1, iso-639-2b, iso-639-2t, iso-639-3, iso-639-5
@@ -112,8 +114,8 @@ def get_iso_lang_data(lang: str) -> Tuple[Dict, Union[Dict, None]]:
112
114
 
113
115
 
114
116
  def find_language_names(
115
- query: str, lang_data: Optional[Dict] = None
116
- ) -> Tuple[str, str]:
117
+ query: str, lang_data: Optional[dict] = None
118
+ ) -> tuple[str, str]:
117
119
  """(native, english) language names for lang with help from language_details dict
118
120
 
119
121
  Falls back to English name if available or query if not"""
@@ -140,7 +142,7 @@ def find_language_names(
140
142
  return default, default
141
143
 
142
144
 
143
- def update_with_macro(lang_data: Dict, macro_data: Dict):
145
+ def update_with_macro(lang_data: dict, macro_data: dict):
144
146
  """update empty keys from lang_data with ones of macro_data"""
145
147
  if macro_data:
146
148
  for key, value in macro_data.items():
@@ -151,7 +153,7 @@ def update_with_macro(lang_data: Dict, macro_data: Dict):
151
153
 
152
154
  def get_language_details(
153
155
  query: str, failsafe: Optional[bool] = False # noqa: FBT002
154
- ) -> Dict:
156
+ ) -> dict:
155
157
  """language details dict from query.
156
158
 
157
159
  Raises NotFound or return `und` language details if failsafe
@@ -22,12 +22,13 @@
22
22
  can still run on default settings which give
23
23
  a bit less size than the original images but maintain a high quality. """
24
24
 
25
+ from __future__ import annotations
25
26
 
26
27
  import io
27
28
  import os
28
29
  import pathlib
29
30
  import subprocess
30
- from typing import Optional, Tuple, Union
31
+ from typing import Optional, Union
31
32
 
32
33
  import piexif
33
34
  from optimize_images.img_aux_processing import do_reduce_colors, rebuild_palette
@@ -57,7 +58,7 @@ def optimize_png(
57
58
  max_colors: Optional[int] = 256,
58
59
  fast_mode: Optional[bool] = True, # noqa: FBT002
59
60
  remove_transparency: Optional[bool] = False, # noqa: FBT002
60
- background_color: Optional[Tuple[int, int, int]] = (255, 255, 255),
61
+ background_color: Optional[tuple[int, int, int]] = (255, 255, 255),
61
62
  **options, # noqa: ARG001
62
63
  ) -> Union[pathlib.Path, io.BytesIO]:
63
64
  """method to optimize PNG files using a pure python external optimizer
@@ -1,7 +1,9 @@
1
1
  #!/usr/bin/env python3
2
2
  # vim: ai ts=4 sts=4 et sw=4 nu
3
3
 
4
- from typing import ClassVar, Dict, Optional, Union
4
+ from __future__ import annotations
5
+
6
+ from typing import ClassVar, Optional, Union
5
7
 
6
8
  """ presets for ImageOptimizer in zimscraperlib.image.optimization module """
7
9
 
@@ -20,7 +22,7 @@ class WebpLow:
20
22
  ext = "webp"
21
23
  mimetype = f"{preset_type}/webp"
22
24
 
23
- options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = {
25
+ options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
24
26
  "lossless": False,
25
27
  "quality": 40,
26
28
  "method": 6,
@@ -39,7 +41,7 @@ class WebpMedium:
39
41
  ext = "webp"
40
42
  mimetype = f"{preset_type}/webp"
41
43
 
42
- options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = {
44
+ options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
43
45
  "lossless": False,
44
46
  "quality": 50,
45
47
  "method": 6,
@@ -58,7 +60,7 @@ class WebpHigh:
58
60
  ext = "webp"
59
61
  mimetype = f"{preset_type}/webp"
60
62
 
61
- options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = {
63
+ options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
62
64
  "lossless": False,
63
65
  "quality": 90,
64
66
  "method": 6,
@@ -79,7 +81,7 @@ class GifLow:
79
81
  ext = "gif"
80
82
  mimetype = f"{preset_type}/gif"
81
83
 
82
- options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = {
84
+ options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
83
85
  "optimize_level": 3,
84
86
  "max_colors": 256,
85
87
  "lossiness": 80,
@@ -102,7 +104,7 @@ class GifMedium:
102
104
  ext = "gif"
103
105
  mimetype = f"{preset_type}/gif"
104
106
 
105
- options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = {
107
+ options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
106
108
  "optimize_level": 3,
107
109
  "lossiness": 20,
108
110
  "no_extensions": True,
@@ -124,7 +126,7 @@ class GifHigh:
124
126
  ext = "gif"
125
127
  mimetype = f"{preset_type}/gif"
126
128
 
127
- options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = {
129
+ options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
128
130
  "optimize_level": 2,
129
131
  "lossiness": None,
130
132
  "no_extensions": True,
@@ -143,7 +145,7 @@ class PngLow:
143
145
  ext = "png"
144
146
  mimetype = f"{preset_type}/png"
145
147
 
146
- options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = {
148
+ options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
147
149
  "reduce_colors": True,
148
150
  "remove_transparency": False,
149
151
  "max_colors": 256,
@@ -162,7 +164,7 @@ class PngMedium:
162
164
  ext = "png"
163
165
  mimetype = f"{preset_type}/png"
164
166
 
165
- options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = {
167
+ options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
166
168
  "reduce_colors": False,
167
169
  "remove_transparency": False,
168
170
  "fast_mode": False,
@@ -180,7 +182,7 @@ class PngHigh:
180
182
  ext = "png"
181
183
  mimetype = f"{preset_type}/png"
182
184
 
183
- options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = {
185
+ options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
184
186
  "reduce_colors": False,
185
187
  "remove_transparency": False,
186
188
  "fast_mode": True,
@@ -199,7 +201,7 @@ class JpegLow:
199
201
  ext = "png"
200
202
  mimetype = f"{preset_type}/png"
201
203
 
202
- options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = {
204
+ options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
203
205
  "quality": 45,
204
206
  "keep_exif": False,
205
207
  "fast_mode": True,
@@ -218,7 +220,7 @@ class JpegMedium:
218
220
  ext = "jpg"
219
221
  mimetype = f"{preset_type}/jpeg"
220
222
 
221
- options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = {
223
+ options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
222
224
  "quality": 65,
223
225
  "keep_exif": False,
224
226
  "fast_mode": True,
@@ -237,7 +239,7 @@ class JpegHigh:
237
239
  ext = "jpg"
238
240
  mimetype = f"{preset_type}/jpeg"
239
241
 
240
- options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = {
242
+ options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
241
243
  "quality": 80,
242
244
  "keep_exif": True,
243
245
  "fast_mode": True,
@@ -1,11 +1,13 @@
1
1
  #!/usr/bin/env python3
2
2
  # vim: ai ts=4 sts=4 et sw=4 nu
3
3
 
4
+ from __future__ import annotations
5
+
4
6
  import colorsys
5
7
  import io
6
8
  import pathlib
7
9
  import re
8
- from typing import Optional, Tuple, Union
10
+ from typing import Optional, Union
9
11
 
10
12
  import colorthief
11
13
  import PIL.Image
@@ -13,14 +15,14 @@ import PIL.Image
13
15
 
14
16
  def get_colors(
15
17
  src: pathlib.Path, use_palette: Optional[bool] = True # noqa: FBT002
16
- ) -> Tuple[str, str]:
18
+ ) -> tuple[str, str]:
17
19
  """(main, secondary) HTML color codes from an image path"""
18
20
 
19
21
  def rgb_to_hex(r: int, g: int, b: int) -> str:
20
22
  """hexadecimal HTML-friendly color code for RGB tuple"""
21
23
  return "#{}{}{}".format(*[str(hex(x)[2:]).zfill(2) for x in (r, g, b)]).upper()
22
24
 
23
- def solarize(r: int, g: int, b: int) -> Tuple[int, int, int]:
25
+ def solarize(r: int, g: int, b: int) -> tuple[int, int, int]:
24
26
  # calculate solarized color for main
25
27
  h, l, s = colorsys.rgb_to_hls( # noqa: E741
26
28
  float(r) / 256, float(g) / 256, float(b) / 256
@@ -71,7 +73,7 @@ def format_for(
71
73
  def is_valid_image(
72
74
  image: Union[pathlib.Path, io.IOBase, bytes],
73
75
  imformat: str,
74
- size: Optional[Tuple[int, int]] = None,
76
+ size: Optional[tuple[int, int]] = None,
75
77
  ) -> bool:
76
78
  """whether image is a valid imformat (PNG) image, optionnaly of requested size"""
77
79
  if isinstance(image, bytes):
@@ -1,12 +1,15 @@
1
1
  #!/usr/bin/env python3
2
2
  # vim: ai ts=4 sts=4 et sw=4 nu
3
3
 
4
+ from __future__ import annotations
5
+
4
6
  import pathlib
5
7
  import shutil
6
8
  import tempfile
7
- from typing import Optional, Tuple, Union
9
+ from typing import Optional, Union
8
10
 
9
11
  from zimscraperlib import logger
12
+ from zimscraperlib.constants import DEFAULT_USER_AGENT
10
13
  from zimscraperlib.constants import (
11
14
  MAXIMUM_DESCRIPTION_METADATA_LENGTH as MAX_DESC_LENGTH,
12
15
  )
@@ -21,6 +24,7 @@ def handle_user_provided_file(
21
24
  dest: Optional[pathlib.Path] = None,
22
25
  in_dir: Optional[pathlib.Path] = None,
23
26
  nocopy: bool = False, # noqa: FBT001, FBT002
27
+ user_agent: Optional[str] = DEFAULT_USER_AGENT,
24
28
  ) -> Union[pathlib.Path, None]:
25
29
  """path to downloaded or copied a user provided file (URL or path)
26
30
 
@@ -42,7 +46,8 @@ def handle_user_provided_file(
42
46
 
43
47
  if str(source).startswith("http"):
44
48
  logger.debug(f"download {source} -> {dest}")
45
- stream_file(url=str(source), fpath=dest)
49
+ headers = {"User-Agent": user_agent} if user_agent else None
50
+ stream_file(url=str(source), fpath=dest, headers=headers)
46
51
  else:
47
52
  source = pathlib.Path(source).expanduser().resolve()
48
53
  if not source.exists():
@@ -60,7 +65,7 @@ def compute_descriptions(
60
65
  default_description: str,
61
66
  user_description: Optional[str],
62
67
  user_long_description: Optional[str],
63
- ) -> Tuple[str, Optional[str]]:
68
+ ) -> tuple[str, Optional[str]]:
64
69
  """Computes short and long descriptions compliant with ZIM standard.
65
70
 
66
71
  Based on provided parameters, the function computes a short and a long description
@@ -5,8 +5,9 @@ import io
5
5
  import logging
6
6
  import pathlib
7
7
  import sys
8
+ from collections.abc import Iterable
8
9
  from logging.handlers import RotatingFileHandler
9
- from typing import Iterable, Optional
10
+ from typing import Optional
10
11
 
11
12
  from zimscraperlib.constants import NAME
12
13
 
@@ -15,13 +15,15 @@
15
15
  Should your scraper need additional mapping, use mimetypes.add_type() and it will
16
16
  be automatically used. """
17
17
 
18
+ from __future__ import annotations
19
+
18
20
  import mimetypes
19
21
  import pathlib
20
- from typing import List, Optional, Union
22
+ from typing import Optional, Union
21
23
 
22
24
  ARTICLE_MIME: str = "text/html"
23
25
  FALLBACK_MIME: str = "application/octet-stream"
24
- FONT_MIMES: List[str] = [
26
+ FONT_MIMES: list[str] = [
25
27
  "font/ttf",
26
28
  "application/font-ttf",
27
29
  "font/sfnt",
@@ -1,18 +1,20 @@
1
1
  #!/usr/bin/env python3
2
2
  # vim: ai ts=4 sts=4 et sw=4 nu
3
3
 
4
- from typing import ClassVar, Dict, Optional, Union
4
+ from __future__ import annotations
5
+
6
+ from typing import ClassVar, Optional, Union
5
7
 
6
8
 
7
9
  class Config(dict):
8
10
  VERSION = 1
9
11
  ext = "dat"
10
12
  mimetype = "application/data"
11
- options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = {}
12
- defaults: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = {
13
+ options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {}
14
+ defaults: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
13
15
  "-max_muxing_queue_size": "9999"
14
16
  }
15
- mapping: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = {
17
+ mapping: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
16
18
  "video_codec": "-codec:v",
17
19
  "audio_codec": "-codec:a",
18
20
  "max_video_bitrate": "-maxrate",
@@ -1,12 +1,14 @@
1
1
  #!/usr/bin/env python3
2
2
  # vim: ai ts=4 sts=4 et sw=4 nu
3
3
 
4
+ from __future__ import annotations
4
5
 
5
6
  import pathlib
6
7
  import shutil
7
8
  import subprocess
8
9
  import tempfile
9
- from typing import List, Optional
10
+ from copy import deepcopy
11
+ from typing import Optional
10
12
 
11
13
  from zimscraperlib import logger
12
14
  from zimscraperlib.logging import nicer_args_join
@@ -15,9 +17,10 @@ from zimscraperlib.logging import nicer_args_join
15
17
  def _build_ffmpeg_args(
16
18
  src_path: pathlib.Path,
17
19
  tmp_path: pathlib.Path,
18
- ffmpeg_args: List[str],
20
+ ffmpeg_args: list[str],
19
21
  threads: Optional[int],
20
- ) -> List[str]:
22
+ ) -> list[str]:
23
+ ffmpeg_args = deepcopy(ffmpeg_args)
21
24
  if threads:
22
25
  if "-threads" in ffmpeg_args:
23
26
  raise AttributeError("Cannot set the number of threads, already set")