zimscraperlib 3.4.0__tar.gz → 4.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/.pre-commit-config.yaml +1 -0
  2. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/CHANGELOG.md +39 -0
  3. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/PKG-INFO +4 -3
  4. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/README.md +0 -2
  5. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/pyproject.toml +6 -0
  6. zimscraperlib-4.0.0/src/zimscraperlib/__about__.py +1 -0
  7. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/download.py +20 -16
  8. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/filesystem.py +1 -1
  9. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/html.py +10 -10
  10. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/i18n.py +69 -68
  11. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/image/__init__.py +1 -1
  12. zimscraperlib-3.4.0/src/zimscraperlib/image/convertion.py → zimscraperlib-4.0.0/src/zimscraperlib/image/conversion.py +44 -9
  13. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/image/optimization.py +81 -51
  14. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/image/probing.py +38 -13
  15. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/image/transformation.py +8 -4
  16. zimscraperlib-4.0.0/src/zimscraperlib/image/utils.py +21 -0
  17. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/inputs.py +26 -1
  18. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/logging.py +22 -21
  19. zimscraperlib-4.0.0/src/zimscraperlib/misc.py +12 -0
  20. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/types.py +4 -6
  21. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/uri.py +9 -8
  22. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/video/encoding.py +8 -7
  23. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/zim/_libkiwix.py +3 -3
  24. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/zim/archive.py +1 -1
  25. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/zim/creator.py +88 -24
  26. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/zim/filesystem.py +23 -17
  27. zimscraperlib-4.0.0/src/zimscraperlib/zim/indexing.py +119 -0
  28. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/zim/items.py +78 -2
  29. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/zim/metadata.py +34 -13
  30. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/zim/providers.py +2 -4
  31. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/conftest.py +26 -0
  32. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/download/test_download.py +8 -8
  33. zimscraperlib-4.0.0/tests/files/encrypted.pdf +0 -0
  34. zimscraperlib-4.0.0/tests/files/encrypted.txt +3 -0
  35. zimscraperlib-4.0.0/tests/files/milderm.pdf +15239 -37
  36. zimscraperlib-4.0.0/tests/files/milderm.txt +47481 -0
  37. zimscraperlib-4.0.0/tests/files/single_wave_icon.gbr +0 -0
  38. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/html/conftest.py +20 -0
  39. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/html/test_html.py +3 -1
  40. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/i18n/test_i18n.py +57 -25
  41. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/image/test_image.py +203 -67
  42. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/inputs/test_inputs.py +30 -1
  43. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/video/test_video.py +2 -1
  44. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/zim/test_fs.py +2 -0
  45. zimscraperlib-4.0.0/tests/zim/test_indexing.py +354 -0
  46. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/zim/test_libkiwix.py +6 -7
  47. zimscraperlib-4.0.0/tests/zim/test_metadata.py +63 -0
  48. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/zim/test_zim_creator.py +74 -14
  49. zimscraperlib-3.4.0/src/zimscraperlib/__about__.py +0 -1
  50. zimscraperlib-3.4.0/src/zimscraperlib/image/utils.py +0 -20
  51. zimscraperlib-3.4.0/src/zimscraperlib/misc.py +0 -8
  52. zimscraperlib-3.4.0/tests/i18n/locale/fr/LC_MESSAGES/messages.po +0 -23
  53. zimscraperlib-3.4.0/tests/i18n/locale/pt/LC_MESSAGES/messages.po +0 -24
  54. zimscraperlib-3.4.0/tests/zim/test_metadata.py +0 -34
  55. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/.gitignore +0 -0
  56. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/LICENSE +0 -0
  57. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/contrib/README.md +0 -0
  58. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/contrib/encode_video.py +0 -0
  59. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/__init__.py +0 -0
  60. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/constants.py +0 -0
  61. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/fix_ogvjs_dist.py +0 -0
  62. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/image/presets.py +0 -0
  63. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/video/__init__.py +0 -0
  64. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/video/config.py +0 -0
  65. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/video/presets.py +0 -0
  66. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/video/probing.py +0 -0
  67. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/zim/__init__.py +0 -0
  68. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tasks.py +0 -0
  69. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/files/DroidSans.ttf +0 -0
  70. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/files/audio.mp3 +0 -0
  71. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/files/blue.jpg +0 -0
  72. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/files/commons.png +0 -0
  73. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/files/commons48.png +0 -0
  74. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/files/mail.gif +0 -0
  75. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/files/ninja.webp +0 -0
  76. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/files/pluto.jpg +0 -0
  77. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/files/square.jpg +0 -0
  78. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/files/square.png +0 -0
  79. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/files/star.svg +0 -0
  80. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/files/video.mkv +0 -0
  81. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/files/video.mp4 +0 -0
  82. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/files/video.webm +0 -0
  83. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/filesystem/test_filesystem.py +0 -0
  84. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/logging/conftest.py +0 -0
  85. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/logging/test_logging.py +0 -0
  86. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/misc/test_misc.py +0 -0
  87. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/ogvjs/conftest.py +0 -0
  88. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/ogvjs/test_ogvjs.py +0 -0
  89. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/types/test_types.py +0 -0
  90. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/uri/test_uri.py +0 -0
  91. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/video/conftest.py +0 -0
  92. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/video/test_encoding.py +0 -0
  93. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/zim/conftest.py +0 -0
  94. {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/zim/test_archive.py +0 -0
@@ -1,5 +1,6 @@
1
1
  # See https://pre-commit.com for more information
2
2
  # See https://pre-commit.com/hooks.html for more hooks
3
+ exclude: ^tests/files # these are raw test files, no need to mess with them
3
4
  repos:
4
5
  - repo: https://github.com/pre-commit/pre-commit-hooks
5
6
  rev: v4.5.0
@@ -5,6 +5,45 @@ All notable changes to this project are documented in this file.
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0).
7
7
 
8
+ ## [4.0.0] - 2024-08-05
9
+
10
+ ### Added
11
+
12
+ - Add utility function to compute ZIM Tags #164, including deduplication #156
13
+ - Metadata does not automatically drops control characters #159
14
+ - New `indexing.IndexData` class to hold title, content and keywords to pass to libzim to index an item
15
+ - Automatically index PDF documents content #167
16
+ - Automatically set proper title on PDF documents #168
17
+ - Expose new `optimization.get_optimization_method` to get the proper optimization method to call for a given image format
18
+ - Add `optimization.get_optimization_method` to get the proper optimization method to call for a given image format
19
+ - New `creator.Creator.convert_and_check_metadata` to convert metadata to bytes or str for known use cases and check proper type is passed to libzim
20
+ - Add svg2png image conversion function #113
21
+ - Add `conversion.convert_svg2png` image conversion function + support for SVG in `probing.format_for` #113
22
+ - Add `i18n.Lang` class used as typed result of i18n operations #151
23
+
24
+ ### Changed
25
+
26
+ - **BREAKING** Renamed `zimscraperlib.image.convertion` to `zimscraperlib.image.conversion` to fix typo
27
+ - **BREAKING** Many changes in type hints to match the real underlying code
28
+ - **BREAKING** Force all boolean arguments (and some other non-obvious parameters) to be keyword-only in function calls for clarity / disambiguation (see ruff rule FBT002)
29
+ - Prefer to use `IO[bytes]` to `io.BytesIO` when possible since it is more generic
30
+ - **BREAKING** `i18n.NotFound` renamed `i18n.NotFoundError`
31
+ - **BREAKING** `types.get_mime_for_name` now returns `str | None`
32
+ - **BREAKING** `creator.Creator.add_metadata` and `creator.Creator.validate_metadata` now only accepts `bytes | str` as value (it must have been converted before call)
33
+ - **BREAKING** second argument of `creator.Creator.add_metadata` has been renamed to `value` instead of `content` to align with other methods
34
+ - When a type issue arises in metadata checks, wrong value type is displayed in exception
35
+ - **BREAKING** `i18n.get_language_details()`, `i18n.get_iso_lang_data()`, `i18n.find_language_names()` and `i18n.update_with_macro` now process / return a new typed `Lang` class #151
36
+ - **BREAKING** Rename `i18.NotFound` to `i18n.NotFoundError`
37
+
38
+ ### Removed
39
+ - **BREAKING** Remove translation features in `i18n`: `Locale` class + `_` and `setlocale` functions #134
40
+
41
+ ### Fixed
42
+
43
+ - Metadata length validation is buggy for unicode strings #158
44
+ - Pillow 10.4.0 reveals improper type hints for image probing functions #177
45
+ - Enhance error when locale fails to setup #157
46
+
8
47
  ## [3.4.0] - 2024-06-21
9
48
 
10
49
  ### Added
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: zimscraperlib
3
- Version: 3.4.0
3
+ Version: 4.0.0
4
4
  Summary: Collection of python tools to re-use common code across scrapers
5
5
  Project-URL: Donate, https://www.kiwix.org/en/support-us/
6
6
  Project-URL: Homepage, https://github.com/openzim/python-scraperlib
@@ -20,13 +20,16 @@ Classifier: Programming Language :: Python :: 3.12
20
20
  Requires-Python: <3.13,>=3.8
21
21
  Requires-Dist: babel<3.0,>=2.9
22
22
  Requires-Dist: beautifulsoup4<5.0,>=4.9.3
23
+ Requires-Dist: cairosvg<3.0,>=2.2.0
23
24
  Requires-Dist: colorthief==0.2.1
24
25
  Requires-Dist: iso639-lang<3.0,>=2.2.3
25
26
  Requires-Dist: libzim<4.0,>=3.4.0
26
27
  Requires-Dist: lxml<6.0,>=4.6.3
27
28
  Requires-Dist: optimize-images<2.0,>=1.3.6
29
+ Requires-Dist: pymupdf<2.0,>=1.24.0
28
30
  Requires-Dist: python-magic<0.5,>=0.4.3
29
31
  Requires-Dist: python-resize-image<1.2,>=1.1.19
32
+ Requires-Dist: regex>=2020.7.14
30
33
  Requires-Dist: requests<3.0,>=2.25.1
31
34
  Requires-Dist: yt-dlp
32
35
  Provides-Extra: check
@@ -103,8 +106,6 @@ sudo apt install libmagic1 wget ffmpeg \
103
106
  apk add ffmpeg gifsicle libmagic wget libjpeg
104
107
  ```
105
108
 
106
- **Nota:** i18n features do not work on Alpine, see https://github.com/openzim/python-scraperlib/issues/134 ; there is one corresponding test which is failing.
107
-
108
109
  # Contribution
109
110
 
110
111
  This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing).
@@ -51,8 +51,6 @@ sudo apt install libmagic1 wget ffmpeg \
51
51
  apk add ffmpeg gifsicle libmagic wget libjpeg
52
52
  ```
53
53
 
54
- **Nota:** i18n features do not work on Alpine, see https://github.com/openzim/python-scraperlib/issues/134 ; there is one corresponding test which is failing.
55
-
56
54
  # Contribution
57
55
 
58
56
  This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing).
@@ -18,6 +18,12 @@ dependencies = [
18
18
  "beautifulsoup4>=4.9.3,<5.0",
19
19
  "lxml>=4.6.3,<6.0",
20
20
  "optimize-images>=1.3.6,<2.0",
21
+ # regex has no upper-bound due to "date-based" release numbers, no semver, so their
22
+ # promise is that they will never (or always) break the API, and the API is very
23
+ # limited and we use only a very small subset of it.
24
+ "regex>=2020.7.14",
25
+ "pymupdf>=1.24.0,<2.0",
26
+ "CairoSVG>=2.2.0,<3.0",
21
27
  # youtube-dl should be updated as frequently as possible
22
28
  "yt-dlp"
23
29
  ]
@@ -0,0 +1 @@
1
+ __version__ = "4.0.0"
@@ -3,13 +3,15 @@
3
3
 
4
4
  from __future__ import annotations
5
5
 
6
- import io
7
6
  import pathlib
8
7
  import subprocess
9
8
  from concurrent.futures import Future, ThreadPoolExecutor
10
- from typing import ClassVar
9
+ from typing import IO, ClassVar
11
10
 
12
11
  import requests
12
+ import requests.adapters
13
+ import requests.structures
14
+ import urllib3.util
13
15
  import yt_dlp as youtube_dl
14
16
 
15
17
  from zimscraperlib import logger
@@ -42,7 +44,8 @@ class YoutubeDownloader:
42
44
  self,
43
45
  url: str,
44
46
  options: dict | None,
45
- wait: bool | None = True, # noqa: FBT002
47
+ *,
48
+ wait: bool | None = True,
46
49
  ) -> bool | Future:
47
50
  """Downloads video using initialized executor.
48
51
 
@@ -52,9 +55,7 @@ class YoutubeDownloader:
52
55
 
53
56
  Returns download result of future (wait=False)"""
54
57
 
55
- future = self.executor.submit(
56
- self._run_youtube_dl, url, options # pyright: ignore
57
- )
58
+ future = self.executor.submit(self._run_youtube_dl, url, options or {})
58
59
  if not wait:
59
60
  return future
60
61
  if not future.exception():
@@ -143,8 +144,8 @@ def save_large_file(url: str, fpath: pathlib.Path) -> None:
143
144
 
144
145
  def _get_retry_adapter(
145
146
  max_retries: int | None = 5,
146
- ) -> requests.adapters.BaseAdapter: # pyright: ignore
147
- retries = requests.packages.urllib3.util.retry.Retry( # pyright: ignore
147
+ ) -> requests.adapters.BaseAdapter:
148
+ retries = urllib3.util.retry.Retry(
148
149
  total=max_retries, # total number of retries
149
150
  connect=max_retries, # connection errors
150
151
  read=max_retries, # read errors
@@ -161,7 +162,7 @@ def _get_retry_adapter(
161
162
  ], # force retry on the following codes
162
163
  )
163
164
 
164
- return requests.adapters.HTTPAdapter(max_retries=retries) # pyright: ignore
165
+ return requests.adapters.HTTPAdapter(max_retries=retries)
165
166
 
166
167
 
167
168
  def get_session(max_retries: int | None = 5) -> requests.Session:
@@ -174,14 +175,15 @@ def get_session(max_retries: int | None = 5) -> requests.Session:
174
175
  def stream_file(
175
176
  url: str,
176
177
  fpath: pathlib.Path | None = None,
177
- byte_stream: io.BytesIO | None = None,
178
+ byte_stream: IO[bytes] | None = None,
178
179
  block_size: int | None = 1024,
179
180
  proxies: dict | None = None,
180
- only_first_block: bool | None = False, # noqa: FBT002
181
181
  max_retries: int | None = 5,
182
182
  headers: dict[str, str] | None = None,
183
183
  session: requests.Session | None = None,
184
- ) -> tuple[int, requests.structures.CaseInsensitiveDict]: # pyright: ignore
184
+ *,
185
+ only_first_block: bool | None = False,
186
+ ) -> tuple[int, requests.structures.CaseInsensitiveDict]:
185
187
  """Stream data from a URL to either a BytesIO object or a file
186
188
  Arguments -
187
189
  fpath - Path of the file where data is sent
@@ -212,12 +214,14 @@ def stream_file(
212
214
  total_downloaded = 0
213
215
  if fpath is not None:
214
216
  fp = open(fpath, "wb")
215
- else:
217
+ elif (
218
+ byte_stream is not None
219
+ ): # pragma: no branch (we use a precise condition to help type checker)
216
220
  fp = byte_stream
217
221
 
218
222
  for data in resp.iter_content(block_size):
219
223
  total_downloaded += len(data)
220
- fp.write(data) # pyright: ignore
224
+ fp.write(data)
221
225
 
222
226
  # stop downloading/reading if we're just testing first block
223
227
  if only_first_block:
@@ -226,7 +230,7 @@ def stream_file(
226
230
  logger.debug(f"Downloaded {total_downloaded} bytes from {url}")
227
231
 
228
232
  if fpath:
229
- fp.close() # pyright: ignore
233
+ fp.close()
230
234
  else:
231
- fp.seek(0) # pyright: ignore
235
+ fp.seek(0)
232
236
  return total_downloaded, resp.headers
@@ -30,7 +30,7 @@ def get_file_mimetype(fpath: pathlib.Path) -> str:
30
30
  return get_content_mimetype(fh.read(2048))
31
31
 
32
32
 
33
- def get_content_mimetype(content: bytes) -> str:
33
+ def get_content_mimetype(content: bytes | str) -> str:
34
34
  """MIME Type of content retrieved from magic headers"""
35
35
 
36
36
  try:
@@ -7,24 +7,22 @@ from __future__ import annotations
7
7
  import pathlib
8
8
  from typing import BinaryIO, TextIO
9
9
 
10
- from bs4 import BeautifulSoup
10
+ from bs4 import BeautifulSoup, element
11
11
 
12
12
  from zimscraperlib.types import ARTICLE_MIME
13
13
 
14
14
 
15
- def find_title_in(content: str | BinaryIO | TextIO, mime_type: str) -> str:
15
+ def find_title_in(content: str | BinaryIO | TextIO, mime_type: str | None) -> str:
16
16
  """Extracted title from HTML content
17
17
 
18
18
  blank on failure to extract and non-HTML files"""
19
19
  if mime_type != ARTICLE_MIME:
20
20
  return ""
21
- try:
22
- return BeautifulSoup(content, "lxml").find("title").text # pyright: ignore
23
- except Exception:
24
- return ""
21
+ title_tag = BeautifulSoup(content, "lxml").find("title")
22
+ return title_tag.text if title_tag else ""
25
23
 
26
24
 
27
- def find_title_in_file(fpath: pathlib.Path, mime_type: str) -> str:
25
+ def find_title_in_file(fpath: pathlib.Path, mime_type: str | None) -> str:
28
26
  """Extracted title from an HTML file"""
29
27
  try:
30
28
  with open(fpath) as fh:
@@ -45,15 +43,17 @@ def find_language_in(content: str | BinaryIO | TextIO, mime_type: str) -> str:
45
43
  for key in keylist:
46
44
  node = soup.find(nodename)
47
45
  if node:
48
- if not node.has_attr(key): # pyright: ignore
46
+ if not isinstance(node, element.Tag) or (
47
+ isinstance(node, element.Tag) and not node.has_attr(key)
48
+ ):
49
49
  continue
50
50
  if (
51
51
  nodename == "meta"
52
- and not node.attrs.get("http-equiv", "").lower() # pyright: ignore
52
+ and not node.attrs.get("http-equiv", "").lower()
53
53
  == "content-language"
54
54
  ):
55
55
  continue
56
- return node.attrs[key] # pyright: ignore
56
+ return node.attrs[key]
57
57
  return ""
58
58
 
59
59
 
@@ -3,9 +3,6 @@
3
3
 
4
4
  from __future__ import annotations
5
5
 
6
- import gettext
7
- import locale
8
- import pathlib
9
6
  import re
10
7
 
11
8
  import babel
@@ -15,57 +12,67 @@ import iso639.exceptions
15
12
  ISO_LEVELS = ["1", "2b", "2t", "3", "5"]
16
13
 
17
14
 
18
- class NotFound(ValueError): # noqa: N818
15
+ class NotFoundError(ValueError):
19
16
  pass
20
17
 
21
18
 
22
- class Locale:
23
- short = "en"
24
- name = "en_US.UTF-8"
25
- locale_dir = None
26
- domain = "messages"
27
- translation = gettext.translation("messages", fallback=True)
19
+ class Lang(dict):
28
20
 
29
- @classmethod
30
- def setup(cls, locale_dir: pathlib.Path, locale_name: str):
31
- cls.name = locale_name
32
- cls.locale_dir = str(locale_dir)
21
+ @property
22
+ def iso_639_1(self) -> str | None:
23
+ """ISO-639-1 language code"""
24
+ return self["iso-639-1"]
33
25
 
34
- if "." in locale_name:
35
- cls.lang, cls.encoding = locale_name.split(".")
36
- else:
37
- cls.lang, cls.encoding = locale_name, "UTF-8"
26
+ @property
27
+ def iso_639_2b(self) -> str | None:
28
+ """ISO-639-2b language code"""
29
+ return self["iso-639-2b"]
38
30
 
39
- computed = locale.setlocale(locale.LC_ALL, (cls.lang, cls.encoding))
31
+ @property
32
+ def iso_639_2t(self) -> str | None:
33
+ """ISO-639-2t language code"""
34
+ return self["iso-639-2t"]
40
35
 
41
- gettext.bindtextdomain(cls.domain, cls.locale_dir)
42
- gettext.textdomain(cls.domain)
36
+ @property
37
+ def iso_639_3(self) -> str | None:
38
+ """ISO-639-3 language code"""
39
+ return self["iso-639-3"]
43
40
 
44
- cls.translation = gettext.translation(
45
- cls.domain, cls.locale_dir, languages=[cls.lang], fallback=True
46
- )
47
- return computed
41
+ @property
42
+ def iso_639_5(self) -> str | None:
43
+ """ISO-639-5 language code"""
44
+ return self["iso-639-5"]
48
45
 
46
+ @property
47
+ def english(self) -> str:
48
+ """language name in English"""
49
+ return self["english"]
49
50
 
50
- def _(text: str) -> str:
51
- """translates text according to setup'd locale"""
52
- return Locale.translation.gettext(text)
51
+ @property
52
+ def native(self) -> str:
53
+ """language name in native language"""
54
+ return self["native"]
53
55
 
56
+ @property
57
+ def iso_types(self) -> list[str]:
58
+ """list of supported iso types"""
59
+ return self["iso_types"]
54
60
 
55
- def setlocale(root_dir: pathlib.Path, locale_name: str):
56
- """set the desired locale for gettext.
61
+ @property
62
+ def query(self) -> list[str]:
63
+ """Query issued for these language details"""
64
+ return self["query"]
57
65
 
58
- call this early"""
59
- return Locale.setup(root_dir / "locale", locale_name)
66
+ @property
67
+ def querytype(self) -> list[str]:
68
+ """Type of query issued to retrieve language details"""
69
+ return self["querytype"]
60
70
 
61
71
 
62
- def get_iso_lang_data(lang: str) -> tuple[dict, dict | None]:
63
- """ISO-639-x languages details for lang. Raises NotFound
72
+ def get_iso_lang_data(lang: str) -> tuple[Lang, Lang | None]:
73
+ """ISO-639-x languages details for lang. Raises NotFoundError
64
74
 
65
- Included keys: iso-639-1, iso-639-2b, iso-639-2t, iso-639-3, iso-639-5
66
- english, iso_types
67
-
68
- See get_language_details() for details"""
75
+ Returns a tuple (main_language, macro_language | None)"""
69
76
 
70
77
  iso_types = []
71
78
 
@@ -75,7 +82,7 @@ def get_iso_lang_data(lang: str) -> tuple[dict, dict | None]:
75
82
  iso639.exceptions.InvalidLanguageValue,
76
83
  iso639.exceptions.DeprecatedLanguageValue,
77
84
  ) as exc:
78
- raise NotFound("Not a valid iso language name/code") from exc
85
+ raise NotFoundError("Not a valid iso language name/code") from exc
79
86
 
80
87
  def replace_types(new_type: str) -> str:
81
88
  # convert new iso_types from iso639-lang Pypi package to old iso_types from
@@ -99,9 +106,9 @@ def get_iso_lang_data(lang: str) -> tuple[dict, dict | None]:
99
106
  if str(getattr(isolang, code_type)).lower() == lang.lower():
100
107
  iso_types.append(replace_types(code_type))
101
108
 
102
- lang_data = {
103
- f"iso-639-{lang_}": getattr(isolang, f"pt{lang_}") for lang_ in ISO_LEVELS
104
- }
109
+ lang_data = Lang(
110
+ **{f"iso-639-{lang_}": getattr(isolang, f"pt{lang_}") for lang_ in ISO_LEVELS}
111
+ )
105
112
  lang_data.update({"english": isolang.name, "iso_types": iso_types})
106
113
 
107
114
  if isolang.macro():
@@ -112,17 +119,20 @@ def get_iso_lang_data(lang: str) -> tuple[dict, dict | None]:
112
119
  return lang_data, None
113
120
 
114
121
 
115
- def find_language_names(query: str, lang_data: dict | None = None) -> tuple[str, str]:
116
- """(native, english) language names for lang with help from language_details dict
122
+ def find_language_names(query: str, lang_data: Lang | None = None) -> tuple[str, str]:
123
+ """(native, english) language names for lang with help from lang_data
117
124
 
118
125
  Falls back to English name if available or query if not"""
119
126
  if lang_data is None:
120
- lang_data = get_language_details(query, failsafe=True) or {}
127
+ lang_data = get_language_details(query, failsafe=True)
128
+ if not lang_data:
129
+ return query, query
130
+
121
131
  try:
122
132
  query_locale = babel.Locale.parse(query)
123
- return query_locale.get_display_name(), query_locale.get_display_name(
124
- "en"
125
- ) # pyright: ignore
133
+ if native_display_name := query_locale.get_display_name():
134
+ if english_display_name := query_locale.get_display_name("en"):
135
+ return native_display_name, english_display_name
126
136
  except (babel.UnknownLocaleError, TypeError, ValueError, AttributeError):
127
137
  pass
128
138
 
@@ -130,39 +140,30 @@ def find_language_names(query: str, lang_data: dict | None = None) -> tuple[str,
130
140
  for iso_level in [f"iso-639-{lang_}" for lang_ in reversed(ISO_LEVELS)]:
131
141
  try:
132
142
  query_locale = babel.Locale.parse(lang_data.get(iso_level))
133
- return query_locale.get_display_name(), query_locale.get_display_name(
134
- "en"
135
- ) # pyright: ignore
143
+ if native_display_name := query_locale.get_display_name():
144
+ if english_display_name := query_locale.get_display_name("en"):
145
+ return native_display_name, english_display_name
136
146
  except (babel.UnknownLocaleError, TypeError, ValueError, AttributeError):
137
147
  pass
138
- default = lang_data.get("english", query)
148
+ default = lang_data.get("english") or query
139
149
  return default, default
140
150
 
141
151
 
142
- def update_with_macro(lang_data: dict, macro_data: dict):
152
+ def update_with_macro(lang_data: Lang, macro_data: Lang | None):
143
153
  """update empty keys from lang_data with ones of macro_data"""
144
154
  if macro_data:
145
155
  for key, value in macro_data.items():
146
- if key in lang_data and not lang_data[key]:
156
+ if key in lang_data and not lang_data.get(key):
147
157
  lang_data[key] = value
148
158
  return lang_data
149
159
 
150
160
 
151
161
  def get_language_details(
152
162
  query: str, failsafe: bool | None = False # noqa: FBT002
153
- ) -> dict:
163
+ ) -> Lang | None:
154
164
  """language details dict from query.
155
165
 
156
- Raises NotFound or return `und` language details if failsafe
157
-
158
- iso-639-1: str ISO-639-1 language code
159
- iso-639-2b: str ISO-639-2b language code
160
- iso-639-2t: str ISO-639-2t language code
161
- iso-639-3: str ISO-639-3 language code
162
- iso-639-5: str ISO-639-5 language code
163
- english: str language name in English
164
- native: str language name in is native language
165
- iso_types: [str] list of supported iso types
166
+ When query fails, either raises NotFoundError or return None, based on failsafe
166
167
 
167
168
  """
168
169
 
@@ -186,12 +187,12 @@ def get_language_details(
186
187
 
187
188
  try:
188
189
  lang_data, macro_data = get_iso_lang_data(adjusted_query)
189
- except NotFound as exc:
190
+ except NotFoundError as exc:
190
191
  if failsafe:
191
- return None # pyright: ignore
192
+ return None
192
193
  raise exc
193
194
 
194
- iso_data = update_with_macro(lang_data, macro_data) # pyright: ignore
195
+ iso_data = update_with_macro(lang_data, macro_data)
195
196
  native_name, english_name = find_language_names(native_query, iso_data)
196
197
  iso_data.update(
197
198
  {
@@ -3,7 +3,7 @@
3
3
  # vim: ai ts=4 sts=4 et sw=4 nu
4
4
 
5
5
  # flake8: noqa
6
- from .convertion import convert_image
6
+ from .conversion import convert_image
7
7
  from .optimization import optimize_image
8
8
  from .probing import is_valid_image
9
9
  from .transformation import resize_image
@@ -5,8 +5,10 @@ from __future__ import annotations
5
5
 
6
6
  import io
7
7
  import pathlib
8
+ from typing import IO
8
9
 
9
- import PIL
10
+ import cairosvg.svg
11
+ from PIL.Image import open as pilopen
10
12
 
11
13
  from zimscraperlib.constants import ALPHA_NOT_SUPPORTED
12
14
  from zimscraperlib.image.probing import format_for
@@ -15,8 +17,8 @@ from zimscraperlib.image.utils import save_image
15
17
 
16
18
 
17
19
  def convert_image(
18
- src: pathlib.Path | io.BytesIO,
19
- dst: pathlib.Path | io.BytesIO,
20
+ src: pathlib.Path | IO[bytes],
21
+ dst: pathlib.Path | IO[bytes],
20
22
  **params: str,
21
23
  ) -> None:
22
24
  """convert an image file from one format to another
@@ -29,29 +31,62 @@ def convert_image(
29
31
  to RGB. ex: RGB, ARGB, CMYK (and other PIL colorspaces)"""
30
32
 
31
33
  colorspace = params.get("colorspace") # requested colorspace
32
- fmt = (
33
- params.pop("fmt").upper() if "fmt" in params else None # pyright: ignore
34
- ) # requested format
34
+ fmt = params.pop("fmt").upper() if "fmt" in params else None # requested format
35
35
  if not fmt:
36
36
  fmt = format_for(dst)
37
- with PIL.Image.open(src) as image: # pyright: ignore
37
+ if not fmt:
38
+ raise ValueError("Impossible to guess destination image format")
39
+ with pilopen(src) as image:
38
40
  if image.mode == "RGBA" and fmt in ALPHA_NOT_SUPPORTED or colorspace:
39
41
  image = image.convert(colorspace or "RGB") # noqa: PLW2901
40
42
  save_image(image, dst, fmt, **params)
41
43
 
42
44
 
45
+ def convert_svg2png(
46
+ src: str | pathlib.Path | io.BytesIO,
47
+ dst: pathlib.Path | IO[bytes],
48
+ width: int | None = None,
49
+ height: int | None = None,
50
+ ):
51
+ """Convert a SVG to a PNG
52
+
53
+ Output width and height might be specified if resize is needed.
54
+ PNG background is transparent.
55
+ """
56
+ kwargs = {}
57
+ if isinstance(src, pathlib.Path):
58
+ src = str(src)
59
+ if isinstance(src, str):
60
+ kwargs["url"] = src
61
+ else:
62
+ kwargs["bytestring"] = src.getvalue()
63
+ if width:
64
+ kwargs["output_width"] = width
65
+ if height:
66
+ kwargs["output_height"] = height
67
+ if isinstance(dst, pathlib.Path):
68
+ cairosvg.svg2png(write_to=str(dst), **kwargs)
69
+ else:
70
+ result = cairosvg.svg2png(**kwargs)
71
+ if not isinstance(result, bytes):
72
+ raise Exception(
73
+ "Unexpected type returned by cairosvg.svg2png"
74
+ ) # pragma: no cover
75
+ dst.write(result)
76
+
77
+
43
78
  def create_favicon(src: pathlib.Path, dst: pathlib.Path) -> None:
44
79
  """generate a squared favicon from a source image"""
45
80
  if dst.suffix != ".ico":
46
81
  raise ValueError("favicon extension must be ICO")
47
82
 
48
- img = PIL.Image.open(src) # pyright: ignore
83
+ img = pilopen(src)
49
84
  w, h = img.size
50
85
  # resize image to square first
51
86
  if w != h:
52
87
  size = min([w, h])
53
88
  resized = dst.parent.joinpath(f"{src.stem}.tmp.{src.suffix}")
54
89
  resize_image(src, size, size, resized, "contain")
55
- img = PIL.Image.open(resized) # pyright: ignore
90
+ img = pilopen(resized)
56
91
  # now convert to ICO
57
92
  save_image(img, dst, "ICO")