zimscraperlib 3.4.0__tar.gz → 4.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/.pre-commit-config.yaml +1 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/CHANGELOG.md +39 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/PKG-INFO +4 -3
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/README.md +0 -2
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/pyproject.toml +6 -0
- zimscraperlib-4.0.0/src/zimscraperlib/__about__.py +1 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/download.py +20 -16
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/filesystem.py +1 -1
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/html.py +10 -10
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/i18n.py +69 -68
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/image/__init__.py +1 -1
- zimscraperlib-3.4.0/src/zimscraperlib/image/convertion.py → zimscraperlib-4.0.0/src/zimscraperlib/image/conversion.py +44 -9
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/image/optimization.py +81 -51
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/image/probing.py +38 -13
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/image/transformation.py +8 -4
- zimscraperlib-4.0.0/src/zimscraperlib/image/utils.py +21 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/inputs.py +26 -1
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/logging.py +22 -21
- zimscraperlib-4.0.0/src/zimscraperlib/misc.py +12 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/types.py +4 -6
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/uri.py +9 -8
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/video/encoding.py +8 -7
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/zim/_libkiwix.py +3 -3
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/zim/archive.py +1 -1
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/zim/creator.py +88 -24
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/zim/filesystem.py +23 -17
- zimscraperlib-4.0.0/src/zimscraperlib/zim/indexing.py +119 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/zim/items.py +78 -2
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/zim/metadata.py +34 -13
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/zim/providers.py +2 -4
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/conftest.py +26 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/download/test_download.py +8 -8
- zimscraperlib-4.0.0/tests/files/encrypted.pdf +0 -0
- zimscraperlib-4.0.0/tests/files/encrypted.txt +3 -0
- zimscraperlib-4.0.0/tests/files/milderm.pdf +15239 -37
- zimscraperlib-4.0.0/tests/files/milderm.txt +47481 -0
- zimscraperlib-4.0.0/tests/files/single_wave_icon.gbr +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/html/conftest.py +20 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/html/test_html.py +3 -1
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/i18n/test_i18n.py +57 -25
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/image/test_image.py +203 -67
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/inputs/test_inputs.py +30 -1
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/video/test_video.py +2 -1
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/zim/test_fs.py +2 -0
- zimscraperlib-4.0.0/tests/zim/test_indexing.py +354 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/zim/test_libkiwix.py +6 -7
- zimscraperlib-4.0.0/tests/zim/test_metadata.py +63 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/zim/test_zim_creator.py +74 -14
- zimscraperlib-3.4.0/src/zimscraperlib/__about__.py +0 -1
- zimscraperlib-3.4.0/src/zimscraperlib/image/utils.py +0 -20
- zimscraperlib-3.4.0/src/zimscraperlib/misc.py +0 -8
- zimscraperlib-3.4.0/tests/i18n/locale/fr/LC_MESSAGES/messages.po +0 -23
- zimscraperlib-3.4.0/tests/i18n/locale/pt/LC_MESSAGES/messages.po +0 -24
- zimscraperlib-3.4.0/tests/zim/test_metadata.py +0 -34
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/.gitignore +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/LICENSE +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/contrib/README.md +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/contrib/encode_video.py +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/__init__.py +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/constants.py +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/fix_ogvjs_dist.py +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/image/presets.py +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/video/__init__.py +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/video/config.py +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/video/presets.py +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/video/probing.py +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/src/zimscraperlib/zim/__init__.py +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tasks.py +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/files/DroidSans.ttf +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/files/audio.mp3 +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/files/blue.jpg +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/files/commons.png +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/files/commons48.png +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/files/mail.gif +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/files/ninja.webp +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/files/pluto.jpg +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/files/square.jpg +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/files/square.png +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/files/star.svg +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/files/video.mkv +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/files/video.mp4 +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/files/video.webm +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/filesystem/test_filesystem.py +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/logging/conftest.py +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/logging/test_logging.py +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/misc/test_misc.py +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/ogvjs/conftest.py +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/ogvjs/test_ogvjs.py +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/types/test_types.py +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/uri/test_uri.py +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/video/conftest.py +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/video/test_encoding.py +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/zim/conftest.py +0 -0
- {zimscraperlib-3.4.0 → zimscraperlib-4.0.0}/tests/zim/test_archive.py +0 -0
|
@@ -5,6 +5,45 @@ All notable changes to this project are documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0).
|
|
7
7
|
|
|
8
|
+
## [4.0.0] - 2024-08-05
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- Add utility function to compute ZIM Tags #164, including deduplication #156
|
|
13
|
+
- Metadata does not automatically drops control characters #159
|
|
14
|
+
- New `indexing.IndexData` class to hold title, content and keywords to pass to libzim to index an item
|
|
15
|
+
- Automatically index PDF documents content #167
|
|
16
|
+
- Automatically set proper title on PDF documents #168
|
|
17
|
+
- Expose new `optimization.get_optimization_method` to get the proper optimization method to call for a given image format
|
|
18
|
+
- Add `optimization.get_optimization_method` to get the proper optimization method to call for a given image format
|
|
19
|
+
- New `creator.Creator.convert_and_check_metadata` to convert metadata to bytes or str for known use cases and check proper type is passed to libzim
|
|
20
|
+
- Add svg2png image conversion function #113
|
|
21
|
+
- Add `conversion.convert_svg2png` image conversion function + support for SVG in `probing.format_for` #113
|
|
22
|
+
- Add `i18n.Lang` class used as typed result of i18n operations #151
|
|
23
|
+
|
|
24
|
+
### Changed
|
|
25
|
+
|
|
26
|
+
- **BREAKING** Renamed `zimscraperlib.image.convertion` to `zimscraperlib.image.conversion` to fix typo
|
|
27
|
+
- **BREAKING** Many changes in type hints to match the real underlying code
|
|
28
|
+
- **BREAKING** Force all boolean arguments (and some other non-obvious parameters) to be keyword-only in function calls for clarity / disambiguation (see ruff rule FBT002)
|
|
29
|
+
- Prefer to use `IO[bytes]` to `io.BytesIO` when possible since it is more generic
|
|
30
|
+
- **BREAKING** `i18n.NotFound` renamed `i18n.NotFoundError`
|
|
31
|
+
- **BREAKING** `types.get_mime_for_name` now returns `str | None`
|
|
32
|
+
- **BREAKING** `creator.Creator.add_metadata` and `creator.Creator.validate_metadata` now only accepts `bytes | str` as value (it must have been converted before call)
|
|
33
|
+
- **BREAKING** second argument of `creator.Creator.add_metadata` has been renamed to `value` instead of `content` to align with other methods
|
|
34
|
+
- When a type issue arises in metadata checks, wrong value type is displayed in exception
|
|
35
|
+
- **BREAKING** `i18n.get_language_details()`, `i18n.get_iso_lang_data()`, `i18n.find_language_names()` and `i18n.update_with_macro` now process / return a new typed `Lang` class #151
|
|
36
|
+
- **BREAKING** Rename `i18.NotFound` to `i18n.NotFoundError`
|
|
37
|
+
|
|
38
|
+
### Removed
|
|
39
|
+
- **BREAKING** Remove translation features in `i18n`: `Locale` class + `_` and `setlocale` functions #134
|
|
40
|
+
|
|
41
|
+
### Fixed
|
|
42
|
+
|
|
43
|
+
- Metadata length validation is buggy for unicode strings #158
|
|
44
|
+
- Pillow 10.4.0 reveals improper type hints for image probing functions #177
|
|
45
|
+
- Enhance error when locale fails to setup #157
|
|
46
|
+
|
|
8
47
|
## [3.4.0] - 2024-06-21
|
|
9
48
|
|
|
10
49
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: zimscraperlib
|
|
3
|
-
Version:
|
|
3
|
+
Version: 4.0.0
|
|
4
4
|
Summary: Collection of python tools to re-use common code across scrapers
|
|
5
5
|
Project-URL: Donate, https://www.kiwix.org/en/support-us/
|
|
6
6
|
Project-URL: Homepage, https://github.com/openzim/python-scraperlib
|
|
@@ -20,13 +20,16 @@ Classifier: Programming Language :: Python :: 3.12
|
|
|
20
20
|
Requires-Python: <3.13,>=3.8
|
|
21
21
|
Requires-Dist: babel<3.0,>=2.9
|
|
22
22
|
Requires-Dist: beautifulsoup4<5.0,>=4.9.3
|
|
23
|
+
Requires-Dist: cairosvg<3.0,>=2.2.0
|
|
23
24
|
Requires-Dist: colorthief==0.2.1
|
|
24
25
|
Requires-Dist: iso639-lang<3.0,>=2.2.3
|
|
25
26
|
Requires-Dist: libzim<4.0,>=3.4.0
|
|
26
27
|
Requires-Dist: lxml<6.0,>=4.6.3
|
|
27
28
|
Requires-Dist: optimize-images<2.0,>=1.3.6
|
|
29
|
+
Requires-Dist: pymupdf<2.0,>=1.24.0
|
|
28
30
|
Requires-Dist: python-magic<0.5,>=0.4.3
|
|
29
31
|
Requires-Dist: python-resize-image<1.2,>=1.1.19
|
|
32
|
+
Requires-Dist: regex>=2020.7.14
|
|
30
33
|
Requires-Dist: requests<3.0,>=2.25.1
|
|
31
34
|
Requires-Dist: yt-dlp
|
|
32
35
|
Provides-Extra: check
|
|
@@ -103,8 +106,6 @@ sudo apt install libmagic1 wget ffmpeg \
|
|
|
103
106
|
apk add ffmpeg gifsicle libmagic wget libjpeg
|
|
104
107
|
```
|
|
105
108
|
|
|
106
|
-
**Nota:** i18n features do not work on Alpine, see https://github.com/openzim/python-scraperlib/issues/134 ; there is one corresponding test which is failing.
|
|
107
|
-
|
|
108
109
|
# Contribution
|
|
109
110
|
|
|
110
111
|
This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing).
|
|
@@ -51,8 +51,6 @@ sudo apt install libmagic1 wget ffmpeg \
|
|
|
51
51
|
apk add ffmpeg gifsicle libmagic wget libjpeg
|
|
52
52
|
```
|
|
53
53
|
|
|
54
|
-
**Nota:** i18n features do not work on Alpine, see https://github.com/openzim/python-scraperlib/issues/134 ; there is one corresponding test which is failing.
|
|
55
|
-
|
|
56
54
|
# Contribution
|
|
57
55
|
|
|
58
56
|
This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing).
|
|
@@ -18,6 +18,12 @@ dependencies = [
|
|
|
18
18
|
"beautifulsoup4>=4.9.3,<5.0",
|
|
19
19
|
"lxml>=4.6.3,<6.0",
|
|
20
20
|
"optimize-images>=1.3.6,<2.0",
|
|
21
|
+
# regex has no upper-bound due to "date-based" release numbers, no semver, so their
|
|
22
|
+
# promise is that they will never (or always) break the API, and the API is very
|
|
23
|
+
# limited and we use only a very small subset of it.
|
|
24
|
+
"regex>=2020.7.14",
|
|
25
|
+
"pymupdf>=1.24.0,<2.0",
|
|
26
|
+
"CairoSVG>=2.2.0,<3.0",
|
|
21
27
|
# youtube-dl should be updated as frequently as possible
|
|
22
28
|
"yt-dlp"
|
|
23
29
|
]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "4.0.0"
|
|
@@ -3,13 +3,15 @@
|
|
|
3
3
|
|
|
4
4
|
from __future__ import annotations
|
|
5
5
|
|
|
6
|
-
import io
|
|
7
6
|
import pathlib
|
|
8
7
|
import subprocess
|
|
9
8
|
from concurrent.futures import Future, ThreadPoolExecutor
|
|
10
|
-
from typing import ClassVar
|
|
9
|
+
from typing import IO, ClassVar
|
|
11
10
|
|
|
12
11
|
import requests
|
|
12
|
+
import requests.adapters
|
|
13
|
+
import requests.structures
|
|
14
|
+
import urllib3.util
|
|
13
15
|
import yt_dlp as youtube_dl
|
|
14
16
|
|
|
15
17
|
from zimscraperlib import logger
|
|
@@ -42,7 +44,8 @@ class YoutubeDownloader:
|
|
|
42
44
|
self,
|
|
43
45
|
url: str,
|
|
44
46
|
options: dict | None,
|
|
45
|
-
|
|
47
|
+
*,
|
|
48
|
+
wait: bool | None = True,
|
|
46
49
|
) -> bool | Future:
|
|
47
50
|
"""Downloads video using initialized executor.
|
|
48
51
|
|
|
@@ -52,9 +55,7 @@ class YoutubeDownloader:
|
|
|
52
55
|
|
|
53
56
|
Returns download result of future (wait=False)"""
|
|
54
57
|
|
|
55
|
-
future = self.executor.submit(
|
|
56
|
-
self._run_youtube_dl, url, options # pyright: ignore
|
|
57
|
-
)
|
|
58
|
+
future = self.executor.submit(self._run_youtube_dl, url, options or {})
|
|
58
59
|
if not wait:
|
|
59
60
|
return future
|
|
60
61
|
if not future.exception():
|
|
@@ -143,8 +144,8 @@ def save_large_file(url: str, fpath: pathlib.Path) -> None:
|
|
|
143
144
|
|
|
144
145
|
def _get_retry_adapter(
|
|
145
146
|
max_retries: int | None = 5,
|
|
146
|
-
) -> requests.adapters.BaseAdapter:
|
|
147
|
-
retries =
|
|
147
|
+
) -> requests.adapters.BaseAdapter:
|
|
148
|
+
retries = urllib3.util.retry.Retry(
|
|
148
149
|
total=max_retries, # total number of retries
|
|
149
150
|
connect=max_retries, # connection errors
|
|
150
151
|
read=max_retries, # read errors
|
|
@@ -161,7 +162,7 @@ def _get_retry_adapter(
|
|
|
161
162
|
], # force retry on the following codes
|
|
162
163
|
)
|
|
163
164
|
|
|
164
|
-
return requests.adapters.HTTPAdapter(max_retries=retries)
|
|
165
|
+
return requests.adapters.HTTPAdapter(max_retries=retries)
|
|
165
166
|
|
|
166
167
|
|
|
167
168
|
def get_session(max_retries: int | None = 5) -> requests.Session:
|
|
@@ -174,14 +175,15 @@ def get_session(max_retries: int | None = 5) -> requests.Session:
|
|
|
174
175
|
def stream_file(
|
|
175
176
|
url: str,
|
|
176
177
|
fpath: pathlib.Path | None = None,
|
|
177
|
-
byte_stream:
|
|
178
|
+
byte_stream: IO[bytes] | None = None,
|
|
178
179
|
block_size: int | None = 1024,
|
|
179
180
|
proxies: dict | None = None,
|
|
180
|
-
only_first_block: bool | None = False, # noqa: FBT002
|
|
181
181
|
max_retries: int | None = 5,
|
|
182
182
|
headers: dict[str, str] | None = None,
|
|
183
183
|
session: requests.Session | None = None,
|
|
184
|
-
|
|
184
|
+
*,
|
|
185
|
+
only_first_block: bool | None = False,
|
|
186
|
+
) -> tuple[int, requests.structures.CaseInsensitiveDict]:
|
|
185
187
|
"""Stream data from a URL to either a BytesIO object or a file
|
|
186
188
|
Arguments -
|
|
187
189
|
fpath - Path of the file where data is sent
|
|
@@ -212,12 +214,14 @@ def stream_file(
|
|
|
212
214
|
total_downloaded = 0
|
|
213
215
|
if fpath is not None:
|
|
214
216
|
fp = open(fpath, "wb")
|
|
215
|
-
|
|
217
|
+
elif (
|
|
218
|
+
byte_stream is not None
|
|
219
|
+
): # pragma: no branch (we use a precise condition to help type checker)
|
|
216
220
|
fp = byte_stream
|
|
217
221
|
|
|
218
222
|
for data in resp.iter_content(block_size):
|
|
219
223
|
total_downloaded += len(data)
|
|
220
|
-
fp.write(data)
|
|
224
|
+
fp.write(data)
|
|
221
225
|
|
|
222
226
|
# stop downloading/reading if we're just testing first block
|
|
223
227
|
if only_first_block:
|
|
@@ -226,7 +230,7 @@ def stream_file(
|
|
|
226
230
|
logger.debug(f"Downloaded {total_downloaded} bytes from {url}")
|
|
227
231
|
|
|
228
232
|
if fpath:
|
|
229
|
-
fp.close()
|
|
233
|
+
fp.close()
|
|
230
234
|
else:
|
|
231
|
-
fp.seek(0)
|
|
235
|
+
fp.seek(0)
|
|
232
236
|
return total_downloaded, resp.headers
|
|
@@ -30,7 +30,7 @@ def get_file_mimetype(fpath: pathlib.Path) -> str:
|
|
|
30
30
|
return get_content_mimetype(fh.read(2048))
|
|
31
31
|
|
|
32
32
|
|
|
33
|
-
def get_content_mimetype(content: bytes) -> str:
|
|
33
|
+
def get_content_mimetype(content: bytes | str) -> str:
|
|
34
34
|
"""MIME Type of content retrieved from magic headers"""
|
|
35
35
|
|
|
36
36
|
try:
|
|
@@ -7,24 +7,22 @@ from __future__ import annotations
|
|
|
7
7
|
import pathlib
|
|
8
8
|
from typing import BinaryIO, TextIO
|
|
9
9
|
|
|
10
|
-
from bs4 import BeautifulSoup
|
|
10
|
+
from bs4 import BeautifulSoup, element
|
|
11
11
|
|
|
12
12
|
from zimscraperlib.types import ARTICLE_MIME
|
|
13
13
|
|
|
14
14
|
|
|
15
|
-
def find_title_in(content: str | BinaryIO | TextIO, mime_type: str) -> str:
|
|
15
|
+
def find_title_in(content: str | BinaryIO | TextIO, mime_type: str | None) -> str:
|
|
16
16
|
"""Extracted title from HTML content
|
|
17
17
|
|
|
18
18
|
blank on failure to extract and non-HTML files"""
|
|
19
19
|
if mime_type != ARTICLE_MIME:
|
|
20
20
|
return ""
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
except Exception:
|
|
24
|
-
return ""
|
|
21
|
+
title_tag = BeautifulSoup(content, "lxml").find("title")
|
|
22
|
+
return title_tag.text if title_tag else ""
|
|
25
23
|
|
|
26
24
|
|
|
27
|
-
def find_title_in_file(fpath: pathlib.Path, mime_type: str) -> str:
|
|
25
|
+
def find_title_in_file(fpath: pathlib.Path, mime_type: str | None) -> str:
|
|
28
26
|
"""Extracted title from an HTML file"""
|
|
29
27
|
try:
|
|
30
28
|
with open(fpath) as fh:
|
|
@@ -45,15 +43,17 @@ def find_language_in(content: str | BinaryIO | TextIO, mime_type: str) -> str:
|
|
|
45
43
|
for key in keylist:
|
|
46
44
|
node = soup.find(nodename)
|
|
47
45
|
if node:
|
|
48
|
-
if not node.
|
|
46
|
+
if not isinstance(node, element.Tag) or (
|
|
47
|
+
isinstance(node, element.Tag) and not node.has_attr(key)
|
|
48
|
+
):
|
|
49
49
|
continue
|
|
50
50
|
if (
|
|
51
51
|
nodename == "meta"
|
|
52
|
-
and not node.attrs.get("http-equiv", "").lower()
|
|
52
|
+
and not node.attrs.get("http-equiv", "").lower()
|
|
53
53
|
== "content-language"
|
|
54
54
|
):
|
|
55
55
|
continue
|
|
56
|
-
return node.attrs[key]
|
|
56
|
+
return node.attrs[key]
|
|
57
57
|
return ""
|
|
58
58
|
|
|
59
59
|
|
|
@@ -3,9 +3,6 @@
|
|
|
3
3
|
|
|
4
4
|
from __future__ import annotations
|
|
5
5
|
|
|
6
|
-
import gettext
|
|
7
|
-
import locale
|
|
8
|
-
import pathlib
|
|
9
6
|
import re
|
|
10
7
|
|
|
11
8
|
import babel
|
|
@@ -15,57 +12,67 @@ import iso639.exceptions
|
|
|
15
12
|
ISO_LEVELS = ["1", "2b", "2t", "3", "5"]
|
|
16
13
|
|
|
17
14
|
|
|
18
|
-
class
|
|
15
|
+
class NotFoundError(ValueError):
|
|
19
16
|
pass
|
|
20
17
|
|
|
21
18
|
|
|
22
|
-
class
|
|
23
|
-
short = "en"
|
|
24
|
-
name = "en_US.UTF-8"
|
|
25
|
-
locale_dir = None
|
|
26
|
-
domain = "messages"
|
|
27
|
-
translation = gettext.translation("messages", fallback=True)
|
|
19
|
+
class Lang(dict):
|
|
28
20
|
|
|
29
|
-
@
|
|
30
|
-
def
|
|
31
|
-
|
|
32
|
-
|
|
21
|
+
@property
|
|
22
|
+
def iso_639_1(self) -> str | None:
|
|
23
|
+
"""ISO-639-1 language code"""
|
|
24
|
+
return self["iso-639-1"]
|
|
33
25
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
26
|
+
@property
|
|
27
|
+
def iso_639_2b(self) -> str | None:
|
|
28
|
+
"""ISO-639-2b language code"""
|
|
29
|
+
return self["iso-639-2b"]
|
|
38
30
|
|
|
39
|
-
|
|
31
|
+
@property
|
|
32
|
+
def iso_639_2t(self) -> str | None:
|
|
33
|
+
"""ISO-639-2t language code"""
|
|
34
|
+
return self["iso-639-2t"]
|
|
40
35
|
|
|
41
|
-
|
|
42
|
-
|
|
36
|
+
@property
|
|
37
|
+
def iso_639_3(self) -> str | None:
|
|
38
|
+
"""ISO-639-3 language code"""
|
|
39
|
+
return self["iso-639-3"]
|
|
43
40
|
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
return
|
|
41
|
+
@property
|
|
42
|
+
def iso_639_5(self) -> str | None:
|
|
43
|
+
"""ISO-639-5 language code"""
|
|
44
|
+
return self["iso-639-5"]
|
|
48
45
|
|
|
46
|
+
@property
|
|
47
|
+
def english(self) -> str:
|
|
48
|
+
"""language name in English"""
|
|
49
|
+
return self["english"]
|
|
49
50
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
51
|
+
@property
|
|
52
|
+
def native(self) -> str:
|
|
53
|
+
"""language name in native language"""
|
|
54
|
+
return self["native"]
|
|
53
55
|
|
|
56
|
+
@property
|
|
57
|
+
def iso_types(self) -> list[str]:
|
|
58
|
+
"""list of supported iso types"""
|
|
59
|
+
return self["iso_types"]
|
|
54
60
|
|
|
55
|
-
|
|
56
|
-
|
|
61
|
+
@property
|
|
62
|
+
def query(self) -> list[str]:
|
|
63
|
+
"""Query issued for these language details"""
|
|
64
|
+
return self["query"]
|
|
57
65
|
|
|
58
|
-
|
|
59
|
-
|
|
66
|
+
@property
|
|
67
|
+
def querytype(self) -> list[str]:
|
|
68
|
+
"""Type of query issued to retrieve language details"""
|
|
69
|
+
return self["querytype"]
|
|
60
70
|
|
|
61
71
|
|
|
62
|
-
def get_iso_lang_data(lang: str) -> tuple[
|
|
63
|
-
"""ISO-639-x languages details for lang. Raises
|
|
72
|
+
def get_iso_lang_data(lang: str) -> tuple[Lang, Lang | None]:
|
|
73
|
+
"""ISO-639-x languages details for lang. Raises NotFoundError
|
|
64
74
|
|
|
65
|
-
|
|
66
|
-
english, iso_types
|
|
67
|
-
|
|
68
|
-
See get_language_details() for details"""
|
|
75
|
+
Returns a tuple (main_language, macro_language | None)"""
|
|
69
76
|
|
|
70
77
|
iso_types = []
|
|
71
78
|
|
|
@@ -75,7 +82,7 @@ def get_iso_lang_data(lang: str) -> tuple[dict, dict | None]:
|
|
|
75
82
|
iso639.exceptions.InvalidLanguageValue,
|
|
76
83
|
iso639.exceptions.DeprecatedLanguageValue,
|
|
77
84
|
) as exc:
|
|
78
|
-
raise
|
|
85
|
+
raise NotFoundError("Not a valid iso language name/code") from exc
|
|
79
86
|
|
|
80
87
|
def replace_types(new_type: str) -> str:
|
|
81
88
|
# convert new iso_types from iso639-lang Pypi package to old iso_types from
|
|
@@ -99,9 +106,9 @@ def get_iso_lang_data(lang: str) -> tuple[dict, dict | None]:
|
|
|
99
106
|
if str(getattr(isolang, code_type)).lower() == lang.lower():
|
|
100
107
|
iso_types.append(replace_types(code_type))
|
|
101
108
|
|
|
102
|
-
lang_data =
|
|
103
|
-
f"iso-639-{lang_}": getattr(isolang, f"pt{lang_}") for lang_ in ISO_LEVELS
|
|
104
|
-
|
|
109
|
+
lang_data = Lang(
|
|
110
|
+
**{f"iso-639-{lang_}": getattr(isolang, f"pt{lang_}") for lang_ in ISO_LEVELS}
|
|
111
|
+
)
|
|
105
112
|
lang_data.update({"english": isolang.name, "iso_types": iso_types})
|
|
106
113
|
|
|
107
114
|
if isolang.macro():
|
|
@@ -112,17 +119,20 @@ def get_iso_lang_data(lang: str) -> tuple[dict, dict | None]:
|
|
|
112
119
|
return lang_data, None
|
|
113
120
|
|
|
114
121
|
|
|
115
|
-
def find_language_names(query: str, lang_data:
|
|
116
|
-
"""(native, english) language names for lang with help from
|
|
122
|
+
def find_language_names(query: str, lang_data: Lang | None = None) -> tuple[str, str]:
|
|
123
|
+
"""(native, english) language names for lang with help from lang_data
|
|
117
124
|
|
|
118
125
|
Falls back to English name if available or query if not"""
|
|
119
126
|
if lang_data is None:
|
|
120
|
-
lang_data = get_language_details(query, failsafe=True)
|
|
127
|
+
lang_data = get_language_details(query, failsafe=True)
|
|
128
|
+
if not lang_data:
|
|
129
|
+
return query, query
|
|
130
|
+
|
|
121
131
|
try:
|
|
122
132
|
query_locale = babel.Locale.parse(query)
|
|
123
|
-
|
|
124
|
-
"en"
|
|
125
|
-
|
|
133
|
+
if native_display_name := query_locale.get_display_name():
|
|
134
|
+
if english_display_name := query_locale.get_display_name("en"):
|
|
135
|
+
return native_display_name, english_display_name
|
|
126
136
|
except (babel.UnknownLocaleError, TypeError, ValueError, AttributeError):
|
|
127
137
|
pass
|
|
128
138
|
|
|
@@ -130,39 +140,30 @@ def find_language_names(query: str, lang_data: dict | None = None) -> tuple[str,
|
|
|
130
140
|
for iso_level in [f"iso-639-{lang_}" for lang_ in reversed(ISO_LEVELS)]:
|
|
131
141
|
try:
|
|
132
142
|
query_locale = babel.Locale.parse(lang_data.get(iso_level))
|
|
133
|
-
|
|
134
|
-
"en"
|
|
135
|
-
|
|
143
|
+
if native_display_name := query_locale.get_display_name():
|
|
144
|
+
if english_display_name := query_locale.get_display_name("en"):
|
|
145
|
+
return native_display_name, english_display_name
|
|
136
146
|
except (babel.UnknownLocaleError, TypeError, ValueError, AttributeError):
|
|
137
147
|
pass
|
|
138
|
-
default = lang_data.get("english"
|
|
148
|
+
default = lang_data.get("english") or query
|
|
139
149
|
return default, default
|
|
140
150
|
|
|
141
151
|
|
|
142
|
-
def update_with_macro(lang_data:
|
|
152
|
+
def update_with_macro(lang_data: Lang, macro_data: Lang | None):
|
|
143
153
|
"""update empty keys from lang_data with ones of macro_data"""
|
|
144
154
|
if macro_data:
|
|
145
155
|
for key, value in macro_data.items():
|
|
146
|
-
if key in lang_data and not lang_data
|
|
156
|
+
if key in lang_data and not lang_data.get(key):
|
|
147
157
|
lang_data[key] = value
|
|
148
158
|
return lang_data
|
|
149
159
|
|
|
150
160
|
|
|
151
161
|
def get_language_details(
|
|
152
162
|
query: str, failsafe: bool | None = False # noqa: FBT002
|
|
153
|
-
) ->
|
|
163
|
+
) -> Lang | None:
|
|
154
164
|
"""language details dict from query.
|
|
155
165
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
iso-639-1: str ISO-639-1 language code
|
|
159
|
-
iso-639-2b: str ISO-639-2b language code
|
|
160
|
-
iso-639-2t: str ISO-639-2t language code
|
|
161
|
-
iso-639-3: str ISO-639-3 language code
|
|
162
|
-
iso-639-5: str ISO-639-5 language code
|
|
163
|
-
english: str language name in English
|
|
164
|
-
native: str language name in is native language
|
|
165
|
-
iso_types: [str] list of supported iso types
|
|
166
|
+
When query fails, either raises NotFoundError or return None, based on failsafe
|
|
166
167
|
|
|
167
168
|
"""
|
|
168
169
|
|
|
@@ -186,12 +187,12 @@ def get_language_details(
|
|
|
186
187
|
|
|
187
188
|
try:
|
|
188
189
|
lang_data, macro_data = get_iso_lang_data(adjusted_query)
|
|
189
|
-
except
|
|
190
|
+
except NotFoundError as exc:
|
|
190
191
|
if failsafe:
|
|
191
|
-
return None
|
|
192
|
+
return None
|
|
192
193
|
raise exc
|
|
193
194
|
|
|
194
|
-
iso_data = update_with_macro(lang_data, macro_data)
|
|
195
|
+
iso_data = update_with_macro(lang_data, macro_data)
|
|
195
196
|
native_name, english_name = find_language_names(native_query, iso_data)
|
|
196
197
|
iso_data.update(
|
|
197
198
|
{
|
|
@@ -5,8 +5,10 @@ from __future__ import annotations
|
|
|
5
5
|
|
|
6
6
|
import io
|
|
7
7
|
import pathlib
|
|
8
|
+
from typing import IO
|
|
8
9
|
|
|
9
|
-
import
|
|
10
|
+
import cairosvg.svg
|
|
11
|
+
from PIL.Image import open as pilopen
|
|
10
12
|
|
|
11
13
|
from zimscraperlib.constants import ALPHA_NOT_SUPPORTED
|
|
12
14
|
from zimscraperlib.image.probing import format_for
|
|
@@ -15,8 +17,8 @@ from zimscraperlib.image.utils import save_image
|
|
|
15
17
|
|
|
16
18
|
|
|
17
19
|
def convert_image(
|
|
18
|
-
src: pathlib.Path |
|
|
19
|
-
dst: pathlib.Path |
|
|
20
|
+
src: pathlib.Path | IO[bytes],
|
|
21
|
+
dst: pathlib.Path | IO[bytes],
|
|
20
22
|
**params: str,
|
|
21
23
|
) -> None:
|
|
22
24
|
"""convert an image file from one format to another
|
|
@@ -29,29 +31,62 @@ def convert_image(
|
|
|
29
31
|
to RGB. ex: RGB, ARGB, CMYK (and other PIL colorspaces)"""
|
|
30
32
|
|
|
31
33
|
colorspace = params.get("colorspace") # requested colorspace
|
|
32
|
-
fmt = (
|
|
33
|
-
params.pop("fmt").upper() if "fmt" in params else None # pyright: ignore
|
|
34
|
-
) # requested format
|
|
34
|
+
fmt = params.pop("fmt").upper() if "fmt" in params else None # requested format
|
|
35
35
|
if not fmt:
|
|
36
36
|
fmt = format_for(dst)
|
|
37
|
-
|
|
37
|
+
if not fmt:
|
|
38
|
+
raise ValueError("Impossible to guess destination image format")
|
|
39
|
+
with pilopen(src) as image:
|
|
38
40
|
if image.mode == "RGBA" and fmt in ALPHA_NOT_SUPPORTED or colorspace:
|
|
39
41
|
image = image.convert(colorspace or "RGB") # noqa: PLW2901
|
|
40
42
|
save_image(image, dst, fmt, **params)
|
|
41
43
|
|
|
42
44
|
|
|
45
|
+
def convert_svg2png(
|
|
46
|
+
src: str | pathlib.Path | io.BytesIO,
|
|
47
|
+
dst: pathlib.Path | IO[bytes],
|
|
48
|
+
width: int | None = None,
|
|
49
|
+
height: int | None = None,
|
|
50
|
+
):
|
|
51
|
+
"""Convert a SVG to a PNG
|
|
52
|
+
|
|
53
|
+
Output width and height might be specified if resize is needed.
|
|
54
|
+
PNG background is transparent.
|
|
55
|
+
"""
|
|
56
|
+
kwargs = {}
|
|
57
|
+
if isinstance(src, pathlib.Path):
|
|
58
|
+
src = str(src)
|
|
59
|
+
if isinstance(src, str):
|
|
60
|
+
kwargs["url"] = src
|
|
61
|
+
else:
|
|
62
|
+
kwargs["bytestring"] = src.getvalue()
|
|
63
|
+
if width:
|
|
64
|
+
kwargs["output_width"] = width
|
|
65
|
+
if height:
|
|
66
|
+
kwargs["output_height"] = height
|
|
67
|
+
if isinstance(dst, pathlib.Path):
|
|
68
|
+
cairosvg.svg2png(write_to=str(dst), **kwargs)
|
|
69
|
+
else:
|
|
70
|
+
result = cairosvg.svg2png(**kwargs)
|
|
71
|
+
if not isinstance(result, bytes):
|
|
72
|
+
raise Exception(
|
|
73
|
+
"Unexpected type returned by cairosvg.svg2png"
|
|
74
|
+
) # pragma: no cover
|
|
75
|
+
dst.write(result)
|
|
76
|
+
|
|
77
|
+
|
|
43
78
|
def create_favicon(src: pathlib.Path, dst: pathlib.Path) -> None:
|
|
44
79
|
"""generate a squared favicon from a source image"""
|
|
45
80
|
if dst.suffix != ".ico":
|
|
46
81
|
raise ValueError("favicon extension must be ICO")
|
|
47
82
|
|
|
48
|
-
img =
|
|
83
|
+
img = pilopen(src)
|
|
49
84
|
w, h = img.size
|
|
50
85
|
# resize image to square first
|
|
51
86
|
if w != h:
|
|
52
87
|
size = min([w, h])
|
|
53
88
|
resized = dst.parent.joinpath(f"{src.stem}.tmp.{src.suffix}")
|
|
54
89
|
resize_image(src, size, size, resized, "contain")
|
|
55
|
-
img =
|
|
90
|
+
img = pilopen(resized)
|
|
56
91
|
# now convert to ICO
|
|
57
92
|
save_image(img, dst, "ICO")
|