zimscraperlib 3.3.2__tar.gz → 3.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/.pre-commit-config.yaml +3 -3
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/CHANGELOG.md +139 -123
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/PKG-INFO +15 -10
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/README.md +3 -1
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/pyproject.toml +14 -14
- zimscraperlib-3.4.0/src/zimscraperlib/__about__.py +1 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/download.py +21 -21
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/filesystem.py +5 -3
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/fix_ogvjs_dist.py +1 -2
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/html.py +4 -3
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/i18n.py +3 -6
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/image/convertion.py +4 -3
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/image/optimization.py +27 -28
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/image/presets.py +13 -13
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/image/probing.py +4 -5
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/image/transformation.py +7 -6
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/image/utils.py +3 -3
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/inputs.py +8 -9
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/logging.py +12 -11
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/misc.py +2 -2
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/types.py +2 -3
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/uri.py +11 -10
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/video/config.py +4 -4
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/video/encoding.py +2 -3
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/video/presets.py +24 -16
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/zim/_libkiwix.py +2 -2
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/zim/archive.py +2 -3
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/zim/creator.py +85 -34
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/zim/filesystem.py +88 -3
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/zim/items.py +18 -17
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/zim/metadata.py +4 -4
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/zim/providers.py +9 -10
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tasks.py +2 -1
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/conftest.py +12 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/download/test_download.py +2 -2
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/inputs/test_inputs.py +4 -5
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/ogvjs/test_ogvjs.py +2 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/video/test_encoding.py +2 -3
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/video/test_video.py +25 -17
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/zim/test_fs.py +45 -1
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/zim/test_metadata.py +2 -3
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/zim/test_zim_creator.py +101 -0
- zimscraperlib-3.3.2/src/zimscraperlib/__about__.py +0 -1
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/.gitignore +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/LICENSE +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/contrib/README.md +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/contrib/encode_video.py +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/__init__.py +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/constants.py +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/image/__init__.py +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/video/__init__.py +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/video/probing.py +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/zim/__init__.py +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/files/DroidSans.ttf +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/files/audio.mp3 +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/files/blue.jpg +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/files/commons.png +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/files/commons48.png +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/files/mail.gif +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/files/ninja.webp +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/files/pluto.jpg +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/files/square.jpg +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/files/square.png +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/files/star.svg +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/files/video.mkv +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/files/video.mp4 +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/files/video.webm +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/filesystem/test_filesystem.py +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/html/conftest.py +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/html/test_html.py +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/i18n/locale/fr/LC_MESSAGES/messages.po +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/i18n/locale/pt/LC_MESSAGES/messages.po +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/i18n/test_i18n.py +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/image/test_image.py +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/logging/conftest.py +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/logging/test_logging.py +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/misc/test_misc.py +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/ogvjs/conftest.py +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/types/test_types.py +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/uri/test_uri.py +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/video/conftest.py +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/zim/conftest.py +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/zim/test_archive.py +0 -0
- {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/zim/test_libkiwix.py +0 -0
|
@@ -7,15 +7,15 @@ repos:
|
|
|
7
7
|
- id: trailing-whitespace
|
|
8
8
|
- id: end-of-file-fixer
|
|
9
9
|
- repo: https://github.com/psf/black
|
|
10
|
-
rev: "24.
|
|
10
|
+
rev: "24.4.2"
|
|
11
11
|
hooks:
|
|
12
12
|
- id: black
|
|
13
13
|
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
14
|
-
rev: v0.
|
|
14
|
+
rev: v0.4.9
|
|
15
15
|
hooks:
|
|
16
16
|
- id: ruff
|
|
17
17
|
- repo: https://github.com/RobertCraigie/pyright-python
|
|
18
|
-
rev: v1.1.
|
|
18
|
+
rev: v1.1.368
|
|
19
19
|
hooks:
|
|
20
20
|
- id: pyright
|
|
21
21
|
name: pyright (system)
|
|
@@ -5,6 +5,24 @@ All notable changes to this project are documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0).
|
|
7
7
|
|
|
8
|
+
## [3.4.0] - 2024-06-21
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- `zim.creator.Creator._log_metadata()` to log (DEBUG) all metadata set on `_metadata` (prior to start()) #155
|
|
13
|
+
- New utility function to confirm ZIM can be created at given location / name #163
|
|
14
|
+
|
|
15
|
+
### Changed
|
|
16
|
+
|
|
17
|
+
- Migrate the **VideoWebmLow** and **VideoWebmHigh** presets to VP9 for smaller file size #79
|
|
18
|
+
- New preset versions are v3 and v2 respectively
|
|
19
|
+
- Simplify type annotations by replacing Union and Optional with pipe character ("|") for improved readability and clarity #150
|
|
20
|
+
- Calling `Creator._log_metadata()` on `Creator.start()` if running in DEBUG #155
|
|
21
|
+
|
|
22
|
+
### Fixed
|
|
23
|
+
|
|
24
|
+
- Add back the `--runinstalled` flag for test execution to allow smooth testing on other build chains #139
|
|
25
|
+
|
|
8
26
|
## [3.3.2] - 2024-03-25
|
|
9
27
|
|
|
10
28
|
### Added
|
|
@@ -137,6 +155,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
137
155
|
- [libzim] `Archive.media_count`
|
|
138
156
|
|
|
139
157
|
### Changed
|
|
158
|
+
|
|
140
159
|
- [libzim] `Archive.article_count` updated to match scraperlib's version
|
|
141
160
|
- `Archive.article_counter` now deprecated. Now returns `Archive.article_count`
|
|
142
161
|
- `Archive.media_counter` now deprecated. Now returns `Archive.media_count`
|
|
@@ -214,197 +233,194 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
214
233
|
- Unreachable code removed in `imaging` module.
|
|
215
234
|
- [tests] “Sanskrit” removed from tests as output not predicatble depending on plaftform.
|
|
216
235
|
|
|
217
|
-
|
|
218
236
|
## [1.4.3]
|
|
219
237
|
|
|
220
|
-
|
|
238
|
+
- `zim.Archive.counters` wont fail on missing `Counter` metadata
|
|
221
239
|
|
|
222
240
|
## [1.4.2]
|
|
223
241
|
|
|
224
|
-
|
|
225
|
-
|
|
242
|
+
- Fixed leak in `zim.Archive`'s `.counters`
|
|
243
|
+
- New `.get_text_metadata()` method on `zim.Archive` to save UTF-8 decoding
|
|
226
244
|
|
|
227
245
|
## [1.4.1]
|
|
228
246
|
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
247
|
+
- New `Counter` metadata based properties for Archive:
|
|
248
|
+
- `.counters`: parsed dict of the Counter metadata
|
|
249
|
+
- `.article_counter`: libkiwix's calculation for nb or article
|
|
250
|
+
- `.media_counter`: libkiwix's calculation for nb or media
|
|
251
|
+
- Fixed `i18n.find_language_names()` failing on some languages
|
|
252
|
+
- Added `uri` module with `rebuild_uri()`
|
|
236
253
|
|
|
237
254
|
## [1.4.0]
|
|
238
255
|
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
+
- Using new python-libzim based on libzim v7
|
|
257
|
+
- New Creator API
|
|
258
|
+
- Removed all namespace references
|
|
259
|
+
- Renamed `url` mentions to `path`
|
|
260
|
+
- Removed all links rewriting
|
|
261
|
+
- Removed Article/CSS/Binary seggreation
|
|
262
|
+
- Kept zimwriterfs mode (except it doesn't rewrite for namespaces)
|
|
263
|
+
- New `html` module for HTML document manipulations
|
|
264
|
+
- New callback system on `add_item_for()` and `add_item()`
|
|
265
|
+
- New Archive API with easier search/suggestions and content access
|
|
266
|
+
- Changed download log level to DEBUG (was INFO)
|
|
267
|
+
- `filesystem.get_file_mimetype` now passes bytes to libmagic instead of filename due to release issue in libmagic
|
|
268
|
+
- safer `inputs.handle_user_provided_file` regarding input as str instead of Path
|
|
269
|
+
- `image.presets` and `video.presets` now all includes `ext` and `mimetype` properties
|
|
270
|
+
- Video convert log now DEBUG instead of INFO
|
|
271
|
+
- Fixed `image.save_image()` saving to disk even when using a bytes stream
|
|
272
|
+
- Fixed `image.transformation.resize_image()` when resizing a byte stream without a dst
|
|
256
273
|
|
|
257
274
|
## [1.3.6 (internal)]
|
|
258
275
|
|
|
259
276
|
Intermediate release using unreleased libzim to support development of libzim7.
|
|
260
277
|
Don't use it.
|
|
261
278
|
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
279
|
+
- requesting newer libzim version (not released ATM)
|
|
280
|
+
- New ZIM API for non-namespace libzim (v7)
|
|
281
|
+
- updated all requirements
|
|
282
|
+
- Fixed download test inconsistency
|
|
283
|
+
- fix_ogvjs mostly useless: only allows webm types
|
|
284
|
+
- exposing retry_adapter for refactoring
|
|
285
|
+
- Changed download log level to DEBUG (was INFO)
|
|
286
|
+
- guess more-defined mime from filename if magic says it's text
|
|
287
|
+
- get_file_mimetype now passes bytes to libmagic
|
|
288
|
+
- safer regarding input as str instead of Path
|
|
289
|
+
- fixed static item for empty content
|
|
290
|
+
- ext and mimetype properties for all presets
|
|
291
|
+
- Video convert log now DEBUG instead of INFO
|
|
292
|
+
- Added delete_fpath to add_item_for() and fixed StaticItem's auto remove
|
|
293
|
+
- Updated badges for new repo name
|
|
277
294
|
|
|
278
295
|
## [1.3.5]
|
|
279
296
|
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
297
|
+
- add `stream_file()` to stream content from a URL into a file or a `BytesIO` object
|
|
298
|
+
- deprecated `save_file()`
|
|
299
|
+
- fixed `add_binary` when used without an fpath (#69)
|
|
300
|
+
- deprecated `make_grayscale` option in image optimization
|
|
301
|
+
- Added support for in-memory optimization for PNG, JPEG, and WebP images
|
|
302
|
+
- allows enabling debug logs via ZIMSCRAPERLIB_DEBUG environ
|
|
286
303
|
|
|
287
304
|
## [1.3.4]
|
|
288
305
|
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
306
|
+
- added `wait` option in `YoutubeDownloader` to allow parallelism while using context manager
|
|
307
|
+
- do not use extension for finding format in `ensure_matches()` in `image.optimization` module
|
|
308
|
+
- added `VideoWebmHigh` and `VideoMp4High` presets for high quality WebM and Mp4 convertion respectively
|
|
309
|
+
- updated presets `WebpHigh`, `JpegMedium`, `JpegLow` and `PngMedium` in `image.presets`
|
|
310
|
+
- `save_image` moved from `image` to `image.utils`
|
|
311
|
+
- added `convert_image` `optimize_image` `resize_image` functions to `image` module
|
|
295
312
|
|
|
296
313
|
## [1.3.3]
|
|
297
314
|
|
|
298
|
-
|
|
315
|
+
- added `YoutubeDownloader` to `download` to download YT videos using a capped nb of threads
|
|
299
316
|
|
|
300
317
|
## [1.3.2]
|
|
301
318
|
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
319
|
+
- fixed rewriting of links with empty target
|
|
320
|
+
- added support for image optimization using `zimscraperlib.image.optimization` for webp, gif, jpeg and png formats
|
|
321
|
+
- added `format_for()` in `zimscraperlib.image.probing` to get PIL image format from the suffix
|
|
305
322
|
|
|
306
323
|
## [1.3.1]
|
|
307
324
|
|
|
308
|
-
|
|
325
|
+
- replaced BeautifoulSoup parser in rewriting (`html.parser` –> `lxml`)
|
|
309
326
|
|
|
310
327
|
## [1.3.0]
|
|
311
328
|
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
329
|
+
- detect mimetypes from filenames for all text files
|
|
330
|
+
- fixed non-filename based StaticArticle
|
|
331
|
+
- enable rewriting of links in poster attribute of audio element
|
|
332
|
+
- added find_language_in() and find_language_in_file() to get language from HTML content and HTML file respectively
|
|
333
|
+
- add a mime mapping to deal with inconsistencies in mimetypes detected by magic on different platforms
|
|
334
|
+
- convert_image signature changed:
|
|
335
|
+
- `target_format` positional argument removed. Replaced with optionnal `fmt` key of keyword arguments.
|
|
336
|
+
- `colorspace` optionnal positional argument removed. Replaced with optionnal `colorspace` key of keyword arguments.
|
|
337
|
+
- prevent rewriting of links with special schemes `mailto`, 'tel', etc. in HTML links rewriting
|
|
338
|
+
- replaced `imaging` module with exploded `image` module (`convertion`, `probing`, `transformation`)
|
|
339
|
+
- changed `create_favicon()` param names (`source_image` -> `src`, `dest_ico` -> `dst`)
|
|
340
|
+
- changed `save_image()` param names (`image` -> `src`)
|
|
341
|
+
- changed `get_colors()` param names (`image_path` -> `src`)
|
|
342
|
+
- changed `resize_image()` param names (`fpath` -> `src`)
|
|
326
343
|
|
|
327
344
|
## [1.2.1]
|
|
328
345
|
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
346
|
+
- fixed URL rewriting when running from /
|
|
347
|
+
- added support for link rewriting in `<object>` element
|
|
348
|
+
- prevent from raising error if element doesn't have the attribute with url
|
|
349
|
+
- use non greedy match for CSS URL links (shortest string matching `url()` format)
|
|
350
|
+
- fix namespace of target only if link doesn't have a netloc
|
|
334
351
|
|
|
335
352
|
## [1.2.0]
|
|
336
353
|
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
354
|
+
- added UTF8 to constants
|
|
355
|
+
- added mime_type discovery via magic (filesystem)
|
|
356
|
+
- Added types: mime types guessing from file names
|
|
357
|
+
- Revamped zim API
|
|
358
|
+
- Removed ZimInfo which role was tu hold metadata for zimwriterfs call
|
|
359
|
+
- Removed calling zimwriterfs binary but kept function name
|
|
360
|
+
- Added zim.filesystem: zimwriterfs-like creation from a build folder
|
|
361
|
+
- Added zim.creator: create files by manually adding each article
|
|
362
|
+
- Added zim.rewriting: tools to rewrite links/urls in HTML/CSS
|
|
363
|
+
- add timeout and retries to save_file() and make it return headers
|
|
347
364
|
|
|
348
365
|
## [1.1.2]
|
|
349
366
|
|
|
350
|
-
|
|
367
|
+
- fixed `convert_image()` which tried to use a closed file
|
|
351
368
|
|
|
352
369
|
## [1.1.1]
|
|
353
370
|
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
371
|
+
- exposed reencode, Config and get_media_info in zimscraperlib.video
|
|
372
|
+
- added save_image() and convert_image() in zimscraperlib.imaging
|
|
373
|
+
- added support for upscaling in resize_image() via allow_upscaling
|
|
374
|
+
- resize_image() now supports params given by user and preservs image colorspace
|
|
375
|
+
- fixed tests for zimscraperlib.imaging
|
|
359
376
|
|
|
360
377
|
## [1.1.0]
|
|
361
378
|
|
|
362
|
-
|
|
363
|
-
|
|
379
|
+
- added video module with reencode, presets, config builder and video file probing
|
|
380
|
+
- `make_zim_file()` accepts extra kwargs for zimwriterfs
|
|
364
381
|
|
|
365
382
|
## [1.0.6]
|
|
366
383
|
|
|
367
|
-
|
|
384
|
+
- added translation support to i18n
|
|
368
385
|
|
|
369
386
|
## [1.0.5]
|
|
370
387
|
|
|
371
|
-
|
|
372
|
-
|
|
388
|
+
- added s3transfer to verbose dependencies list
|
|
389
|
+
- changed default log format to include module name
|
|
373
390
|
|
|
374
391
|
## [1.0.4]
|
|
375
392
|
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
393
|
+
- verbose dependencies (urllib3, boto3) now logged at WARNING level by default
|
|
394
|
+
- ability to set verbose dependencies log level and add modules to the list
|
|
395
|
+
- zimscraperlib's logging level now aligned with scraper's requested one
|
|
380
396
|
|
|
381
397
|
## [1.0.3]
|
|
382
398
|
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
399
|
+
- fix_ogvjs_dist script more generic (#1)
|
|
400
|
+
- updated zim to support other zimwriterfs params (#10)
|
|
401
|
+
- more flexible requirements for requests dependency
|
|
386
402
|
|
|
387
403
|
## [1.0.2]
|
|
388
404
|
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
405
|
+
- fixed return value of `get_language_details` on non-existent language
|
|
406
|
+
- fixed crash on `resize_image` with method `height`
|
|
407
|
+
- fixed root logger level (now DEBUG)
|
|
408
|
+
- removed useless `console=True` `getLogger` param
|
|
409
|
+
- completed tests (100% coverage)
|
|
410
|
+
- added `./test` script for quick local testing
|
|
411
|
+
- improved tox.ini
|
|
412
|
+
- added `create_favicon` to generate a squared favicon
|
|
413
|
+
- added `handle_user_provided_file` to handle user file/URL from param
|
|
398
414
|
|
|
399
415
|
## [1.0.1]
|
|
400
416
|
|
|
401
|
-
|
|
417
|
+
- fixed fix_ogvjs_dist
|
|
402
418
|
|
|
403
419
|
## [1.0.0]
|
|
404
420
|
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
421
|
+
- initial version providing
|
|
422
|
+
- download: save_file, save_large_file
|
|
423
|
+
- fix_ogvjs_dist
|
|
424
|
+
- i18n: setlocale, get_language_details
|
|
425
|
+
- imaging: get_colors, resize_image, is_hex_color
|
|
426
|
+
- zim: ZimInfo, make_zim_file
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: zimscraperlib
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.4.0
|
|
4
4
|
Summary: Collection of python tools to re-use common code across scrapers
|
|
5
5
|
Project-URL: Donate, https://www.kiwix.org/en/support-us/
|
|
6
6
|
Project-URL: Homepage, https://github.com/openzim/python-scraperlib
|
|
@@ -8,6 +8,8 @@ Author-email: openZIM <dev@openzim.org>
|
|
|
8
8
|
License: GPL-3.0-or-later
|
|
9
9
|
License-File: LICENSE
|
|
10
10
|
Keywords: offline,openzim,zim
|
|
11
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
11
13
|
Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
|
|
12
14
|
Classifier: Programming Language :: Python :: 3
|
|
13
15
|
Classifier: Programming Language :: Python :: 3.8
|
|
@@ -28,23 +30,24 @@ Requires-Dist: python-resize-image<1.2,>=1.1.19
|
|
|
28
30
|
Requires-Dist: requests<3.0,>=2.25.1
|
|
29
31
|
Requires-Dist: yt-dlp
|
|
30
32
|
Provides-Extra: check
|
|
31
|
-
Requires-Dist: pyright==1.1.
|
|
33
|
+
Requires-Dist: pyright==1.1.368; extra == 'check'
|
|
34
|
+
Requires-Dist: pytest==8.2.2; extra == 'check'
|
|
32
35
|
Provides-Extra: dev
|
|
33
|
-
Requires-Dist:
|
|
34
|
-
Requires-Dist: pre-commit==3.
|
|
36
|
+
Requires-Dist: ipython==8.25.0; extra == 'dev'
|
|
37
|
+
Requires-Dist: pre-commit==3.7.1; extra == 'dev'
|
|
35
38
|
Requires-Dist: zimscraperlib[check]; extra == 'dev'
|
|
36
39
|
Requires-Dist: zimscraperlib[lint]; extra == 'dev'
|
|
37
40
|
Requires-Dist: zimscraperlib[scripts]; extra == 'dev'
|
|
38
41
|
Requires-Dist: zimscraperlib[test]; extra == 'dev'
|
|
39
42
|
Provides-Extra: lint
|
|
40
|
-
Requires-Dist: black==24.
|
|
41
|
-
Requires-Dist: ruff==0.
|
|
43
|
+
Requires-Dist: black==24.4.2; extra == 'lint'
|
|
44
|
+
Requires-Dist: ruff==0.4.9; extra == 'lint'
|
|
42
45
|
Provides-Extra: scripts
|
|
43
46
|
Requires-Dist: invoke==2.2.0; extra == 'scripts'
|
|
44
47
|
Provides-Extra: test
|
|
45
|
-
Requires-Dist: coverage==7.
|
|
46
|
-
Requires-Dist: pytest-mock==3.
|
|
47
|
-
Requires-Dist: pytest==8.
|
|
48
|
+
Requires-Dist: coverage==7.5.3; extra == 'test'
|
|
49
|
+
Requires-Dist: pytest-mock==3.14.0; extra == 'test'
|
|
50
|
+
Requires-Dist: pytest==8.2.2; extra == 'test'
|
|
48
51
|
Description-Content-Type: text/markdown
|
|
49
52
|
|
|
50
53
|
zimscraperlib
|
|
@@ -104,7 +107,9 @@ apk add ffmpeg gifsicle libmagic wget libjpeg
|
|
|
104
107
|
|
|
105
108
|
# Contribution
|
|
106
109
|
|
|
107
|
-
This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing)
|
|
110
|
+
This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing).
|
|
111
|
+
|
|
112
|
+
This project has implemented openZIM's [Python bootstrap, conventions and policies](https://github.com/openzim/_python-bootstrap/docs/Policy.md) **v1.0.2**.
|
|
108
113
|
|
|
109
114
|
```shell
|
|
110
115
|
pip install hatch
|
|
@@ -55,7 +55,9 @@ apk add ffmpeg gifsicle libmagic wget libjpeg
|
|
|
55
55
|
|
|
56
56
|
# Contribution
|
|
57
57
|
|
|
58
|
-
This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing)
|
|
58
|
+
This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing).
|
|
59
|
+
|
|
60
|
+
This project has implemented openZIM's [Python bootstrap, conventions and policies](https://github.com/openzim/_python-bootstrap/docs/Policy.md) **v1.0.2**.
|
|
59
61
|
|
|
60
62
|
```shell
|
|
61
63
|
pip install hatch
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
[build-system]
|
|
2
|
-
requires = ["hatchling", "hatch-openzim"]
|
|
2
|
+
requires = ["hatchling", "hatch-openzim>=0.2"]
|
|
3
3
|
build-backend = "hatchling.build"
|
|
4
4
|
|
|
5
5
|
[project]
|
|
@@ -25,31 +25,31 @@ dynamic = ["authors", "classifiers", "keywords", "license", "version", "urls"]
|
|
|
25
25
|
|
|
26
26
|
[tool.hatch.metadata.hooks.openzim-metadata]
|
|
27
27
|
kind = "scraper"
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
# ]
|
|
28
|
+
additional-classifiers = [
|
|
29
|
+
"Development Status :: 5 - Production/Stable",
|
|
30
|
+
"Intended Audience :: Developers",
|
|
31
|
+
]
|
|
33
32
|
|
|
34
33
|
[project.optional-dependencies]
|
|
35
34
|
scripts = [
|
|
36
35
|
"invoke==2.2.0",
|
|
37
36
|
]
|
|
38
37
|
lint = [
|
|
39
|
-
"black==24.
|
|
40
|
-
"ruff==0.
|
|
38
|
+
"black==24.4.2",
|
|
39
|
+
"ruff==0.4.9",
|
|
41
40
|
]
|
|
42
41
|
check = [
|
|
43
|
-
"pyright==1.1.
|
|
42
|
+
"pyright==1.1.368",
|
|
43
|
+
"pytest==8.2.2",
|
|
44
44
|
]
|
|
45
45
|
test = [
|
|
46
|
-
"pytest==8.
|
|
47
|
-
"pytest-mock==3.
|
|
48
|
-
"coverage==7.
|
|
46
|
+
"pytest==8.2.2",
|
|
47
|
+
"pytest-mock==3.14.0",
|
|
48
|
+
"coverage==7.5.3",
|
|
49
49
|
]
|
|
50
50
|
dev = [
|
|
51
|
-
"
|
|
52
|
-
"
|
|
51
|
+
"ipython==8.25.0",
|
|
52
|
+
"pre-commit==3.7.1",
|
|
53
53
|
"zimscraperlib[scripts]",
|
|
54
54
|
"zimscraperlib[lint]",
|
|
55
55
|
"zimscraperlib[test]",
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "3.4.0"
|
|
@@ -7,7 +7,7 @@ import io
|
|
|
7
7
|
import pathlib
|
|
8
8
|
import subprocess
|
|
9
9
|
from concurrent.futures import Future, ThreadPoolExecutor
|
|
10
|
-
from typing import ClassVar
|
|
10
|
+
from typing import ClassVar
|
|
11
11
|
|
|
12
12
|
import requests
|
|
13
13
|
import yt_dlp as youtube_dl
|
|
@@ -21,7 +21,7 @@ class YoutubeDownloader:
|
|
|
21
21
|
Shutdown method must be run explicitly to
|
|
22
22
|
free any occupied resources"""
|
|
23
23
|
|
|
24
|
-
def __init__(self, threads:
|
|
24
|
+
def __init__(self, threads: int | None = 1) -> None:
|
|
25
25
|
self.executor = ThreadPoolExecutor(max_workers=threads)
|
|
26
26
|
|
|
27
27
|
def __enter__(self):
|
|
@@ -41,9 +41,9 @@ class YoutubeDownloader:
|
|
|
41
41
|
def download(
|
|
42
42
|
self,
|
|
43
43
|
url: str,
|
|
44
|
-
options:
|
|
45
|
-
wait:
|
|
46
|
-
) ->
|
|
44
|
+
options: dict | None,
|
|
45
|
+
wait: bool | None = True, # noqa: FBT002
|
|
46
|
+
) -> bool | Future:
|
|
47
47
|
"""Downloads video using initialized executor.
|
|
48
48
|
|
|
49
49
|
url: URL or Video ID
|
|
@@ -65,8 +65,8 @@ class YoutubeDownloader:
|
|
|
65
65
|
|
|
66
66
|
|
|
67
67
|
class YoutubeConfig(dict):
|
|
68
|
-
options: ClassVar[dict[str,
|
|
69
|
-
defaults: ClassVar[dict[str,
|
|
68
|
+
options: ClassVar[dict[str, str | bool | int | None]] = {}
|
|
69
|
+
defaults: ClassVar[dict[str, str | bool | int | None]] = {
|
|
70
70
|
"writethumbnail": True,
|
|
71
71
|
"write_all_thumbnails": True,
|
|
72
72
|
"writesubtitles": True,
|
|
@@ -88,8 +88,8 @@ class YoutubeConfig(dict):
|
|
|
88
88
|
@classmethod
|
|
89
89
|
def get_options(
|
|
90
90
|
cls,
|
|
91
|
-
target_dir:
|
|
92
|
-
filepath:
|
|
91
|
+
target_dir: pathlib.Path | None = None,
|
|
92
|
+
filepath: pathlib.Path | None = None,
|
|
93
93
|
**options,
|
|
94
94
|
):
|
|
95
95
|
if "outtmpl" not in options:
|
|
@@ -109,14 +109,14 @@ class YoutubeConfig(dict):
|
|
|
109
109
|
|
|
110
110
|
|
|
111
111
|
class BestWebm(YoutubeConfig):
|
|
112
|
-
options: ClassVar[dict[str,
|
|
112
|
+
options: ClassVar[dict[str, str | bool | int | None]] = {
|
|
113
113
|
"preferredcodec": "webm",
|
|
114
114
|
"format": "best[ext=webm]/bestvideo[ext=webm]+bestaudio[ext=webm]/best",
|
|
115
115
|
}
|
|
116
116
|
|
|
117
117
|
|
|
118
118
|
class BestMp4(YoutubeConfig):
|
|
119
|
-
options: ClassVar[dict[str,
|
|
119
|
+
options: ClassVar[dict[str, str | bool | int | None]] = {
|
|
120
120
|
"preferredcodec": "mp4",
|
|
121
121
|
"format": "best[ext=mp4]/bestvideo[ext=mp4]+bestaudio[ext=m4a]/best",
|
|
122
122
|
}
|
|
@@ -142,7 +142,7 @@ def save_large_file(url: str, fpath: pathlib.Path) -> None:
|
|
|
142
142
|
|
|
143
143
|
|
|
144
144
|
def _get_retry_adapter(
|
|
145
|
-
max_retries:
|
|
145
|
+
max_retries: int | None = 5,
|
|
146
146
|
) -> requests.adapters.BaseAdapter: # pyright: ignore
|
|
147
147
|
retries = requests.packages.urllib3.util.retry.Retry( # pyright: ignore
|
|
148
148
|
total=max_retries, # total number of retries
|
|
@@ -164,7 +164,7 @@ def _get_retry_adapter(
|
|
|
164
164
|
return requests.adapters.HTTPAdapter(max_retries=retries) # pyright: ignore
|
|
165
165
|
|
|
166
166
|
|
|
167
|
-
def get_session(max_retries:
|
|
167
|
+
def get_session(max_retries: int | None = 5) -> requests.Session:
|
|
168
168
|
"""Session to hold cookies and connection pool together"""
|
|
169
169
|
session = requests.Session()
|
|
170
170
|
session.mount("http", _get_retry_adapter(max_retries)) # tied to http and https
|
|
@@ -173,14 +173,14 @@ def get_session(max_retries: Optional[int] = 5) -> requests.Session:
|
|
|
173
173
|
|
|
174
174
|
def stream_file(
|
|
175
175
|
url: str,
|
|
176
|
-
fpath:
|
|
177
|
-
byte_stream:
|
|
178
|
-
block_size:
|
|
179
|
-
proxies:
|
|
180
|
-
only_first_block:
|
|
181
|
-
max_retries:
|
|
182
|
-
headers:
|
|
183
|
-
session:
|
|
176
|
+
fpath: pathlib.Path | None = None,
|
|
177
|
+
byte_stream: io.BytesIO | None = None,
|
|
178
|
+
block_size: int | None = 1024,
|
|
179
|
+
proxies: dict | None = None,
|
|
180
|
+
only_first_block: bool | None = False, # noqa: FBT002
|
|
181
|
+
max_retries: int | None = 5,
|
|
182
|
+
headers: dict[str, str] | None = None,
|
|
183
|
+
session: requests.Session | None = None,
|
|
184
184
|
) -> tuple[int, requests.structures.CaseInsensitiveDict]: # pyright: ignore
|
|
185
185
|
"""Stream data from a URL to either a BytesIO object or a file
|
|
186
186
|
Arguments -
|
|
@@ -5,10 +5,12 @@
|
|
|
5
5
|
|
|
6
6
|
Shortcuts to retrieve mime type using magic"""
|
|
7
7
|
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
8
10
|
import os
|
|
9
11
|
import pathlib
|
|
10
12
|
from collections.abc import Callable
|
|
11
|
-
from typing import Any
|
|
13
|
+
from typing import Any
|
|
12
14
|
|
|
13
15
|
import magic
|
|
14
16
|
|
|
@@ -43,8 +45,8 @@ def get_content_mimetype(content: bytes) -> str:
|
|
|
43
45
|
|
|
44
46
|
|
|
45
47
|
def delete_callback(
|
|
46
|
-
fpath:
|
|
47
|
-
callback:
|
|
48
|
+
fpath: str | pathlib.Path,
|
|
49
|
+
callback: Callable | None = None,
|
|
48
50
|
*callback_args: Any,
|
|
49
51
|
):
|
|
50
52
|
"""helper deleting passed filepath, optionnaly calling an additional callback"""
|
|
@@ -9,13 +9,12 @@ from __future__ import annotations
|
|
|
9
9
|
import logging
|
|
10
10
|
import pathlib
|
|
11
11
|
import sys
|
|
12
|
-
from typing import Union
|
|
13
12
|
|
|
14
13
|
logging.basicConfig(format="%(levelname)s:%(message)s", level=logging.DEBUG)
|
|
15
14
|
logger = logging.getLogger(__name__)
|
|
16
15
|
|
|
17
16
|
|
|
18
|
-
def fix_source_dir(source_vendors_path:
|
|
17
|
+
def fix_source_dir(source_vendors_path: pathlib.Path | str):
|
|
19
18
|
"""update ogvjs plugin to trigger on webm mimetype"""
|
|
20
19
|
root = pathlib.Path(source_vendors_path)
|
|
21
20
|
logger.info("fixing videosjs-ogvjs.js")
|