zimscraperlib 3.3.1__tar.gz → 3.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/.pre-commit-config.yaml +3 -3
  2. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/CHANGELOG.md +150 -123
  3. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/PKG-INFO +18 -13
  4. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/README.md +3 -1
  5. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/pyproject.toml +17 -17
  6. zimscraperlib-3.4.0/src/zimscraperlib/__about__.py +1 -0
  7. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/download.py +21 -21
  8. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/filesystem.py +5 -3
  9. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/fix_ogvjs_dist.py +1 -2
  10. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/html.py +4 -3
  11. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/i18n.py +3 -6
  12. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/image/convertion.py +4 -3
  13. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/image/optimization.py +27 -28
  14. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/image/presets.py +13 -13
  15. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/image/probing.py +4 -5
  16. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/image/transformation.py +7 -6
  17. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/image/utils.py +3 -3
  18. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/inputs.py +8 -9
  19. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/logging.py +12 -11
  20. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/misc.py +2 -2
  21. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/types.py +2 -3
  22. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/uri.py +11 -10
  23. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/video/config.py +4 -4
  24. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/video/encoding.py +2 -3
  25. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/video/presets.py +24 -16
  26. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/zim/_libkiwix.py +2 -2
  27. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/zim/archive.py +2 -3
  28. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/zim/creator.py +85 -34
  29. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/zim/filesystem.py +96 -4
  30. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/zim/items.py +18 -17
  31. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/zim/metadata.py +4 -4
  32. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/zim/providers.py +9 -10
  33. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tasks.py +2 -1
  34. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/conftest.py +12 -0
  35. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/download/test_download.py +2 -2
  36. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/inputs/test_inputs.py +4 -5
  37. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/ogvjs/test_ogvjs.py +2 -0
  38. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/video/test_encoding.py +2 -3
  39. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/video/test_video.py +25 -17
  40. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/zim/test_fs.py +45 -1
  41. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/zim/test_metadata.py +2 -3
  42. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/zim/test_zim_creator.py +101 -0
  43. zimscraperlib-3.3.1/src/zimscraperlib/__about__.py +0 -1
  44. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/.gitignore +0 -0
  45. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/LICENSE +0 -0
  46. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/contrib/README.md +0 -0
  47. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/contrib/encode_video.py +0 -0
  48. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/__init__.py +0 -0
  49. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/constants.py +0 -0
  50. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/image/__init__.py +0 -0
  51. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/video/__init__.py +0 -0
  52. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/video/probing.py +0 -0
  53. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/zim/__init__.py +0 -0
  54. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/files/DroidSans.ttf +0 -0
  55. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/files/audio.mp3 +0 -0
  56. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/files/blue.jpg +0 -0
  57. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/files/commons.png +0 -0
  58. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/files/commons48.png +0 -0
  59. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/files/mail.gif +0 -0
  60. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/files/ninja.webp +0 -0
  61. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/files/pluto.jpg +0 -0
  62. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/files/square.jpg +0 -0
  63. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/files/square.png +0 -0
  64. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/files/star.svg +0 -0
  65. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/files/video.mkv +0 -0
  66. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/files/video.mp4 +0 -0
  67. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/files/video.webm +0 -0
  68. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/filesystem/test_filesystem.py +0 -0
  69. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/html/conftest.py +0 -0
  70. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/html/test_html.py +0 -0
  71. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/i18n/locale/fr/LC_MESSAGES/messages.po +0 -0
  72. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/i18n/locale/pt/LC_MESSAGES/messages.po +0 -0
  73. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/i18n/test_i18n.py +0 -0
  74. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/image/test_image.py +0 -0
  75. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/logging/conftest.py +0 -0
  76. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/logging/test_logging.py +0 -0
  77. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/misc/test_misc.py +0 -0
  78. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/ogvjs/conftest.py +0 -0
  79. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/types/test_types.py +0 -0
  80. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/uri/test_uri.py +0 -0
  81. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/video/conftest.py +0 -0
  82. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/zim/conftest.py +0 -0
  83. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/zim/test_archive.py +0 -0
  84. {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/zim/test_libkiwix.py +0 -0
@@ -7,15 +7,15 @@ repos:
7
7
  - id: trailing-whitespace
8
8
  - id: end-of-file-fixer
9
9
  - repo: https://github.com/psf/black
10
- rev: "24.1.1"
10
+ rev: "24.4.2"
11
11
  hooks:
12
12
  - id: black
13
13
  - repo: https://github.com/astral-sh/ruff-pre-commit
14
- rev: v0.2.1
14
+ rev: v0.4.9
15
15
  hooks:
16
16
  - id: ruff
17
17
  - repo: https://github.com/RobertCraigie/pyright-python
18
- rev: v1.1.350
18
+ rev: v1.1.368
19
19
  hooks:
20
20
  - id: pyright
21
21
  name: pyright (system)
@@ -5,6 +5,35 @@ All notable changes to this project are documented in this file.
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0).
7
7
 
8
+ ## [3.4.0] - 2024-06-21
9
+
10
+ ### Added
11
+
12
+ - `zim.creator.Creator._log_metadata()` to log (DEBUG) all metadata set on `_metadata` (prior to start()) #155
13
+ - New utility function to confirm ZIM can be created at given location / name #163
14
+
15
+ ### Changed
16
+
17
+ - Migrate the **VideoWebmLow** and **VideoWebmHigh** presets to VP9 for smaller file size #79
18
+ - New preset versions are v3 and v2 respectively
19
+ - Simplify type annotations by replacing Union and Optional with pipe character ("|") for improved readability and clarity #150
20
+ - Calling `Creator._log_metadata()` on `Creator.start()` if running in DEBUG #155
21
+
22
+ ### Fixed
23
+
24
+ - Add back the `--runinstalled` flag for test execution to allow smooth testing on other build chains #139
25
+
26
+ ## [3.3.2] - 2024-03-25
27
+
28
+ ### Added
29
+
30
+ - Add support for `disable_metadata_checks` and `ignore_duplicates` arguments in `make_zim_file` function ("zimwritefs-mode")
31
+
32
+ ### Changed
33
+
34
+ - Relaxed constraints on Python dependencies
35
+ - Upgraded optional dependencies used for test and QA
36
+
8
37
  ## [3.3.1] - 2024-02-27
9
38
 
10
39
  ### Added
@@ -126,6 +155,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
126
155
  - [libzim] `Archive.media_count`
127
156
 
128
157
  ### Changed
158
+
129
159
  - [libzim] `Archive.article_count` updated to match scraperlib's version
130
160
  - `Archive.article_counter` now deprecated. Now returns `Archive.article_count`
131
161
  - `Archive.media_counter` now deprecated. Now returns `Archive.media_count`
@@ -203,197 +233,194 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
203
233
  - Unreachable code removed in `imaging` module.
204
234
  - [tests] “Sanskrit” removed from tests as output not predicatble depending on plaftform.
205
235
 
206
-
207
236
  ## [1.4.3]
208
237
 
209
- * `zim.Archive.counters` wont fail on missing `Counter` metadata
238
+ - `zim.Archive.counters` wont fail on missing `Counter` metadata
210
239
 
211
240
  ## [1.4.2]
212
241
 
213
- * Fixed leak in `zim.Archive`'s `.counters`
214
- * New `.get_text_metadata()` method on `zim.Archive` to save UTF-8 decoding
242
+ - Fixed leak in `zim.Archive`'s `.counters`
243
+ - New `.get_text_metadata()` method on `zim.Archive` to save UTF-8 decoding
215
244
 
216
245
  ## [1.4.1]
217
246
 
218
- * New `Counter` metadata based properties for Archive:
219
- * `.counters`: parsed dict of the Counter metadata
220
- * `.article_counter`: libkiwix's calculation for nb or article
221
- * `.media_counter`: libkiwix's calculation for nb or media
222
- * Fixed `i18n.find_language_names()` failing on some languages
223
- * Added `uri` module with `rebuild_uri()`
224
-
247
+ - New `Counter` metadata based properties for Archive:
248
+ - `.counters`: parsed dict of the Counter metadata
249
+ - `.article_counter`: libkiwix's calculation for nb or article
250
+ - `.media_counter`: libkiwix's calculation for nb or media
251
+ - Fixed `i18n.find_language_names()` failing on some languages
252
+ - Added `uri` module with `rebuild_uri()`
225
253
 
226
254
  ## [1.4.0]
227
255
 
228
- * Using new python-libzim based on libzim v7
229
- * New Creator API
230
- * Removed all namespace references
231
- * Renamed `url` mentions to `path`
232
- * Removed all links rewriting
233
- * Removed Article/CSS/Binary seggreation
234
- * Kept zimwriterfs mode (except it doesn't rewrite for namespaces)
235
- * New `html` module for HTML document manipulations
236
- * New callback system on `add_item_for()` and `add_item()`
237
- * New Archive API with easier search/suggestions and content access
238
- * Changed download log level to DEBUG (was INFO)
239
- * `filesystem.get_file_mimetype` now passes bytes to libmagic instead of filename due to release issue in libmagic
240
- * safer `inputs.handle_user_provided_file` regarding input as str instead of Path
241
- * `image.presets` and `video.presets` now all includes `ext` and `mimetype` properties
242
- * Video convert log now DEBUG instead of INFO
243
- * Fixed `image.save_image()` saving to disk even when using a bytes stream
244
- * Fixed `image.transformation.resize_image()` when resizing a byte stream without a dst
256
+ - Using new python-libzim based on libzim v7
257
+ - New Creator API
258
+ - Removed all namespace references
259
+ - Renamed `url` mentions to `path`
260
+ - Removed all links rewriting
261
+ - Removed Article/CSS/Binary seggreation
262
+ - Kept zimwriterfs mode (except it doesn't rewrite for namespaces)
263
+ - New `html` module for HTML document manipulations
264
+ - New callback system on `add_item_for()` and `add_item()`
265
+ - New Archive API with easier search/suggestions and content access
266
+ - Changed download log level to DEBUG (was INFO)
267
+ - `filesystem.get_file_mimetype` now passes bytes to libmagic instead of filename due to release issue in libmagic
268
+ - safer `inputs.handle_user_provided_file` regarding input as str instead of Path
269
+ - `image.presets` and `video.presets` now all includes `ext` and `mimetype` properties
270
+ - Video convert log now DEBUG instead of INFO
271
+ - Fixed `image.save_image()` saving to disk even when using a bytes stream
272
+ - Fixed `image.transformation.resize_image()` when resizing a byte stream without a dst
245
273
 
246
274
  ## [1.3.6 (internal)]
247
275
 
248
276
  Intermediate release using unreleased libzim to support development of libzim7.
249
277
  Don't use it.
250
278
 
251
- * requesting newer libzim version (not released ATM)
252
- * New ZIM API for non-namespace libzim (v7)
253
- * updated all requirements
254
- * Fixed download test inconsistency
255
- * fix_ogvjs mostly useless: only allows webm types
256
- * exposing retry_adapter for refactoring
257
- * Changed download log level to DEBUG (was INFO)
258
- * guess more-defined mime from filename if magic says it's text
259
- * get_file_mimetype now passes bytes to libmagic
260
- * safer regarding input as str instead of Path
261
- * fixed static item for empty content
262
- * ext and mimetype properties for all presets
263
- * Video convert log now DEBUG instead of INFO
264
- * Added delete_fpath to add_item_for() and fixed StaticItem's auto remove
265
- * Updated badges for new repo name
279
+ - requesting newer libzim version (not released ATM)
280
+ - New ZIM API for non-namespace libzim (v7)
281
+ - updated all requirements
282
+ - Fixed download test inconsistency
283
+ - fix_ogvjs mostly useless: only allows webm types
284
+ - exposing retry_adapter for refactoring
285
+ - Changed download log level to DEBUG (was INFO)
286
+ - guess more-defined mime from filename if magic says it's text
287
+ - get_file_mimetype now passes bytes to libmagic
288
+ - safer regarding input as str instead of Path
289
+ - fixed static item for empty content
290
+ - ext and mimetype properties for all presets
291
+ - Video convert log now DEBUG instead of INFO
292
+ - Added delete_fpath to add_item_for() and fixed StaticItem's auto remove
293
+ - Updated badges for new repo name
266
294
 
267
295
  ## [1.3.5]
268
296
 
269
- * add `stream_file()` to stream content from a URL into a file or a `BytesIO` object
270
- * deprecated `save_file()`
271
- * fixed `add_binary` when used without an fpath (#69)
272
- * deprecated `make_grayscale` option in image optimization
273
- * Added support for in-memory optimization for PNG, JPEG, and WebP images
274
- * allows enabling debug logs via ZIMSCRAPERLIB_DEBUG environ
297
+ - add `stream_file()` to stream content from a URL into a file or a `BytesIO` object
298
+ - deprecated `save_file()`
299
+ - fixed `add_binary` when used without an fpath (#69)
300
+ - deprecated `make_grayscale` option in image optimization
301
+ - Added support for in-memory optimization for PNG, JPEG, and WebP images
302
+ - allows enabling debug logs via ZIMSCRAPERLIB_DEBUG environ
275
303
 
276
304
  ## [1.3.4]
277
305
 
278
- * added `wait` option in `YoutubeDownloader` to allow parallelism while using context manager
279
- * do not use extension for finding format in `ensure_matches()` in `image.optimization` module
280
- * added `VideoWebmHigh` and `VideoMp4High` presets for high quality WebM and Mp4 convertion respectively
281
- * updated presets `WebpHigh`, `JpegMedium`, `JpegLow` and `PngMedium` in `image.presets`
282
- * `save_image` moved from `image` to `image.utils`
283
- * added `convert_image` `optimize_image` `resize_image` functions to `image` module
306
+ - added `wait` option in `YoutubeDownloader` to allow parallelism while using context manager
307
+ - do not use extension for finding format in `ensure_matches()` in `image.optimization` module
308
+ - added `VideoWebmHigh` and `VideoMp4High` presets for high quality WebM and Mp4 convertion respectively
309
+ - updated presets `WebpHigh`, `JpegMedium`, `JpegLow` and `PngMedium` in `image.presets`
310
+ - `save_image` moved from `image` to `image.utils`
311
+ - added `convert_image` `optimize_image` `resize_image` functions to `image` module
284
312
 
285
313
  ## [1.3.3]
286
314
 
287
- * added `YoutubeDownloader` to `download` to download YT videos using a capped nb of threads
315
+ - added `YoutubeDownloader` to `download` to download YT videos using a capped nb of threads
288
316
 
289
317
  ## [1.3.2]
290
318
 
291
- * fixed rewriting of links with empty target
292
- * added support for image optimization using `zimscraperlib.image.optimization` for webp, gif, jpeg and png formats
293
- * added `format_for()` in `zimscraperlib.image.probing` to get PIL image format from the suffix
319
+ - fixed rewriting of links with empty target
320
+ - added support for image optimization using `zimscraperlib.image.optimization` for webp, gif, jpeg and png formats
321
+ - added `format_for()` in `zimscraperlib.image.probing` to get PIL image format from the suffix
294
322
 
295
323
  ## [1.3.1]
296
324
 
297
- * replaced BeautifoulSoup parser in rewriting (`html.parser` –> `lxml`)
325
+ - replaced BeautifoulSoup parser in rewriting (`html.parser` –> `lxml`)
298
326
 
299
327
  ## [1.3.0]
300
328
 
301
- * detect mimetypes from filenames for all text files
302
- * fixed non-filename based StaticArticle
303
- * enable rewriting of links in poster attribute of audio element
304
- * added find_language_in() and find_language_in_file() to get language from HTML content and HTML file respectively
305
- * add a mime mapping to deal with inconsistencies in mimetypes detected by magic on different platforms
306
- * convert_image signature changed:
307
- * `target_format` positional argument removed. Replaced with optionnal `fmt` key of keyword arguments.
308
- * `colorspace` optionnal positional argument removed. Replaced with optionnal `colorspace` key of keyword arguments.
309
- * prevent rewriting of links with special schemes `mailto`, 'tel', etc. in HTML links rewriting
310
- * replaced `imaging` module with exploded `image` module (`convertion`, `probing`, `transformation`)
311
- * changed `create_favicon()` param names (`source_image` -> `src`, `dest_ico` -> `dst`)
312
- * changed `save_image()` param names (`image` -> `src`)
313
- * changed `get_colors()` param names (`image_path` -> `src`)
314
- * changed `resize_image()` param names (`fpath` -> `src`)
329
+ - detect mimetypes from filenames for all text files
330
+ - fixed non-filename based StaticArticle
331
+ - enable rewriting of links in poster attribute of audio element
332
+ - added find_language_in() and find_language_in_file() to get language from HTML content and HTML file respectively
333
+ - add a mime mapping to deal with inconsistencies in mimetypes detected by magic on different platforms
334
+ - convert_image signature changed:
335
+ - `target_format` positional argument removed. Replaced with optionnal `fmt` key of keyword arguments.
336
+ - `colorspace` optionnal positional argument removed. Replaced with optionnal `colorspace` key of keyword arguments.
337
+ - prevent rewriting of links with special schemes `mailto`, 'tel', etc. in HTML links rewriting
338
+ - replaced `imaging` module with exploded `image` module (`convertion`, `probing`, `transformation`)
339
+ - changed `create_favicon()` param names (`source_image` -> `src`, `dest_ico` -> `dst`)
340
+ - changed `save_image()` param names (`image` -> `src`)
341
+ - changed `get_colors()` param names (`image_path` -> `src`)
342
+ - changed `resize_image()` param names (`fpath` -> `src`)
315
343
 
316
344
  ## [1.2.1]
317
345
 
318
- * fixed URL rewriting when running from /
319
- * added support for link rewriting in `<object>` element
320
- * prevent from raising error if element doesn't have the attribute with url
321
- * use non greedy match for CSS URL links (shortest string matching `url()` format)
322
- * fix namespace of target only if link doesn't have a netloc
346
+ - fixed URL rewriting when running from /
347
+ - added support for link rewriting in `<object>` element
348
+ - prevent from raising error if element doesn't have the attribute with url
349
+ - use non greedy match for CSS URL links (shortest string matching `url()` format)
350
+ - fix namespace of target only if link doesn't have a netloc
323
351
 
324
352
  ## [1.2.0]
325
353
 
326
- * added UTF8 to constants
327
- * added mime_type discovery via magic (filesystem)
328
- * Added types: mime types guessing from file names
329
- * Revamped zim API
330
- * Removed ZimInfo which role was tu hold metadata for zimwriterfs call
331
- * Removed calling zimwriterfs binary but kept function name
332
- * Added zim.filesystem: zimwriterfs-like creation from a build folder
333
- * Added zim.creator: create files by manually adding each article
334
- * Added zim.rewriting: tools to rewrite links/urls in HTML/CSS
335
- * add timeout and retries to save_file() and make it return headers
354
+ - added UTF8 to constants
355
+ - added mime_type discovery via magic (filesystem)
356
+ - Added types: mime types guessing from file names
357
+ - Revamped zim API
358
+ - Removed ZimInfo which role was tu hold metadata for zimwriterfs call
359
+ - Removed calling zimwriterfs binary but kept function name
360
+ - Added zim.filesystem: zimwriterfs-like creation from a build folder
361
+ - Added zim.creator: create files by manually adding each article
362
+ - Added zim.rewriting: tools to rewrite links/urls in HTML/CSS
363
+ - add timeout and retries to save_file() and make it return headers
336
364
 
337
365
  ## [1.1.2]
338
366
 
339
- * fixed `convert_image()` which tried to use a closed file
367
+ - fixed `convert_image()` which tried to use a closed file
340
368
 
341
369
  ## [1.1.1]
342
370
 
343
- * exposed reencode, Config and get_media_info in zimscraperlib.video
344
- * added save_image() and convert_image() in zimscraperlib.imaging
345
- * added support for upscaling in resize_image() via allow_upscaling
346
- * resize_image() now supports params given by user and preservs image colorspace
347
- * fixed tests for zimscraperlib.imaging
371
+ - exposed reencode, Config and get_media_info in zimscraperlib.video
372
+ - added save_image() and convert_image() in zimscraperlib.imaging
373
+ - added support for upscaling in resize_image() via allow_upscaling
374
+ - resize_image() now supports params given by user and preservs image colorspace
375
+ - fixed tests for zimscraperlib.imaging
348
376
 
349
377
  ## [1.1.0]
350
378
 
351
- * added video module with reencode, presets, config builder and video file probing
352
- * `make_zim_file()` accepts extra kwargs for zimwriterfs
379
+ - added video module with reencode, presets, config builder and video file probing
380
+ - `make_zim_file()` accepts extra kwargs for zimwriterfs
353
381
 
354
382
  ## [1.0.6]
355
383
 
356
- * added translation support to i18n
384
+ - added translation support to i18n
357
385
 
358
386
  ## [1.0.5]
359
387
 
360
- * added s3transfer to verbose dependencies list
361
- * changed default log format to include module name
388
+ - added s3transfer to verbose dependencies list
389
+ - changed default log format to include module name
362
390
 
363
391
  ## [1.0.4]
364
392
 
365
- * verbose dependencies (urllib3, boto3) now logged at WARNING level by default
366
- * ability to set verbose dependencies log level and add modules to the list
367
- * zimscraperlib's logging level now aligned with scraper's requested one
368
-
393
+ - verbose dependencies (urllib3, boto3) now logged at WARNING level by default
394
+ - ability to set verbose dependencies log level and add modules to the list
395
+ - zimscraperlib's logging level now aligned with scraper's requested one
369
396
 
370
397
  ## [1.0.3]
371
398
 
372
- * fix_ogvjs_dist script more generic (#1)
373
- * updated zim to support other zimwriterfs params (#10)
374
- * more flexible requirements for requests dependency
399
+ - fix_ogvjs_dist script more generic (#1)
400
+ - updated zim to support other zimwriterfs params (#10)
401
+ - more flexible requirements for requests dependency
375
402
 
376
403
  ## [1.0.2]
377
404
 
378
- * fixed return value of `get_language_details` on non-existent language
379
- * fixed crash on `resize_image` with method `height`
380
- * fixed root logger level (now DEBUG)
381
- * removed useless `console=True` `getLogger` param
382
- * completed tests (100% coverage)
383
- * added `./test` script for quick local testing
384
- * improved tox.ini
385
- * added `create_favicon` to generate a squared favicon
386
- * added `handle_user_provided_file` to handle user file/URL from param
405
+ - fixed return value of `get_language_details` on non-existent language
406
+ - fixed crash on `resize_image` with method `height`
407
+ - fixed root logger level (now DEBUG)
408
+ - removed useless `console=True` `getLogger` param
409
+ - completed tests (100% coverage)
410
+ - added `./test` script for quick local testing
411
+ - improved tox.ini
412
+ - added `create_favicon` to generate a squared favicon
413
+ - added `handle_user_provided_file` to handle user file/URL from param
387
414
 
388
415
  ## [1.0.1]
389
416
 
390
- * fixed fix_ogvjs_dist
417
+ - fixed fix_ogvjs_dist
391
418
 
392
419
  ## [1.0.0]
393
420
 
394
- * initial version providing
395
- * download: save_file, save_large_file
396
- * fix_ogvjs_dist
397
- * i18n: setlocale, get_language_details
398
- * imaging: get_colors, resize_image, is_hex_color
399
- * zim: ZimInfo, make_zim_file
421
+ - initial version providing
422
+ - download: save_file, save_large_file
423
+ - fix_ogvjs_dist
424
+ - i18n: setlocale, get_language_details
425
+ - imaging: get_colors, resize_image, is_hex_color
426
+ - zim: ZimInfo, make_zim_file
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: zimscraperlib
3
- Version: 3.3.1
3
+ Version: 3.4.0
4
4
  Summary: Collection of python tools to re-use common code across scrapers
5
5
  Project-URL: Donate, https://www.kiwix.org/en/support-us/
6
6
  Project-URL: Homepage, https://github.com/openzim/python-scraperlib
@@ -8,6 +8,8 @@ Author-email: openZIM <dev@openzim.org>
8
8
  License: GPL-3.0-or-later
9
9
  License-File: LICENSE
10
10
  Keywords: offline,openzim,zim
11
+ Classifier: Development Status :: 5 - Production/Stable
12
+ Classifier: Intended Audience :: Developers
11
13
  Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
12
14
  Classifier: Programming Language :: Python :: 3
13
15
  Classifier: Programming Language :: Python :: 3.8
@@ -17,34 +19,35 @@ Classifier: Programming Language :: Python :: 3.11
17
19
  Classifier: Programming Language :: Python :: 3.12
18
20
  Requires-Python: <3.13,>=3.8
19
21
  Requires-Dist: babel<3.0,>=2.9
20
- Requires-Dist: beautifulsoup4<4.10,>=4.9.3
22
+ Requires-Dist: beautifulsoup4<5.0,>=4.9.3
21
23
  Requires-Dist: colorthief==0.2.1
22
24
  Requires-Dist: iso639-lang<3.0,>=2.2.3
23
25
  Requires-Dist: libzim<4.0,>=3.4.0
24
- Requires-Dist: lxml<4.10,>=4.6.3
25
- Requires-Dist: optimize-images<1.6,>=1.3.6
26
+ Requires-Dist: lxml<6.0,>=4.6.3
27
+ Requires-Dist: optimize-images<2.0,>=1.3.6
26
28
  Requires-Dist: python-magic<0.5,>=0.4.3
27
29
  Requires-Dist: python-resize-image<1.2,>=1.1.19
28
30
  Requires-Dist: requests<3.0,>=2.25.1
29
31
  Requires-Dist: yt-dlp
30
32
  Provides-Extra: check
31
- Requires-Dist: pyright==1.1.350; extra == 'check'
33
+ Requires-Dist: pyright==1.1.368; extra == 'check'
34
+ Requires-Dist: pytest==8.2.2; extra == 'check'
32
35
  Provides-Extra: dev
33
- Requires-Dist: debugpy==1.8.0; extra == 'dev'
34
- Requires-Dist: pre-commit==3.5.0; extra == 'dev'
36
+ Requires-Dist: ipython==8.25.0; extra == 'dev'
37
+ Requires-Dist: pre-commit==3.7.1; extra == 'dev'
35
38
  Requires-Dist: zimscraperlib[check]; extra == 'dev'
36
39
  Requires-Dist: zimscraperlib[lint]; extra == 'dev'
37
40
  Requires-Dist: zimscraperlib[scripts]; extra == 'dev'
38
41
  Requires-Dist: zimscraperlib[test]; extra == 'dev'
39
42
  Provides-Extra: lint
40
- Requires-Dist: black==24.1.1; extra == 'lint'
41
- Requires-Dist: ruff==0.2.1; extra == 'lint'
43
+ Requires-Dist: black==24.4.2; extra == 'lint'
44
+ Requires-Dist: ruff==0.4.9; extra == 'lint'
42
45
  Provides-Extra: scripts
43
46
  Requires-Dist: invoke==2.2.0; extra == 'scripts'
44
47
  Provides-Extra: test
45
- Requires-Dist: coverage==7.4.1; extra == 'test'
46
- Requires-Dist: pytest-mock==3.12.0; extra == 'test'
47
- Requires-Dist: pytest==8.0.0; extra == 'test'
48
+ Requires-Dist: coverage==7.5.3; extra == 'test'
49
+ Requires-Dist: pytest-mock==3.14.0; extra == 'test'
50
+ Requires-Dist: pytest==8.2.2; extra == 'test'
48
51
  Description-Content-Type: text/markdown
49
52
 
50
53
  zimscraperlib
@@ -104,7 +107,9 @@ apk add ffmpeg gifsicle libmagic wget libjpeg
104
107
 
105
108
  # Contribution
106
109
 
107
- This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing)
110
+ This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing).
111
+
112
+ This project has implemented openZIM's [Python bootstrap, conventions and policies](https://github.com/openzim/_python-bootstrap/docs/Policy.md) **v1.0.2**.
108
113
 
109
114
  ```shell
110
115
  pip install hatch
@@ -55,7 +55,9 @@ apk add ffmpeg gifsicle libmagic wget libjpeg
55
55
 
56
56
  # Contribution
57
57
 
58
- This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing)
58
+ This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing).
59
+
60
+ This project has implemented openZIM's [Python bootstrap, conventions and policies](https://github.com/openzim/_python-bootstrap/docs/Policy.md) **v1.0.2**.
59
61
 
60
62
  ```shell
61
63
  pip install hatch
@@ -1,5 +1,5 @@
1
1
  [build-system]
2
- requires = ["hatchling", "hatch-openzim"]
2
+ requires = ["hatchling", "hatch-openzim>=0.2"]
3
3
  build-backend = "hatchling.build"
4
4
 
5
5
  [project]
@@ -15,9 +15,9 @@ dependencies = [
15
15
  "Babel>=2.9,<3.0",
16
16
  "python-magic>=0.4.3,<0.5",
17
17
  "libzim>=3.4.0,<4.0",
18
- "beautifulsoup4>=4.9.3,<4.10", # upgrade to 4.10 and later to be done
19
- "lxml>=4.6.3,<4.10", # upgrade to 4.10 and later to be done
20
- "optimize-images>=1.3.6,<1.6",
18
+ "beautifulsoup4>=4.9.3,<5.0",
19
+ "lxml>=4.6.3,<6.0",
20
+ "optimize-images>=1.3.6,<2.0",
21
21
  # youtube-dl should be updated as frequently as possible
22
22
  "yt-dlp"
23
23
  ]
@@ -25,31 +25,31 @@ dynamic = ["authors", "classifiers", "keywords", "license", "version", "urls"]
25
25
 
26
26
  [tool.hatch.metadata.hooks.openzim-metadata]
27
27
  kind = "scraper"
28
- # not yet supported in hatch-openzim 0.1
29
- # additional-classifiers = [
30
- # "Development Status :: 5 - Production/Stable",
31
- # "Intended Audience :: Developers",
32
- # ]
28
+ additional-classifiers = [
29
+ "Development Status :: 5 - Production/Stable",
30
+ "Intended Audience :: Developers",
31
+ ]
33
32
 
34
33
  [project.optional-dependencies]
35
34
  scripts = [
36
35
  "invoke==2.2.0",
37
36
  ]
38
37
  lint = [
39
- "black==24.1.1",
40
- "ruff==0.2.1",
38
+ "black==24.4.2",
39
+ "ruff==0.4.9",
41
40
  ]
42
41
  check = [
43
- "pyright==1.1.350",
42
+ "pyright==1.1.368",
43
+ "pytest==8.2.2",
44
44
  ]
45
45
  test = [
46
- "pytest==8.0.0",
47
- "pytest-mock==3.12.0",
48
- "coverage==7.4.1",
46
+ "pytest==8.2.2",
47
+ "pytest-mock==3.14.0",
48
+ "coverage==7.5.3",
49
49
  ]
50
50
  dev = [
51
- "pre-commit==3.5.0",
52
- "debugpy==1.8.0",
51
+ "ipython==8.25.0",
52
+ "pre-commit==3.7.1",
53
53
  "zimscraperlib[scripts]",
54
54
  "zimscraperlib[lint]",
55
55
  "zimscraperlib[test]",
@@ -0,0 +1 @@
1
+ __version__ = "3.4.0"
@@ -7,7 +7,7 @@ import io
7
7
  import pathlib
8
8
  import subprocess
9
9
  from concurrent.futures import Future, ThreadPoolExecutor
10
- from typing import ClassVar, Optional, Union
10
+ from typing import ClassVar
11
11
 
12
12
  import requests
13
13
  import yt_dlp as youtube_dl
@@ -21,7 +21,7 @@ class YoutubeDownloader:
21
21
  Shutdown method must be run explicitly to
22
22
  free any occupied resources"""
23
23
 
24
- def __init__(self, threads: Optional[int] = 1) -> None:
24
+ def __init__(self, threads: int | None = 1) -> None:
25
25
  self.executor = ThreadPoolExecutor(max_workers=threads)
26
26
 
27
27
  def __enter__(self):
@@ -41,9 +41,9 @@ class YoutubeDownloader:
41
41
  def download(
42
42
  self,
43
43
  url: str,
44
- options: Optional[dict],
45
- wait: Optional[bool] = True, # noqa: FBT002
46
- ) -> Union[bool, Future]:
44
+ options: dict | None,
45
+ wait: bool | None = True, # noqa: FBT002
46
+ ) -> bool | Future:
47
47
  """Downloads video using initialized executor.
48
48
 
49
49
  url: URL or Video ID
@@ -65,8 +65,8 @@ class YoutubeDownloader:
65
65
 
66
66
 
67
67
  class YoutubeConfig(dict):
68
- options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {}
69
- defaults: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
68
+ options: ClassVar[dict[str, str | bool | int | None]] = {}
69
+ defaults: ClassVar[dict[str, str | bool | int | None]] = {
70
70
  "writethumbnail": True,
71
71
  "write_all_thumbnails": True,
72
72
  "writesubtitles": True,
@@ -88,8 +88,8 @@ class YoutubeConfig(dict):
88
88
  @classmethod
89
89
  def get_options(
90
90
  cls,
91
- target_dir: Optional[pathlib.Path] = None,
92
- filepath: Optional[pathlib.Path] = None,
91
+ target_dir: pathlib.Path | None = None,
92
+ filepath: pathlib.Path | None = None,
93
93
  **options,
94
94
  ):
95
95
  if "outtmpl" not in options:
@@ -109,14 +109,14 @@ class YoutubeConfig(dict):
109
109
 
110
110
 
111
111
  class BestWebm(YoutubeConfig):
112
- options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
112
+ options: ClassVar[dict[str, str | bool | int | None]] = {
113
113
  "preferredcodec": "webm",
114
114
  "format": "best[ext=webm]/bestvideo[ext=webm]+bestaudio[ext=webm]/best",
115
115
  }
116
116
 
117
117
 
118
118
  class BestMp4(YoutubeConfig):
119
- options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
119
+ options: ClassVar[dict[str, str | bool | int | None]] = {
120
120
  "preferredcodec": "mp4",
121
121
  "format": "best[ext=mp4]/bestvideo[ext=mp4]+bestaudio[ext=m4a]/best",
122
122
  }
@@ -142,7 +142,7 @@ def save_large_file(url: str, fpath: pathlib.Path) -> None:
142
142
 
143
143
 
144
144
  def _get_retry_adapter(
145
- max_retries: Optional[int] = 5,
145
+ max_retries: int | None = 5,
146
146
  ) -> requests.adapters.BaseAdapter: # pyright: ignore
147
147
  retries = requests.packages.urllib3.util.retry.Retry( # pyright: ignore
148
148
  total=max_retries, # total number of retries
@@ -164,7 +164,7 @@ def _get_retry_adapter(
164
164
  return requests.adapters.HTTPAdapter(max_retries=retries) # pyright: ignore
165
165
 
166
166
 
167
- def get_session(max_retries: Optional[int] = 5) -> requests.Session:
167
+ def get_session(max_retries: int | None = 5) -> requests.Session:
168
168
  """Session to hold cookies and connection pool together"""
169
169
  session = requests.Session()
170
170
  session.mount("http", _get_retry_adapter(max_retries)) # tied to http and https
@@ -173,14 +173,14 @@ def get_session(max_retries: Optional[int] = 5) -> requests.Session:
173
173
 
174
174
  def stream_file(
175
175
  url: str,
176
- fpath: Optional[pathlib.Path] = None,
177
- byte_stream: Optional[io.BytesIO] = None,
178
- block_size: Optional[int] = 1024,
179
- proxies: Optional[dict] = None,
180
- only_first_block: Optional[bool] = False, # noqa: FBT002
181
- max_retries: Optional[int] = 5,
182
- headers: Optional[dict[str, str]] = None,
183
- session: Optional[requests.Session] = None,
176
+ fpath: pathlib.Path | None = None,
177
+ byte_stream: io.BytesIO | None = None,
178
+ block_size: int | None = 1024,
179
+ proxies: dict | None = None,
180
+ only_first_block: bool | None = False, # noqa: FBT002
181
+ max_retries: int | None = 5,
182
+ headers: dict[str, str] | None = None,
183
+ session: requests.Session | None = None,
184
184
  ) -> tuple[int, requests.structures.CaseInsensitiveDict]: # pyright: ignore
185
185
  """Stream data from a URL to either a BytesIO object or a file
186
186
  Arguments -