zimscraperlib 3.3.2__tar.gz → 3.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/.pre-commit-config.yaml +3 -3
  2. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/CHANGELOG.md +139 -123
  3. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/PKG-INFO +15 -10
  4. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/README.md +3 -1
  5. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/pyproject.toml +14 -14
  6. zimscraperlib-3.4.0/src/zimscraperlib/__about__.py +1 -0
  7. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/download.py +21 -21
  8. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/filesystem.py +5 -3
  9. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/fix_ogvjs_dist.py +1 -2
  10. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/html.py +4 -3
  11. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/i18n.py +3 -6
  12. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/image/convertion.py +4 -3
  13. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/image/optimization.py +27 -28
  14. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/image/presets.py +13 -13
  15. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/image/probing.py +4 -5
  16. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/image/transformation.py +7 -6
  17. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/image/utils.py +3 -3
  18. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/inputs.py +8 -9
  19. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/logging.py +12 -11
  20. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/misc.py +2 -2
  21. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/types.py +2 -3
  22. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/uri.py +11 -10
  23. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/video/config.py +4 -4
  24. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/video/encoding.py +2 -3
  25. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/video/presets.py +24 -16
  26. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/zim/_libkiwix.py +2 -2
  27. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/zim/archive.py +2 -3
  28. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/zim/creator.py +85 -34
  29. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/zim/filesystem.py +88 -3
  30. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/zim/items.py +18 -17
  31. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/zim/metadata.py +4 -4
  32. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/zim/providers.py +9 -10
  33. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tasks.py +2 -1
  34. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/conftest.py +12 -0
  35. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/download/test_download.py +2 -2
  36. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/inputs/test_inputs.py +4 -5
  37. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/ogvjs/test_ogvjs.py +2 -0
  38. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/video/test_encoding.py +2 -3
  39. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/video/test_video.py +25 -17
  40. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/zim/test_fs.py +45 -1
  41. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/zim/test_metadata.py +2 -3
  42. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/zim/test_zim_creator.py +101 -0
  43. zimscraperlib-3.3.2/src/zimscraperlib/__about__.py +0 -1
  44. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/.gitignore +0 -0
  45. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/LICENSE +0 -0
  46. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/contrib/README.md +0 -0
  47. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/contrib/encode_video.py +0 -0
  48. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/__init__.py +0 -0
  49. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/constants.py +0 -0
  50. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/image/__init__.py +0 -0
  51. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/video/__init__.py +0 -0
  52. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/video/probing.py +0 -0
  53. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/src/zimscraperlib/zim/__init__.py +0 -0
  54. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/files/DroidSans.ttf +0 -0
  55. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/files/audio.mp3 +0 -0
  56. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/files/blue.jpg +0 -0
  57. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/files/commons.png +0 -0
  58. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/files/commons48.png +0 -0
  59. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/files/mail.gif +0 -0
  60. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/files/ninja.webp +0 -0
  61. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/files/pluto.jpg +0 -0
  62. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/files/square.jpg +0 -0
  63. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/files/square.png +0 -0
  64. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/files/star.svg +0 -0
  65. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/files/video.mkv +0 -0
  66. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/files/video.mp4 +0 -0
  67. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/files/video.webm +0 -0
  68. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/filesystem/test_filesystem.py +0 -0
  69. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/html/conftest.py +0 -0
  70. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/html/test_html.py +0 -0
  71. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/i18n/locale/fr/LC_MESSAGES/messages.po +0 -0
  72. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/i18n/locale/pt/LC_MESSAGES/messages.po +0 -0
  73. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/i18n/test_i18n.py +0 -0
  74. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/image/test_image.py +0 -0
  75. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/logging/conftest.py +0 -0
  76. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/logging/test_logging.py +0 -0
  77. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/misc/test_misc.py +0 -0
  78. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/ogvjs/conftest.py +0 -0
  79. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/types/test_types.py +0 -0
  80. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/uri/test_uri.py +0 -0
  81. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/video/conftest.py +0 -0
  82. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/zim/conftest.py +0 -0
  83. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/zim/test_archive.py +0 -0
  84. {zimscraperlib-3.3.2 → zimscraperlib-3.4.0}/tests/zim/test_libkiwix.py +0 -0
@@ -7,15 +7,15 @@ repos:
7
7
  - id: trailing-whitespace
8
8
  - id: end-of-file-fixer
9
9
  - repo: https://github.com/psf/black
10
- rev: "24.3.0"
10
+ rev: "24.4.2"
11
11
  hooks:
12
12
  - id: black
13
13
  - repo: https://github.com/astral-sh/ruff-pre-commit
14
- rev: v0.3.3
14
+ rev: v0.4.9
15
15
  hooks:
16
16
  - id: ruff
17
17
  - repo: https://github.com/RobertCraigie/pyright-python
18
- rev: v1.1.354
18
+ rev: v1.1.368
19
19
  hooks:
20
20
  - id: pyright
21
21
  name: pyright (system)
@@ -5,6 +5,24 @@ All notable changes to this project are documented in this file.
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0).
7
7
 
8
+ ## [3.4.0] - 2024-06-21
9
+
10
+ ### Added
11
+
12
+ - `zim.creator.Creator._log_metadata()` to log (DEBUG) all metadata set on `_metadata` (prior to start()) #155
13
+ - New utility function to confirm ZIM can be created at given location / name #163
14
+
15
+ ### Changed
16
+
17
+ - Migrate the **VideoWebmLow** and **VideoWebmHigh** presets to VP9 for smaller file size #79
18
+ - New preset versions are v3 and v2 respectively
19
+ - Simplify type annotations by replacing Union and Optional with pipe character ("|") for improved readability and clarity #150
20
+ - Calling `Creator._log_metadata()` on `Creator.start()` if running in DEBUG #155
21
+
22
+ ### Fixed
23
+
24
+ - Add back the `--runinstalled` flag for test execution to allow smooth testing on other build chains #139
25
+
8
26
  ## [3.3.2] - 2024-03-25
9
27
 
10
28
  ### Added
@@ -137,6 +155,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
137
155
  - [libzim] `Archive.media_count`
138
156
 
139
157
  ### Changed
158
+
140
159
  - [libzim] `Archive.article_count` updated to match scraperlib's version
141
160
  - `Archive.article_counter` now deprecated. Now returns `Archive.article_count`
142
161
  - `Archive.media_counter` now deprecated. Now returns `Archive.media_count`
@@ -214,197 +233,194 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
214
233
  - Unreachable code removed in `imaging` module.
215
234
  - [tests] “Sanskrit” removed from tests as output not predicatble depending on plaftform.
216
235
 
217
-
218
236
  ## [1.4.3]
219
237
 
220
- * `zim.Archive.counters` wont fail on missing `Counter` metadata
238
+ - `zim.Archive.counters` wont fail on missing `Counter` metadata
221
239
 
222
240
  ## [1.4.2]
223
241
 
224
- * Fixed leak in `zim.Archive`'s `.counters`
225
- * New `.get_text_metadata()` method on `zim.Archive` to save UTF-8 decoding
242
+ - Fixed leak in `zim.Archive`'s `.counters`
243
+ - New `.get_text_metadata()` method on `zim.Archive` to save UTF-8 decoding
226
244
 
227
245
  ## [1.4.1]
228
246
 
229
- * New `Counter` metadata based properties for Archive:
230
- * `.counters`: parsed dict of the Counter metadata
231
- * `.article_counter`: libkiwix's calculation for nb or article
232
- * `.media_counter`: libkiwix's calculation for nb or media
233
- * Fixed `i18n.find_language_names()` failing on some languages
234
- * Added `uri` module with `rebuild_uri()`
235
-
247
+ - New `Counter` metadata based properties for Archive:
248
+ - `.counters`: parsed dict of the Counter metadata
249
+ - `.article_counter`: libkiwix's calculation for nb or article
250
+ - `.media_counter`: libkiwix's calculation for nb or media
251
+ - Fixed `i18n.find_language_names()` failing on some languages
252
+ - Added `uri` module with `rebuild_uri()`
236
253
 
237
254
  ## [1.4.0]
238
255
 
239
- * Using new python-libzim based on libzim v7
240
- * New Creator API
241
- * Removed all namespace references
242
- * Renamed `url` mentions to `path`
243
- * Removed all links rewriting
244
- * Removed Article/CSS/Binary seggreation
245
- * Kept zimwriterfs mode (except it doesn't rewrite for namespaces)
246
- * New `html` module for HTML document manipulations
247
- * New callback system on `add_item_for()` and `add_item()`
248
- * New Archive API with easier search/suggestions and content access
249
- * Changed download log level to DEBUG (was INFO)
250
- * `filesystem.get_file_mimetype` now passes bytes to libmagic instead of filename due to release issue in libmagic
251
- * safer `inputs.handle_user_provided_file` regarding input as str instead of Path
252
- * `image.presets` and `video.presets` now all includes `ext` and `mimetype` properties
253
- * Video convert log now DEBUG instead of INFO
254
- * Fixed `image.save_image()` saving to disk even when using a bytes stream
255
- * Fixed `image.transformation.resize_image()` when resizing a byte stream without a dst
256
+ - Using new python-libzim based on libzim v7
257
+ - New Creator API
258
+ - Removed all namespace references
259
+ - Renamed `url` mentions to `path`
260
+ - Removed all links rewriting
261
+ - Removed Article/CSS/Binary seggreation
262
+ - Kept zimwriterfs mode (except it doesn't rewrite for namespaces)
263
+ - New `html` module for HTML document manipulations
264
+ - New callback system on `add_item_for()` and `add_item()`
265
+ - New Archive API with easier search/suggestions and content access
266
+ - Changed download log level to DEBUG (was INFO)
267
+ - `filesystem.get_file_mimetype` now passes bytes to libmagic instead of filename due to release issue in libmagic
268
+ - safer `inputs.handle_user_provided_file` regarding input as str instead of Path
269
+ - `image.presets` and `video.presets` now all includes `ext` and `mimetype` properties
270
+ - Video convert log now DEBUG instead of INFO
271
+ - Fixed `image.save_image()` saving to disk even when using a bytes stream
272
+ - Fixed `image.transformation.resize_image()` when resizing a byte stream without a dst
256
273
 
257
274
  ## [1.3.6 (internal)]
258
275
 
259
276
  Intermediate release using unreleased libzim to support development of libzim7.
260
277
  Don't use it.
261
278
 
262
- * requesting newer libzim version (not released ATM)
263
- * New ZIM API for non-namespace libzim (v7)
264
- * updated all requirements
265
- * Fixed download test inconsistency
266
- * fix_ogvjs mostly useless: only allows webm types
267
- * exposing retry_adapter for refactoring
268
- * Changed download log level to DEBUG (was INFO)
269
- * guess more-defined mime from filename if magic says it's text
270
- * get_file_mimetype now passes bytes to libmagic
271
- * safer regarding input as str instead of Path
272
- * fixed static item for empty content
273
- * ext and mimetype properties for all presets
274
- * Video convert log now DEBUG instead of INFO
275
- * Added delete_fpath to add_item_for() and fixed StaticItem's auto remove
276
- * Updated badges for new repo name
279
+ - requesting newer libzim version (not released ATM)
280
+ - New ZIM API for non-namespace libzim (v7)
281
+ - updated all requirements
282
+ - Fixed download test inconsistency
283
+ - fix_ogvjs mostly useless: only allows webm types
284
+ - exposing retry_adapter for refactoring
285
+ - Changed download log level to DEBUG (was INFO)
286
+ - guess more-defined mime from filename if magic says it's text
287
+ - get_file_mimetype now passes bytes to libmagic
288
+ - safer regarding input as str instead of Path
289
+ - fixed static item for empty content
290
+ - ext and mimetype properties for all presets
291
+ - Video convert log now DEBUG instead of INFO
292
+ - Added delete_fpath to add_item_for() and fixed StaticItem's auto remove
293
+ - Updated badges for new repo name
277
294
 
278
295
  ## [1.3.5]
279
296
 
280
- * add `stream_file()` to stream content from a URL into a file or a `BytesIO` object
281
- * deprecated `save_file()`
282
- * fixed `add_binary` when used without an fpath (#69)
283
- * deprecated `make_grayscale` option in image optimization
284
- * Added support for in-memory optimization for PNG, JPEG, and WebP images
285
- * allows enabling debug logs via ZIMSCRAPERLIB_DEBUG environ
297
+ - add `stream_file()` to stream content from a URL into a file or a `BytesIO` object
298
+ - deprecated `save_file()`
299
+ - fixed `add_binary` when used without an fpath (#69)
300
+ - deprecated `make_grayscale` option in image optimization
301
+ - Added support for in-memory optimization for PNG, JPEG, and WebP images
302
+ - allows enabling debug logs via ZIMSCRAPERLIB_DEBUG environ
286
303
 
287
304
  ## [1.3.4]
288
305
 
289
- * added `wait` option in `YoutubeDownloader` to allow parallelism while using context manager
290
- * do not use extension for finding format in `ensure_matches()` in `image.optimization` module
291
- * added `VideoWebmHigh` and `VideoMp4High` presets for high quality WebM and Mp4 convertion respectively
292
- * updated presets `WebpHigh`, `JpegMedium`, `JpegLow` and `PngMedium` in `image.presets`
293
- * `save_image` moved from `image` to `image.utils`
294
- * added `convert_image` `optimize_image` `resize_image` functions to `image` module
306
+ - added `wait` option in `YoutubeDownloader` to allow parallelism while using context manager
307
+ - do not use extension for finding format in `ensure_matches()` in `image.optimization` module
308
+ - added `VideoWebmHigh` and `VideoMp4High` presets for high quality WebM and Mp4 convertion respectively
309
+ - updated presets `WebpHigh`, `JpegMedium`, `JpegLow` and `PngMedium` in `image.presets`
310
+ - `save_image` moved from `image` to `image.utils`
311
+ - added `convert_image` `optimize_image` `resize_image` functions to `image` module
295
312
 
296
313
  ## [1.3.3]
297
314
 
298
- * added `YoutubeDownloader` to `download` to download YT videos using a capped nb of threads
315
+ - added `YoutubeDownloader` to `download` to download YT videos using a capped nb of threads
299
316
 
300
317
  ## [1.3.2]
301
318
 
302
- * fixed rewriting of links with empty target
303
- * added support for image optimization using `zimscraperlib.image.optimization` for webp, gif, jpeg and png formats
304
- * added `format_for()` in `zimscraperlib.image.probing` to get PIL image format from the suffix
319
+ - fixed rewriting of links with empty target
320
+ - added support for image optimization using `zimscraperlib.image.optimization` for webp, gif, jpeg and png formats
321
+ - added `format_for()` in `zimscraperlib.image.probing` to get PIL image format from the suffix
305
322
 
306
323
  ## [1.3.1]
307
324
 
308
- * replaced BeautifoulSoup parser in rewriting (`html.parser` –> `lxml`)
325
+ - replaced BeautifoulSoup parser in rewriting (`html.parser` –> `lxml`)
309
326
 
310
327
  ## [1.3.0]
311
328
 
312
- * detect mimetypes from filenames for all text files
313
- * fixed non-filename based StaticArticle
314
- * enable rewriting of links in poster attribute of audio element
315
- * added find_language_in() and find_language_in_file() to get language from HTML content and HTML file respectively
316
- * add a mime mapping to deal with inconsistencies in mimetypes detected by magic on different platforms
317
- * convert_image signature changed:
318
- * `target_format` positional argument removed. Replaced with optionnal `fmt` key of keyword arguments.
319
- * `colorspace` optionnal positional argument removed. Replaced with optionnal `colorspace` key of keyword arguments.
320
- * prevent rewriting of links with special schemes `mailto`, 'tel', etc. in HTML links rewriting
321
- * replaced `imaging` module with exploded `image` module (`convertion`, `probing`, `transformation`)
322
- * changed `create_favicon()` param names (`source_image` -> `src`, `dest_ico` -> `dst`)
323
- * changed `save_image()` param names (`image` -> `src`)
324
- * changed `get_colors()` param names (`image_path` -> `src`)
325
- * changed `resize_image()` param names (`fpath` -> `src`)
329
+ - detect mimetypes from filenames for all text files
330
+ - fixed non-filename based StaticArticle
331
+ - enable rewriting of links in poster attribute of audio element
332
+ - added find_language_in() and find_language_in_file() to get language from HTML content and HTML file respectively
333
+ - add a mime mapping to deal with inconsistencies in mimetypes detected by magic on different platforms
334
+ - convert_image signature changed:
335
+ - `target_format` positional argument removed. Replaced with optionnal `fmt` key of keyword arguments.
336
+ - `colorspace` optionnal positional argument removed. Replaced with optionnal `colorspace` key of keyword arguments.
337
+ - prevent rewriting of links with special schemes `mailto`, 'tel', etc. in HTML links rewriting
338
+ - replaced `imaging` module with exploded `image` module (`convertion`, `probing`, `transformation`)
339
+ - changed `create_favicon()` param names (`source_image` -> `src`, `dest_ico` -> `dst`)
340
+ - changed `save_image()` param names (`image` -> `src`)
341
+ - changed `get_colors()` param names (`image_path` -> `src`)
342
+ - changed `resize_image()` param names (`fpath` -> `src`)
326
343
 
327
344
  ## [1.2.1]
328
345
 
329
- * fixed URL rewriting when running from /
330
- * added support for link rewriting in `<object>` element
331
- * prevent from raising error if element doesn't have the attribute with url
332
- * use non greedy match for CSS URL links (shortest string matching `url()` format)
333
- * fix namespace of target only if link doesn't have a netloc
346
+ - fixed URL rewriting when running from /
347
+ - added support for link rewriting in `<object>` element
348
+ - prevent from raising error if element doesn't have the attribute with url
349
+ - use non greedy match for CSS URL links (shortest string matching `url()` format)
350
+ - fix namespace of target only if link doesn't have a netloc
334
351
 
335
352
  ## [1.2.0]
336
353
 
337
- * added UTF8 to constants
338
- * added mime_type discovery via magic (filesystem)
339
- * Added types: mime types guessing from file names
340
- * Revamped zim API
341
- * Removed ZimInfo which role was tu hold metadata for zimwriterfs call
342
- * Removed calling zimwriterfs binary but kept function name
343
- * Added zim.filesystem: zimwriterfs-like creation from a build folder
344
- * Added zim.creator: create files by manually adding each article
345
- * Added zim.rewriting: tools to rewrite links/urls in HTML/CSS
346
- * add timeout and retries to save_file() and make it return headers
354
+ - added UTF8 to constants
355
+ - added mime_type discovery via magic (filesystem)
356
+ - Added types: mime types guessing from file names
357
+ - Revamped zim API
358
+ - Removed ZimInfo which role was tu hold metadata for zimwriterfs call
359
+ - Removed calling zimwriterfs binary but kept function name
360
+ - Added zim.filesystem: zimwriterfs-like creation from a build folder
361
+ - Added zim.creator: create files by manually adding each article
362
+ - Added zim.rewriting: tools to rewrite links/urls in HTML/CSS
363
+ - add timeout and retries to save_file() and make it return headers
347
364
 
348
365
  ## [1.1.2]
349
366
 
350
- * fixed `convert_image()` which tried to use a closed file
367
+ - fixed `convert_image()` which tried to use a closed file
351
368
 
352
369
  ## [1.1.1]
353
370
 
354
- * exposed reencode, Config and get_media_info in zimscraperlib.video
355
- * added save_image() and convert_image() in zimscraperlib.imaging
356
- * added support for upscaling in resize_image() via allow_upscaling
357
- * resize_image() now supports params given by user and preservs image colorspace
358
- * fixed tests for zimscraperlib.imaging
371
+ - exposed reencode, Config and get_media_info in zimscraperlib.video
372
+ - added save_image() and convert_image() in zimscraperlib.imaging
373
+ - added support for upscaling in resize_image() via allow_upscaling
374
+ - resize_image() now supports params given by user and preservs image colorspace
375
+ - fixed tests for zimscraperlib.imaging
359
376
 
360
377
  ## [1.1.0]
361
378
 
362
- * added video module with reencode, presets, config builder and video file probing
363
- * `make_zim_file()` accepts extra kwargs for zimwriterfs
379
+ - added video module with reencode, presets, config builder and video file probing
380
+ - `make_zim_file()` accepts extra kwargs for zimwriterfs
364
381
 
365
382
  ## [1.0.6]
366
383
 
367
- * added translation support to i18n
384
+ - added translation support to i18n
368
385
 
369
386
  ## [1.0.5]
370
387
 
371
- * added s3transfer to verbose dependencies list
372
- * changed default log format to include module name
388
+ - added s3transfer to verbose dependencies list
389
+ - changed default log format to include module name
373
390
 
374
391
  ## [1.0.4]
375
392
 
376
- * verbose dependencies (urllib3, boto3) now logged at WARNING level by default
377
- * ability to set verbose dependencies log level and add modules to the list
378
- * zimscraperlib's logging level now aligned with scraper's requested one
379
-
393
+ - verbose dependencies (urllib3, boto3) now logged at WARNING level by default
394
+ - ability to set verbose dependencies log level and add modules to the list
395
+ - zimscraperlib's logging level now aligned with scraper's requested one
380
396
 
381
397
  ## [1.0.3]
382
398
 
383
- * fix_ogvjs_dist script more generic (#1)
384
- * updated zim to support other zimwriterfs params (#10)
385
- * more flexible requirements for requests dependency
399
+ - fix_ogvjs_dist script more generic (#1)
400
+ - updated zim to support other zimwriterfs params (#10)
401
+ - more flexible requirements for requests dependency
386
402
 
387
403
  ## [1.0.2]
388
404
 
389
- * fixed return value of `get_language_details` on non-existent language
390
- * fixed crash on `resize_image` with method `height`
391
- * fixed root logger level (now DEBUG)
392
- * removed useless `console=True` `getLogger` param
393
- * completed tests (100% coverage)
394
- * added `./test` script for quick local testing
395
- * improved tox.ini
396
- * added `create_favicon` to generate a squared favicon
397
- * added `handle_user_provided_file` to handle user file/URL from param
405
+ - fixed return value of `get_language_details` on non-existent language
406
+ - fixed crash on `resize_image` with method `height`
407
+ - fixed root logger level (now DEBUG)
408
+ - removed useless `console=True` `getLogger` param
409
+ - completed tests (100% coverage)
410
+ - added `./test` script for quick local testing
411
+ - improved tox.ini
412
+ - added `create_favicon` to generate a squared favicon
413
+ - added `handle_user_provided_file` to handle user file/URL from param
398
414
 
399
415
  ## [1.0.1]
400
416
 
401
- * fixed fix_ogvjs_dist
417
+ - fixed fix_ogvjs_dist
402
418
 
403
419
  ## [1.0.0]
404
420
 
405
- * initial version providing
406
- * download: save_file, save_large_file
407
- * fix_ogvjs_dist
408
- * i18n: setlocale, get_language_details
409
- * imaging: get_colors, resize_image, is_hex_color
410
- * zim: ZimInfo, make_zim_file
421
+ - initial version providing
422
+ - download: save_file, save_large_file
423
+ - fix_ogvjs_dist
424
+ - i18n: setlocale, get_language_details
425
+ - imaging: get_colors, resize_image, is_hex_color
426
+ - zim: ZimInfo, make_zim_file
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: zimscraperlib
3
- Version: 3.3.2
3
+ Version: 3.4.0
4
4
  Summary: Collection of python tools to re-use common code across scrapers
5
5
  Project-URL: Donate, https://www.kiwix.org/en/support-us/
6
6
  Project-URL: Homepage, https://github.com/openzim/python-scraperlib
@@ -8,6 +8,8 @@ Author-email: openZIM <dev@openzim.org>
8
8
  License: GPL-3.0-or-later
9
9
  License-File: LICENSE
10
10
  Keywords: offline,openzim,zim
11
+ Classifier: Development Status :: 5 - Production/Stable
12
+ Classifier: Intended Audience :: Developers
11
13
  Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
12
14
  Classifier: Programming Language :: Python :: 3
13
15
  Classifier: Programming Language :: Python :: 3.8
@@ -28,23 +30,24 @@ Requires-Dist: python-resize-image<1.2,>=1.1.19
28
30
  Requires-Dist: requests<3.0,>=2.25.1
29
31
  Requires-Dist: yt-dlp
30
32
  Provides-Extra: check
31
- Requires-Dist: pyright==1.1.354; extra == 'check'
33
+ Requires-Dist: pyright==1.1.368; extra == 'check'
34
+ Requires-Dist: pytest==8.2.2; extra == 'check'
32
35
  Provides-Extra: dev
33
- Requires-Dist: debugpy==1.8.1; extra == 'dev'
34
- Requires-Dist: pre-commit==3.6.2; extra == 'dev'
36
+ Requires-Dist: ipython==8.25.0; extra == 'dev'
37
+ Requires-Dist: pre-commit==3.7.1; extra == 'dev'
35
38
  Requires-Dist: zimscraperlib[check]; extra == 'dev'
36
39
  Requires-Dist: zimscraperlib[lint]; extra == 'dev'
37
40
  Requires-Dist: zimscraperlib[scripts]; extra == 'dev'
38
41
  Requires-Dist: zimscraperlib[test]; extra == 'dev'
39
42
  Provides-Extra: lint
40
- Requires-Dist: black==24.3.0; extra == 'lint'
41
- Requires-Dist: ruff==0.3.3; extra == 'lint'
43
+ Requires-Dist: black==24.4.2; extra == 'lint'
44
+ Requires-Dist: ruff==0.4.9; extra == 'lint'
42
45
  Provides-Extra: scripts
43
46
  Requires-Dist: invoke==2.2.0; extra == 'scripts'
44
47
  Provides-Extra: test
45
- Requires-Dist: coverage==7.4.4; extra == 'test'
46
- Requires-Dist: pytest-mock==3.12.0; extra == 'test'
47
- Requires-Dist: pytest==8.1.1; extra == 'test'
48
+ Requires-Dist: coverage==7.5.3; extra == 'test'
49
+ Requires-Dist: pytest-mock==3.14.0; extra == 'test'
50
+ Requires-Dist: pytest==8.2.2; extra == 'test'
48
51
  Description-Content-Type: text/markdown
49
52
 
50
53
  zimscraperlib
@@ -104,7 +107,9 @@ apk add ffmpeg gifsicle libmagic wget libjpeg
104
107
 
105
108
  # Contribution
106
109
 
107
- This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing)
110
+ This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing).
111
+
112
+ This project has implemented openZIM's [Python bootstrap, conventions and policies](https://github.com/openzim/_python-bootstrap/docs/Policy.md) **v1.0.2**.
108
113
 
109
114
  ```shell
110
115
  pip install hatch
@@ -55,7 +55,9 @@ apk add ffmpeg gifsicle libmagic wget libjpeg
55
55
 
56
56
  # Contribution
57
57
 
58
- This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing)
58
+ This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing).
59
+
60
+ This project has implemented openZIM's [Python bootstrap, conventions and policies](https://github.com/openzim/_python-bootstrap/docs/Policy.md) **v1.0.2**.
59
61
 
60
62
  ```shell
61
63
  pip install hatch
@@ -1,5 +1,5 @@
1
1
  [build-system]
2
- requires = ["hatchling", "hatch-openzim"]
2
+ requires = ["hatchling", "hatch-openzim>=0.2"]
3
3
  build-backend = "hatchling.build"
4
4
 
5
5
  [project]
@@ -25,31 +25,31 @@ dynamic = ["authors", "classifiers", "keywords", "license", "version", "urls"]
25
25
 
26
26
  [tool.hatch.metadata.hooks.openzim-metadata]
27
27
  kind = "scraper"
28
- # not yet supported in hatch-openzim 0.1
29
- # additional-classifiers = [
30
- # "Development Status :: 5 - Production/Stable",
31
- # "Intended Audience :: Developers",
32
- # ]
28
+ additional-classifiers = [
29
+ "Development Status :: 5 - Production/Stable",
30
+ "Intended Audience :: Developers",
31
+ ]
33
32
 
34
33
  [project.optional-dependencies]
35
34
  scripts = [
36
35
  "invoke==2.2.0",
37
36
  ]
38
37
  lint = [
39
- "black==24.3.0",
40
- "ruff==0.3.3",
38
+ "black==24.4.2",
39
+ "ruff==0.4.9",
41
40
  ]
42
41
  check = [
43
- "pyright==1.1.354",
42
+ "pyright==1.1.368",
43
+ "pytest==8.2.2",
44
44
  ]
45
45
  test = [
46
- "pytest==8.1.1",
47
- "pytest-mock==3.12.0",
48
- "coverage==7.4.4",
46
+ "pytest==8.2.2",
47
+ "pytest-mock==3.14.0",
48
+ "coverage==7.5.3",
49
49
  ]
50
50
  dev = [
51
- "pre-commit==3.6.2",
52
- "debugpy==1.8.1",
51
+ "ipython==8.25.0",
52
+ "pre-commit==3.7.1",
53
53
  "zimscraperlib[scripts]",
54
54
  "zimscraperlib[lint]",
55
55
  "zimscraperlib[test]",
@@ -0,0 +1 @@
1
+ __version__ = "3.4.0"
@@ -7,7 +7,7 @@ import io
7
7
  import pathlib
8
8
  import subprocess
9
9
  from concurrent.futures import Future, ThreadPoolExecutor
10
- from typing import ClassVar, Optional, Union
10
+ from typing import ClassVar
11
11
 
12
12
  import requests
13
13
  import yt_dlp as youtube_dl
@@ -21,7 +21,7 @@ class YoutubeDownloader:
21
21
  Shutdown method must be run explicitly to
22
22
  free any occupied resources"""
23
23
 
24
- def __init__(self, threads: Optional[int] = 1) -> None:
24
+ def __init__(self, threads: int | None = 1) -> None:
25
25
  self.executor = ThreadPoolExecutor(max_workers=threads)
26
26
 
27
27
  def __enter__(self):
@@ -41,9 +41,9 @@ class YoutubeDownloader:
41
41
  def download(
42
42
  self,
43
43
  url: str,
44
- options: Optional[dict],
45
- wait: Optional[bool] = True, # noqa: FBT002
46
- ) -> Union[bool, Future]:
44
+ options: dict | None,
45
+ wait: bool | None = True, # noqa: FBT002
46
+ ) -> bool | Future:
47
47
  """Downloads video using initialized executor.
48
48
 
49
49
  url: URL or Video ID
@@ -65,8 +65,8 @@ class YoutubeDownloader:
65
65
 
66
66
 
67
67
  class YoutubeConfig(dict):
68
- options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {}
69
- defaults: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
68
+ options: ClassVar[dict[str, str | bool | int | None]] = {}
69
+ defaults: ClassVar[dict[str, str | bool | int | None]] = {
70
70
  "writethumbnail": True,
71
71
  "write_all_thumbnails": True,
72
72
  "writesubtitles": True,
@@ -88,8 +88,8 @@ class YoutubeConfig(dict):
88
88
  @classmethod
89
89
  def get_options(
90
90
  cls,
91
- target_dir: Optional[pathlib.Path] = None,
92
- filepath: Optional[pathlib.Path] = None,
91
+ target_dir: pathlib.Path | None = None,
92
+ filepath: pathlib.Path | None = None,
93
93
  **options,
94
94
  ):
95
95
  if "outtmpl" not in options:
@@ -109,14 +109,14 @@ class YoutubeConfig(dict):
109
109
 
110
110
 
111
111
  class BestWebm(YoutubeConfig):
112
- options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
112
+ options: ClassVar[dict[str, str | bool | int | None]] = {
113
113
  "preferredcodec": "webm",
114
114
  "format": "best[ext=webm]/bestvideo[ext=webm]+bestaudio[ext=webm]/best",
115
115
  }
116
116
 
117
117
 
118
118
  class BestMp4(YoutubeConfig):
119
- options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {
119
+ options: ClassVar[dict[str, str | bool | int | None]] = {
120
120
  "preferredcodec": "mp4",
121
121
  "format": "best[ext=mp4]/bestvideo[ext=mp4]+bestaudio[ext=m4a]/best",
122
122
  }
@@ -142,7 +142,7 @@ def save_large_file(url: str, fpath: pathlib.Path) -> None:
142
142
 
143
143
 
144
144
  def _get_retry_adapter(
145
- max_retries: Optional[int] = 5,
145
+ max_retries: int | None = 5,
146
146
  ) -> requests.adapters.BaseAdapter: # pyright: ignore
147
147
  retries = requests.packages.urllib3.util.retry.Retry( # pyright: ignore
148
148
  total=max_retries, # total number of retries
@@ -164,7 +164,7 @@ def _get_retry_adapter(
164
164
  return requests.adapters.HTTPAdapter(max_retries=retries) # pyright: ignore
165
165
 
166
166
 
167
- def get_session(max_retries: Optional[int] = 5) -> requests.Session:
167
+ def get_session(max_retries: int | None = 5) -> requests.Session:
168
168
  """Session to hold cookies and connection pool together"""
169
169
  session = requests.Session()
170
170
  session.mount("http", _get_retry_adapter(max_retries)) # tied to http and https
@@ -173,14 +173,14 @@ def get_session(max_retries: Optional[int] = 5) -> requests.Session:
173
173
 
174
174
  def stream_file(
175
175
  url: str,
176
- fpath: Optional[pathlib.Path] = None,
177
- byte_stream: Optional[io.BytesIO] = None,
178
- block_size: Optional[int] = 1024,
179
- proxies: Optional[dict] = None,
180
- only_first_block: Optional[bool] = False, # noqa: FBT002
181
- max_retries: Optional[int] = 5,
182
- headers: Optional[dict[str, str]] = None,
183
- session: Optional[requests.Session] = None,
176
+ fpath: pathlib.Path | None = None,
177
+ byte_stream: io.BytesIO | None = None,
178
+ block_size: int | None = 1024,
179
+ proxies: dict | None = None,
180
+ only_first_block: bool | None = False, # noqa: FBT002
181
+ max_retries: int | None = 5,
182
+ headers: dict[str, str] | None = None,
183
+ session: requests.Session | None = None,
184
184
  ) -> tuple[int, requests.structures.CaseInsensitiveDict]: # pyright: ignore
185
185
  """Stream data from a URL to either a BytesIO object or a file
186
186
  Arguments -
@@ -5,10 +5,12 @@
5
5
 
6
6
  Shortcuts to retrieve mime type using magic"""
7
7
 
8
+ from __future__ import annotations
9
+
8
10
  import os
9
11
  import pathlib
10
12
  from collections.abc import Callable
11
- from typing import Any, Optional, Union
13
+ from typing import Any
12
14
 
13
15
  import magic
14
16
 
@@ -43,8 +45,8 @@ def get_content_mimetype(content: bytes) -> str:
43
45
 
44
46
 
45
47
  def delete_callback(
46
- fpath: Union[str, pathlib.Path],
47
- callback: Optional[Callable] = None,
48
+ fpath: str | pathlib.Path,
49
+ callback: Callable | None = None,
48
50
  *callback_args: Any,
49
51
  ):
50
52
  """helper deleting passed filepath, optionnaly calling an additional callback"""
@@ -9,13 +9,12 @@ from __future__ import annotations
9
9
  import logging
10
10
  import pathlib
11
11
  import sys
12
- from typing import Union
13
12
 
14
13
  logging.basicConfig(format="%(levelname)s:%(message)s", level=logging.DEBUG)
15
14
  logger = logging.getLogger(__name__)
16
15
 
17
16
 
18
- def fix_source_dir(source_vendors_path: Union[pathlib.Path, str]):
17
+ def fix_source_dir(source_vendors_path: pathlib.Path | str):
19
18
  """update ogvjs plugin to trigger on webm mimetype"""
20
19
  root = pathlib.Path(source_vendors_path)
21
20
  logger.info("fixing videosjs-ogvjs.js")