zimscraperlib 3.3.1__tar.gz → 3.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/.pre-commit-config.yaml +3 -3
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/CHANGELOG.md +150 -123
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/PKG-INFO +18 -13
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/README.md +3 -1
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/pyproject.toml +17 -17
- zimscraperlib-3.4.0/src/zimscraperlib/__about__.py +1 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/download.py +21 -21
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/filesystem.py +5 -3
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/fix_ogvjs_dist.py +1 -2
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/html.py +4 -3
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/i18n.py +3 -6
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/image/convertion.py +4 -3
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/image/optimization.py +27 -28
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/image/presets.py +13 -13
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/image/probing.py +4 -5
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/image/transformation.py +7 -6
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/image/utils.py +3 -3
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/inputs.py +8 -9
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/logging.py +12 -11
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/misc.py +2 -2
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/types.py +2 -3
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/uri.py +11 -10
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/video/config.py +4 -4
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/video/encoding.py +2 -3
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/video/presets.py +24 -16
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/zim/_libkiwix.py +2 -2
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/zim/archive.py +2 -3
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/zim/creator.py +85 -34
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/zim/filesystem.py +96 -4
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/zim/items.py +18 -17
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/zim/metadata.py +4 -4
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/zim/providers.py +9 -10
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tasks.py +2 -1
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/conftest.py +12 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/download/test_download.py +2 -2
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/inputs/test_inputs.py +4 -5
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/ogvjs/test_ogvjs.py +2 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/video/test_encoding.py +2 -3
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/video/test_video.py +25 -17
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/zim/test_fs.py +45 -1
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/zim/test_metadata.py +2 -3
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/zim/test_zim_creator.py +101 -0
- zimscraperlib-3.3.1/src/zimscraperlib/__about__.py +0 -1
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/.gitignore +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/LICENSE +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/contrib/README.md +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/contrib/encode_video.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/__init__.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/constants.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/image/__init__.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/video/__init__.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/video/probing.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/src/zimscraperlib/zim/__init__.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/files/DroidSans.ttf +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/files/audio.mp3 +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/files/blue.jpg +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/files/commons.png +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/files/commons48.png +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/files/mail.gif +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/files/ninja.webp +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/files/pluto.jpg +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/files/square.jpg +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/files/square.png +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/files/star.svg +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/files/video.mkv +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/files/video.mp4 +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/files/video.webm +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/filesystem/test_filesystem.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/html/conftest.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/html/test_html.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/i18n/locale/fr/LC_MESSAGES/messages.po +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/i18n/locale/pt/LC_MESSAGES/messages.po +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/i18n/test_i18n.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/image/test_image.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/logging/conftest.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/logging/test_logging.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/misc/test_misc.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/ogvjs/conftest.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/types/test_types.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/uri/test_uri.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/video/conftest.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/zim/conftest.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/zim/test_archive.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.4.0}/tests/zim/test_libkiwix.py +0 -0
|
@@ -7,15 +7,15 @@ repos:
|
|
|
7
7
|
- id: trailing-whitespace
|
|
8
8
|
- id: end-of-file-fixer
|
|
9
9
|
- repo: https://github.com/psf/black
|
|
10
|
-
rev: "24.
|
|
10
|
+
rev: "24.4.2"
|
|
11
11
|
hooks:
|
|
12
12
|
- id: black
|
|
13
13
|
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
14
|
-
rev: v0.
|
|
14
|
+
rev: v0.4.9
|
|
15
15
|
hooks:
|
|
16
16
|
- id: ruff
|
|
17
17
|
- repo: https://github.com/RobertCraigie/pyright-python
|
|
18
|
-
rev: v1.1.
|
|
18
|
+
rev: v1.1.368
|
|
19
19
|
hooks:
|
|
20
20
|
- id: pyright
|
|
21
21
|
name: pyright (system)
|
|
@@ -5,6 +5,35 @@ All notable changes to this project are documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0).
|
|
7
7
|
|
|
8
|
+
## [3.4.0] - 2024-06-21
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- `zim.creator.Creator._log_metadata()` to log (DEBUG) all metadata set on `_metadata` (prior to start()) #155
|
|
13
|
+
- New utility function to confirm ZIM can be created at given location / name #163
|
|
14
|
+
|
|
15
|
+
### Changed
|
|
16
|
+
|
|
17
|
+
- Migrate the **VideoWebmLow** and **VideoWebmHigh** presets to VP9 for smaller file size #79
|
|
18
|
+
- New preset versions are v3 and v2 respectively
|
|
19
|
+
- Simplify type annotations by replacing Union and Optional with pipe character ("|") for improved readability and clarity #150
|
|
20
|
+
- Calling `Creator._log_metadata()` on `Creator.start()` if running in DEBUG #155
|
|
21
|
+
|
|
22
|
+
### Fixed
|
|
23
|
+
|
|
24
|
+
- Add back the `--runinstalled` flag for test execution to allow smooth testing on other build chains #139
|
|
25
|
+
|
|
26
|
+
## [3.3.2] - 2024-03-25
|
|
27
|
+
|
|
28
|
+
### Added
|
|
29
|
+
|
|
30
|
+
- Add support for `disable_metadata_checks` and `ignore_duplicates` arguments in `make_zim_file` function ("zimwritefs-mode")
|
|
31
|
+
|
|
32
|
+
### Changed
|
|
33
|
+
|
|
34
|
+
- Relaxed constraints on Python dependencies
|
|
35
|
+
- Upgraded optional dependencies used for test and QA
|
|
36
|
+
|
|
8
37
|
## [3.3.1] - 2024-02-27
|
|
9
38
|
|
|
10
39
|
### Added
|
|
@@ -126,6 +155,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
126
155
|
- [libzim] `Archive.media_count`
|
|
127
156
|
|
|
128
157
|
### Changed
|
|
158
|
+
|
|
129
159
|
- [libzim] `Archive.article_count` updated to match scraperlib's version
|
|
130
160
|
- `Archive.article_counter` now deprecated. Now returns `Archive.article_count`
|
|
131
161
|
- `Archive.media_counter` now deprecated. Now returns `Archive.media_count`
|
|
@@ -203,197 +233,194 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
203
233
|
- Unreachable code removed in `imaging` module.
|
|
204
234
|
- [tests] “Sanskrit” removed from tests as output not predicatble depending on plaftform.
|
|
205
235
|
|
|
206
|
-
|
|
207
236
|
## [1.4.3]
|
|
208
237
|
|
|
209
|
-
|
|
238
|
+
- `zim.Archive.counters` wont fail on missing `Counter` metadata
|
|
210
239
|
|
|
211
240
|
## [1.4.2]
|
|
212
241
|
|
|
213
|
-
|
|
214
|
-
|
|
242
|
+
- Fixed leak in `zim.Archive`'s `.counters`
|
|
243
|
+
- New `.get_text_metadata()` method on `zim.Archive` to save UTF-8 decoding
|
|
215
244
|
|
|
216
245
|
## [1.4.1]
|
|
217
246
|
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
247
|
+
- New `Counter` metadata based properties for Archive:
|
|
248
|
+
- `.counters`: parsed dict of the Counter metadata
|
|
249
|
+
- `.article_counter`: libkiwix's calculation for nb or article
|
|
250
|
+
- `.media_counter`: libkiwix's calculation for nb or media
|
|
251
|
+
- Fixed `i18n.find_language_names()` failing on some languages
|
|
252
|
+
- Added `uri` module with `rebuild_uri()`
|
|
225
253
|
|
|
226
254
|
## [1.4.0]
|
|
227
255
|
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
256
|
+
- Using new python-libzim based on libzim v7
|
|
257
|
+
- New Creator API
|
|
258
|
+
- Removed all namespace references
|
|
259
|
+
- Renamed `url` mentions to `path`
|
|
260
|
+
- Removed all links rewriting
|
|
261
|
+
- Removed Article/CSS/Binary seggreation
|
|
262
|
+
- Kept zimwriterfs mode (except it doesn't rewrite for namespaces)
|
|
263
|
+
- New `html` module for HTML document manipulations
|
|
264
|
+
- New callback system on `add_item_for()` and `add_item()`
|
|
265
|
+
- New Archive API with easier search/suggestions and content access
|
|
266
|
+
- Changed download log level to DEBUG (was INFO)
|
|
267
|
+
- `filesystem.get_file_mimetype` now passes bytes to libmagic instead of filename due to release issue in libmagic
|
|
268
|
+
- safer `inputs.handle_user_provided_file` regarding input as str instead of Path
|
|
269
|
+
- `image.presets` and `video.presets` now all includes `ext` and `mimetype` properties
|
|
270
|
+
- Video convert log now DEBUG instead of INFO
|
|
271
|
+
- Fixed `image.save_image()` saving to disk even when using a bytes stream
|
|
272
|
+
- Fixed `image.transformation.resize_image()` when resizing a byte stream without a dst
|
|
245
273
|
|
|
246
274
|
## [1.3.6 (internal)]
|
|
247
275
|
|
|
248
276
|
Intermediate release using unreleased libzim to support development of libzim7.
|
|
249
277
|
Don't use it.
|
|
250
278
|
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
279
|
+
- requesting newer libzim version (not released ATM)
|
|
280
|
+
- New ZIM API for non-namespace libzim (v7)
|
|
281
|
+
- updated all requirements
|
|
282
|
+
- Fixed download test inconsistency
|
|
283
|
+
- fix_ogvjs mostly useless: only allows webm types
|
|
284
|
+
- exposing retry_adapter for refactoring
|
|
285
|
+
- Changed download log level to DEBUG (was INFO)
|
|
286
|
+
- guess more-defined mime from filename if magic says it's text
|
|
287
|
+
- get_file_mimetype now passes bytes to libmagic
|
|
288
|
+
- safer regarding input as str instead of Path
|
|
289
|
+
- fixed static item for empty content
|
|
290
|
+
- ext and mimetype properties for all presets
|
|
291
|
+
- Video convert log now DEBUG instead of INFO
|
|
292
|
+
- Added delete_fpath to add_item_for() and fixed StaticItem's auto remove
|
|
293
|
+
- Updated badges for new repo name
|
|
266
294
|
|
|
267
295
|
## [1.3.5]
|
|
268
296
|
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
297
|
+
- add `stream_file()` to stream content from a URL into a file or a `BytesIO` object
|
|
298
|
+
- deprecated `save_file()`
|
|
299
|
+
- fixed `add_binary` when used without an fpath (#69)
|
|
300
|
+
- deprecated `make_grayscale` option in image optimization
|
|
301
|
+
- Added support for in-memory optimization for PNG, JPEG, and WebP images
|
|
302
|
+
- allows enabling debug logs via ZIMSCRAPERLIB_DEBUG environ
|
|
275
303
|
|
|
276
304
|
## [1.3.4]
|
|
277
305
|
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
306
|
+
- added `wait` option in `YoutubeDownloader` to allow parallelism while using context manager
|
|
307
|
+
- do not use extension for finding format in `ensure_matches()` in `image.optimization` module
|
|
308
|
+
- added `VideoWebmHigh` and `VideoMp4High` presets for high quality WebM and Mp4 convertion respectively
|
|
309
|
+
- updated presets `WebpHigh`, `JpegMedium`, `JpegLow` and `PngMedium` in `image.presets`
|
|
310
|
+
- `save_image` moved from `image` to `image.utils`
|
|
311
|
+
- added `convert_image` `optimize_image` `resize_image` functions to `image` module
|
|
284
312
|
|
|
285
313
|
## [1.3.3]
|
|
286
314
|
|
|
287
|
-
|
|
315
|
+
- added `YoutubeDownloader` to `download` to download YT videos using a capped nb of threads
|
|
288
316
|
|
|
289
317
|
## [1.3.2]
|
|
290
318
|
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
319
|
+
- fixed rewriting of links with empty target
|
|
320
|
+
- added support for image optimization using `zimscraperlib.image.optimization` for webp, gif, jpeg and png formats
|
|
321
|
+
- added `format_for()` in `zimscraperlib.image.probing` to get PIL image format from the suffix
|
|
294
322
|
|
|
295
323
|
## [1.3.1]
|
|
296
324
|
|
|
297
|
-
|
|
325
|
+
- replaced BeautifoulSoup parser in rewriting (`html.parser` –> `lxml`)
|
|
298
326
|
|
|
299
327
|
## [1.3.0]
|
|
300
328
|
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
329
|
+
- detect mimetypes from filenames for all text files
|
|
330
|
+
- fixed non-filename based StaticArticle
|
|
331
|
+
- enable rewriting of links in poster attribute of audio element
|
|
332
|
+
- added find_language_in() and find_language_in_file() to get language from HTML content and HTML file respectively
|
|
333
|
+
- add a mime mapping to deal with inconsistencies in mimetypes detected by magic on different platforms
|
|
334
|
+
- convert_image signature changed:
|
|
335
|
+
- `target_format` positional argument removed. Replaced with optionnal `fmt` key of keyword arguments.
|
|
336
|
+
- `colorspace` optionnal positional argument removed. Replaced with optionnal `colorspace` key of keyword arguments.
|
|
337
|
+
- prevent rewriting of links with special schemes `mailto`, 'tel', etc. in HTML links rewriting
|
|
338
|
+
- replaced `imaging` module with exploded `image` module (`convertion`, `probing`, `transformation`)
|
|
339
|
+
- changed `create_favicon()` param names (`source_image` -> `src`, `dest_ico` -> `dst`)
|
|
340
|
+
- changed `save_image()` param names (`image` -> `src`)
|
|
341
|
+
- changed `get_colors()` param names (`image_path` -> `src`)
|
|
342
|
+
- changed `resize_image()` param names (`fpath` -> `src`)
|
|
315
343
|
|
|
316
344
|
## [1.2.1]
|
|
317
345
|
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
346
|
+
- fixed URL rewriting when running from /
|
|
347
|
+
- added support for link rewriting in `<object>` element
|
|
348
|
+
- prevent from raising error if element doesn't have the attribute with url
|
|
349
|
+
- use non greedy match for CSS URL links (shortest string matching `url()` format)
|
|
350
|
+
- fix namespace of target only if link doesn't have a netloc
|
|
323
351
|
|
|
324
352
|
## [1.2.0]
|
|
325
353
|
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
354
|
+
- added UTF8 to constants
|
|
355
|
+
- added mime_type discovery via magic (filesystem)
|
|
356
|
+
- Added types: mime types guessing from file names
|
|
357
|
+
- Revamped zim API
|
|
358
|
+
- Removed ZimInfo which role was tu hold metadata for zimwriterfs call
|
|
359
|
+
- Removed calling zimwriterfs binary but kept function name
|
|
360
|
+
- Added zim.filesystem: zimwriterfs-like creation from a build folder
|
|
361
|
+
- Added zim.creator: create files by manually adding each article
|
|
362
|
+
- Added zim.rewriting: tools to rewrite links/urls in HTML/CSS
|
|
363
|
+
- add timeout and retries to save_file() and make it return headers
|
|
336
364
|
|
|
337
365
|
## [1.1.2]
|
|
338
366
|
|
|
339
|
-
|
|
367
|
+
- fixed `convert_image()` which tried to use a closed file
|
|
340
368
|
|
|
341
369
|
## [1.1.1]
|
|
342
370
|
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
371
|
+
- exposed reencode, Config and get_media_info in zimscraperlib.video
|
|
372
|
+
- added save_image() and convert_image() in zimscraperlib.imaging
|
|
373
|
+
- added support for upscaling in resize_image() via allow_upscaling
|
|
374
|
+
- resize_image() now supports params given by user and preservs image colorspace
|
|
375
|
+
- fixed tests for zimscraperlib.imaging
|
|
348
376
|
|
|
349
377
|
## [1.1.0]
|
|
350
378
|
|
|
351
|
-
|
|
352
|
-
|
|
379
|
+
- added video module with reencode, presets, config builder and video file probing
|
|
380
|
+
- `make_zim_file()` accepts extra kwargs for zimwriterfs
|
|
353
381
|
|
|
354
382
|
## [1.0.6]
|
|
355
383
|
|
|
356
|
-
|
|
384
|
+
- added translation support to i18n
|
|
357
385
|
|
|
358
386
|
## [1.0.5]
|
|
359
387
|
|
|
360
|
-
|
|
361
|
-
|
|
388
|
+
- added s3transfer to verbose dependencies list
|
|
389
|
+
- changed default log format to include module name
|
|
362
390
|
|
|
363
391
|
## [1.0.4]
|
|
364
392
|
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
393
|
+
- verbose dependencies (urllib3, boto3) now logged at WARNING level by default
|
|
394
|
+
- ability to set verbose dependencies log level and add modules to the list
|
|
395
|
+
- zimscraperlib's logging level now aligned with scraper's requested one
|
|
369
396
|
|
|
370
397
|
## [1.0.3]
|
|
371
398
|
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
399
|
+
- fix_ogvjs_dist script more generic (#1)
|
|
400
|
+
- updated zim to support other zimwriterfs params (#10)
|
|
401
|
+
- more flexible requirements for requests dependency
|
|
375
402
|
|
|
376
403
|
## [1.0.2]
|
|
377
404
|
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
405
|
+
- fixed return value of `get_language_details` on non-existent language
|
|
406
|
+
- fixed crash on `resize_image` with method `height`
|
|
407
|
+
- fixed root logger level (now DEBUG)
|
|
408
|
+
- removed useless `console=True` `getLogger` param
|
|
409
|
+
- completed tests (100% coverage)
|
|
410
|
+
- added `./test` script for quick local testing
|
|
411
|
+
- improved tox.ini
|
|
412
|
+
- added `create_favicon` to generate a squared favicon
|
|
413
|
+
- added `handle_user_provided_file` to handle user file/URL from param
|
|
387
414
|
|
|
388
415
|
## [1.0.1]
|
|
389
416
|
|
|
390
|
-
|
|
417
|
+
- fixed fix_ogvjs_dist
|
|
391
418
|
|
|
392
419
|
## [1.0.0]
|
|
393
420
|
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
421
|
+
- initial version providing
|
|
422
|
+
- download: save_file, save_large_file
|
|
423
|
+
- fix_ogvjs_dist
|
|
424
|
+
- i18n: setlocale, get_language_details
|
|
425
|
+
- imaging: get_colors, resize_image, is_hex_color
|
|
426
|
+
- zim: ZimInfo, make_zim_file
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: zimscraperlib
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.4.0
|
|
4
4
|
Summary: Collection of python tools to re-use common code across scrapers
|
|
5
5
|
Project-URL: Donate, https://www.kiwix.org/en/support-us/
|
|
6
6
|
Project-URL: Homepage, https://github.com/openzim/python-scraperlib
|
|
@@ -8,6 +8,8 @@ Author-email: openZIM <dev@openzim.org>
|
|
|
8
8
|
License: GPL-3.0-or-later
|
|
9
9
|
License-File: LICENSE
|
|
10
10
|
Keywords: offline,openzim,zim
|
|
11
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
11
13
|
Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
|
|
12
14
|
Classifier: Programming Language :: Python :: 3
|
|
13
15
|
Classifier: Programming Language :: Python :: 3.8
|
|
@@ -17,34 +19,35 @@ Classifier: Programming Language :: Python :: 3.11
|
|
|
17
19
|
Classifier: Programming Language :: Python :: 3.12
|
|
18
20
|
Requires-Python: <3.13,>=3.8
|
|
19
21
|
Requires-Dist: babel<3.0,>=2.9
|
|
20
|
-
Requires-Dist: beautifulsoup4<
|
|
22
|
+
Requires-Dist: beautifulsoup4<5.0,>=4.9.3
|
|
21
23
|
Requires-Dist: colorthief==0.2.1
|
|
22
24
|
Requires-Dist: iso639-lang<3.0,>=2.2.3
|
|
23
25
|
Requires-Dist: libzim<4.0,>=3.4.0
|
|
24
|
-
Requires-Dist: lxml<
|
|
25
|
-
Requires-Dist: optimize-images<
|
|
26
|
+
Requires-Dist: lxml<6.0,>=4.6.3
|
|
27
|
+
Requires-Dist: optimize-images<2.0,>=1.3.6
|
|
26
28
|
Requires-Dist: python-magic<0.5,>=0.4.3
|
|
27
29
|
Requires-Dist: python-resize-image<1.2,>=1.1.19
|
|
28
30
|
Requires-Dist: requests<3.0,>=2.25.1
|
|
29
31
|
Requires-Dist: yt-dlp
|
|
30
32
|
Provides-Extra: check
|
|
31
|
-
Requires-Dist: pyright==1.1.
|
|
33
|
+
Requires-Dist: pyright==1.1.368; extra == 'check'
|
|
34
|
+
Requires-Dist: pytest==8.2.2; extra == 'check'
|
|
32
35
|
Provides-Extra: dev
|
|
33
|
-
Requires-Dist:
|
|
34
|
-
Requires-Dist: pre-commit==3.
|
|
36
|
+
Requires-Dist: ipython==8.25.0; extra == 'dev'
|
|
37
|
+
Requires-Dist: pre-commit==3.7.1; extra == 'dev'
|
|
35
38
|
Requires-Dist: zimscraperlib[check]; extra == 'dev'
|
|
36
39
|
Requires-Dist: zimscraperlib[lint]; extra == 'dev'
|
|
37
40
|
Requires-Dist: zimscraperlib[scripts]; extra == 'dev'
|
|
38
41
|
Requires-Dist: zimscraperlib[test]; extra == 'dev'
|
|
39
42
|
Provides-Extra: lint
|
|
40
|
-
Requires-Dist: black==24.
|
|
41
|
-
Requires-Dist: ruff==0.
|
|
43
|
+
Requires-Dist: black==24.4.2; extra == 'lint'
|
|
44
|
+
Requires-Dist: ruff==0.4.9; extra == 'lint'
|
|
42
45
|
Provides-Extra: scripts
|
|
43
46
|
Requires-Dist: invoke==2.2.0; extra == 'scripts'
|
|
44
47
|
Provides-Extra: test
|
|
45
|
-
Requires-Dist: coverage==7.
|
|
46
|
-
Requires-Dist: pytest-mock==3.
|
|
47
|
-
Requires-Dist: pytest==8.
|
|
48
|
+
Requires-Dist: coverage==7.5.3; extra == 'test'
|
|
49
|
+
Requires-Dist: pytest-mock==3.14.0; extra == 'test'
|
|
50
|
+
Requires-Dist: pytest==8.2.2; extra == 'test'
|
|
48
51
|
Description-Content-Type: text/markdown
|
|
49
52
|
|
|
50
53
|
zimscraperlib
|
|
@@ -104,7 +107,9 @@ apk add ffmpeg gifsicle libmagic wget libjpeg
|
|
|
104
107
|
|
|
105
108
|
# Contribution
|
|
106
109
|
|
|
107
|
-
This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing)
|
|
110
|
+
This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing).
|
|
111
|
+
|
|
112
|
+
This project has implemented openZIM's [Python bootstrap, conventions and policies](https://github.com/openzim/_python-bootstrap/docs/Policy.md) **v1.0.2**.
|
|
108
113
|
|
|
109
114
|
```shell
|
|
110
115
|
pip install hatch
|
|
@@ -55,7 +55,9 @@ apk add ffmpeg gifsicle libmagic wget libjpeg
|
|
|
55
55
|
|
|
56
56
|
# Contribution
|
|
57
57
|
|
|
58
|
-
This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing)
|
|
58
|
+
This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing).
|
|
59
|
+
|
|
60
|
+
This project has implemented openZIM's [Python bootstrap, conventions and policies](https://github.com/openzim/_python-bootstrap/docs/Policy.md) **v1.0.2**.
|
|
59
61
|
|
|
60
62
|
```shell
|
|
61
63
|
pip install hatch
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
[build-system]
|
|
2
|
-
requires = ["hatchling", "hatch-openzim"]
|
|
2
|
+
requires = ["hatchling", "hatch-openzim>=0.2"]
|
|
3
3
|
build-backend = "hatchling.build"
|
|
4
4
|
|
|
5
5
|
[project]
|
|
@@ -15,9 +15,9 @@ dependencies = [
|
|
|
15
15
|
"Babel>=2.9,<3.0",
|
|
16
16
|
"python-magic>=0.4.3,<0.5",
|
|
17
17
|
"libzim>=3.4.0,<4.0",
|
|
18
|
-
"beautifulsoup4>=4.9.3,<
|
|
19
|
-
"lxml>=4.6.3,<
|
|
20
|
-
"optimize-images>=1.3.6,<
|
|
18
|
+
"beautifulsoup4>=4.9.3,<5.0",
|
|
19
|
+
"lxml>=4.6.3,<6.0",
|
|
20
|
+
"optimize-images>=1.3.6,<2.0",
|
|
21
21
|
# youtube-dl should be updated as frequently as possible
|
|
22
22
|
"yt-dlp"
|
|
23
23
|
]
|
|
@@ -25,31 +25,31 @@ dynamic = ["authors", "classifiers", "keywords", "license", "version", "urls"]
|
|
|
25
25
|
|
|
26
26
|
[tool.hatch.metadata.hooks.openzim-metadata]
|
|
27
27
|
kind = "scraper"
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
# ]
|
|
28
|
+
additional-classifiers = [
|
|
29
|
+
"Development Status :: 5 - Production/Stable",
|
|
30
|
+
"Intended Audience :: Developers",
|
|
31
|
+
]
|
|
33
32
|
|
|
34
33
|
[project.optional-dependencies]
|
|
35
34
|
scripts = [
|
|
36
35
|
"invoke==2.2.0",
|
|
37
36
|
]
|
|
38
37
|
lint = [
|
|
39
|
-
"black==24.
|
|
40
|
-
"ruff==0.
|
|
38
|
+
"black==24.4.2",
|
|
39
|
+
"ruff==0.4.9",
|
|
41
40
|
]
|
|
42
41
|
check = [
|
|
43
|
-
"pyright==1.1.
|
|
42
|
+
"pyright==1.1.368",
|
|
43
|
+
"pytest==8.2.2",
|
|
44
44
|
]
|
|
45
45
|
test = [
|
|
46
|
-
"pytest==8.
|
|
47
|
-
"pytest-mock==3.
|
|
48
|
-
"coverage==7.
|
|
46
|
+
"pytest==8.2.2",
|
|
47
|
+
"pytest-mock==3.14.0",
|
|
48
|
+
"coverage==7.5.3",
|
|
49
49
|
]
|
|
50
50
|
dev = [
|
|
51
|
-
"
|
|
52
|
-
"
|
|
51
|
+
"ipython==8.25.0",
|
|
52
|
+
"pre-commit==3.7.1",
|
|
53
53
|
"zimscraperlib[scripts]",
|
|
54
54
|
"zimscraperlib[lint]",
|
|
55
55
|
"zimscraperlib[test]",
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "3.4.0"
|
|
@@ -7,7 +7,7 @@ import io
|
|
|
7
7
|
import pathlib
|
|
8
8
|
import subprocess
|
|
9
9
|
from concurrent.futures import Future, ThreadPoolExecutor
|
|
10
|
-
from typing import ClassVar
|
|
10
|
+
from typing import ClassVar
|
|
11
11
|
|
|
12
12
|
import requests
|
|
13
13
|
import yt_dlp as youtube_dl
|
|
@@ -21,7 +21,7 @@ class YoutubeDownloader:
|
|
|
21
21
|
Shutdown method must be run explicitly to
|
|
22
22
|
free any occupied resources"""
|
|
23
23
|
|
|
24
|
-
def __init__(self, threads:
|
|
24
|
+
def __init__(self, threads: int | None = 1) -> None:
|
|
25
25
|
self.executor = ThreadPoolExecutor(max_workers=threads)
|
|
26
26
|
|
|
27
27
|
def __enter__(self):
|
|
@@ -41,9 +41,9 @@ class YoutubeDownloader:
|
|
|
41
41
|
def download(
|
|
42
42
|
self,
|
|
43
43
|
url: str,
|
|
44
|
-
options:
|
|
45
|
-
wait:
|
|
46
|
-
) ->
|
|
44
|
+
options: dict | None,
|
|
45
|
+
wait: bool | None = True, # noqa: FBT002
|
|
46
|
+
) -> bool | Future:
|
|
47
47
|
"""Downloads video using initialized executor.
|
|
48
48
|
|
|
49
49
|
url: URL or Video ID
|
|
@@ -65,8 +65,8 @@ class YoutubeDownloader:
|
|
|
65
65
|
|
|
66
66
|
|
|
67
67
|
class YoutubeConfig(dict):
|
|
68
|
-
options: ClassVar[dict[str,
|
|
69
|
-
defaults: ClassVar[dict[str,
|
|
68
|
+
options: ClassVar[dict[str, str | bool | int | None]] = {}
|
|
69
|
+
defaults: ClassVar[dict[str, str | bool | int | None]] = {
|
|
70
70
|
"writethumbnail": True,
|
|
71
71
|
"write_all_thumbnails": True,
|
|
72
72
|
"writesubtitles": True,
|
|
@@ -88,8 +88,8 @@ class YoutubeConfig(dict):
|
|
|
88
88
|
@classmethod
|
|
89
89
|
def get_options(
|
|
90
90
|
cls,
|
|
91
|
-
target_dir:
|
|
92
|
-
filepath:
|
|
91
|
+
target_dir: pathlib.Path | None = None,
|
|
92
|
+
filepath: pathlib.Path | None = None,
|
|
93
93
|
**options,
|
|
94
94
|
):
|
|
95
95
|
if "outtmpl" not in options:
|
|
@@ -109,14 +109,14 @@ class YoutubeConfig(dict):
|
|
|
109
109
|
|
|
110
110
|
|
|
111
111
|
class BestWebm(YoutubeConfig):
|
|
112
|
-
options: ClassVar[dict[str,
|
|
112
|
+
options: ClassVar[dict[str, str | bool | int | None]] = {
|
|
113
113
|
"preferredcodec": "webm",
|
|
114
114
|
"format": "best[ext=webm]/bestvideo[ext=webm]+bestaudio[ext=webm]/best",
|
|
115
115
|
}
|
|
116
116
|
|
|
117
117
|
|
|
118
118
|
class BestMp4(YoutubeConfig):
|
|
119
|
-
options: ClassVar[dict[str,
|
|
119
|
+
options: ClassVar[dict[str, str | bool | int | None]] = {
|
|
120
120
|
"preferredcodec": "mp4",
|
|
121
121
|
"format": "best[ext=mp4]/bestvideo[ext=mp4]+bestaudio[ext=m4a]/best",
|
|
122
122
|
}
|
|
@@ -142,7 +142,7 @@ def save_large_file(url: str, fpath: pathlib.Path) -> None:
|
|
|
142
142
|
|
|
143
143
|
|
|
144
144
|
def _get_retry_adapter(
|
|
145
|
-
max_retries:
|
|
145
|
+
max_retries: int | None = 5,
|
|
146
146
|
) -> requests.adapters.BaseAdapter: # pyright: ignore
|
|
147
147
|
retries = requests.packages.urllib3.util.retry.Retry( # pyright: ignore
|
|
148
148
|
total=max_retries, # total number of retries
|
|
@@ -164,7 +164,7 @@ def _get_retry_adapter(
|
|
|
164
164
|
return requests.adapters.HTTPAdapter(max_retries=retries) # pyright: ignore
|
|
165
165
|
|
|
166
166
|
|
|
167
|
-
def get_session(max_retries:
|
|
167
|
+
def get_session(max_retries: int | None = 5) -> requests.Session:
|
|
168
168
|
"""Session to hold cookies and connection pool together"""
|
|
169
169
|
session = requests.Session()
|
|
170
170
|
session.mount("http", _get_retry_adapter(max_retries)) # tied to http and https
|
|
@@ -173,14 +173,14 @@ def get_session(max_retries: Optional[int] = 5) -> requests.Session:
|
|
|
173
173
|
|
|
174
174
|
def stream_file(
|
|
175
175
|
url: str,
|
|
176
|
-
fpath:
|
|
177
|
-
byte_stream:
|
|
178
|
-
block_size:
|
|
179
|
-
proxies:
|
|
180
|
-
only_first_block:
|
|
181
|
-
max_retries:
|
|
182
|
-
headers:
|
|
183
|
-
session:
|
|
176
|
+
fpath: pathlib.Path | None = None,
|
|
177
|
+
byte_stream: io.BytesIO | None = None,
|
|
178
|
+
block_size: int | None = 1024,
|
|
179
|
+
proxies: dict | None = None,
|
|
180
|
+
only_first_block: bool | None = False, # noqa: FBT002
|
|
181
|
+
max_retries: int | None = 5,
|
|
182
|
+
headers: dict[str, str] | None = None,
|
|
183
|
+
session: requests.Session | None = None,
|
|
184
184
|
) -> tuple[int, requests.structures.CaseInsensitiveDict]: # pyright: ignore
|
|
185
185
|
"""Stream data from a URL to either a BytesIO object or a file
|
|
186
186
|
Arguments -
|