zimscraperlib 3.3.2__tar.gz → 4.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/.pre-commit-config.yaml +4 -3
  2. zimscraperlib-4.0.0/CHANGELOG.md +465 -0
  3. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/PKG-INFO +18 -12
  4. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/README.md +3 -3
  5. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/pyproject.toml +20 -14
  6. zimscraperlib-4.0.0/src/zimscraperlib/__about__.py +1 -0
  7. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/src/zimscraperlib/download.py +37 -33
  8. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/src/zimscraperlib/filesystem.py +6 -4
  9. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/src/zimscraperlib/fix_ogvjs_dist.py +1 -2
  10. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/src/zimscraperlib/html.py +13 -12
  11. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/src/zimscraperlib/i18n.py +70 -72
  12. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/src/zimscraperlib/image/__init__.py +1 -1
  13. zimscraperlib-3.3.2/src/zimscraperlib/image/convertion.py → zimscraperlib-4.0.0/src/zimscraperlib/image/conversion.py +46 -10
  14. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/src/zimscraperlib/image/optimization.py +93 -64
  15. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/src/zimscraperlib/image/presets.py +13 -13
  16. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/src/zimscraperlib/image/probing.py +39 -15
  17. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/src/zimscraperlib/image/transformation.py +14 -9
  18. zimscraperlib-4.0.0/src/zimscraperlib/image/utils.py +21 -0
  19. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/src/zimscraperlib/inputs.py +34 -10
  20. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/src/zimscraperlib/logging.py +27 -25
  21. zimscraperlib-4.0.0/src/zimscraperlib/misc.py +12 -0
  22. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/src/zimscraperlib/types.py +6 -9
  23. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/src/zimscraperlib/uri.py +20 -18
  24. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/src/zimscraperlib/video/config.py +4 -4
  25. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/src/zimscraperlib/video/encoding.py +10 -10
  26. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/src/zimscraperlib/video/presets.py +24 -16
  27. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/src/zimscraperlib/zim/_libkiwix.py +4 -4
  28. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/src/zimscraperlib/zim/archive.py +3 -4
  29. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/src/zimscraperlib/zim/creator.py +168 -53
  30. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/src/zimscraperlib/zim/filesystem.py +111 -20
  31. zimscraperlib-4.0.0/src/zimscraperlib/zim/indexing.py +119 -0
  32. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/src/zimscraperlib/zim/items.py +96 -19
  33. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/src/zimscraperlib/zim/metadata.py +38 -17
  34. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/src/zimscraperlib/zim/providers.py +11 -14
  35. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tasks.py +2 -1
  36. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/conftest.py +38 -0
  37. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/download/test_download.py +10 -10
  38. zimscraperlib-4.0.0/tests/files/encrypted.pdf +0 -0
  39. zimscraperlib-4.0.0/tests/files/encrypted.txt +3 -0
  40. zimscraperlib-4.0.0/tests/files/milderm.pdf +15239 -37
  41. zimscraperlib-4.0.0/tests/files/milderm.txt +47481 -0
  42. zimscraperlib-4.0.0/tests/files/single_wave_icon.gbr +0 -0
  43. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/html/conftest.py +20 -0
  44. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/html/test_html.py +3 -1
  45. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/i18n/test_i18n.py +57 -25
  46. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/image/test_image.py +203 -67
  47. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/inputs/test_inputs.py +33 -5
  48. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/ogvjs/test_ogvjs.py +2 -0
  49. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/video/test_encoding.py +2 -3
  50. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/video/test_video.py +27 -18
  51. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/zim/test_fs.py +47 -1
  52. zimscraperlib-4.0.0/tests/zim/test_indexing.py +354 -0
  53. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/zim/test_libkiwix.py +6 -7
  54. zimscraperlib-4.0.0/tests/zim/test_metadata.py +63 -0
  55. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/zim/test_zim_creator.py +175 -14
  56. zimscraperlib-3.3.2/CHANGELOG.md +0 -410
  57. zimscraperlib-3.3.2/src/zimscraperlib/__about__.py +0 -1
  58. zimscraperlib-3.3.2/src/zimscraperlib/image/utils.py +0 -20
  59. zimscraperlib-3.3.2/src/zimscraperlib/misc.py +0 -8
  60. zimscraperlib-3.3.2/tests/i18n/locale/fr/LC_MESSAGES/messages.po +0 -23
  61. zimscraperlib-3.3.2/tests/i18n/locale/pt/LC_MESSAGES/messages.po +0 -24
  62. zimscraperlib-3.3.2/tests/zim/test_metadata.py +0 -35
  63. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/.gitignore +0 -0
  64. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/LICENSE +0 -0
  65. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/contrib/README.md +0 -0
  66. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/contrib/encode_video.py +0 -0
  67. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/src/zimscraperlib/__init__.py +0 -0
  68. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/src/zimscraperlib/constants.py +0 -0
  69. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/src/zimscraperlib/video/__init__.py +0 -0
  70. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/src/zimscraperlib/video/probing.py +0 -0
  71. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/src/zimscraperlib/zim/__init__.py +0 -0
  72. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/files/DroidSans.ttf +0 -0
  73. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/files/audio.mp3 +0 -0
  74. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/files/blue.jpg +0 -0
  75. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/files/commons.png +0 -0
  76. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/files/commons48.png +0 -0
  77. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/files/mail.gif +0 -0
  78. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/files/ninja.webp +0 -0
  79. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/files/pluto.jpg +0 -0
  80. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/files/square.jpg +0 -0
  81. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/files/square.png +0 -0
  82. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/files/star.svg +0 -0
  83. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/files/video.mkv +0 -0
  84. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/files/video.mp4 +0 -0
  85. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/files/video.webm +0 -0
  86. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/filesystem/test_filesystem.py +0 -0
  87. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/logging/conftest.py +0 -0
  88. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/logging/test_logging.py +0 -0
  89. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/misc/test_misc.py +0 -0
  90. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/ogvjs/conftest.py +0 -0
  91. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/types/test_types.py +0 -0
  92. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/uri/test_uri.py +0 -0
  93. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/video/conftest.py +0 -0
  94. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/zim/conftest.py +0 -0
  95. {zimscraperlib-3.3.2 → zimscraperlib-4.0.0}/tests/zim/test_archive.py +0 -0
@@ -1,5 +1,6 @@
1
1
  # See https://pre-commit.com for more information
2
2
  # See https://pre-commit.com/hooks.html for more hooks
3
+ exclude: ^tests/files # these are raw test files, no need to mess with them
3
4
  repos:
4
5
  - repo: https://github.com/pre-commit/pre-commit-hooks
5
6
  rev: v4.5.0
@@ -7,15 +8,15 @@ repos:
7
8
  - id: trailing-whitespace
8
9
  - id: end-of-file-fixer
9
10
  - repo: https://github.com/psf/black
10
- rev: "24.3.0"
11
+ rev: "24.4.2"
11
12
  hooks:
12
13
  - id: black
13
14
  - repo: https://github.com/astral-sh/ruff-pre-commit
14
- rev: v0.3.3
15
+ rev: v0.4.9
15
16
  hooks:
16
17
  - id: ruff
17
18
  - repo: https://github.com/RobertCraigie/pyright-python
18
- rev: v1.1.354
19
+ rev: v1.1.368
19
20
  hooks:
20
21
  - id: pyright
21
22
  name: pyright (system)
@@ -0,0 +1,465 @@
1
+ ## Changelog
2
+
3
+ All notable changes to this project are documented in this file.
4
+
5
+ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
6
+ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0).
7
+
8
+ ## [4.0.0] - 2024-08-05
9
+
10
+ ### Added
11
+
12
+ - Add utility function to compute ZIM Tags #164, including deduplication #156
13
+ - Metadata does not automatically drops control characters #159
14
+ - New `indexing.IndexData` class to hold title, content and keywords to pass to libzim to index an item
15
+ - Automatically index PDF documents content #167
16
+ - Automatically set proper title on PDF documents #168
17
+ - Expose new `optimization.get_optimization_method` to get the proper optimization method to call for a given image format
18
+ - Add `optimization.get_optimization_method` to get the proper optimization method to call for a given image format
19
+ - New `creator.Creator.convert_and_check_metadata` to convert metadata to bytes or str for known use cases and check proper type is passed to libzim
20
+ - Add svg2png image conversion function #113
21
+ - Add `conversion.convert_svg2png` image conversion function + support for SVG in `probing.format_for` #113
22
+ - Add `i18n.Lang` class used as typed result of i18n operations #151
23
+
24
+ ### Changed
25
+
26
+ - **BREAKING** Renamed `zimscraperlib.image.convertion` to `zimscraperlib.image.conversion` to fix typo
27
+ - **BREAKING** Many changes in type hints to match the real underlying code
28
+ - **BREAKING** Force all boolean arguments (and some other non-obvious parameters) to be keyword-only in function calls for clarity / disambiguation (see ruff rule FBT002)
29
+ - Prefer to use `IO[bytes]` to `io.BytesIO` when possible since it is more generic
30
+ - **BREAKING** `i18n.NotFound` renamed `i18n.NotFoundError`
31
+ - **BREAKING** `types.get_mime_for_name` now returns `str | None`
32
+ - **BREAKING** `creator.Creator.add_metadata` and `creator.Creator.validate_metadata` now only accepts `bytes | str` as value (it must have been converted before call)
33
+ - **BREAKING** second argument of `creator.Creator.add_metadata` has been renamed to `value` instead of `content` to align with other methods
34
+ - When a type issue arises in metadata checks, wrong value type is displayed in exception
35
+ - **BREAKING** `i18n.get_language_details()`, `i18n.get_iso_lang_data()`, `i18n.find_language_names()` and `i18n.update_with_macro` now process / return a new typed `Lang` class #151
36
+ - **BREAKING** Rename `i18.NotFound` to `i18n.NotFoundError`
37
+
38
+ ### Removed
39
+ - **BREAKING** Remove translation features in `i18n`: `Locale` class + `_` and `setlocale` functions #134
40
+
41
+ ### Fixed
42
+
43
+ - Metadata length validation is buggy for unicode strings #158
44
+ - Pillow 10.4.0 reveals improper type hints for image probing functions #177
45
+ - Enhance error when locale fails to setup #157
46
+
47
+ ## [3.4.0] - 2024-06-21
48
+
49
+ ### Added
50
+
51
+ - `zim.creator.Creator._log_metadata()` to log (DEBUG) all metadata set on `_metadata` (prior to start()) #155
52
+ - New utility function to confirm ZIM can be created at given location / name #163
53
+
54
+ ### Changed
55
+
56
+ - Migrate the **VideoWebmLow** and **VideoWebmHigh** presets to VP9 for smaller file size #79
57
+ - New preset versions are v3 and v2 respectively
58
+ - Simplify type annotations by replacing Union and Optional with pipe character ("|") for improved readability and clarity #150
59
+ - Calling `Creator._log_metadata()` on `Creator.start()` if running in DEBUG #155
60
+
61
+ ### Fixed
62
+
63
+ - Add back the `--runinstalled` flag for test execution to allow smooth testing on other build chains #139
64
+
65
+ ## [3.3.2] - 2024-03-25
66
+
67
+ ### Added
68
+
69
+ - Add support for `disable_metadata_checks` and `ignore_duplicates` arguments in `make_zim_file` function ("zimwritefs-mode")
70
+
71
+ ### Changed
72
+
73
+ - Relaxed constraints on Python dependencies
74
+ - Upgraded optional dependencies used for test and QA
75
+
76
+ ## [3.3.1] - 2024-02-27
77
+
78
+ ### Added
79
+
80
+ - Set a user-agent for `handle_user_provided_file` #103
81
+
82
+ ### Changed
83
+
84
+ - Migrate to generic syntax in all std collections #140
85
+
86
+ ### Fixed
87
+
88
+ - Do not modify the ffmpeg_args in reencode function #144
89
+
90
+ ## [3.3.0] - 2024-02-14
91
+
92
+ ### Added
93
+
94
+ - New `disable_metadata_checks` parameter in `zimscraperlib.zim.creator.Creator` initializer, allowing to disable metadata check at startup (assuming the user will validate them on its own) #119
95
+
96
+ ### Changed
97
+
98
+ - Rework the **VideoWebmLow** preset for faster encoding and smaller file size #122
99
+ - preset has been bumped to **version 2**
100
+ - when using an S3 cache, all videos using this preset will be reencoded and uploaded to cache again (it will replace the same file encoded with preset version 1)
101
+ - When reencoding a video, ffmpeg now uses only 1 CPU thread by default (new arg to `reencode` allows to override this default value)
102
+ - Using openZIM Python bootstrap conventions (including hatch-openzim plugin) #120
103
+ - Add support for Python 3.12, drop Python 3.7 support #118
104
+ - Replace "iso-369" by "iso639-lang" library
105
+ - Replace "file-magic" by "python-magic" library for Alpine Linux support and better maintenance
106
+
107
+ ## Fixed
108
+
109
+ - Fixed type hints of `zimscraperlib.zim.Item` and subclasses, and `zimscraperlib.image.optimization:convert_image`
110
+
111
+ ## [3.2.0] - 2023-12-16
112
+
113
+ ### Added
114
+
115
+ - Add utility function to compute/check ZIM descriptions #110
116
+
117
+ ### Changed
118
+
119
+ - Using pylibzim `3.4.0`
120
+
121
+ ### Removed
122
+
123
+ - Support for Python 3.7 (EOL)
124
+
125
+ ## [3.1.1] - 2023-07-18
126
+
127
+ ### Changed
128
+
129
+ - Fixed declared (hint) return type of `download.stream_file` #104
130
+ - Fixed declared (hint) type of `content` param for `Creator.add_item_for` #107
131
+
132
+ ## [3.1.0] - 2023-05-05
133
+
134
+ ### Changed
135
+
136
+ - Using pylibzim `3.1.0`
137
+ - ZIM metadata check now allows multiple values (comma-separated) for `Language`
138
+ - Using `yt_dlp` instead of `youtube_dl`
139
+
140
+ ### Removed
141
+
142
+ - Dropped support for Python 3.6
143
+
144
+ ## [3.0.0] - 2023-03-31
145
+
146
+ ⚠️ Warning: this release introduce several API changes to `zim.creator.Creator` and `zim.filesystem.make_zim_file`
147
+
148
+ ### Added
149
+
150
+ - `zim.creator.Creator.config_metadata` method (returning Self) exposing all mandatory Metdata, all standard ones and allowing extra text metdadata.
151
+ - `zim.creator.Creator.config_dev_metadata` method setting stub metdata for all mandatory ones (allowing overrides)
152
+ - `zim.metadata` module with a list of per-metadata validation functions
153
+ - `zim.creator.Creator.validate_metadata` (called on `start`) to verify metadata respects the spec (and its recommendations)
154
+ - `zim.filesystem.make_zim_file` accepts a new optional `long_description` param.
155
+ - `i18n.is_valid_iso_639_3` to check ISO-639-3 codes
156
+ - `image.probing.is_valid_image` to check Image format and size
157
+
158
+ ### Changed
159
+
160
+ - `zim.creator.Creator` `main_path` argument now mandatory
161
+ - `zim.creator.Creator.start` now fails on missing required or invalid metadata
162
+ - `zim.creator.Creator.add_metadata` nows enforces validation checks
163
+ - `zim.filesystem.make_zim_file` renamed its `favicon_path` param to `illustration_path`
164
+ - `zim.creator.Creator.config_indexing` `language` argument now optionnal when `indexing=False`
165
+ - `zim.creator.Creator.config_indexing` now validates `language` is ISO- 639-3 when `indexing=True`
166
+
167
+ ### Removed
168
+
169
+ - `zim.creator.Creator.update_metadata`. See `.config_metadata()` instead
170
+ - `zim.creator.Creator` `language` argument. See `.config_metadata()` instead
171
+ - `zim.creator.Creator` keyword arguments. See `.config_metadata()` instead
172
+ - `zim.creator.Creator.add_default_illustration`. See `.config_metadata()` instead
173
+ - `zim.archibe.Archive.media_counter` (deprecated in `2.0.0`)
174
+
175
+ ## [2.1.0] - 2023-03-06
176
+
177
+ ## Added
178
+
179
+ - `zim.creator.Creator(language=)` can be specified as `List[str]`. `["eng", "fra"]`, `["eng"]`, `"eng,fra"`, "eng" are all valid values.
180
+
181
+ ### Changed
182
+
183
+ - Fixed `zim.providers.URLProvider` returning incomplete streams under certain circumstances (from https://github.com/openzim/kolibri/issues/40)
184
+ - Fixed `zim.creator.Creator` not supporting multiple values in for Language metadata, as required by the spec
185
+
186
+ ## [2.0.0] - 2022-12-06
187
+
188
+ - Using pylibzim v2.1.0 (using libzim 8.1.0)
189
+
190
+ ### Added
191
+
192
+ - [libzim] `Entry.get_redirect_entry()`
193
+ - [libzim] `Item.get_indexdata()` to implement custom IndexData per entry (writer)
194
+ - [libzim] `Archive.media_count`
195
+
196
+ ### Changed
197
+
198
+ - [libzim] `Archive.article_count` updated to match scraperlib's version
199
+ - `Archive.article_counter` now deprecated. Now returns `Archive.article_count`
200
+ - `Archive.media_counter` now deprecated. Now returns `Archive.media_count`
201
+
202
+ ### Removed
203
+
204
+ - [libzim] `lzma` compression algorithm
205
+
206
+ ## [1.8.0] - 2022-08-05
207
+
208
+ ### Added
209
+
210
+ - `download.get_session()` to build a new requests Session
211
+
212
+ ### Changed
213
+
214
+ - `download.stream_file()` accepts a `session` param to use instead of creating one
215
+
216
+ ## [1.7.0] - 2022-08-02
217
+
218
+ ### Added
219
+
220
+ - `zim.Creator` now supports `ignore_duplicates: bool` parameter to
221
+ prevent duplicates from raising exceptions
222
+ - `zim.Creator.add_item`, `zim.Creator.add_redirect` and `zim.Creator.add_item_for`
223
+ now supports a `duplicate_ok: bool` parameter to prevent an exception
224
+ should this item/redirect be a duplicate
225
+
226
+ ## [1.6.3] - 2022-08-02
227
+
228
+ ### Added
229
+
230
+ - `download.stream_file()` supports passing `headers` (scrapers were already using it)
231
+
232
+ ## [1.6.2] - 2022-07-29
233
+
234
+ ### Changed
235
+
236
+ - Fixed `filesystem.get_content_mimetype()` crashing on non-guessable byte stream
237
+
238
+ ## [1.6.1] - 2022-07-26
239
+
240
+ ### Changed
241
+
242
+ - Wider range of accepted lxml dependency version as 4.9.1 fixes a security issue
243
+
244
+ ## [1.6.0] - 2022-05-23
245
+
246
+ ## Added
247
+
248
+ - `Archive.get_metadata_item()` to retrieve full item instead of just value
249
+
250
+ ### Changed
251
+
252
+ - Using pylibzim v1.1.0 (using libzim 7.2.1)
253
+ - Adding duplicate entries now raises RuntimeError
254
+ - filesize is fixed for larger ZIMs
255
+
256
+ ## [1.5.0] - 2022-05-09
257
+
258
+ ### Added
259
+
260
+ - `zim.Archive.tags` and `zim.Archive.get_tags()` to retrieve parsed Tags
261
+ with optionnal `libkiwix` param to include libkiwix's hints
262
+ - [tests] Counter tests now also uses a libzim6 file.
263
+
264
+ ### Changed
265
+
266
+ - `zim.Archive.article_counter` follows libkiwix's new bahavior of
267
+ returning libzim's `article_count` for libzim 7+ ZIMs and
268
+ returning previously returned (parsed) value for older ZIMs.
269
+
270
+ ### Removed
271
+
272
+ - Unreachable code removed in `imaging` module.
273
+ - [tests] “Sanskrit” removed from tests as output not predicatble depending on plaftform.
274
+
275
+ ## [1.4.3]
276
+
277
+ - `zim.Archive.counters` wont fail on missing `Counter` metadata
278
+
279
+ ## [1.4.2]
280
+
281
+ - Fixed leak in `zim.Archive`'s `.counters`
282
+ - New `.get_text_metadata()` method on `zim.Archive` to save UTF-8 decoding
283
+
284
+ ## [1.4.1]
285
+
286
+ - New `Counter` metadata based properties for Archive:
287
+ - `.counters`: parsed dict of the Counter metadata
288
+ - `.article_counter`: libkiwix's calculation for nb or article
289
+ - `.media_counter`: libkiwix's calculation for nb or media
290
+ - Fixed `i18n.find_language_names()` failing on some languages
291
+ - Added `uri` module with `rebuild_uri()`
292
+
293
+ ## [1.4.0]
294
+
295
+ - Using new python-libzim based on libzim v7
296
+ - New Creator API
297
+ - Removed all namespace references
298
+ - Renamed `url` mentions to `path`
299
+ - Removed all links rewriting
300
+ - Removed Article/CSS/Binary seggreation
301
+ - Kept zimwriterfs mode (except it doesn't rewrite for namespaces)
302
+ - New `html` module for HTML document manipulations
303
+ - New callback system on `add_item_for()` and `add_item()`
304
+ - New Archive API with easier search/suggestions and content access
305
+ - Changed download log level to DEBUG (was INFO)
306
+ - `filesystem.get_file_mimetype` now passes bytes to libmagic instead of filename due to release issue in libmagic
307
+ - safer `inputs.handle_user_provided_file` regarding input as str instead of Path
308
+ - `image.presets` and `video.presets` now all includes `ext` and `mimetype` properties
309
+ - Video convert log now DEBUG instead of INFO
310
+ - Fixed `image.save_image()` saving to disk even when using a bytes stream
311
+ - Fixed `image.transformation.resize_image()` when resizing a byte stream without a dst
312
+
313
+ ## [1.3.6 (internal)]
314
+
315
+ Intermediate release using unreleased libzim to support development of libzim7.
316
+ Don't use it.
317
+
318
+ - requesting newer libzim version (not released ATM)
319
+ - New ZIM API for non-namespace libzim (v7)
320
+ - updated all requirements
321
+ - Fixed download test inconsistency
322
+ - fix_ogvjs mostly useless: only allows webm types
323
+ - exposing retry_adapter for refactoring
324
+ - Changed download log level to DEBUG (was INFO)
325
+ - guess more-defined mime from filename if magic says it's text
326
+ - get_file_mimetype now passes bytes to libmagic
327
+ - safer regarding input as str instead of Path
328
+ - fixed static item for empty content
329
+ - ext and mimetype properties for all presets
330
+ - Video convert log now DEBUG instead of INFO
331
+ - Added delete_fpath to add_item_for() and fixed StaticItem's auto remove
332
+ - Updated badges for new repo name
333
+
334
+ ## [1.3.5]
335
+
336
+ - add `stream_file()` to stream content from a URL into a file or a `BytesIO` object
337
+ - deprecated `save_file()`
338
+ - fixed `add_binary` when used without an fpath (#69)
339
+ - deprecated `make_grayscale` option in image optimization
340
+ - Added support for in-memory optimization for PNG, JPEG, and WebP images
341
+ - allows enabling debug logs via ZIMSCRAPERLIB_DEBUG environ
342
+
343
+ ## [1.3.4]
344
+
345
+ - added `wait` option in `YoutubeDownloader` to allow parallelism while using context manager
346
+ - do not use extension for finding format in `ensure_matches()` in `image.optimization` module
347
+ - added `VideoWebmHigh` and `VideoMp4High` presets for high quality WebM and Mp4 convertion respectively
348
+ - updated presets `WebpHigh`, `JpegMedium`, `JpegLow` and `PngMedium` in `image.presets`
349
+ - `save_image` moved from `image` to `image.utils`
350
+ - added `convert_image` `optimize_image` `resize_image` functions to `image` module
351
+
352
+ ## [1.3.3]
353
+
354
+ - added `YoutubeDownloader` to `download` to download YT videos using a capped nb of threads
355
+
356
+ ## [1.3.2]
357
+
358
+ - fixed rewriting of links with empty target
359
+ - added support for image optimization using `zimscraperlib.image.optimization` for webp, gif, jpeg and png formats
360
+ - added `format_for()` in `zimscraperlib.image.probing` to get PIL image format from the suffix
361
+
362
+ ## [1.3.1]
363
+
364
+ - replaced BeautifoulSoup parser in rewriting (`html.parser` –> `lxml`)
365
+
366
+ ## [1.3.0]
367
+
368
+ - detect mimetypes from filenames for all text files
369
+ - fixed non-filename based StaticArticle
370
+ - enable rewriting of links in poster attribute of audio element
371
+ - added find_language_in() and find_language_in_file() to get language from HTML content and HTML file respectively
372
+ - add a mime mapping to deal with inconsistencies in mimetypes detected by magic on different platforms
373
+ - convert_image signature changed:
374
+ - `target_format` positional argument removed. Replaced with optionnal `fmt` key of keyword arguments.
375
+ - `colorspace` optionnal positional argument removed. Replaced with optionnal `colorspace` key of keyword arguments.
376
+ - prevent rewriting of links with special schemes `mailto`, 'tel', etc. in HTML links rewriting
377
+ - replaced `imaging` module with exploded `image` module (`convertion`, `probing`, `transformation`)
378
+ - changed `create_favicon()` param names (`source_image` -> `src`, `dest_ico` -> `dst`)
379
+ - changed `save_image()` param names (`image` -> `src`)
380
+ - changed `get_colors()` param names (`image_path` -> `src`)
381
+ - changed `resize_image()` param names (`fpath` -> `src`)
382
+
383
+ ## [1.2.1]
384
+
385
+ - fixed URL rewriting when running from /
386
+ - added support for link rewriting in `<object>` element
387
+ - prevent from raising error if element doesn't have the attribute with url
388
+ - use non greedy match for CSS URL links (shortest string matching `url()` format)
389
+ - fix namespace of target only if link doesn't have a netloc
390
+
391
+ ## [1.2.0]
392
+
393
+ - added UTF8 to constants
394
+ - added mime_type discovery via magic (filesystem)
395
+ - Added types: mime types guessing from file names
396
+ - Revamped zim API
397
+ - Removed ZimInfo which role was tu hold metadata for zimwriterfs call
398
+ - Removed calling zimwriterfs binary but kept function name
399
+ - Added zim.filesystem: zimwriterfs-like creation from a build folder
400
+ - Added zim.creator: create files by manually adding each article
401
+ - Added zim.rewriting: tools to rewrite links/urls in HTML/CSS
402
+ - add timeout and retries to save_file() and make it return headers
403
+
404
+ ## [1.1.2]
405
+
406
+ - fixed `convert_image()` which tried to use a closed file
407
+
408
+ ## [1.1.1]
409
+
410
+ - exposed reencode, Config and get_media_info in zimscraperlib.video
411
+ - added save_image() and convert_image() in zimscraperlib.imaging
412
+ - added support for upscaling in resize_image() via allow_upscaling
413
+ - resize_image() now supports params given by user and preservs image colorspace
414
+ - fixed tests for zimscraperlib.imaging
415
+
416
+ ## [1.1.0]
417
+
418
+ - added video module with reencode, presets, config builder and video file probing
419
+ - `make_zim_file()` accepts extra kwargs for zimwriterfs
420
+
421
+ ## [1.0.6]
422
+
423
+ - added translation support to i18n
424
+
425
+ ## [1.0.5]
426
+
427
+ - added s3transfer to verbose dependencies list
428
+ - changed default log format to include module name
429
+
430
+ ## [1.0.4]
431
+
432
+ - verbose dependencies (urllib3, boto3) now logged at WARNING level by default
433
+ - ability to set verbose dependencies log level and add modules to the list
434
+ - zimscraperlib's logging level now aligned with scraper's requested one
435
+
436
+ ## [1.0.3]
437
+
438
+ - fix_ogvjs_dist script more generic (#1)
439
+ - updated zim to support other zimwriterfs params (#10)
440
+ - more flexible requirements for requests dependency
441
+
442
+ ## [1.0.2]
443
+
444
+ - fixed return value of `get_language_details` on non-existent language
445
+ - fixed crash on `resize_image` with method `height`
446
+ - fixed root logger level (now DEBUG)
447
+ - removed useless `console=True` `getLogger` param
448
+ - completed tests (100% coverage)
449
+ - added `./test` script for quick local testing
450
+ - improved tox.ini
451
+ - added `create_favicon` to generate a squared favicon
452
+ - added `handle_user_provided_file` to handle user file/URL from param
453
+
454
+ ## [1.0.1]
455
+
456
+ - fixed fix_ogvjs_dist
457
+
458
+ ## [1.0.0]
459
+
460
+ - initial version providing
461
+ - download: save_file, save_large_file
462
+ - fix_ogvjs_dist
463
+ - i18n: setlocale, get_language_details
464
+ - imaging: get_colors, resize_image, is_hex_color
465
+ - zim: ZimInfo, make_zim_file
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: zimscraperlib
3
- Version: 3.3.2
3
+ Version: 4.0.0
4
4
  Summary: Collection of python tools to re-use common code across scrapers
5
5
  Project-URL: Donate, https://www.kiwix.org/en/support-us/
6
6
  Project-URL: Homepage, https://github.com/openzim/python-scraperlib
@@ -8,6 +8,8 @@ Author-email: openZIM <dev@openzim.org>
8
8
  License: GPL-3.0-or-later
9
9
  License-File: LICENSE
10
10
  Keywords: offline,openzim,zim
11
+ Classifier: Development Status :: 5 - Production/Stable
12
+ Classifier: Intended Audience :: Developers
11
13
  Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
12
14
  Classifier: Programming Language :: Python :: 3
13
15
  Classifier: Programming Language :: Python :: 3.8
@@ -18,33 +20,37 @@ Classifier: Programming Language :: Python :: 3.12
18
20
  Requires-Python: <3.13,>=3.8
19
21
  Requires-Dist: babel<3.0,>=2.9
20
22
  Requires-Dist: beautifulsoup4<5.0,>=4.9.3
23
+ Requires-Dist: cairosvg<3.0,>=2.2.0
21
24
  Requires-Dist: colorthief==0.2.1
22
25
  Requires-Dist: iso639-lang<3.0,>=2.2.3
23
26
  Requires-Dist: libzim<4.0,>=3.4.0
24
27
  Requires-Dist: lxml<6.0,>=4.6.3
25
28
  Requires-Dist: optimize-images<2.0,>=1.3.6
29
+ Requires-Dist: pymupdf<2.0,>=1.24.0
26
30
  Requires-Dist: python-magic<0.5,>=0.4.3
27
31
  Requires-Dist: python-resize-image<1.2,>=1.1.19
32
+ Requires-Dist: regex>=2020.7.14
28
33
  Requires-Dist: requests<3.0,>=2.25.1
29
34
  Requires-Dist: yt-dlp
30
35
  Provides-Extra: check
31
- Requires-Dist: pyright==1.1.354; extra == 'check'
36
+ Requires-Dist: pyright==1.1.368; extra == 'check'
37
+ Requires-Dist: pytest==8.2.2; extra == 'check'
32
38
  Provides-Extra: dev
33
- Requires-Dist: debugpy==1.8.1; extra == 'dev'
34
- Requires-Dist: pre-commit==3.6.2; extra == 'dev'
39
+ Requires-Dist: ipython==8.25.0; extra == 'dev'
40
+ Requires-Dist: pre-commit==3.7.1; extra == 'dev'
35
41
  Requires-Dist: zimscraperlib[check]; extra == 'dev'
36
42
  Requires-Dist: zimscraperlib[lint]; extra == 'dev'
37
43
  Requires-Dist: zimscraperlib[scripts]; extra == 'dev'
38
44
  Requires-Dist: zimscraperlib[test]; extra == 'dev'
39
45
  Provides-Extra: lint
40
- Requires-Dist: black==24.3.0; extra == 'lint'
41
- Requires-Dist: ruff==0.3.3; extra == 'lint'
46
+ Requires-Dist: black==24.4.2; extra == 'lint'
47
+ Requires-Dist: ruff==0.4.9; extra == 'lint'
42
48
  Provides-Extra: scripts
43
49
  Requires-Dist: invoke==2.2.0; extra == 'scripts'
44
50
  Provides-Extra: test
45
- Requires-Dist: coverage==7.4.4; extra == 'test'
46
- Requires-Dist: pytest-mock==3.12.0; extra == 'test'
47
- Requires-Dist: pytest==8.1.1; extra == 'test'
51
+ Requires-Dist: coverage==7.5.3; extra == 'test'
52
+ Requires-Dist: pytest-mock==3.14.0; extra == 'test'
53
+ Requires-Dist: pytest==8.2.2; extra == 'test'
48
54
  Description-Content-Type: text/markdown
49
55
 
50
56
  zimscraperlib
@@ -100,11 +106,11 @@ sudo apt install libmagic1 wget ffmpeg \
100
106
  apk add ffmpeg gifsicle libmagic wget libjpeg
101
107
  ```
102
108
 
103
- **Nota:** i18n features do not work on Alpine, see https://github.com/openzim/python-scraperlib/issues/134 ; there is one corresponding test which is failing.
104
-
105
109
  # Contribution
106
110
 
107
- This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing)
111
+ This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing).
112
+
113
+ This project has implemented openZIM's [Python bootstrap, conventions and policies](https://github.com/openzim/_python-bootstrap/docs/Policy.md) **v1.0.2**.
108
114
 
109
115
  ```shell
110
116
  pip install hatch
@@ -51,11 +51,11 @@ sudo apt install libmagic1 wget ffmpeg \
51
51
  apk add ffmpeg gifsicle libmagic wget libjpeg
52
52
  ```
53
53
 
54
- **Nota:** i18n features do not work on Alpine, see https://github.com/openzim/python-scraperlib/issues/134 ; there is one corresponding test which is failing.
55
-
56
54
  # Contribution
57
55
 
58
- This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing)
56
+ This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing).
57
+
58
+ This project has implemented openZIM's [Python bootstrap, conventions and policies](https://github.com/openzim/_python-bootstrap/docs/Policy.md) **v1.0.2**.
59
59
 
60
60
  ```shell
61
61
  pip install hatch
@@ -1,5 +1,5 @@
1
1
  [build-system]
2
- requires = ["hatchling", "hatch-openzim"]
2
+ requires = ["hatchling", "hatch-openzim>=0.2"]
3
3
  build-backend = "hatchling.build"
4
4
 
5
5
  [project]
@@ -18,6 +18,12 @@ dependencies = [
18
18
  "beautifulsoup4>=4.9.3,<5.0",
19
19
  "lxml>=4.6.3,<6.0",
20
20
  "optimize-images>=1.3.6,<2.0",
21
+ # regex has no upper-bound due to "date-based" release numbers, no semver, so their
22
+ # promise is that they will never (or always) break the API, and the API is very
23
+ # limited and we use only a very small subset of it.
24
+ "regex>=2020.7.14",
25
+ "pymupdf>=1.24.0,<2.0",
26
+ "CairoSVG>=2.2.0,<3.0",
21
27
  # youtube-dl should be updated as frequently as possible
22
28
  "yt-dlp"
23
29
  ]
@@ -25,31 +31,31 @@ dynamic = ["authors", "classifiers", "keywords", "license", "version", "urls"]
25
31
 
26
32
  [tool.hatch.metadata.hooks.openzim-metadata]
27
33
  kind = "scraper"
28
- # not yet supported in hatch-openzim 0.1
29
- # additional-classifiers = [
30
- # "Development Status :: 5 - Production/Stable",
31
- # "Intended Audience :: Developers",
32
- # ]
34
+ additional-classifiers = [
35
+ "Development Status :: 5 - Production/Stable",
36
+ "Intended Audience :: Developers",
37
+ ]
33
38
 
34
39
  [project.optional-dependencies]
35
40
  scripts = [
36
41
  "invoke==2.2.0",
37
42
  ]
38
43
  lint = [
39
- "black==24.3.0",
40
- "ruff==0.3.3",
44
+ "black==24.4.2",
45
+ "ruff==0.4.9",
41
46
  ]
42
47
  check = [
43
- "pyright==1.1.354",
48
+ "pyright==1.1.368",
49
+ "pytest==8.2.2",
44
50
  ]
45
51
  test = [
46
- "pytest==8.1.1",
47
- "pytest-mock==3.12.0",
48
- "coverage==7.4.4",
52
+ "pytest==8.2.2",
53
+ "pytest-mock==3.14.0",
54
+ "coverage==7.5.3",
49
55
  ]
50
56
  dev = [
51
- "pre-commit==3.6.2",
52
- "debugpy==1.8.1",
57
+ "ipython==8.25.0",
58
+ "pre-commit==3.7.1",
53
59
  "zimscraperlib[scripts]",
54
60
  "zimscraperlib[lint]",
55
61
  "zimscraperlib[test]",
@@ -0,0 +1 @@
1
+ __version__ = "4.0.0"