zimscraperlib 3.2.0__tar.gz → 3.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. zimscraperlib-3.3.0/.gitignore +254 -0
  2. zimscraperlib-3.3.0/.pre-commit-config.yaml +27 -0
  3. {zimscraperlib-3.2.0 → zimscraperlib-3.3.0}/CHANGELOG.md +21 -0
  4. {zimscraperlib-3.2.0/src/zimscraperlib.egg-info → zimscraperlib-3.3.0}/PKG-INFO +60 -35
  5. {zimscraperlib-3.2.0 → zimscraperlib-3.3.0}/README.md +23 -14
  6. zimscraperlib-3.3.0/contrib/README.md +3 -0
  7. zimscraperlib-3.3.0/contrib/encode_video.py +42 -0
  8. zimscraperlib-3.3.0/pyproject.toml +243 -0
  9. zimscraperlib-3.3.0/src/zimscraperlib/__about__.py +1 -0
  10. {zimscraperlib-3.2.0 → zimscraperlib-3.3.0}/src/zimscraperlib/__init__.py +2 -3
  11. {zimscraperlib-3.2.0 → zimscraperlib-3.3.0}/src/zimscraperlib/constants.py +3 -4
  12. {zimscraperlib-3.2.0 → zimscraperlib-3.3.0}/src/zimscraperlib/download.py +24 -19
  13. {zimscraperlib-3.2.0 → zimscraperlib-3.3.0}/src/zimscraperlib/filesystem.py +6 -3
  14. {zimscraperlib-3.2.0 → zimscraperlib-3.3.0}/src/zimscraperlib/fix_ogvjs_dist.py +10 -10
  15. {zimscraperlib-3.2.0 → zimscraperlib-3.3.0}/src/zimscraperlib/html.py +7 -8
  16. {zimscraperlib-3.2.0 → zimscraperlib-3.3.0}/src/zimscraperlib/i18n.py +48 -24
  17. {zimscraperlib-3.2.0 → zimscraperlib-3.3.0}/src/zimscraperlib/image/convertion.py +16 -12
  18. {zimscraperlib-3.2.0 → zimscraperlib-3.3.0}/src/zimscraperlib/image/optimization.py +50 -49
  19. {zimscraperlib-3.2.0 → zimscraperlib-3.3.0}/src/zimscraperlib/image/presets.py +14 -13
  20. {zimscraperlib-3.2.0 → zimscraperlib-3.3.0}/src/zimscraperlib/image/probing.py +15 -9
  21. {zimscraperlib-3.2.0 → zimscraperlib-3.3.0}/src/zimscraperlib/image/transformation.py +17 -9
  22. zimscraperlib-3.3.0/src/zimscraperlib/image/utils.py +20 -0
  23. {zimscraperlib-3.2.0 → zimscraperlib-3.3.0}/src/zimscraperlib/inputs.py +10 -7
  24. {zimscraperlib-3.2.0 → zimscraperlib-3.3.0}/src/zimscraperlib/logging.py +16 -15
  25. {zimscraperlib-3.2.0 → zimscraperlib-3.3.0}/src/zimscraperlib/misc.py +2 -2
  26. {zimscraperlib-3.2.0 → zimscraperlib-3.3.0}/src/zimscraperlib/types.py +4 -3
  27. {zimscraperlib-3.2.0 → zimscraperlib-3.3.0}/src/zimscraperlib/uri.py +20 -20
  28. {zimscraperlib-3.2.0 → zimscraperlib-3.3.0}/src/zimscraperlib/video/config.py +10 -7
  29. zimscraperlib-3.3.0/src/zimscraperlib/video/encoding.py +86 -0
  30. {zimscraperlib-3.2.0 → zimscraperlib-3.3.0}/src/zimscraperlib/video/presets.py +14 -16
  31. {zimscraperlib-3.2.0 → zimscraperlib-3.3.0}/src/zimscraperlib/video/probing.py +3 -4
  32. {zimscraperlib-3.2.0 → zimscraperlib-3.3.0}/src/zimscraperlib/zim/__init__.py +11 -7
  33. {zimscraperlib-3.2.0 → zimscraperlib-3.3.0}/src/zimscraperlib/zim/_libkiwix.py +15 -12
  34. {zimscraperlib-3.2.0 → zimscraperlib-3.3.0}/src/zimscraperlib/zim/archive.py +7 -8
  35. {zimscraperlib-3.2.0 → zimscraperlib-3.3.0}/src/zimscraperlib/zim/creator.py +85 -57
  36. {zimscraperlib-3.2.0 → zimscraperlib-3.3.0}/src/zimscraperlib/zim/filesystem.py +25 -24
  37. {zimscraperlib-3.2.0 → zimscraperlib-3.3.0}/src/zimscraperlib/zim/items.py +76 -20
  38. {zimscraperlib-3.2.0 → zimscraperlib-3.3.0}/src/zimscraperlib/zim/metadata.py +12 -7
  39. {zimscraperlib-3.2.0 → zimscraperlib-3.3.0}/src/zimscraperlib/zim/providers.py +10 -9
  40. zimscraperlib-3.3.0/tasks.py +109 -0
  41. zimscraperlib-3.3.0/tests/conftest.py +170 -0
  42. zimscraperlib-3.3.0/tests/download/test_download.py +283 -0
  43. zimscraperlib-3.3.0/tests/files/DroidSans.ttf +0 -0
  44. zimscraperlib-3.3.0/tests/files/audio.mp3 +0 -0
  45. zimscraperlib-3.3.0/tests/files/blue.jpg +0 -0
  46. zimscraperlib-3.3.0/tests/files/commons.png +0 -0
  47. zimscraperlib-3.3.0/tests/files/commons48.png +0 -0
  48. zimscraperlib-3.3.0/tests/files/mail.gif +0 -0
  49. zimscraperlib-3.3.0/tests/files/ninja.webp +0 -0
  50. zimscraperlib-3.3.0/tests/files/pluto.jpg +0 -0
  51. zimscraperlib-3.3.0/tests/files/square.jpg +0 -0
  52. zimscraperlib-3.3.0/tests/files/square.png +0 -0
  53. zimscraperlib-3.3.0/tests/files/star.svg +1 -0
  54. zimscraperlib-3.3.0/tests/files/video.mkv +0 -0
  55. zimscraperlib-3.3.0/tests/files/video.mp4 +0 -0
  56. zimscraperlib-3.3.0/tests/files/video.webm +0 -0
  57. zimscraperlib-3.3.0/tests/filesystem/test_filesystem.py +71 -0
  58. zimscraperlib-3.3.0/tests/html/conftest.py +25 -0
  59. zimscraperlib-3.3.0/tests/html/test_html.py +93 -0
  60. zimscraperlib-3.3.0/tests/i18n/locale/fr/LC_MESSAGES/messages.po +23 -0
  61. zimscraperlib-3.3.0/tests/i18n/locale/pt/LC_MESSAGES/messages.po +24 -0
  62. zimscraperlib-3.3.0/tests/i18n/test_i18n.py +216 -0
  63. zimscraperlib-3.3.0/tests/image/test_image.py +618 -0
  64. zimscraperlib-3.3.0/tests/inputs/test_inputs.py +246 -0
  65. zimscraperlib-3.3.0/tests/logging/conftest.py +17 -0
  66. zimscraperlib-3.3.0/tests/logging/test_logging.py +155 -0
  67. zimscraperlib-3.3.0/tests/misc/test_misc.py +16 -0
  68. zimscraperlib-3.3.0/tests/ogvjs/conftest.py +22 -0
  69. zimscraperlib-3.3.0/tests/ogvjs/test_ogvjs.py +92 -0
  70. zimscraperlib-3.3.0/tests/types/test_types.py +55 -0
  71. zimscraperlib-3.3.0/tests/uri/test_uri.py +76 -0
  72. zimscraperlib-3.3.0/tests/video/conftest.py +17 -0
  73. zimscraperlib-3.3.0/tests/video/test_encoding.py +94 -0
  74. zimscraperlib-3.3.0/tests/video/test_video.py +469 -0
  75. zimscraperlib-3.3.0/tests/zim/conftest.py +103 -0
  76. zimscraperlib-3.3.0/tests/zim/test_archive.py +146 -0
  77. zimscraperlib-3.3.0/tests/zim/test_fs.py +145 -0
  78. zimscraperlib-3.3.0/tests/zim/test_libkiwix.py +83 -0
  79. zimscraperlib-3.3.0/tests/zim/test_metadata.py +32 -0
  80. zimscraperlib-3.3.0/tests/zim/test_zim_creator.py +712 -0
  81. zimscraperlib-3.2.0/MANIFEST.in +0 -4
  82. zimscraperlib-3.2.0/PKG-INFO +0 -106
  83. zimscraperlib-3.2.0/requirements.txt +0 -12
  84. zimscraperlib-3.2.0/setup.cfg +0 -4
  85. zimscraperlib-3.2.0/setup.py +0 -53
  86. zimscraperlib-3.2.0/src/zimscraperlib/VERSION +0 -1
  87. zimscraperlib-3.2.0/src/zimscraperlib/image/utils.py +0 -17
  88. zimscraperlib-3.2.0/src/zimscraperlib/video/encoding.py +0 -54
  89. zimscraperlib-3.2.0/src/zimscraperlib.egg-info/SOURCES.txt +0 -46
  90. zimscraperlib-3.2.0/src/zimscraperlib.egg-info/dependency_links.txt +0 -1
  91. zimscraperlib-3.2.0/src/zimscraperlib.egg-info/entry_points.txt +0 -2
  92. zimscraperlib-3.2.0/src/zimscraperlib.egg-info/not-zip-safe +0 -1
  93. zimscraperlib-3.2.0/src/zimscraperlib.egg-info/requires.txt +0 -11
  94. zimscraperlib-3.2.0/src/zimscraperlib.egg-info/top_level.txt +0 -1
  95. {zimscraperlib-3.2.0 → zimscraperlib-3.3.0}/LICENSE +0 -0
  96. {zimscraperlib-3.2.0 → zimscraperlib-3.3.0}/src/zimscraperlib/image/__init__.py +0 -0
  97. {zimscraperlib-3.2.0 → zimscraperlib-3.3.0}/src/zimscraperlib/video/__init__.py +0 -0
@@ -0,0 +1,254 @@
1
+ # Created by https://www.toptal.com/developers/gitignore/api/python,macos,windows,linux
2
+ # Edit at https://www.toptal.com/developers/gitignore?templates=python,macos,windows,linux
3
+
4
+ ### Linux ###
5
+ *~
6
+
7
+ # temporary files which can be created if a process still has a handle open of a deleted file
8
+ .fuse_hidden*
9
+
10
+ # KDE directory preferences
11
+ .directory
12
+
13
+ # Linux trash folder which might appear on any partition or disk
14
+ .Trash-*
15
+
16
+ # .nfs files are created when an open file is removed but is still being accessed
17
+ .nfs*
18
+
19
+ ### macOS ###
20
+ # General
21
+ .DS_Store
22
+ .AppleDouble
23
+ .LSOverride
24
+
25
+ # Icon must end with two \r
26
+ Icon
27
+
28
+
29
+ # Thumbnails
30
+ ._*
31
+
32
+ # Files that might appear in the root of a volume
33
+ .DocumentRevisions-V100
34
+ .fseventsd
35
+ .Spotlight-V100
36
+ .TemporaryItems
37
+ .Trashes
38
+ .VolumeIcon.icns
39
+ .com.apple.timemachine.donotpresent
40
+
41
+ # Directories potentially created on remote AFP share
42
+ .AppleDB
43
+ .AppleDesktop
44
+ Network Trash Folder
45
+ Temporary Items
46
+ .apdisk
47
+
48
+ ### macOS Patch ###
49
+ # iCloud generated files
50
+ *.icloud
51
+
52
+ ### Python ###
53
+ # Byte-compiled / optimized / DLL files
54
+ __pycache__/
55
+ *.py[cod]
56
+ *$py.class
57
+
58
+ # C extensions
59
+ *.so
60
+
61
+ # Distribution / packaging
62
+ .Python
63
+ build/
64
+ develop-eggs/
65
+ dist/
66
+ downloads/
67
+ eggs/
68
+ .eggs/
69
+ lib/
70
+ lib64/
71
+ parts/
72
+ sdist/
73
+ var/
74
+ wheels/
75
+ share/python-wheels/
76
+ *.egg-info/
77
+ .installed.cfg
78
+ *.egg
79
+ MANIFEST
80
+
81
+ # PyInstaller
82
+ # Usually these files are written by a python script from a template
83
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
84
+ *.manifest
85
+ *.spec
86
+
87
+ # Installer logs
88
+ pip-log.txt
89
+ pip-delete-this-directory.txt
90
+
91
+ # Unit test / coverage reports
92
+ htmlcov/
93
+ .tox/
94
+ .nox/
95
+ .coverage
96
+ .coverage.*
97
+ .cache
98
+ nosetests.xml
99
+ coverage.xml
100
+ *.cover
101
+ *.py,cover
102
+ .hypothesis/
103
+ .pytest_cache/
104
+ cover/
105
+
106
+ # Translations
107
+ *.mo
108
+ *.pot
109
+
110
+ # Django stuff:
111
+ *.log
112
+ local_settings.py
113
+ db.sqlite3
114
+ db.sqlite3-journal
115
+
116
+ # Flask stuff:
117
+ instance/
118
+ .webassets-cache
119
+
120
+ # Scrapy stuff:
121
+ .scrapy
122
+
123
+ # Sphinx documentation
124
+ docs/_build/
125
+
126
+ # PyBuilder
127
+ .pybuilder/
128
+ target/
129
+
130
+ # Jupyter Notebook
131
+ .ipynb_checkpoints
132
+
133
+ # IPython
134
+ profile_default/
135
+ ipython_config.py
136
+
137
+ # pyenv
138
+ # For a library or package, you might want to ignore these files since the code is
139
+ # intended to run in multiple environments; otherwise, check them in:
140
+ # .python-version
141
+
142
+ # pipenv
143
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
144
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
145
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
146
+ # install all needed dependencies.
147
+ #Pipfile.lock
148
+
149
+ # poetry
150
+ # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
151
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
152
+ # commonly ignored for libraries.
153
+ # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
154
+ #poetry.lock
155
+
156
+ # pdm
157
+ # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
158
+ #pdm.lock
159
+ # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
160
+ # in version control.
161
+ # https://pdm.fming.dev/#use-with-ide
162
+ .pdm.toml
163
+
164
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
165
+ __pypackages__/
166
+
167
+ # Celery stuff
168
+ celerybeat-schedule
169
+ celerybeat.pid
170
+
171
+ # SageMath parsed files
172
+ *.sage.py
173
+
174
+ # Environments
175
+ .env
176
+ .venv
177
+ env/
178
+ venv/
179
+ ENV/
180
+ env.bak/
181
+ venv.bak/
182
+
183
+ # Spyder project settings
184
+ .spyderproject
185
+ .spyproject
186
+
187
+ # Rope project settings
188
+ .ropeproject
189
+
190
+ # mkdocs documentation
191
+ /site
192
+
193
+ # mypy
194
+ .mypy_cache/
195
+ .dmypy.json
196
+ dmypy.json
197
+
198
+ # Pyre type checker
199
+ .pyre/
200
+
201
+ # pytype static type analyzer
202
+ .pytype/
203
+
204
+ # Cython debug symbols
205
+ cython_debug/
206
+
207
+ # PyCharm
208
+ # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
209
+ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
210
+ # and can be added to the global gitignore or merged into this file. For a more nuclear
211
+ # option (not recommended) you can uncomment the following to ignore the entire idea folder.
212
+ #.idea/
213
+
214
+ ### Python Patch ###
215
+ # Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration
216
+ poetry.toml
217
+
218
+ # ruff
219
+ .ruff_cache/
220
+
221
+ # LSP config files
222
+ pyrightconfig.json
223
+
224
+ ### Windows ###
225
+ # Windows thumbnail cache files
226
+ Thumbs.db
227
+ Thumbs.db:encryptable
228
+ ehthumbs.db
229
+ ehthumbs_vista.db
230
+
231
+ # Dump file
232
+ *.stackdump
233
+
234
+ # Folder config file
235
+ [Dd]esktop.ini
236
+
237
+ # Recycle Bin used on file shares
238
+ $RECYCLE.BIN/
239
+
240
+ # Windows Installer files
241
+ *.cab
242
+ *.msi
243
+ *.msix
244
+ *.msm
245
+ *.msp
246
+
247
+ # Windows shortcuts
248
+ *.lnk
249
+
250
+ # End of https://www.toptal.com/developers/gitignore/api/python,macos,windows,linux
251
+
252
+ # ignore all vscode, this is not standard configuration in this place
253
+ .vscode
254
+ src/libzim-stubs
@@ -0,0 +1,27 @@
1
+ # See https://pre-commit.com for more information
2
+ # See https://pre-commit.com/hooks.html for more hooks
3
+ repos:
4
+ - repo: https://github.com/pre-commit/pre-commit-hooks
5
+ rev: v4.5.0
6
+ hooks:
7
+ - id: trailing-whitespace
8
+ - id: end-of-file-fixer
9
+ - repo: https://github.com/psf/black
10
+ rev: "24.1.1"
11
+ hooks:
12
+ - id: black
13
+ - repo: https://github.com/astral-sh/ruff-pre-commit
14
+ rev: v0.2.1
15
+ hooks:
16
+ - id: ruff
17
+ - repo: https://github.com/RobertCraigie/pyright-python
18
+ rev: v1.1.350
19
+ hooks:
20
+ - id: pyright
21
+ name: pyright (system)
22
+ description: 'pyright static type checker'
23
+ entry: pyright
24
+ language: system
25
+ 'types_or': [python, pyi]
26
+ require_serial: true
27
+ minimum_pre_commit_version: '2.9.2'
@@ -5,6 +5,27 @@ All notable changes to this project are documented in this file.
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0).
7
7
 
8
+ ## [3.3.0] - 2024-02-14
9
+
10
+ ### Added
11
+
12
+ - New `disable_metadata_checks` parameter in `zimscraperlib.zim.creator.Creator` initializer, allowing to disable metadata check at startup (assuming the user will validate them on its own) #119
13
+
14
+ ### Changed
15
+
16
+ - Rework the **VideoWebmLow** preset for faster encoding and smaller file size #122
17
+ - preset has been bumped to **version 2**
18
+ - when using an S3 cache, all videos using this preset will be reencoded and uploaded to cache again (it will replace the same file encoded with preset version 1)
19
+ - When reencoding a video, ffmpeg now uses only 1 CPU thread by default (new arg to `reencode` allows to override this default value)
20
+ - Using openZIM Python bootstrap conventions (including hatch-openzim plugin) #120
21
+ - Add support for Python 3.12, drop Python 3.7 support #118
22
+ - Replace "iso-369" by "iso639-lang" library
23
+ - Replace "file-magic" by "python-magic" library for Alpine Linux support and better maintenance
24
+
25
+ ## Fixed
26
+
27
+ - Fixed type hints of `zimscraperlib.zim.Item` and subclasses, and `zimscraperlib.image.optimization:convert_image`
28
+
8
29
  ## [3.2.0] - 2023-12-16
9
30
 
10
31
  ### Added
@@ -1,35 +1,51 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: zimscraperlib
3
- Version: 3.2.0
3
+ Version: 3.3.0
4
4
  Summary: Collection of python tools to re-use common code across scrapers
5
- Home-page: https://github.com/openzim/python_scraperlib
6
- Author: kiwix
7
- Author-email: reg@kiwix.org
8
- License: GPLv3+
9
- Keywords: kiwix zim offline
10
- Classifier: Development Status :: 5 - Production/Stable
11
- Classifier: Intended Audience :: Developers
12
- Classifier: Programming Language :: Python
13
- Classifier: Programming Language :: Python :: 3.7
5
+ Project-URL: Donate, https://www.kiwix.org/en/support-us/
6
+ Project-URL: Homepage, https://github.com/openzim/python-scraperlib
7
+ Author-email: openZIM <dev@openzim.org>
8
+ License: GPL-3.0-or-later
9
+ License-File: LICENSE
10
+ Keywords: offline,openzim,zim
11
+ Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
12
+ Classifier: Programming Language :: Python :: 3
14
13
  Classifier: Programming Language :: Python :: 3.8
15
14
  Classifier: Programming Language :: Python :: 3.9
16
15
  Classifier: Programming Language :: Python :: 3.10
17
16
  Classifier: Programming Language :: Python :: 3.11
18
- Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
19
- Requires-Python: >=3.7
20
- Description-Content-Type: text/markdown
21
- License-File: LICENSE
22
- Requires-Dist: iso-639==0.4.5
23
- Requires-Dist: requests<3.0,>=2.25.1
24
- Requires-Dist: colorthief==0.2.1
25
- Requires-Dist: python-resize-image<1.2,>=1.1.19
26
- Requires-Dist: Babel<3.0,>=2.9
27
- Requires-Dist: file-magic<0.5,>=0.4.0
28
- Requires-Dist: libzim<3.5,>=3.4.0
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Requires-Python: <3.13,>=3.8
19
+ Requires-Dist: babel<3.0,>=2.9
29
20
  Requires-Dist: beautifulsoup4<4.10,>=4.9.3
21
+ Requires-Dist: colorthief==0.2.1
22
+ Requires-Dist: iso639-lang<3.0,>=2.2.3
23
+ Requires-Dist: libzim<4.0,>=3.4.0
30
24
  Requires-Dist: lxml<4.10,>=4.6.3
31
25
  Requires-Dist: optimize-images<1.6,>=1.3.6
26
+ Requires-Dist: python-magic<0.5,>=0.4.3
27
+ Requires-Dist: python-resize-image<1.2,>=1.1.19
28
+ Requires-Dist: requests<3.0,>=2.25.1
32
29
  Requires-Dist: yt-dlp
30
+ Provides-Extra: check
31
+ Requires-Dist: pyright==1.1.350; extra == 'check'
32
+ Provides-Extra: dev
33
+ Requires-Dist: debugpy==1.8.0; extra == 'dev'
34
+ Requires-Dist: pre-commit==3.5.0; extra == 'dev'
35
+ Requires-Dist: zimscraperlib[check]; extra == 'dev'
36
+ Requires-Dist: zimscraperlib[lint]; extra == 'dev'
37
+ Requires-Dist: zimscraperlib[scripts]; extra == 'dev'
38
+ Requires-Dist: zimscraperlib[test]; extra == 'dev'
39
+ Provides-Extra: lint
40
+ Requires-Dist: black==24.1.1; extra == 'lint'
41
+ Requires-Dist: ruff==0.2.1; extra == 'lint'
42
+ Provides-Extra: scripts
43
+ Requires-Dist: invoke==2.2.0; extra == 'scripts'
44
+ Provides-Extra: test
45
+ Requires-Dist: coverage==7.4.1; extra == 'test'
46
+ Requires-Dist: pytest-mock==3.12.0; extra == 'test'
47
+ Requires-Dist: pytest==8.0.0; extra == 'test'
48
+ Description-Content-Type: text/markdown
33
49
 
34
50
  zimscraperlib
35
51
  =============
@@ -78,29 +94,38 @@ sudo apt install libmagic1 wget ffmpeg \
78
94
  libharfbuzz-dev libfribidi-dev libxcb1-dev gifsicle
79
95
  ```
80
96
 
97
+ ## Alpine
98
+ ```
99
+ apk add ffmpeg gifsicle libmagic wget libjpeg
100
+ ```
101
+
102
+ **Nota:** i18n features do not work on Alpine, see https://github.com/openzim/python-scraperlib/issues/134 ; there is one corresponding test which is failing.
103
+
81
104
  # Contribution
82
105
 
106
+ This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing)
107
+
83
108
  ```shell
84
- pip -r requirements.txt
85
- pip install tox pre-commit
109
+ pip install hatch
110
+ pip install ".[dev]"
86
111
  pre-commit install
87
112
  # For tests
88
- tox
113
+ invoke coverage
89
114
  ```
90
115
 
91
116
  # Users
92
117
 
93
118
  Non-exhaustive list of scrapers using it (check status when updating API):
94
119
 
95
- * [openzim/youtube](https://github.com/openzim/youtube)
120
+ * [openzim/freecodecamp](https://github.com/openzim/freecodecamp)
121
+ * [openzim/gutenberg](https://github.com/openzim/gutenberg)
122
+ * [openzim/ifixit](https://github.com/openzim/ifixit)
123
+ * [openzim/kolibri](https://github.com/openzim/kolibri)
96
124
  * [openzim/nautilus](https://github.com/openzim/nautilus)
97
-
98
- # releasing
99
-
100
- * Update your dependencies: `pip install -U setuptools wheel twine`
101
- * Make sure CHANGELOG.md is up-to-date
102
- * Bump version on `src/zimscraperlib/VERSION`
103
- * Build packages `python ./setup.py sdist bdist_wheel`
104
- * Upload to PyPI `twine upload dist/zimscraperlib-2.0.0*`.
105
- * Commit your Changelog + version bump changes
106
- * Tag version on git `git tag -a v2.0.0`
125
+ * [openzim/nautilus](https://github.com/openzim/nautilus)
126
+ * [openzim/openedx](https://github.com/openzim/openedx)
127
+ * [openzim/sotoki](https://github.com/openzim/sotoki)
128
+ * [openzim/ted](https://github.com/openzim/ted)
129
+ * [openzim/warc2zim](https://github.com/openzim/warc2zim)
130
+ * [openzim/wikihow](https://github.com/openzim/wikihow)
131
+ * [openzim/youtube](https://github.com/openzim/youtube)
@@ -45,29 +45,38 @@ sudo apt install libmagic1 wget ffmpeg \
45
45
  libharfbuzz-dev libfribidi-dev libxcb1-dev gifsicle
46
46
  ```
47
47
 
48
+ ## Alpine
49
+ ```
50
+ apk add ffmpeg gifsicle libmagic wget libjpeg
51
+ ```
52
+
53
+ **Nota:** i18n features do not work on Alpine, see https://github.com/openzim/python-scraperlib/issues/134 ; there is one corresponding test which is failing.
54
+
48
55
  # Contribution
49
56
 
57
+ This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing)
58
+
50
59
  ```shell
51
- pip -r requirements.txt
52
- pip install tox pre-commit
60
+ pip install hatch
61
+ pip install ".[dev]"
53
62
  pre-commit install
54
63
  # For tests
55
- tox
64
+ invoke coverage
56
65
  ```
57
66
 
58
67
  # Users
59
68
 
60
69
  Non-exhaustive list of scrapers using it (check status when updating API):
61
70
 
62
- * [openzim/youtube](https://github.com/openzim/youtube)
71
+ * [openzim/freecodecamp](https://github.com/openzim/freecodecamp)
72
+ * [openzim/gutenberg](https://github.com/openzim/gutenberg)
73
+ * [openzim/ifixit](https://github.com/openzim/ifixit)
74
+ * [openzim/kolibri](https://github.com/openzim/kolibri)
63
75
  * [openzim/nautilus](https://github.com/openzim/nautilus)
64
-
65
- # releasing
66
-
67
- * Update your dependencies: `pip install -U setuptools wheel twine`
68
- * Make sure CHANGELOG.md is up-to-date
69
- * Bump version on `src/zimscraperlib/VERSION`
70
- * Build packages `python ./setup.py sdist bdist_wheel`
71
- * Upload to PyPI `twine upload dist/zimscraperlib-2.0.0*`.
72
- * Commit your Changelog + version bump changes
73
- * Tag version on git `git tag -a v2.0.0`
76
+ * [openzim/nautilus](https://github.com/openzim/nautilus)
77
+ * [openzim/openedx](https://github.com/openzim/openedx)
78
+ * [openzim/sotoki](https://github.com/openzim/sotoki)
79
+ * [openzim/ted](https://github.com/openzim/ted)
80
+ * [openzim/warc2zim](https://github.com/openzim/warc2zim)
81
+ * [openzim/wikihow](https://github.com/openzim/wikihow)
82
+ * [openzim/youtube](https://github.com/openzim/youtube)
@@ -0,0 +1,3 @@
1
+
2
+ This folder contains some tooling around zimscraperlib:
3
+ - `encode_video.py`: a small utility to encode a video with an existing video preset, just like a scraper would do
@@ -0,0 +1,42 @@
1
+ import sys
2
+ from pathlib import Path
3
+ from typing import List
4
+
5
+ from zimscraperlib import logger
6
+ from zimscraperlib.video import presets, reencode
7
+
8
+
9
+ def encode_video(src_path: Path, dst_path: Path, preset: str):
10
+ if not src_path.exists():
11
+ raise ValueError(f"{src_path} does not exists")
12
+ try:
13
+ preset_cls = getattr(presets, preset)
14
+ except AttributeError:
15
+ logger.error(f"{preset} preset not found")
16
+ raise
17
+ logger.info(f"Encoding video {src_path} with {preset} version {preset_cls.VERSION}")
18
+ success, process = reencode(
19
+ src_path=src_path,
20
+ dst_path=dst_path,
21
+ ffmpeg_args=preset_cls().to_ffmpeg_args(),
22
+ with_process=True,
23
+ ) # pyright: ignore[reportGeneralTypeIssues] (returned type is variable, depending on `with_process` value)
24
+ if not success:
25
+ logger.error(f"conversion failed:\n{process.stdout}")
26
+
27
+
28
+ def run(args: List[str] = sys.argv):
29
+ if len(args) < 4: # noqa: PLR2004
30
+ print(f"Usage: {args[0]} <src_path> <dst_path> <preset>") # noqa: T201
31
+ print( # noqa: T201
32
+ "\t<src_path>\tpath to the video to encode."
33
+ "\t<dst_path>\tpath to the store the reencoded video."
34
+ "\t<preset>\tname of preset to use."
35
+ )
36
+ return 1
37
+ encode_video(Path(args[1]), Path(args[2]), args[3])
38
+ return 0
39
+
40
+
41
+ if __name__ == "__main__":
42
+ sys.exit(run())