zimscraperlib 3.1.1__tar.gz → 3.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- zimscraperlib-3.3.0/.gitignore +254 -0
- zimscraperlib-3.3.0/.pre-commit-config.yaml +27 -0
- {zimscraperlib-3.1.1 → zimscraperlib-3.3.0}/CHANGELOG.md +36 -1
- zimscraperlib-3.3.0/PKG-INFO +131 -0
- {zimscraperlib-3.1.1 → zimscraperlib-3.3.0}/README.md +23 -14
- zimscraperlib-3.3.0/contrib/README.md +3 -0
- zimscraperlib-3.3.0/contrib/encode_video.py +42 -0
- zimscraperlib-3.3.0/pyproject.toml +243 -0
- zimscraperlib-3.3.0/src/zimscraperlib/__about__.py +1 -0
- {zimscraperlib-3.1.1 → zimscraperlib-3.3.0}/src/zimscraperlib/__init__.py +2 -3
- {zimscraperlib-3.1.1 → zimscraperlib-3.3.0}/src/zimscraperlib/constants.py +4 -4
- {zimscraperlib-3.1.1 → zimscraperlib-3.3.0}/src/zimscraperlib/download.py +24 -19
- {zimscraperlib-3.1.1 → zimscraperlib-3.3.0}/src/zimscraperlib/filesystem.py +6 -3
- {zimscraperlib-3.1.1 → zimscraperlib-3.3.0}/src/zimscraperlib/fix_ogvjs_dist.py +10 -10
- {zimscraperlib-3.1.1 → zimscraperlib-3.3.0}/src/zimscraperlib/html.py +7 -8
- {zimscraperlib-3.1.1 → zimscraperlib-3.3.0}/src/zimscraperlib/i18n.py +48 -24
- {zimscraperlib-3.1.1 → zimscraperlib-3.3.0}/src/zimscraperlib/image/convertion.py +16 -12
- {zimscraperlib-3.1.1 → zimscraperlib-3.3.0}/src/zimscraperlib/image/optimization.py +50 -49
- {zimscraperlib-3.1.1 → zimscraperlib-3.3.0}/src/zimscraperlib/image/presets.py +14 -13
- {zimscraperlib-3.1.1 → zimscraperlib-3.3.0}/src/zimscraperlib/image/probing.py +15 -9
- {zimscraperlib-3.1.1 → zimscraperlib-3.3.0}/src/zimscraperlib/image/transformation.py +17 -9
- zimscraperlib-3.3.0/src/zimscraperlib/image/utils.py +20 -0
- zimscraperlib-3.3.0/src/zimscraperlib/inputs.py +109 -0
- {zimscraperlib-3.1.1 → zimscraperlib-3.3.0}/src/zimscraperlib/logging.py +16 -15
- {zimscraperlib-3.1.1 → zimscraperlib-3.3.0}/src/zimscraperlib/misc.py +2 -2
- {zimscraperlib-3.1.1 → zimscraperlib-3.3.0}/src/zimscraperlib/types.py +4 -3
- {zimscraperlib-3.1.1 → zimscraperlib-3.3.0}/src/zimscraperlib/uri.py +20 -20
- {zimscraperlib-3.1.1 → zimscraperlib-3.3.0}/src/zimscraperlib/video/config.py +10 -7
- zimscraperlib-3.3.0/src/zimscraperlib/video/encoding.py +86 -0
- {zimscraperlib-3.1.1 → zimscraperlib-3.3.0}/src/zimscraperlib/video/presets.py +14 -16
- {zimscraperlib-3.1.1 → zimscraperlib-3.3.0}/src/zimscraperlib/video/probing.py +3 -4
- {zimscraperlib-3.1.1 → zimscraperlib-3.3.0}/src/zimscraperlib/zim/__init__.py +11 -7
- {zimscraperlib-3.1.1 → zimscraperlib-3.3.0}/src/zimscraperlib/zim/_libkiwix.py +15 -12
- {zimscraperlib-3.1.1 → zimscraperlib-3.3.0}/src/zimscraperlib/zim/archive.py +7 -8
- {zimscraperlib-3.1.1 → zimscraperlib-3.3.0}/src/zimscraperlib/zim/creator.py +85 -57
- {zimscraperlib-3.1.1 → zimscraperlib-3.3.0}/src/zimscraperlib/zim/filesystem.py +25 -24
- {zimscraperlib-3.1.1 → zimscraperlib-3.3.0}/src/zimscraperlib/zim/items.py +76 -20
- {zimscraperlib-3.1.1 → zimscraperlib-3.3.0}/src/zimscraperlib/zim/metadata.py +14 -8
- {zimscraperlib-3.1.1 → zimscraperlib-3.3.0}/src/zimscraperlib/zim/providers.py +10 -9
- zimscraperlib-3.3.0/tasks.py +109 -0
- zimscraperlib-3.3.0/tests/conftest.py +170 -0
- zimscraperlib-3.3.0/tests/download/test_download.py +283 -0
- zimscraperlib-3.3.0/tests/files/DroidSans.ttf +0 -0
- zimscraperlib-3.3.0/tests/files/audio.mp3 +0 -0
- zimscraperlib-3.3.0/tests/files/blue.jpg +0 -0
- zimscraperlib-3.3.0/tests/files/commons.png +0 -0
- zimscraperlib-3.3.0/tests/files/commons48.png +0 -0
- zimscraperlib-3.3.0/tests/files/mail.gif +0 -0
- zimscraperlib-3.3.0/tests/files/ninja.webp +0 -0
- zimscraperlib-3.3.0/tests/files/pluto.jpg +0 -0
- zimscraperlib-3.3.0/tests/files/square.jpg +0 -0
- zimscraperlib-3.3.0/tests/files/square.png +0 -0
- zimscraperlib-3.3.0/tests/files/star.svg +1 -0
- zimscraperlib-3.3.0/tests/files/video.mkv +0 -0
- zimscraperlib-3.3.0/tests/files/video.mp4 +0 -0
- zimscraperlib-3.3.0/tests/files/video.webm +0 -0
- zimscraperlib-3.3.0/tests/filesystem/test_filesystem.py +71 -0
- zimscraperlib-3.3.0/tests/html/conftest.py +25 -0
- zimscraperlib-3.3.0/tests/html/test_html.py +93 -0
- zimscraperlib-3.3.0/tests/i18n/locale/fr/LC_MESSAGES/messages.po +23 -0
- zimscraperlib-3.3.0/tests/i18n/locale/pt/LC_MESSAGES/messages.po +24 -0
- zimscraperlib-3.3.0/tests/i18n/test_i18n.py +216 -0
- zimscraperlib-3.3.0/tests/image/test_image.py +618 -0
- zimscraperlib-3.3.0/tests/inputs/test_inputs.py +246 -0
- zimscraperlib-3.3.0/tests/logging/conftest.py +17 -0
- zimscraperlib-3.3.0/tests/logging/test_logging.py +155 -0
- zimscraperlib-3.3.0/tests/misc/test_misc.py +16 -0
- zimscraperlib-3.3.0/tests/ogvjs/conftest.py +22 -0
- zimscraperlib-3.3.0/tests/ogvjs/test_ogvjs.py +92 -0
- zimscraperlib-3.3.0/tests/types/test_types.py +55 -0
- zimscraperlib-3.3.0/tests/uri/test_uri.py +76 -0
- zimscraperlib-3.3.0/tests/video/conftest.py +17 -0
- zimscraperlib-3.3.0/tests/video/test_encoding.py +94 -0
- zimscraperlib-3.3.0/tests/video/test_video.py +469 -0
- zimscraperlib-3.3.0/tests/zim/conftest.py +103 -0
- zimscraperlib-3.3.0/tests/zim/test_archive.py +146 -0
- zimscraperlib-3.3.0/tests/zim/test_fs.py +145 -0
- zimscraperlib-3.3.0/tests/zim/test_libkiwix.py +83 -0
- zimscraperlib-3.3.0/tests/zim/test_metadata.py +32 -0
- zimscraperlib-3.3.0/tests/zim/test_zim_creator.py +712 -0
- zimscraperlib-3.1.1/MANIFEST.in +0 -4
- zimscraperlib-3.1.1/PKG-INFO +0 -95
- zimscraperlib-3.1.1/requirements.txt +0 -12
- zimscraperlib-3.1.1/setup.cfg +0 -4
- zimscraperlib-3.1.1/setup.py +0 -53
- zimscraperlib-3.1.1/src/zimscraperlib/VERSION +0 -1
- zimscraperlib-3.1.1/src/zimscraperlib/image/utils.py +0 -17
- zimscraperlib-3.1.1/src/zimscraperlib/inputs.py +0 -51
- zimscraperlib-3.1.1/src/zimscraperlib/video/encoding.py +0 -54
- zimscraperlib-3.1.1/src/zimscraperlib.egg-info/PKG-INFO +0 -95
- zimscraperlib-3.1.1/src/zimscraperlib.egg-info/SOURCES.txt +0 -46
- zimscraperlib-3.1.1/src/zimscraperlib.egg-info/dependency_links.txt +0 -1
- zimscraperlib-3.1.1/src/zimscraperlib.egg-info/entry_points.txt +0 -2
- zimscraperlib-3.1.1/src/zimscraperlib.egg-info/not-zip-safe +0 -1
- zimscraperlib-3.1.1/src/zimscraperlib.egg-info/requires.txt +0 -11
- zimscraperlib-3.1.1/src/zimscraperlib.egg-info/top_level.txt +0 -1
- {zimscraperlib-3.1.1 → zimscraperlib-3.3.0}/LICENSE +0 -0
- {zimscraperlib-3.1.1 → zimscraperlib-3.3.0}/src/zimscraperlib/image/__init__.py +0 -0
- {zimscraperlib-3.1.1 → zimscraperlib-3.3.0}/src/zimscraperlib/video/__init__.py +0 -0
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
# Created by https://www.toptal.com/developers/gitignore/api/python,macos,windows,linux
|
|
2
|
+
# Edit at https://www.toptal.com/developers/gitignore?templates=python,macos,windows,linux
|
|
3
|
+
|
|
4
|
+
### Linux ###
|
|
5
|
+
*~
|
|
6
|
+
|
|
7
|
+
# temporary files which can be created if a process still has a handle open of a deleted file
|
|
8
|
+
.fuse_hidden*
|
|
9
|
+
|
|
10
|
+
# KDE directory preferences
|
|
11
|
+
.directory
|
|
12
|
+
|
|
13
|
+
# Linux trash folder which might appear on any partition or disk
|
|
14
|
+
.Trash-*
|
|
15
|
+
|
|
16
|
+
# .nfs files are created when an open file is removed but is still being accessed
|
|
17
|
+
.nfs*
|
|
18
|
+
|
|
19
|
+
### macOS ###
|
|
20
|
+
# General
|
|
21
|
+
.DS_Store
|
|
22
|
+
.AppleDouble
|
|
23
|
+
.LSOverride
|
|
24
|
+
|
|
25
|
+
# Icon must end with two \r
|
|
26
|
+
Icon
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
# Thumbnails
|
|
30
|
+
._*
|
|
31
|
+
|
|
32
|
+
# Files that might appear in the root of a volume
|
|
33
|
+
.DocumentRevisions-V100
|
|
34
|
+
.fseventsd
|
|
35
|
+
.Spotlight-V100
|
|
36
|
+
.TemporaryItems
|
|
37
|
+
.Trashes
|
|
38
|
+
.VolumeIcon.icns
|
|
39
|
+
.com.apple.timemachine.donotpresent
|
|
40
|
+
|
|
41
|
+
# Directories potentially created on remote AFP share
|
|
42
|
+
.AppleDB
|
|
43
|
+
.AppleDesktop
|
|
44
|
+
Network Trash Folder
|
|
45
|
+
Temporary Items
|
|
46
|
+
.apdisk
|
|
47
|
+
|
|
48
|
+
### macOS Patch ###
|
|
49
|
+
# iCloud generated files
|
|
50
|
+
*.icloud
|
|
51
|
+
|
|
52
|
+
### Python ###
|
|
53
|
+
# Byte-compiled / optimized / DLL files
|
|
54
|
+
__pycache__/
|
|
55
|
+
*.py[cod]
|
|
56
|
+
*$py.class
|
|
57
|
+
|
|
58
|
+
# C extensions
|
|
59
|
+
*.so
|
|
60
|
+
|
|
61
|
+
# Distribution / packaging
|
|
62
|
+
.Python
|
|
63
|
+
build/
|
|
64
|
+
develop-eggs/
|
|
65
|
+
dist/
|
|
66
|
+
downloads/
|
|
67
|
+
eggs/
|
|
68
|
+
.eggs/
|
|
69
|
+
lib/
|
|
70
|
+
lib64/
|
|
71
|
+
parts/
|
|
72
|
+
sdist/
|
|
73
|
+
var/
|
|
74
|
+
wheels/
|
|
75
|
+
share/python-wheels/
|
|
76
|
+
*.egg-info/
|
|
77
|
+
.installed.cfg
|
|
78
|
+
*.egg
|
|
79
|
+
MANIFEST
|
|
80
|
+
|
|
81
|
+
# PyInstaller
|
|
82
|
+
# Usually these files are written by a python script from a template
|
|
83
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
84
|
+
*.manifest
|
|
85
|
+
*.spec
|
|
86
|
+
|
|
87
|
+
# Installer logs
|
|
88
|
+
pip-log.txt
|
|
89
|
+
pip-delete-this-directory.txt
|
|
90
|
+
|
|
91
|
+
# Unit test / coverage reports
|
|
92
|
+
htmlcov/
|
|
93
|
+
.tox/
|
|
94
|
+
.nox/
|
|
95
|
+
.coverage
|
|
96
|
+
.coverage.*
|
|
97
|
+
.cache
|
|
98
|
+
nosetests.xml
|
|
99
|
+
coverage.xml
|
|
100
|
+
*.cover
|
|
101
|
+
*.py,cover
|
|
102
|
+
.hypothesis/
|
|
103
|
+
.pytest_cache/
|
|
104
|
+
cover/
|
|
105
|
+
|
|
106
|
+
# Translations
|
|
107
|
+
*.mo
|
|
108
|
+
*.pot
|
|
109
|
+
|
|
110
|
+
# Django stuff:
|
|
111
|
+
*.log
|
|
112
|
+
local_settings.py
|
|
113
|
+
db.sqlite3
|
|
114
|
+
db.sqlite3-journal
|
|
115
|
+
|
|
116
|
+
# Flask stuff:
|
|
117
|
+
instance/
|
|
118
|
+
.webassets-cache
|
|
119
|
+
|
|
120
|
+
# Scrapy stuff:
|
|
121
|
+
.scrapy
|
|
122
|
+
|
|
123
|
+
# Sphinx documentation
|
|
124
|
+
docs/_build/
|
|
125
|
+
|
|
126
|
+
# PyBuilder
|
|
127
|
+
.pybuilder/
|
|
128
|
+
target/
|
|
129
|
+
|
|
130
|
+
# Jupyter Notebook
|
|
131
|
+
.ipynb_checkpoints
|
|
132
|
+
|
|
133
|
+
# IPython
|
|
134
|
+
profile_default/
|
|
135
|
+
ipython_config.py
|
|
136
|
+
|
|
137
|
+
# pyenv
|
|
138
|
+
# For a library or package, you might want to ignore these files since the code is
|
|
139
|
+
# intended to run in multiple environments; otherwise, check them in:
|
|
140
|
+
# .python-version
|
|
141
|
+
|
|
142
|
+
# pipenv
|
|
143
|
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
|
144
|
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
|
145
|
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
|
146
|
+
# install all needed dependencies.
|
|
147
|
+
#Pipfile.lock
|
|
148
|
+
|
|
149
|
+
# poetry
|
|
150
|
+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
|
151
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
152
|
+
# commonly ignored for libraries.
|
|
153
|
+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
|
154
|
+
#poetry.lock
|
|
155
|
+
|
|
156
|
+
# pdm
|
|
157
|
+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
|
158
|
+
#pdm.lock
|
|
159
|
+
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
|
160
|
+
# in version control.
|
|
161
|
+
# https://pdm.fming.dev/#use-with-ide
|
|
162
|
+
.pdm.toml
|
|
163
|
+
|
|
164
|
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
|
165
|
+
__pypackages__/
|
|
166
|
+
|
|
167
|
+
# Celery stuff
|
|
168
|
+
celerybeat-schedule
|
|
169
|
+
celerybeat.pid
|
|
170
|
+
|
|
171
|
+
# SageMath parsed files
|
|
172
|
+
*.sage.py
|
|
173
|
+
|
|
174
|
+
# Environments
|
|
175
|
+
.env
|
|
176
|
+
.venv
|
|
177
|
+
env/
|
|
178
|
+
venv/
|
|
179
|
+
ENV/
|
|
180
|
+
env.bak/
|
|
181
|
+
venv.bak/
|
|
182
|
+
|
|
183
|
+
# Spyder project settings
|
|
184
|
+
.spyderproject
|
|
185
|
+
.spyproject
|
|
186
|
+
|
|
187
|
+
# Rope project settings
|
|
188
|
+
.ropeproject
|
|
189
|
+
|
|
190
|
+
# mkdocs documentation
|
|
191
|
+
/site
|
|
192
|
+
|
|
193
|
+
# mypy
|
|
194
|
+
.mypy_cache/
|
|
195
|
+
.dmypy.json
|
|
196
|
+
dmypy.json
|
|
197
|
+
|
|
198
|
+
# Pyre type checker
|
|
199
|
+
.pyre/
|
|
200
|
+
|
|
201
|
+
# pytype static type analyzer
|
|
202
|
+
.pytype/
|
|
203
|
+
|
|
204
|
+
# Cython debug symbols
|
|
205
|
+
cython_debug/
|
|
206
|
+
|
|
207
|
+
# PyCharm
|
|
208
|
+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
|
209
|
+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
|
210
|
+
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
|
211
|
+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
|
212
|
+
#.idea/
|
|
213
|
+
|
|
214
|
+
### Python Patch ###
|
|
215
|
+
# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration
|
|
216
|
+
poetry.toml
|
|
217
|
+
|
|
218
|
+
# ruff
|
|
219
|
+
.ruff_cache/
|
|
220
|
+
|
|
221
|
+
# LSP config files
|
|
222
|
+
pyrightconfig.json
|
|
223
|
+
|
|
224
|
+
### Windows ###
|
|
225
|
+
# Windows thumbnail cache files
|
|
226
|
+
Thumbs.db
|
|
227
|
+
Thumbs.db:encryptable
|
|
228
|
+
ehthumbs.db
|
|
229
|
+
ehthumbs_vista.db
|
|
230
|
+
|
|
231
|
+
# Dump file
|
|
232
|
+
*.stackdump
|
|
233
|
+
|
|
234
|
+
# Folder config file
|
|
235
|
+
[Dd]esktop.ini
|
|
236
|
+
|
|
237
|
+
# Recycle Bin used on file shares
|
|
238
|
+
$RECYCLE.BIN/
|
|
239
|
+
|
|
240
|
+
# Windows Installer files
|
|
241
|
+
*.cab
|
|
242
|
+
*.msi
|
|
243
|
+
*.msix
|
|
244
|
+
*.msm
|
|
245
|
+
*.msp
|
|
246
|
+
|
|
247
|
+
# Windows shortcuts
|
|
248
|
+
*.lnk
|
|
249
|
+
|
|
250
|
+
# End of https://www.toptal.com/developers/gitignore/api/python,macos,windows,linux
|
|
251
|
+
|
|
252
|
+
# ignore all vscode, this is not standard configuration in this place
|
|
253
|
+
.vscode
|
|
254
|
+
src/libzim-stubs
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# See https://pre-commit.com for more information
|
|
2
|
+
# See https://pre-commit.com/hooks.html for more hooks
|
|
3
|
+
repos:
|
|
4
|
+
- repo: https://github.com/pre-commit/pre-commit-hooks
|
|
5
|
+
rev: v4.5.0
|
|
6
|
+
hooks:
|
|
7
|
+
- id: trailing-whitespace
|
|
8
|
+
- id: end-of-file-fixer
|
|
9
|
+
- repo: https://github.com/psf/black
|
|
10
|
+
rev: "24.1.1"
|
|
11
|
+
hooks:
|
|
12
|
+
- id: black
|
|
13
|
+
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
14
|
+
rev: v0.2.1
|
|
15
|
+
hooks:
|
|
16
|
+
- id: ruff
|
|
17
|
+
- repo: https://github.com/RobertCraigie/pyright-python
|
|
18
|
+
rev: v1.1.350
|
|
19
|
+
hooks:
|
|
20
|
+
- id: pyright
|
|
21
|
+
name: pyright (system)
|
|
22
|
+
description: 'pyright static type checker'
|
|
23
|
+
entry: pyright
|
|
24
|
+
language: system
|
|
25
|
+
'types_or': [python, pyi]
|
|
26
|
+
require_serial: true
|
|
27
|
+
minimum_pre_commit_version: '2.9.2'
|
|
@@ -5,7 +5,42 @@ All notable changes to this project are documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0).
|
|
7
7
|
|
|
8
|
-
## [3.
|
|
8
|
+
## [3.3.0] - 2024-02-14
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- New `disable_metadata_checks` parameter in `zimscraperlib.zim.creator.Creator` initializer, allowing to disable metadata check at startup (assuming the user will validate them on its own) #119
|
|
13
|
+
|
|
14
|
+
### Changed
|
|
15
|
+
|
|
16
|
+
- Rework the **VideoWebmLow** preset for faster encoding and smaller file size #122
|
|
17
|
+
- preset has been bumped to **version 2**
|
|
18
|
+
- when using an S3 cache, all videos using this preset will be reencoded and uploaded to cache again (it will replace the same file encoded with preset version 1)
|
|
19
|
+
- When reencoding a video, ffmpeg now uses only 1 CPU thread by default (new arg to `reencode` allows to override this default value)
|
|
20
|
+
- Using openZIM Python bootstrap conventions (including hatch-openzim plugin) #120
|
|
21
|
+
- Add support for Python 3.12, drop Python 3.7 support #118
|
|
22
|
+
- Replace "iso-369" by "iso639-lang" library
|
|
23
|
+
- Replace "file-magic" by "python-magic" library for Alpine Linux support and better maintenance
|
|
24
|
+
|
|
25
|
+
## Fixed
|
|
26
|
+
|
|
27
|
+
- Fixed type hints of `zimscraperlib.zim.Item` and subclasses, and `zimscraperlib.image.optimization:convert_image`
|
|
28
|
+
|
|
29
|
+
## [3.2.0] - 2023-12-16
|
|
30
|
+
|
|
31
|
+
### Added
|
|
32
|
+
|
|
33
|
+
- Add utility function to compute/check ZIM descriptions #110
|
|
34
|
+
|
|
35
|
+
### Changed
|
|
36
|
+
|
|
37
|
+
- Using pylibzim `3.4.0`
|
|
38
|
+
|
|
39
|
+
### Removed
|
|
40
|
+
|
|
41
|
+
- Support for Python 3.7 (EOL)
|
|
42
|
+
|
|
43
|
+
## [3.1.1] - 2023-07-18
|
|
9
44
|
|
|
10
45
|
### Changed
|
|
11
46
|
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: zimscraperlib
|
|
3
|
+
Version: 3.3.0
|
|
4
|
+
Summary: Collection of python tools to re-use common code across scrapers
|
|
5
|
+
Project-URL: Donate, https://www.kiwix.org/en/support-us/
|
|
6
|
+
Project-URL: Homepage, https://github.com/openzim/python-scraperlib
|
|
7
|
+
Author-email: openZIM <dev@openzim.org>
|
|
8
|
+
License: GPL-3.0-or-later
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: offline,openzim,zim
|
|
11
|
+
Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Requires-Python: <3.13,>=3.8
|
|
19
|
+
Requires-Dist: babel<3.0,>=2.9
|
|
20
|
+
Requires-Dist: beautifulsoup4<4.10,>=4.9.3
|
|
21
|
+
Requires-Dist: colorthief==0.2.1
|
|
22
|
+
Requires-Dist: iso639-lang<3.0,>=2.2.3
|
|
23
|
+
Requires-Dist: libzim<4.0,>=3.4.0
|
|
24
|
+
Requires-Dist: lxml<4.10,>=4.6.3
|
|
25
|
+
Requires-Dist: optimize-images<1.6,>=1.3.6
|
|
26
|
+
Requires-Dist: python-magic<0.5,>=0.4.3
|
|
27
|
+
Requires-Dist: python-resize-image<1.2,>=1.1.19
|
|
28
|
+
Requires-Dist: requests<3.0,>=2.25.1
|
|
29
|
+
Requires-Dist: yt-dlp
|
|
30
|
+
Provides-Extra: check
|
|
31
|
+
Requires-Dist: pyright==1.1.350; extra == 'check'
|
|
32
|
+
Provides-Extra: dev
|
|
33
|
+
Requires-Dist: debugpy==1.8.0; extra == 'dev'
|
|
34
|
+
Requires-Dist: pre-commit==3.5.0; extra == 'dev'
|
|
35
|
+
Requires-Dist: zimscraperlib[check]; extra == 'dev'
|
|
36
|
+
Requires-Dist: zimscraperlib[lint]; extra == 'dev'
|
|
37
|
+
Requires-Dist: zimscraperlib[scripts]; extra == 'dev'
|
|
38
|
+
Requires-Dist: zimscraperlib[test]; extra == 'dev'
|
|
39
|
+
Provides-Extra: lint
|
|
40
|
+
Requires-Dist: black==24.1.1; extra == 'lint'
|
|
41
|
+
Requires-Dist: ruff==0.2.1; extra == 'lint'
|
|
42
|
+
Provides-Extra: scripts
|
|
43
|
+
Requires-Dist: invoke==2.2.0; extra == 'scripts'
|
|
44
|
+
Provides-Extra: test
|
|
45
|
+
Requires-Dist: coverage==7.4.1; extra == 'test'
|
|
46
|
+
Requires-Dist: pytest-mock==3.12.0; extra == 'test'
|
|
47
|
+
Requires-Dist: pytest==8.0.0; extra == 'test'
|
|
48
|
+
Description-Content-Type: text/markdown
|
|
49
|
+
|
|
50
|
+
zimscraperlib
|
|
51
|
+
=============
|
|
52
|
+
|
|
53
|
+
[](https://github.com/openzim/python-scraperlib/actions?query=branch%3Amain)
|
|
54
|
+
[](https://www.codefactor.io/repository/github/openzim/python-scraperlib)
|
|
55
|
+
[](https://www.gnu.org/licenses/gpl-3.0)
|
|
56
|
+
[](https://pypi.org/project/zimscraperlib/)
|
|
57
|
+
[](https://codecov.io/gh/openzim/python-scraperlib)
|
|
58
|
+
|
|
59
|
+
Collection of python code to re-use across python-based scrapers
|
|
60
|
+
|
|
61
|
+
# Usage
|
|
62
|
+
|
|
63
|
+
* This library is meant to be installed via PyPI ([`zimscraperlib`](https://pypi.org/project/zimscraperlib/)).
|
|
64
|
+
* Make sure to reference it using a version code as the API is subject to frequent changes.
|
|
65
|
+
* API should remain the same only within the same *minor* version.
|
|
66
|
+
|
|
67
|
+
Example usage:
|
|
68
|
+
|
|
69
|
+
``` pip
|
|
70
|
+
zimscraperlib>=1.1,<1.2
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
# Dependencies
|
|
74
|
+
|
|
75
|
+
* libmagic
|
|
76
|
+
* wget
|
|
77
|
+
* libzim (auto-installed, not available on Windows)
|
|
78
|
+
* Pillow
|
|
79
|
+
* FFmpeg
|
|
80
|
+
* gifsicle (>=1.92)
|
|
81
|
+
|
|
82
|
+
## macOS
|
|
83
|
+
|
|
84
|
+
```sh
|
|
85
|
+
brew install libmagic wget libtiff libjpeg webp little-cms2 ffmpeg gifsicle
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
## Linux
|
|
89
|
+
|
|
90
|
+
```sh
|
|
91
|
+
sudo apt install libmagic1 wget ffmpeg \
|
|
92
|
+
libtiff5-dev libjpeg8-dev libopenjp2-7-dev zlib1g-dev \
|
|
93
|
+
libfreetype6-dev liblcms2-dev libwebp-dev tcl8.6-dev tk8.6-dev python3-tk \
|
|
94
|
+
libharfbuzz-dev libfribidi-dev libxcb1-dev gifsicle
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
## Alpine
|
|
98
|
+
```
|
|
99
|
+
apk add ffmpeg gifsicle libmagic wget libjpeg
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
**Nota:** i18n features do not work on Alpine, see https://github.com/openzim/python-scraperlib/issues/134 ; there is one corresponding test which is failing.
|
|
103
|
+
|
|
104
|
+
# Contribution
|
|
105
|
+
|
|
106
|
+
This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing)
|
|
107
|
+
|
|
108
|
+
```shell
|
|
109
|
+
pip install hatch
|
|
110
|
+
pip install ".[dev]"
|
|
111
|
+
pre-commit install
|
|
112
|
+
# For tests
|
|
113
|
+
invoke coverage
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
# Users
|
|
117
|
+
|
|
118
|
+
Non-exhaustive list of scrapers using it (check status when updating API):
|
|
119
|
+
|
|
120
|
+
* [openzim/freecodecamp](https://github.com/openzim/freecodecamp)
|
|
121
|
+
* [openzim/gutenberg](https://github.com/openzim/gutenberg)
|
|
122
|
+
* [openzim/ifixit](https://github.com/openzim/ifixit)
|
|
123
|
+
* [openzim/kolibri](https://github.com/openzim/kolibri)
|
|
124
|
+
* [openzim/nautilus](https://github.com/openzim/nautilus)
|
|
125
|
+
* [openzim/nautilus](https://github.com/openzim/nautilus)
|
|
126
|
+
* [openzim/openedx](https://github.com/openzim/openedx)
|
|
127
|
+
* [openzim/sotoki](https://github.com/openzim/sotoki)
|
|
128
|
+
* [openzim/ted](https://github.com/openzim/ted)
|
|
129
|
+
* [openzim/warc2zim](https://github.com/openzim/warc2zim)
|
|
130
|
+
* [openzim/wikihow](https://github.com/openzim/wikihow)
|
|
131
|
+
* [openzim/youtube](https://github.com/openzim/youtube)
|
|
@@ -45,29 +45,38 @@ sudo apt install libmagic1 wget ffmpeg \
|
|
|
45
45
|
libharfbuzz-dev libfribidi-dev libxcb1-dev gifsicle
|
|
46
46
|
```
|
|
47
47
|
|
|
48
|
+
## Alpine
|
|
49
|
+
```
|
|
50
|
+
apk add ffmpeg gifsicle libmagic wget libjpeg
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
**Nota:** i18n features do not work on Alpine, see https://github.com/openzim/python-scraperlib/issues/134 ; there is one corresponding test which is failing.
|
|
54
|
+
|
|
48
55
|
# Contribution
|
|
49
56
|
|
|
57
|
+
This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing)
|
|
58
|
+
|
|
50
59
|
```shell
|
|
51
|
-
pip
|
|
52
|
-
pip install
|
|
60
|
+
pip install hatch
|
|
61
|
+
pip install ".[dev]"
|
|
53
62
|
pre-commit install
|
|
54
63
|
# For tests
|
|
55
|
-
|
|
64
|
+
invoke coverage
|
|
56
65
|
```
|
|
57
66
|
|
|
58
67
|
# Users
|
|
59
68
|
|
|
60
69
|
Non-exhaustive list of scrapers using it (check status when updating API):
|
|
61
70
|
|
|
62
|
-
* [openzim/
|
|
71
|
+
* [openzim/freecodecamp](https://github.com/openzim/freecodecamp)
|
|
72
|
+
* [openzim/gutenberg](https://github.com/openzim/gutenberg)
|
|
73
|
+
* [openzim/ifixit](https://github.com/openzim/ifixit)
|
|
74
|
+
* [openzim/kolibri](https://github.com/openzim/kolibri)
|
|
63
75
|
* [openzim/nautilus](https://github.com/openzim/nautilus)
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
*
|
|
68
|
-
*
|
|
69
|
-
*
|
|
70
|
-
*
|
|
71
|
-
* Upload to PyPI `twine upload dist/zimscraperlib-2.0.0*`.
|
|
72
|
-
* Commit your Changelog + version bump changes
|
|
73
|
-
* Tag version on git `git tag -a v2.0.0`
|
|
76
|
+
* [openzim/nautilus](https://github.com/openzim/nautilus)
|
|
77
|
+
* [openzim/openedx](https://github.com/openzim/openedx)
|
|
78
|
+
* [openzim/sotoki](https://github.com/openzim/sotoki)
|
|
79
|
+
* [openzim/ted](https://github.com/openzim/ted)
|
|
80
|
+
* [openzim/warc2zim](https://github.com/openzim/warc2zim)
|
|
81
|
+
* [openzim/wikihow](https://github.com/openzim/wikihow)
|
|
82
|
+
* [openzim/youtube](https://github.com/openzim/youtube)
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from typing import List
|
|
4
|
+
|
|
5
|
+
from zimscraperlib import logger
|
|
6
|
+
from zimscraperlib.video import presets, reencode
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def encode_video(src_path: Path, dst_path: Path, preset: str):
|
|
10
|
+
if not src_path.exists():
|
|
11
|
+
raise ValueError(f"{src_path} does not exists")
|
|
12
|
+
try:
|
|
13
|
+
preset_cls = getattr(presets, preset)
|
|
14
|
+
except AttributeError:
|
|
15
|
+
logger.error(f"{preset} preset not found")
|
|
16
|
+
raise
|
|
17
|
+
logger.info(f"Encoding video {src_path} with {preset} version {preset_cls.VERSION}")
|
|
18
|
+
success, process = reencode(
|
|
19
|
+
src_path=src_path,
|
|
20
|
+
dst_path=dst_path,
|
|
21
|
+
ffmpeg_args=preset_cls().to_ffmpeg_args(),
|
|
22
|
+
with_process=True,
|
|
23
|
+
) # pyright: ignore[reportGeneralTypeIssues] (returned type is variable, depending on `with_process` value)
|
|
24
|
+
if not success:
|
|
25
|
+
logger.error(f"conversion failed:\n{process.stdout}")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def run(args: List[str] = sys.argv):
|
|
29
|
+
if len(args) < 4: # noqa: PLR2004
|
|
30
|
+
print(f"Usage: {args[0]} <src_path> <dst_path> <preset>") # noqa: T201
|
|
31
|
+
print( # noqa: T201
|
|
32
|
+
"\t<src_path>\tpath to the video to encode."
|
|
33
|
+
"\t<dst_path>\tpath to the store the reencoded video."
|
|
34
|
+
"\t<preset>\tname of preset to use."
|
|
35
|
+
)
|
|
36
|
+
return 1
|
|
37
|
+
encode_video(Path(args[1]), Path(args[2]), args[3])
|
|
38
|
+
return 0
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
if __name__ == "__main__":
|
|
42
|
+
sys.exit(run())
|