zimscraperlib 3.3.1__tar.gz → 3.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/.pre-commit-config.yaml +3 -3
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/CHANGELOG.md +11 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/PKG-INFO +11 -11
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/pyproject.toml +10 -10
- zimscraperlib-3.3.2/src/zimscraperlib/__about__.py +1 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/src/zimscraperlib/zim/filesystem.py +8 -1
- zimscraperlib-3.3.1/src/zimscraperlib/__about__.py +0 -1
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/.gitignore +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/LICENSE +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/README.md +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/contrib/README.md +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/contrib/encode_video.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/src/zimscraperlib/__init__.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/src/zimscraperlib/constants.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/src/zimscraperlib/download.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/src/zimscraperlib/filesystem.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/src/zimscraperlib/fix_ogvjs_dist.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/src/zimscraperlib/html.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/src/zimscraperlib/i18n.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/src/zimscraperlib/image/__init__.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/src/zimscraperlib/image/convertion.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/src/zimscraperlib/image/optimization.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/src/zimscraperlib/image/presets.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/src/zimscraperlib/image/probing.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/src/zimscraperlib/image/transformation.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/src/zimscraperlib/image/utils.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/src/zimscraperlib/inputs.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/src/zimscraperlib/logging.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/src/zimscraperlib/misc.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/src/zimscraperlib/types.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/src/zimscraperlib/uri.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/src/zimscraperlib/video/__init__.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/src/zimscraperlib/video/config.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/src/zimscraperlib/video/encoding.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/src/zimscraperlib/video/presets.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/src/zimscraperlib/video/probing.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/src/zimscraperlib/zim/__init__.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/src/zimscraperlib/zim/_libkiwix.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/src/zimscraperlib/zim/archive.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/src/zimscraperlib/zim/creator.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/src/zimscraperlib/zim/items.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/src/zimscraperlib/zim/metadata.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/src/zimscraperlib/zim/providers.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tasks.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/conftest.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/download/test_download.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/files/DroidSans.ttf +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/files/audio.mp3 +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/files/blue.jpg +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/files/commons.png +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/files/commons48.png +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/files/mail.gif +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/files/ninja.webp +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/files/pluto.jpg +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/files/square.jpg +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/files/square.png +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/files/star.svg +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/files/video.mkv +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/files/video.mp4 +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/files/video.webm +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/filesystem/test_filesystem.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/html/conftest.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/html/test_html.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/i18n/locale/fr/LC_MESSAGES/messages.po +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/i18n/locale/pt/LC_MESSAGES/messages.po +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/i18n/test_i18n.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/image/test_image.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/inputs/test_inputs.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/logging/conftest.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/logging/test_logging.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/misc/test_misc.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/ogvjs/conftest.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/ogvjs/test_ogvjs.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/types/test_types.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/uri/test_uri.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/video/conftest.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/video/test_encoding.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/video/test_video.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/zim/conftest.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/zim/test_archive.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/zim/test_fs.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/zim/test_libkiwix.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/zim/test_metadata.py +0 -0
- {zimscraperlib-3.3.1 → zimscraperlib-3.3.2}/tests/zim/test_zim_creator.py +0 -0
|
@@ -7,15 +7,15 @@ repos:
|
|
|
7
7
|
- id: trailing-whitespace
|
|
8
8
|
- id: end-of-file-fixer
|
|
9
9
|
- repo: https://github.com/psf/black
|
|
10
|
-
rev: "24.
|
|
10
|
+
rev: "24.3.0"
|
|
11
11
|
hooks:
|
|
12
12
|
- id: black
|
|
13
13
|
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
14
|
-
rev: v0.
|
|
14
|
+
rev: v0.3.3
|
|
15
15
|
hooks:
|
|
16
16
|
- id: ruff
|
|
17
17
|
- repo: https://github.com/RobertCraigie/pyright-python
|
|
18
|
-
rev: v1.1.
|
|
18
|
+
rev: v1.1.354
|
|
19
19
|
hooks:
|
|
20
20
|
- id: pyright
|
|
21
21
|
name: pyright (system)
|
|
@@ -5,6 +5,17 @@ All notable changes to this project are documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0).
|
|
7
7
|
|
|
8
|
+
## [3.3.2] - 2024-03-25
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- Add support for `disable_metadata_checks` and `ignore_duplicates` arguments in `make_zim_file` function ("zimwritefs-mode")
|
|
13
|
+
|
|
14
|
+
### Changed
|
|
15
|
+
|
|
16
|
+
- Relaxed constraints on Python dependencies
|
|
17
|
+
- Upgraded optional dependencies used for test and QA
|
|
18
|
+
|
|
8
19
|
## [3.3.1] - 2024-02-27
|
|
9
20
|
|
|
10
21
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: zimscraperlib
|
|
3
|
-
Version: 3.3.
|
|
3
|
+
Version: 3.3.2
|
|
4
4
|
Summary: Collection of python tools to re-use common code across scrapers
|
|
5
5
|
Project-URL: Donate, https://www.kiwix.org/en/support-us/
|
|
6
6
|
Project-URL: Homepage, https://github.com/openzim/python-scraperlib
|
|
@@ -17,34 +17,34 @@ Classifier: Programming Language :: Python :: 3.11
|
|
|
17
17
|
Classifier: Programming Language :: Python :: 3.12
|
|
18
18
|
Requires-Python: <3.13,>=3.8
|
|
19
19
|
Requires-Dist: babel<3.0,>=2.9
|
|
20
|
-
Requires-Dist: beautifulsoup4<
|
|
20
|
+
Requires-Dist: beautifulsoup4<5.0,>=4.9.3
|
|
21
21
|
Requires-Dist: colorthief==0.2.1
|
|
22
22
|
Requires-Dist: iso639-lang<3.0,>=2.2.3
|
|
23
23
|
Requires-Dist: libzim<4.0,>=3.4.0
|
|
24
|
-
Requires-Dist: lxml<
|
|
25
|
-
Requires-Dist: optimize-images<
|
|
24
|
+
Requires-Dist: lxml<6.0,>=4.6.3
|
|
25
|
+
Requires-Dist: optimize-images<2.0,>=1.3.6
|
|
26
26
|
Requires-Dist: python-magic<0.5,>=0.4.3
|
|
27
27
|
Requires-Dist: python-resize-image<1.2,>=1.1.19
|
|
28
28
|
Requires-Dist: requests<3.0,>=2.25.1
|
|
29
29
|
Requires-Dist: yt-dlp
|
|
30
30
|
Provides-Extra: check
|
|
31
|
-
Requires-Dist: pyright==1.1.
|
|
31
|
+
Requires-Dist: pyright==1.1.354; extra == 'check'
|
|
32
32
|
Provides-Extra: dev
|
|
33
|
-
Requires-Dist: debugpy==1.8.
|
|
34
|
-
Requires-Dist: pre-commit==3.
|
|
33
|
+
Requires-Dist: debugpy==1.8.1; extra == 'dev'
|
|
34
|
+
Requires-Dist: pre-commit==3.6.2; extra == 'dev'
|
|
35
35
|
Requires-Dist: zimscraperlib[check]; extra == 'dev'
|
|
36
36
|
Requires-Dist: zimscraperlib[lint]; extra == 'dev'
|
|
37
37
|
Requires-Dist: zimscraperlib[scripts]; extra == 'dev'
|
|
38
38
|
Requires-Dist: zimscraperlib[test]; extra == 'dev'
|
|
39
39
|
Provides-Extra: lint
|
|
40
|
-
Requires-Dist: black==24.
|
|
41
|
-
Requires-Dist: ruff==0.
|
|
40
|
+
Requires-Dist: black==24.3.0; extra == 'lint'
|
|
41
|
+
Requires-Dist: ruff==0.3.3; extra == 'lint'
|
|
42
42
|
Provides-Extra: scripts
|
|
43
43
|
Requires-Dist: invoke==2.2.0; extra == 'scripts'
|
|
44
44
|
Provides-Extra: test
|
|
45
|
-
Requires-Dist: coverage==7.4.
|
|
45
|
+
Requires-Dist: coverage==7.4.4; extra == 'test'
|
|
46
46
|
Requires-Dist: pytest-mock==3.12.0; extra == 'test'
|
|
47
|
-
Requires-Dist: pytest==8.
|
|
47
|
+
Requires-Dist: pytest==8.1.1; extra == 'test'
|
|
48
48
|
Description-Content-Type: text/markdown
|
|
49
49
|
|
|
50
50
|
zimscraperlib
|
|
@@ -15,9 +15,9 @@ dependencies = [
|
|
|
15
15
|
"Babel>=2.9,<3.0",
|
|
16
16
|
"python-magic>=0.4.3,<0.5",
|
|
17
17
|
"libzim>=3.4.0,<4.0",
|
|
18
|
-
"beautifulsoup4>=4.9.3,<
|
|
19
|
-
"lxml>=4.6.3,<
|
|
20
|
-
"optimize-images>=1.3.6,<
|
|
18
|
+
"beautifulsoup4>=4.9.3,<5.0",
|
|
19
|
+
"lxml>=4.6.3,<6.0",
|
|
20
|
+
"optimize-images>=1.3.6,<2.0",
|
|
21
21
|
# youtube-dl should be updated as frequently as possible
|
|
22
22
|
"yt-dlp"
|
|
23
23
|
]
|
|
@@ -36,20 +36,20 @@ scripts = [
|
|
|
36
36
|
"invoke==2.2.0",
|
|
37
37
|
]
|
|
38
38
|
lint = [
|
|
39
|
-
"black==24.
|
|
40
|
-
"ruff==0.
|
|
39
|
+
"black==24.3.0",
|
|
40
|
+
"ruff==0.3.3",
|
|
41
41
|
]
|
|
42
42
|
check = [
|
|
43
|
-
"pyright==1.1.
|
|
43
|
+
"pyright==1.1.354",
|
|
44
44
|
]
|
|
45
45
|
test = [
|
|
46
|
-
"pytest==8.
|
|
46
|
+
"pytest==8.1.1",
|
|
47
47
|
"pytest-mock==3.12.0",
|
|
48
|
-
"coverage==7.4.
|
|
48
|
+
"coverage==7.4.4",
|
|
49
49
|
]
|
|
50
50
|
dev = [
|
|
51
|
-
"pre-commit==3.
|
|
52
|
-
"debugpy==1.8.
|
|
51
|
+
"pre-commit==3.6.2",
|
|
52
|
+
"debugpy==1.8.1",
|
|
53
53
|
"zimscraperlib[scripts]",
|
|
54
54
|
"zimscraperlib[lint]",
|
|
55
55
|
"zimscraperlib[test]",
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "3.3.2"
|
|
@@ -135,6 +135,8 @@ def make_zim_file(
|
|
|
135
135
|
redirects_file: pathlib.Path = None, # noqa: RUF013 # pyright: ignore
|
|
136
136
|
rewrite_links: bool = True, # noqa: FBT001, FBT002, ARG001
|
|
137
137
|
workaround_nocancel: bool = True, # noqa: FBT001, FBT002
|
|
138
|
+
ignore_duplicates: bool = True, # noqa: FBT001, FBT002
|
|
139
|
+
disable_metadata_checks: bool = False, # noqa: FBT001, FBT002
|
|
138
140
|
):
|
|
139
141
|
"""Creates a zimwriterfs-like ZIM file at {fpath} from {build_dir}
|
|
140
142
|
|
|
@@ -157,7 +159,12 @@ def make_zim_file(
|
|
|
157
159
|
with open(illustration_path, "rb") as fh:
|
|
158
160
|
illustration_data = fh.read()
|
|
159
161
|
|
|
160
|
-
zim_file = Creator(
|
|
162
|
+
zim_file = Creator(
|
|
163
|
+
filename=fpath,
|
|
164
|
+
main_path=main_page,
|
|
165
|
+
ignore_duplicates=ignore_duplicates,
|
|
166
|
+
disable_metadata_checks=disable_metadata_checks,
|
|
167
|
+
).config_metadata(
|
|
161
168
|
**{
|
|
162
169
|
k: v
|
|
163
170
|
for k, v in {
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "3.3.1"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|