arkindex-base-worker 0.4.0rc5__py3-none-any.whl → 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {arkindex_base_worker-0.4.0rc5.dist-info → arkindex_base_worker-0.5.0.dist-info}/METADATA +10 -13
- arkindex_base_worker-0.5.0.dist-info/RECORD +60 -0
- {arkindex_base_worker-0.4.0rc5.dist-info → arkindex_base_worker-0.5.0.dist-info}/WHEEL +1 -1
- {arkindex_base_worker-0.4.0rc5.dist-info → arkindex_base_worker-0.5.0.dist-info}/top_level.txt +1 -0
- arkindex_worker/__init__.py +3 -0
- arkindex_worker/cache.py +6 -25
- arkindex_worker/image.py +105 -66
- arkindex_worker/utils.py +2 -1
- arkindex_worker/worker/__init__.py +22 -32
- arkindex_worker/worker/base.py +16 -9
- arkindex_worker/worker/classification.py +36 -34
- arkindex_worker/worker/corpus.py +3 -3
- arkindex_worker/worker/dataset.py +9 -9
- arkindex_worker/worker/element.py +261 -231
- arkindex_worker/worker/entity.py +137 -206
- arkindex_worker/worker/image.py +3 -3
- arkindex_worker/worker/metadata.py +27 -38
- arkindex_worker/worker/process.py +24 -0
- arkindex_worker/worker/task.py +9 -9
- arkindex_worker/worker/training.py +15 -11
- arkindex_worker/worker/transcription.py +77 -71
- examples/standalone/python/worker.py +171 -0
- examples/tooled/python/worker.py +50 -0
- tests/conftest.py +22 -36
- tests/test_base_worker.py +1 -1
- tests/test_cache.py +1 -2
- tests/test_dataset_worker.py +1 -1
- tests/test_elements_worker/test_element.py +200 -26
- tests/test_elements_worker/{test_entity_create.py → test_entity.py} +220 -227
- tests/test_elements_worker/test_metadata.py +0 -47
- tests/test_elements_worker/test_process.py +89 -0
- tests/test_elements_worker/test_training.py +8 -8
- tests/test_elements_worker/test_worker.py +61 -14
- tests/test_image.py +244 -126
- tests/test_merge.py +0 -7
- tests/test_utils.py +37 -0
- arkindex_base_worker-0.4.0rc5.dist-info/RECORD +0 -60
- arkindex_worker/worker/version.py +0 -58
- tests/test_elements_worker/test_entity_list_and_check.py +0 -160
- tests/test_elements_worker/test_version.py +0 -60
- {arkindex_base_worker-0.4.0rc5.dist-info → arkindex_base_worker-0.5.0.dist-info/licenses}/LICENSE +0 -0
tests/test_image.py
CHANGED
|
@@ -9,6 +9,7 @@ import pytest
|
|
|
9
9
|
from PIL import Image, ImageChops, ImageOps
|
|
10
10
|
from requests import HTTPError
|
|
11
11
|
|
|
12
|
+
import arkindex_worker.image
|
|
12
13
|
from arkindex_worker.cache import CachedElement, create_tables, init_cache_db
|
|
13
14
|
from arkindex_worker.image import (
|
|
14
15
|
IIIF_FULL,
|
|
@@ -35,9 +36,18 @@ ROTATED_MIRRORED_IMAGE = FIXTURES_DIR / "rotated_mirrored_image.jpg"
|
|
|
35
36
|
TEST_IMAGE = {"width": 800, "height": 300}
|
|
36
37
|
|
|
37
38
|
|
|
38
|
-
@pytest.fixture
|
|
39
|
+
@pytest.fixture
|
|
39
40
|
def mock_page():
|
|
40
41
|
class Page(Element):
|
|
42
|
+
@property
|
|
43
|
+
def crop(self):
|
|
44
|
+
# Image from Socface (https://socface.site.ined.fr/) project (AD026)
|
|
45
|
+
image = Image.open(FIXTURES_DIR / "AD026_6M_00505_0001_0373.jpg")
|
|
46
|
+
x, y, element_width, element_height = polygon_bounding_box(
|
|
47
|
+
self.zone.polygon
|
|
48
|
+
)
|
|
49
|
+
return image.crop(box=(x, y, x + element_width, y + element_height))
|
|
50
|
+
|
|
41
51
|
def open_image(
|
|
42
52
|
self,
|
|
43
53
|
*args,
|
|
@@ -46,18 +56,21 @@ def mock_page():
|
|
|
46
56
|
use_full_image: bool | None = False,
|
|
47
57
|
**kwargs,
|
|
48
58
|
) -> Image.Image:
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
59
|
+
crop = self.crop.copy()
|
|
60
|
+
crop.thumbnail(
|
|
61
|
+
size=(
|
|
62
|
+
max_width or self.zone.image.width,
|
|
63
|
+
max_height or self.zone.image.height,
|
|
64
|
+
)
|
|
65
|
+
)
|
|
66
|
+
return crop
|
|
54
67
|
|
|
55
68
|
return Page(
|
|
56
69
|
id="page_id",
|
|
57
70
|
name="1",
|
|
58
71
|
zone={
|
|
59
|
-
"polygon": [[0, 0], [
|
|
60
|
-
"image": {"width":
|
|
72
|
+
"polygon": [[0, 0], [2000, 0], [2000, 3000], [0, 3000], [0, 0]],
|
|
73
|
+
"image": {"width": 2000, "height": 3000},
|
|
61
74
|
},
|
|
62
75
|
rotation_angle=0,
|
|
63
76
|
mirrored=False,
|
|
@@ -221,7 +234,7 @@ def test_open_image_rotate_mirror(rotation_angle, mirrored, expected_path):
|
|
|
221
234
|
@pytest.mark.parametrize(
|
|
222
235
|
("polygon", "error"),
|
|
223
236
|
[
|
|
224
|
-
# Polygon
|
|
237
|
+
# Polygon isn't a list or tuple
|
|
225
238
|
(
|
|
226
239
|
{
|
|
227
240
|
"polygon": [
|
|
@@ -239,7 +252,15 @@ def test_open_image_rotate_mirror(rotation_angle, mirrored, expected_path):
|
|
|
239
252
|
[99, 208],
|
|
240
253
|
]
|
|
241
254
|
},
|
|
242
|
-
"
|
|
255
|
+
"Polygon must be a valid list or tuple of points.",
|
|
256
|
+
),
|
|
257
|
+
# Polygon hasn't enough points
|
|
258
|
+
(
|
|
259
|
+
[
|
|
260
|
+
[99, 200],
|
|
261
|
+
[25, 224],
|
|
262
|
+
],
|
|
263
|
+
"Polygon should have at least three points.",
|
|
243
264
|
),
|
|
244
265
|
# Point coordinates are not integers
|
|
245
266
|
(
|
|
@@ -624,186 +645,283 @@ def test_upload_image(responses):
|
|
|
624
645
|
|
|
625
646
|
|
|
626
647
|
@pytest.mark.parametrize(
|
|
627
|
-
(
|
|
648
|
+
(
|
|
649
|
+
"max_pixels_short",
|
|
650
|
+
"max_pixels_long",
|
|
651
|
+
"max_bytes",
|
|
652
|
+
"expected_sizes",
|
|
653
|
+
"expected_logs",
|
|
654
|
+
),
|
|
628
655
|
[
|
|
629
|
-
# No limits
|
|
656
|
+
# No limits provided
|
|
630
657
|
(
|
|
631
658
|
None,
|
|
632
659
|
None,
|
|
660
|
+
None,
|
|
661
|
+
[(2000, 3000), (1000, 1500), (200, 300)],
|
|
633
662
|
[
|
|
634
|
-
(
|
|
635
|
-
(
|
|
636
|
-
(
|
|
637
|
-
(1594, 2400),
|
|
638
|
-
(1494, 2250),
|
|
639
|
-
(1395, 2100),
|
|
640
|
-
(1195, 1800),
|
|
641
|
-
(996, 1500),
|
|
642
|
-
(797, 1200),
|
|
643
|
-
(598, 900),
|
|
644
|
-
],
|
|
645
|
-
[
|
|
646
|
-
(logging.WARNING, "The image was resized to (1992 x 3000)."),
|
|
647
|
-
(logging.WARNING, "The image was resized to (1793 x 2700)."),
|
|
648
|
-
(logging.WARNING, "The image was resized to (1694 x 2550)."),
|
|
649
|
-
(logging.WARNING, "The image was resized to (1594 x 2400)."),
|
|
650
|
-
(logging.WARNING, "The image was resized to (1494 x 2250)."),
|
|
651
|
-
(logging.WARNING, "The image was resized to (1395 x 2100)."),
|
|
652
|
-
(logging.WARNING, "The image was resized to (1195 x 1800)."),
|
|
653
|
-
(logging.WARNING, "The image was resized to (996 x 1500)."),
|
|
654
|
-
(logging.WARNING, "The image was resized to (797 x 1200)."),
|
|
655
|
-
(logging.WARNING, "The image was resized to (598 x 900)."),
|
|
663
|
+
(logging.INFO, "This element's image dimensions are (2000 x 3000)."),
|
|
664
|
+
(logging.WARNING, "The image was resized to (1000 x 1500)."),
|
|
665
|
+
(logging.WARNING, "The image was resized to (200 x 300)."),
|
|
656
666
|
],
|
|
657
667
|
),
|
|
658
|
-
# Image already under
|
|
668
|
+
# Image already under all three limits
|
|
659
669
|
(
|
|
670
|
+
10000,
|
|
660
671
|
10000,
|
|
661
672
|
4000000, # 4MB
|
|
673
|
+
[(2000, 3000), (1000, 1500), (200, 300)],
|
|
662
674
|
[
|
|
663
|
-
(
|
|
664
|
-
(
|
|
665
|
-
(
|
|
666
|
-
(1594, 2400),
|
|
667
|
-
(1494, 2250),
|
|
668
|
-
(1395, 2100),
|
|
669
|
-
(1195, 1800),
|
|
670
|
-
(996, 1500),
|
|
671
|
-
(797, 1200),
|
|
672
|
-
(598, 900),
|
|
675
|
+
(logging.INFO, "This element's image dimensions are (2000 x 3000)."),
|
|
676
|
+
(logging.WARNING, "The image was resized to (1000 x 1500)."),
|
|
677
|
+
(logging.WARNING, "The image was resized to (200 x 300)."),
|
|
673
678
|
],
|
|
679
|
+
),
|
|
680
|
+
# Image above the "short side in pixels" limit
|
|
681
|
+
(
|
|
682
|
+
1000,
|
|
683
|
+
None,
|
|
684
|
+
None,
|
|
685
|
+
[(1000, 1500), (500, 750), (100, 150)],
|
|
674
686
|
[
|
|
675
|
-
(logging.
|
|
676
|
-
(
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
(logging.WARNING, "The image
|
|
681
|
-
(logging.WARNING, "The image was resized to (
|
|
682
|
-
(logging.WARNING, "The image was resized to (
|
|
683
|
-
(logging.WARNING, "The image was resized to (
|
|
684
|
-
(logging.WARNING, "The image was resized to (598 x 900)."),
|
|
687
|
+
(logging.INFO, "This element's image dimensions are (2000 x 3000)."),
|
|
688
|
+
(
|
|
689
|
+
logging.WARNING,
|
|
690
|
+
"Maximum image dimensions supported are (1000 x 3000).",
|
|
691
|
+
),
|
|
692
|
+
(logging.WARNING, "The image will be resized."),
|
|
693
|
+
(logging.WARNING, "The image was resized to (1000 x 1500)."),
|
|
694
|
+
(logging.WARNING, "The image was resized to (500 x 750)."),
|
|
695
|
+
(logging.WARNING, "The image was resized to (100 x 150)."),
|
|
685
696
|
],
|
|
686
697
|
),
|
|
687
|
-
# Image above the
|
|
698
|
+
# Image above the "long side in pixels" limit
|
|
688
699
|
(
|
|
689
700
|
None,
|
|
690
|
-
|
|
691
|
-
|
|
701
|
+
2000,
|
|
702
|
+
None,
|
|
703
|
+
[(1333, 2000), (667, 1000), (133, 200)],
|
|
692
704
|
[
|
|
693
|
-
(logging.
|
|
694
|
-
(
|
|
695
|
-
|
|
705
|
+
(logging.INFO, "This element's image dimensions are (2000 x 3000)."),
|
|
706
|
+
(
|
|
707
|
+
logging.WARNING,
|
|
708
|
+
"Maximum image dimensions supported are (2000 x 2000).",
|
|
709
|
+
),
|
|
696
710
|
(logging.WARNING, "The image will be resized."),
|
|
697
|
-
(logging.WARNING, "The image was resized to (
|
|
698
|
-
(logging.WARNING, "The image
|
|
711
|
+
(logging.WARNING, "The image was resized to (1333 x 2000)."),
|
|
712
|
+
(logging.WARNING, "The image was resized to (667 x 1000)."),
|
|
713
|
+
(logging.WARNING, "The image was resized to (133 x 200)."),
|
|
714
|
+
],
|
|
715
|
+
),
|
|
716
|
+
# Image above the "size in bytes" limit
|
|
717
|
+
(
|
|
718
|
+
None,
|
|
719
|
+
None,
|
|
720
|
+
100000, # 100kB
|
|
721
|
+
[(200, 300)],
|
|
722
|
+
[
|
|
723
|
+
(logging.INFO, "This element's image dimensions are (2000 x 3000)."),
|
|
724
|
+
(logging.WARNING, "The image size is 1.3 MB."),
|
|
699
725
|
(logging.WARNING, "Maximum image input size supported is 100.0 kB."),
|
|
700
726
|
(logging.WARNING, "The image will be resized."),
|
|
701
|
-
(logging.WARNING, "The image was resized to (
|
|
702
|
-
(logging.WARNING, "The image size is
|
|
727
|
+
(logging.WARNING, "The image was resized to (1000 x 1500)."),
|
|
728
|
+
(logging.WARNING, "The image size is 214.2 kB."),
|
|
703
729
|
(logging.WARNING, "Maximum image input size supported is 100.0 kB."),
|
|
704
730
|
(logging.WARNING, "The image will be resized."),
|
|
705
|
-
(logging.WARNING, "The image was resized to (
|
|
706
|
-
|
|
707
|
-
|
|
731
|
+
(logging.WARNING, "The image was resized to (200 x 300)."),
|
|
732
|
+
],
|
|
733
|
+
),
|
|
734
|
+
# Image above all three limits
|
|
735
|
+
(
|
|
736
|
+
1000,
|
|
737
|
+
2000,
|
|
738
|
+
100000, # 100kB
|
|
739
|
+
[(500, 750), (100, 150)],
|
|
740
|
+
[
|
|
741
|
+
(logging.INFO, "This element's image dimensions are (2000 x 3000)."),
|
|
742
|
+
(
|
|
743
|
+
logging.WARNING,
|
|
744
|
+
"Maximum image dimensions supported are (1000 x 2000).",
|
|
745
|
+
),
|
|
708
746
|
(logging.WARNING, "The image will be resized."),
|
|
709
|
-
(logging.WARNING, "The image was resized to (
|
|
710
|
-
(logging.WARNING, "The image size is
|
|
747
|
+
(logging.WARNING, "The image was resized to (1000 x 1500)."),
|
|
748
|
+
(logging.WARNING, "The image size is 214.2 kB."),
|
|
711
749
|
(logging.WARNING, "Maximum image input size supported is 100.0 kB."),
|
|
712
750
|
(logging.WARNING, "The image will be resized."),
|
|
713
|
-
(logging.WARNING, "The image was resized to (
|
|
714
|
-
(logging.WARNING, "The image
|
|
715
|
-
|
|
751
|
+
(logging.WARNING, "The image was resized to (500 x 750)."),
|
|
752
|
+
(logging.WARNING, "The image was resized to (100 x 150)."),
|
|
753
|
+
],
|
|
754
|
+
),
|
|
755
|
+
# Image always above all three limits
|
|
756
|
+
(
|
|
757
|
+
50,
|
|
758
|
+
50,
|
|
759
|
+
50, # 50B
|
|
760
|
+
[],
|
|
761
|
+
[
|
|
762
|
+
(logging.INFO, "This element's image dimensions are (2000 x 3000)."),
|
|
763
|
+
(logging.WARNING, "Maximum image dimensions supported are (50 x 50)."),
|
|
716
764
|
(logging.WARNING, "The image will be resized."),
|
|
717
|
-
(logging.WARNING, "The image was resized to (
|
|
718
|
-
(logging.WARNING, "The image size is
|
|
719
|
-
(logging.WARNING, "Maximum image input size supported is
|
|
765
|
+
(logging.WARNING, "The image was resized to (33 x 50)."),
|
|
766
|
+
(logging.WARNING, "The image size is 1.0 kB."),
|
|
767
|
+
(logging.WARNING, "Maximum image input size supported is 50 Bytes."),
|
|
720
768
|
(logging.WARNING, "The image will be resized."),
|
|
721
|
-
(logging.WARNING, "The image was resized to (
|
|
722
|
-
(logging.WARNING, "The image size is
|
|
723
|
-
(logging.WARNING, "Maximum image input size supported is
|
|
769
|
+
(logging.WARNING, "The image was resized to (17 x 25)."),
|
|
770
|
+
(logging.WARNING, "The image size is 785 Bytes."),
|
|
771
|
+
(logging.WARNING, "Maximum image input size supported is 50 Bytes."),
|
|
724
772
|
(logging.WARNING, "The image will be resized."),
|
|
725
|
-
(logging.WARNING, "The image was resized to (
|
|
726
|
-
(logging.WARNING, "The image size is
|
|
727
|
-
(logging.WARNING, "Maximum image input size supported is
|
|
773
|
+
(logging.WARNING, "The image was resized to (3 x 5)."),
|
|
774
|
+
(logging.WARNING, "The image size is 689 Bytes."),
|
|
775
|
+
(logging.WARNING, "Maximum image input size supported is 50 Bytes."),
|
|
728
776
|
(logging.WARNING, "The image will be resized."),
|
|
729
|
-
(logging.WARNING, "The image was resized to (598 x 900)."),
|
|
730
777
|
],
|
|
731
778
|
),
|
|
732
|
-
|
|
779
|
+
],
|
|
780
|
+
)
|
|
781
|
+
def test_resized_images_portrait_format(
|
|
782
|
+
monkeypatch,
|
|
783
|
+
max_pixels_short,
|
|
784
|
+
max_pixels_long,
|
|
785
|
+
max_bytes,
|
|
786
|
+
expected_sizes,
|
|
787
|
+
expected_logs,
|
|
788
|
+
mock_page,
|
|
789
|
+
caplog,
|
|
790
|
+
):
|
|
791
|
+
monkeypatch.setattr(arkindex_worker.image, "IMAGE_RATIOS", [1.0, 0.5, 0.1])
|
|
792
|
+
|
|
793
|
+
# Short side is the width, long side is the height
|
|
794
|
+
assert mock_page.zone.image.width < mock_page.zone.image.height
|
|
795
|
+
|
|
796
|
+
assert [
|
|
797
|
+
Image.open(image).size
|
|
798
|
+
for image in resized_images(
|
|
799
|
+
element=mock_page,
|
|
800
|
+
max_pixels_short=max_pixels_short,
|
|
801
|
+
max_pixels_long=max_pixels_long,
|
|
802
|
+
max_bytes=max_bytes,
|
|
803
|
+
)
|
|
804
|
+
] == expected_sizes
|
|
805
|
+
|
|
806
|
+
assert [
|
|
807
|
+
(record.levelno, record.message) for record in caplog.records
|
|
808
|
+
] == expected_logs
|
|
809
|
+
|
|
810
|
+
|
|
811
|
+
@pytest.mark.parametrize(
|
|
812
|
+
("max_pixels_short", "max_pixels_long", "expected_sizes", "expected_logs"),
|
|
813
|
+
[
|
|
814
|
+
# Image above the "short side in pixels" limit
|
|
733
815
|
(
|
|
734
|
-
|
|
816
|
+
1000,
|
|
735
817
|
None,
|
|
736
|
-
[(
|
|
818
|
+
[(1500, 1000), (750, 500), (150, 100)],
|
|
737
819
|
[
|
|
820
|
+
(logging.INFO, "This element's image dimensions are (3000 x 2000)."),
|
|
738
821
|
(
|
|
739
822
|
logging.WARNING,
|
|
740
|
-
"Maximum image
|
|
823
|
+
"Maximum image dimensions supported are (3000 x 1000).",
|
|
741
824
|
),
|
|
742
825
|
(logging.WARNING, "The image will be resized."),
|
|
743
|
-
(logging.WARNING, "The image was resized to (
|
|
744
|
-
(logging.WARNING, "The image was resized to (
|
|
745
|
-
(logging.WARNING, "The image was resized to (
|
|
746
|
-
(logging.WARNING, "The image was resized to (797 x 1200)."),
|
|
747
|
-
(logging.WARNING, "The image was resized to (598 x 900)."),
|
|
826
|
+
(logging.WARNING, "The image was resized to (1500 x 1000)."),
|
|
827
|
+
(logging.WARNING, "The image was resized to (750 x 500)."),
|
|
828
|
+
(logging.WARNING, "The image was resized to (150 x 100)."),
|
|
748
829
|
],
|
|
749
830
|
),
|
|
750
|
-
# Image above the
|
|
831
|
+
# Image above the "long side in pixels" limit
|
|
751
832
|
(
|
|
833
|
+
None,
|
|
752
834
|
2000,
|
|
753
|
-
|
|
754
|
-
[(598, 900)],
|
|
835
|
+
[(2000, 1333), (1000, 667), (200, 133)],
|
|
755
836
|
[
|
|
837
|
+
(logging.INFO, "This element's image dimensions are (3000 x 2000)."),
|
|
756
838
|
(
|
|
757
839
|
logging.WARNING,
|
|
758
|
-
"Maximum image
|
|
840
|
+
"Maximum image dimensions supported are (2000 x 2000).",
|
|
759
841
|
),
|
|
760
842
|
(logging.WARNING, "The image will be resized."),
|
|
761
|
-
(logging.WARNING, "The image was resized to (
|
|
762
|
-
(logging.WARNING, "The image
|
|
763
|
-
(logging.WARNING, "
|
|
764
|
-
(logging.WARNING, "The image will be resized."),
|
|
765
|
-
(logging.WARNING, "The image was resized to (1195 x 1800)."),
|
|
766
|
-
(logging.WARNING, "The image size is 258.6 kB."),
|
|
767
|
-
(logging.WARNING, "Maximum image input size supported is 100.0 kB."),
|
|
768
|
-
(logging.WARNING, "The image will be resized."),
|
|
769
|
-
(logging.WARNING, "The image was resized to (996 x 1500)."),
|
|
770
|
-
(logging.WARNING, "The image size is 179.0 kB."),
|
|
771
|
-
(logging.WARNING, "Maximum image input size supported is 100.0 kB."),
|
|
772
|
-
(logging.WARNING, "The image will be resized."),
|
|
773
|
-
(logging.WARNING, "The image was resized to (797 x 1200)."),
|
|
774
|
-
(logging.WARNING, "The image size is 115.7 kB."),
|
|
775
|
-
(logging.WARNING, "Maximum image input size supported is 100.0 kB."),
|
|
776
|
-
(logging.WARNING, "The image will be resized."),
|
|
777
|
-
(logging.WARNING, "The image was resized to (598 x 900)."),
|
|
843
|
+
(logging.WARNING, "The image was resized to (2000 x 1333)."),
|
|
844
|
+
(logging.WARNING, "The image was resized to (1000 x 667)."),
|
|
845
|
+
(logging.WARNING, "The image was resized to (200 x 133)."),
|
|
778
846
|
],
|
|
779
847
|
),
|
|
780
|
-
# Image
|
|
848
|
+
# Image above the two pixels limits
|
|
781
849
|
(
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
[],
|
|
850
|
+
1000,
|
|
851
|
+
2000,
|
|
852
|
+
[(1500, 1000), (750, 500), (150, 100)],
|
|
785
853
|
[
|
|
786
|
-
(logging.
|
|
787
|
-
(
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
854
|
+
(logging.INFO, "This element's image dimensions are (3000 x 2000)."),
|
|
855
|
+
(
|
|
856
|
+
logging.WARNING,
|
|
857
|
+
"Maximum image dimensions supported are (2000 x 1000).",
|
|
858
|
+
),
|
|
791
859
|
(logging.WARNING, "The image will be resized."),
|
|
860
|
+
(logging.WARNING, "The image was resized to (1500 x 1000)."),
|
|
861
|
+
(logging.WARNING, "The image was resized to (750 x 500)."),
|
|
862
|
+
(logging.WARNING, "The image was resized to (150 x 100)."),
|
|
792
863
|
],
|
|
793
864
|
),
|
|
794
865
|
],
|
|
795
866
|
)
|
|
796
|
-
def
|
|
797
|
-
|
|
867
|
+
def test_resized_images_landscape_format(
|
|
868
|
+
monkeypatch,
|
|
869
|
+
max_pixels_short,
|
|
870
|
+
max_pixels_long,
|
|
871
|
+
expected_sizes,
|
|
872
|
+
expected_logs,
|
|
873
|
+
mock_page,
|
|
874
|
+
caplog,
|
|
798
875
|
):
|
|
799
|
-
|
|
876
|
+
monkeypatch.setattr(arkindex_worker.image, "IMAGE_RATIOS", [1.0, 0.5, 0.1])
|
|
877
|
+
|
|
878
|
+
# Short side is the height, long side is the width
|
|
879
|
+
mock_page.zone = {
|
|
880
|
+
"polygon": [[0, 0], [3000, 0], [3000, 2000], [0, 2000], [0, 0]],
|
|
881
|
+
"image": {"width": 3000, "height": 2000},
|
|
882
|
+
}
|
|
883
|
+
assert mock_page.zone.image.height < mock_page.zone.image.width
|
|
800
884
|
|
|
801
885
|
assert [
|
|
802
886
|
Image.open(image).size
|
|
803
887
|
for image in resized_images(
|
|
804
|
-
element=mock_page,
|
|
888
|
+
element=mock_page,
|
|
889
|
+
max_pixels_short=max_pixels_short,
|
|
890
|
+
max_pixels_long=max_pixels_long,
|
|
891
|
+
max_bytes=None,
|
|
805
892
|
)
|
|
806
893
|
] == expected_sizes
|
|
894
|
+
|
|
807
895
|
assert [
|
|
808
896
|
(record.levelno, record.message) for record in caplog.records
|
|
809
897
|
] == expected_logs
|
|
898
|
+
|
|
899
|
+
|
|
900
|
+
def test_resized_images_use_base64(monkeypatch, mock_page, caplog):
|
|
901
|
+
monkeypatch.setattr(arkindex_worker.image, "IMAGE_RATIOS", [1.0, 0.5, 0.25, 0.1])
|
|
902
|
+
|
|
903
|
+
assert list(
|
|
904
|
+
map(
|
|
905
|
+
len,
|
|
906
|
+
resized_images(
|
|
907
|
+
element=mock_page,
|
|
908
|
+
max_pixels_short=None,
|
|
909
|
+
max_pixels_long=None,
|
|
910
|
+
max_bytes=100000,
|
|
911
|
+
use_base64=True,
|
|
912
|
+
),
|
|
913
|
+
)
|
|
914
|
+
) == [65280, 11892]
|
|
915
|
+
|
|
916
|
+
assert [(record.levelno, record.message) for record in caplog.records] == [
|
|
917
|
+
(logging.INFO, "This element's image dimensions are (2000 x 3000)."),
|
|
918
|
+
(logging.WARNING, "The image size is 1.7 MB."),
|
|
919
|
+
(logging.WARNING, "Maximum image input size supported is 100.0 kB."),
|
|
920
|
+
(logging.WARNING, "The image will be resized."),
|
|
921
|
+
(logging.WARNING, "The image was resized to (1000 x 1500)."),
|
|
922
|
+
(logging.WARNING, "The image size is 285.6 kB."),
|
|
923
|
+
(logging.WARNING, "Maximum image input size supported is 100.0 kB."),
|
|
924
|
+
(logging.WARNING, "The image will be resized."),
|
|
925
|
+
(logging.WARNING, "The image was resized to (500 x 750)."),
|
|
926
|
+
(logging.WARNING, "The image was resized to (200 x 300)."),
|
|
927
|
+
]
|
tests/test_merge.py
CHANGED
|
@@ -7,7 +7,6 @@ from arkindex_worker.cache import (
|
|
|
7
7
|
SQL_VERSION,
|
|
8
8
|
CachedClassification,
|
|
9
9
|
CachedElement,
|
|
10
|
-
CachedEntity,
|
|
11
10
|
CachedImage,
|
|
12
11
|
CachedTranscription,
|
|
13
12
|
CachedTranscriptionEntity,
|
|
@@ -85,7 +84,6 @@ def test_merge_databases(
|
|
|
85
84
|
assert CachedElement.select().count() == 0
|
|
86
85
|
assert CachedTranscription.select().count() == 0
|
|
87
86
|
assert CachedClassification.select().count() == 0
|
|
88
|
-
assert CachedEntity.select().count() == 0
|
|
89
87
|
assert CachedTranscriptionEntity.select().count() == 0
|
|
90
88
|
|
|
91
89
|
# Retrieve parents databases paths
|
|
@@ -103,7 +101,6 @@ def test_merge_databases(
|
|
|
103
101
|
assert CachedElement.select().count() == len(expected_elements)
|
|
104
102
|
assert CachedTranscription.select().count() == len(expected_transcriptions)
|
|
105
103
|
assert CachedClassification.select().count() == 0
|
|
106
|
-
assert CachedEntity.select().count() == 0
|
|
107
104
|
assert CachedTranscriptionEntity.select().count() == 0
|
|
108
105
|
assert [
|
|
109
106
|
e.id for e in CachedElement.select().order_by("id")
|
|
@@ -124,7 +121,6 @@ def test_merge_chunk(mock_databases, tmp_path):
|
|
|
124
121
|
assert CachedElement.select().count() == 0
|
|
125
122
|
assert CachedTranscription.select().count() == 0
|
|
126
123
|
assert CachedClassification.select().count() == 0
|
|
127
|
-
assert CachedEntity.select().count() == 0
|
|
128
124
|
assert CachedTranscriptionEntity.select().count() == 0
|
|
129
125
|
|
|
130
126
|
# Check filenames
|
|
@@ -144,7 +140,6 @@ def test_merge_chunk(mock_databases, tmp_path):
|
|
|
144
140
|
assert CachedElement.select().count() == 3
|
|
145
141
|
assert CachedTranscription.select().count() == 0
|
|
146
142
|
assert CachedClassification.select().count() == 0
|
|
147
|
-
assert CachedEntity.select().count() == 0
|
|
148
143
|
assert CachedTranscriptionEntity.select().count() == 0
|
|
149
144
|
assert [e.id for e in CachedElement.select().order_by("id")] == [
|
|
150
145
|
UUID("42424242-4242-4242-4242-424242424242"),
|
|
@@ -171,7 +166,6 @@ def test_merge_from_worker(
|
|
|
171
166
|
assert CachedElement.select().count() == 0
|
|
172
167
|
assert CachedTranscription.select().count() == 0
|
|
173
168
|
assert CachedClassification.select().count() == 0
|
|
174
|
-
assert CachedEntity.select().count() == 0
|
|
175
169
|
assert CachedTranscriptionEntity.select().count() == 0
|
|
176
170
|
|
|
177
171
|
# Configure worker with a specific data directory
|
|
@@ -191,7 +185,6 @@ def test_merge_from_worker(
|
|
|
191
185
|
assert CachedElement.select().count() == 3
|
|
192
186
|
assert CachedTranscription.select().count() == 1
|
|
193
187
|
assert CachedClassification.select().count() == 0
|
|
194
|
-
assert CachedEntity.select().count() == 0
|
|
195
188
|
assert CachedTranscriptionEntity.select().count() == 0
|
|
196
189
|
assert [e.id for e in CachedElement.select().order_by("id")] == [
|
|
197
190
|
UUID("12341234-1234-1234-1234-123412341234"),
|
tests/test_utils.py
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
|
|
1
3
|
import pytest
|
|
2
4
|
|
|
5
|
+
from arkindex_worker.cache import unsupported_cache
|
|
3
6
|
from arkindex_worker.utils import (
|
|
4
7
|
DEFAULT_BATCH_SIZE,
|
|
5
8
|
batch_publication,
|
|
@@ -58,10 +61,24 @@ def test_close_delete_file(tmp_path):
|
|
|
58
61
|
|
|
59
62
|
|
|
60
63
|
class TestMixin:
|
|
64
|
+
def __init__(self, use_cache: bool = False):
|
|
65
|
+
"""
|
|
66
|
+
Args:
|
|
67
|
+
use_cache (bool, optional): To mock BaseWorker.use_cache attribute. Defaults to False.
|
|
68
|
+
"""
|
|
69
|
+
self.use_cache = use_cache
|
|
70
|
+
|
|
61
71
|
@batch_publication
|
|
62
72
|
def custom_publication_in_batches(self, batch_size: int = DEFAULT_BATCH_SIZE):
|
|
63
73
|
return batch_size
|
|
64
74
|
|
|
75
|
+
@unsupported_cache
|
|
76
|
+
@batch_publication
|
|
77
|
+
def custom_publication_in_batches_without_cache(
|
|
78
|
+
self, batch_size: int = DEFAULT_BATCH_SIZE
|
|
79
|
+
):
|
|
80
|
+
return batch_size
|
|
81
|
+
|
|
65
82
|
|
|
66
83
|
def test_batch_publication_decorator_no_parameter():
|
|
67
84
|
assert TestMixin().custom_publication_in_batches() == DEFAULT_BATCH_SIZE
|
|
@@ -81,3 +98,23 @@ def test_batch_publication_decorator_right_parameter(batch_size):
|
|
|
81
98
|
assert (
|
|
82
99
|
TestMixin().custom_publication_in_batches(batch_size=batch_size) == batch_size
|
|
83
100
|
)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def test_batch_publication_decorator_alongside_unsupported_cache(caplog):
|
|
104
|
+
# Capture log messages
|
|
105
|
+
caplog.clear()
|
|
106
|
+
with caplog.at_level(logging.WARNING):
|
|
107
|
+
# Call the helper
|
|
108
|
+
assert (
|
|
109
|
+
TestMixin(use_cache=True).custom_publication_in_batches_without_cache()
|
|
110
|
+
== DEFAULT_BATCH_SIZE
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
# Check logs
|
|
114
|
+
assert caplog.record_tuples == [
|
|
115
|
+
(
|
|
116
|
+
"arkindex_worker",
|
|
117
|
+
logging.WARNING,
|
|
118
|
+
"This API helper `custom_publication_in_batches_without_cache` did not update the cache database",
|
|
119
|
+
),
|
|
120
|
+
]
|
|
@@ -1,60 +0,0 @@
|
|
|
1
|
-
arkindex_worker/__init__.py,sha256=OlgCtTC9MaWeejviY0a3iQpALcRQGMVArFVVYwTF6I8,162
|
|
2
|
-
arkindex_worker/cache.py,sha256=qTblc_zKdYC47Wip6_O9Jf5qBkQW2ozQQrg-nsx1WuY,11221
|
|
3
|
-
arkindex_worker/image.py,sha256=oEgVCrSHiGh3D5-UXfM6PvT17TttSxC0115irpvB3Dw,18581
|
|
4
|
-
arkindex_worker/models.py,sha256=bPQzGZNs5a6z6DEcygsa8T33VOqPlMUbwKzHqlKzwbw,9923
|
|
5
|
-
arkindex_worker/utils.py,sha256=q1EeLdC6ebYIH-C0LOAqw2cNpjCjVoP-Vbr-39mF4w0,9884
|
|
6
|
-
arkindex_worker/worker/__init__.py,sha256=m255Cle3nE_FtAXbbJj_v1aS9ClT6hpDlrUFXTPiqkI,15985
|
|
7
|
-
arkindex_worker/worker/base.py,sha256=7Pmw-UQSxV-xkW8NO5cXsxJ8W8szzyppMaNjq_az81A,19844
|
|
8
|
-
arkindex_worker/worker/classification.py,sha256=zECSNzGCZFzoPoDVZN4kuGYRNLzMQLBaRt3q1jnBSaA,10952
|
|
9
|
-
arkindex_worker/worker/corpus.py,sha256=0TQFOwZ6Te-CZi6lgkZY1wzyJ5wO9LAmcVQtqHvZpPk,2291
|
|
10
|
-
arkindex_worker/worker/dataset.py,sha256=LwzKwNFX4FqfLxh29LSvJydPwRw3VHaB1wjuFhUshsE,5267
|
|
11
|
-
arkindex_worker/worker/element.py,sha256=Qvvq9kJnAHNATHW7zi96eIY1x-0MsR-T5rrSJg6e9Y4,45309
|
|
12
|
-
arkindex_worker/worker/entity.py,sha256=s5wjX6_JfTyk4qfMoV0OWfOXUx6T-9WpOiEpaoaCEFM,14808
|
|
13
|
-
arkindex_worker/worker/image.py,sha256=t_Az6IGnj0EZyvcA4XxfPikOUjn_pztgsyxTkFZhaXU,621
|
|
14
|
-
arkindex_worker/worker/metadata.py,sha256=VRajtd2kaBvar9GercX4knvR6l1WFYjoCdJWU9ccKgk,7291
|
|
15
|
-
arkindex_worker/worker/process.py,sha256=IAJaiiCizK4vpPmMQD0yYSB6IIoyy7yU-5JKaiuPb7o,1073
|
|
16
|
-
arkindex_worker/worker/task.py,sha256=r1j7_qbdNu2Z8H8HbGzO3P3qdx-2N1pBbUPFDca0rqg,1519
|
|
17
|
-
arkindex_worker/worker/training.py,sha256=H8FmCdzGcDW-WMMwcgvmZPlN5tPHwGo0BXn12qmzj8g,10875
|
|
18
|
-
arkindex_worker/worker/transcription.py,sha256=52RY9kYsiR1sz9FxOigyo12Ker3VDbQ4U42gK9DpR3g,21146
|
|
19
|
-
arkindex_worker/worker/version.py,sha256=JIT7OI3Mo7RPkNrjOB9hfqrsG-FYygz_zi4l8PbkuAo,1960
|
|
20
|
-
hooks/pre_gen_project.py,sha256=xQJERv3vv9VzIqcBHI281eeWLWREXUF4mMw7PvJHHXM,269
|
|
21
|
-
tests/__init__.py,sha256=DG--S6IpGl399rzSAjDdHL76CkOIeZIjajCcyUSDhOQ,241
|
|
22
|
-
tests/conftest.py,sha256=2ocZ2x-mZQrNe9zvWwhWk2_4ExdaBHIB74SvtDlExRE,21580
|
|
23
|
-
tests/test_base_worker.py,sha256=2EIYcd_3f9O0zB5WiGIQV0Cn9wndLvnEnSfcAE1qWWU,30607
|
|
24
|
-
tests/test_cache.py,sha256=ii0gyr0DrG7ChEs7pmT8hMdSguAOAcCze4bRMiFQxuk,10640
|
|
25
|
-
tests/test_dataset_worker.py,sha256=gApYz0LArHr1cNn079_fa_BQABF6RVQYuM1Tc4m3NsQ,22089
|
|
26
|
-
tests/test_element.py,sha256=2G9M15TLxQRmvrWM9Kw2ucnElh4kSv_oF_5FYwwAxTY,13181
|
|
27
|
-
tests/test_image.py,sha256=03E24JVa7TZJfuwQyfVEBe3RAq3R993IMl1AHXRr7zY,25497
|
|
28
|
-
tests/test_merge.py,sha256=TuOeUS0UCz66DPOQFFhc4NQBxIjZL9f5czi4XnvGrr4,8270
|
|
29
|
-
tests/test_utils.py,sha256=_WJUPnt-pM_TQ0er4yjPZy-u_LePrHq1lxwk_teky7M,2544
|
|
30
|
-
tests/test_elements_worker/__init__.py,sha256=Fh4nkbbyJSMv_VtjQxnWrOqTnxXaaWI8S9WU0VrzCHs,179
|
|
31
|
-
tests/test_elements_worker/test_classification.py,sha256=nya7veSPR_O9G41Enodp2-o6AifMBcaSTWJP2vXSSJ4,30133
|
|
32
|
-
tests/test_elements_worker/test_cli.py,sha256=a23i1pUDbXi23MUtbWwGEcLLrmc_YlrbDgOG3h66wLM,2620
|
|
33
|
-
tests/test_elements_worker/test_corpus.py,sha256=kscJyM8k1njYJJFGuvliVzn89lWh41mEyDCCawnp3W8,5483
|
|
34
|
-
tests/test_elements_worker/test_dataset.py,sha256=00IlOZv9YFlZ23rGXyR-HLbKLQxGelZ1Bf9lEZYA0IY,11412
|
|
35
|
-
tests/test_elements_worker/test_element.py,sha256=lb5tLjl0jsixX0OWVhBAaKLE9GKkBw79kFHDNGommaQ,12535
|
|
36
|
-
tests/test_elements_worker/test_element_create_multiple.py,sha256=arYFGmxc0517ZUii6k__G_UQQatuNIASTC8MXvUrSwk,21887
|
|
37
|
-
tests/test_elements_worker/test_element_create_single.py,sha256=Fa9zm12J2rQ3VrUe3yIlHAc7Vty_eQYb_YGnNPQB3IE,16697
|
|
38
|
-
tests/test_elements_worker/test_element_list_children.py,sha256=2zH4h663w3EduqpzQr-7bf9zIDzO1x2WxdUYYHsIHkI,31358
|
|
39
|
-
tests/test_elements_worker/test_element_list_parents.py,sha256=TXeGW-a3W-7GmB2QrhJH9mMnvxuybeAwQ4tL3iIxwXo,16734
|
|
40
|
-
tests/test_elements_worker/test_entity_create.py,sha256=9Tjr9KA2yo44VFV283q_cs6XbbVguUMDNfCj-DILSJg,29353
|
|
41
|
-
tests/test_elements_worker/test_entity_list_and_check.py,sha256=ENBLaqbXlRUDbHRvQla3080a0HJltrWAPYWNohUA9NU,4992
|
|
42
|
-
tests/test_elements_worker/test_image.py,sha256=BljMNKgec_9a5bzNzFpYZIvSbuvwsWDfdqLHVJaTa7M,2079
|
|
43
|
-
tests/test_elements_worker/test_metadata.py,sha256=Xfggy-vxw5DZ3hFKx3sB7OYb2d1tu1RiNK8fvKJIaBs,22294
|
|
44
|
-
tests/test_elements_worker/test_task.py,sha256=wTUWqN9UhfKmJn3IcFY75EW4I1ulRhisflmY1kmP47s,5574
|
|
45
|
-
tests/test_elements_worker/test_training.py,sha256=Qxi9EzGr_uKcn2Fh5aE6jNrq1K8QKLiOiSew4upASPs,8721
|
|
46
|
-
tests/test_elements_worker/test_transcription_create.py,sha256=yznO9B_BVsOR0Z_VY5ZL8gJp0ZPCz_4sPUs5dXtixAg,29281
|
|
47
|
-
tests/test_elements_worker/test_transcription_create_with_elements.py,sha256=tmcyglgssEqMnt1Mdy_u6X1m2wgLWTo_HdWst3GrK2k,33056
|
|
48
|
-
tests/test_elements_worker/test_transcription_list.py,sha256=ikz7HYPCoQWTdTRCd382SB-y-T2BbigPLlIcx5Eow-I,15324
|
|
49
|
-
tests/test_elements_worker/test_version.py,sha256=xqCgcgukTFJzkMgYfQG-8mTbu0o2fdYjWC07FktThfw,2125
|
|
50
|
-
tests/test_elements_worker/test_worker.py,sha256=fnFwkcDb6tx5i7lbelifeHD-BrGz6o5UBEojPCdtuAo,23474
|
|
51
|
-
worker-demo/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
52
|
-
worker-demo/tests/conftest.py,sha256=XzNMNeg6pmABUAH8jN6eZTlZSFGLYjS3-DTXjiRN6Yc,1002
|
|
53
|
-
worker-demo/tests/test_worker.py,sha256=3DLd4NRK4bfyatG5P_PK4k9P9tJHx9XQq5_ryFEEFVg,304
|
|
54
|
-
worker-demo/worker_demo/__init__.py,sha256=2BPomV8ZMNf3YXJgloatKeHQCE6QOkwmsHGkO6MkQuM,125
|
|
55
|
-
worker-demo/worker_demo/worker.py,sha256=Rt-DjWa5iBP08k58NDZMfeyPuFbtNcbX6nc5jFX7GNo,440
|
|
56
|
-
arkindex_base_worker-0.4.0rc5.dist-info/LICENSE,sha256=NVshRi1efwVezMfW7xXYLrdDr2Li1AfwfGOd5WuH1kQ,1063
|
|
57
|
-
arkindex_base_worker-0.4.0rc5.dist-info/METADATA,sha256=EFoMgnh4SUvYV0yIdBSfA7aoaZiTA7kYrJTrH2la3mY,3338
|
|
58
|
-
arkindex_base_worker-0.4.0rc5.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
|
|
59
|
-
arkindex_base_worker-0.4.0rc5.dist-info/top_level.txt,sha256=58NuslgxQC2vT4DiqZEgO4JqJRrYa2yeNI9QvkbfGQU,40
|
|
60
|
-
arkindex_base_worker-0.4.0rc5.dist-info/RECORD,,
|