arkindex-base-worker 0.4.0__py3-none-any.whl → 0.4.0a2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {arkindex_base_worker-0.4.0.dist-info → arkindex_base_worker-0.4.0a2.dist-info}/METADATA +13 -15
- arkindex_base_worker-0.4.0a2.dist-info/RECORD +51 -0
- {arkindex_base_worker-0.4.0.dist-info → arkindex_base_worker-0.4.0a2.dist-info}/WHEEL +1 -1
- arkindex_worker/cache.py +1 -1
- arkindex_worker/image.py +1 -120
- arkindex_worker/utils.py +0 -82
- arkindex_worker/worker/__init__.py +161 -46
- arkindex_worker/worker/base.py +11 -36
- arkindex_worker/worker/classification.py +18 -34
- arkindex_worker/worker/corpus.py +4 -21
- arkindex_worker/worker/dataset.py +1 -71
- arkindex_worker/worker/element.py +91 -352
- arkindex_worker/worker/entity.py +11 -11
- arkindex_worker/worker/metadata.py +9 -19
- arkindex_worker/worker/task.py +4 -5
- arkindex_worker/worker/training.py +6 -6
- arkindex_worker/worker/transcription.py +68 -89
- arkindex_worker/worker/version.py +1 -3
- tests/__init__.py +1 -1
- tests/conftest.py +45 -33
- tests/test_base_worker.py +3 -204
- tests/test_dataset_worker.py +4 -7
- tests/test_elements_worker/{test_classification.py → test_classifications.py} +61 -194
- tests/test_elements_worker/test_corpus.py +1 -32
- tests/test_elements_worker/test_dataset.py +1 -1
- tests/test_elements_worker/test_elements.py +2734 -0
- tests/test_elements_worker/{test_entity_create.py → test_entities.py} +160 -26
- tests/test_elements_worker/test_image.py +1 -2
- tests/test_elements_worker/test_metadata.py +99 -224
- tests/test_elements_worker/test_task.py +1 -1
- tests/test_elements_worker/test_training.py +2 -2
- tests/test_elements_worker/test_transcriptions.py +2102 -0
- tests/test_elements_worker/test_worker.py +280 -563
- tests/test_image.py +204 -429
- tests/test_merge.py +2 -1
- tests/test_utils.py +3 -66
- arkindex_base_worker-0.4.0.dist-info/RECORD +0 -61
- arkindex_worker/worker/process.py +0 -92
- tests/test_elements_worker/test_element.py +0 -427
- tests/test_elements_worker/test_element_create_multiple.py +0 -715
- tests/test_elements_worker/test_element_create_single.py +0 -528
- tests/test_elements_worker/test_element_list_children.py +0 -969
- tests/test_elements_worker/test_element_list_parents.py +0 -530
- tests/test_elements_worker/test_entity_list_and_check.py +0 -160
- tests/test_elements_worker/test_process.py +0 -89
- tests/test_elements_worker/test_transcription_create.py +0 -873
- tests/test_elements_worker/test_transcription_create_with_elements.py +0 -951
- tests/test_elements_worker/test_transcription_list.py +0 -450
- tests/test_elements_worker/test_version.py +0 -60
- {arkindex_base_worker-0.4.0.dist-info → arkindex_base_worker-0.4.0a2.dist-info}/LICENSE +0 -0
- {arkindex_base_worker-0.4.0.dist-info → arkindex_base_worker-0.4.0a2.dist-info}/top_level.txt +0 -0
tests/test_merge.py
CHANGED
|
@@ -161,7 +161,7 @@ def test_merge_from_worker(
|
|
|
161
161
|
"""
|
|
162
162
|
responses.add(
|
|
163
163
|
responses.GET,
|
|
164
|
-
"http://testserver/api/v1/task/my_task/",
|
|
164
|
+
"http://testserver/api/v1/task/my_task/from-agent/",
|
|
165
165
|
status=200,
|
|
166
166
|
json={"parents": ["first", "second"]},
|
|
167
167
|
)
|
|
@@ -181,6 +181,7 @@ def test_merge_from_worker(
|
|
|
181
181
|
(tmp_path / "my_task").mkdir()
|
|
182
182
|
mock_base_worker_with_cache.args = mock_base_worker_with_cache.parser.parse_args()
|
|
183
183
|
mock_base_worker_with_cache.configure()
|
|
184
|
+
mock_base_worker_with_cache.configure_cache()
|
|
184
185
|
# Store parent tasks IDs as attribute
|
|
185
186
|
assert mock_base_worker_with_cache.task_parents == ["first", "second"]
|
|
186
187
|
|
tests/test_utils.py
CHANGED
|
@@ -1,18 +1,15 @@
|
|
|
1
|
-
import
|
|
1
|
+
from pathlib import Path
|
|
2
2
|
|
|
3
3
|
import pytest
|
|
4
4
|
|
|
5
|
-
from arkindex_worker.cache import unsupported_cache
|
|
6
5
|
from arkindex_worker.utils import (
|
|
7
|
-
DEFAULT_BATCH_SIZE,
|
|
8
|
-
batch_publication,
|
|
9
6
|
close_delete_file,
|
|
10
7
|
extract_tar_zst_archive,
|
|
11
8
|
parse_source_id,
|
|
12
9
|
)
|
|
13
|
-
from tests import FIXTURES_DIR
|
|
14
10
|
|
|
15
|
-
|
|
11
|
+
FIXTURES = Path(__file__).absolute().parent / "data"
|
|
12
|
+
ARCHIVE = FIXTURES / "archive.tar.zst"
|
|
16
13
|
|
|
17
14
|
|
|
18
15
|
@pytest.mark.parametrize(
|
|
@@ -58,63 +55,3 @@ def test_close_delete_file(tmp_path):
|
|
|
58
55
|
close_delete_file(archive_fd, archive_path)
|
|
59
56
|
|
|
60
57
|
assert not archive_path.exists()
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
class TestMixin:
|
|
64
|
-
def __init__(self, use_cache: bool = False):
|
|
65
|
-
"""
|
|
66
|
-
Args:
|
|
67
|
-
use_cache (bool, optional): To mock BaseWorker.use_cache attribute. Defaults to False.
|
|
68
|
-
"""
|
|
69
|
-
self.use_cache = use_cache
|
|
70
|
-
|
|
71
|
-
@batch_publication
|
|
72
|
-
def custom_publication_in_batches(self, batch_size: int = DEFAULT_BATCH_SIZE):
|
|
73
|
-
return batch_size
|
|
74
|
-
|
|
75
|
-
@unsupported_cache
|
|
76
|
-
@batch_publication
|
|
77
|
-
def custom_publication_in_batches_without_cache(
|
|
78
|
-
self, batch_size: int = DEFAULT_BATCH_SIZE
|
|
79
|
-
):
|
|
80
|
-
return batch_size
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
def test_batch_publication_decorator_no_parameter():
|
|
84
|
-
assert TestMixin().custom_publication_in_batches() == DEFAULT_BATCH_SIZE
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
@pytest.mark.parametrize("wrong_batch_size", [None, "not an int", 0])
|
|
88
|
-
def test_batch_publication_decorator_wrong_parameter(wrong_batch_size):
|
|
89
|
-
with pytest.raises(
|
|
90
|
-
AssertionError,
|
|
91
|
-
match="batch_size shouldn't be null and should be a strictly positive integer",
|
|
92
|
-
):
|
|
93
|
-
TestMixin().custom_publication_in_batches(batch_size=wrong_batch_size)
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
@pytest.mark.parametrize("batch_size", [1, 10, DEFAULT_BATCH_SIZE])
|
|
97
|
-
def test_batch_publication_decorator_right_parameter(batch_size):
|
|
98
|
-
assert (
|
|
99
|
-
TestMixin().custom_publication_in_batches(batch_size=batch_size) == batch_size
|
|
100
|
-
)
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
def test_batch_publication_decorator_alongside_unsupported_cache(caplog):
|
|
104
|
-
# Capture log messages
|
|
105
|
-
caplog.clear()
|
|
106
|
-
with caplog.at_level(logging.WARNING):
|
|
107
|
-
# Call the helper
|
|
108
|
-
assert (
|
|
109
|
-
TestMixin(use_cache=True).custom_publication_in_batches_without_cache()
|
|
110
|
-
== DEFAULT_BATCH_SIZE
|
|
111
|
-
)
|
|
112
|
-
|
|
113
|
-
# Check logs
|
|
114
|
-
assert caplog.record_tuples == [
|
|
115
|
-
(
|
|
116
|
-
"arkindex_worker",
|
|
117
|
-
logging.WARNING,
|
|
118
|
-
"This API helper `custom_publication_in_batches_without_cache` did not update the cache database",
|
|
119
|
-
),
|
|
120
|
-
]
|
|
@@ -1,61 +0,0 @@
|
|
|
1
|
-
arkindex_worker/__init__.py,sha256=OlgCtTC9MaWeejviY0a3iQpALcRQGMVArFVVYwTF6I8,162
|
|
2
|
-
arkindex_worker/cache.py,sha256=qTblc_zKdYC47Wip6_O9Jf5qBkQW2ozQQrg-nsx1WuY,11221
|
|
3
|
-
arkindex_worker/image.py,sha256=oEgVCrSHiGh3D5-UXfM6PvT17TttSxC0115irpvB3Dw,18581
|
|
4
|
-
arkindex_worker/models.py,sha256=bPQzGZNs5a6z6DEcygsa8T33VOqPlMUbwKzHqlKzwbw,9923
|
|
5
|
-
arkindex_worker/utils.py,sha256=MbbJT8oh8DMHHR-vidFeXdUH0TSXGWm7ZDGWzrRXoEY,9933
|
|
6
|
-
arkindex_worker/worker/__init__.py,sha256=0_YHeOe31KR_8ynbnYMIMwnSQTVbKkkeLGmnlTMhFx0,16234
|
|
7
|
-
arkindex_worker/worker/base.py,sha256=7Pmw-UQSxV-xkW8NO5cXsxJ8W8szzyppMaNjq_az81A,19844
|
|
8
|
-
arkindex_worker/worker/classification.py,sha256=zECSNzGCZFzoPoDVZN4kuGYRNLzMQLBaRt3q1jnBSaA,10952
|
|
9
|
-
arkindex_worker/worker/corpus.py,sha256=0TQFOwZ6Te-CZi6lgkZY1wzyJ5wO9LAmcVQtqHvZpPk,2291
|
|
10
|
-
arkindex_worker/worker/dataset.py,sha256=LwzKwNFX4FqfLxh29LSvJydPwRw3VHaB1wjuFhUshsE,5267
|
|
11
|
-
arkindex_worker/worker/element.py,sha256=Qvvq9kJnAHNATHW7zi96eIY1x-0MsR-T5rrSJg6e9Y4,45309
|
|
12
|
-
arkindex_worker/worker/entity.py,sha256=DG8oVAdy-r18fliTjnzGI1j6l7SOFmyIBmE6JlE6A8g,14799
|
|
13
|
-
arkindex_worker/worker/image.py,sha256=t_Az6IGnj0EZyvcA4XxfPikOUjn_pztgsyxTkFZhaXU,621
|
|
14
|
-
arkindex_worker/worker/metadata.py,sha256=VRajtd2kaBvar9GercX4knvR6l1WFYjoCdJWU9ccKgk,7291
|
|
15
|
-
arkindex_worker/worker/process.py,sha256=9TEHpMcBax1wc6PrWMMrdXe2uNfqyVj7n_dAYZRBGnY,1854
|
|
16
|
-
arkindex_worker/worker/task.py,sha256=r1j7_qbdNu2Z8H8HbGzO3P3qdx-2N1pBbUPFDca0rqg,1519
|
|
17
|
-
arkindex_worker/worker/training.py,sha256=H8FmCdzGcDW-WMMwcgvmZPlN5tPHwGo0BXn12qmzj8g,10875
|
|
18
|
-
arkindex_worker/worker/transcription.py,sha256=52RY9kYsiR1sz9FxOigyo12Ker3VDbQ4U42gK9DpR3g,21146
|
|
19
|
-
arkindex_worker/worker/version.py,sha256=JIT7OI3Mo7RPkNrjOB9hfqrsG-FYygz_zi4l8PbkuAo,1960
|
|
20
|
-
hooks/pre_gen_project.py,sha256=xQJERv3vv9VzIqcBHI281eeWLWREXUF4mMw7PvJHHXM,269
|
|
21
|
-
tests/__init__.py,sha256=DG--S6IpGl399rzSAjDdHL76CkOIeZIjajCcyUSDhOQ,241
|
|
22
|
-
tests/conftest.py,sha256=Z9amrKmVtFltzTUUm07fGDrT4m540biaTpjedmplyzc,21536
|
|
23
|
-
tests/test_base_worker.py,sha256=2EIYcd_3f9O0zB5WiGIQV0Cn9wndLvnEnSfcAE1qWWU,30607
|
|
24
|
-
tests/test_cache.py,sha256=ii0gyr0DrG7ChEs7pmT8hMdSguAOAcCze4bRMiFQxuk,10640
|
|
25
|
-
tests/test_dataset_worker.py,sha256=z8ydliUlwW2j-irgLAotJMacgJXkVvF5TgsWLyCn1Jo,22087
|
|
26
|
-
tests/test_element.py,sha256=2G9M15TLxQRmvrWM9Kw2ucnElh4kSv_oF_5FYwwAxTY,13181
|
|
27
|
-
tests/test_image.py,sha256=nFxf7fnh8DyYF6ZWhvDAtN5rEk96zdJEm03vMiyz7pA,25495
|
|
28
|
-
tests/test_merge.py,sha256=TuOeUS0UCz66DPOQFFhc4NQBxIjZL9f5czi4XnvGrr4,8270
|
|
29
|
-
tests/test_utils.py,sha256=nYL1s2ViZoLoMiNpLGDaWwxf8dJ1D8aT522AO-PVaEQ,3607
|
|
30
|
-
tests/test_elements_worker/__init__.py,sha256=Fh4nkbbyJSMv_VtjQxnWrOqTnxXaaWI8S9WU0VrzCHs,179
|
|
31
|
-
tests/test_elements_worker/test_classification.py,sha256=nya7veSPR_O9G41Enodp2-o6AifMBcaSTWJP2vXSSJ4,30133
|
|
32
|
-
tests/test_elements_worker/test_cli.py,sha256=a23i1pUDbXi23MUtbWwGEcLLrmc_YlrbDgOG3h66wLM,2620
|
|
33
|
-
tests/test_elements_worker/test_corpus.py,sha256=kscJyM8k1njYJJFGuvliVzn89lWh41mEyDCCawnp3W8,5483
|
|
34
|
-
tests/test_elements_worker/test_dataset.py,sha256=00IlOZv9YFlZ23rGXyR-HLbKLQxGelZ1Bf9lEZYA0IY,11412
|
|
35
|
-
tests/test_elements_worker/test_element.py,sha256=lb5tLjl0jsixX0OWVhBAaKLE9GKkBw79kFHDNGommaQ,12535
|
|
36
|
-
tests/test_elements_worker/test_element_create_multiple.py,sha256=arYFGmxc0517ZUii6k__G_UQQatuNIASTC8MXvUrSwk,21887
|
|
37
|
-
tests/test_elements_worker/test_element_create_single.py,sha256=Fa9zm12J2rQ3VrUe3yIlHAc7Vty_eQYb_YGnNPQB3IE,16697
|
|
38
|
-
tests/test_elements_worker/test_element_list_children.py,sha256=2zH4h663w3EduqpzQr-7bf9zIDzO1x2WxdUYYHsIHkI,31358
|
|
39
|
-
tests/test_elements_worker/test_element_list_parents.py,sha256=TXeGW-a3W-7GmB2QrhJH9mMnvxuybeAwQ4tL3iIxwXo,16734
|
|
40
|
-
tests/test_elements_worker/test_entity_create.py,sha256=9Tjr9KA2yo44VFV283q_cs6XbbVguUMDNfCj-DILSJg,29353
|
|
41
|
-
tests/test_elements_worker/test_entity_list_and_check.py,sha256=ENBLaqbXlRUDbHRvQla3080a0HJltrWAPYWNohUA9NU,4992
|
|
42
|
-
tests/test_elements_worker/test_image.py,sha256=BljMNKgec_9a5bzNzFpYZIvSbuvwsWDfdqLHVJaTa7M,2079
|
|
43
|
-
tests/test_elements_worker/test_metadata.py,sha256=Xfggy-vxw5DZ3hFKx3sB7OYb2d1tu1RiNK8fvKJIaBs,22294
|
|
44
|
-
tests/test_elements_worker/test_process.py,sha256=y4RoVhPfyHzR795fw7-_FXElBcKo3fy4Ew_HI-kxJic,3088
|
|
45
|
-
tests/test_elements_worker/test_task.py,sha256=wTUWqN9UhfKmJn3IcFY75EW4I1ulRhisflmY1kmP47s,5574
|
|
46
|
-
tests/test_elements_worker/test_training.py,sha256=3W2LzpqxekvRiX42m_PvWcVel7ynQJmzO8gKcLmCMQI,8717
|
|
47
|
-
tests/test_elements_worker/test_transcription_create.py,sha256=yznO9B_BVsOR0Z_VY5ZL8gJp0ZPCz_4sPUs5dXtixAg,29281
|
|
48
|
-
tests/test_elements_worker/test_transcription_create_with_elements.py,sha256=tmcyglgssEqMnt1Mdy_u6X1m2wgLWTo_HdWst3GrK2k,33056
|
|
49
|
-
tests/test_elements_worker/test_transcription_list.py,sha256=ikz7HYPCoQWTdTRCd382SB-y-T2BbigPLlIcx5Eow-I,15324
|
|
50
|
-
tests/test_elements_worker/test_version.py,sha256=xqCgcgukTFJzkMgYfQG-8mTbu0o2fdYjWC07FktThfw,2125
|
|
51
|
-
tests/test_elements_worker/test_worker.py,sha256=pLUgjyrrXrzVD6T-kdH1ppk5Yn_iDuI8JdFGweTEMXE,25156
|
|
52
|
-
worker-demo/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
53
|
-
worker-demo/tests/conftest.py,sha256=XzNMNeg6pmABUAH8jN6eZTlZSFGLYjS3-DTXjiRN6Yc,1002
|
|
54
|
-
worker-demo/tests/test_worker.py,sha256=3DLd4NRK4bfyatG5P_PK4k9P9tJHx9XQq5_ryFEEFVg,304
|
|
55
|
-
worker-demo/worker_demo/__init__.py,sha256=2BPomV8ZMNf3YXJgloatKeHQCE6QOkwmsHGkO6MkQuM,125
|
|
56
|
-
worker-demo/worker_demo/worker.py,sha256=Rt-DjWa5iBP08k58NDZMfeyPuFbtNcbX6nc5jFX7GNo,440
|
|
57
|
-
arkindex_base_worker-0.4.0.dist-info/LICENSE,sha256=NVshRi1efwVezMfW7xXYLrdDr2Li1AfwfGOd5WuH1kQ,1063
|
|
58
|
-
arkindex_base_worker-0.4.0.dist-info/METADATA,sha256=OjqRHv4B7-BiEwyDeDgWt6HhBWJ7AkpBe6qCCHnG_W0,3334
|
|
59
|
-
arkindex_base_worker-0.4.0.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
|
|
60
|
-
arkindex_base_worker-0.4.0.dist-info/top_level.txt,sha256=58NuslgxQC2vT4DiqZEgO4JqJRrYa2yeNI9QvkbfGQU,40
|
|
61
|
-
arkindex_base_worker-0.4.0.dist-info/RECORD,,
|
|
@@ -1,92 +0,0 @@
|
|
|
1
|
-
from collections.abc import Iterator
|
|
2
|
-
from enum import Enum
|
|
3
|
-
|
|
4
|
-
from arkindex_worker.cache import unsupported_cache
|
|
5
|
-
|
|
6
|
-
# Increases the number of elements returned per page by the API
|
|
7
|
-
PROCESS_ELEMENTS_PAGE_SIZE = 500
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
class ActivityState(Enum):
|
|
11
|
-
"""
|
|
12
|
-
Processing state of an element.
|
|
13
|
-
"""
|
|
14
|
-
|
|
15
|
-
Queued = "queued"
|
|
16
|
-
"""
|
|
17
|
-
The element has not yet been processed by a worker.
|
|
18
|
-
"""
|
|
19
|
-
|
|
20
|
-
Started = "started"
|
|
21
|
-
"""
|
|
22
|
-
The element is being processed by a worker.
|
|
23
|
-
"""
|
|
24
|
-
|
|
25
|
-
Processed = "processed"
|
|
26
|
-
"""
|
|
27
|
-
The element has been successfully processed by a worker.
|
|
28
|
-
"""
|
|
29
|
-
|
|
30
|
-
Error = "error"
|
|
31
|
-
"""
|
|
32
|
-
An error occurred while processing this element.
|
|
33
|
-
"""
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
class ProcessMode(Enum):
|
|
37
|
-
"""
|
|
38
|
-
Mode of the process of the worker.
|
|
39
|
-
"""
|
|
40
|
-
|
|
41
|
-
Files = "files"
|
|
42
|
-
"""
|
|
43
|
-
Processes of files (images, PDFs, IIIF, ...) imports.
|
|
44
|
-
"""
|
|
45
|
-
|
|
46
|
-
Workers = "workers"
|
|
47
|
-
"""
|
|
48
|
-
Processes of worker executions.
|
|
49
|
-
"""
|
|
50
|
-
|
|
51
|
-
Template = "template"
|
|
52
|
-
"""
|
|
53
|
-
Process templates.
|
|
54
|
-
"""
|
|
55
|
-
|
|
56
|
-
S3 = "s3"
|
|
57
|
-
"""
|
|
58
|
-
Processes of imports from an S3-compatible storage.
|
|
59
|
-
"""
|
|
60
|
-
|
|
61
|
-
Local = "local"
|
|
62
|
-
"""
|
|
63
|
-
Local processes.
|
|
64
|
-
"""
|
|
65
|
-
|
|
66
|
-
Dataset = "dataset"
|
|
67
|
-
"""
|
|
68
|
-
Dataset processes.
|
|
69
|
-
"""
|
|
70
|
-
|
|
71
|
-
Export = "export"
|
|
72
|
-
"""
|
|
73
|
-
Export processes.
|
|
74
|
-
"""
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
class ProcessMixin:
|
|
78
|
-
@unsupported_cache
|
|
79
|
-
def list_process_elements(self, with_image: bool = False) -> Iterator[dict]:
|
|
80
|
-
"""
|
|
81
|
-
List the elements of a process.
|
|
82
|
-
|
|
83
|
-
:param with_image: whether or not to include zone and image information in the elements response.
|
|
84
|
-
:returns: the process' elements.
|
|
85
|
-
"""
|
|
86
|
-
return self.api_client.paginate(
|
|
87
|
-
"ListProcessElements",
|
|
88
|
-
id=self.process_information["id"],
|
|
89
|
-
with_image=with_image,
|
|
90
|
-
allow_missing_data=True,
|
|
91
|
-
page_size=PROCESS_ELEMENTS_PAGE_SIZE,
|
|
92
|
-
)
|
|
@@ -1,427 +0,0 @@
|
|
|
1
|
-
import json
|
|
2
|
-
import re
|
|
3
|
-
|
|
4
|
-
import pytest
|
|
5
|
-
from responses import matchers
|
|
6
|
-
|
|
7
|
-
from arkindex.exceptions import ErrorResponse
|
|
8
|
-
from arkindex_worker.cache import (
|
|
9
|
-
CachedElement,
|
|
10
|
-
CachedImage,
|
|
11
|
-
)
|
|
12
|
-
from arkindex_worker.models import Element
|
|
13
|
-
from arkindex_worker.worker.element import MissingTypeError
|
|
14
|
-
from tests import CORPUS_ID
|
|
15
|
-
|
|
16
|
-
from . import BASE_API_CALLS
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
def test_list_corpus_types(responses, mock_elements_worker):
|
|
20
|
-
responses.add(
|
|
21
|
-
responses.GET,
|
|
22
|
-
f"http://testserver/api/v1/corpus/{CORPUS_ID}/",
|
|
23
|
-
json={
|
|
24
|
-
"id": CORPUS_ID,
|
|
25
|
-
"types": [{"slug": "folder"}, {"slug": "page"}],
|
|
26
|
-
},
|
|
27
|
-
)
|
|
28
|
-
|
|
29
|
-
mock_elements_worker.list_corpus_types()
|
|
30
|
-
|
|
31
|
-
assert mock_elements_worker.corpus_types == {
|
|
32
|
-
"folder": {"slug": "folder"},
|
|
33
|
-
"page": {"slug": "page"},
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
def test_check_required_types_argument_types(mock_elements_worker):
|
|
38
|
-
with pytest.raises(
|
|
39
|
-
AssertionError, match="At least one element type slug is required."
|
|
40
|
-
):
|
|
41
|
-
mock_elements_worker.check_required_types()
|
|
42
|
-
|
|
43
|
-
with pytest.raises(AssertionError, match="Element type slugs must be strings."):
|
|
44
|
-
mock_elements_worker.check_required_types("lol", 42)
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
def test_check_required_types(mock_elements_worker):
|
|
48
|
-
mock_elements_worker.corpus_types = {
|
|
49
|
-
"folder": {"slug": "folder"},
|
|
50
|
-
"page": {"slug": "page"},
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
assert mock_elements_worker.check_required_types("page")
|
|
54
|
-
assert mock_elements_worker.check_required_types("page", "folder")
|
|
55
|
-
|
|
56
|
-
with pytest.raises(
|
|
57
|
-
MissingTypeError,
|
|
58
|
-
match=re.escape(
|
|
59
|
-
"Element types act, text_line were not found in corpus (11111111-1111-1111-1111-111111111111)."
|
|
60
|
-
),
|
|
61
|
-
):
|
|
62
|
-
assert mock_elements_worker.check_required_types("page", "text_line", "act")
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
def test_check_required_types_create_missing(responses, mock_elements_worker):
|
|
66
|
-
mock_elements_worker.corpus_types = {
|
|
67
|
-
"folder": {"slug": "folder"},
|
|
68
|
-
"page": {"slug": "page"},
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
responses.add(
|
|
72
|
-
responses.POST,
|
|
73
|
-
"http://testserver/api/v1/elements/type/",
|
|
74
|
-
match=[
|
|
75
|
-
matchers.json_params_matcher(
|
|
76
|
-
{
|
|
77
|
-
"slug": "text_line",
|
|
78
|
-
"display_name": "text_line",
|
|
79
|
-
"folder": False,
|
|
80
|
-
"corpus": CORPUS_ID,
|
|
81
|
-
}
|
|
82
|
-
)
|
|
83
|
-
],
|
|
84
|
-
)
|
|
85
|
-
responses.add(
|
|
86
|
-
responses.POST,
|
|
87
|
-
"http://testserver/api/v1/elements/type/",
|
|
88
|
-
match=[
|
|
89
|
-
matchers.json_params_matcher(
|
|
90
|
-
{
|
|
91
|
-
"slug": "act",
|
|
92
|
-
"display_name": "act",
|
|
93
|
-
"folder": False,
|
|
94
|
-
"corpus": CORPUS_ID,
|
|
95
|
-
}
|
|
96
|
-
)
|
|
97
|
-
],
|
|
98
|
-
)
|
|
99
|
-
|
|
100
|
-
assert mock_elements_worker.check_required_types(
|
|
101
|
-
"page", "text_line", "act", create_missing=True
|
|
102
|
-
)
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
@pytest.mark.parametrize(
|
|
106
|
-
("payload", "error"),
|
|
107
|
-
[
|
|
108
|
-
# Element
|
|
109
|
-
(
|
|
110
|
-
{"element": None},
|
|
111
|
-
"element shouldn't be null and should be an Element or CachedElement",
|
|
112
|
-
),
|
|
113
|
-
(
|
|
114
|
-
{"element": "not element type"},
|
|
115
|
-
"element shouldn't be null and should be an Element or CachedElement",
|
|
116
|
-
),
|
|
117
|
-
],
|
|
118
|
-
)
|
|
119
|
-
def test_partial_update_element_wrong_param_element(
|
|
120
|
-
mock_elements_worker, payload, error
|
|
121
|
-
):
|
|
122
|
-
api_payload = {
|
|
123
|
-
"element": Element({"zone": None}),
|
|
124
|
-
**payload,
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
with pytest.raises(AssertionError, match=error):
|
|
128
|
-
mock_elements_worker.partial_update_element(
|
|
129
|
-
**api_payload,
|
|
130
|
-
)
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
@pytest.mark.parametrize(
|
|
134
|
-
("payload", "error"),
|
|
135
|
-
[
|
|
136
|
-
# Type
|
|
137
|
-
({"type": 1234}, "type should be a str"),
|
|
138
|
-
({"type": None}, "type should be a str"),
|
|
139
|
-
],
|
|
140
|
-
)
|
|
141
|
-
def test_partial_update_element_wrong_param_type(mock_elements_worker, payload, error):
|
|
142
|
-
api_payload = {
|
|
143
|
-
"element": Element({"zone": None}),
|
|
144
|
-
**payload,
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
with pytest.raises(AssertionError, match=error):
|
|
148
|
-
mock_elements_worker.partial_update_element(
|
|
149
|
-
**api_payload,
|
|
150
|
-
)
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
@pytest.mark.parametrize(
|
|
154
|
-
("payload", "error"),
|
|
155
|
-
[
|
|
156
|
-
# Name
|
|
157
|
-
({"name": 1234}, "name should be a str"),
|
|
158
|
-
({"name": None}, "name should be a str"),
|
|
159
|
-
],
|
|
160
|
-
)
|
|
161
|
-
def test_partial_update_element_wrong_param_name(mock_elements_worker, payload, error):
|
|
162
|
-
api_payload = {
|
|
163
|
-
"element": Element({"zone": None}),
|
|
164
|
-
**payload,
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
with pytest.raises(AssertionError, match=error):
|
|
168
|
-
mock_elements_worker.partial_update_element(
|
|
169
|
-
**api_payload,
|
|
170
|
-
)
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
@pytest.mark.parametrize(
|
|
174
|
-
("payload", "error"),
|
|
175
|
-
[
|
|
176
|
-
# Polygon
|
|
177
|
-
({"polygon": "not a polygon"}, "polygon should be a list"),
|
|
178
|
-
({"polygon": None}, "polygon should be a list"),
|
|
179
|
-
({"polygon": [[1, 1], [2, 2]]}, "polygon should have at least three points"),
|
|
180
|
-
(
|
|
181
|
-
{"polygon": [[1, 1, 1], [2, 2, 1], [2, 1, 1], [1, 2, 1]]},
|
|
182
|
-
"polygon points should be lists of two items",
|
|
183
|
-
),
|
|
184
|
-
(
|
|
185
|
-
{"polygon": [[1], [2], [2], [1]]},
|
|
186
|
-
"polygon points should be lists of two items",
|
|
187
|
-
),
|
|
188
|
-
(
|
|
189
|
-
{"polygon": [["not a coord", 1], [2, 2], [2, 1], [1, 2]]},
|
|
190
|
-
"polygon points should be lists of two numbers",
|
|
191
|
-
),
|
|
192
|
-
],
|
|
193
|
-
)
|
|
194
|
-
def test_partial_update_element_wrong_param_polygon(
|
|
195
|
-
mock_elements_worker, payload, error
|
|
196
|
-
):
|
|
197
|
-
api_payload = {
|
|
198
|
-
"element": Element({"zone": None}),
|
|
199
|
-
**payload,
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
with pytest.raises(AssertionError, match=error):
|
|
203
|
-
mock_elements_worker.partial_update_element(
|
|
204
|
-
**api_payload,
|
|
205
|
-
)
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
@pytest.mark.parametrize(
|
|
209
|
-
("payload", "error"),
|
|
210
|
-
[
|
|
211
|
-
# Confidence
|
|
212
|
-
({"confidence": "lol"}, "confidence should be None or a float in [0..1] range"),
|
|
213
|
-
({"confidence": "0.2"}, "confidence should be None or a float in [0..1] range"),
|
|
214
|
-
({"confidence": -1.0}, "confidence should be None or a float in [0..1] range"),
|
|
215
|
-
({"confidence": 1.42}, "confidence should be None or a float in [0..1] range"),
|
|
216
|
-
(
|
|
217
|
-
{"confidence": float("inf")},
|
|
218
|
-
"confidence should be None or a float in [0..1] range",
|
|
219
|
-
),
|
|
220
|
-
],
|
|
221
|
-
)
|
|
222
|
-
def test_partial_update_element_wrong_param_conf(mock_elements_worker, payload, error):
|
|
223
|
-
api_payload = {
|
|
224
|
-
"element": Element({"zone": None}),
|
|
225
|
-
**payload,
|
|
226
|
-
}
|
|
227
|
-
|
|
228
|
-
with pytest.raises(AssertionError, match=re.escape(error)):
|
|
229
|
-
mock_elements_worker.partial_update_element(
|
|
230
|
-
**api_payload,
|
|
231
|
-
)
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
@pytest.mark.parametrize(
|
|
235
|
-
("payload", "error"),
|
|
236
|
-
[
|
|
237
|
-
# Rotation angle
|
|
238
|
-
({"rotation_angle": "lol"}, "rotation_angle should be a positive integer"),
|
|
239
|
-
({"rotation_angle": -1}, "rotation_angle should be a positive integer"),
|
|
240
|
-
({"rotation_angle": 0.5}, "rotation_angle should be a positive integer"),
|
|
241
|
-
({"rotation_angle": None}, "rotation_angle should be a positive integer"),
|
|
242
|
-
],
|
|
243
|
-
)
|
|
244
|
-
def test_partial_update_element_wrong_param_rota(mock_elements_worker, payload, error):
|
|
245
|
-
api_payload = {
|
|
246
|
-
"element": Element({"zone": None}),
|
|
247
|
-
**payload,
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
with pytest.raises(AssertionError, match=error):
|
|
251
|
-
mock_elements_worker.partial_update_element(
|
|
252
|
-
**api_payload,
|
|
253
|
-
)
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
@pytest.mark.parametrize(
|
|
257
|
-
("payload", "error"),
|
|
258
|
-
[
|
|
259
|
-
# Mirrored
|
|
260
|
-
({"mirrored": "lol"}, "mirrored should be a boolean"),
|
|
261
|
-
({"mirrored": 1234}, "mirrored should be a boolean"),
|
|
262
|
-
({"mirrored": None}, "mirrored should be a boolean"),
|
|
263
|
-
],
|
|
264
|
-
)
|
|
265
|
-
def test_partial_update_element_wrong_param_mir(mock_elements_worker, payload, error):
|
|
266
|
-
api_payload = {
|
|
267
|
-
"element": Element({"zone": None}),
|
|
268
|
-
**payload,
|
|
269
|
-
}
|
|
270
|
-
|
|
271
|
-
with pytest.raises(AssertionError, match=error):
|
|
272
|
-
mock_elements_worker.partial_update_element(
|
|
273
|
-
**api_payload,
|
|
274
|
-
)
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
@pytest.mark.parametrize(
|
|
278
|
-
("payload", "error"),
|
|
279
|
-
[
|
|
280
|
-
# Image
|
|
281
|
-
({"image": "lol"}, "image should be a UUID"),
|
|
282
|
-
({"image": 1234}, "image should be a UUID"),
|
|
283
|
-
({"image": None}, "image should be a UUID"),
|
|
284
|
-
],
|
|
285
|
-
)
|
|
286
|
-
def test_partial_update_element_wrong_param_image(mock_elements_worker, payload, error):
|
|
287
|
-
api_payload = {
|
|
288
|
-
"element": Element({"zone": None}),
|
|
289
|
-
**payload,
|
|
290
|
-
}
|
|
291
|
-
|
|
292
|
-
with pytest.raises(AssertionError, match=error):
|
|
293
|
-
mock_elements_worker.partial_update_element(
|
|
294
|
-
**api_payload,
|
|
295
|
-
)
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
def test_partial_update_element_api_error(responses, mock_elements_worker):
|
|
299
|
-
elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
|
|
300
|
-
responses.add(
|
|
301
|
-
responses.PATCH,
|
|
302
|
-
f"http://testserver/api/v1/element/{elt.id}/",
|
|
303
|
-
status=418,
|
|
304
|
-
)
|
|
305
|
-
|
|
306
|
-
with pytest.raises(ErrorResponse):
|
|
307
|
-
mock_elements_worker.partial_update_element(
|
|
308
|
-
element=elt,
|
|
309
|
-
type="something",
|
|
310
|
-
name="0",
|
|
311
|
-
polygon=[[1, 1], [2, 2], [2, 1], [1, 2]],
|
|
312
|
-
)
|
|
313
|
-
|
|
314
|
-
assert len(responses.calls) == len(BASE_API_CALLS) + 1
|
|
315
|
-
assert [
|
|
316
|
-
(call.request.method, call.request.url) for call in responses.calls
|
|
317
|
-
] == BASE_API_CALLS + [("PATCH", f"http://testserver/api/v1/element/{elt.id}/")]
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
@pytest.mark.usefixtures("_mock_cached_elements", "_mock_cached_images")
|
|
321
|
-
@pytest.mark.parametrize(
|
|
322
|
-
"payload",
|
|
323
|
-
[
|
|
324
|
-
(
|
|
325
|
-
{
|
|
326
|
-
"polygon": [[10, 10], [20, 20], [20, 10], [10, 20]],
|
|
327
|
-
"confidence": None,
|
|
328
|
-
}
|
|
329
|
-
),
|
|
330
|
-
(
|
|
331
|
-
{
|
|
332
|
-
"rotation_angle": 45,
|
|
333
|
-
"mirrored": False,
|
|
334
|
-
}
|
|
335
|
-
),
|
|
336
|
-
(
|
|
337
|
-
{
|
|
338
|
-
"polygon": [[10, 10], [20, 20], [20, 10], [10, 20]],
|
|
339
|
-
"confidence": None,
|
|
340
|
-
"rotation_angle": 45,
|
|
341
|
-
"mirrored": False,
|
|
342
|
-
}
|
|
343
|
-
),
|
|
344
|
-
],
|
|
345
|
-
)
|
|
346
|
-
def test_partial_update_element(responses, mock_elements_worker_with_cache, payload):
|
|
347
|
-
elt = CachedElement.select().first()
|
|
348
|
-
new_image = CachedImage.select().first()
|
|
349
|
-
|
|
350
|
-
elt_response = {
|
|
351
|
-
"image": str(new_image.id),
|
|
352
|
-
**payload,
|
|
353
|
-
}
|
|
354
|
-
responses.add(
|
|
355
|
-
responses.PATCH,
|
|
356
|
-
f"http://testserver/api/v1/element/{elt.id}/",
|
|
357
|
-
status=200,
|
|
358
|
-
# UUID not allowed in JSON
|
|
359
|
-
json=elt_response,
|
|
360
|
-
)
|
|
361
|
-
|
|
362
|
-
element_update_response = mock_elements_worker_with_cache.partial_update_element(
|
|
363
|
-
element=elt,
|
|
364
|
-
**{**elt_response, "image": new_image.id},
|
|
365
|
-
)
|
|
366
|
-
|
|
367
|
-
assert len(responses.calls) == len(BASE_API_CALLS) + 1
|
|
368
|
-
assert [
|
|
369
|
-
(call.request.method, call.request.url) for call in responses.calls
|
|
370
|
-
] == BASE_API_CALLS + [
|
|
371
|
-
(
|
|
372
|
-
"PATCH",
|
|
373
|
-
f"http://testserver/api/v1/element/{elt.id}/",
|
|
374
|
-
),
|
|
375
|
-
]
|
|
376
|
-
assert json.loads(responses.calls[-1].request.body) == elt_response
|
|
377
|
-
assert element_update_response == elt_response
|
|
378
|
-
|
|
379
|
-
cached_element = CachedElement.get(CachedElement.id == elt.id)
|
|
380
|
-
# Always present in payload
|
|
381
|
-
assert str(cached_element.image_id) == elt_response["image"]
|
|
382
|
-
# Optional params
|
|
383
|
-
if "polygon" in payload:
|
|
384
|
-
# Cast to string as this is the only difference compared to model
|
|
385
|
-
elt_response["polygon"] = str(elt_response["polygon"])
|
|
386
|
-
|
|
387
|
-
for param in payload:
|
|
388
|
-
assert getattr(cached_element, param) == elt_response[param]
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
@pytest.mark.usefixtures("_mock_cached_elements")
|
|
392
|
-
@pytest.mark.parametrize("confidence", [None, 0.42])
|
|
393
|
-
def test_partial_update_element_confidence(
|
|
394
|
-
responses, mock_elements_worker_with_cache, confidence
|
|
395
|
-
):
|
|
396
|
-
elt = CachedElement.select().first()
|
|
397
|
-
elt_response = {
|
|
398
|
-
"polygon": [[10, 10], [20, 20], [20, 10], [10, 20]],
|
|
399
|
-
"confidence": confidence,
|
|
400
|
-
}
|
|
401
|
-
responses.add(
|
|
402
|
-
responses.PATCH,
|
|
403
|
-
f"http://testserver/api/v1/element/{elt.id}/",
|
|
404
|
-
status=200,
|
|
405
|
-
json=elt_response,
|
|
406
|
-
)
|
|
407
|
-
|
|
408
|
-
element_update_response = mock_elements_worker_with_cache.partial_update_element(
|
|
409
|
-
element=elt,
|
|
410
|
-
**elt_response,
|
|
411
|
-
)
|
|
412
|
-
|
|
413
|
-
assert len(responses.calls) == len(BASE_API_CALLS) + 1
|
|
414
|
-
assert [
|
|
415
|
-
(call.request.method, call.request.url) for call in responses.calls
|
|
416
|
-
] == BASE_API_CALLS + [
|
|
417
|
-
(
|
|
418
|
-
"PATCH",
|
|
419
|
-
f"http://testserver/api/v1/element/{elt.id}/",
|
|
420
|
-
),
|
|
421
|
-
]
|
|
422
|
-
assert json.loads(responses.calls[-1].request.body) == elt_response
|
|
423
|
-
assert element_update_response == elt_response
|
|
424
|
-
|
|
425
|
-
cached_element = CachedElement.get(CachedElement.id == elt.id)
|
|
426
|
-
assert cached_element.polygon == str(elt_response["polygon"])
|
|
427
|
-
assert cached_element.confidence == confidence
|