arkindex-base-worker 0.3.6rc4__py3-none-any.whl → 0.3.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- arkindex_base_worker-0.3.7.dist-info/LICENSE +21 -0
- arkindex_base_worker-0.3.7.dist-info/METADATA +77 -0
- arkindex_base_worker-0.3.7.dist-info/RECORD +47 -0
- {arkindex_base_worker-0.3.6rc4.dist-info → arkindex_base_worker-0.3.7.dist-info}/WHEEL +1 -1
- {arkindex_base_worker-0.3.6rc4.dist-info → arkindex_base_worker-0.3.7.dist-info}/top_level.txt +2 -0
- arkindex_worker/cache.py +14 -0
- arkindex_worker/image.py +29 -19
- arkindex_worker/models.py +14 -2
- arkindex_worker/utils.py +17 -3
- arkindex_worker/worker/__init__.py +122 -125
- arkindex_worker/worker/base.py +24 -24
- arkindex_worker/worker/classification.py +18 -25
- arkindex_worker/worker/dataset.py +24 -18
- arkindex_worker/worker/element.py +100 -19
- arkindex_worker/worker/entity.py +35 -4
- arkindex_worker/worker/metadata.py +21 -11
- arkindex_worker/worker/training.py +13 -0
- arkindex_worker/worker/transcription.py +45 -5
- arkindex_worker/worker/version.py +22 -0
- hooks/pre_gen_project.py +3 -0
- tests/conftest.py +16 -8
- tests/test_base_worker.py +0 -6
- tests/test_dataset_worker.py +291 -409
- tests/test_elements_worker/test_classifications.py +365 -539
- tests/test_elements_worker/test_cli.py +1 -1
- tests/test_elements_worker/test_dataset.py +97 -116
- tests/test_elements_worker/test_elements.py +354 -76
- tests/test_elements_worker/test_entities.py +22 -2
- tests/test_elements_worker/test_metadata.py +53 -27
- tests/test_elements_worker/test_training.py +35 -0
- tests/test_elements_worker/test_transcriptions.py +149 -16
- tests/test_elements_worker/test_worker.py +19 -6
- tests/test_image.py +37 -0
- tests/test_utils.py +23 -1
- worker-demo/tests/__init__.py +0 -0
- worker-demo/tests/conftest.py +32 -0
- worker-demo/tests/test_worker.py +12 -0
- worker-demo/worker_demo/__init__.py +6 -0
- worker-demo/worker_demo/worker.py +19 -0
- arkindex_base_worker-0.3.6rc4.dist-info/METADATA +0 -47
- arkindex_base_worker-0.3.6rc4.dist-info/RECORD +0 -40
tests/conftest.py
CHANGED
|
@@ -22,7 +22,7 @@ from arkindex_worker.cache import (
|
|
|
22
22
|
create_version_table,
|
|
23
23
|
init_cache_db,
|
|
24
24
|
)
|
|
25
|
-
from arkindex_worker.models import Artifact, Dataset
|
|
25
|
+
from arkindex_worker.models import Artifact, Dataset, Set
|
|
26
26
|
from arkindex_worker.worker import BaseWorker, DatasetWorker, ElementsWorker
|
|
27
27
|
from arkindex_worker.worker.dataset import DatasetState
|
|
28
28
|
from arkindex_worker.worker.transcription import TextOrientation
|
|
@@ -139,7 +139,7 @@ def _mock_worker_run_api(responses):
|
|
|
139
139
|
"docker_image_name": None,
|
|
140
140
|
"state": "created",
|
|
141
141
|
"gpu_usage": "disabled",
|
|
142
|
-
"model_usage":
|
|
142
|
+
"model_usage": "disabled",
|
|
143
143
|
"worker": {
|
|
144
144
|
"id": "deadbeef-1234-5678-1234-worker",
|
|
145
145
|
"name": "Fake worker",
|
|
@@ -206,7 +206,7 @@ def _mock_worker_run_no_revision_api(responses):
|
|
|
206
206
|
"docker_image_name": None,
|
|
207
207
|
"state": "created",
|
|
208
208
|
"gpu_usage": "disabled",
|
|
209
|
-
"model_usage":
|
|
209
|
+
"model_usage": "disabled",
|
|
210
210
|
"worker": {
|
|
211
211
|
"id": "deadbeef-1234-5678-1234-worker",
|
|
212
212
|
"name": "Fake worker",
|
|
@@ -466,6 +466,7 @@ def _mock_cached_transcriptions(mock_cache_db):
|
|
|
466
466
|
confidence=0.42,
|
|
467
467
|
orientation=TextOrientation.HorizontalLeftToRight,
|
|
468
468
|
worker_version_id=UUID("56785678-5678-5678-5678-567856785678"),
|
|
469
|
+
worker_run_id=UUID("56785678-5678-5678-5678-567856785678"),
|
|
469
470
|
)
|
|
470
471
|
CachedTranscription.create(
|
|
471
472
|
id=UUID("22222222-2222-2222-2222-222222222222"),
|
|
@@ -506,6 +507,7 @@ def _mock_cached_transcriptions(mock_cache_db):
|
|
|
506
507
|
confidence=0.42,
|
|
507
508
|
orientation=TextOrientation.HorizontalLeftToRight,
|
|
508
509
|
worker_version_id=None,
|
|
510
|
+
worker_run_id=None,
|
|
509
511
|
)
|
|
510
512
|
|
|
511
513
|
|
|
@@ -595,11 +597,11 @@ def mock_databases(tmp_path):
|
|
|
595
597
|
@pytest.fixture()
|
|
596
598
|
def default_dataset():
|
|
597
599
|
return Dataset(
|
|
598
|
-
|
|
600
|
+
{
|
|
599
601
|
"id": "dataset_id",
|
|
600
602
|
"name": "My dataset",
|
|
601
603
|
"description": "A super dataset built by me",
|
|
602
|
-
"sets": ["set_1", "set_2", "set_3"],
|
|
604
|
+
"sets": ["set_1", "set_2", "set_3", "set_4"],
|
|
603
605
|
"state": DatasetState.Open.value,
|
|
604
606
|
"corpus_id": "corpus_id",
|
|
605
607
|
"creator": "creator@teklia.com",
|
|
@@ -610,6 +612,11 @@ def default_dataset():
|
|
|
610
612
|
)
|
|
611
613
|
|
|
612
614
|
|
|
615
|
+
@pytest.fixture()
|
|
616
|
+
def default_train_set(default_dataset):
|
|
617
|
+
return Set(name="train", dataset=default_dataset)
|
|
618
|
+
|
|
619
|
+
|
|
613
620
|
@pytest.fixture()
|
|
614
621
|
def mock_dataset_worker(monkeypatch, mocker, _mock_worker_run_api):
|
|
615
622
|
monkeypatch.setenv("PONOS_TASK", "my_task")
|
|
@@ -632,9 +639,10 @@ def mock_dev_dataset_worker(mocker):
|
|
|
632
639
|
[
|
|
633
640
|
"worker",
|
|
634
641
|
"--dev",
|
|
635
|
-
"--
|
|
636
|
-
"11111111-1111-1111-1111-111111111111",
|
|
637
|
-
"
|
|
642
|
+
"--set",
|
|
643
|
+
"11111111-1111-1111-1111-111111111111:train",
|
|
644
|
+
"11111111-1111-1111-1111-111111111111:val",
|
|
645
|
+
"22222222-2222-2222-2222-222222222222:my_set",
|
|
638
646
|
],
|
|
639
647
|
)
|
|
640
648
|
|
tests/test_base_worker.py
CHANGED
|
@@ -86,7 +86,6 @@ def test_cli_default(mocker):
|
|
|
86
86
|
assert logger.level == logging.NOTSET
|
|
87
87
|
assert worker.api_client
|
|
88
88
|
assert worker.config == {"someKey": "someValue"} # from API
|
|
89
|
-
assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
|
|
90
89
|
|
|
91
90
|
logger.setLevel(logging.NOTSET)
|
|
92
91
|
|
|
@@ -106,7 +105,6 @@ def test_cli_arg_verbose_given(mocker):
|
|
|
106
105
|
assert logger.level == logging.DEBUG
|
|
107
106
|
assert worker.api_client
|
|
108
107
|
assert worker.config == {"someKey": "someValue"} # from API
|
|
109
|
-
assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
|
|
110
108
|
|
|
111
109
|
logger.setLevel(logging.NOTSET)
|
|
112
110
|
|
|
@@ -126,7 +124,6 @@ def test_cli_envvar_debug_given(mocker, monkeypatch):
|
|
|
126
124
|
assert logger.level == logging.DEBUG
|
|
127
125
|
assert worker.api_client
|
|
128
126
|
assert worker.config == {"someKey": "someValue"} # from API
|
|
129
|
-
assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
|
|
130
127
|
|
|
131
128
|
logger.setLevel(logging.NOTSET)
|
|
132
129
|
|
|
@@ -142,7 +139,6 @@ def test_configure_dev_mode(mocker):
|
|
|
142
139
|
|
|
143
140
|
assert worker.args.dev is True
|
|
144
141
|
assert worker.process_information is None
|
|
145
|
-
assert worker.worker_run_id == "56785678-5678-5678-5678-567856785678"
|
|
146
142
|
assert worker.is_read_only is True
|
|
147
143
|
assert worker.user_configuration == {}
|
|
148
144
|
|
|
@@ -212,7 +208,6 @@ def test_configure_worker_run(mocker, responses, caplog):
|
|
|
212
208
|
("arkindex_worker", logging.INFO, "Loaded user configuration from WorkerRun"),
|
|
213
209
|
]
|
|
214
210
|
|
|
215
|
-
assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
|
|
216
211
|
assert worker.user_configuration == {"a": "b"}
|
|
217
212
|
|
|
218
213
|
|
|
@@ -482,7 +477,6 @@ def test_configure_load_model_configuration(mocker, responses):
|
|
|
482
477
|
|
|
483
478
|
worker.configure()
|
|
484
479
|
|
|
485
|
-
assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
|
|
486
480
|
assert worker.model_configuration == {
|
|
487
481
|
"param1": "value1",
|
|
488
482
|
"param2": 2,
|