arkindex-base-worker 0.3.6rc4__py3-none-any.whl → 0.3.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. arkindex_base_worker-0.3.7.dist-info/LICENSE +21 -0
  2. arkindex_base_worker-0.3.7.dist-info/METADATA +77 -0
  3. arkindex_base_worker-0.3.7.dist-info/RECORD +47 -0
  4. {arkindex_base_worker-0.3.6rc4.dist-info → arkindex_base_worker-0.3.7.dist-info}/WHEEL +1 -1
  5. {arkindex_base_worker-0.3.6rc4.dist-info → arkindex_base_worker-0.3.7.dist-info}/top_level.txt +2 -0
  6. arkindex_worker/cache.py +14 -0
  7. arkindex_worker/image.py +29 -19
  8. arkindex_worker/models.py +14 -2
  9. arkindex_worker/utils.py +17 -3
  10. arkindex_worker/worker/__init__.py +122 -125
  11. arkindex_worker/worker/base.py +24 -24
  12. arkindex_worker/worker/classification.py +18 -25
  13. arkindex_worker/worker/dataset.py +24 -18
  14. arkindex_worker/worker/element.py +100 -19
  15. arkindex_worker/worker/entity.py +35 -4
  16. arkindex_worker/worker/metadata.py +21 -11
  17. arkindex_worker/worker/training.py +13 -0
  18. arkindex_worker/worker/transcription.py +45 -5
  19. arkindex_worker/worker/version.py +22 -0
  20. hooks/pre_gen_project.py +3 -0
  21. tests/conftest.py +16 -8
  22. tests/test_base_worker.py +0 -6
  23. tests/test_dataset_worker.py +291 -409
  24. tests/test_elements_worker/test_classifications.py +365 -539
  25. tests/test_elements_worker/test_cli.py +1 -1
  26. tests/test_elements_worker/test_dataset.py +97 -116
  27. tests/test_elements_worker/test_elements.py +354 -76
  28. tests/test_elements_worker/test_entities.py +22 -2
  29. tests/test_elements_worker/test_metadata.py +53 -27
  30. tests/test_elements_worker/test_training.py +35 -0
  31. tests/test_elements_worker/test_transcriptions.py +149 -16
  32. tests/test_elements_worker/test_worker.py +19 -6
  33. tests/test_image.py +37 -0
  34. tests/test_utils.py +23 -1
  35. worker-demo/tests/__init__.py +0 -0
  36. worker-demo/tests/conftest.py +32 -0
  37. worker-demo/tests/test_worker.py +12 -0
  38. worker-demo/worker_demo/__init__.py +6 -0
  39. worker-demo/worker_demo/worker.py +19 -0
  40. arkindex_base_worker-0.3.6rc4.dist-info/METADATA +0 -47
  41. arkindex_base_worker-0.3.6rc4.dist-info/RECORD +0 -40
tests/conftest.py CHANGED
@@ -22,7 +22,7 @@ from arkindex_worker.cache import (
22
22
  create_version_table,
23
23
  init_cache_db,
24
24
  )
25
- from arkindex_worker.models import Artifact, Dataset
25
+ from arkindex_worker.models import Artifact, Dataset, Set
26
26
  from arkindex_worker.worker import BaseWorker, DatasetWorker, ElementsWorker
27
27
  from arkindex_worker.worker.dataset import DatasetState
28
28
  from arkindex_worker.worker.transcription import TextOrientation
@@ -139,7 +139,7 @@ def _mock_worker_run_api(responses):
139
139
  "docker_image_name": None,
140
140
  "state": "created",
141
141
  "gpu_usage": "disabled",
142
- "model_usage": False,
142
+ "model_usage": "disabled",
143
143
  "worker": {
144
144
  "id": "deadbeef-1234-5678-1234-worker",
145
145
  "name": "Fake worker",
@@ -206,7 +206,7 @@ def _mock_worker_run_no_revision_api(responses):
206
206
  "docker_image_name": None,
207
207
  "state": "created",
208
208
  "gpu_usage": "disabled",
209
- "model_usage": False,
209
+ "model_usage": "disabled",
210
210
  "worker": {
211
211
  "id": "deadbeef-1234-5678-1234-worker",
212
212
  "name": "Fake worker",
@@ -466,6 +466,7 @@ def _mock_cached_transcriptions(mock_cache_db):
466
466
  confidence=0.42,
467
467
  orientation=TextOrientation.HorizontalLeftToRight,
468
468
  worker_version_id=UUID("56785678-5678-5678-5678-567856785678"),
469
+ worker_run_id=UUID("56785678-5678-5678-5678-567856785678"),
469
470
  )
470
471
  CachedTranscription.create(
471
472
  id=UUID("22222222-2222-2222-2222-222222222222"),
@@ -506,6 +507,7 @@ def _mock_cached_transcriptions(mock_cache_db):
506
507
  confidence=0.42,
507
508
  orientation=TextOrientation.HorizontalLeftToRight,
508
509
  worker_version_id=None,
510
+ worker_run_id=None,
509
511
  )
510
512
 
511
513
 
@@ -595,11 +597,11 @@ def mock_databases(tmp_path):
595
597
  @pytest.fixture()
596
598
  def default_dataset():
597
599
  return Dataset(
598
- **{
600
+ {
599
601
  "id": "dataset_id",
600
602
  "name": "My dataset",
601
603
  "description": "A super dataset built by me",
602
- "sets": ["set_1", "set_2", "set_3"],
604
+ "sets": ["set_1", "set_2", "set_3", "set_4"],
603
605
  "state": DatasetState.Open.value,
604
606
  "corpus_id": "corpus_id",
605
607
  "creator": "creator@teklia.com",
@@ -610,6 +612,11 @@ def default_dataset():
610
612
  )
611
613
 
612
614
 
615
+ @pytest.fixture()
616
+ def default_train_set(default_dataset):
617
+ return Set(name="train", dataset=default_dataset)
618
+
619
+
613
620
  @pytest.fixture()
614
621
  def mock_dataset_worker(monkeypatch, mocker, _mock_worker_run_api):
615
622
  monkeypatch.setenv("PONOS_TASK", "my_task")
@@ -632,9 +639,10 @@ def mock_dev_dataset_worker(mocker):
632
639
  [
633
640
  "worker",
634
641
  "--dev",
635
- "--dataset",
636
- "11111111-1111-1111-1111-111111111111",
637
- "22222222-2222-2222-2222-222222222222",
642
+ "--set",
643
+ "11111111-1111-1111-1111-111111111111:train",
644
+ "11111111-1111-1111-1111-111111111111:val",
645
+ "22222222-2222-2222-2222-222222222222:my_set",
638
646
  ],
639
647
  )
640
648
 
tests/test_base_worker.py CHANGED
@@ -86,7 +86,6 @@ def test_cli_default(mocker):
86
86
  assert logger.level == logging.NOTSET
87
87
  assert worker.api_client
88
88
  assert worker.config == {"someKey": "someValue"} # from API
89
- assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
90
89
 
91
90
  logger.setLevel(logging.NOTSET)
92
91
 
@@ -106,7 +105,6 @@ def test_cli_arg_verbose_given(mocker):
106
105
  assert logger.level == logging.DEBUG
107
106
  assert worker.api_client
108
107
  assert worker.config == {"someKey": "someValue"} # from API
109
- assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
110
108
 
111
109
  logger.setLevel(logging.NOTSET)
112
110
 
@@ -126,7 +124,6 @@ def test_cli_envvar_debug_given(mocker, monkeypatch):
126
124
  assert logger.level == logging.DEBUG
127
125
  assert worker.api_client
128
126
  assert worker.config == {"someKey": "someValue"} # from API
129
- assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
130
127
 
131
128
  logger.setLevel(logging.NOTSET)
132
129
 
@@ -142,7 +139,6 @@ def test_configure_dev_mode(mocker):
142
139
 
143
140
  assert worker.args.dev is True
144
141
  assert worker.process_information is None
145
- assert worker.worker_run_id == "56785678-5678-5678-5678-567856785678"
146
142
  assert worker.is_read_only is True
147
143
  assert worker.user_configuration == {}
148
144
 
@@ -212,7 +208,6 @@ def test_configure_worker_run(mocker, responses, caplog):
212
208
  ("arkindex_worker", logging.INFO, "Loaded user configuration from WorkerRun"),
213
209
  ]
214
210
 
215
- assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
216
211
  assert worker.user_configuration == {"a": "b"}
217
212
 
218
213
 
@@ -482,7 +477,6 @@ def test_configure_load_model_configuration(mocker, responses):
482
477
 
483
478
  worker.configure()
484
479
 
485
- assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
486
480
  assert worker.model_configuration == {
487
481
  "param1": "value1",
488
482
  "param2": 2,