arkindex-base-worker 0.3.6rc1__py3-none-any.whl → 0.3.6rc2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. arkindex_base_worker-0.3.6rc2.dist-info/METADATA +39 -0
  2. arkindex_base_worker-0.3.6rc2.dist-info/RECORD +40 -0
  3. arkindex_worker/__init__.py +0 -1
  4. arkindex_worker/cache.py +19 -25
  5. arkindex_worker/image.py +16 -17
  6. arkindex_worker/models.py +17 -21
  7. arkindex_worker/utils.py +16 -17
  8. arkindex_worker/worker/__init__.py +14 -23
  9. arkindex_worker/worker/base.py +12 -7
  10. arkindex_worker/worker/classification.py +13 -15
  11. arkindex_worker/worker/dataset.py +3 -4
  12. arkindex_worker/worker/element.py +80 -75
  13. arkindex_worker/worker/entity.py +27 -29
  14. arkindex_worker/worker/metadata.py +19 -25
  15. arkindex_worker/worker/task.py +2 -3
  16. arkindex_worker/worker/training.py +21 -22
  17. arkindex_worker/worker/transcription.py +37 -34
  18. arkindex_worker/worker/version.py +1 -2
  19. tests/conftest.py +55 -75
  20. tests/test_base_worker.py +37 -31
  21. tests/test_cache.py +14 -7
  22. tests/test_dataset_worker.py +4 -4
  23. tests/test_element.py +0 -1
  24. tests/test_elements_worker/__init__.py +0 -1
  25. tests/test_elements_worker/test_classifications.py +0 -1
  26. tests/test_elements_worker/test_cli.py +22 -17
  27. tests/test_elements_worker/test_dataset.py +9 -10
  28. tests/test_elements_worker/test_elements.py +58 -63
  29. tests/test_elements_worker/test_entities.py +10 -20
  30. tests/test_elements_worker/test_metadata.py +72 -96
  31. tests/test_elements_worker/test_task.py +9 -10
  32. tests/test_elements_worker/test_training.py +20 -13
  33. tests/test_elements_worker/test_transcriptions.py +6 -10
  34. tests/test_elements_worker/test_worker.py +16 -14
  35. tests/test_image.py +21 -20
  36. tests/test_merge.py +5 -6
  37. tests/test_utils.py +0 -1
  38. arkindex_base_worker-0.3.6rc1.dist-info/METADATA +0 -27
  39. arkindex_base_worker-0.3.6rc1.dist-info/RECORD +0 -42
  40. arkindex_worker/git.py +0 -392
  41. tests/test_git.py +0 -480
  42. {arkindex_base_worker-0.3.6rc1.dist-info → arkindex_base_worker-0.3.6rc2.dist-info}/WHEEL +0 -0
  43. {arkindex_base_worker-0.3.6rc1.dist-info → arkindex_base_worker-0.3.6rc2.dist-info}/top_level.txt +0 -0
tests/test_git.py DELETED
@@ -1,480 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- from pathlib import Path
3
-
4
- import pytest
5
- from gitlab import GitlabCreateError, GitlabError
6
- from requests import ConnectionError
7
- from responses import matchers
8
-
9
- from arkindex_worker.git import GitlabHelper
10
-
11
- PROJECT_ID = 21259233
12
- MERGE_REQUEST_ID = 7
13
- SOURCE_BRANCH = "new_branch"
14
- TARGET_BRANCH = "master"
15
- MR_TITLE = "merge request title"
16
- CREATE_MR_RESPONSE_JSON = {
17
- "id": 107,
18
- "iid": MERGE_REQUEST_ID,
19
- "project_id": PROJECT_ID,
20
- "title": MR_TITLE,
21
- "target_branch": TARGET_BRANCH,
22
- "source_branch": SOURCE_BRANCH,
23
- # several fields omitted
24
- }
25
-
26
-
27
- @pytest.fixture
28
- def fake_responses(responses):
29
- responses.add(
30
- responses.GET,
31
- "https://gitlab.com/api/v4/projects/balsac_exporter%2Fbalsac-exported-xmls-testing",
32
- json={
33
- "id": PROJECT_ID,
34
- # several fields omitted
35
- },
36
- )
37
- return responses
38
-
39
-
40
- def test_clone_done(fake_git_helper):
41
- assert not fake_git_helper.is_clone_finished
42
- fake_git_helper._clone_done(None, None, None)
43
- assert fake_git_helper.is_clone_finished
44
-
45
-
46
- def test_clone(fake_git_helper):
47
- command = fake_git_helper.run_clone_in_background()
48
- cmd_str = " ".join(list(map(str, command.cmd)))
49
-
50
- assert "git" in cmd_str
51
- assert "clone" in cmd_str
52
-
53
-
54
- def _get_fn_name_from_call(call):
55
- # call.add(2, 3) => "add"
56
- return str(call)[len("call.") :].split("(")[0]
57
-
58
-
59
- def test_save_files(fake_git_helper, mocker):
60
- mocker.patch("sh.wc", return_value=2)
61
- fake_git_helper._git = mocker.MagicMock()
62
- fake_git_helper.is_clone_finished = True
63
- fake_git_helper.success = True
64
-
65
- fake_git_helper.save_files(Path("/tmp/test_1234/tmp/"))
66
-
67
- expected_calls = ["checkout", "add", "commit", "show", "push"]
68
- actual_calls = list(map(_get_fn_name_from_call, fake_git_helper._git.mock_calls))
69
-
70
- assert actual_calls == expected_calls
71
- assert fake_git_helper.gitlab_helper.merge.call_count == 1
72
-
73
-
74
- def test_save_files__fail_with_failed_clone(fake_git_helper, mocker):
75
- mocker.patch("sh.wc", return_value=2)
76
- fake_git_helper._git = mocker.MagicMock()
77
- fake_git_helper.is_clone_finished = True
78
-
79
- with pytest.raises(Exception) as execinfo:
80
- fake_git_helper.save_files(Path("/tmp/test_1234/tmp/"))
81
-
82
- assert execinfo.value.args[0] == "Clone was not a success"
83
-
84
-
85
- def test_merge(mocker):
86
- api = mocker.MagicMock()
87
- project = mocker.MagicMock()
88
- api.projects.get.return_value = project
89
- merqe_request = mocker.MagicMock()
90
- project.mergerequests.create.return_value = merqe_request
91
- mocker.patch("gitlab.Gitlab", return_value=api)
92
-
93
- gitlab_helper = GitlabHelper("project_id", "url", "token", "branch")
94
-
95
- gitlab_helper._wait_for_rebase_to_finish = mocker.MagicMock()
96
- gitlab_helper._wait_for_rebase_to_finish.return_value = True
97
-
98
- success = gitlab_helper.merge("source", "merge title")
99
-
100
- assert success
101
- assert project.mergerequests.create.call_count == 1
102
- assert merqe_request.merge.call_count == 1
103
-
104
-
105
- def test_merge__rebase_failed(mocker):
106
- api = mocker.MagicMock()
107
- project = mocker.MagicMock()
108
- api.projects.get.return_value = project
109
- merqe_request = mocker.MagicMock()
110
- project.mergerequests.create.return_value = merqe_request
111
- mocker.patch("gitlab.Gitlab", return_value=api)
112
-
113
- gitlab_helper = GitlabHelper("project_id", "url", "token", "branch")
114
-
115
- gitlab_helper._wait_for_rebase_to_finish = mocker.MagicMock()
116
- gitlab_helper._wait_for_rebase_to_finish.return_value = False
117
-
118
- success = gitlab_helper.merge("source", "merge title")
119
-
120
- assert not success
121
- assert project.mergerequests.create.call_count == 1
122
- assert merqe_request.merge.call_count == 0
123
-
124
-
125
- def test_wait_for_rebase_to_finish(fake_responses, fake_gitlab_helper_factory):
126
- get_mr_url = f"https://gitlab.com/api/v4/projects/{PROJECT_ID}/merge_requests/{MERGE_REQUEST_ID}?include_rebase_in_progress=True"
127
-
128
- fake_responses.add(
129
- fake_responses.GET,
130
- get_mr_url,
131
- json={
132
- "rebase_in_progress": True,
133
- "merge_error": None,
134
- },
135
- )
136
-
137
- fake_responses.add(
138
- fake_responses.GET,
139
- get_mr_url,
140
- json={
141
- "rebase_in_progress": True,
142
- "merge_error": None,
143
- },
144
- )
145
-
146
- fake_responses.add(
147
- fake_responses.GET,
148
- get_mr_url,
149
- json={
150
- "rebase_in_progress": False,
151
- "merge_error": None,
152
- },
153
- )
154
-
155
- gitlab_helper = fake_gitlab_helper_factory()
156
-
157
- success = gitlab_helper._wait_for_rebase_to_finish(MERGE_REQUEST_ID)
158
-
159
- assert success
160
- assert len(fake_responses.calls) == 4
161
- assert gitlab_helper.is_rebase_finished
162
-
163
-
164
- def test_wait_for_rebase_to_finish__fail_connection_error(
165
- fake_responses, fake_gitlab_helper_factory
166
- ):
167
- get_mr_url = f"https://gitlab.com/api/v4/projects/{PROJECT_ID}/merge_requests/{MERGE_REQUEST_ID}?include_rebase_in_progress=True"
168
-
169
- fake_responses.add(
170
- fake_responses.GET,
171
- get_mr_url,
172
- body=ConnectionError(),
173
- )
174
-
175
- gitlab_helper = fake_gitlab_helper_factory()
176
-
177
- with pytest.raises(ConnectionError):
178
- gitlab_helper._wait_for_rebase_to_finish(MERGE_REQUEST_ID)
179
-
180
- assert len(fake_responses.calls) == 2
181
- assert not gitlab_helper.is_rebase_finished
182
-
183
-
184
- def test_wait_for_rebase_to_finish__fail_server_error(
185
- fake_responses, fake_gitlab_helper_factory
186
- ):
187
- get_mr_url = f"https://gitlab.com/api/v4/projects/{PROJECT_ID}/merge_requests/{MERGE_REQUEST_ID}?include_rebase_in_progress=True"
188
-
189
- fake_responses.add(
190
- fake_responses.GET,
191
- get_mr_url,
192
- body="Service Unavailable",
193
- status=503,
194
- )
195
-
196
- gitlab_helper = fake_gitlab_helper_factory()
197
-
198
- with pytest.raises(GitlabError):
199
- gitlab_helper._wait_for_rebase_to_finish(MERGE_REQUEST_ID)
200
-
201
- assert len(fake_responses.calls) == 2
202
- assert not gitlab_helper.is_rebase_finished
203
-
204
-
205
- def test_merge_request(fake_responses, fake_gitlab_helper_factory, mocker):
206
- fake_responses.add(
207
- fake_responses.POST,
208
- f"https://gitlab.com/api/v4/projects/{PROJECT_ID}/merge_requests",
209
- json=CREATE_MR_RESPONSE_JSON,
210
- )
211
-
212
- fake_responses.add(
213
- fake_responses.PUT,
214
- f"https://gitlab.com/api/v4/projects/{PROJECT_ID}/merge_requests/{MERGE_REQUEST_ID}/rebase",
215
- json={},
216
- )
217
-
218
- fake_responses.add(
219
- fake_responses.PUT,
220
- f"https://gitlab.com/api/v4/projects/{PROJECT_ID}/merge_requests/{MERGE_REQUEST_ID}/merge",
221
- json={
222
- "iid": MERGE_REQUEST_ID,
223
- "state": "merged",
224
- # several fields omitted
225
- },
226
- match=[matchers.json_params_matcher({"should_remove_source_branch": True})],
227
- )
228
-
229
- # the fake_responses are defined in the same order as they are expected to be called
230
- expected_http_methods = [r.method for r in fake_responses.registered()]
231
- expected_urls = [r.url for r in fake_responses.registered()]
232
-
233
- gitlab_helper = fake_gitlab_helper_factory()
234
- gitlab_helper._wait_for_rebase_to_finish = mocker.MagicMock()
235
- gitlab_helper._wait_for_rebase_to_finish.return_value = True
236
-
237
- success = gitlab_helper.merge(SOURCE_BRANCH, MR_TITLE)
238
- assert success
239
- assert len(fake_responses.calls) == 4
240
- assert [c.request.method for c in fake_responses.calls] == expected_http_methods
241
- assert [c.request.url for c in fake_responses.calls] == expected_urls
242
-
243
-
244
- def test_merge_request_fail(fake_responses, fake_gitlab_helper_factory, mocker):
245
- fake_responses.add(
246
- fake_responses.POST,
247
- f"https://gitlab.com/api/v4/projects/{PROJECT_ID}/merge_requests",
248
- json=CREATE_MR_RESPONSE_JSON,
249
- )
250
-
251
- fake_responses.add(
252
- fake_responses.PUT,
253
- f"https://gitlab.com/api/v4/projects/{PROJECT_ID}/merge_requests/{MERGE_REQUEST_ID}/rebase",
254
- json={},
255
- )
256
-
257
- fake_responses.add(
258
- fake_responses.PUT,
259
- f"https://gitlab.com/api/v4/projects/{PROJECT_ID}/merge_requests/{MERGE_REQUEST_ID}/merge",
260
- json={"error": "Method not allowed"},
261
- status=405,
262
- match=[matchers.json_params_matcher({"should_remove_source_branch": True})],
263
- )
264
-
265
- # the fake_responses are defined in the same order as they are expected to be called
266
- expected_http_methods = [r.method for r in fake_responses.registered()]
267
- expected_urls = [r.url for r in fake_responses.registered()]
268
-
269
- gitlab_helper = fake_gitlab_helper_factory()
270
- gitlab_helper._wait_for_rebase_to_finish = mocker.MagicMock()
271
- gitlab_helper._wait_for_rebase_to_finish.return_value = True
272
-
273
- success = gitlab_helper.merge(SOURCE_BRANCH, MR_TITLE)
274
-
275
- assert not success
276
- assert len(fake_responses.calls) == 4
277
- assert [c.request.method for c in fake_responses.calls] == expected_http_methods
278
- assert [c.request.url for c in fake_responses.calls] == expected_urls
279
-
280
-
281
- def test_merge_request__success_after_errors(
282
- fake_responses, fake_gitlab_helper_factory
283
- ):
284
- fake_responses.add(
285
- fake_responses.POST,
286
- f"https://gitlab.com/api/v4/projects/{PROJECT_ID}/merge_requests",
287
- json=CREATE_MR_RESPONSE_JSON,
288
- )
289
-
290
- rebase_url = f"https://gitlab.com/api/v4/projects/{PROJECT_ID}/merge_requests/{MERGE_REQUEST_ID}/rebase"
291
-
292
- fake_responses.add(
293
- fake_responses.PUT,
294
- rebase_url,
295
- json={"rebase_in_progress": True},
296
- )
297
-
298
- get_mr_url = f"https://gitlab.com/api/v4/projects/{PROJECT_ID}/merge_requests/{MERGE_REQUEST_ID}?include_rebase_in_progress=True"
299
-
300
- fake_responses.add(
301
- fake_responses.GET,
302
- get_mr_url,
303
- body="Service Unavailable",
304
- status=503,
305
- )
306
-
307
- fake_responses.add(
308
- fake_responses.PUT,
309
- rebase_url,
310
- json={"rebase_in_progress": True},
311
- )
312
-
313
- fake_responses.add(
314
- fake_responses.GET,
315
- get_mr_url,
316
- body=ConnectionError(),
317
- )
318
-
319
- fake_responses.add(
320
- fake_responses.PUT,
321
- rebase_url,
322
- json={"rebase_in_progress": True},
323
- )
324
-
325
- fake_responses.add(
326
- fake_responses.GET,
327
- get_mr_url,
328
- json={
329
- "rebase_in_progress": True,
330
- "merge_error": None,
331
- },
332
- )
333
-
334
- fake_responses.add(
335
- fake_responses.GET,
336
- get_mr_url,
337
- json={
338
- "rebase_in_progress": False,
339
- "merge_error": None,
340
- },
341
- )
342
-
343
- fake_responses.add(
344
- fake_responses.PUT,
345
- f"https://gitlab.com/api/v4/projects/{PROJECT_ID}/merge_requests/{MERGE_REQUEST_ID}/merge",
346
- json={
347
- "iid": MERGE_REQUEST_ID,
348
- "state": "merged",
349
- # several fields omitted
350
- },
351
- match=[matchers.json_params_matcher({"should_remove_source_branch": True})],
352
- )
353
-
354
- # the fake_responses are defined in the same order as they are expected to be called
355
- expected_http_methods = [r.method for r in fake_responses.registered()]
356
- expected_urls = [r.url for r in fake_responses.registered()]
357
-
358
- gitlab_helper = fake_gitlab_helper_factory()
359
-
360
- success = gitlab_helper.merge(SOURCE_BRANCH, MR_TITLE)
361
-
362
- assert success
363
- assert len(fake_responses.calls) == 10
364
- assert [c.request.method for c in fake_responses.calls] == expected_http_methods
365
- assert [c.request.url for c in fake_responses.calls] == expected_urls
366
-
367
-
368
- def test_merge_request__fail_bad_request(fake_responses, fake_gitlab_helper_factory):
369
- fake_responses.add(
370
- fake_responses.POST,
371
- f"https://gitlab.com/api/v4/projects/{PROJECT_ID}/merge_requests",
372
- json=CREATE_MR_RESPONSE_JSON,
373
- )
374
-
375
- rebase_url = f"https://gitlab.com/api/v4/projects/{PROJECT_ID}/merge_requests/{MERGE_REQUEST_ID}/rebase"
376
-
377
- fake_responses.add(
378
- fake_responses.PUT,
379
- rebase_url,
380
- json={"rebase_in_progress": True},
381
- )
382
-
383
- get_mr_url = f"https://gitlab.com/api/v4/projects/{PROJECT_ID}/merge_requests/{MERGE_REQUEST_ID}?include_rebase_in_progress=True"
384
-
385
- fake_responses.add(
386
- fake_responses.GET,
387
- get_mr_url,
388
- body="Bad Request",
389
- status=400,
390
- )
391
-
392
- # the fake_responses are defined in the same order as they are expected to be called
393
- expected_http_methods = [r.method for r in fake_responses.registered()]
394
- expected_urls = [r.url for r in fake_responses.registered()]
395
-
396
- gitlab_helper = fake_gitlab_helper_factory()
397
-
398
- with pytest.raises(GitlabError):
399
- gitlab_helper.merge(SOURCE_BRANCH, MR_TITLE)
400
-
401
- assert len(fake_responses.calls) == 4
402
- assert [c.request.method for c in fake_responses.calls] == expected_http_methods
403
- assert [c.request.url for c in fake_responses.calls] == expected_urls
404
-
405
-
406
- def test_create_merge_request__no_retry_5xx_error(
407
- fake_responses, fake_gitlab_helper_factory
408
- ):
409
- request_url = f"https://gitlab.com/api/v4/projects/{PROJECT_ID}/merge_requests"
410
-
411
- fake_responses.add(
412
- fake_responses.POST,
413
- request_url,
414
- body="Service Unavailable",
415
- status=503,
416
- )
417
-
418
- # the fake_responses are defined in the same order as they are expected to be called
419
- expected_http_methods = [r.method for r in fake_responses.registered()]
420
- expected_urls = [r.url for r in fake_responses.registered()]
421
-
422
- gitlab_helper = fake_gitlab_helper_factory()
423
-
424
- with pytest.raises(GitlabCreateError):
425
- gitlab_helper.project.mergerequests.create(
426
- {
427
- "source_branch": "branch",
428
- "target_branch": gitlab_helper.branch,
429
- "title": "MR title",
430
- }
431
- )
432
-
433
- assert len(fake_responses.calls) == 2
434
- assert [c.request.method for c in fake_responses.calls] == expected_http_methods
435
- assert [c.request.url for c in fake_responses.calls] == expected_urls
436
-
437
-
438
- def test_create_merge_request__retry_5xx_error(
439
- fake_responses, fake_gitlab_helper_factory
440
- ):
441
- request_url = f"https://gitlab.com/api/v4/projects/{PROJECT_ID}/merge_requests"
442
-
443
- fake_responses.add(
444
- fake_responses.POST,
445
- request_url,
446
- body="Service Unavailable",
447
- status=503,
448
- )
449
-
450
- fake_responses.add(
451
- fake_responses.POST,
452
- request_url,
453
- body="Service Unavailable",
454
- status=503,
455
- )
456
-
457
- fake_responses.add(
458
- fake_responses.POST,
459
- request_url,
460
- json=CREATE_MR_RESPONSE_JSON,
461
- )
462
-
463
- # the fake_responses are defined in the same order as they are expected to be called
464
- expected_http_methods = [r.method for r in fake_responses.registered()]
465
- expected_urls = [r.url for r in fake_responses.registered()]
466
-
467
- gitlab_helper = fake_gitlab_helper_factory()
468
-
469
- gitlab_helper.project.mergerequests.create(
470
- {
471
- "source_branch": "branch",
472
- "target_branch": gitlab_helper.branch,
473
- "title": "MR title",
474
- },
475
- retry_transient_errors=True,
476
- )
477
-
478
- assert len(fake_responses.calls) == 4
479
- assert [c.request.method for c in fake_responses.calls] == expected_http_methods
480
- assert [c.request.url for c in fake_responses.calls] == expected_urls