udata 10.8.2.dev36872__py2.py3-none-any.whl → 10.8.2.dev36980__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of udata might be problematic. Click here for more details.
- udata/core/organization/api.py +1 -1
- udata/harvest/actions.py +24 -28
- udata/harvest/api.py +25 -33
- udata/harvest/commands.py +7 -7
- udata/harvest/tasks.py +1 -1
- udata/harvest/tests/ckan/test_ckan_backend.py +5 -5
- udata/harvest/tests/ckan/test_ckan_backend_errors.py +6 -6
- udata/harvest/tests/ckan/test_ckan_backend_filters.py +5 -5
- udata/harvest/tests/ckan/test_dkan_backend.py +1 -1
- udata/harvest/tests/test_actions.py +27 -32
- udata/harvest/tests/test_api.py +23 -18
- udata/harvest/tests/test_dcat_backend.py +29 -29
- udata/routing.py +6 -0
- udata/static/chunks/{13.2d06442dd9a05d9777b5.js → 13.f29411b06be1883356a3.js} +2 -2
- udata/static/chunks/{13.2d06442dd9a05d9777b5.js.map → 13.f29411b06be1883356a3.js.map} +1 -1
- udata/static/chunks/{17.e8e4caaad5cb0cc0bacc.js → 17.3bd0340930d4a314ce9c.js} +2 -2
- udata/static/chunks/{17.e8e4caaad5cb0cc0bacc.js.map → 17.3bd0340930d4a314ce9c.js.map} +1 -1
- udata/static/chunks/{19.f03a102365af4315f9db.js → 19.3e0e8651d948e04b8cf2.js} +3 -3
- udata/static/chunks/{19.f03a102365af4315f9db.js.map → 19.3e0e8651d948e04b8cf2.js.map} +1 -1
- udata/static/chunks/{5.5660483641193b7f8295.js → 5.0fa1408dae4e76b87b2e.js} +3 -3
- udata/static/chunks/{5.5660483641193b7f8295.js.map → 5.0fa1408dae4e76b87b2e.js.map} +1 -1
- udata/static/chunks/{6.30dce49d17db07600b06.js → 6.d663709d877baa44a71e.js} +3 -3
- udata/static/chunks/{6.30dce49d17db07600b06.js.map → 6.d663709d877baa44a71e.js.map} +1 -1
- udata/static/chunks/{8.b58fcd977fcaf3415571.js → 8.494b003a94383b142c18.js} +2 -2
- udata/static/chunks/{8.b58fcd977fcaf3415571.js.map → 8.494b003a94383b142c18.js.map} +1 -1
- udata/static/common.js +1 -1
- udata/static/common.js.map +1 -1
- udata/tests/api/test_organizations_api.py +5 -0
- {udata-10.8.2.dev36872.dist-info → udata-10.8.2.dev36980.dist-info}/METADATA +3 -1
- {udata-10.8.2.dev36872.dist-info → udata-10.8.2.dev36980.dist-info}/RECORD +34 -34
- {udata-10.8.2.dev36872.dist-info → udata-10.8.2.dev36980.dist-info}/LICENSE +0 -0
- {udata-10.8.2.dev36872.dist-info → udata-10.8.2.dev36980.dist-info}/WHEEL +0 -0
- {udata-10.8.2.dev36872.dist-info → udata-10.8.2.dev36980.dist-info}/entry_points.txt +0 -0
- {udata-10.8.2.dev36872.dist-info → udata-10.8.2.dev36980.dist-info}/top_level.txt +0 -0
|
@@ -152,7 +152,7 @@ class HarvestActionsTest:
|
|
|
152
152
|
data["url"] = new_url
|
|
153
153
|
|
|
154
154
|
with assert_emit(signals.harvest_source_updated):
|
|
155
|
-
source = actions.update_source(source
|
|
155
|
+
source = actions.update_source(source, data)
|
|
156
156
|
|
|
157
157
|
assert source.url == new_url
|
|
158
158
|
source.reload()
|
|
@@ -162,14 +162,14 @@ class HarvestActionsTest:
|
|
|
162
162
|
def test_validate_source(self, mock):
|
|
163
163
|
source = HarvestSourceFactory()
|
|
164
164
|
|
|
165
|
-
actions.validate_source(source
|
|
165
|
+
actions.validate_source(source)
|
|
166
166
|
|
|
167
167
|
source.reload()
|
|
168
168
|
assert source.validation.state == VALIDATION_ACCEPTED
|
|
169
169
|
assert source.validation.on is not None
|
|
170
170
|
assert source.validation.by is None
|
|
171
171
|
assert source.validation.comment is None
|
|
172
|
-
mock.assert_called_once_with(source
|
|
172
|
+
mock.assert_called_once_with(source)
|
|
173
173
|
|
|
174
174
|
assert source.periodic_task is not None
|
|
175
175
|
|
|
@@ -177,7 +177,7 @@ class HarvestActionsTest:
|
|
|
177
177
|
def test_validate_source_with_comment(self, mock):
|
|
178
178
|
source = HarvestSourceFactory()
|
|
179
179
|
|
|
180
|
-
actions.validate_source(source
|
|
180
|
+
actions.validate_source(source, "comment")
|
|
181
181
|
|
|
182
182
|
source.reload()
|
|
183
183
|
|
|
@@ -185,12 +185,12 @@ class HarvestActionsTest:
|
|
|
185
185
|
assert source.validation.on is not None
|
|
186
186
|
assert source.validation.by is None
|
|
187
187
|
assert source.validation.comment == "comment"
|
|
188
|
-
mock.assert_called_once_with(source
|
|
188
|
+
mock.assert_called_once_with(source)
|
|
189
189
|
|
|
190
190
|
def test_reject_source(self):
|
|
191
191
|
source = HarvestSourceFactory()
|
|
192
192
|
|
|
193
|
-
actions.reject_source(source
|
|
193
|
+
actions.reject_source(source, "comment")
|
|
194
194
|
|
|
195
195
|
source.reload()
|
|
196
196
|
assert source.validation.state == VALIDATION_REFUSED
|
|
@@ -200,20 +200,16 @@ class HarvestActionsTest:
|
|
|
200
200
|
|
|
201
201
|
def test_get_source_by_slug(self):
|
|
202
202
|
source = HarvestSourceFactory()
|
|
203
|
-
assert
|
|
203
|
+
assert HarvestSource.get(source.slug) == source
|
|
204
204
|
|
|
205
205
|
def test_get_source_by_id(self):
|
|
206
206
|
source = HarvestSourceFactory()
|
|
207
|
-
assert
|
|
208
|
-
|
|
209
|
-
def test_get_source_by_objectid(self):
|
|
210
|
-
source = HarvestSourceFactory()
|
|
211
|
-
assert actions.get_source(source.id) == source
|
|
207
|
+
assert HarvestSource.get(str(source.id)) == source
|
|
212
208
|
|
|
213
209
|
def test_delete_source_by_slug(self):
|
|
214
210
|
source = HarvestSourceFactory()
|
|
215
211
|
with assert_emit(signals.harvest_source_deleted):
|
|
216
|
-
deleted_source = actions.delete_source(source
|
|
212
|
+
deleted_source = actions.delete_source(source)
|
|
217
213
|
|
|
218
214
|
assert deleted_source.deleted is not None
|
|
219
215
|
assert deleted_source.id == source.id
|
|
@@ -223,7 +219,7 @@ class HarvestActionsTest:
|
|
|
223
219
|
def test_delete_source_by_id(self):
|
|
224
220
|
source = HarvestSourceFactory()
|
|
225
221
|
with assert_emit(signals.harvest_source_deleted):
|
|
226
|
-
deleted_source = actions.delete_source(
|
|
222
|
+
deleted_source = actions.delete_source(source)
|
|
227
223
|
|
|
228
224
|
assert deleted_source.deleted is not None
|
|
229
225
|
assert deleted_source.id == source.id
|
|
@@ -233,19 +229,18 @@ class HarvestActionsTest:
|
|
|
233
229
|
def test_delete_source_by_objectid(self):
|
|
234
230
|
source = HarvestSourceFactory()
|
|
235
231
|
with assert_emit(signals.harvest_source_deleted):
|
|
236
|
-
deleted_source = actions.delete_source(source
|
|
232
|
+
deleted_source = actions.delete_source(source)
|
|
237
233
|
|
|
238
234
|
assert deleted_source.deleted is not None
|
|
239
235
|
assert deleted_source.id == source.id
|
|
240
236
|
deleted_sources = HarvestSource.objects(deleted__exists=True)
|
|
241
237
|
assert len(deleted_sources) == 1
|
|
242
238
|
|
|
243
|
-
|
|
244
|
-
def test_clean_source(self, by_attr):
|
|
239
|
+
def test_clean_source(self):
|
|
245
240
|
source = HarvestSourceFactory()
|
|
246
241
|
for _ in range(5):
|
|
247
242
|
DatasetFactory(harvest=HarvestDatasetMetadata(source_id=str(source.id)))
|
|
248
|
-
actions.clean_source(
|
|
243
|
+
actions.clean_source(source)
|
|
249
244
|
datasets = Dataset.objects.filter(harvest__source_id=str(source.id))
|
|
250
245
|
assert len(datasets) == 5
|
|
251
246
|
for dataset in datasets:
|
|
@@ -262,7 +257,7 @@ class HarvestActionsTest:
|
|
|
262
257
|
def test_schedule(self):
|
|
263
258
|
source = HarvestSourceFactory()
|
|
264
259
|
with assert_emit(signals.harvest_source_scheduled):
|
|
265
|
-
source = actions.schedule(
|
|
260
|
+
source = actions.schedule(source, hour=0)
|
|
266
261
|
|
|
267
262
|
assert len(PeriodicTask.objects) == 1
|
|
268
263
|
periodic_task = source.periodic_task
|
|
@@ -279,7 +274,7 @@ class HarvestActionsTest:
|
|
|
279
274
|
def test_schedule_from_cron(self):
|
|
280
275
|
source = HarvestSourceFactory()
|
|
281
276
|
with assert_emit(signals.harvest_source_scheduled):
|
|
282
|
-
source = actions.schedule(
|
|
277
|
+
source = actions.schedule(source, "0 1 2 3 sunday")
|
|
283
278
|
|
|
284
279
|
assert len(PeriodicTask.objects) == 1
|
|
285
280
|
periodic_task = source.periodic_task
|
|
@@ -296,10 +291,10 @@ class HarvestActionsTest:
|
|
|
296
291
|
def test_reschedule(self):
|
|
297
292
|
source = HarvestSourceFactory()
|
|
298
293
|
with assert_emit(signals.harvest_source_scheduled):
|
|
299
|
-
source = actions.schedule(
|
|
294
|
+
source = actions.schedule(source, hour=0)
|
|
300
295
|
|
|
301
296
|
with assert_emit(signals.harvest_source_scheduled):
|
|
302
|
-
source = actions.schedule(
|
|
297
|
+
source = actions.schedule(source, minute=0)
|
|
303
298
|
|
|
304
299
|
assert len(PeriodicTask.objects) == 1
|
|
305
300
|
periodic_task = source.periodic_task
|
|
@@ -323,7 +318,7 @@ class HarvestActionsTest:
|
|
|
323
318
|
)
|
|
324
319
|
source = HarvestSourceFactory(periodic_task=periodic_task)
|
|
325
320
|
with assert_emit(signals.harvest_source_unscheduled):
|
|
326
|
-
actions.unschedule(
|
|
321
|
+
actions.unschedule(source)
|
|
327
322
|
|
|
328
323
|
source.reload()
|
|
329
324
|
assert len(PeriodicTask.objects) == 0
|
|
@@ -460,7 +455,7 @@ class ExecutionTestMixin(MockBackendsMixin):
|
|
|
460
455
|
org = OrganizationFactory()
|
|
461
456
|
source = HarvestSourceFactory(backend="factory", organization=org)
|
|
462
457
|
with assert_emit(signals.before_harvest_job, signals.after_harvest_job):
|
|
463
|
-
self.action(source
|
|
458
|
+
self.action(source)
|
|
464
459
|
|
|
465
460
|
source.reload()
|
|
466
461
|
assert len(HarvestJob.objects(source=source)) == 1
|
|
@@ -496,7 +491,7 @@ class ExecutionTestMixin(MockBackendsMixin):
|
|
|
496
491
|
|
|
497
492
|
source = HarvestSourceFactory(backend="factory")
|
|
498
493
|
with assert_emit(signals.before_harvest_job), mock_initialize.connected_to(init):
|
|
499
|
-
self.action(source
|
|
494
|
+
self.action(source)
|
|
500
495
|
|
|
501
496
|
source.reload()
|
|
502
497
|
assert len(HarvestJob.objects(source=source)) == 1
|
|
@@ -523,7 +518,7 @@ class ExecutionTestMixin(MockBackendsMixin):
|
|
|
523
518
|
assert_emit(signals.before_harvest_job, signals.after_harvest_job),
|
|
524
519
|
mock_process.connected_to(process),
|
|
525
520
|
):
|
|
526
|
-
self.action(source
|
|
521
|
+
self.action(source)
|
|
527
522
|
|
|
528
523
|
source.reload()
|
|
529
524
|
assert len(HarvestJob.objects(source=source)) == 1
|
|
@@ -559,7 +554,7 @@ class ExecutionTestMixin(MockBackendsMixin):
|
|
|
559
554
|
def test_empty(self):
|
|
560
555
|
source = HarvestSourceFactory(backend="factory", config={"count": 0})
|
|
561
556
|
with assert_emit(signals.before_harvest_job, signals.after_harvest_job):
|
|
562
|
-
self.action(source
|
|
557
|
+
self.action(source)
|
|
563
558
|
|
|
564
559
|
source.reload()
|
|
565
560
|
assert len(HarvestJob.objects(source=source)) == 1
|
|
@@ -580,7 +575,7 @@ class ExecutionTestMixin(MockBackendsMixin):
|
|
|
580
575
|
org = OrganizationFactory()
|
|
581
576
|
source = HarvestSourceFactory(backend="factory", organization=org, config={"count": 10})
|
|
582
577
|
|
|
583
|
-
self.action(source
|
|
578
|
+
self.action(source)
|
|
584
579
|
assert len(Dataset.objects) == 5
|
|
585
580
|
|
|
586
581
|
|
|
@@ -599,7 +594,7 @@ class HarvestPreviewTest(MockBackendsMixin):
|
|
|
599
594
|
org = OrganizationFactory()
|
|
600
595
|
source = HarvestSourceFactory(backend="factory", organization=org)
|
|
601
596
|
|
|
602
|
-
job = actions.preview(source
|
|
597
|
+
job = actions.preview(source)
|
|
603
598
|
|
|
604
599
|
assert job.status == "done"
|
|
605
600
|
assert job.errors == []
|
|
@@ -628,7 +623,7 @@ class HarvestPreviewTest(MockBackendsMixin):
|
|
|
628
623
|
org = OrganizationFactory()
|
|
629
624
|
source = HarvestSourceFactory(backend="factory", organization=org, config={"count": 10})
|
|
630
625
|
|
|
631
|
-
job = actions.preview(source
|
|
626
|
+
job = actions.preview(source)
|
|
632
627
|
|
|
633
628
|
assert len(job.items) == 5
|
|
634
629
|
|
|
@@ -639,7 +634,7 @@ class HarvestPreviewTest(MockBackendsMixin):
|
|
|
639
634
|
source = HarvestSourceFactory(backend="factory")
|
|
640
635
|
|
|
641
636
|
with mock_initialize.connected_to(init):
|
|
642
|
-
job = actions.preview(source
|
|
637
|
+
job = actions.preview(source)
|
|
643
638
|
|
|
644
639
|
assert job.status == "failed"
|
|
645
640
|
assert len(job.errors) == 1
|
|
@@ -660,7 +655,7 @@ class HarvestPreviewTest(MockBackendsMixin):
|
|
|
660
655
|
source = HarvestSourceFactory(backend="factory")
|
|
661
656
|
|
|
662
657
|
with mock_process.connected_to(process):
|
|
663
|
-
job = actions.preview(source
|
|
658
|
+
job = actions.preview(source)
|
|
664
659
|
|
|
665
660
|
assert job.status == "done-errors"
|
|
666
661
|
assert job.started is not None
|
udata/harvest/tests/test_api.py
CHANGED
|
@@ -395,7 +395,7 @@ class HarvestAPITest(MockBackendsMixin):
|
|
|
395
395
|
"url": new_url,
|
|
396
396
|
"backend": "factory",
|
|
397
397
|
}
|
|
398
|
-
api_url = url_for("api.harvest_source",
|
|
398
|
+
api_url = url_for("api.harvest_source", source=source)
|
|
399
399
|
response = api.put(api_url, data)
|
|
400
400
|
assert200(response)
|
|
401
401
|
assert response.json["url"] == new_url
|
|
@@ -403,7 +403,7 @@ class HarvestAPITest(MockBackendsMixin):
|
|
|
403
403
|
# Source is now owned by orga, with user as member
|
|
404
404
|
source.organization = OrganizationFactory(members=[Member(user=user)])
|
|
405
405
|
source.save()
|
|
406
|
-
api_url = url_for("api.harvest_source",
|
|
406
|
+
api_url = url_for("api.harvest_source", source=source)
|
|
407
407
|
response = api.put(api_url, data)
|
|
408
408
|
assert200(response)
|
|
409
409
|
|
|
@@ -418,7 +418,7 @@ class HarvestAPITest(MockBackendsMixin):
|
|
|
418
418
|
"url": new_url,
|
|
419
419
|
"backend": "factory",
|
|
420
420
|
}
|
|
421
|
-
api_url: str = url_for("api.harvest_source",
|
|
421
|
+
api_url: str = url_for("api.harvest_source", source=source)
|
|
422
422
|
response = api.put(api_url, data)
|
|
423
423
|
|
|
424
424
|
assert403(response)
|
|
@@ -429,7 +429,7 @@ class HarvestAPITest(MockBackendsMixin):
|
|
|
429
429
|
source = HarvestSourceFactory()
|
|
430
430
|
|
|
431
431
|
data = {"state": VALIDATION_ACCEPTED}
|
|
432
|
-
url = url_for("api.validate_harvest_source",
|
|
432
|
+
url = url_for("api.validate_harvest_source", source=source)
|
|
433
433
|
response = api.post(url, data)
|
|
434
434
|
assert200(response)
|
|
435
435
|
|
|
@@ -443,7 +443,7 @@ class HarvestAPITest(MockBackendsMixin):
|
|
|
443
443
|
source = HarvestSourceFactory()
|
|
444
444
|
|
|
445
445
|
data = {"state": VALIDATION_REFUSED, "comment": "Not valid"}
|
|
446
|
-
url = url_for("api.validate_harvest_source",
|
|
446
|
+
url = url_for("api.validate_harvest_source", source=source)
|
|
447
447
|
response = api.post(url, data)
|
|
448
448
|
assert200(response)
|
|
449
449
|
|
|
@@ -458,22 +458,27 @@ class HarvestAPITest(MockBackendsMixin):
|
|
|
458
458
|
source = HarvestSourceFactory()
|
|
459
459
|
|
|
460
460
|
data = {"validate": True}
|
|
461
|
-
url = url_for("api.validate_harvest_source",
|
|
461
|
+
url = url_for("api.validate_harvest_source", source=source)
|
|
462
462
|
response = api.post(url, data)
|
|
463
463
|
assert403(response)
|
|
464
464
|
|
|
465
465
|
def test_get_source(self, api):
|
|
466
466
|
source = HarvestSourceFactory()
|
|
467
467
|
|
|
468
|
-
url = url_for("api.harvest_source",
|
|
468
|
+
url = url_for("api.harvest_source", source=source)
|
|
469
469
|
response = api.get(url)
|
|
470
470
|
assert200(response)
|
|
471
471
|
|
|
472
|
+
def test_get_missing_source(self, api):
|
|
473
|
+
url = url_for("api.harvest_source", source="685bb38b9cb9284b93fd9e72")
|
|
474
|
+
response = api.get(url)
|
|
475
|
+
assert404(response)
|
|
476
|
+
|
|
472
477
|
def test_source_preview(self, api):
|
|
473
478
|
api.login()
|
|
474
479
|
source = HarvestSourceFactory(backend="factory")
|
|
475
480
|
|
|
476
|
-
url = url_for("api.preview_harvest_source",
|
|
481
|
+
url = url_for("api.preview_harvest_source", source=source)
|
|
477
482
|
response = api.get(url)
|
|
478
483
|
assert200(response)
|
|
479
484
|
|
|
@@ -488,7 +493,7 @@ class HarvestAPITest(MockBackendsMixin):
|
|
|
488
493
|
validation=HarvestSourceValidation(state=VALIDATION_ACCEPTED),
|
|
489
494
|
)
|
|
490
495
|
|
|
491
|
-
url = url_for("api.run_harvest_source",
|
|
496
|
+
url = url_for("api.run_harvest_source", source=source)
|
|
492
497
|
response = api.post(url)
|
|
493
498
|
assert200(response)
|
|
494
499
|
|
|
@@ -505,7 +510,7 @@ class HarvestAPITest(MockBackendsMixin):
|
|
|
505
510
|
validation=HarvestSourceValidation(state=VALIDATION_ACCEPTED),
|
|
506
511
|
)
|
|
507
512
|
|
|
508
|
-
url = url_for("api.run_harvest_source",
|
|
513
|
+
url = url_for("api.run_harvest_source", source=source)
|
|
509
514
|
response = api.post(url)
|
|
510
515
|
assert400(response)
|
|
511
516
|
|
|
@@ -523,7 +528,7 @@ class HarvestAPITest(MockBackendsMixin):
|
|
|
523
528
|
validation=HarvestSourceValidation(state=VALIDATION_ACCEPTED),
|
|
524
529
|
)
|
|
525
530
|
|
|
526
|
-
url = url_for("api.run_harvest_source",
|
|
531
|
+
url = url_for("api.run_harvest_source", source=source)
|
|
527
532
|
response = api.post(url)
|
|
528
533
|
assert403(response)
|
|
529
534
|
|
|
@@ -540,7 +545,7 @@ class HarvestAPITest(MockBackendsMixin):
|
|
|
540
545
|
validation=HarvestSourceValidation(state=VALIDATION_PENDING),
|
|
541
546
|
)
|
|
542
547
|
|
|
543
|
-
url = url_for("api.run_harvest_source",
|
|
548
|
+
url = url_for("api.run_harvest_source", source=source)
|
|
544
549
|
response = api.post(url)
|
|
545
550
|
assert400(response)
|
|
546
551
|
|
|
@@ -556,7 +561,7 @@ class HarvestAPITest(MockBackendsMixin):
|
|
|
556
561
|
user = api.login()
|
|
557
562
|
source = HarvestSourceFactory(owner=user)
|
|
558
563
|
|
|
559
|
-
url = url_for("api.harvest_source",
|
|
564
|
+
url = url_for("api.harvest_source", source=source)
|
|
560
565
|
response = api.delete(url)
|
|
561
566
|
assert204(response)
|
|
562
567
|
|
|
@@ -568,7 +573,7 @@ class HarvestAPITest(MockBackendsMixin):
|
|
|
568
573
|
api.login()
|
|
569
574
|
source = HarvestSourceFactory()
|
|
570
575
|
|
|
571
|
-
url = url_for("api.harvest_source",
|
|
576
|
+
url = url_for("api.harvest_source", source=source)
|
|
572
577
|
response = api.delete(url)
|
|
573
578
|
|
|
574
579
|
assert403(response)
|
|
@@ -579,7 +584,7 @@ class HarvestAPITest(MockBackendsMixin):
|
|
|
579
584
|
source = HarvestSourceFactory()
|
|
580
585
|
|
|
581
586
|
data = "0 0 * * *"
|
|
582
|
-
url = url_for("api.schedule_harvest_source",
|
|
587
|
+
url = url_for("api.schedule_harvest_source", source=source)
|
|
583
588
|
response = api.post(url, data)
|
|
584
589
|
assert200(response)
|
|
585
590
|
|
|
@@ -601,7 +606,7 @@ class HarvestAPITest(MockBackendsMixin):
|
|
|
601
606
|
source = HarvestSourceFactory()
|
|
602
607
|
|
|
603
608
|
data = "0 0 * * *"
|
|
604
|
-
url = url_for("api.schedule_harvest_source",
|
|
609
|
+
url = url_for("api.schedule_harvest_source", source=source)
|
|
605
610
|
response = api.post(url, data)
|
|
606
611
|
assert403(response)
|
|
607
612
|
|
|
@@ -620,7 +625,7 @@ class HarvestAPITest(MockBackendsMixin):
|
|
|
620
625
|
)
|
|
621
626
|
source = HarvestSourceFactory(periodic_task=periodic_task)
|
|
622
627
|
|
|
623
|
-
url = url_for("api.schedule_harvest_source",
|
|
628
|
+
url = url_for("api.schedule_harvest_source", source=source)
|
|
624
629
|
response = api.delete(url)
|
|
625
630
|
assert204(response)
|
|
626
631
|
|
|
@@ -639,7 +644,7 @@ class HarvestAPITest(MockBackendsMixin):
|
|
|
639
644
|
)
|
|
640
645
|
source = HarvestSourceFactory(periodic_task=periodic_task)
|
|
641
646
|
|
|
642
|
-
url = url_for("api.schedule_harvest_source",
|
|
647
|
+
url = url_for("api.schedule_harvest_source", source=source)
|
|
643
648
|
response = api.delete(url)
|
|
644
649
|
assert403(response)
|
|
645
650
|
|
|
@@ -74,7 +74,7 @@ class DcatBackendTest:
|
|
|
74
74
|
org = OrganizationFactory()
|
|
75
75
|
source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
|
|
76
76
|
|
|
77
|
-
actions.run(source
|
|
77
|
+
actions.run(source)
|
|
78
78
|
|
|
79
79
|
source.reload()
|
|
80
80
|
|
|
@@ -123,7 +123,7 @@ class DcatBackendTest:
|
|
|
123
123
|
org = OrganizationFactory()
|
|
124
124
|
source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
|
|
125
125
|
|
|
126
|
-
actions.run(source
|
|
126
|
+
actions.run(source)
|
|
127
127
|
|
|
128
128
|
datasets = {d.harvest.dct_identifier: d for d in Dataset.objects}
|
|
129
129
|
|
|
@@ -154,7 +154,7 @@ class DcatBackendTest:
|
|
|
154
154
|
org = OrganizationFactory()
|
|
155
155
|
source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
|
|
156
156
|
|
|
157
|
-
actions.run(source
|
|
157
|
+
actions.run(source)
|
|
158
158
|
|
|
159
159
|
datasets = {d.harvest.dct_identifier: d for d in Dataset.objects}
|
|
160
160
|
|
|
@@ -171,7 +171,7 @@ class DcatBackendTest:
|
|
|
171
171
|
org = OrganizationFactory()
|
|
172
172
|
source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
|
|
173
173
|
|
|
174
|
-
actions.run(source
|
|
174
|
+
actions.run(source)
|
|
175
175
|
|
|
176
176
|
dataservices = Dataservice.objects
|
|
177
177
|
|
|
@@ -192,7 +192,7 @@ class DcatBackendTest:
|
|
|
192
192
|
org = OrganizationFactory()
|
|
193
193
|
source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
|
|
194
194
|
|
|
195
|
-
actions.run(source
|
|
195
|
+
actions.run(source)
|
|
196
196
|
|
|
197
197
|
datasets = {d.harvest.dct_identifier: d for d in Dataset.objects}
|
|
198
198
|
assert len(datasets) == 8
|
|
@@ -244,7 +244,7 @@ class DcatBackendTest:
|
|
|
244
244
|
org = OrganizationFactory()
|
|
245
245
|
source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
|
|
246
246
|
|
|
247
|
-
actions.run(source
|
|
247
|
+
actions.run(source)
|
|
248
248
|
|
|
249
249
|
datasets = {d.harvest.dct_identifier: d for d in Dataset.objects}
|
|
250
250
|
|
|
@@ -306,7 +306,7 @@ class DcatBackendTest:
|
|
|
306
306
|
org = OrganizationFactory()
|
|
307
307
|
source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
|
|
308
308
|
|
|
309
|
-
actions.run(source
|
|
309
|
+
actions.run(source)
|
|
310
310
|
|
|
311
311
|
assert Dataset.objects.count() == 2
|
|
312
312
|
assert HarvestJob.objects.first().status == "done"
|
|
@@ -320,7 +320,7 @@ class DcatBackendTest:
|
|
|
320
320
|
org = OrganizationFactory()
|
|
321
321
|
source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
|
|
322
322
|
|
|
323
|
-
actions.run(source
|
|
323
|
+
actions.run(source)
|
|
324
324
|
|
|
325
325
|
datasets = {d.harvest.dct_identifier: d for d in Dataset.objects}
|
|
326
326
|
|
|
@@ -345,7 +345,7 @@ class DcatBackendTest:
|
|
|
345
345
|
org = OrganizationFactory()
|
|
346
346
|
source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
|
|
347
347
|
|
|
348
|
-
actions.run(source
|
|
348
|
+
actions.run(source)
|
|
349
349
|
|
|
350
350
|
datasets = {d.harvest.dct_identifier: d for d in Dataset.objects}
|
|
351
351
|
|
|
@@ -388,7 +388,7 @@ class DcatBackendTest:
|
|
|
388
388
|
url = mock_dcat(rmock, filename)
|
|
389
389
|
source = HarvestSourceFactory(backend="dcat", url=url, organization=OrganizationFactory())
|
|
390
390
|
|
|
391
|
-
actions.run(source
|
|
391
|
+
actions.run(source)
|
|
392
392
|
|
|
393
393
|
source.reload()
|
|
394
394
|
|
|
@@ -416,8 +416,8 @@ class DcatBackendTest:
|
|
|
416
416
|
source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
|
|
417
417
|
|
|
418
418
|
# Run the same havester twice
|
|
419
|
-
actions.run(source
|
|
420
|
-
actions.run(source
|
|
419
|
+
actions.run(source)
|
|
420
|
+
actions.run(source)
|
|
421
421
|
|
|
422
422
|
datasets = {d.harvest.dct_identifier: d for d in Dataset.objects}
|
|
423
423
|
|
|
@@ -431,7 +431,7 @@ class DcatBackendTest:
|
|
|
431
431
|
org = OrganizationFactory()
|
|
432
432
|
source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
|
|
433
433
|
|
|
434
|
-
actions.run(source
|
|
434
|
+
actions.run(source)
|
|
435
435
|
|
|
436
436
|
source.reload()
|
|
437
437
|
|
|
@@ -443,7 +443,7 @@ class DcatBackendTest:
|
|
|
443
443
|
org = OrganizationFactory()
|
|
444
444
|
source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
|
|
445
445
|
|
|
446
|
-
actions.run(source
|
|
446
|
+
actions.run(source)
|
|
447
447
|
|
|
448
448
|
source.reload()
|
|
449
449
|
|
|
@@ -456,7 +456,7 @@ class DcatBackendTest:
|
|
|
456
456
|
org = OrganizationFactory()
|
|
457
457
|
source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
|
|
458
458
|
|
|
459
|
-
actions.run(source
|
|
459
|
+
actions.run(source)
|
|
460
460
|
|
|
461
461
|
source.reload()
|
|
462
462
|
|
|
@@ -470,7 +470,7 @@ class DcatBackendTest:
|
|
|
470
470
|
org = OrganizationFactory()
|
|
471
471
|
source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
|
|
472
472
|
|
|
473
|
-
actions.run(source
|
|
473
|
+
actions.run(source)
|
|
474
474
|
|
|
475
475
|
source.reload()
|
|
476
476
|
|
|
@@ -493,7 +493,7 @@ class DcatBackendTest:
|
|
|
493
493
|
org = OrganizationFactory()
|
|
494
494
|
source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
|
|
495
495
|
|
|
496
|
-
actions.run(source
|
|
496
|
+
actions.run(source)
|
|
497
497
|
|
|
498
498
|
# test dct:license support
|
|
499
499
|
dataset = Dataset.objects.get(harvest__dct_identifier="3")
|
|
@@ -587,7 +587,7 @@ class DcatBackendTest:
|
|
|
587
587
|
url = mock_dcat(rmock, "geonetwork.xml", path="catalog.xml")
|
|
588
588
|
org = OrganizationFactory()
|
|
589
589
|
source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
|
|
590
|
-
actions.run(source
|
|
590
|
+
actions.run(source)
|
|
591
591
|
dataset = Dataset.objects.filter(organization=org).first()
|
|
592
592
|
assert dataset is not None
|
|
593
593
|
assert dataset.harvest is not None
|
|
@@ -616,7 +616,7 @@ class DcatBackendTest:
|
|
|
616
616
|
url = mock_dcat(rmock, "sig.oreme.rdf")
|
|
617
617
|
org = OrganizationFactory()
|
|
618
618
|
source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
|
|
619
|
-
actions.run(source
|
|
619
|
+
actions.run(source)
|
|
620
620
|
dataset = Dataset.objects.filter(organization=org).first()
|
|
621
621
|
|
|
622
622
|
assert dataset is not None
|
|
@@ -646,7 +646,7 @@ class DcatBackendTest:
|
|
|
646
646
|
url = mock_dcat(rmock, "udata.xml")
|
|
647
647
|
org = OrganizationFactory()
|
|
648
648
|
source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
|
|
649
|
-
actions.run(source
|
|
649
|
+
actions.run(source)
|
|
650
650
|
|
|
651
651
|
source.reload()
|
|
652
652
|
job = source.get_last_job()
|
|
@@ -712,7 +712,7 @@ class DcatBackendTest:
|
|
|
712
712
|
get_mock = rmock.get(url)
|
|
713
713
|
org = OrganizationFactory()
|
|
714
714
|
source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
|
|
715
|
-
actions.run(source
|
|
715
|
+
actions.run(source)
|
|
716
716
|
|
|
717
717
|
assert "User-Agent" in get_mock.last_request.headers
|
|
718
718
|
assert get_mock.last_request.headers["User-Agent"] == "uData/0.1 dcat"
|
|
@@ -723,7 +723,7 @@ class DcatBackendTest:
|
|
|
723
723
|
org = OrganizationFactory()
|
|
724
724
|
source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
|
|
725
725
|
|
|
726
|
-
actions.run(source
|
|
726
|
+
actions.run(source)
|
|
727
727
|
|
|
728
728
|
source.reload()
|
|
729
729
|
|
|
@@ -741,7 +741,7 @@ class DcatBackendTest:
|
|
|
741
741
|
org = OrganizationFactory()
|
|
742
742
|
source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
|
|
743
743
|
|
|
744
|
-
actions.run(source
|
|
744
|
+
actions.run(source)
|
|
745
745
|
|
|
746
746
|
source.reload()
|
|
747
747
|
|
|
@@ -773,7 +773,7 @@ class DcatBackendTest:
|
|
|
773
773
|
rmock.head(url, headers={"Content-Type": "application/json"})
|
|
774
774
|
org = OrganizationFactory()
|
|
775
775
|
source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
|
|
776
|
-
actions.run(source
|
|
776
|
+
actions.run(source)
|
|
777
777
|
|
|
778
778
|
source.reload()
|
|
779
779
|
|
|
@@ -787,7 +787,7 @@ class DcatBackendTest:
|
|
|
787
787
|
rmock.get(url, status_code=404)
|
|
788
788
|
|
|
789
789
|
source = HarvestSourceFactory(backend="dcat", url=url, organization=OrganizationFactory())
|
|
790
|
-
actions.run(source
|
|
790
|
+
actions.run(source)
|
|
791
791
|
source.reload()
|
|
792
792
|
|
|
793
793
|
job = source.get_last_job()
|
|
@@ -800,7 +800,7 @@ class DcatBackendTest:
|
|
|
800
800
|
rmock.head(url, status_code=404)
|
|
801
801
|
|
|
802
802
|
source = HarvestSourceFactory(backend="dcat", url=url, organization=OrganizationFactory())
|
|
803
|
-
actions.run(source
|
|
803
|
+
actions.run(source)
|
|
804
804
|
source.reload()
|
|
805
805
|
|
|
806
806
|
job = source.get_last_job()
|
|
@@ -817,7 +817,7 @@ class CswDcatBackendTest:
|
|
|
817
817
|
org = OrganizationFactory()
|
|
818
818
|
source = HarvestSourceFactory(backend="csw-dcat", url=url, organization=org)
|
|
819
819
|
|
|
820
|
-
actions.run(source
|
|
820
|
+
actions.run(source)
|
|
821
821
|
|
|
822
822
|
source.reload()
|
|
823
823
|
|
|
@@ -863,7 +863,7 @@ class CswDcatBackendTest:
|
|
|
863
863
|
org = OrganizationFactory()
|
|
864
864
|
source = HarvestSourceFactory(backend="csw-dcat", url=url, organization=org)
|
|
865
865
|
|
|
866
|
-
actions.run(source
|
|
866
|
+
actions.run(source)
|
|
867
867
|
|
|
868
868
|
assert "User-Agent" in get_mock.last_request.headers
|
|
869
869
|
assert get_mock.last_request.headers["User-Agent"] == "uData/0.1 csw-dcat"
|
|
@@ -902,7 +902,7 @@ class CswIso19139DcatBackendTest:
|
|
|
902
902
|
},
|
|
903
903
|
)
|
|
904
904
|
|
|
905
|
-
actions.run(source
|
|
905
|
+
actions.run(source)
|
|
906
906
|
|
|
907
907
|
source.reload()
|
|
908
908
|
|
udata/routing.py
CHANGED
|
@@ -10,6 +10,7 @@ from werkzeug.urls import url_quote
|
|
|
10
10
|
from udata import models
|
|
11
11
|
from udata.core.dataservices.models import Dataservice
|
|
12
12
|
from udata.core.spatial.models import GeoZone
|
|
13
|
+
from udata.harvest.models import HarvestSource
|
|
13
14
|
from udata.i18n import ISO_639_1_CODES
|
|
14
15
|
from udata.mongo import db
|
|
15
16
|
from udata.uris import cdata_url, homepage_url
|
|
@@ -136,6 +137,10 @@ class DataserviceConverter(ModelConverter):
|
|
|
136
137
|
model = Dataservice
|
|
137
138
|
|
|
138
139
|
|
|
140
|
+
class HarvestSourceConverter(ModelConverter):
|
|
141
|
+
model = HarvestSource
|
|
142
|
+
|
|
143
|
+
|
|
139
144
|
class CommunityResourceConverter(ModelConverter):
|
|
140
145
|
model = models.CommunityResource
|
|
141
146
|
|
|
@@ -239,6 +244,7 @@ def init_app(app):
|
|
|
239
244
|
app.url_map.converters["dataset"] = DatasetConverter
|
|
240
245
|
app.url_map.converters["dataset_without_resources"] = DatasetWithoutResourcesConverter
|
|
241
246
|
app.url_map.converters["dataservice"] = DataserviceConverter
|
|
247
|
+
app.url_map.converters["harvest_source"] = HarvestSourceConverter
|
|
242
248
|
app.url_map.converters["crid"] = CommunityResourceConverter
|
|
243
249
|
app.url_map.converters["org"] = OrganizationConverter
|
|
244
250
|
app.url_map.converters["reuse"] = ReuseConverter
|