udata 10.8.2.dev36872__py2.py3-none-any.whl → 10.8.2.dev36938__py2.py3-none-any.whl

udata/harvest/actions.py

@@ -29,6 +29,11 @@ log = logging.getLogger(__name__)
 DEFAULT_PAGE_SIZE = 10
 
 
+def get_source(ident):
+    """Get an harvest source given its ID or its slug"""
+    return HarvestSource.get(ident)
+
+
 def list_backends():
     """List all available backends"""
     return backends.get_all(current_app).values()
@@ -44,11 +49,6 @@ def list_sources(owner=None, deleted=False):
     return list(sources)
 
 
-def get_source(ident):
-    """Get an harvest source given its ID or its slug"""
-    return HarvestSource.get(ident)
-
-
 def get_job(ident):
     """Get an harvest job given its ID"""
     return HarvestJob.objects.get(id=ident)
@@ -89,31 +89,28 @@ def create_source(
     return source
 
 
-def update_source(ident, data):
+def update_source(source: HarvestSource, data):
     """Update an harvest source"""
-    source = get_source(ident)
     source.modify(**data)
     signals.harvest_source_updated.send(source)
     return source
 
 
-def validate_source(ident, comment=None):
+def validate_source(source: HarvestSource, comment=None):
     """Validate a source for automatic harvesting"""
-    source = get_source(ident)
     source.validation.on = datetime.utcnow()
     source.validation.comment = comment
     source.validation.state = VALIDATION_ACCEPTED
     if current_user.is_authenticated:
         source.validation.by = current_user._get_current_object()
     source.save()
-    schedule(ident, cron=current_app.config["HARVEST_DEFAULT_SCHEDULE"])
-    launch(ident)
+    schedule(source, cron=current_app.config["HARVEST_DEFAULT_SCHEDULE"])
+    launch(source)
     return source
 
 
-def reject_source(ident, comment):
+def reject_source(source: HarvestSource, comment):
     """Reject a source for automatic harvesting"""
-    source = get_source(ident)
     source.validation.on = datetime.utcnow()
     source.validation.comment = comment
     source.validation.state = VALIDATION_REFUSED
@@ -123,18 +120,16 @@ def reject_source(ident, comment):
     return source
 
 
-def delete_source(ident):
+def delete_source(source: HarvestSource):
     """Delete an harvest source"""
-    source = get_source(ident)
     source.deleted = datetime.utcnow()
     source.save()
     signals.harvest_source_deleted.send(source)
     return source
 
 
-def clean_source(ident):
+def clean_source(source: HarvestSource):
     """Deletes all datasets linked to a harvest source"""
-    source = get_source(ident)
     datasets = Dataset.objects.filter(harvest__source_id=str(source.id))
     for dataset in datasets:
         dataset.deleted = datetime.utcnow()
@@ -180,22 +175,20 @@ def purge_jobs():
     return HarvestJob.objects(created__lt=expiration).delete()
 
 
-def run(ident):
+def run(source: HarvestSource):
     """Launch or resume an harvesting for a given source if none is running"""
-    source = get_source(ident)
     cls = backends.get(current_app, source.backend)
     backend = cls(source)
     backend.harvest()
 
 
-def launch(ident):
+def launch(source: HarvestSource):
     """Launch or resume an harvesting for a given source if none is running"""
-    return harvest.delay(ident)
+    return harvest.delay(source.id)
 
 
-def preview(ident):
+def preview(source: HarvestSource):
     """Preview an harvesting for a given source"""
-    source = get_source(ident)
     cls = backends.get(current_app, source.backend)
     max_items = current_app.config["HARVEST_PREVIEW_MAX_ITEMS"]
     backend = cls(source, dryrun=True, max_items=max_items)
@@ -240,11 +233,15 @@ def preview_from_config(
 
 
 def schedule(
-    ident, cron=None, minute="*", hour="*", day_of_week="*", day_of_month="*", month_of_year="*"
+    source: HarvestSource,
+    cron=None,
+    minute="*",
+    hour="*",
+    day_of_week="*",
+    day_of_month="*",
+    month_of_year="*",
 ):
     """Schedule an harvesting on a source given a crontab"""
-    source = get_source(ident)
-
     if cron:
         minute, hour, day_of_month, month_of_year, day_of_week = cron.split()
 
@@ -273,9 +270,8 @@ def schedule(
     return source
 
 
-def unschedule(ident):
+def unschedule(source: HarvestSource):
     """Unschedule an harvesting on a source"""
-    source = get_source(ident)
     if not source.periodic_task:
         msg = "Harvesting on source {0} is ot scheduled".format(source.name)
         raise ValueError(msg)

udata/harvest/api.py

@@ -300,59 +300,54 @@ class SourcesAPI(API):
         return source, 201
 
 
-@ns.route("/source/<string:ident>/", endpoint="harvest_source")
-@api.param("ident", "A source ID or slug")
+@ns.route("/source/<harvest_source:source>/", endpoint="harvest_source")
 class SourceAPI(API):
     @api.doc("get_harvest_source")
     @api.marshal_with(source_fields)
-    def get(self, ident):
+    def get(self, source: HarvestSource):
         """Get a single source given an ID or a slug"""
-        return actions.get_source(ident)
+        return source
 
     @api.secure
     @api.doc("update_harvest_source")
     @api.expect(source_fields)
     @api.marshal_with(source_fields)
-    def put(self, ident):
+    def put(self, source: HarvestSource):
         """Update a harvest source"""
-        source = actions.get_source(ident)
         OwnablePermission(source).test()
         form = api.validate(HarvestSourceForm, source)
-        source = actions.update_source(ident, form.data)
+        source = actions.update_source(source, form.data)
         return source
 
     @api.secure
     @api.doc("delete_harvest_source")
     @api.marshal_with(source_fields)
-    def delete(self, ident):
-        source: HarvestSource = actions.get_source(ident)
+    def delete(self, source: HarvestSource):
         OwnablePermission(source).test()
-        return actions.delete_source(ident), 204
+        return actions.delete_source(source), 204
 
 
-@ns.route("/source/<string:ident>/validate/", endpoint="validate_harvest_source")
-@api.param("ident", "A source ID or slug")
+@ns.route("/source/<harvest_source:source>/validate/", endpoint="validate_harvest_source")
 class ValidateSourceAPI(API):
     @api.doc("validate_harvest_source")
     @api.secure(admin_permission)
     @api.expect(validation_fields)
     @api.marshal_with(source_fields)
-    def post(self, ident):
+    def post(self, source: HarvestSource):
         """Validate or reject an harvest source"""
         form = api.validate(HarvestSourceValidationForm)
         if form.state.data == VALIDATION_ACCEPTED:
-            return actions.validate_source(ident, form.comment.data)
+            return actions.validate_source(source, form.comment.data)
         else:
-            return actions.reject_source(ident, form.comment.data)
+            return actions.reject_source(source, form.comment.data)
 
 
-@ns.route("/source/<string:ident>/run/", endpoint="run_harvest_source")
-@api.param("ident", "A source ID or slug")
+@ns.route("/source/<harvest_source:source>/run/", endpoint="run_harvest_source")
 class RunSourceAPI(API):
     @api.doc("run_harvest_source")
     @api.secure
     @api.marshal_with(source_fields)
-    def post(self, ident):
+    def post(self, source: HarvestSource):
         enabled = current_app.config.get("HARVEST_ENABLE_MANUAL_RUN")
         if not enabled and not current_user.sysadmin:
             api.abort(
@@ -360,39 +355,37 @@ class RunSourceAPI(API):
                 "Cannot run source manually. Please contact the platform if you need to reschedule the harvester.",
             )
 
-        source: HarvestSource = actions.get_source(ident)
         OwnablePermission(source).test()
 
         if source.validation.state != VALIDATION_ACCEPTED:
             api.abort(400, "Source is not validated. Please validate the source before running.")
 
-        actions.launch(ident)
+        actions.launch(source)
 
         return source
 
 
-@ns.route("/source/<string:ident>/schedule/", endpoint="schedule_harvest_source")
-@api.param("ident", "A source ID or slug")
+@ns.route("/source/<harvest_source:source>/schedule/", endpoint="schedule_harvest_source")
 class ScheduleSourceAPI(API):
     @api.doc("schedule_harvest_source")
     @api.secure(admin_permission)
     @api.expect((str, "A cron expression"))
     @api.marshal_with(source_fields)
-    def post(self, ident):
+    def post(self, source: HarvestSource):
         """Schedule an harvest source"""
         # Handle both syntax: quoted and unquoted
         try:
             data = request.json
         except BadRequest:
             data = request.data.decode("utf-8")
-        return actions.schedule(ident, data)
+        return actions.schedule(source, data)
 
     @api.doc("unschedule_harvest_source")
     @api.secure(admin_permission)
     @api.marshal_with(source_fields)
-    def delete(self, ident):
+    def delete(self, source: HarvestSource):
         """Unschedule an harvest source"""
-        return actions.unschedule(ident), 204
+        return actions.unschedule(source), 204
 
 
 @ns.route("/source/preview/", endpoint="preview_harvest_source_config")
@@ -409,15 +402,14 @@ class PreviewSourceConfigAPI(API):
         return actions.preview_from_config(**form.data)
 
 
-@ns.route("/source/<string:ident>/preview/", endpoint="preview_harvest_source")
-@api.param("ident", "A source ID or slug")
+@ns.route("/source/<harvest_source:source>/preview/", endpoint="preview_harvest_source")
 class PreviewSourceAPI(API):
     @api.secure
     @api.doc("preview_harvest_source")
     @api.marshal_with(preview_job_fields)
-    def get(self, ident):
+    def get(self, source: HarvestSource):
         """Preview a single harvest source given an ID or a slug"""
-        return actions.preview(ident)
+        return actions.preview(source)
 
 
 parser = api.parser()
@@ -427,15 +419,15 @@ parser.add_argument(
 )
 
 
-@ns.route("/source/<string:ident>/jobs/", endpoint="harvest_jobs")
+@ns.route("/source/<harvest_source:source>/jobs/", endpoint="harvest_jobs")
 class JobsAPI(API):
     @api.doc("list_harvest_jobs")
     @api.expect(parser)
     @api.marshal_with(job_page_fields)
-    def get(self, ident):
+    def get(self, source: HarvestSource):
         """List all jobs for a given source"""
         args = parser.parse_args()
-        qs = HarvestJob.objects(source=ident)
+        qs = HarvestJob.objects(source=source)
         qs = qs.order_by("-created")
         return qs.paginate(args["page"], args["page_size"])
 

udata/harvest/commands.py

@@ -44,7 +44,7 @@ def create(name, url, backend, frequency=None, owner=None, org=None):
 @click.argument("identifier")
 def validate(identifier):
     """Validate a source given its identifier"""
-    source = actions.validate_source(identifier)
+    source = actions.validate_source(actions.get_source(identifier))
     log.info("Source %s (%s) has been validated", source.slug, str(source.id))
 
 
@@ -52,7 +52,7 @@ def validate(identifier):
 def delete(identifier):
     """Delete a harvest source"""
     log.info('Deleting source "%s"', identifier)
-    actions.delete_source(identifier)
+    actions.delete_source(actions.get_source(identifier))
     log.info('Deleted source "%s"', identifier)
 
 
@@ -61,7 +61,7 @@ def delete(identifier):
 def clean(identifier):
     """Delete all datasets linked to a harvest source"""
     log.info(f'Cleaning source "{identifier}"')
-    num_of_datasets = actions.clean_source(identifier)
+    num_of_datasets = actions.clean_source(actions.get_source(identifier))
     log.info(f'Cleaned source "{identifier}" - deleted {num_of_datasets} dataset(s)')
 
 
@@ -99,7 +99,7 @@ def backends():
 def launch(identifier):
     """Launch a source harvesting on the workers"""
     log.info('Launching harvest job for source "%s"', identifier)
-    actions.launch(identifier)
+    actions.launch(actions.get_source(identifier))
 
 
 @grp.command()
@@ -107,7 +107,7 @@ def launch(identifier):
 def run(identifier):
     """Run a harvester synchronously"""
     log.info('Harvesting source "%s"', identifier)
-    actions.run(identifier)
+    actions.run(actions.get_source(identifier))
 
 
 @grp.command()
@@ -121,7 +121,7 @@ def run(identifier):
 @click.option("-M", "--month-of-year", default="*", help="The crontab expression for month of year")
 def schedule(identifier, **kwargs):
     """Schedule a harvest job to run periodically"""
-    source = actions.schedule(identifier, **kwargs)
+    source = actions.schedule(actions.get_source(identifier), **kwargs)
     msg = "Scheduled {source.name} with the following crontab: {cron}"
     log.info(msg.format(source=source, cron=source.periodic_task.crontab))
 
@@ -130,7 +130,7 @@ def schedule(identifier, **kwargs):
 @click.argument("identifier")
 def unschedule(identifier):
     """Unschedule a periodical harvest job"""
-    source = actions.unschedule(identifier)
+    source = actions.unschedule(actions.get_source(identifier))
     log.info('Unscheduled harvest source "%s"', source.name)
 
 

udata/harvest/tasks.py

@@ -14,7 +14,7 @@ def harvest(self, ident):
 
     source = HarvestSource.get(ident)
     if source.deleted or not source.active:
-        log.info('Ignoring inactive or deleted source "%s"', ident)
+        log.info('Ignoring inactive or deleted source "%s"', source.id)
         return  # Ignore deleted and inactive sources
     Backend = backends.get(current_app, source.backend)
     backend = Backend(source)

udata/harvest/tests/ckan/test_ckan_backend.py

@@ -96,7 +96,7 @@ def harvest_ckan(request, source, ckan, app, rmock):
     )
 
     with app.app_context():
-        actions.run(source.slug)
+        actions.run(source)
         source.reload()
         job = source.get_last_job()
         assert len(job.items) == 1
@@ -565,7 +565,7 @@ def test_minimal_ckan_response(app, rmock):
         status_code=200,
         headers={"Content-Type": "application/json"},
     )
-    actions.run(source.slug)
+    actions.run(source)
     source.reload()
     assert source.get_last_job().status == "done"
 
@@ -600,7 +600,7 @@ def test_flawed_ckan_response(app, rmock):
         status_code=200,
         headers={"Content-Type": "application/json"},
     )
-    actions.run(source.slug)
+    actions.run(source)
     source.reload()
     assert source.get_last_job().status == "done-errors"
     assert source.get_last_job().items[0].remote_id == _id
@@ -617,7 +617,7 @@ def test_flawed_ckan_response(app, rmock):
         status_code=200,
         headers={"Content-Type": "application/json"},
     )
-    actions.run(source.slug)
+    actions.run(source)
     source.reload()
     assert source.get_last_job().status == "done-errors"
     assert source.get_last_job().items[0].remote_id == name
@@ -662,7 +662,7 @@ def test_max_items(app, rmock):
         status_code=200,
         headers={"Content-Type": "application/json"},
     )
-    actions.run(source.slug)
+    actions.run(source)
     source.reload()
     assert source.get_last_job().status == "done"
     assert len(source.get_last_job().items) == 1

udata/harvest/tests/ckan/test_ckan_backend_errors.py

@@ -25,7 +25,7 @@ def test_html_error(rmock, code):
 
     rmock.get(API_URL, text=html, status_code=code, headers={"Content-Type": "text/html"})
 
-    actions.run(source.slug)
+    actions.run(source)
 
     source.reload()
 
@@ -45,7 +45,7 @@ def test_plain_text_error(rmock, code):
         API_URL, text='"Some error"', status_code=code, headers={"Content-Type": "text/plain"}
     )
 
-    actions.run(source.slug)
+    actions.run(source)
 
     source.reload()
 
@@ -66,7 +66,7 @@ def test_200_plain_text_error(rmock):
 
     rmock.get(API_URL, text='"Some error"', status_code=200, headers={"Content-Type": "text/plain"})
 
-    actions.run(source.slug)
+    actions.run(source)
 
     source.reload()
 
@@ -84,7 +84,7 @@ def test_standard_api_json_error(rmock):
 
     rmock.get(API_URL, json=json, status_code=200, headers={"Content-Type": "application/json"})
 
-    actions.run(source.slug)
+    actions.run(source)
 
     source.reload()
 
@@ -106,7 +106,7 @@ def test_standard_api_json_error_with_details(rmock):
 
     rmock.get(API_URL, json=json, status_code=200, headers={"Content-Type": "application/json"})
 
-    actions.run(source.slug)
+    actions.run(source)
 
     source.reload()
 
@@ -129,7 +129,7 @@ def test_standard_api_json_error_with_details_and_type(rmock):
 
     rmock.get(API_URL, json=json, status_code=200, headers={"Content-Type": "application/json"})
 
-    actions.run(source.slug)
+    actions.run(source)
 
     source.reload()
 

udata/harvest/tests/ckan/test_ckan_backend_filters.py

@@ -26,7 +26,7 @@ def test_include_org_filter(ckan, rmock):
         headers={"Content-Type": "application/json"},
     )
 
-    actions.run(source.slug)
+    actions.run(source)
     source.reload()
 
     assert rmock.call_count == 1
@@ -50,7 +50,7 @@ def test_exclude_org_filter(ckan, rmock):
         headers={"Content-Type": "application/json"},
     )
 
-    actions.run(source.slug)
+    actions.run(source)
     source.reload()
 
     assert rmock.call_count == 1
@@ -72,7 +72,7 @@ def test_tag_filter(ckan, rmock):
         headers={"Content-Type": "application/json"},
     )
 
-    actions.run(source.slug)
+    actions.run(source)
     source.reload()
 
     assert rmock.call_count == 1
@@ -95,7 +95,7 @@ def test_exclude_tag_filter(ckan, rmock):
         headers={"Content-Type": "application/json"},
     )
 
-    actions.run(source.slug)
+    actions.run(source)
     source.reload()
 
     assert rmock.call_count == 1
@@ -122,7 +122,7 @@ def test_can_have_multiple_filters(ckan, rmock):
         headers={"Content-Type": "application/json"},
     )
 
-    actions.run(source.slug)
+    actions.run(source)
     source.reload()
 
     assert rmock.call_count == 1

udata/harvest/tests/ckan/test_dkan_backend.py

@@ -54,7 +54,7 @@ def test_dkan_french_w_license(app, rmock):
     rmock.get(
         PACKAGE_SHOW_URL, json=data, status_code=200, headers={"Content-Type": "application/json"}
     )
-    actions.run(source.slug)
+    actions.run(source)
     source.reload()
     assert source.get_last_job().status == "done"