udata 12.0.2.dev15__py3-none-any.whl → 13.0.1.dev21__py3-none-any.whl

udata/harvest/backends/__init__.py

@@ -1,17 +1,26 @@
-from udata.entrypoints import EntrypointError, get_enabled
+from fnmatch import fnmatch
+from importlib.metadata import entry_points
 
+from flask import current_app
 
-def get(app, name):
-    """Get a backend given its name"""
-    backend = get_all(app).get(name)
-    if not backend:
-        msg = 'Harvest backend "{0}" is not registered'.format(name)
-        raise EntrypointError(msg)
-    return backend
+from .base import BaseBackend, HarvestExtraConfig, HarvestFeature, HarvestFilter  # noqa
 
 
-def get_all(app):
-    return get_enabled("udata.harvesters", app)
+def get_backend(name: str) -> type[BaseBackend] | None:
+    return get_enabled_backends().get(name)
 
 
-from .base import BaseBackend, HarvestFeature, HarvestFilter, HarvestExtraConfig  # noqa
+def get_all_backends() -> dict[str, type[BaseBackend]]:
+    # Note that we use the `BaseBackend.name` and not `ep.name`. The entrypoint name
+    # is not used anymore.
+    return {ep.load().name: ep.load() for ep in entry_points(group="udata.harvesters")}
+
+
+def is_backend_enabled(backend: type[BaseBackend]) -> bool:
+    return any(fnmatch(backend.name, g) for g in current_app.config["HARVESTER_BACKENDS"])
+
+
+def get_enabled_backends() -> dict[str, type[BaseBackend]]:
+    return {
+        name: backend for name, backend in get_all_backends().items() if is_backend_enabled(backend)
+    }

udata/harvest/backends/base.py

@@ -85,8 +85,8 @@ class BaseBackend(object):
     Also provides a few helpers needed on all or some backends.
     """
 
-    name = None
-    display_name = None
+    name: str
+    display_name: str | None = None
     verify_ssl = True
 
     # Define some allowed filters on the backend

udata/harvest/backends/ckan/harvesters.py

@@ -25,6 +25,7 @@ ALLOWED_RESOURCE_TYPES = ("dkan", "file", "file.upload", "api", "metadata")
 
 
 class CkanBackend(BaseBackend):
+    name = "ckan"
     display_name = "CKAN"
     filters = (
         HarvestFilter(_("Organization"), "organization", str, _("A CKAN Organization name")),
@@ -158,7 +159,6 @@ class CkanBackend(BaseBackend):
         dataset.tags = [t["name"] for t in data["tags"] if t["name"]]
 
         dataset.harvest.created_at = data["metadata_created"]
-        dataset.harvest.modified_at = data["metadata_modified"]
 
         dataset.harvest.ckan_name = data["name"]
 
@@ -266,5 +266,6 @@ class CkanBackend(BaseBackend):
 
 
 class DkanBackend(CkanBackend):
+    name = "dkan"
     schema = dkan_schema
     filters = []

udata/harvest/backends/dcat.py

@@ -63,6 +63,7 @@ def extract_graph(source, target, node, specs):
 
 
 class DcatBackend(BaseBackend):
+    name = "dcat"
     display_name = "DCAT"
 
     def __init__(self, *args, **kwargs):
@@ -256,6 +257,7 @@ class CswDcatBackend(DcatBackend):
     The parsing of items is then the same as for the DcatBackend.
     """
 
+    name = "csw-dcat"
     display_name = "CSW-DCAT"
 
     # CSW_REQUEST is based on:
@@ -424,6 +426,7 @@ class CswIso19139DcatBackend(CswDcatBackend):
     The parsing of items is then the same as for the DcatBackend.
     """
 
+    name = "csw-iso-19139"
     display_name = "CSW-ISO-19139"
 
     extra_configs = (

udata/harvest/backends/maaf.py

@@ -129,6 +129,7 @@ def dictize(element):
 
 
 class MaafBackend(BaseBackend):
+    name = "maaf"
     display_name = "MAAF"
     verify_ssl = False
 

udata/harvest/commands.py

@@ -2,7 +2,8 @@ import logging
 
 import click
 
-from udata.commands import cli
+from udata.commands import KO, OK, cli, green, red
+from udata.harvest.backends import get_all_backends, is_backend_enabled
 
 from . import actions
 
@@ -89,9 +90,10 @@ def sources(scheduled=False):
 @grp.command()
 def backends():
     """List available backends"""
-    log.info("Available backends:")
-    for backend in actions.list_backends():
-        log.info("%s (%s)", backend.name, backend.display_name or backend.name)
+    print("Available backends:")
+    for backend in get_all_backends().values():
+        status = green(OK) if is_backend_enabled(backend) else red(KO)
+        click.echo("{0} {1} ({2})".format(status, backend.display_name, backend.name))
 
 
 @grp.command()

udata/harvest/forms.py

@@ -1,8 +1,8 @@
 from udata.forms import Form, fields, validators
+from udata.harvest.backends import get_backend, get_enabled_backends
 from udata.i18n import lazy_gettext as _
 from udata.utils import safe_unicode
 
-from .actions import list_backends
 from .models import VALIDATION_REFUSED, VALIDATION_STATES
 
 __all__ = "HarvestSourceForm", "HarvestSourceValidationForm"
@@ -13,9 +13,6 @@ class HarvestConfigField(fields.DictField):
     A DictField with extras validations on known configurations
     """
 
-    def get_backend(self, form):
-        return next(b for b in list_backends() if b.name == form.backend.data)
-
     def get_filter_specs(self, backend, key):
         candidates = (f for f in backend.filters if f.key == key)
         return next(candidates, None)
@@ -30,7 +27,10 @@ class HarvestConfigField(fields.DictField):
 
     def pre_validate(self, form):
         if self.data:
-            backend = self.get_backend(form)
+            backend = get_backend(form.backend.data)
+            if backend is None:
+                return  # Should have been catch by the enum check for `form.backend`
+
             # Validate filters
             for f in self.data.get("filters") or []:
                 if not ("key" in f and "value" in f):
@@ -49,6 +49,7 @@ class HarvestConfigField(fields.DictField):
                     msg = '"{0}" filter should of type "{1}"'
                     msg = msg.format(specs.key, specs.type.__name__)
                     raise validators.ValidationError(msg)
+
             # Validate extras configs
             for f in self.data.get("extra_configs") or []:
                 if not ("key" in f and "value" in f):
@@ -63,6 +64,7 @@ class HarvestConfigField(fields.DictField):
                     msg = '"{0}" extra config should be of type "{1}"'
                     msg = msg.format(specs.key, specs.type.__name__)
                     raise validators.ValidationError(msg)
+
             # Validate features
             for key, value in (self.data.get("features") or {}).items():
                 if not isinstance(value, bool):
@@ -81,7 +83,8 @@ class HarvestSourceForm(Form):
     )
     url = fields.URLField(_("URL"), [validators.DataRequired()])
     backend = fields.SelectField(
-        _("Backend"), choices=lambda: [(b.name, b.display_name) for b in list_backends()]
+        _("Backend"),
+        choices=lambda: [(b.name, b.display_name) for b in get_enabled_backends().values()],
     )
     owner = fields.CurrentUserField()
     organization = fields.PublishAsField(_("Publish as"))

udata/harvest/tasks.py

@@ -1,5 +1,3 @@
-from flask import current_app
-
 from udata.tasks import get_logger, job, task
 
 from . import backends
@@ -16,7 +14,7 @@ def harvest(self, ident):
     if source.deleted or not source.active:
         log.info('Ignoring inactive or deleted source "%s"', source.id)
         return  # Ignore deleted and inactive sources
-    Backend = backends.get(current_app, source.backend)
+    Backend = backends.get_backend(source.backend)
     backend = Backend(source)
 
     backend.harvest()
@@ -27,7 +25,7 @@ def harvest_job_item(job_id, item_id):
     log.info('Harvesting item %s for job "%s"', item_id, job_id)
 
     job = HarvestJob.objects.get(pk=job_id)
-    Backend = backends.get(current_app, job.source.backend)
+    Backend = backends.get_backend(job.source.backend)
     backend = Backend(job)
 
     item = next(i for i in job.items if i.remote_id == item_id)
@@ -40,7 +38,7 @@ def harvest_job_item(job_id, item_id):
 def harvest_job_finalize(results, job_id):
     log.info('Finalize harvesting for job "%s"', job_id)
     job = HarvestJob.objects.get(pk=job_id)
-    Backend = backends.get(current_app, job.source.backend)
+    Backend = backends.get_backend(job.source.backend)
     backend = Backend(job)
     backend.finalize()