udata 13.0.1.dev12__py3-none-any.whl → 14.4.1.dev7__py3-none-any.whl

udata/frontend/__init__.py

@@ -1,132 +1,13 @@
-import inspect
 import logging
-from importlib import import_module
 
-import pkg_resources
-from jinja2 import pass_context
-from markupsafe import Markup
-
-from udata import entrypoints
-from udata.i18n import I18nBlueprint
-
-from .markdown import UdataCleaner
 from .markdown import init_app as init_markdown
 
 log = logging.getLogger(__name__)
 
 
-hook = I18nBlueprint("hook", __name__)
-
-_template_hooks = {}
-
-
-@hook.app_template_global()
-def package_version(name: str) -> str:
-    return pkg_resources.get_distribution(name).version
-
-
-@hook.app_template_filter()
-def avatar_placeholder(url):
-    if url:
-        return url
-
-    return "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAMgAAADICAYAAACtWK6eAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAjrSURBVHgB7d0LjwxLGMbxtpa4W/dLCBG+/xcSISuuiyWEEJzzzKpV807PszPdNT1t6v9LhHPIzpJ6uu5vH9vb2/vdAGi11QCYi4AABgEBDAICGAQEMAgIYBAQwCAggEFAAIOAAAYBAQwCAhgEBDAICGAQEMAgIIBBQACDgAAGAQEMAgIYBAQwCAhgEBDAICCAQUAAg4AABgEBDAICGAQEMAgIYBAQwCAggEFAAIOAAAYBAQwCAhgEBDAICGAQEMAgIIBBQACDgAAGAQEMAgIYBAQwCAhgEBDAICCAQUAAg4AABgEBDAICGAQEMAgIYBAQwNhusDbfvn1rvn79Ovn558+fzffv3w9/7+TJk82JEyea06dPT37o1xgeARnYly9fmk+fPjUfPnyYhML9udypU6eac+fONVevXiUsAzq2t7f3u8HKKRT//1vPNPwuLl261Ny4cYOgDICArJiGTy9evFgoGHmD//Hjx5F/nqCsHgFZIfUYL1++bP29ra2t5vLly83Zs2fnzjHSHEXh+vjxY/Pr16+ZP6O5yr179yZDMJRHQFZkd3e32d/fn/n/CoSe+vp5GZqvaJj2+vXr1t7l1q1bk/kJyiIgK/DkyZOZIZV6iLt37y4djDZv3ryZBCW6fv36JHwoh4AU1tZz6Mmuxnv8+PGmFC0JK4ixN6EnKYuNwoL0VI/h0BNdjbZkOERzj0ePHjUXLlyY+v+a85RYKcMBAlKI5gca+uQUDvUcq6LQaYIeQ/L8+XO7x4LFEZACtLoUV6tWHY7cnTt3plaxNPyat3qG5RCQAt6+fTt1TEQT8qHCIaknyYdx2qlnqNUfAelJk+Q473jw4EEzNM1JYijbVrqwHALS0+fPn6d6D+1uq7Gug1av8qGWehB6kX4ISE9xYj7k0KpN3AfRDjy6IyA96CjIWHqPRCta+VykbTcfiyMgPWh4lYvLreuys7Nz+Gst9zLM6o6A9KC9j1yJYyQlXLx4ceq/deAR3RCQHvLTtZocl94t7yqe7NVQEN0QkB7yJ/O65x45BTU/Pp/Pk7AcAtJRbHRju4+xvf33NvUil6/QjoBsKF3IQn/8KwIGAekoTsjHNozJFxC4s94dAelIAclDMrbj5UzMyyAgPeQrV2NaSk2F6BIKOnRHQHo4c+bM4a/1xB7LMCtuDKrgHLohID2oXE9uLAcDdRckRw/SHQHpIZ69ikdP1kG9WH72SsdfxrSJ+a8hID1okp6fvxrD/Yt4SUonjNEdAekp3r9Y5y0+9R758ErLu2M5QPmvIiA9qQHGXkQlR9chhlOlTRle9UNACoi9iG4ZDr2ipVDG3iO/F4JuCEgB6kHyCbv2IFT1cKjNQ+17tJUdovfoj4AUotpU8Yj5ECFROPQ5OU3MmZyXQUAK0YrW/fv3p46fqPE+fvx4ZcMtDaliCBVSlTpFGQSkIG3IxcaZepK4edeHDiLqpTyxxKjCoZpcY7nZuAmo7r4CCoMab6R5yu3bt3udrtXX1iJAPIyYwsG8oywCsiIaXj19+rR1eKVJveYIKq6wyMUmfY3379837969a53T6OvF0qMog4CskJ7yetq74VV6BZuCkj/90+FHlRZycxj1SFeuXGmwGgRkAArIvFendaVgaeWMIdVqEZABKSh9qq6rl0nvSucIyTAIyBpo+KSTvwqKhlBtb69NNPlWKNJmJPOMYRGQEdDEW8OvfAKuoZN6DAKxXtsN1i7eb8d4sFEIGAQEMAgIYBAQwCAggMEqVkFpp1wrUkMXj9ZeSlomXsfnbyoC0oEao4qzabNPP8dKhkk6Z6UNvtI73/oeVIcrbTa2HWPR8Xvtp+iztdlIfazlsVG4BDVGhULHRZa9KZgaqsKixtrlCa8QKBT6HuaF8qjvQW/h1edT0HoxBGQBCoYOG5aseZWe7unVbW0bhWmHXUdT9Nmlru/qs3QCWFVPCIpHQAwNY169ejW5hzGPegINo9TYU2NLQzD9cOesSknntfLPV6j0wxXVTj0K99fnIyBzqGE9e/as9TUCeurrspOGS0eN6/Xk15DsqHsdy0rzCjVu1wuo19HfRd+DhmdtgdXpYAWF4y6zCEgLNSaV0YlDGjVKldPpOuFWQ1VQFJqjnu65dJkqTbS7nurVZ+7v709uJ8aw6uvryi5DrmkEJGi7T64GevPmzZXc3EtzjDbpbbWln+zzbjoSklkEJKPVIQ2rcnpi6773Jt7ca+spCck0dpP+0JM89hwa329ypRD9/R4+fDhT8G53d7fBAQLyRyzApnG+7nxv+sS1rcfQHCmWMq0VAWkOqqLn8wA1FoWjFgpJrAqpYtjrftfJGFQfEA2tNGFNNCGvsTqh5lpa6s2t810nY1F9QGIjuHbtWrWldGK1lLSHU7OqA9L2Rqb4FK1NfNcJAamYNsxysXHUqO2NWTXPRaoOiHaVE/UenEk6EB8UY3m99TpUGxAd88hXrnSMAwfUg+TLvvmDpDbVBkRnonLxnee1i6+UG/qdi2NRbUDiuJpat9NijxofKLWoNiD5rnm6tIS/4jF+3W2pUbUByecfhGNW3Asa4uLXGFUbkHxMzTs22sVDjDXiLBZgEBDAICCAQUAAg4AABgEBDAICGAQEMKoNSL57Xqrm7aapdfc8V21A8t3zkoWhN0WsYF/rqxOqDcj58+cPf62GQJmbv2IhC1Et4hpV+wIdFShQ1fb0lNQTUz3Jzs5O1WezUqHr2HvUeh2g6tKjqv1Ez+HpwOImV5c8StWrWOpFKNQwn8KxqXWJF0Xx6mZ+tfNaqXie6oOpmn3td2UISNDl3X+bJH9TFnjL7QzeBIscO+mAQUAAg4AABgEBDAICGAQEMAgIYBAQwCAggEFAAIOAAAYBAQwCAhgEBDAICGAQEMAgIIBBQACDgAAGAQEMAgIYBAQwCAhgEBDAICCAQUAAg4AABgEBDAICGAQEMAgIYBAQwCAggEFAAIOAAAYBAQwCAhgEBDAICGAQEMAgIIBBQACDgAAGAQEMAgIYBAQwCAhgEBDAICCAQUAAg4AABgEBDAICGAQEMP4DgXjtsl0eo1YAAAAASUVORK5CYII="
-
-
-def _wrapper(func, name=None, when=None):
-    name = name or func.__name__
-    if name not in _template_hooks:
-        _template_hooks[name] = []
-    _template_hooks[name].append((func, when))
-    return func
-
-
-def template_hook(func_or_name, when=None):
-    if callable(func_or_name):
-        return _wrapper(func_or_name)
-    elif isinstance(func_or_name, str):
-
-        def wrapper(func):
-            return _wrapper(func, func_or_name, when=when)
-
-        return wrapper
-
-
-def has_template_hook(name):
-    return name in _template_hooks
-
-
-class HookRenderer:
-    def __init__(self, funcs, ctx, *args, **kwargs):
-        self.funcs = funcs
-        self.ctx = ctx
-        self.args = args
-        self.kwargs = kwargs
-
-    def __html__(self):
-        return Markup(
-            "".join(
-                f(self.ctx, *self.args, **self.kwargs)
-                for f, w in self.funcs
-                if w is None or w(self.ctx)
-            )
-        )
-
-    def __iter__(self):
-        for func, when in self.funcs:
-            if when is None or when(self.ctx):
-                yield Markup(func(self.ctx, *self.args, **self.kwargs))
-
-
-@pass_context
-def render_template_hook(ctx, name, *args, **kwargs):
-    if not has_template_hook(name):
-        return ""
-    return HookRenderer(_template_hooks[name], ctx, *args, **kwargs)
-
-
-@hook.app_context_processor
-def inject_hooks():
-    return {
-        "hook": render_template_hook,
-        "has_hook": has_template_hook,
-    }
-
-
-class SafeMarkup(Markup):
-    """Markup object bypasses Jinja's escaping. This override allows to sanitize the resulting html."""
-
-    def __new__(cls, base, *args, **kwargs):
-        cleaner = UdataCleaner()
-        return super().__new__(cls, cleaner.clean(base), *args, **kwargs)
-
-
-def _load_views(app, module):
-    views = module if inspect.ismodule(module) else import_module(module)
-    blueprint = getattr(views, "blueprint", None)
-    if blueprint:
-        app.register_blueprint(blueprint)
-
-
-VIEWS = ["core.storages"]
-
-
-def init_app(app, views=None):
-    views = views or VIEWS
+def init_app(app):
+    from udata.core.storages.views import blueprint as storage_blueprint
 
     init_markdown(app)
 
-    for view in views:
-        _load_views(app, "udata.{}.views".format(view))
-
-    # Load hook blueprint
-    app.register_blueprint(hook)
-
-    # Load all plugins views and blueprints
-    for module in entrypoints.get_enabled("udata.views", app).values():
-        _load_views(app, module)
-
-    # Load all plugins views and blueprints
-    for module in entrypoints.get_enabled("udata.front", app).values():
-        front_module = module if inspect.ismodule(module) else import_module(module)
-        front_module.init_app(app)
+    app.register_blueprint(storage_blueprint)

udata/frontend/markdown.py

@@ -7,8 +7,9 @@ import html2text
 import mistune
 from bleach.css_sanitizer import CSSSanitizer
 from bleach.linkifier import LinkifyFilter
-from flask import Markup, current_app, request
+from flask import current_app, request
 from jinja2.filters import do_striptags, do_truncate
+from markupsafe import Markup
 from werkzeug.local import LocalProxy
 
 from udata.i18n import _

udata/harvest/actions.py

@@ -34,11 +34,6 @@ def get_source(ident):
     return HarvestSource.get(ident)
 
 
-def list_backends():
-    """List all available backends"""
-    return backends.get_all(current_app).values()
-
-
 def list_sources(owner=None, deleted=False):
     """List all harvest sources"""
     sources = HarvestSource.objects
@@ -177,7 +172,7 @@ def purge_jobs():
 
 def run(source: HarvestSource):
     """Launch or resume an harvesting for a given source if none is running"""
-    cls = backends.get(current_app, source.backend)
+    cls = backends.get_backend(source.backend)
     backend = cls(source)
     backend.harvest()
 
@@ -189,7 +184,7 @@ def launch(source: HarvestSource):
 
 def preview(source: HarvestSource):
     """Preview an harvesting for a given source"""
-    cls = backends.get(current_app, source.backend)
+    cls = backends.get_backend(source.backend)
     max_items = current_app.config["HARVEST_PREVIEW_MAX_ITEMS"]
     backend = cls(source, dryrun=True, max_items=max_items)
     return backend.harvest()
@@ -226,7 +221,7 @@ def preview_from_config(
         active=active,
         autoarchive=autoarchive,
     )
-    cls = backends.get(current_app, source.backend)
+    cls = backends.get_backend(source.backend)
     max_items = current_app.config["HARVEST_PREVIEW_MAX_ITEMS"]
     backend = cls(source, dryrun=True, max_items=max_items)
     return backend.harvest()
@@ -259,7 +254,7 @@ def schedule(
         source.modify(
             periodic_task=PeriodicTask.objects.create(
                 task="harvest",
-                name="Harvest {0}".format(source.name),
+                name=f"Harvest {source.name} ({source.id})",
                 description="Periodic Harvesting",
                 enabled=True,
                 args=[str(source.id)],
@@ -322,3 +317,23 @@ def attach(domain, filename):
             count += 1
 
     return AttachResult(count, errors)
+
+
+def detach(dataset: Dataset):
+    """Detach a dataset from its harvest source
+
+    The dataset will be cleaned from harvested information
+    and will no longer be updated or archived by harvesting.
+    """
+    dataset.harvest = None
+    for resource in dataset.resources:
+        resource.harvest = None
+    dataset.save()
+
+
+def detach_all_from_source(source: HarvestSource):
+    """Detach all datasets linked to a harvest source"""
+    datasets = Dataset.objects.filter(harvest__source_id=str(source.id))
+    for dataset in datasets:
+        detach(dataset)
+    return len(datasets)

udata/harvest/api.py

@@ -6,10 +6,10 @@ from udata.api import API, api, fields
 from udata.auth import admin_permission
 from udata.core.dataservices.models import Dataservice
 from udata.core.dataset.api_fields import dataset_fields, dataset_ref_fields
-from udata.core.dataset.permissions import OwnablePermission
 from udata.core.organization.api_fields import org_ref_fields
 from udata.core.organization.permissions import EditOrganizationPermission
 from udata.core.user.api_fields import user_ref_fields
+from udata.harvest.backends import get_enabled_backends
 
 from . import actions
 from .forms import HarvestSourceForm, HarvestSourceValidationForm
@@ -25,10 +25,6 @@ from .models import (
 ns = api.namespace("harvest", "Harvest related operations")
 
 
-def backends_ids():
-    return [b.name for b in actions.list_backends()]
-
-
 error_fields = api.model(
     "HarvestError",
     {
@@ -58,6 +54,7 @@ item_fields = api.model(
     "HarvestItem",
     {
         "remote_id": fields.String(description="The item remote ID to process", required=True),
+        "remote_url": fields.String(description="The item remote url (if available)"),
         "dataset": fields.Nested(
             dataset_ref_fields, description="The processed dataset", allow_null=True
         ),
@@ -118,6 +115,18 @@ validation_fields = api.model(
     },
 )
 
+source_permissions_fields = api.model(
+    "HarvestSourcePermissions",
+    {
+        "edit": fields.Permission(),
+        "delete": fields.Permission(),
+        "run": fields.Permission(),
+        "preview": fields.Permission(),
+        "validate": fields.Permission(),
+        "schedule": fields.Permission(),
+    },
+)
+
 source_fields = api.model(
     "HarvestSource",
     {
@@ -126,7 +135,9 @@ source_fields = api.model(
         "description": fields.Markdown(description="The source description"),
         "url": fields.String(description="The source base URL", required=True),
         "backend": fields.String(
-            description="The source backend", enum=backends_ids, required=True
+            description="The source backend",
+            enum=lambda: list(get_enabled_backends().keys()),
+            required=True,
         ),
         "config": fields.Raw(description="The configuration as key-value pairs"),
         "created_at": fields.ISODateTime(
@@ -154,6 +165,7 @@ source_fields = api.model(
         "schedule": fields.String(
             description="The source schedule (interval or cron expression)", readonly=True
         ),
+        "permissions": fields.Nested(source_permissions_fields, readonly=True),
     },
 )
 
@@ -314,7 +326,7 @@ class SourceAPI(API):
     @api.marshal_with(source_fields)
     def put(self, source: HarvestSource):
         """Update a harvest source"""
-        OwnablePermission(source).test()
+        source.permissions["edit"].test()
         form = api.validate(HarvestSourceForm, source)
         source = actions.update_source(source, form.data)
         return source
@@ -323,18 +335,19 @@ class SourceAPI(API):
     @api.doc("delete_harvest_source")
     @api.marshal_with(source_fields)
     def delete(self, source: HarvestSource):
-        OwnablePermission(source).test()
+        source.permissions["delete"].test()
         return actions.delete_source(source), 204
 
 
 @ns.route("/source/<harvest_source:source>/validate/", endpoint="validate_harvest_source")
 class ValidateSourceAPI(API):
     @api.doc("validate_harvest_source")
-    @api.secure(admin_permission)
+    @api.secure
     @api.expect(validation_fields)
     @api.marshal_with(source_fields)
     def post(self, source: HarvestSource):
         """Validate or reject an harvest source"""
+        source.permissions["validate"].test()
         form = api.validate(HarvestSourceValidationForm)
         if form.state.data == VALIDATION_ACCEPTED:
             return actions.validate_source(source, form.comment.data)
@@ -355,7 +368,7 @@ class RunSourceAPI(API):
                 "Cannot run source manually. Please contact the platform if you need to reschedule the harvester.",
             )
 
-        OwnablePermission(source).test()
+        source.permissions["run"].test()
 
         if source.validation.state != VALIDATION_ACCEPTED:
             api.abort(400, "Source is not validated. Please validate the source before running.")
@@ -368,11 +381,12 @@ class RunSourceAPI(API):
 @ns.route("/source/<harvest_source:source>/schedule/", endpoint="schedule_harvest_source")
 class ScheduleSourceAPI(API):
     @api.doc("schedule_harvest_source")
-    @api.secure(admin_permission)
+    @api.secure
     @api.expect((str, "A cron expression"))
     @api.marshal_with(source_fields)
     def post(self, source: HarvestSource):
         """Schedule an harvest source"""
+        source.permissions["schedule"].test()
         # Handle both syntax: quoted and unquoted
         try:
             data = request.json
@@ -381,10 +395,11 @@ class ScheduleSourceAPI(API):
         return actions.schedule(source, data)
 
     @api.doc("unschedule_harvest_source")
-    @api.secure(admin_permission)
+    @api.secure
     @api.marshal_with(source_fields)
     def delete(self, source: HarvestSource):
         """Unschedule an harvest source"""
+        source.permissions["schedule"].test()
         return actions.unschedule(source), 204
 
 
@@ -409,6 +424,7 @@ class PreviewSourceAPI(API):
     @api.marshal_with(preview_job_fields)
     def get(self, source: HarvestSource):
         """Preview a single harvest source given an ID or a slug"""
+        source.permissions["preview"].test()
         return actions.preview(source)
 
 
@@ -438,7 +454,7 @@ class JobAPI(API):
     @api.expect(parser)
     @api.marshal_with(job_fields)
     def get(self, ident):
-        """List all jobs for a given source"""
+        """Get a single job given an ID"""
         return actions.get_job(ident)
 
 
@@ -457,15 +473,7 @@ class ListBackendsAPI(API):
                     "features": [f.as_dict() for f in b.features],
                     "extra_configs": [f.as_dict() for f in b.extra_configs],
                 }
-                for b in actions.list_backends()
+                for b in get_enabled_backends().values()
             ],
             key=lambda b: b["label"],
         )
-
-
-@ns.route("/job_status/", endpoint="havest_job_status")
-class ListHarvesterAPI(API):
-    @api.doc(model=[str])
-    def get(self):
-        """List all available harvesters"""
-        return actions.list_backends()

udata/harvest/backends/__init__.py

@@ -1,17 +1,29 @@
-from udata.entrypoints import EntrypointError, get_enabled
+from fnmatch import fnmatch
+from importlib.metadata import entry_points
 
+from flask import current_app
 
-def get(app, name):
-    """Get a backend given its name"""
-    backend = get_all(app).get(name)
+from .base import BaseBackend, HarvestExtraConfig, HarvestFeature, HarvestFilter  # noqa
+
+
+def get_backend(name: str) -> type[BaseBackend] | None:
+    backend = get_enabled_backends().get(name)
     if not backend:
-        msg = 'Harvest backend "{0}" is not registered'.format(name)
-        raise EntrypointError(msg)
+        raise ValueError(f"Backend {name} unknown. Make sure it is declared in HARVESTER_BACKENDS.")
     return backend
 
 
-def get_all(app):
-    return get_enabled("udata.harvesters", app)
+def get_all_backends() -> dict[str, type[BaseBackend]]:
+    # Note that we use the `BaseBackend.name` and not `ep.name`. The entrypoint name
+    # is not used anymore.
+    return {ep.load().name: ep.load() for ep in entry_points(group="udata.harvesters")}
+
+
+def is_backend_enabled(backend: type[BaseBackend]) -> bool:
+    return any(fnmatch(backend.name, g) for g in current_app.config["HARVESTER_BACKENDS"])
 
 
-from .base import BaseBackend, HarvestFeature, HarvestFilter, HarvestExtraConfig  # noqa
+def get_enabled_backends() -> dict[str, type[BaseBackend]]:
+    return {
+        name: backend for name, backend in get_all_backends().items() if is_backend_enabled(backend)
+    }

udata/harvest/backends/base.py

@@ -85,8 +85,8 @@ class BaseBackend(object):
     Also provides a few helpers needed on all or some backends.
     """
 
-    name = None
-    display_name = None
+    name: str
+    display_name: str | None = None
     verify_ssl = True
 
     # Define some allowed filters on the backend
@@ -166,6 +166,7 @@ class BaseBackend(object):
         log.debug(f"Starting harvesting {self.source.name} ({self.source.url})…")
         factory = HarvestJob if self.dryrun else HarvestJob.objects.create
         self.job = factory(status="initialized", started=datetime.utcnow(), source=self.source)
+        self.remote_ids = set()
 
         before_harvest_job.send(self)
         # Set harvest_activity_user on global context during the run
@@ -190,6 +191,7 @@ class BaseBackend(object):
 
             if any(i.status == "failed" for i in self.job.items):
                 self.job.status += "-errors"
+
         except HarvestValidationError as e:
             log.exception(
                 f'Harvesting validation failed for "{safe_unicode(self.source.name)}" ({self.source.backend})'
@@ -199,6 +201,15 @@ class BaseBackend(object):
 
             error = HarvestError(message=safe_unicode(e))
             self.job.errors.append(error)
+        except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e:
+            log.warning(
+                f'Harvesting connection error for "{safe_unicode(self.source.name)}" ({self.source.backend}): {e}'
+            )
+
+            self.job.status = "failed"
+
+            error = HarvestError(message=safe_unicode(e), details=traceback.format_exc())
+            self.job.errors.append(error)
         except Exception as e:
             log.exception(
                 f'Harvesting failed for "{safe_unicode(self.source.name)}" ({self.source.backend})'
@@ -232,8 +243,13 @@ class BaseBackend(object):
 
             current_app.logger.addHandler(log_catcher)
             dataset = self.inner_process_dataset(item, **kwargs)
+            if dataset.harvest:
+                item.remote_url = dataset.harvest.remote_url
+
+            # Use `item.remote_id` from this point, because `inner_process_dataset` could have modified it.
+
+            self.ensure_unique_remote_id(item)
 
-            # Use `item.remote_id` because `inner_process_dataset` could have modified it.
             dataset.harvest = self.update_dataset_harvest_info(dataset.harvest, item.remote_id)
             dataset.archived = None
 
@@ -291,6 +307,10 @@ class BaseBackend(object):
                 raise HarvestSkipException("missing identifier")
 
             dataservice = self.inner_process_dataservice(item, **kwargs)
+            if dataservice.harvest:
+                item.remote_url = dataservice.harvest.remote_url
+
+            self.ensure_unique_remote_id(item)
 
             dataservice.harvest = self.update_dataservice_harvest_info(
                 dataservice.harvest, remote_id
@@ -325,6 +345,12 @@ class BaseBackend(object):
             item.ended = datetime.utcnow()
             self.save_job()
 
+    def ensure_unique_remote_id(self, item):
+        if item.remote_id in self.remote_ids:
+            raise HarvestValidationError(f"Identifier '{item.remote_id}' already exists")
+
+        self.remote_ids.add(item.remote_id)
+
     def update_dataset_harvest_info(self, harvest: HarvestDatasetMetadata | None, remote_id: int):
         if not harvest:
             harvest = HarvestDatasetMetadata()

udata/harvest/backends/ckan/harvesters.py

@@ -25,6 +25,7 @@ ALLOWED_RESOURCE_TYPES = ("dkan", "file", "file.upload", "api", "metadata")
 
 
 class CkanBackend(BaseBackend):
+    name = "ckan"
     display_name = "CKAN"
     filters = (
         HarvestFilter(_("Organization"), "organization", str, _("A CKAN Organization name")),
@@ -172,7 +173,10 @@ class CkanBackend(BaseBackend):
                 continue
             elif key == "spatial":
                 # GeoJSON representation (Polygon or Point)
-                spatial_geom = json.loads(value)
+                if isinstance(value, dict):
+                    spatial_geom = value
+                else:
+                    spatial_geom = json.loads(value)
             elif key == "spatial-text":
                 # Textual representation of the extent / location
                 qs = GeoZone.objects(db.Q(name=value) | db.Q(slug=value))
@@ -212,12 +216,17 @@ class CkanBackend(BaseBackend):
             dataset.spatial.zones = [spatial_zone]
 
         if spatial_geom:
+            if "type" not in spatial_geom:
+                raise HarvestException(f"Spatial geometry {spatial_geom} without `type`")
+
             if spatial_geom["type"] == "Polygon":
                 coordinates = [spatial_geom["coordinates"]]
             elif spatial_geom["type"] == "MultiPolygon":
                 coordinates = spatial_geom["coordinates"]
             else:
-                raise HarvestException("Unsupported spatial geometry")
+                raise HarvestException(
+                    f"Unsupported spatial geometry {spatial_geom['type']} in {spatial_geom}. (Supported types are `Polygon` and `MultiPolygon`)"
+                )
             dataset.spatial.geom = {"type": "MultiPolygon", "coordinates": coordinates}
 
         if temporal_start and temporal_end:
@@ -265,5 +274,7 @@ class CkanBackend(BaseBackend):
 
 
 class DkanBackend(CkanBackend):
+    name = "dkan"
+    display_name = "DKAN"
     schema = dkan_schema
     filters = []

udata/harvest/backends/dcat.py

@@ -63,6 +63,7 @@ def extract_graph(source, target, node, specs):
 
 
 class DcatBackend(BaseBackend):
+    name = "dcat"
     display_name = "DCAT"
 
     def __init__(self, *args, **kwargs):
@@ -256,6 +257,7 @@ class CswDcatBackend(DcatBackend):
     The parsing of items is then the same as for the DcatBackend.
     """
 
+    name = "csw-dcat"
     display_name = "CSW-DCAT"
 
     # CSW_REQUEST is based on:
@@ -424,6 +426,7 @@ class CswIso19139DcatBackend(CswDcatBackend):
     The parsing of items is then the same as for the DcatBackend.
     """
 
+    name = "csw-iso-19139"
     display_name = "CSW-ISO-19139"
 
     extra_configs = (

udata/harvest/backends/maaf.py

@@ -129,6 +129,7 @@ def dictize(element):
 
 
 class MaafBackend(BaseBackend):
+    name = "maaf"
     display_name = "MAAF"
     verify_ssl = False
 

udata/harvest/commands.py

@@ -2,7 +2,9 @@ import logging
 
 import click
 
-from udata.commands import cli
+from udata.commands import KO, OK, cli, green, red
+from udata.harvest.backends import get_all_backends, is_backend_enabled
+from udata.models import Dataset
 
 from . import actions
 
@@ -89,9 +91,10 @@ def sources(scheduled=False):
 @grp.command()
 def backends():
     """List available backends"""
-    log.info("Available backends:")
-    for backend in actions.list_backends():
-        log.info("%s (%s)", backend.name, backend.display_name or backend.name)
+    print("Available backends:")
+    for backend in get_all_backends().values():
+        status = green(OK) if is_backend_enabled(backend) else red(KO)
+        click.echo("{0} {1} ({2})".format(status, backend.display_name, backend.name))
 
 
 @grp.command()
@@ -154,3 +157,35 @@ def attach(domain, filename):
     log.info("Attaching datasets for domain %s", domain)
     result = actions.attach(domain, filename)
     log.info("Attached %s datasets to %s", result.success, domain)
+
+
+@grp.command()
+@click.argument("dataset_id")
+def detach(dataset_id):
+    """
+    Detach a dataset_id from its harvest source
+
+    The dataset will be cleaned from harvested information
+    """
+    log.info(f"Detaching dataset {dataset_id}")
+    dataset = Dataset.get(dataset_id)
+    actions.detach(dataset)
+    log.info("Done")
+
+
+@grp.command()
+@click.argument("identifier")
+def detach_all_from_source(identifier):
+    """
+    Detach all datasets from a harvest source
+
+    All the datasets will be cleaned from harvested information.
+    Make sure the harvest source won't create new duplicate datasets,
+    either by deactivating it or filtering its scope, etc.
+    """
+    log.info(f"Detaching datasets from harvest source {identifier}")
+    count = actions.detach_all_from_source(actions.get_source(identifier))
+    log.info(f"Detached {count} datasets")
+    log.warning(
+        "Make sure the harvest source won't create new duplicate datasets, either by deactivating it or filtering its scope, etc."
+    )