udata 10.8.1.dev36703__py2.py3-none-any.whl → 10.8.2__py2.py3-none-any.whl

udata/harvest/api.py

@@ -300,59 +300,54 @@ class SourcesAPI(API):
         return source, 201
 
 
-@ns.route("/source/<string:ident>", endpoint="harvest_source")
-@api.param("ident", "A source ID or slug")
+@ns.route("/source/<harvest_source:source>/", endpoint="harvest_source")
 class SourceAPI(API):
     @api.doc("get_harvest_source")
     @api.marshal_with(source_fields)
-    def get(self, ident):
+    def get(self, source: HarvestSource):
         """Get a single source given an ID or a slug"""
-        return actions.get_source(ident)
+        return source
 
     @api.secure
     @api.doc("update_harvest_source")
     @api.expect(source_fields)
     @api.marshal_with(source_fields)
-    def put(self, ident):
+    def put(self, source: HarvestSource):
         """Update a harvest source"""
-        source = actions.get_source(ident)
         OwnablePermission(source).test()
         form = api.validate(HarvestSourceForm, source)
-        source = actions.update_source(ident, form.data)
+        source = actions.update_source(source, form.data)
         return source
 
     @api.secure
     @api.doc("delete_harvest_source")
     @api.marshal_with(source_fields)
-    def delete(self, ident):
-        source: HarvestSource = actions.get_source(ident)
+    def delete(self, source: HarvestSource):
         OwnablePermission(source).test()
-        return actions.delete_source(ident), 204
+        return actions.delete_source(source), 204
 
 
-@ns.route("/source/<string:ident>/validate", endpoint="validate_harvest_source")
-@api.param("ident", "A source ID or slug")
+@ns.route("/source/<harvest_source:source>/validate/", endpoint="validate_harvest_source")
 class ValidateSourceAPI(API):
     @api.doc("validate_harvest_source")
     @api.secure(admin_permission)
     @api.expect(validation_fields)
     @api.marshal_with(source_fields)
-    def post(self, ident):
+    def post(self, source: HarvestSource):
         """Validate or reject an harvest source"""
         form = api.validate(HarvestSourceValidationForm)
         if form.state.data == VALIDATION_ACCEPTED:
-            return actions.validate_source(ident, form.comment.data)
+            return actions.validate_source(source, form.comment.data)
         else:
-            return actions.reject_source(ident, form.comment.data)
+            return actions.reject_source(source, form.comment.data)
 
 
-@ns.route("/source/<string:ident>/run", endpoint="run_harvest_source")
-@api.param("ident", "A source ID or slug")
+@ns.route("/source/<harvest_source:source>/run/", endpoint="run_harvest_source")
 class RunSourceAPI(API):
     @api.doc("run_harvest_source")
     @api.secure
     @api.marshal_with(source_fields)
-    def post(self, ident):
+    def post(self, source: HarvestSource):
         enabled = current_app.config.get("HARVEST_ENABLE_MANUAL_RUN")
         if not enabled and not current_user.sysadmin:
             api.abort(
@@ -360,42 +355,40 @@ class RunSourceAPI(API):
                 "Cannot run source manually. Please contact the platform if you need to reschedule the harvester.",
             )
 
-        source: HarvestSource = actions.get_source(ident)
         OwnablePermission(source).test()
 
         if source.validation.state != VALIDATION_ACCEPTED:
             api.abort(400, "Source is not validated. Please validate the source before running.")
 
-        actions.launch(ident)
+        actions.launch(source)
 
         return source
 
 
-@ns.route("/source/<string:ident>/schedule", endpoint="schedule_harvest_source")
-@api.param("ident", "A source ID or slug")
+@ns.route("/source/<harvest_source:source>/schedule/", endpoint="schedule_harvest_source")
 class ScheduleSourceAPI(API):
     @api.doc("schedule_harvest_source")
     @api.secure(admin_permission)
     @api.expect((str, "A cron expression"))
     @api.marshal_with(source_fields)
-    def post(self, ident):
+    def post(self, source: HarvestSource):
         """Schedule an harvest source"""
         # Handle both syntax: quoted and unquoted
         try:
             data = request.json
         except BadRequest:
             data = request.data.decode("utf-8")
-        return actions.schedule(ident, data)
+        return actions.schedule(source, data)
 
     @api.doc("unschedule_harvest_source")
     @api.secure(admin_permission)
     @api.marshal_with(source_fields)
-    def delete(self, ident):
+    def delete(self, source: HarvestSource):
         """Unschedule an harvest source"""
-        return actions.unschedule(ident), 204
+        return actions.unschedule(source), 204
 
 
-@ns.route("/source/preview", endpoint="preview_harvest_source_config")
+@ns.route("/source/preview/", endpoint="preview_harvest_source_config")
 class PreviewSourceConfigAPI(API):
     @api.secure
     @api.expect(source_fields)
@@ -409,15 +402,14 @@ class PreviewSourceConfigAPI(API):
         return actions.preview_from_config(**form.data)
 
 
-@ns.route("/source/<string:ident>/preview", endpoint="preview_harvest_source")
-@api.param("ident", "A source ID or slug")
+@ns.route("/source/<harvest_source:source>/preview/", endpoint="preview_harvest_source")
 class PreviewSourceAPI(API):
     @api.secure
     @api.doc("preview_harvest_source")
     @api.marshal_with(preview_job_fields)
-    def get(self, ident):
+    def get(self, source: HarvestSource):
         """Preview a single harvest source given an ID or a slug"""
-        return actions.preview(ident)
+        return actions.preview(source)
 
 
 parser = api.parser()
@@ -427,15 +419,15 @@ parser.add_argument(
 )
 
 
-@ns.route("/source/<string:ident>/jobs/", endpoint="harvest_jobs")
+@ns.route("/source/<harvest_source:source>/jobs/", endpoint="harvest_jobs")
 class JobsAPI(API):
     @api.doc("list_harvest_jobs")
     @api.expect(parser)
     @api.marshal_with(job_page_fields)
-    def get(self, ident):
+    def get(self, source: HarvestSource):
         """List all jobs for a given source"""
         args = parser.parse_args()
-        qs = HarvestJob.objects(source=ident)
+        qs = HarvestJob.objects(source=source)
         qs = qs.order_by("-created")
         return qs.paginate(args["page"], args["page_size"])
 
@@ -450,7 +442,7 @@ class JobAPI(API):
         return actions.get_job(ident)
 
 
-@ns.route("/backends", endpoint="harvest_backends")
+@ns.route("/backends/", endpoint="harvest_backends")
 class ListBackendsAPI(API):
     @api.doc("harvest_backends")
     @api.marshal_with(backend_fields)
@@ -471,7 +463,7 @@ class ListBackendsAPI(API):
         )
 
 
-@ns.route("/job_status", endpoint="havest_job_status")
+@ns.route("/job_status/", endpoint="havest_job_status")
 class ListHarvesterAPI(API):
     @api.doc(model=[str])
     def get(self):

udata/harvest/backends/ckan/__init__.py

@@ -0,0 +1,3 @@
+"""
+CKAN integration for udata
+"""

udata/harvest/backends/ckan/harvesters.py

@@ -0,0 +1,274 @@
+import json
+import logging
+from urllib.parse import urljoin
+from uuid import UUID
+
+from udata import uris
+from udata.harvest.models import HarvestItem
+from udata.i18n import lazy_gettext as _
+
+try:
+    from udata.core.dataset.constants import UPDATE_FREQUENCIES
+except ImportError:
+    # legacy import of constants in udata
+    from udata.models import UPDATE_FREQUENCIES
+from udata.core.dataset.models import HarvestDatasetMetadata, HarvestResourceMetadata
+from udata.core.dataset.rdf import frequency_from_rdf
+from udata.frontend.markdown import parse_html
+from udata.harvest.backends.base import BaseBackend, HarvestFilter
+from udata.harvest.exceptions import HarvestException, HarvestSkipException
+from udata.models import GeoZone, License, Resource, SpatialCoverage, db
+from udata.utils import daterange_end, daterange_start, get_by
+
+from .schemas.ckan import schema as ckan_schema
+from .schemas.dkan import schema as dkan_schema
+
+log = logging.getLogger(__name__)
+
+# dkan is a dummy value for dkan that does not provide resource_type
+ALLOWED_RESOURCE_TYPES = ("dkan", "file", "file.upload", "api", "metadata")
+
+
+class CkanBackend(BaseBackend):
+    display_name = "CKAN"
+    filters = (
+        HarvestFilter(_("Organization"), "organization", str, _("A CKAN Organization name")),
+        HarvestFilter(_("Tag"), "tags", str, _("A CKAN tag name")),
+    )
+    schema = ckan_schema
+
+    def get_headers(self):
+        headers = super(CkanBackend, self).get_headers()
+        headers["content-type"] = "application/json"
+        if self.config.get("apikey"):
+            headers["Authorization"] = self.config["apikey"]
+        return headers
+
+    def action_url(self, endpoint):
+        path = "/".join(["api/3/action", endpoint])
+        return urljoin(self.source.url, path)
+
+    def dataset_url(self, name):
+        path = "/".join(["dataset", name])
+        return urljoin(self.source.url, path)
+
+    def get_action(self, endpoint, fix=False, **kwargs):
+        url = self.action_url(endpoint)
+        if fix:
+            response = self.post(url, "{}", params=kwargs)
+        else:
+            response = self.get(url, params=kwargs)
+
+        response.raise_for_status()
+        content_type = response.headers.get("Content-Type", "")
+        mime_type = content_type.split(";", 1)[0]
+
+        if mime_type == "application/json":  # Standard API JSON response
+            data = response.json()
+            # CKAN API can returns 200 even on errors
+            # Only the `success` property allows to detect errors
+            if data.get("success", False):
+                return data
+            else:
+                error = data.get("error")
+                if isinstance(error, dict):
+                    # Error object with message
+                    msg = error.get("message", "Unknown error")
+                    if "__type" in error:
+                        # Typed error
+                        msg = ": ".join((error["__type"], msg))
+                else:
+                    # Error only contains a message
+                    msg = error
+                raise HarvestException(msg)
+
+        elif mime_type == "text/html":  # Standard html error page
+            raise HarvestException("Unknown Error: {} returned HTML".format(url))
+        else:
+            # If it's not HTML, CKAN respond with raw quoted text
+            msg = response.text.strip('"')
+            raise HarvestException(msg)
+
+    def get_status(self):
+        url = urljoin(self.source.url, "/api/util/status")
+        response = self.get(url)
+        return response.json()
+
+    def inner_harvest(self):
+        """List all datasets for a given ..."""
+        fix = False  # Fix should be True for CKAN < '1.8'
+
+        filters = self.config.get("filters", [])
+        if len(filters) > 0:
+            # Build a q search query based on filters
+            # use package_search because package_list doesn't allow filtering
+            # use q parameters because fq is broken with multiple filters
+            params = []
+            for f in filters:
+                param = "{key}:{value}".format(**f)
+                if f.get("type") == "exclude":
+                    param = "-" + param
+                params.append(param)
+            q = " AND ".join(params)
+            # max out rows count to 1000 as per
+            # https://docs.ckan.org/en/latest/api/#ckan.logic.action.get.package_search
+            response = self.get_action("package_search", fix=fix, q=q, rows=1000)
+            names = [r["name"] for r in response["result"]["results"]]
+        else:
+            response = self.get_action("package_list", fix=fix)
+            names = response["result"]
+
+        for name in names:
+            # We use `name` as `remote_id` for now, we'll be replace at the beginning of the process
+            self.process_dataset(name)
+            if self.has_reached_max_items():
+                return
+
+    def inner_process_dataset(self, item: HarvestItem):
+        response = self.get_action("package_show", id=item.remote_id)
+
+        result = response["result"]
+        # DKAN returns a list where CKAN returns an object
+        # we "unlist" here instead of after schema validation in order to get the id easily
+        if type(result) is list:
+            result = result[0]
+
+        # Replace the `remote_id` from `name` to `id`.
+        if result.get("id"):
+            item.remote_id = result["id"]
+
+        data = self.validate(result, self.schema)
+
+        # Skip if no resource
+        if not len(data.get("resources", [])):
+            raise HarvestSkipException(f"Dataset {data['name']} has no record")
+
+        dataset = self.get_dataset(item.remote_id)
+
+        if not dataset.harvest:
+            dataset.harvest = HarvestDatasetMetadata()
+
+        # Core attributes
+        if not dataset.slug:
+            dataset.slug = data["name"]
+        dataset.title = data["title"]
+        dataset.description = parse_html(data["notes"])
+
+        # Detect license
+        default_license = dataset.license or License.default()
+        dataset.license = License.guess(
+            data["license_id"], data["license_title"], default=default_license
+        )
+
+        dataset.tags = [t["name"] for t in data["tags"] if t["name"]]
+
+        dataset.harvest.created_at = data["metadata_created"]
+        dataset.harvest.modified_at = data["metadata_modified"]
+
+        dataset.harvest.ckan_name = data["name"]
+
+        temporal_start, temporal_end = None, None
+        spatial_geom, spatial_zone = None, None
+
+        for extra in data["extras"]:
+            key = extra["key"]
+            value = extra["value"]
+            if value is None or (isinstance(value, str) and not value.strip()):
+                # Skip empty extras
+                continue
+            elif key == "spatial":
+                # GeoJSON representation (Polygon or Point)
+                spatial_geom = json.loads(value)
+            elif key == "spatial-text":
+                # Textual representation of the extent / location
+                qs = GeoZone.objects(db.Q(name=value) | db.Q(slug=value))
+                if qs.count() == 1:
+                    spatial_zone = qs.first()
+                else:
+                    dataset.extras["ckan:spatial-text"] = value
+                    log.debug("spatial-text value not handled: %s", value)
+            elif key == "spatial-uri":
+                # Linked Data URI representing the place name
+                dataset.extras["ckan:spatial-uri"] = value
+                log.debug("spatial-uri value not handled: %s", value)
+            elif key == "frequency":
+                # Update frequency
+                freq = frequency_from_rdf(value)
+                if freq:
+                    dataset.frequency = freq
+                elif value in UPDATE_FREQUENCIES:
+                    dataset.frequency = value
+                else:
+                    dataset.extras["ckan:frequency"] = value
+                    log.debug("frequency value not handled: %s", value)
+            # Temporal coverage start
+            elif key == "temporal_start":
+                temporal_start = daterange_start(value)
+            # Temporal coverage end
+            elif key == "temporal_end":
+                temporal_end = daterange_end(value)
+            else:
+                dataset.extras[extra["key"]] = value
+
+        if spatial_geom or spatial_zone:
+            dataset.spatial = SpatialCoverage()
+
+        if spatial_zone:
+            dataset.spatial.zones = [spatial_zone]
+
+        if spatial_geom:
+            if spatial_geom["type"] == "Polygon":
+                coordinates = [spatial_geom["coordinates"]]
+            elif spatial_geom["type"] == "MultiPolygon":
+                coordinates = spatial_geom["coordinates"]
+            else:
+                raise HarvestException("Unsupported spatial geometry")
+            dataset.spatial.geom = {"type": "MultiPolygon", "coordinates": coordinates}
+
+        if temporal_start and temporal_end:
+            dataset.temporal_coverage = db.DateRange(
+                start=temporal_start,
+                end=temporal_end,
+            )
+
+        # Remote URL
+        dataset.harvest.remote_url = self.dataset_url(data["name"])
+        if data.get("url"):
+            try:
+                url = uris.validate(data["url"])
+            except uris.ValidationError:
+                dataset.harvest.ckan_source = data["url"]
+            else:
+                # use declared `url` as `remote_url` if any
+                dataset.harvest.remote_url = url
+
+        # Resources
+        for res in data["resources"]:
+            if res["resource_type"] not in ALLOWED_RESOURCE_TYPES:
+                continue
+            try:
+                resource = get_by(dataset.resources, "id", UUID(res["id"]))
+            except Exception:
+                log.error("Unable to parse resource ID %s", res["id"])
+                continue
+            if not resource:
+                resource = Resource(id=res["id"])
+                dataset.resources.append(resource)
+            if not resource.harvest:
+                resource.harvest = HarvestResourceMetadata()
+            resource.title = res.get("name", "") or ""
+            resource.description = parse_html(res.get("description"))
+            resource.url = res["url"]
+            resource.filetype = "remote"
+            resource.format = res.get("format")
+            resource.mime = res.get("mimetype")
+            resource.hash = res.get("hash")
+            resource.harvest.created_at = res["created"]
+            resource.harvest.modified_at = res["last_modified"]
+
+        return dataset
+
+
+class DkanBackend(CkanBackend):
+    schema = dkan_schema
+    filters = []

udata/harvest/backends/ckan/schemas/ckan.py

@@ -0,0 +1,86 @@
+from voluptuous import All, Any, Coerce, DefaultTo, Lower, Optional, Schema
+
+from udata.harvest.filters import (
+    boolean,
+    email,
+    empty_none,
+    hash,
+    is_url,
+    normalize_string,
+    normalize_tag,
+    slug,
+    to_date,
+)
+
+RESOURCE_TYPES = ("file", "file.upload", "api", "documentation", "image", "visualization")
+
+
+resource = {
+    "id": str,
+    "position": int,
+    "name": All(DefaultTo(""), str),
+    "description": Any(All(str, normalize_string), None),
+    "format": All(str, Lower),
+    "mimetype": Any(All(str, Lower), None),
+    "size": Any(Coerce(int), None),
+    "hash": Any(All(str, hash), None),
+    "created": All(str, to_date),
+    "last_modified": Any(All(str, to_date), None),
+    "url": All(str, is_url()),
+    "resource_type": All(empty_none, DefaultTo("file"), str, Any(*RESOURCE_TYPES)),
+}
+
+tag = {
+    "id": str,
+    Optional("vocabulary_id"): Any(str, None),
+    Optional("display_name"): str,
+    "name": All(str, normalize_tag),
+    Optional("state"): str,
+}
+
+organization = {
+    "id": str,
+    "description": str,
+    "created": All(str, to_date),
+    "title": str,
+    "name": All(str, slug),
+    "revision_timestamp": All(str, to_date),
+    "is_organization": boolean,
+    "state": str,
+    "image_url": str,
+    "revision_id": str,
+    "type": "organization",
+    "approval_status": "approved",
+}
+
+schema = Schema(
+    {
+        "id": str,
+        "name": str,
+        "title": str,
+        "notes": Any(All(str, normalize_string), None),
+        "license_id": All(DefaultTo("not-specified"), str),
+        "license_title": Any(str, None),
+        "tags": [tag],
+        "metadata_created": All(str, to_date),
+        "metadata_modified": All(str, to_date),
+        "organization": Any(organization, None),
+        "resources": [resource],
+        Optional("revision_id"): str,
+        Optional("extras", default=list): [
+            {
+                "key": str,
+                "value": Any(str, int, float, boolean, dict, list),
+            }
+        ],
+        "private": boolean,
+        "type": "dataset",
+        "author": Any(str, None),
+        "author_email": All(empty_none, Any(All(str, email), None)),
+        "maintainer": Any(str, None),
+        "maintainer_email": All(empty_none, Any(All(str, email), None)),
+        "state": Any(str, None),
+    },
+    required=True,
+    extra=True,
+)

udata/harvest/backends/ckan/schemas/dkan.py

@@ -0,0 +1,98 @@
+import dateutil.parser
+from humanfriendly import parse_size
+from voluptuous import All, Any, DefaultTo, Lower, Optional, Schema
+
+from udata.harvest.filters import boolean, email, empty_none, hash, is_url, normalize_string, slug
+
+from .ckan import tag
+
+
+class FrenchParserInfo(dateutil.parser.parserinfo):
+    WEEKDAYS = [
+        ("Lun", "Lundi"),
+        ("Mar", "Mardi"),
+        ("Mer", "Mercredi"),
+        ("Jeu", "Jeudi"),
+        ("Ven", "Vendredi"),
+        ("Sam", "Samedi"),
+        ("Dim", "Dimanche"),
+    ]
+
+
+def parse_date(value, **kwargs):
+    return dateutil.parser.parse(value, **kwargs).date()
+
+
+def to_date(value):
+    """
+    Try w/ french weekdays then dateutil's default
+    `fuzzy` is used when 'Date changed' is in the value
+    """
+    try:
+        return parse_date(value, fuzzy=True, parserinfo=FrenchParserInfo(), dayfirst=True)
+    except ValueError:
+        return parse_date(value, fuzzy=True)
+
+
+def dkan_parse_size(value):
+    if value:
+        # not strictly true but should be enough
+        value = value.replace("octets", "bytes")
+        return parse_size(value)
+
+
+resource = {
+    "id": str,
+    "name": All(DefaultTo(""), str),
+    "description": All(str, normalize_string),
+    "format": All(str, Lower),
+    "mimetype": Any(All(str, Lower), None),
+    "size": All(str, dkan_parse_size),
+    Optional("hash"): Any(All(str, hash), None),
+    "created": All(str, to_date),
+    "last_modified": Any(All(str, to_date), None),
+    "url": All(str, is_url()),
+    Optional("resource_type", default="dkan"): All(
+        empty_none,
+        str,
+    ),
+}
+
+group = {
+    "id": str,
+    "description": str,
+    "image_display_url": str,
+    "title": str,
+    "name": All(str, slug),
+}
+
+schema = Schema(
+    {
+        "id": str,
+        "name": str,
+        "title": str,
+        "notes": Any(All(str, normalize_string), None),
+        Optional("license_id", default=None): All(DefaultTo("not-specified"), str),
+        Optional("license_title", default=None): Any(str, None),
+        Optional("tags", default=list): [tag],
+        "metadata_created": All(str, to_date),
+        "metadata_modified": All(str, to_date),
+        Optional("groups"): [Any(group, None)],
+        "resources": [resource],
+        Optional("extras", default=list): [
+            {
+                "key": str,
+                "value": Any(str, int, float, boolean, dict, list),
+            }
+        ],
+        "private": boolean,
+        "type": "Dataset",
+        Optional("author"): Any(str, None),
+        Optional("author_email"): All(empty_none, Any(All(str, email), None)),
+        "maintainer": Any(str, None),
+        "maintainer_email": All(empty_none, Any(All(str, email), None)),
+        "state": Any(str, None),
+    },
+    required=True,
+    extra=True,
+)