udata 10.1.3.dev34224__py2.py3-none-any.whl → 10.1.3.dev34275__py2.py3-none-any.whl

udata/commands/fixtures.py

@@ -39,7 +39,7 @@ COMMUNITY_RES_URL = "/api/1/datasets/community_resources"
 DISCUSSION_URL = "/api/1/discussions"
 
 
-DEFAULT_FIXTURE_FILE_TAG: str = "v5.0.0"
+DEFAULT_FIXTURE_FILE_TAG: str = "v6.0.0"
 DEFAULT_FIXTURE_FILE: str = f"https://raw.githubusercontent.com/opendatateam/udata-fixtures/{DEFAULT_FIXTURE_FILE_TAG}/results.json"  # noqa
 
 DEFAULT_FIXTURES_RESULTS_FILENAME: str = "results.json"

udata/core/dataservices/constants.py

@@ -0,0 +1,11 @@
+DATASERVICE_FORMATS = ["REST", "WMS", "WSL"]
+
+
+DATASERVICE_ACCESS_TYPE_OPEN = "open"
+DATASERVICE_ACCESS_TYPE_OPEN_WITH_ACCOUNT = "open_with_account"
+DATASERVICE_ACCESS_TYPE_RESTRICTED = "restricted"
+DATASERVICE_ACCESS_TYPES = [
+    DATASERVICE_ACCESS_TYPE_OPEN,
+    DATASERVICE_ACCESS_TYPE_OPEN_WITH_ACCOUNT,
+    DATASERVICE_ACCESS_TYPE_RESTRICTED,
+]

udata/core/dataservices/csv.py

@@ -13,13 +13,13 @@ class DataserviceCsvAdapter(csv.Adapter):
         ("url", lambda d: d.self_web_url()),
         "description",
         "base_api_url",
-        "endpoint_description_url",
+        "machine_documentation_url",
+        "technical_documentation_url",
         "business_documentation_url",
         "authorization_request_url",
         "availability",
         "rate_limiting",
-        "is_restricted",
-        "has_token",
+        "access_type",
         "license",
         ("organization", "organization.name"),
         ("organization_id", "organization.id"),

udata/core/dataservices/models.py

@@ -8,6 +8,7 @@ from mongoengine.signals import post_save
 import udata.core.contact_point.api_fields as contact_api_fields
 import udata.core.dataset.api_fields as datasets_api_fields
 from udata.api_fields import field, function_field, generate_fields
+from udata.core.dataservices.constants import DATASERVICE_ACCESS_TYPES, DATASERVICE_FORMATS
 from udata.core.dataset.models import Dataset
 from udata.core.metrics.models import WithMetrics
 from udata.core.owned import Owned, OwnedQuerySet
@@ -24,8 +25,6 @@ from udata.uris import endpoint_for
 # "spatial"
 # "temporal_coverage"
 
-DATASERVICE_FORMATS = ["REST", "WMS", "WSL"]
-
 
 class DataserviceQuerySet(OwnedQuerySet):
     def visible(self):
@@ -100,11 +99,17 @@ class HarvestMetadata(db.EmbeddedDocument):
 @generate_fields(
     searchable=True,
     additional_filters={"organization_badge": "organization.badges"},
+    additional_sorts=[
+        {"key": "followers", "value": "metrics.followers"},
+        {"key": "views", "value": "metrics.views"},
+    ],
 )
 class Dataservice(WithMetrics, Owned, db.Document):
     meta = {
         "indexes": [
             "$title",
+            "metrics.followers",
+            "metrics.views",
         ]
         + Owned.meta["indexes"],
         "queryset_class": DataserviceQuerySet,
@@ -132,13 +137,22 @@ class Dataservice(WithMetrics, Owned, db.Document):
     )
     description = field(db.StringField(default=""), description="In markdown")
     base_api_url = field(db.URLField(), sortable=True)
-    endpoint_description_url = field(db.URLField())
+
+    machine_documentation_url = field(
+        db.URLField(), description="Swagger link, OpenAPI format, WMS XML…"
+    )
+    technical_documentation_url = field(db.URLField(), description="HTML version of a Swagger…")
     business_documentation_url = field(db.URLField())
-    authorization_request_url = field(db.URLField())
-    availability = field(db.FloatField(min=0, max=100), example="99.99")
+
     rate_limiting = field(db.StringField())
-    is_restricted = field(db.BooleanField(), filterable={})
-    has_token = field(db.BooleanField())
+    rate_limiting_url = field(db.URLField())
+
+    availability = field(db.FloatField(min=0, max=100), example="99.99")
+    availability_url = field(db.URLField())
+
+    access_type = field(db.StringField(choices=DATASERVICE_ACCESS_TYPES), filterable={})
+    authorization_request_url = field(db.URLField())
+
     format = field(db.StringField(choices=DATASERVICE_FORMATS))
 
     license = field(
@@ -223,11 +237,11 @@ class Dataservice(WithMetrics, Owned, db.Document):
     def self_web_url(self):
         return endpoint_for("dataservices.show", dataservice=self, _external=True)
 
-    # TODO
-    # frequency = db.StringField(choices=list(UPDATE_FREQUENCIES.keys()))
-    # temporal_coverage = db.EmbeddedDocumentField(db.DateRange)
-    # spatial = db.EmbeddedDocumentField(SpatialCoverage)
-    # harvest = db.EmbeddedDocumentField(HarvestDatasetMetadata)
+    __metrics_keys__ = [
+        "discussions",
+        "followers",
+        "views",
+    ]
 
     @property
     def is_hidden(self):

udata/core/dataservices/rdf.py

@@ -42,7 +42,9 @@ def dataservice_from_rdf(
     dataservice.description = sanitize_html(d.value(DCT.description) or d.value(DCT.abstract))
 
     dataservice.base_api_url = url_from_rdf(d, DCAT.endpointURL)
-    dataservice.endpoint_description_url = url_from_rdf(d, DCAT.endpointDescription)
+
+    # TODO detect if it's human-readable or not?
+    dataservice.machine_documentation_url = url_from_rdf(d, DCAT.endpointDescription)
 
     roles = [  # Imbricated list of contact points for each role
         contact_points_from_rdf(d, rdf_entity, role, dataservice)
@@ -145,8 +147,8 @@ def dataservice_to_rdf(dataservice: Dataservice, graph=None):
             ),
         )
 
-    if dataservice.endpoint_description_url:
-        d.set(DCAT.endpointDescription, URIRef(dataservice.endpoint_description_url))
+    if dataservice.machine_documentation_url:
+        d.set(DCAT.endpointDescription, URIRef(dataservice.machine_documentation_url))
 
     # Add DCAT-AP HVD properties if the dataservice is tagged hvd.
     # See https://semiceu.github.io/DCAT-AP/releases/2.2.0-hvd/

udata/core/dataservices/search.py

@@ -5,6 +5,11 @@ from flask_restx.inputs import boolean
 
 from udata.api import api
 from udata.api.parsers import ModelApiParser
+from udata.core.dataservices.constants import (
+    DATASERVICE_ACCESS_TYPE_OPEN,
+    DATASERVICE_ACCESS_TYPE_OPEN_WITH_ACCOUNT,
+    DATASERVICE_ACCESS_TYPE_RESTRICTED,
+)
 from udata.models import Dataservice, Organization, User
 from udata.search import (
     BoolFilter,
@@ -47,7 +52,11 @@ class DataserviceApiParser(ModelApiParser):
                 api.abort(400, "Organization arg must be an identifier")
             dataservices = dataservices.filter(organization=args["organization"])
         if "is_restricted" in args:
-            dataservices = dataservices.filter(is_restricted=boolean(args["is_restricted"]))
+            dataservices = dataservices.filter(
+                access_type__in=[DATASERVICE_ACCESS_TYPE_RESTRICTED]
+                if boolean(args["is_restricted"])
+                else [DATASERVICE_ACCESS_TYPE_OPEN, DATASERVICE_ACCESS_TYPE_OPEN_WITH_ACCOUNT]
+            )
         return dataservices
 
 
@@ -56,9 +65,7 @@ class DataserviceSearch(ModelSearchAdapter):
     model = Dataservice
     search_url = "dataservices/"
 
-    sorts = {
-        "created": "created_at",
-    }
+    sorts = {"created": "created_at", "views": "views", "followers": "followers"}
 
     filters = {
         "tag": Filter(),
@@ -114,5 +121,6 @@ class DataserviceSearch(ModelSearchAdapter):
             "tags": dataservice.tags,
             "extras": extras,
             "followers": dataservice.metrics.get("followers", 0),
-            "is_restricted": dataservice.is_restricted or False,
+            "is_restricted": dataservice.access_type == DATASERVICE_ACCESS_TYPE_RESTRICTED,
+            "views": dataservice.metrics.get("views", 0),
         }

udata/core/metrics/commands.py

@@ -4,6 +4,7 @@ import click
 from flask import current_app
 
 from udata.commands import cli, success
+from udata.core.dataservices.models import Dataservice
 from udata.models import Dataset, GeoZone, Organization, Reuse, Site, User
 
 log = logging.getLogger(__name__)
@@ -19,6 +20,7 @@ def grp():
 @click.option("-s", "--site", is_flag=True, help="Update site metrics")
 @click.option("-o", "--organizations", is_flag=True, help="Compute organizations metrics")
 @click.option("-d", "--datasets", is_flag=True, help="Compute datasets metrics")
+@click.option("--dataservices", is_flag=True, help="Compute dataservices metrics")
 @click.option("-r", "--reuses", is_flag=True, help="Compute reuses metrics")
 @click.option("-u", "--users", is_flag=True, help="Compute users metrics")
 @click.option("-g", "--geozones", is_flag=True, help="Compute geo levels metrics")
@@ -28,12 +30,13 @@ def update(
     organizations=False,
     users=False,
     datasets=False,
+    dataservices=False,
     reuses=False,
     geozones=False,
     drop=False,
 ):
     """Update all metrics for the current date"""
-    do_all = not any((site, organizations, users, datasets, reuses, geozones))
+    do_all = not any((site, organizations, users, datasets, dataservices, reuses, geozones))
 
     if do_all or site:
         log.info("Update site metrics")
@@ -75,6 +78,22 @@ def update(
                     log.info(f"Error during update: {e}")
                     continue
 
+    if do_all or dataservices:
+        log.info("Update dataservices metrics")
+        all_dataservices = Dataservice.objects.visible().timeout(False)
+        with click.progressbar(
+            all_dataservices, length=Dataservice.objects.count()
+        ) as dataservice_bar:
+            for dataservice in dataservice_bar:
+                try:
+                    if drop:
+                        dataservice.metrics.clear()
+                    dataservice.count_discussions()
+                    dataservice.count_followers()
+                except Exception as e:
+                    log.info(f"Error during update: {e}")
+                    continue
+
     if do_all or reuses:
         log.info("Update reuses metrics")
         all_reuses = Reuse.objects.visible().timeout(False)

udata/harvest/tests/test_dcat_backend.py

@@ -179,7 +179,7 @@ class DcatBackendTest:
         assert dataservices[0].title == "Explore API v2"
         assert dataservices[0].base_api_url == "https://data.paris2024.org/api/explore/v2.1/"
         assert (
-            dataservices[0].endpoint_description_url
+            dataservices[0].machine_documentation_url
             == "https://data.paris2024.org/api/explore/v2.1/swagger.json"
         )
         assert (

udata/migrations/2025-01-05-dataservices-fields-changes.py

@@ -0,0 +1,136 @@
+"""
+This migration keeps only the "Local authority" badge if the organization also has the "Public service" badge.
+"""
+
+import logging
+from typing import List
+
+from mongoengine.connection import get_db
+
+from udata.core.dataservices.constants import (
+    DATASERVICE_ACCESS_TYPE_OPEN,
+    DATASERVICE_ACCESS_TYPE_OPEN_WITH_ACCOUNT,
+    DATASERVICE_ACCESS_TYPE_RESTRICTED,
+)
+from udata.core.dataservices.models import Dataservice
+
+log = logging.getLogger(__name__)
+
+
+def migrate(db):
+    log.info("Preprocessing dataservices…")
+
+    count = get_db().dataservice.update_many(
+        filter={
+            "$or": [
+                {"is_restricted": None},
+                {"is_restricted": {"$exists": False}},
+            ]
+        },
+        update={"$set": {"is_restricted": False}},
+    )
+    log.info(
+        f"\tConverted {count.modified_count} dataservices from `is_restricted=None` to `is_restricted=False`"
+    )
+
+    count = get_db().dataservice.update_many(
+        filter={
+            "$or": [
+                {"has_token": None},
+                {"has_token": {"$exists": False}},
+            ]
+        },
+        update={"$set": {"has_token": False}},
+    )
+    log.info(
+        f"\tConverted {count.modified_count} dataservices from `has_token=None` to `has_token=False`"
+    )
+
+    for dataservice in get_db().dataservice.find({"is_restricted": True, "has_token": False}):
+        log.info(
+            f"\tDataservice #{dataservice['_id']} {dataservice['title']} is restricted but without token. (will be set to access_type={DATASERVICE_ACCESS_TYPE_RESTRICTED})"
+        )
+
+    log.info("Processing dataservices…")
+
+    count = get_db().dataservice.update_many(
+        filter={
+            "is_restricted": True,
+            # `has_token` could be True or False, we don't care
+        },
+        update={"$set": {"access_type": DATASERVICE_ACCESS_TYPE_RESTRICTED}},
+    )
+    log.info(
+        f"\t{count.modified_count} restricted dataservices to DATASERVICE_ACCESS_TYPE_RESTRICTED"
+    )
+
+    count = get_db().dataservice.update_many(
+        filter={
+            "is_restricted": False,
+            "has_token": True,
+        },
+        update={"$set": {"access_type": DATASERVICE_ACCESS_TYPE_OPEN_WITH_ACCOUNT}},
+    )
+    log.info(
+        f"\t{count.modified_count} dataservices not restricted but with token to DATASERVICE_ACCESS_TYPE_OPEN_WITH_ACCOUNT"
+    )
+
+    count = get_db().dataservice.update_many(
+        filter={
+            "is_restricted": False,
+            "has_token": False,
+        },
+        update={"$set": {"access_type": DATASERVICE_ACCESS_TYPE_OPEN}},
+    )
+    log.info(f"\t{count.modified_count} open dataservices to DATASERVICE_ACCESS_TYPE_OPEN")
+
+    dataservices: List[Dataservice] = get_db().dataservice.find()
+    for dataservice in dataservices:
+        if (
+            "endpoint_description_url" not in dataservice
+            or not dataservice["endpoint_description_url"]
+        ):
+            continue
+
+        to_set = {}
+        if (
+            dataservice["endpoint_description_url"].endswith(".json")
+            or dataservice["endpoint_description_url"].endswith(".yaml")
+            or dataservice["endpoint_description_url"].endswith(".yml")
+            or dataservice["endpoint_description_url"].endswith("?format=openapi-json")
+            or "getcapabilities" in dataservice["endpoint_description_url"].lower()
+            or "getresourcedescription" in dataservice["endpoint_description_url"].lower()
+            or dataservice["endpoint_description_url"].startswith(
+                "https://api.insee.fr/catalogue/api-docs/carbon.super"
+            )
+        ):
+            # log.info(f"[MACHINE] {dataservice["endpoint_description_url"]}")
+            to_set["machine_documentation_url"] = dataservice["endpoint_description_url"]
+        else:
+            # log.info(f"[ HUMAN ] {dataservice["endpoint_description_url"]}")
+            to_set["technical_documentation_url"] = dataservice["endpoint_description_url"]
+
+        result = get_db().dataservice.update_one(
+            filter={
+                "_id": dataservice["_id"],
+            },
+            update={"$set": to_set},
+        )
+        assert result.modified_count == 1
+        assert result.matched_count == 1
+
+    log.info("Postprocessing dataservices…")
+
+    count = get_db().dataservice.update_many(
+        {},
+        {
+            "$unset": {
+                "endpoint_description_url": "",
+                "is_restricted": "",
+                "has_token": "",
+            }
+        },
+    )
+    log.info(f"\tUnset legacy fields on {count.modified_count} dataservices")
+
+    log.info("Done")