udata 10.1.3.dev34251__py2.py3-none-any.whl → 10.1.3.dev34283__py2.py3-none-any.whl

udata/commands/fixtures.py

@@ -39,7 +39,7 @@ COMMUNITY_RES_URL = "/api/1/datasets/community_resources"
 DISCUSSION_URL = "/api/1/discussions"
 
 
-DEFAULT_FIXTURE_FILE_TAG: str = "v5.0.0"
+DEFAULT_FIXTURE_FILE_TAG: str = "v6.0.0"
 DEFAULT_FIXTURE_FILE: str = f"https://raw.githubusercontent.com/opendatateam/udata-fixtures/{DEFAULT_FIXTURE_FILE_TAG}/results.json"  # noqa
 
 DEFAULT_FIXTURES_RESULTS_FILENAME: str = "results.json"

udata/core/dataservices/constants.py

@@ -0,0 +1,11 @@
+DATASERVICE_FORMATS = ["REST", "WMS", "WSL"]
+
+
+DATASERVICE_ACCESS_TYPE_OPEN = "open"
+DATASERVICE_ACCESS_TYPE_OPEN_WITH_ACCOUNT = "open_with_account"
+DATASERVICE_ACCESS_TYPE_RESTRICTED = "restricted"
+DATASERVICE_ACCESS_TYPES = [
+    DATASERVICE_ACCESS_TYPE_OPEN,
+    DATASERVICE_ACCESS_TYPE_OPEN_WITH_ACCOUNT,
+    DATASERVICE_ACCESS_TYPE_RESTRICTED,
+]

udata/core/dataservices/csv.py

@@ -13,13 +13,13 @@ class DataserviceCsvAdapter(csv.Adapter):
         ("url", lambda d: d.self_web_url()),
         "description",
         "base_api_url",
-        "endpoint_description_url",
+        "machine_documentation_url",
+        "technical_documentation_url",
         "business_documentation_url",
         "authorization_request_url",
         "availability",
         "rate_limiting",
-        "is_restricted",
-        "has_token",
+        "access_type",
         "license",
         ("organization", "organization.name"),
         ("organization_id", "organization.id"),

udata/core/dataservices/models.py

@@ -8,6 +8,7 @@ from mongoengine.signals import post_save
 import udata.core.contact_point.api_fields as contact_api_fields
 import udata.core.dataset.api_fields as datasets_api_fields
 from udata.api_fields import field, function_field, generate_fields
+from udata.core.dataservices.constants import DATASERVICE_ACCESS_TYPES, DATASERVICE_FORMATS
 from udata.core.dataset.models import Dataset
 from udata.core.metrics.models import WithMetrics
 from udata.core.owned import Owned, OwnedQuerySet
@@ -24,8 +25,6 @@ from udata.uris import endpoint_for
 # "spatial"
 # "temporal_coverage"
 
-DATASERVICE_FORMATS = ["REST", "WMS", "WSL"]
-
 
 class DataserviceQuerySet(OwnedQuerySet):
     def visible(self):
@@ -95,6 +94,7 @@ class HarvestMetadata(db.EmbeddedDocument):
     )
     last_update = field(db.DateTimeField(), description="Date of the last harvesting")
     archived_at = field(db.DateTimeField())
+    archived_reason = field(db.StringField())
 
 
 @generate_fields(
@@ -138,13 +138,22 @@ class Dataservice(WithMetrics, Owned, db.Document):
     )
     description = field(db.StringField(default=""), description="In markdown")
     base_api_url = field(db.URLField(), sortable=True)
-    endpoint_description_url = field(db.URLField())
+
+    machine_documentation_url = field(
+        db.URLField(), description="Swagger link, OpenAPI format, WMS XML…"
+    )
+    technical_documentation_url = field(db.URLField(), description="HTML version of a Swagger…")
     business_documentation_url = field(db.URLField())
-    authorization_request_url = field(db.URLField())
-    availability = field(db.FloatField(min=0, max=100), example="99.99")
+
     rate_limiting = field(db.StringField())
-    is_restricted = field(db.BooleanField(), filterable={})
-    has_token = field(db.BooleanField())
+    rate_limiting_url = field(db.URLField())
+
+    availability = field(db.FloatField(min=0, max=100), example="99.99")
+    availability_url = field(db.URLField())
+
+    access_type = field(db.StringField(choices=DATASERVICE_ACCESS_TYPES), filterable={})
+    authorization_request_url = field(db.URLField())
+
     format = field(db.StringField(choices=DATASERVICE_FORMATS))
 
     license = field(

udata/core/dataservices/rdf.py

@@ -42,7 +42,9 @@ def dataservice_from_rdf(
     dataservice.description = sanitize_html(d.value(DCT.description) or d.value(DCT.abstract))
 
     dataservice.base_api_url = url_from_rdf(d, DCAT.endpointURL)
-    dataservice.endpoint_description_url = url_from_rdf(d, DCAT.endpointDescription)
+
+    # TODO detect if it's human-readable or not?
+    dataservice.machine_documentation_url = url_from_rdf(d, DCAT.endpointDescription)
 
     roles = [  # Imbricated list of contact points for each role
         contact_points_from_rdf(d, rdf_entity, role, dataservice)
@@ -145,8 +147,8 @@ def dataservice_to_rdf(dataservice: Dataservice, graph=None):
             ),
         )
 
-    if dataservice.endpoint_description_url:
-        d.set(DCAT.endpointDescription, URIRef(dataservice.endpoint_description_url))
+    if dataservice.machine_documentation_url:
+        d.set(DCAT.endpointDescription, URIRef(dataservice.machine_documentation_url))
 
     # Add DCAT-AP HVD properties if the dataservice is tagged hvd.
     # See https://semiceu.github.io/DCAT-AP/releases/2.2.0-hvd/

udata/core/dataservices/search.py

@@ -5,6 +5,11 @@ from flask_restx.inputs import boolean
 
 from udata.api import api
 from udata.api.parsers import ModelApiParser
+from udata.core.dataservices.constants import (
+    DATASERVICE_ACCESS_TYPE_OPEN,
+    DATASERVICE_ACCESS_TYPE_OPEN_WITH_ACCOUNT,
+    DATASERVICE_ACCESS_TYPE_RESTRICTED,
+)
 from udata.models import Dataservice, Organization, User
 from udata.search import (
     BoolFilter,
@@ -47,7 +52,11 @@ class DataserviceApiParser(ModelApiParser):
                 api.abort(400, "Organization arg must be an identifier")
             dataservices = dataservices.filter(organization=args["organization"])
         if "is_restricted" in args:
-            dataservices = dataservices.filter(is_restricted=boolean(args["is_restricted"]))
+            dataservices = dataservices.filter(
+                access_type__in=[DATASERVICE_ACCESS_TYPE_RESTRICTED]
+                if boolean(args["is_restricted"])
+                else [DATASERVICE_ACCESS_TYPE_OPEN, DATASERVICE_ACCESS_TYPE_OPEN_WITH_ACCOUNT]
+            )
         return dataservices
 
 
@@ -112,6 +121,6 @@ class DataserviceSearch(ModelSearchAdapter):
             "tags": dataservice.tags,
             "extras": extras,
             "followers": dataservice.metrics.get("followers", 0),
+            "is_restricted": dataservice.access_type == DATASERVICE_ACCESS_TYPE_RESTRICTED,
             "views": dataservice.metrics.get("views", 0),
-            "is_restricted": dataservice.is_restricted or False,
         }

udata/harvest/actions.py

@@ -7,6 +7,7 @@ from bson import ObjectId
 from flask import current_app
 
 from udata.auth import current_user
+from udata.core.dataservices.models import Dataservice
 from udata.core.dataset.models import HarvestDatasetMetadata
 from udata.models import Dataset, Organization, PeriodicTask, User
 from udata.storage.s3 import delete_file
@@ -18,6 +19,7 @@ from .models import (
     VALIDATION_REFUSED,
     HarvestJob,
     HarvestSource,
+    archive_harvested_dataservice,
     archive_harvested_dataset,
 )
 from .tasks import harvest
@@ -161,6 +163,9 @@ def purge_sources():
         datasets = Dataset.objects.filter(harvest__source_id=str(source.id))
         for dataset in datasets:
             archive_harvested_dataset(dataset, reason="harvester-deleted", dryrun=False)
+        dataservices = Dataservice.objects.filter(harvest__source_id=str(source.id))
+        for dataservice in dataservices:
+            archive_harvested_dataservice(dataservice, reason="harvester-deleted", dryrun=False)
         source.delete()
     return count
 

udata/harvest/backends/base.py

@@ -20,6 +20,7 @@ from ..models import (
     HarvestItem,
     HarvestJob,
     HarvestLog,
+    archive_harvested_dataservice,
     archive_harvested_dataset,
 )
 from ..signals import after_harvest_job, before_harvest_job
@@ -342,6 +343,7 @@ class BaseBackend(object):
         harvest.last_update = datetime.utcnow()
 
         harvest.archived_at = None
+        harvest.archived_reason = None
 
         return harvest
 
@@ -370,9 +372,10 @@ class BaseBackend(object):
             "harvest__remote_id__nin": remote_ids,
             "harvest__last_update__lt": limit_date,
         }
-        local_items_not_on_remote = Dataset.objects.filter(**q)
+        local_datasets_not_on_remote = Dataset.objects.filter(**q)
+        local_dataservices_not_on_remote = Dataservice.objects.filter(**q)
 
-        for dataset in local_items_not_on_remote:
+        for dataset in local_datasets_not_on_remote:
             if not dataset.harvest.archived_at:
                 archive_harvested_dataset(dataset, reason="not-on-remote", dryrun=self.dryrun)
             # add a HarvestItem to the job list (useful for report)
@@ -385,6 +388,23 @@ class BaseBackend(object):
 
             self.save_job()
 
+        for dataservice in local_dataservices_not_on_remote:
+            if not dataservice.harvest.archived_at:
+                archive_harvested_dataservice(
+                    dataservice, reason="not-on-remote", dryrun=self.dryrun
+                )
+            # add a HarvestItem to the job list (useful for report)
+            # even when archiving has already been done (useful for debug)
+            self.job.items.append(
+                HarvestItem(
+                    remote_id=str(dataservice.harvest.remote_id),
+                    dataservice=dataservice,
+                    status="archived",
+                )
+            )
+
+            self.save_job()
+
     def get_dataset(self, remote_id):
         """Get or create a dataset given its remote ID (and its source)
         We first try to match `source_id` to be source domain independent

udata/harvest/models.py

@@ -6,6 +6,7 @@ from urllib.parse import urlparse
 from werkzeug.utils import cached_property
 
 from udata.core.dataservices.models import Dataservice
+from udata.core.dataservices.models import HarvestMetadata as HarvestDataserviceMetadata
 from udata.core.dataset.models import HarvestDatasetMetadata
 from udata.core.owned import Owned, OwnedQuerySet
 from udata.i18n import lazy_gettext as _
@@ -203,3 +204,21 @@ def archive_harvested_dataset(dataset, reason, dryrun=False):
         dataset.validate()
     else:
         dataset.save()
+
+
+def archive_harvested_dataservice(dataservice, reason, dryrun=False):
+    """
+    Archive an harvested dataservice, setting extras accordingly.
+    If `dryrun` is True, the dataservice is not saved but validated only.
+    """
+    log.debug("Archiving dataservice %s", dataservice.id)
+    archival_date = datetime.utcnow()
+    dataservice.archived_at = archival_date
+    if not dataservice.harvest:
+        dataservice.harvest = HarvestDataserviceMetadata()
+    dataservice.harvest.archived_reason = reason
+    dataservice.harvest.archived_at = archival_date
+    if dryrun:
+        dataservice.validate()
+    else:
+        dataservice.save()

udata/harvest/tests/test_actions.py

@@ -6,6 +6,8 @@ from tempfile import NamedTemporaryFile
 import pytest
 from mock import patch
 
+from udata.core.dataservices.factories import DataserviceFactory
+from udata.core.dataservices.models import HarvestMetadata as HarvestDataserviceMetadata
 from udata.core.dataset.factories import DatasetFactory
 from udata.core.dataset.models import HarvestDatasetMetadata
 from udata.core.organization.factories import OrganizationFactory
@@ -396,17 +398,27 @@ class HarvestActionsTest:
         dataset_to_archive = DatasetFactory(
             harvest=HarvestDatasetMetadata(source_id=str(to_delete[0].id))
         )
+        dataservice_to_archive = DataserviceFactory(
+            harvest=HarvestDataserviceMetadata(source_id=str(to_delete[0].id))
+        )
 
         result = actions.purge_sources()
         dataset_to_archive.reload()
+        dataservice_to_archive.reload()
 
         assert result == len(to_delete)
         assert len(HarvestSource.objects) == len(to_keep)
         assert PeriodicTask.objects.filter(id=periodic_task.id).count() == 0
         assert HarvestJob.objects(id=harvest_job.id).count() == 0
+
         assert dataset_to_archive.harvest.archived == "harvester-deleted"
+        assert_equal_dates(dataset_to_archive.harvest.archived_at, now)
         assert_equal_dates(dataset_to_archive.archived, now)
 
+        assert dataservice_to_archive.harvest.archived_reason == "harvester-deleted"
+        assert_equal_dates(dataservice_to_archive.harvest.archived_at, now)
+        assert_equal_dates(dataservice_to_archive.archived_at, now)
+
     @pytest.mark.options(HARVEST_JOBS_RETENTION_DAYS=2)
     def test_purge_jobs(self):
         now = datetime.utcnow()

udata/harvest/tests/test_base_backend.py

@@ -4,6 +4,8 @@ from urllib.parse import urlparse
 import pytest
 from voluptuous import Schema
 
+from udata.core.dataservices.factories import DataserviceFactory
+from udata.core.dataservices.models import Dataservice
 from udata.core.dataset import tasks
 from udata.core.dataset.factories import DatasetFactory
 from udata.harvest.models import HarvestItem
@@ -20,9 +22,9 @@ class Unknown:
     pass
 
 
-def gen_remote_IDs(num: int) -> list[str]:
+def gen_remote_IDs(num: int, prefix: str = "") -> list[str]:
     """Generate remote IDs."""
-    return [f"fake-{i}" for i in range(num)]
+    return [f"{prefix}fake-{i}" for i in range(num)]
 
 
 class FakeBackend(BaseBackend):
@@ -45,6 +47,11 @@ class FakeBackend(BaseBackend):
             if self.is_done():
                 return
 
+        for remote_id in self.source.config.get("dataservice_remote_ids", []):
+            self.process_dataservice(remote_id)
+            if self.is_done():
+                return
+
     def inner_process_dataset(self, item: HarvestItem):
         dataset = self.get_dataset(item.remote_id)
 
@@ -55,6 +62,16 @@ class FakeBackend(BaseBackend):
             dataset.last_modified_internal = self.source.config["last_modified"]
         return dataset
 
+    def inner_process_dataservice(self, item: HarvestItem):
+        dataservice = self.get_dataservice(item.remote_id)
+
+        for key, value in DataserviceFactory.as_dict().items():
+            if getattr(dataservice, key) is None:
+                setattr(dataservice, key, value)
+        if self.source.config.get("last_modified"):
+            dataservice.last_modified_internal = self.source.config["last_modified"]
+        return dataservice
+
 
 class HarvestFilterTest:
     @pytest.mark.parametrize("type,expected", HarvestFilter.TYPES.items())
@@ -210,7 +227,13 @@ class BaseBackendTest:
 
     def test_autoarchive(self, app):
         nb_datasets = 3
-        source = HarvestSourceFactory(config={"dataset_remote_ids": gen_remote_IDs(nb_datasets)})
+        nb_dataservices = 3
+        source = HarvestSourceFactory(
+            config={
+                "dataset_remote_ids": gen_remote_IDs(nb_datasets, "dataset-"),
+                "dataservice_remote_ids": gen_remote_IDs(nb_dataservices, "dataservice-"),
+            }
+        )
         backend = FakeBackend(source)
 
         # create a dangling dataset to be archived
@@ -220,7 +243,15 @@ class BaseBackendTest:
             harvest={
                 "domain": source.domain,
                 "source_id": str(source.id),
-                "remote_id": "not-on-remote",
+                "remote_id": "dataset-not-on-remote",
+                "last_update": last_update,
+            }
+        )
+        dataservice_arch = DataserviceFactory(
+            harvest={
+                "domain": source.domain,
+                "source_id": str(source.id),
+                "remote_id": "dataservice-not-on-remote",
                 "last_update": last_update,
             }
         )
@@ -232,7 +263,15 @@ class BaseBackendTest:
             harvest={
                 "domain": source.domain,
                 "source_id": str(source.id),
-                "remote_id": "not-on-remote-two",
+                "remote_id": "dataset-not-on-remote-two",
+                "last_update": last_update,
+            }
+        )
+        dataservice_no_arch = DataserviceFactory(
+            harvest={
+                "domain": source.domain,
+                "source_id": str(source.id),
+                "remote_id": "dataservice-not-on-remote-two",
                 "last_update": last_update,
             }
         )
@@ -240,13 +279,17 @@ class BaseBackendTest:
         job = backend.harvest()
 
         # all datasets except arch : 3 mocks + 1 manual (no_arch)
-        assert len(job.items) == nb_datasets + 1
+        assert len(job.items) == (nb_datasets + 1) + (nb_dataservices + 1)
         # all datasets : 3 mocks + 2 manuals (arch and no_arch)
         assert Dataset.objects.count() == nb_datasets + 2
+        assert Dataservice.objects.count() == nb_dataservices + 2
 
         archived_items = [i for i in job.items if i.status == "archived"]
-        assert len(archived_items) == 1
+        assert len(archived_items) == 2
         assert archived_items[0].dataset == dataset_arch
+        assert archived_items[0].dataservice is None
+        assert archived_items[1].dataset is None
+        assert archived_items[1].dataservice == dataservice_arch
 
         dataset_arch.reload()
         assert dataset_arch.archived is not None
@@ -258,18 +301,41 @@ class BaseBackendTest:
         assert "archived" not in dataset_no_arch.harvest
         assert "archived_at" not in dataset_no_arch.harvest
 
+        dataservice_arch.reload()
+        assert dataservice_arch.archived_at is not None
+        assert "archived_reason" in dataservice_arch.harvest
+        assert "archived_at" in dataservice_arch.harvest
+
+        dataservice_no_arch.reload()
+        assert dataservice_no_arch.archived_at is None
+        assert "archived_reason" not in dataservice_no_arch.harvest
+        assert "archived_at" not in dataservice_no_arch.harvest
+
         # test unarchive: archive manually then relaunch harvest
-        dataset = Dataset.objects.get(**{"harvest__remote_id": "fake-1"})
+        dataset = Dataset.objects.get(**{"harvest__remote_id": "dataset-fake-1"})
         dataset.archived = datetime.utcnow()
         dataset.harvest.archived = "not-on-remote"
         dataset.harvest.archived_at = datetime.utcnow()
         dataset.save()
+
+        dataservice = Dataservice.objects.get(**{"harvest__remote_id": "dataservice-fake-1"})
+        dataservice.archived_at = datetime.utcnow()
+        dataservice.harvest.archived_reason = "not-on-remote"
+        dataservice.harvest.archived_at = datetime.utcnow()
+        dataservice.save()
+
         backend.harvest()
+
         dataset.reload()
         assert dataset.archived is None
         assert "archived" not in dataset.harvest
         assert "archived_at" not in dataset.harvest
 
+        dataservice.reload()
+        assert dataservice.archived_at is None
+        assert "archived_reason" not in dataservice.harvest
+        assert "archived_at" not in dataservice.harvest
+
     def test_harvest_datasets_get_deleted(self):
         nb_datasets = 3
         source = HarvestSourceFactory(config={"dataset_remote_ids": gen_remote_IDs(nb_datasets)})

udata/harvest/tests/test_dcat_backend.py

@@ -179,7 +179,7 @@ class DcatBackendTest:
         assert dataservices[0].title == "Explore API v2"
         assert dataservices[0].base_api_url == "https://data.paris2024.org/api/explore/v2.1/"
         assert (
-            dataservices[0].endpoint_description_url
+            dataservices[0].machine_documentation_url
             == "https://data.paris2024.org/api/explore/v2.1/swagger.json"
         )
         assert (

udata/migrations/2025-01-05-dataservices-fields-changes.py

@@ -0,0 +1,136 @@
+"""
+This migration keeps only the "Local authority" badge if the organization also has the "Public service" badge.
+"""
+
+import logging
+from typing import List
+
+from mongoengine.connection import get_db
+
+from udata.core.dataservices.constants import (
+    DATASERVICE_ACCESS_TYPE_OPEN,
+    DATASERVICE_ACCESS_TYPE_OPEN_WITH_ACCOUNT,
+    DATASERVICE_ACCESS_TYPE_RESTRICTED,
+)
+from udata.core.dataservices.models import Dataservice
+
+log = logging.getLogger(__name__)
+
+
+def migrate(db):
+    log.info("Preprocessing dataservices…")
+
+    count = get_db().dataservice.update_many(
+        filter={
+            "$or": [
+                {"is_restricted": None},
+                {"is_restricted": {"$exists": False}},
+            ]
+        },
+        update={"$set": {"is_restricted": False}},
+    )
+    log.info(
+        f"\tConverted {count.modified_count} dataservices from `is_restricted=None` to `is_restricted=False`"
+    )
+
+    count = get_db().dataservice.update_many(
+        filter={
+            "$or": [
+                {"has_token": None},
+                {"has_token": {"$exists": False}},
+            ]
+        },
+        update={"$set": {"has_token": False}},
+    )
+    log.info(
+        f"\tConverted {count.modified_count} dataservices from `has_token=None` to `has_token=False`"
+    )
+
+    for dataservice in get_db().dataservice.find({"is_restricted": True, "has_token": False}):
+        log.info(
+            f"\tDataservice #{dataservice['_id']} {dataservice['title']} is restricted but without token. (will be set to access_type={DATASERVICE_ACCESS_TYPE_RESTRICTED})"
+        )
+
+    log.info("Processing dataservices…")
+
+    count = get_db().dataservice.update_many(
+        filter={
+            "is_restricted": True,
+            # `has_token` could be True or False, we don't care
+        },
+        update={"$set": {"access_type": DATASERVICE_ACCESS_TYPE_RESTRICTED}},
+    )
+    log.info(
+        f"\t{count.modified_count} restricted dataservices to DATASERVICE_ACCESS_TYPE_RESTRICTED"
+    )
+
+    count = get_db().dataservice.update_many(
+        filter={
+            "is_restricted": False,
+            "has_token": True,
+        },
+        update={"$set": {"access_type": DATASERVICE_ACCESS_TYPE_OPEN_WITH_ACCOUNT}},
+    )
+    log.info(
+        f"\t{count.modified_count} dataservices not restricted but with token to DATASERVICE_ACCESS_TYPE_OPEN_WITH_ACCOUNT"
+    )
+
+    count = get_db().dataservice.update_many(
+        filter={
+            "is_restricted": False,
+            "has_token": False,
+        },
+        update={"$set": {"access_type": DATASERVICE_ACCESS_TYPE_OPEN}},
+    )
+    log.info(f"\t{count.modified_count} open dataservices to DATASERVICE_ACCESS_TYPE_OPEN")
+
+    dataservices: List[Dataservice] = get_db().dataservice.find()
+    for dataservice in dataservices:
+        if (
+            "endpoint_description_url" not in dataservice
+            or not dataservice["endpoint_description_url"]
+        ):
+            continue
+
+        to_set = {}
+        if (
+            dataservice["endpoint_description_url"].endswith(".json")
+            or dataservice["endpoint_description_url"].endswith(".yaml")
+            or dataservice["endpoint_description_url"].endswith(".yml")
+            or dataservice["endpoint_description_url"].endswith("?format=openapi-json")
+            or "getcapabilities" in dataservice["endpoint_description_url"].lower()
+            or "getresourcedescription" in dataservice["endpoint_description_url"].lower()
+            or dataservice["endpoint_description_url"].startswith(
+                "https://api.insee.fr/catalogue/api-docs/carbon.super"
+            )
+        ):
+            # log.info(f"[MACHINE] {dataservice["endpoint_description_url"]}")
+            to_set["machine_documentation_url"] = dataservice["endpoint_description_url"]
+        else:
+            # log.info(f"[ HUMAN ] {dataservice["endpoint_description_url"]}")
+            to_set["technical_documentation_url"] = dataservice["endpoint_description_url"]
+
+        result = get_db().dataservice.update_one(
+            filter={
+                "_id": dataservice["_id"],
+            },
+            update={"$set": to_set},
+        )
+        assert result.modified_count == 1
+        assert result.matched_count == 1
+
+    log.info("Postprocessing dataservices…")
+
+    count = get_db().dataservice.update_many(
+        {},
+        {
+            "$unset": {
+                "endpoint_description_url": "",
+                "is_restricted": "",
+                "has_token": "",
+            }
+        },
+    )
+    log.info(f"\tUnset legacy fields on {count.modified_count} dataservices")
+
+    log.info("Done")