udata 8.0.2.dev29253__py2.py3-none-any.whl → 8.0.2.dev29284__py2.py3-none-any.whl

udata/commands/purge.py

@@ -5,6 +5,7 @@ import click
 from udata.commands import cli, success
 
 from udata.core.dataset.tasks import purge_datasets
+from udata.core.dataservices.tasks import purge_dataservices
 from udata.core.organization.tasks import purge_organizations
 from udata.core.reuse.tasks import purge_reuses
 
@@ -15,13 +16,14 @@ log = logging.getLogger(__name__)
 @click.option('-d', '--datasets', is_flag=True)
 @click.option('-r', '--reuses', is_flag=True)
 @click.option('-o', '--organizations', is_flag=True)
-def purge(datasets, reuses, organizations):
+@click.option('--dataservices', is_flag=True)
+def purge(datasets, reuses, organizations, dataservices):
     '''
     Permanently remove data flagged as deleted.
 
     If no model flag is given, all models are purged.
     '''
-    purge_all = not any((datasets, reuses, organizations))
+    purge_all = not any((datasets, reuses, organizations, dataservices))
 
     if purge_all or datasets:
         log.info('Purging datasets')
@@ -35,4 +37,8 @@ def purge(datasets, reuses, organizations):
         log.info('Purging organizations')
         purge_organizations()
 
+    if purge_all or dataservices:
+        log.info('Purging dataservices')
+        purge_dataservices()
+
     success('Done')

udata/core/dataservices/api.py

@@ -6,11 +6,12 @@ import mongoengine
 from udata.api import api, API
 from udata.api_fields import patch
 from udata.core.dataset.permissions import OwnablePermission
+from udata.core.followers.api import FollowAPI
 from .models import Dataservice
-from udata.models import db
 
 ns = api.namespace('dataservices', 'Dataservices related operations (beta)')
 
+
 @ns.route('/', endpoint='dataservices')
 class DataservicesAPI(API):
     '''Dataservices collection endpoint'''
@@ -39,6 +40,7 @@ class DataservicesAPI(API):
 
         return dataservice, 201
 
+
 @ns.route('/<dataservice:dataservice>/', endpoint='dataservice')
 class DataserviceAPI(API):
     @api.doc('get_dataservice')
@@ -47,7 +49,7 @@ class DataserviceAPI(API):
         if dataservice.deleted_at and not OwnablePermission(dataservice).can():
             api.abort(410, 'Dataservice has been deleted')
         return dataservice
-    
+
     @api.secure
     @api.doc('update_dataservice', responses={400: 'Validation error'})
     @api.expect(Dataservice.__write_fields__)
@@ -75,10 +77,16 @@ class DataserviceAPI(API):
             api.abort(410, 'dataservice has been deleted')
 
         OwnablePermission(dataservice).test()
-
         dataservice.deleted_at = datetime.utcnow()
         dataservice.modified_at = datetime.utcnow()
         dataservice.save()
 
         return '', 204
 
+
+@ns.route('/<id>/followers/', endpoint='dataservice_followers')
+@ns.doc(get={'id': 'list_dataservice_followers'},
+        post={'id': 'follow_dataservice'},
+        delete={'id': 'unfollow_dataservice'})
+class DataserviceFollowersAPI(FollowAPI):
+    model = Dataservice

udata/core/dataservices/models.py

@@ -3,11 +3,11 @@ from udata.api_fields import field, function_field, generate_fields
 from udata.core.dataset.models import Dataset
 from udata.core.metrics.models import WithMetrics
 from udata.core.owned import Owned, OwnedQuerySet
-from udata.i18n import lazy_gettext as _
 import udata.core.contact_point.api_fields as contact_api_fields
 import udata.core.dataset.api_fields as datasets_api_fields
+from udata.i18n import lazy_gettext as _
 
-from udata.models import db
+from udata.models import db, Discussion, Follow
 from udata.uris import endpoint_for
 
 # "frequency"
@@ -128,3 +128,15 @@ class Dataservice(WithMetrics, Owned, db.Document):
     # temporal_coverage = db.EmbeddedDocumentField(db.DateRange)
     # spatial = db.EmbeddedDocumentField(SpatialCoverage)
     # harvest = db.EmbeddedDocumentField(HarvestDatasetMetadata)
+
+    @property
+    def is_hidden(self):
+        return self.private or self.deleted_at or self.archived_at
+
+    def count_discussions(self):
+        self.metrics['discussions'] = Discussion.objects(subject=self, closed=None).count()
+        self.save()
+
+    def count_followers(self):
+        self.metrics['followers'] = Follow.objects(until=None).followers(self).count()
+        self.save()

udata/core/dataservices/permissions.py

@@ -0,0 +1,7 @@
+from udata.core.dataset.permissions import (
+    OwnablePermission
+)
+
+class DataserviceEditPermission(OwnablePermission):
+    '''Permissions to edit a Dataservice'''
+    pass

udata/core/dataservices/tasks.py

@@ -0,0 +1,25 @@
+from celery.utils.log import get_task_logger
+
+from udata.core.dataservices.models import Dataservice
+# from udata.harvest.models import HarvestJob
+from udata.models import (Follow, Discussion, Activity, Transfer)
+from udata.tasks import job
+
+log = get_task_logger(__name__)
+
+
+@job('purge-dataservices')
+def purge_dataservices(self):
+    for dataservice in Dataservice.objects(deleted_at__ne=None):
+        log.info(f'Purging dataservice {dataservice}')
+        # Remove followers
+        Follow.objects(following=dataservice).delete()
+        # Remove discussions
+        Discussion.objects(subject=dataservice).delete()
+        # Remove HarvestItem references
+        # TODO: uncomment when adding dataservice harvest
+        # HarvestJob.objects(items__dataservice=dataservice).update(set__items__S__dataservice=None)
+        # Remove associated Transfers
+        Transfer.objects(subject=dataservice).delete()
+        # Remove dataservice
+        dataservice.delete()

udata/harvest/backends/base.py

@@ -2,6 +2,7 @@ import logging
 import traceback
 
 from datetime import datetime, date, timedelta
+from typing import Optional
 from uuid import UUID
 
 import requests
@@ -68,7 +69,12 @@ class HarvestFeature(object):
 
 
 class BaseBackend(object):
-    '''Base class for Harvester implementations'''
+    """
+    Base class that wrap children methods to add error management and debug logs.
+    Also provides a few helpers needed on all or some backends.
+
+
+    """
 
     name = None
     display_name = None
@@ -127,92 +133,69 @@ class BaseBackend(object):
     def get_filters(self):
         return self.config.get('filters', [])
 
-    def harvest(self):
-        '''Start the harvesting process'''
-        if self.perform_initialization() is not None:
-            self.process_items()
-            self.finalize()
-        return self.job
+    def inner_harvest(self):
+        raise NotImplementedError
+    
+    def inner_process_dataset(self, item: HarvestItem) -> Dataset:
+        raise NotImplementedError
 
-    def perform_initialization(self):
-        '''Initialize the harvesting for a given job'''
-        log.debug('Initializing backend')
+    def harvest(self):
+        log.debug(f'Starting harvesting {self.source.name} ({self.source.url})…')
         factory = HarvestJob if self.dryrun else HarvestJob.objects.create
-        self.job = factory(status='initializing',
+        self.job = factory(status='initialized',
                            started=datetime.utcnow(),
                            source=self.source)
 
         before_harvest_job.send(self)
 
         try:
-            self.initialize()
-            self.job.status = 'initialized'
-            if not self.dryrun:
-                self.job.save()
+            self.inner_harvest()
+
+            if self.source.autoarchive:
+                self.autoarchive()
+
+            self.job.status = 'done'
+
+            if any(i.status == 'failed' for i in self.job.items):
+                self.job.status += '-errors'
         except HarvestValidationError as e:
-            log.info('Initialization failed for "%s" (%s)',
-                     safe_unicode(self.source.name), self.source.backend)
-            error = HarvestError(message=safe_unicode(e))
-            self.job.errors.append(error)
-            self.job.status = 'failed'
-            self.end()
-            return None
-        except Exception as e:
+            log.exception(f'Harvesting validation failed for "{safe_unicode(self.source.name)}" ({self.source.backend})')
+
             self.job.status = 'failed'
+
             error = HarvestError(message=safe_unicode(e))
             self.job.errors.append(error)
-            self.end()
-            msg = 'Initialization failed for "{0.name}" ({0.backend})'
-            log.exception(msg.format(self.source))
-            return None
+        except Exception as e:
+            log.exception(f'Harvesting failed for "{safe_unicode(self.source.name)}" ({self.source.backend})')
 
-        if self.max_items:
-            self.job.items = self.job.items[:self.max_items]
+            self.job.status = 'failed'
 
-        if self.job.items:
-            log.debug('Queued %s items', len(self.job.items))
+            error = HarvestError(message=safe_unicode(e), details=traceback.format_exc())
+            self.job.errors.append(error)
+        finally:
+            self.end_job()
+        
+        return self.job
 
-        return len(self.job.items)
+    def process_dataset(self, remote_id: str, **kwargs):
+        log.debug(f'Processing dataset {remote_id}…')
 
-    def initialize(self):
-        raise NotImplementedError
+        # TODO add `type` to `HarvestItem` to differentiate `Dataset` from `Dataservice`
+        item = HarvestItem(status='started', started=datetime.utcnow(), remote_id=remote_id)
+        self.job.items.append(item)
+        self.save_job()
 
-    def process_items(self):
-        '''Process the data identified in the initialize stage'''
-        for item in self.job.items:
-            self.process_item(item)
+        try:
+            if not remote_id:
+                raise HarvestSkipException("missing identifier")
 
-    def process_item(self, item):
-        log.debug('Processing: %s', item.remote_id)
-        item.status = 'started'
-        item.started = datetime.utcnow()
-        if not self.dryrun:
-            self.job.save()
+            dataset = self.inner_process_dataset(item, **kwargs)
 
-        try:
-            dataset = self.process(item)
-            if not dataset.harvest:
-                dataset.harvest = HarvestDatasetMetadata()
-            dataset.harvest.domain = self.source.domain
-            dataset.harvest.remote_id = item.remote_id
-            dataset.harvest.source_id = str(self.source.id)
-            dataset.harvest.last_update = datetime.utcnow()
-            dataset.harvest.backend = self.display_name
-
-            # unset archived status if needed
-            if dataset.harvest:
-                dataset.harvest.archived_at = None
-                dataset.harvest.archived = None
+            # Use `item.remote_id` because `inner_process_dataset` could have modified it.
+            dataset.harvest = self.update_harvest_info(dataset.harvest, item.remote_id)
             dataset.archived = None
 
-            # TODO permissions checking
-            if not dataset.organization and not dataset.owner:
-                if self.source.organization:
-                    dataset.organization = self.source.organization
-                elif self.source.owner:
-                    dataset.owner = self.source.owner
-
-            # TODO: Apply editble mappings
+            # TODO: Apply editable mappings
 
             if self.dryrun:
                 dataset.validate()
@@ -221,26 +204,54 @@ class BaseBackend(object):
             item.dataset = dataset
             item.status = 'done'
         except HarvestSkipException as e:
-            log.info('Skipped item %s : %s', item.remote_id, safe_unicode(e))
             item.status = 'skipped'
+
+            log.info(f'Skipped item {item.remote_id} : {safe_unicode(e)}')
             item.errors.append(HarvestError(message=safe_unicode(e)))
         except HarvestValidationError as e:
-            log.info('Error validating item %s : %s', item.remote_id, safe_unicode(e))
             item.status = 'failed'
+
+            log.info(f'Error validating item {item.remote_id} : {safe_unicode(e)}')
             item.errors.append(HarvestError(message=safe_unicode(e)))
         except Exception as e:
-            log.exception('Error while processing %s : %s',
-                          item.remote_id,
-                          safe_unicode(e))
-            error = HarvestError(message=safe_unicode(e),
-                                 details=traceback.format_exc())
-            item.errors.append(error)
             item.status = 'failed'
+            log.exception(f'Error while processing {item.remote_id} : {safe_unicode(e)}')
+
+            error = HarvestError(message=safe_unicode(e), details=traceback.format_exc())
+            item.errors.append(error)
+        finally:
+            item.ended = datetime.utcnow()
+            self.save_job()
+
+    def is_done(self) -> bool:
+        '''Should be called after process_dataset to know if we reach the max items'''
+        return self.max_items and len(self.job.items) >= self.max_items
+
+    def update_harvest_info(self, harvest: Optional[HarvestDatasetMetadata], remote_id: int):
+        if not harvest:
+            harvest = HarvestDatasetMetadata()
+        harvest.domain = self.source.domain
+        harvest.remote_id = remote_id
+        harvest.source_id = str(self.source.id)
+        harvest.last_update = datetime.utcnow()
+        harvest.backend = self.display_name
 
-        item.ended = datetime.utcnow()
+        harvest.archived_at = None
+        harvest.archived = None
+
+        return harvest
+
+    def save_job(self):
+        if not self.dryrun:
+            self.job.save()
+
+    def end_job(self):
+        self.job.ended = datetime.utcnow()
         if not self.dryrun:
             self.job.save()
 
+        after_harvest_job.send(self)
+
     def autoarchive(self):
         '''
         Archive items that exist on the local instance but not on remote platform
@@ -262,34 +273,13 @@ class BaseBackend(object):
                 archive_harvested_dataset(dataset, reason='not-on-remote', dryrun=self.dryrun)
             # add a HarvestItem to the job list (useful for report)
             # even when archiving has already been done (useful for debug)
-            item = self.add_item(dataset.harvest.remote_id)
-            item.dataset = dataset
-            item.status = 'archived'
-
-            if not self.dryrun:
-                self.job.save()
+            self.job.items.append(HarvestItem(
+                remote_id=str(dataset.harvest.remote_id),
+                dataset=dataset,
+                status='archived'
+            ))
 
-    def process(self, item):
-        raise NotImplementedError
-
-    def add_item(self, identifier, *args, **kwargs):
-        item = HarvestItem(remote_id=str(identifier), args=args, kwargs=kwargs)
-        self.job.items.append(item)
-        return item
-
-    def finalize(self):
-        if self.source.autoarchive:
-            self.autoarchive()
-        self.job.status = 'done'
-        if any(i.status == 'failed' for i in self.job.items):
-            self.job.status += '-errors'
-        self.end()
-
-    def end(self):
-        self.job.ended = datetime.utcnow()
-        if not self.dryrun:
-            self.job.save()
-        after_harvest_job.send(self)
+            self.save_job()
 
     def get_dataset(self, remote_id):
         '''Get or create a dataset given its remote ID (and its source)
@@ -352,4 +342,4 @@ class BaseBackend(object):
                     msg = str(error)
                 errors.append(msg)
             msg = '\n- '.join(['Validation error:'] + errors)
-            raise HarvestValidationError(msg)
+            raise HarvestValidationError(msg)

udata/harvest/backends/dcat.py

@@ -7,13 +7,15 @@ import boto3
 from flask import current_app
 from datetime import date
 import json
-from typing import List
+from typing import Generator, List
 
+from udata.core.dataset.models import Dataset
 from udata.rdf import (
     DCAT, DCT, HYDRA, SPDX, namespace_manager, guess_format, url_from_rdf
 )
 from udata.core.dataset.rdf import dataset_from_rdf
 from udata.storage.s3 import store_as_json, get_from_json
+from udata.harvest.models import HarvestItem
 
 from .base import BaseBackend
 
@@ -59,14 +61,17 @@ def extract_graph(source, target, node, specs):
 class DcatBackend(BaseBackend):
     display_name = 'DCAT'
 
-    def initialize(self):
-        '''List all datasets for a given ...'''
+    def inner_harvest(self):
         fmt = self.get_format()
-        graphs = self.parse_graph(self.source.url, fmt)
-
         self.job.data = { 'format': fmt }
 
-        serialized_graphs = [graph.serialize(format=fmt, indent=None) for graph in graphs]
+        serialized_graphs = []
+
+        for page_number, page in self.walk_graph(self.source.url, fmt):
+            self.process_one_datasets_page(page_number, page)
+            serialized_graphs.append(page.serialize(format=fmt, indent=None))
+
+        # TODO call `walk_graph` with `process_dataservices`
 
         # The official MongoDB document size in 16MB. The default value here is 15MB to account for other fields in the document (and for difference between * 1024 vs * 1000).
         max_harvest_graph_size_in_mongo = current_app.config.get('HARVEST_MAX_CATALOG_SIZE_IN_MONGO')
@@ -105,13 +110,11 @@ class DcatBackend(BaseBackend):
                 raise ValueError(msg)
         return fmt
 
-    def parse_graph(self, url, fmt) -> List[Graph]:
+    def walk_graph(self, url: str, fmt: str) -> Generator[tuple[int, Graph], None, None]:
         """
-        Returns an instance of rdflib.Graph for each detected page
-        The index in the list is the page number
+        Yield all RDF pages as `Graph` from the source
         """
-        graphs = []
-        page = 0
+        page_number = 0
         while url:
             subgraph = Graph(namespace_manager=namespace_manager)
             response = self.get(url)
@@ -128,19 +131,26 @@ class DcatBackend(BaseBackend):
                     pagination = subgraph.resource(pagination)
                     url = url_from_rdf(pagination, prop)
                     break
-            graphs.append(subgraph)
 
-            for node in subgraph.subjects(RDF.type, DCAT.Dataset):
-                id = subgraph.value(node, DCT.identifier)
-                kwargs = {'page': page}
-                self.add_item(id, **kwargs)
-                if self.max_items and len(self.job.items) >= self.max_items:
-                    # this will stop iterating on pagination
-                    url = None
+            yield page_number, subgraph
+            if self.is_done():
+                return
 
-            page += 1
+            page_number += 1
+    
+    def process_one_datasets_page(self, page_number: int, page: Graph):
+        for node in page.subjects(RDF.type, DCAT.Dataset):
+            remote_id = page.value(node, DCT.identifier)
+            self.process_dataset(remote_id, page_number=page_number, page=page, node=node)
+
+            if self.is_done():
+                return
+            
+    def inner_process_dataset(self, item: HarvestItem, page_number: int, page: Graph, node):
+        item.kwargs['page_number'] = page_number
 
-        return graphs
+        dataset = self.get_dataset(item.remote_id)
+        return dataset_from_rdf(page, dataset, node=node)
 
     def get_node_from_item(self, graph, item):
         for node in graph.subjects(RDF.type, DCAT.Dataset):
@@ -148,33 +158,6 @@ class DcatBackend(BaseBackend):
                 return node
         raise ValueError(f'Unable to find dataset with DCT.identifier:{item.remote_id}')
 
-    def process(self, item):
-        if item.remote_id == 'None':
-            raise ValueError('The DCT.identifier is missing on this DCAT.Dataset record')
-        graph = Graph(namespace_manager=namespace_manager)
-
-        if self.job.data.get('graphs') is not None:
-            graphs = self.job.data['graphs']
-        else:
-            bucket = current_app.config.get('HARVEST_GRAPHS_S3_BUCKET')
-            if bucket is None:
-                raise ValueError(f"No bucket configured but the harvest job item {item.id} on job {self.job.id} doesn't have a graph in MongoDB.")
-
-            graphs = get_from_json(bucket, self.job.data['filename'])
-            if graphs is None:
-                raise ValueError(f"The file '{self.job.data['filename']}' is missing in S3 bucket '{bucket}'")
-
-        data = graphs[item.kwargs['page']]
-        format = self.job.data['format']
-
-        graph.parse(data=bytes(data, encoding='utf8'), format=format)
-        node = self.get_node_from_item(graph, item)
-
-        dataset = self.get_dataset(item.remote_id)
-        dataset = dataset_from_rdf(graph, dataset, node=node)
-        return dataset
-    
-
     def next_record_if_should_continue(self, start, search_results):
         next_record = int(search_results.attrib['nextRecord'])
         matched_count = int(search_results.attrib['numberOfRecordsMatched'])
@@ -209,7 +192,10 @@ class CswDcatBackend(DcatBackend):
 
     DCAT_SCHEMA = 'http://www.w3.org/ns/dcat#'
 
-    def parse_graph(self, url: str, fmt: str) -> List[Graph]:
+    def walk_graph(self, url: str, fmt: str) -> Generator[tuple[int, Graph], None, None]:
+        """
+        Yield all RDF pages as `Graph` from the source
+        """
         body = '''<csw:GetRecords xmlns:csw="http://www.opengis.net/cat/csw/2.0.2"
                                   xmlns:gmd="http://www.isotc211.org/2005/gmd"
                                   service="CSW" version="2.0.2" resultType="results"
@@ -227,8 +213,7 @@ class CswDcatBackend(DcatBackend):
                 </csw:GetRecords>'''
         headers = {'Content-Type': 'application/xml'}
 
-        graphs = []
-        page = 0
+        page_number = 0
         start = 1
 
         response = self.post(url, data=body.format(start=start, schema=self.DCAT_SCHEMA),
@@ -239,36 +224,29 @@ class CswDcatBackend(DcatBackend):
         if tree.tag == '{' + OWS_NAMESPACE + '}ExceptionReport':
             raise ValueError(f'Failed to query CSW:\n{content}')
         while tree:
-            graph = Graph(namespace_manager=namespace_manager)
             search_results = tree.find('csw:SearchResults', {'csw': CSW_NAMESPACE})
             if search_results is None:
-                log.error(f'No search results found for {url} on page {page}')
+                log.error(f'No search results found for {url} on page {page_number}')
                 break
             for child in search_results:
                 subgraph = Graph(namespace_manager=namespace_manager)
                 subgraph.parse(data=ET.tostring(child), format=fmt)
-                graph += subgraph
 
-                for node in subgraph.subjects(RDF.type, DCAT.Dataset):
-                    id = subgraph.value(node, DCT.identifier)
-                    kwargs = {'nid': str(node), 'page': page}
-                    kwargs['type'] = 'uriref' if isinstance(node, URIRef) else 'blank'
-                    self.add_item(id, **kwargs)
-            graphs.append(graph)
+                yield page_number, subgraph
+                if self.is_done():
+                    return
 
             next_record = self.next_record_if_should_continue(start, search_results)
             if not next_record:
                 break
 
             start = next_record
-            page += 1
+            page_number += 1
 
             tree = ET.fromstring(
                 self.post(url, data=body.format(start=start, schema=self.DCAT_SCHEMA),
                           headers=headers).content)
 
-        return graphs
-
 
 class CswIso19139DcatBackend(DcatBackend):
     '''
@@ -282,13 +260,10 @@ class CswIso19139DcatBackend(DcatBackend):
 
     XSL_URL = "https://raw.githubusercontent.com/SEMICeu/iso-19139-to-dcat-ap/master/iso-19139-to-dcat-ap.xsl"
 
-    def parse_graph(self, url: str, fmt: str) -> List[Graph]:
-        '''
-        Parse CSW graph querying ISO schema.
-        Use SEMIC GeoDCAT-AP XSLT to map it to a correct version.
-        See https://github.com/SEMICeu/iso-19139-to-dcat-ap for more information on the XSLT.
-        '''
-
+    def walk_graph(self, url: str, fmt: str) -> Generator[tuple[int, Graph], None, None]:
+        """
+        Yield all RDF pages as `Graph` from the source
+        """
         # Load XSLT
         xsl = ET.fromstring(self.get(self.XSL_URL).content)
         transform = ET.XSLT(xsl)
@@ -320,8 +295,7 @@ class CswIso19139DcatBackend(DcatBackend):
                 </csw:GetRecords>'''
         headers = {'Content-Type': 'application/xml'}
 
-        graphs = []
-        page = 0
+        page_number = 0
         start = 1
 
         response = self.post(url, data=body.format(start=start, schema=self.ISO_SCHEMA),
@@ -338,7 +312,7 @@ class CswIso19139DcatBackend(DcatBackend):
             # infos (useful for pagination)
             search_results = tree_before_transform.find('csw:SearchResults', {'csw': CSW_NAMESPACE})
             if search_results is None:
-                log.error(f'No search results found for {url} on page {page}')
+                log.error(f'No search results found for {url} on page {page_number}')
                 break
 
             subgraph = Graph(namespace_manager=namespace_manager)
@@ -347,19 +321,16 @@ class CswIso19139DcatBackend(DcatBackend):
             if not subgraph.subjects(RDF.type, DCAT.Dataset):
                 raise ValueError("Failed to fetch CSW content")
 
-            for node in subgraph.subjects(RDF.type, DCAT.Dataset):
-                id = subgraph.value(node, DCT.identifier)
-                kwargs = {'nid': str(node), 'page': page}
-                kwargs['type'] = 'uriref' if isinstance(node, URIRef) else 'blank'
-                self.add_item(id, **kwargs)
-            graphs.append(subgraph)
+            yield page_number, subgraph
+            if self.is_done():
+                return
 
             next_record = self.next_record_if_should_continue(start, search_results)
             if not next_record:
                 break
 
             start = next_record
-            page += 1
+            page_number += 1
 
             response = self.post(url, data=body.format(start=start, schema=self.ISO_SCHEMA),
                                  headers=headers)
@@ -367,5 +338,3 @@ class CswIso19139DcatBackend(DcatBackend):
 
             tree_before_transform = ET.fromstring(response.content)
             tree = transform(tree_before_transform, CoupledResourceLookUp="'disabled'")
-
-        return graphs

udata/harvest/tasks.py

@@ -18,20 +18,10 @@ def harvest(self, ident):
         return  # Ignore deleted sources
     Backend = backends.get(current_app, source.backend)
     backend = Backend(source)
-    items = backend.perform_initialization()
-    if items is None:
-        pass
-    elif items == 0:
-        backend.finalize()
-    else:
-        finalize = harvest_job_finalize.s(backend.job.id)
-        items = [
-            harvest_job_item.s(backend.job.id, item.remote_id)
-            for item in backend.job.items
-        ]
-        chord(items)(finalize)
-    
 
+    backend.harvest()
+
+    
 
 @task(ignore_result=False, route='low.harvest')
 def harvest_job_item(job_id, item_id):

udata/harvest/tests/factories.py

@@ -6,9 +6,10 @@ from flask.signals import Namespace
 
 from udata.factories import ModelFactory
 from udata.core.dataset.factories import DatasetFactory
+from udata.core.dataset.models import Dataset
 
 from .. import backends
-from ..models import HarvestSource, HarvestJob
+from ..models import HarvestItem, HarvestSource, HarvestJob
 
 
 def dtfactory(start, end):
@@ -55,14 +56,20 @@ class FactoryBackend(backends.BaseBackend):
         backends.HarvestFeature('toggled', 'Toggled', 'A togglable', True),
     )
 
-    def initialize(self):
+    def inner_harvest(self):
         mock_initialize.send(self)
         for i in range(self.config.get('count', DEFAULT_COUNT)):
-            self.add_item(i)
+            self.process_dataset(str(i))
+            if self.is_done():
+                return
 
-    def process(self, item):
-        mock_process.send(self, item=item)
-        return DatasetFactory.build(title='dataset-{0}'.format(item.remote_id))
+    def inner_process_dataset(self, item: HarvestItem):
+        mock_process.send(self, item=item.remote_id)
+
+        dataset = self.get_dataset(item.remote_id)
+        dataset.title = f'dataset-{item.remote_id}'
+
+        return dataset
 
 
 class MockBackendsMixin(object):

udata/harvest/tests/test_actions.py

@@ -580,7 +580,7 @@ class ExecutionTestMixin(MockBackendsMixin):
 
     def test_error_on_item(self):
         def process(self, item):
-            if item.remote_id == '1':
+            if item == '1':
                 raise ValueError('test')
 
         source = HarvestSourceFactory(backend='factory')
@@ -723,7 +723,7 @@ class HarvestPreviewTest(MockBackendsMixin):
 
     def test_preview_with_error_on_item(self):
         def process(self, item):
-            if item.remote_id == '1':
+            if item == '1':
                 raise ValueError('test')
 
         source = HarvestSourceFactory(backend='factory')

udata/harvest/tests/test_base_backend.py

@@ -5,6 +5,7 @@ from urllib.parse import urlparse
 from dateutil.parser import parse
 from voluptuous import Schema
 
+from udata.harvest.models import HarvestItem
 from udata.utils import faker
 from udata.core.dataset import tasks
 from udata.core.dataset.factories import DatasetFactory
@@ -31,12 +32,16 @@ class FakeBackend(BaseBackend):
         HarvestFeature('enabled', 'A test feature enabled by default', default=True),
     )
 
-    def initialize(self):
+    def inner_harvest(self):
         for i in range(self.source.config.get('nb_datasets', 3)):
-            self.add_item('fake-{0}'.format(i))
+            remote_id = f'fake-{i}'
+            self.process_dataset(remote_id)
+            if self.is_done():
+                return
 
-    def process(self, item):
+    def inner_process_dataset(self, item: HarvestItem):
         dataset = self.get_dataset(item.remote_id)
+
         for key, value in DatasetFactory.as_dict(visible=True).items():
             setattr(dataset, key, value)
         if self.source.config.get('last_modified'):
@@ -219,8 +224,7 @@ class BaseBackendTest:
         assert 'archived_at' not in dataset_no_arch.harvest
 
         # test unarchive: archive manually then relaunch harvest
-        q = {'harvest__remote_id': 'fake-1'}
-        dataset = Dataset.objects.get(**q)
+        dataset = Dataset.objects.get(**{'harvest__remote_id': 'fake-1'})
         dataset.archived = datetime.utcnow()
         dataset.harvest.archived = 'not-on-remote'
         dataset.harvest.archived_at = datetime.utcnow()

udata/harvest/tests/test_dcat_backend.py

@@ -240,6 +240,19 @@ class DcatBackendTest:
         actions.purge_jobs()
         assert get_from_json(current_app.config.get('HARVEST_GRAPHS_S3_BUCKET'), job.data['filename']) is None
 
+    @pytest.mark.options(SCHEMA_CATALOG_URL='https://example.com/schemas', HARVEST_MAX_ITEMS=2)
+    def test_harvest_max_items(self, rmock):
+        rmock.get('https://example.com/schemas', json=ResourceSchemaMockData.get_mock_data())
+
+        filename = 'bnodes.xml'
+        url = mock_dcat(rmock, filename)
+        org = OrganizationFactory()
+        source = HarvestSourceFactory(backend='dcat', url=url, organization=org)
+
+        actions.run(source.slug)
+
+        assert Dataset.objects.count() == 2
+        assert HarvestJob.objects.first().status == 'done'
 
     @pytest.mark.options(SCHEMA_CATALOG_URL='https://example.com/schemas')
     def test_harvest_spatial(self, rmock):

udata/tests/dataservice/test_dataservice_tasks.py

@@ -0,0 +1,46 @@
+import pytest
+
+from udata.core.dataservices.models import Dataservice
+from udata.core.user.factories import UserFactory
+from udata.core.dataservices import tasks
+from udata.models import Discussion, Follow, Message, Transfer
+from udata.utils import faker
+
+
+pytestmark = pytest.mark.usefixtures('clean_db')
+
+
+def test_purge_dataservices():
+    dataservices = [
+        Dataservice.objects.create(
+            title='delete me',
+            base_api_url='https://example.com/api',
+            deleted_at='2016-01-01'),
+        Dataservice.objects.create(
+            title='keep me',
+            base_api_url='https://example.com/api'),
+    ]
+
+    user = UserFactory()
+    transfer = Transfer.objects.create(
+        owner=user,
+        recipient=user,
+        subject=dataservices[0],
+        comment='comment',
+    )
+
+    discussion = Discussion.objects.create(
+        subject=dataservices[0],
+        user=user,
+        title='test discussion',
+        discussion=[Message(content=faker.sentence(), posted_by=user)]
+    )
+
+    follower = Follow.objects.create(follower=user, following=dataservices[0])
+
+    tasks.purge_dataservices()
+
+    assert Dataservice.objects.count() == 1
+    assert Transfer.objects.filter(id=transfer.id).count() == 0
+    assert Discussion.objects.filter(id=discussion.id).count() == 0
+    assert Follow.objects.filter(id=follower.id).count() == 0

{udata-8.0.2.dev29253.dist-info → udata-8.0.2.dev29284.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: udata
-Version: 8.0.2.dev29253
+Version: 8.0.2.dev29284
 Summary: Open data portal
 Home-page: https://github.com/opendatateam/udata
 Author: Opendata Team
@@ -137,7 +137,9 @@ It is collectively taken care of by members of the
 
 ## Current (in progress)
 
-- Nothing yet
+- **breaking change** Harvest backend is now sync [#3030](https://github.com/opendatateam/udata/pull/3030)
+- Allow dataservices to be discussed and followed [#3049](https://github.com/opendatateam/udata/pull/3049)
+- Add purge-dataservices job [#3049](https://github.com/opendatateam/udata/pull/3049)
 
 ## 8.0.1 (2024-05-28)
 

{udata-8.0.2.dev29253.dist-info → udata-8.0.2.dev29284.dist-info}/RECORD

@@ -45,7 +45,7 @@ udata/commands/fixtures.py,sha256=RrzRdUBJ2FbS-iDsEMPu5Nfp0WJXzU2rk3GB9t6JlRc,58
 udata/commands/images.py,sha256=bzVvLj9LZ_nkyTdPlPe4g5U26k53VTH5ADCWmOfM8e4,2037
 udata/commands/info.py,sha256=Gf0AT8yP1tScbKLz8aXurlro55ot19sefrbOhELi034,1548
 udata/commands/init.py,sha256=t0dKNQ0_1Tg4r84h-XDwIZbdhkyEDu6k8jpn5YEAyOg,1524
-udata/commands/purge.py,sha256=qE6Q34UC3QC5XPUjv78sIwYMvG_VfQFDFbsoilxaN9M,963
+udata/commands/purge.py,sha256=weytJTaqDRHHrJd7bODkT-vNR2H8PehvWYhZHnNVAps,1203
 udata/commands/serve.py,sha256=EvXYQUzSr6FhDtgSmiKkNpkPesrx4r74KpMd6iUOUN8,2567
 udata/commands/static.py,sha256=tAB4WsJ9MhFjdmVif57RKGUhXpbQPyh1Oktzu86Baz4,2201
 udata/commands/test.py,sha256=h1fgPpWARY7ucu1XzYRje-OuQflS-ThhpBRA35IO_Os,892
@@ -79,8 +79,10 @@ udata/core/contact_point/factories.py,sha256=ATuV1seBCGKY3CzvPDG5nxfBBqHu-3YtER0
 udata/core/contact_point/forms.py,sha256=ggLhSJ1IRn5MclrhydckjAxwr4fFZxgAD4huSSucSsA,598
 udata/core/contact_point/models.py,sha256=NlNKureCpzgTLJuGviZPjNx-ABYRp4j2L-ur9Gmixao,324
 udata/core/dataservices/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-udata/core/dataservices/api.py,sha256=j0_fQdq_O3zBqY7PGapmNbeMjb2sS4x8jr3cTM-4i_Q,2803
-udata/core/dataservices/models.py,sha256=G8m-YKHf32DdA-3Q4m-hlQCLZ8ZZUFKaRFD0YZtqvbE,4037
+udata/core/dataservices/api.py,sha256=rjCU55NNGgCDRlurfhJUT2byBGJWN5coM8b7AApzEew,3090
+udata/core/dataservices/models.py,sha256=6cE7tQ6GahnFaz1U_8yNmJqlCzoHQxZdet4qXQaqa7k,4449
+udata/core/dataservices/permissions.py,sha256=X9Bh8e0pnx6OgeEf6NowXZUiwyreUa6UY479B16cCqs,175
+udata/core/dataservices/tasks.py,sha256=NOWcTPoLasMrrvq9EkwQMGlUbQQmi_l3s815K-mtZTM,971
 udata/core/dataset/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 udata/core/dataset/actions.py,sha256=3pzBg_qOR-w7fwPpTOKUHXWC9lkjALbOn1UQFmmT-s0,1199
 udata/core/dataset/activities.py,sha256=qQnHNL0hOB1IGtQl7JsnVOiUsWT0gm-pts9uDyR3bvU,1536
@@ -274,16 +276,16 @@ udata/harvest/forms.py,sha256=AxEFhBACjXVT9X1bSK8WzDAZgWg-hdaWBebFb1ihNYo,3389
 udata/harvest/models.py,sha256=7mTZKcIxtagW5GimYIqPyzQ4tqExO3RzKxqOh0tB9PE,5988
 udata/harvest/notifications.py,sha256=aNKUtUl73Caj_kl-ENOEbTX_cgtySqLjHhtH9FEDR9Y,774
 udata/harvest/signals.py,sha256=wlXTi1E7rIVyNvxw0yUqyN5gF3thg276LAOmAF9vDJY,1338
-udata/harvest/tasks.py,sha256=0VhefKCQJSU_puTpdKOpvt3WORXHAFWGEB-R_MhB12M,1981
+udata/harvest/tasks.py,sha256=XbJPoposzkxitecDaK5WYnPqzCatAU-eHX5MUDu9nJo,1644
 udata/harvest/backends/__init__.py,sha256=qcLhHKWO97TeWd93ZwymG_Cc9FO7sMM7h4fs6XYdtS8,447
-udata/harvest/backends/base.py,sha256=oaPQcQ0onIXH5ofUtWH5sM6_5_wSBLawHSOjeeoG6jQ,12258
-udata/harvest/backends/dcat.py,sha256=q5v6sUm8xBFYH437S3MUTAu40Ecka1Y6Oj6VB6xl2B4,15300
+udata/harvest/backends/base.py,sha256=A0itOkDRPPa7_umHI0q5bybBk0ZtEBTqeaq3DIe-bPw,11752
+udata/harvest/backends/dcat.py,sha256=MzuBgUDNREjstFJIo5Z-epHqveT_nychIynJ633rNVU,13940
 udata/harvest/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-udata/harvest/tests/factories.py,sha256=CbQORC1OJ1_Agtv_3LjCXysNumjMYlROwZPSEAHo8sM,2005
-udata/harvest/tests/test_actions.py,sha256=7xSpouCAcf5p_bd38zHCyPN7sKWUUZXA7IlpI-yNVrQ,27603
+udata/harvest/tests/factories.py,sha256=vUFON9GzI5CbD3bP8_ayOs3S9pHbhhHiI7B4GhoQtVE,2218
+udata/harvest/tests/test_actions.py,sha256=TPHb8n8tlQ0l2lu8GzbymKPOpzTeN_VCtFmszZoCAQI,27583
 udata/harvest/tests/test_api.py,sha256=QXhseHfnkBEmMbIJzroMdDYGLDj6Njal1s-2sn0xhEM,14888
-udata/harvest/tests/test_base_backend.py,sha256=JA8Df1Eu-lEPLZfxyK81bsmT6exOjV_3PtKHJekAp5g,12092
-udata/harvest/tests/test_dcat_backend.py,sha256=9Ghhe9faOJipLdz5mEUlZP7ZpCgJk0zTN2VREqZTzAA,32574
+udata/harvest/tests/test_base_backend.py,sha256=idFssHnN1iv2ktP1b1IlDpGglVR4Rzza-XuJr68KIlA,12240
+udata/harvest/tests/test_dcat_backend.py,sha256=QsCyi00TeWeb3HUg_ym-PjO5f4DaEm4HIzbc8Wgs4Yc,33138
 udata/harvest/tests/test_filters.py,sha256=V2HFZlexIJa6r1DX6g2ktvIgjg4gSY11QPfPOd3_Oug,2370
 udata/harvest/tests/test_models.py,sha256=p2VazyrPXSArBuf8Kf19TGPcQ86SnOGCGmvjcMOw0s0,924
 udata/harvest/tests/test_notifications.py,sha256=ZwtwioittW3XcZc0x6zbHjs1dVaAxPytlVymnJa5w0E,817
@@ -608,6 +610,7 @@ udata/tests/cli/test_cli_base.py,sha256=piqoq4Ib5bdZQpuUAJh583qfjCSglWZQclKzhO3Y
 udata/tests/cli/test_db_cli.py,sha256=-hw9SU3RvNT7fvqVtQHxEpKstnjMU4L_DY9tiBH-ybs,1726
 udata/tests/data/image.jpg,sha256=hdmpaCjOhmAAfNGuTqWKEjv7IC4GXJx-nP_rT274hc8,337049
 udata/tests/data/image.png,sha256=GAqXz7w_u7CapODIUF45UpVddmqelnGQkcrwKZq3448,266488
+udata/tests/dataservice/test_dataservice_tasks.py,sha256=bRL6A0arzAA0a_k0mAc_mZ9vkT23Cw3kNr40NIhj4PM,1375
 udata/tests/dataset/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 udata/tests/dataset/test_csv_adapter.py,sha256=2r5z4FuqG9pWgUyfDUw8afQYK5nIKE8XXOCuDHmEcZA,3207
 udata/tests/dataset/test_dataset_actions.py,sha256=4_0F_TCSCfHpzqDt659c8hG4GpkWjVtmT4bAPgmE4B8,723
@@ -681,9 +684,9 @@ udata/translations/pt/LC_MESSAGES/udata.mo,sha256=uttB2K8VsqzkEQG-5HfTtFms_3LtV9
 udata/translations/pt/LC_MESSAGES/udata.po,sha256=8Ql1Lp7Z9KLnvp-qRxw-NhFu1p35Xj-q6Jg9JHsYhcw,43733
 udata/translations/sr/LC_MESSAGES/udata.mo,sha256=US8beNIMPxP5h-zD_jfP1TheDDd4DdRVS5UIiY5XVZ8,28553
 udata/translations/sr/LC_MESSAGES/udata.po,sha256=TM0yMDvKRljyOzgZZMlTX6OfpF6OC4Ngf_9Zc8n6ayA,50313
-udata-8.0.2.dev29253.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
-udata-8.0.2.dev29253.dist-info/METADATA,sha256=tWlzzUKyNCnDXnnDe8KxJ3AcR8OOe-OtGLSCere9U5g,123456
-udata-8.0.2.dev29253.dist-info/WHEEL,sha256=DZajD4pwLWue70CAfc7YaxT1wLUciNBvN_TTcvXpltE,110
-udata-8.0.2.dev29253.dist-info/entry_points.txt,sha256=3SKiqVy4HUqxf6iWspgMqH8d88Htk6KoLbG1BU-UddQ,451
-udata-8.0.2.dev29253.dist-info/top_level.txt,sha256=39OCg-VWFWOq4gCKnjKNu-s3OwFlZIu_dVH8Gl6ndHw,12
-udata-8.0.2.dev29253.dist-info/RECORD,,
+udata-8.0.2.dev29284.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
+udata-8.0.2.dev29284.dist-info/METADATA,sha256=gi8odHkhnGS9sSIVoTNtLJKRsuYqZgEHE2cBYsLF8Gw,123742
+udata-8.0.2.dev29284.dist-info/WHEEL,sha256=DZajD4pwLWue70CAfc7YaxT1wLUciNBvN_TTcvXpltE,110
+udata-8.0.2.dev29284.dist-info/entry_points.txt,sha256=3SKiqVy4HUqxf6iWspgMqH8d88Htk6KoLbG1BU-UddQ,451
+udata-8.0.2.dev29284.dist-info/top_level.txt,sha256=39OCg-VWFWOq4gCKnjKNu-s3OwFlZIu_dVH8Gl6ndHw,12
+udata-8.0.2.dev29284.dist-info/RECORD,,