udata 8.0.2.dev29253__py2.py3-none-any.whl → 8.0.2.dev29263__py2.py3-none-any.whl

udata/harvest/backends/base.py

@@ -2,6 +2,7 @@ import logging
 import traceback
 
 from datetime import datetime, date, timedelta
+from typing import Optional
 from uuid import UUID
 
 import requests
@@ -68,7 +69,12 @@ class HarvestFeature(object):
 
 
 class BaseBackend(object):
-    '''Base class for Harvester implementations'''
+    """
+    Base class that wrap children methods to add error management and debug logs.
+    Also provides a few helpers needed on all or some backends.
+
+
+    """
 
     name = None
     display_name = None
@@ -127,92 +133,69 @@ class BaseBackend(object):
     def get_filters(self):
         return self.config.get('filters', [])
 
-    def harvest(self):
-        '''Start the harvesting process'''
-        if self.perform_initialization() is not None:
-            self.process_items()
-            self.finalize()
-        return self.job
+    def inner_harvest(self):
+        raise NotImplementedError
+    
+    def inner_process_dataset(self, item: HarvestItem) -> Dataset:
+        raise NotImplementedError
 
-    def perform_initialization(self):
-        '''Initialize the harvesting for a given job'''
-        log.debug('Initializing backend')
+    def harvest(self):
+        log.debug(f'Starting harvesting {self.source.name} ({self.source.url})…')
         factory = HarvestJob if self.dryrun else HarvestJob.objects.create
-        self.job = factory(status='initializing',
+        self.job = factory(status='initialized',
                            started=datetime.utcnow(),
                            source=self.source)
 
         before_harvest_job.send(self)
 
         try:
-            self.initialize()
-            self.job.status = 'initialized'
-            if not self.dryrun:
-                self.job.save()
+            self.inner_harvest()
+
+            if self.source.autoarchive:
+                self.autoarchive()
+
+            self.job.status = 'done'
+
+            if any(i.status == 'failed' for i in self.job.items):
+                self.job.status += '-errors'
         except HarvestValidationError as e:
-            log.info('Initialization failed for "%s" (%s)',
-                     safe_unicode(self.source.name), self.source.backend)
-            error = HarvestError(message=safe_unicode(e))
-            self.job.errors.append(error)
-            self.job.status = 'failed'
-            self.end()
-            return None
-        except Exception as e:
+            log.exception(f'Harvesting validation failed for "{safe_unicode(self.source.name)}" ({self.source.backend})')
+
             self.job.status = 'failed'
+
             error = HarvestError(message=safe_unicode(e))
             self.job.errors.append(error)
-            self.end()
-            msg = 'Initialization failed for "{0.name}" ({0.backend})'
-            log.exception(msg.format(self.source))
-            return None
+        except Exception as e:
+            log.exception(f'Harvesting failed for "{safe_unicode(self.source.name)}" ({self.source.backend})')
 
-        if self.max_items:
-            self.job.items = self.job.items[:self.max_items]
+            self.job.status = 'failed'
 
-        if self.job.items:
-            log.debug('Queued %s items', len(self.job.items))
+            error = HarvestError(message=safe_unicode(e), details=traceback.format_exc())
+            self.job.errors.append(error)
+        finally:
+            self.end_job()
+        
+        return self.job
 
-        return len(self.job.items)
+    def process_dataset(self, remote_id: str, **kwargs):
+        log.debug(f'Processing dataset {remote_id}…')
 
-    def initialize(self):
-        raise NotImplementedError
+        # TODO add `type` to `HarvestItem` to differentiate `Dataset` from `Dataservice`
+        item = HarvestItem(status='started', started=datetime.utcnow(), remote_id=remote_id)
+        self.job.items.append(item)
+        self.save_job()
 
-    def process_items(self):
-        '''Process the data identified in the initialize stage'''
-        for item in self.job.items:
-            self.process_item(item)
+        try:
+            if not remote_id:
+                raise HarvestSkipException("missing identifier")
 
-    def process_item(self, item):
-        log.debug('Processing: %s', item.remote_id)
-        item.status = 'started'
-        item.started = datetime.utcnow()
-        if not self.dryrun:
-            self.job.save()
+            dataset = self.inner_process_dataset(item, **kwargs)
 
-        try:
-            dataset = self.process(item)
-            if not dataset.harvest:
-                dataset.harvest = HarvestDatasetMetadata()
-            dataset.harvest.domain = self.source.domain
-            dataset.harvest.remote_id = item.remote_id
-            dataset.harvest.source_id = str(self.source.id)
-            dataset.harvest.last_update = datetime.utcnow()
-            dataset.harvest.backend = self.display_name
-
-            # unset archived status if needed
-            if dataset.harvest:
-                dataset.harvest.archived_at = None
-                dataset.harvest.archived = None
+            # Use `item.remote_id` because `inner_process_dataset` could have modified it.
+            dataset.harvest = self.update_harvest_info(dataset.harvest, item.remote_id)
             dataset.archived = None
 
-            # TODO permissions checking
-            if not dataset.organization and not dataset.owner:
-                if self.source.organization:
-                    dataset.organization = self.source.organization
-                elif self.source.owner:
-                    dataset.owner = self.source.owner
-
-            # TODO: Apply editble mappings
+            # TODO: Apply editable mappings
 
             if self.dryrun:
                 dataset.validate()
@@ -221,26 +204,54 @@ class BaseBackend(object):
             item.dataset = dataset
             item.status = 'done'
         except HarvestSkipException as e:
-            log.info('Skipped item %s : %s', item.remote_id, safe_unicode(e))
             item.status = 'skipped'
+
+            log.info(f'Skipped item {item.remote_id} : {safe_unicode(e)}')
             item.errors.append(HarvestError(message=safe_unicode(e)))
         except HarvestValidationError as e:
-            log.info('Error validating item %s : %s', item.remote_id, safe_unicode(e))
             item.status = 'failed'
+
+            log.info(f'Error validating item {item.remote_id} : {safe_unicode(e)}')
             item.errors.append(HarvestError(message=safe_unicode(e)))
         except Exception as e:
-            log.exception('Error while processing %s : %s',
-                          item.remote_id,
-                          safe_unicode(e))
-            error = HarvestError(message=safe_unicode(e),
-                                 details=traceback.format_exc())
-            item.errors.append(error)
             item.status = 'failed'
+            log.exception(f'Error while processing {item.remote_id} : {safe_unicode(e)}')
+
+            error = HarvestError(message=safe_unicode(e), details=traceback.format_exc())
+            item.errors.append(error)
+        finally:
+            item.ended = datetime.utcnow()
+            self.save_job()
+
+    def is_done(self) -> bool:
+        '''Should be called after process_dataset to know if we reach the max items'''
+        return self.max_items and len(self.job.items) >= self.max_items
+
+    def update_harvest_info(self, harvest: Optional[HarvestDatasetMetadata], remote_id: int):
+        if not harvest:
+            harvest = HarvestDatasetMetadata()
+        harvest.domain = self.source.domain
+        harvest.remote_id = remote_id
+        harvest.source_id = str(self.source.id)
+        harvest.last_update = datetime.utcnow()
+        harvest.backend = self.display_name
 
-        item.ended = datetime.utcnow()
+        harvest.archived_at = None
+        harvest.archived = None
+
+        return harvest
+
+    def save_job(self):
+        if not self.dryrun:
+            self.job.save()
+
+    def end_job(self):
+        self.job.ended = datetime.utcnow()
         if not self.dryrun:
             self.job.save()
 
+        after_harvest_job.send(self)
+
     def autoarchive(self):
         '''
         Archive items that exist on the local instance but not on remote platform
@@ -262,34 +273,13 @@ class BaseBackend(object):
                 archive_harvested_dataset(dataset, reason='not-on-remote', dryrun=self.dryrun)
             # add a HarvestItem to the job list (useful for report)
             # even when archiving has already been done (useful for debug)
-            item = self.add_item(dataset.harvest.remote_id)
-            item.dataset = dataset
-            item.status = 'archived'
-
-            if not self.dryrun:
-                self.job.save()
+            self.job.items.append(HarvestItem(
+                remote_id=str(dataset.harvest.remote_id),
+                dataset=dataset,
+                status='archived'
+            ))
 
-    def process(self, item):
-        raise NotImplementedError
-
-    def add_item(self, identifier, *args, **kwargs):
-        item = HarvestItem(remote_id=str(identifier), args=args, kwargs=kwargs)
-        self.job.items.append(item)
-        return item
-
-    def finalize(self):
-        if self.source.autoarchive:
-            self.autoarchive()
-        self.job.status = 'done'
-        if any(i.status == 'failed' for i in self.job.items):
-            self.job.status += '-errors'
-        self.end()
-
-    def end(self):
-        self.job.ended = datetime.utcnow()
-        if not self.dryrun:
-            self.job.save()
-        after_harvest_job.send(self)
+            self.save_job()
 
     def get_dataset(self, remote_id):
         '''Get or create a dataset given its remote ID (and its source)
@@ -352,4 +342,4 @@ class BaseBackend(object):
                     msg = str(error)
                 errors.append(msg)
             msg = '\n- '.join(['Validation error:'] + errors)
-            raise HarvestValidationError(msg)
+            raise HarvestValidationError(msg)

udata/harvest/backends/dcat.py

@@ -7,13 +7,15 @@ import boto3
 from flask import current_app
 from datetime import date
 import json
-from typing import List
+from typing import Generator, List
 
+from udata.core.dataset.models import Dataset
 from udata.rdf import (
     DCAT, DCT, HYDRA, SPDX, namespace_manager, guess_format, url_from_rdf
 )
 from udata.core.dataset.rdf import dataset_from_rdf
 from udata.storage.s3 import store_as_json, get_from_json
+from udata.harvest.models import HarvestItem
 
 from .base import BaseBackend
 
@@ -59,14 +61,17 @@ def extract_graph(source, target, node, specs):
 class DcatBackend(BaseBackend):
     display_name = 'DCAT'
 
-    def initialize(self):
-        '''List all datasets for a given ...'''
+    def inner_harvest(self):
         fmt = self.get_format()
-        graphs = self.parse_graph(self.source.url, fmt)
-
         self.job.data = { 'format': fmt }
 
-        serialized_graphs = [graph.serialize(format=fmt, indent=None) for graph in graphs]
+        serialized_graphs = []
+
+        for page_number, page in self.walk_graph(self.source.url, fmt):
+            self.process_one_datasets_page(page_number, page)
+            serialized_graphs.append(page.serialize(format=fmt, indent=None))
+
+        # TODO call `walk_graph` with `process_dataservices`
 
         # The official MongoDB document size in 16MB. The default value here is 15MB to account for other fields in the document (and for difference between * 1024 vs * 1000).
         max_harvest_graph_size_in_mongo = current_app.config.get('HARVEST_MAX_CATALOG_SIZE_IN_MONGO')
@@ -105,13 +110,11 @@ class DcatBackend(BaseBackend):
                 raise ValueError(msg)
         return fmt
 
-    def parse_graph(self, url, fmt) -> List[Graph]:
+    def walk_graph(self, url: str, fmt: str) -> Generator[tuple[int, Graph], None, None]:
         """
-        Returns an instance of rdflib.Graph for each detected page
-        The index in the list is the page number
+        Yield all RDF pages as `Graph` from the source
         """
-        graphs = []
-        page = 0
+        page_number = 0
         while url:
             subgraph = Graph(namespace_manager=namespace_manager)
             response = self.get(url)
@@ -128,19 +131,26 @@ class DcatBackend(BaseBackend):
                     pagination = subgraph.resource(pagination)
                     url = url_from_rdf(pagination, prop)
                     break
-            graphs.append(subgraph)
 
-            for node in subgraph.subjects(RDF.type, DCAT.Dataset):
-                id = subgraph.value(node, DCT.identifier)
-                kwargs = {'page': page}
-                self.add_item(id, **kwargs)
-                if self.max_items and len(self.job.items) >= self.max_items:
-                    # this will stop iterating on pagination
-                    url = None
+            yield page_number, subgraph
+            if self.is_done():
+                return
 
-            page += 1
+            page_number += 1
+    
+    def process_one_datasets_page(self, page_number: int, page: Graph):
+        for node in page.subjects(RDF.type, DCAT.Dataset):
+            remote_id = page.value(node, DCT.identifier)
+            self.process_dataset(remote_id, page_number=page_number, page=page, node=node)
+
+            if self.is_done():
+                return
+            
+    def inner_process_dataset(self, item: HarvestItem, page_number: int, page: Graph, node):
+        item.kwargs['page_number'] = page_number
 
-        return graphs
+        dataset = self.get_dataset(item.remote_id)
+        return dataset_from_rdf(page, dataset, node=node)
 
     def get_node_from_item(self, graph, item):
         for node in graph.subjects(RDF.type, DCAT.Dataset):
@@ -148,33 +158,6 @@ class DcatBackend(BaseBackend):
                 return node
         raise ValueError(f'Unable to find dataset with DCT.identifier:{item.remote_id}')
 
-    def process(self, item):
-        if item.remote_id == 'None':
-            raise ValueError('The DCT.identifier is missing on this DCAT.Dataset record')
-        graph = Graph(namespace_manager=namespace_manager)
-
-        if self.job.data.get('graphs') is not None:
-            graphs = self.job.data['graphs']
-        else:
-            bucket = current_app.config.get('HARVEST_GRAPHS_S3_BUCKET')
-            if bucket is None:
-                raise ValueError(f"No bucket configured but the harvest job item {item.id} on job {self.job.id} doesn't have a graph in MongoDB.")
-
-            graphs = get_from_json(bucket, self.job.data['filename'])
-            if graphs is None:
-                raise ValueError(f"The file '{self.job.data['filename']}' is missing in S3 bucket '{bucket}'")
-
-        data = graphs[item.kwargs['page']]
-        format = self.job.data['format']
-
-        graph.parse(data=bytes(data, encoding='utf8'), format=format)
-        node = self.get_node_from_item(graph, item)
-
-        dataset = self.get_dataset(item.remote_id)
-        dataset = dataset_from_rdf(graph, dataset, node=node)
-        return dataset
-    
-
     def next_record_if_should_continue(self, start, search_results):
         next_record = int(search_results.attrib['nextRecord'])
         matched_count = int(search_results.attrib['numberOfRecordsMatched'])
@@ -209,7 +192,10 @@ class CswDcatBackend(DcatBackend):
 
     DCAT_SCHEMA = 'http://www.w3.org/ns/dcat#'
 
-    def parse_graph(self, url: str, fmt: str) -> List[Graph]:
+    def walk_graph(self, url: str, fmt: str) -> Generator[tuple[int, Graph], None, None]:
+        """
+        Yield all RDF pages as `Graph` from the source
+        """
         body = '''<csw:GetRecords xmlns:csw="http://www.opengis.net/cat/csw/2.0.2"
                                   xmlns:gmd="http://www.isotc211.org/2005/gmd"
                                   service="CSW" version="2.0.2" resultType="results"
@@ -227,8 +213,7 @@ class CswDcatBackend(DcatBackend):
                 </csw:GetRecords>'''
         headers = {'Content-Type': 'application/xml'}
 
-        graphs = []
-        page = 0
+        page_number = 0
         start = 1
 
         response = self.post(url, data=body.format(start=start, schema=self.DCAT_SCHEMA),
@@ -239,36 +224,29 @@ class CswDcatBackend(DcatBackend):
         if tree.tag == '{' + OWS_NAMESPACE + '}ExceptionReport':
             raise ValueError(f'Failed to query CSW:\n{content}')
         while tree:
-            graph = Graph(namespace_manager=namespace_manager)
             search_results = tree.find('csw:SearchResults', {'csw': CSW_NAMESPACE})
             if search_results is None:
-                log.error(f'No search results found for {url} on page {page}')
+                log.error(f'No search results found for {url} on page {page_number}')
                 break
             for child in search_results:
                 subgraph = Graph(namespace_manager=namespace_manager)
                 subgraph.parse(data=ET.tostring(child), format=fmt)
-                graph += subgraph
 
-                for node in subgraph.subjects(RDF.type, DCAT.Dataset):
-                    id = subgraph.value(node, DCT.identifier)
-                    kwargs = {'nid': str(node), 'page': page}
-                    kwargs['type'] = 'uriref' if isinstance(node, URIRef) else 'blank'
-                    self.add_item(id, **kwargs)
-            graphs.append(graph)
+                yield page_number, subgraph
+                if self.is_done():
+                    return
 
             next_record = self.next_record_if_should_continue(start, search_results)
             if not next_record:
                 break
 
             start = next_record
-            page += 1
+            page_number += 1
 
             tree = ET.fromstring(
                 self.post(url, data=body.format(start=start, schema=self.DCAT_SCHEMA),
                           headers=headers).content)
 
-        return graphs
-
 
 class CswIso19139DcatBackend(DcatBackend):
     '''
@@ -282,13 +260,10 @@ class CswIso19139DcatBackend(DcatBackend):
 
     XSL_URL = "https://raw.githubusercontent.com/SEMICeu/iso-19139-to-dcat-ap/master/iso-19139-to-dcat-ap.xsl"
 
-    def parse_graph(self, url: str, fmt: str) -> List[Graph]:
-        '''
-        Parse CSW graph querying ISO schema.
-        Use SEMIC GeoDCAT-AP XSLT to map it to a correct version.
-        See https://github.com/SEMICeu/iso-19139-to-dcat-ap for more information on the XSLT.
-        '''
-
+    def walk_graph(self, url: str, fmt: str) -> Generator[tuple[int, Graph], None, None]:
+        """
+        Yield all RDF pages as `Graph` from the source
+        """
         # Load XSLT
         xsl = ET.fromstring(self.get(self.XSL_URL).content)
         transform = ET.XSLT(xsl)
@@ -320,8 +295,7 @@ class CswIso19139DcatBackend(DcatBackend):
                 </csw:GetRecords>'''
         headers = {'Content-Type': 'application/xml'}
 
-        graphs = []
-        page = 0
+        page_number = 0
         start = 1
 
         response = self.post(url, data=body.format(start=start, schema=self.ISO_SCHEMA),
@@ -338,7 +312,7 @@ class CswIso19139DcatBackend(DcatBackend):
             # infos (useful for pagination)
             search_results = tree_before_transform.find('csw:SearchResults', {'csw': CSW_NAMESPACE})
             if search_results is None:
-                log.error(f'No search results found for {url} on page {page}')
+                log.error(f'No search results found for {url} on page {page_number}')
                 break
 
             subgraph = Graph(namespace_manager=namespace_manager)
@@ -347,19 +321,16 @@ class CswIso19139DcatBackend(DcatBackend):
             if not subgraph.subjects(RDF.type, DCAT.Dataset):
                 raise ValueError("Failed to fetch CSW content")
 
-            for node in subgraph.subjects(RDF.type, DCAT.Dataset):
-                id = subgraph.value(node, DCT.identifier)
-                kwargs = {'nid': str(node), 'page': page}
-                kwargs['type'] = 'uriref' if isinstance(node, URIRef) else 'blank'
-                self.add_item(id, **kwargs)
-            graphs.append(subgraph)
+            yield page_number, subgraph
+            if self.is_done():
+                return
 
             next_record = self.next_record_if_should_continue(start, search_results)
             if not next_record:
                 break
 
             start = next_record
-            page += 1
+            page_number += 1
 
             response = self.post(url, data=body.format(start=start, schema=self.ISO_SCHEMA),
                                  headers=headers)
@@ -367,5 +338,3 @@ class CswIso19139DcatBackend(DcatBackend):
 
             tree_before_transform = ET.fromstring(response.content)
             tree = transform(tree_before_transform, CoupledResourceLookUp="'disabled'")
-
-        return graphs

udata/harvest/tasks.py

@@ -18,20 +18,10 @@ def harvest(self, ident):
         return  # Ignore deleted sources
     Backend = backends.get(current_app, source.backend)
     backend = Backend(source)
-    items = backend.perform_initialization()
-    if items is None:
-        pass
-    elif items == 0:
-        backend.finalize()
-    else:
-        finalize = harvest_job_finalize.s(backend.job.id)
-        items = [
-            harvest_job_item.s(backend.job.id, item.remote_id)
-            for item in backend.job.items
-        ]
-        chord(items)(finalize)
-    
 
+    backend.harvest()
+
+    
 
 @task(ignore_result=False, route='low.harvest')
 def harvest_job_item(job_id, item_id):

udata/harvest/tests/factories.py

@@ -6,9 +6,10 @@ from flask.signals import Namespace
 
 from udata.factories import ModelFactory
 from udata.core.dataset.factories import DatasetFactory
+from udata.core.dataset.models import Dataset
 
 from .. import backends
-from ..models import HarvestSource, HarvestJob
+from ..models import HarvestItem, HarvestSource, HarvestJob
 
 
 def dtfactory(start, end):
@@ -55,14 +56,20 @@ class FactoryBackend(backends.BaseBackend):
         backends.HarvestFeature('toggled', 'Toggled', 'A togglable', True),
     )
 
-    def initialize(self):
+    def inner_harvest(self):
         mock_initialize.send(self)
         for i in range(self.config.get('count', DEFAULT_COUNT)):
-            self.add_item(i)
+            self.process_dataset(str(i))
+            if self.is_done():
+                return
 
-    def process(self, item):
-        mock_process.send(self, item=item)
-        return DatasetFactory.build(title='dataset-{0}'.format(item.remote_id))
+    def inner_process_dataset(self, item: HarvestItem):
+        mock_process.send(self, item=item.remote_id)
+
+        dataset = self.get_dataset(item.remote_id)
+        dataset.title = f'dataset-{item.remote_id}'
+
+        return dataset
 
 
 class MockBackendsMixin(object):

udata/harvest/tests/test_actions.py

@@ -580,7 +580,7 @@ class ExecutionTestMixin(MockBackendsMixin):
 
     def test_error_on_item(self):
         def process(self, item):
-            if item.remote_id == '1':
+            if item == '1':
                 raise ValueError('test')
 
         source = HarvestSourceFactory(backend='factory')
@@ -723,7 +723,7 @@ class HarvestPreviewTest(MockBackendsMixin):
 
     def test_preview_with_error_on_item(self):
         def process(self, item):
-            if item.remote_id == '1':
+            if item == '1':
                 raise ValueError('test')
 
         source = HarvestSourceFactory(backend='factory')

udata/harvest/tests/test_base_backend.py

@@ -5,6 +5,7 @@ from urllib.parse import urlparse
 from dateutil.parser import parse
 from voluptuous import Schema
 
+from udata.harvest.models import HarvestItem
 from udata.utils import faker
 from udata.core.dataset import tasks
 from udata.core.dataset.factories import DatasetFactory
@@ -31,12 +32,16 @@ class FakeBackend(BaseBackend):
         HarvestFeature('enabled', 'A test feature enabled by default', default=True),
     )
 
-    def initialize(self):
+    def inner_harvest(self):
         for i in range(self.source.config.get('nb_datasets', 3)):
-            self.add_item('fake-{0}'.format(i))
+            remote_id = f'fake-{i}'
+            self.process_dataset(remote_id)
+            if self.is_done():
+                return
 
-    def process(self, item):
+    def inner_process_dataset(self, item: HarvestItem):
         dataset = self.get_dataset(item.remote_id)
+
         for key, value in DatasetFactory.as_dict(visible=True).items():
             setattr(dataset, key, value)
         if self.source.config.get('last_modified'):
@@ -219,8 +224,7 @@ class BaseBackendTest:
         assert 'archived_at' not in dataset_no_arch.harvest
 
         # test unarchive: archive manually then relaunch harvest
-        q = {'harvest__remote_id': 'fake-1'}
-        dataset = Dataset.objects.get(**q)
+        dataset = Dataset.objects.get(**{'harvest__remote_id': 'fake-1'})
         dataset.archived = datetime.utcnow()
         dataset.harvest.archived = 'not-on-remote'
         dataset.harvest.archived_at = datetime.utcnow()

udata/harvest/tests/test_dcat_backend.py

@@ -240,6 +240,19 @@ class DcatBackendTest:
         actions.purge_jobs()
         assert get_from_json(current_app.config.get('HARVEST_GRAPHS_S3_BUCKET'), job.data['filename']) is None
 
+    @pytest.mark.options(SCHEMA_CATALOG_URL='https://example.com/schemas', HARVEST_MAX_ITEMS=2)
+    def test_harvest_max_items(self, rmock):
+        rmock.get('https://example.com/schemas', json=ResourceSchemaMockData.get_mock_data())
+
+        filename = 'bnodes.xml'
+        url = mock_dcat(rmock, filename)
+        org = OrganizationFactory()
+        source = HarvestSourceFactory(backend='dcat', url=url, organization=org)
+
+        actions.run(source.slug)
+
+        assert Dataset.objects.count() == 2
+        assert HarvestJob.objects.first().status == 'done'
 
     @pytest.mark.options(SCHEMA_CATALOG_URL='https://example.com/schemas')
     def test_harvest_spatial(self, rmock):