udata 9.1.2.dev30355__py2.py3-none-any.whl → 9.1.2.dev30454__py2.py3-none-any.whl

udata/harvest/backends/dcat.py

@@ -1,19 +1,25 @@
 import logging
+from datetime import date
+from typing import Generator
 
-from rdflib import Graph
-from rdflib.namespace import RDF
 import lxml.etree as ET
 from flask import current_app
-from datetime import date
-from typing import Generator
+from rdflib import Graph
+from rdflib.namespace import RDF
 
+from udata.core.dataservices.rdf import dataservice_from_rdf
+from udata.core.dataset.rdf import dataset_from_rdf
+from udata.harvest.models import HarvestItem
 from udata.rdf import (
-    DCAT, DCT, HYDRA, SPDX, namespace_manager, guess_format, url_from_rdf
+    DCAT,
+    DCT,
+    HYDRA,
+    SPDX,
+    guess_format,
+    namespace_manager,
+    url_from_rdf,
 )
-from udata.core.dataset.rdf import dataset_from_rdf
-from udata.core.dataservices.rdf import dataservice_from_rdf
 from udata.storage.s3 import store_as_json
-from udata.harvest.models import HarvestItem
 
 from .base import BaseBackend
 
@@ -23,9 +29,7 @@ log = logging.getLogger(__name__)
 # Attributes representing nested classes to be stored in the graph
 # in order to have a complete graph
 DCAT_NESTING = {
-    DCAT.distribution: {
-        SPDX.checksum: {}
-    },
+    DCAT.distribution: {SPDX.checksum: {}},
     DCT.temporal: {},
     DCT.spatial: {},
 }
@@ -36,16 +40,16 @@ DCAT_NESTING[DCAT.distributions] = DCAT_NESTING[DCAT.distribution]
 # Known pagination class and their next page property
 KNOWN_PAGINATION = (
     (HYDRA.PartialCollectionView, HYDRA.next),
-    (HYDRA.PagedCollection, HYDRA.nextPage)
+    (HYDRA.PagedCollection, HYDRA.nextPage),
 )
 
-CSW_NAMESPACE = 'http://www.opengis.net/cat/csw/2.0.2'
-OWS_NAMESPACE = 'http://www.opengis.net/ows'
+CSW_NAMESPACE = "http://www.opengis.net/cat/csw/2.0.2"
+OWS_NAMESPACE = "http://www.opengis.net/ows"
 
 # Useful to patch essential failing URIs
 URIS_TO_REPLACE = {
     # See https://github.com/etalab/data.gouv.fr/issues/1151
-    'https://project-open-data.cio.gov/v1.1/schema/catalog.jsonld': 'https://gist.githubusercontent.com/maudetes/f019586185d6f59dcfb07f97148a1973/raw/585c3c7bf602b5a4e635b137257d0619792e2c1f/gistfile1.txt'  # noqa
+    "https://project-open-data.cio.gov/v1.1/schema/catalog.jsonld": "https://gist.githubusercontent.com/maudetes/f019586185d6f59dcfb07f97148a1973/raw/585c3c7bf602b5a4e635b137257d0619792e2c1f/gistfile1.txt"  # noqa
 }
 
 
@@ -57,11 +61,11 @@ def extract_graph(source, target, node, specs):
 
 
 class DcatBackend(BaseBackend):
-    display_name = 'DCAT'
+    display_name = "DCAT"
 
     def inner_harvest(self):
         fmt = self.get_format()
-        self.job.data = { 'format': fmt }
+        self.job.data = {"format": fmt}
 
         serialized_graphs = []
 
@@ -73,24 +77,30 @@ class DcatBackend(BaseBackend):
             self.process_one_dataservices_page(page_number, page)
 
         # The official MongoDB document size in 16MB. The default value here is 15MB to account for other fields in the document (and for difference between * 1024 vs * 1000).
-        max_harvest_graph_size_in_mongo = current_app.config.get('HARVEST_MAX_CATALOG_SIZE_IN_MONGO')
+        max_harvest_graph_size_in_mongo = current_app.config.get(
+            "HARVEST_MAX_CATALOG_SIZE_IN_MONGO"
+        )
         if max_harvest_graph_size_in_mongo is None:
             max_harvest_graph_size_in_mongo = 15 * 1000 * 1000
 
-        bucket = current_app.config.get('HARVEST_GRAPHS_S3_BUCKET')
+        bucket = current_app.config.get("HARVEST_GRAPHS_S3_BUCKET")
 
-        if bucket is not None and sum([len(g.encode('utf-8')) for g in serialized_graphs]) >= max_harvest_graph_size_in_mongo:
-            prefix = current_app.config.get('HARVEST_GRAPHS_S3_FILENAME_PREFIX') or ''
+        if (
+            bucket is not None
+            and sum([len(g.encode("utf-8")) for g in serialized_graphs])
+            >= max_harvest_graph_size_in_mongo
+        ):
+            prefix = current_app.config.get("HARVEST_GRAPHS_S3_FILENAME_PREFIX") or ""
 
             # TODO: we could store each page in independant files to allow downloading only the require page in
             # subsequent jobs. (less data to download in each job)
-            filename = f'{prefix}harvest_{self.job.id}_{date.today()}.json'
+            filename = f"{prefix}harvest_{self.job.id}_{date.today()}.json"
 
             store_as_json(bucket, filename, serialized_graphs)
 
-            self.job.data['filename'] = filename
+            self.job.data["filename"] = filename
         else:
-            self.job.data['graphs'] = serialized_graphs
+            self.job.data["graphs"] = serialized_graphs
 
     def get_format(self):
         fmt = guess_format(self.source.url)
@@ -99,9 +109,9 @@ class DcatBackend(BaseBackend):
         if not fmt:
             response = self.head(self.source.url)
             response.raise_for_status()
-            mime_type = response.headers.get('Content-Type', '').split(';', 1)[0]
+            mime_type = response.headers.get("Content-Type", "").split(";", 1)[0]
             if not mime_type:
-                msg = 'Unable to detect format from extension or mime type'
+                msg = "Unable to detect format from extension or mime type"
                 raise ValueError(msg)
             fmt = guess_format(mime_type)
             if not fmt:
@@ -136,7 +146,7 @@ class DcatBackend(BaseBackend):
                 return
 
             page_number += 1
-    
+
     def process_one_datasets_page(self, page_number: int, page: Graph):
         for node in page.subjects(RDF.type, DCAT.Dataset):
             remote_id = page.value(node, DCT.identifier)
@@ -152,47 +162,45 @@ class DcatBackend(BaseBackend):
 
             if self.is_done():
                 return
-            
+
     def inner_process_dataset(self, item: HarvestItem, page_number: int, page: Graph, node):
-        item.kwargs['page_number'] = page_number
+        item.kwargs["page_number"] = page_number
 
         dataset = self.get_dataset(item.remote_id)
         return dataset_from_rdf(page, dataset, node=node)
 
     def inner_process_dataservice(self, item: HarvestItem, page_number: int, page: Graph, node):
-        item.kwargs['page_number'] = page_number
+        item.kwargs["page_number"] = page_number
 
         dataservice = self.get_dataservice(item.remote_id)
-        return dataservice_from_rdf(page, dataservice, node, [item.dataset for item in self.job.items])
+        return dataservice_from_rdf(
+            page, dataservice, node, [item.dataset for item in self.job.items]
+        )
 
     def get_node_from_item(self, graph, item):
         for node in graph.subjects(RDF.type, DCAT.Dataset):
             if str(graph.value(node, DCT.identifier)) == item.remote_id:
                 return node
-        raise ValueError(f'Unable to find dataset with DCT.identifier:{item.remote_id}')
+        raise ValueError(f"Unable to find dataset with DCT.identifier:{item.remote_id}")
 
     def next_record_if_should_continue(self, start, search_results):
-        next_record = int(search_results.attrib['nextRecord'])
-        matched_count = int(search_results.attrib['numberOfRecordsMatched'])
-        returned_count = int(search_results.attrib['numberOfRecordsReturned'])
+        next_record = int(search_results.attrib["nextRecord"])
+        matched_count = int(search_results.attrib["numberOfRecordsMatched"])
+        returned_count = int(search_results.attrib["numberOfRecordsReturned"])
 
         # Break conditions copied gratefully from
         # noqa https://github.com/geonetwork/core-geonetwork/blob/main/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Harvester.java#L338-L369
         break_conditions = (
             # standard CSW: A value of 0 means all records have been returned.
             next_record == 0,
-
             # Misbehaving CSW server returning a next record > matched count
             next_record > matched_count,
-
             # No results returned already
             returned_count == 0,
-
             # Current next record is lower than previous one
             next_record < start,
-
             # Enough items have been harvested already
-            self.max_items and len(self.job.items) >= self.max_items
+            self.max_items and len(self.job.items) >= self.max_items,
         )
 
         if any(break_conditions):
@@ -200,16 +208,17 @@ class DcatBackend(BaseBackend):
         else:
             return next_record
 
+
 class CswDcatBackend(DcatBackend):
-    display_name = 'CSW-DCAT'
+    display_name = "CSW-DCAT"
 
-    DCAT_SCHEMA = 'http://www.w3.org/ns/dcat#'
+    DCAT_SCHEMA = "http://www.w3.org/ns/dcat#"
 
     def walk_graph(self, url: str, fmt: str) -> Generator[tuple[int, Graph], None, None]:
         """
         Yield all RDF pages as `Graph` from the source
         """
-        body = '''<csw:GetRecords xmlns:csw="http://www.opengis.net/cat/csw/2.0.2"
+        body = """<csw:GetRecords xmlns:csw="http://www.opengis.net/cat/csw/2.0.2"
                                   xmlns:gmd="http://www.isotc211.org/2005/gmd"
                                   service="CSW" version="2.0.2" resultType="results"
                                   startPosition="{start}" maxPosition="200"
@@ -223,23 +232,24 @@ class CswDcatBackend(DcatBackend):
                             </ogc:SortProperty>
                         </ogc:SortBy>
                     </csw:Query>
-                </csw:GetRecords>'''
-        headers = {'Content-Type': 'application/xml'}
+                </csw:GetRecords>"""
+        headers = {"Content-Type": "application/xml"}
 
         page_number = 0
         start = 1
 
-        response = self.post(url, data=body.format(start=start, schema=self.DCAT_SCHEMA),
-                             headers=headers)
+        response = self.post(
+            url, data=body.format(start=start, schema=self.DCAT_SCHEMA), headers=headers
+        )
         response.raise_for_status()
         content = response.content
         tree = ET.fromstring(content)
-        if tree.tag == '{' + OWS_NAMESPACE + '}ExceptionReport':
-            raise ValueError(f'Failed to query CSW:\n{content}')
+        if tree.tag == "{" + OWS_NAMESPACE + "}ExceptionReport":
+            raise ValueError(f"Failed to query CSW:\n{content}")
         while tree:
-            search_results = tree.find('csw:SearchResults', {'csw': CSW_NAMESPACE})
+            search_results = tree.find("csw:SearchResults", {"csw": CSW_NAMESPACE})
             if search_results is None:
-                log.error(f'No search results found for {url} on page {page_number}')
+                log.error(f"No search results found for {url} on page {page_number}")
                 break
             for child in search_results:
                 subgraph = Graph(namespace_manager=namespace_manager)
@@ -257,19 +267,21 @@ class CswDcatBackend(DcatBackend):
             page_number += 1
 
             tree = ET.fromstring(
-                self.post(url, data=body.format(start=start, schema=self.DCAT_SCHEMA),
-                          headers=headers).content)
+                self.post(
+                    url, data=body.format(start=start, schema=self.DCAT_SCHEMA), headers=headers
+                ).content
+            )
 
 
 class CswIso19139DcatBackend(DcatBackend):
-    '''
+    """
     An harvester that takes CSW ISO 19139 as input and transforms it to DCAT using SEMIC GeoDCAT-AP XSLT.
     The parsing of items is then the same as for the DcatBackend.
-    '''
+    """
 
-    display_name = 'CSW-ISO-19139'
+    display_name = "CSW-ISO-19139"
 
-    ISO_SCHEMA = 'http://www.isotc211.org/2005/gmd'
+    ISO_SCHEMA = "http://www.isotc211.org/2005/gmd"
 
     XSL_URL = "https://raw.githubusercontent.com/SEMICeu/iso-19139-to-dcat-ap/master/iso-19139-to-dcat-ap.xsl"
 
@@ -287,7 +299,7 @@ class CswIso19139DcatBackend(DcatBackend):
 
         # Start querying and parsing graph
         # Filter on dataset or serie records
-        body = '''<csw:GetRecords xmlns:csw="http://www.opengis.net/cat/csw/2.0.2"
+        body = """<csw:GetRecords xmlns:csw="http://www.opengis.net/cat/csw/2.0.2"
                                   xmlns:gmd="http://www.isotc211.org/2005/gmd"
                                   service="CSW" version="2.0.2" resultType="results"
                                   startPosition="{start}" maxPosition="10"
@@ -313,14 +325,15 @@ class CswIso19139DcatBackend(DcatBackend):
                             </ogc:Filter>
                         </csw:Constraint>
                     </csw:Query>
-                </csw:GetRecords>'''
-        headers = {'Content-Type': 'application/xml'}
+                </csw:GetRecords>"""
+        headers = {"Content-Type": "application/xml"}
 
         page_number = 0
         start = 1
 
-        response = self.post(url, data=body.format(start=start, schema=self.ISO_SCHEMA),
-                             headers=headers)
+        response = self.post(
+            url, data=body.format(start=start, schema=self.ISO_SCHEMA), headers=headers
+        )
         response.raise_for_status()
 
         tree_before_transform = ET.fromstring(response.content)
@@ -331,9 +344,9 @@ class CswIso19139DcatBackend(DcatBackend):
         while tree:
             # We query the tree before the transformation because the XSLT remove the search results
             # infos (useful for pagination)
-            search_results = tree_before_transform.find('csw:SearchResults', {'csw': CSW_NAMESPACE})
+            search_results = tree_before_transform.find("csw:SearchResults", {"csw": CSW_NAMESPACE})
             if search_results is None:
-                log.error(f'No search results found for {url} on page {page_number}')
+                log.error(f"No search results found for {url} on page {page_number}")
                 break
 
             subgraph = Graph(namespace_manager=namespace_manager)
@@ -353,8 +366,9 @@ class CswIso19139DcatBackend(DcatBackend):
             start = next_record
             page_number += 1
 
-            response = self.post(url, data=body.format(start=start, schema=self.ISO_SCHEMA),
-                                 headers=headers)
+            response = self.post(
+                url, data=body.format(start=start, schema=self.ISO_SCHEMA), headers=headers
+            )
             response.raise_for_status()
 
             tree_before_transform = ET.fromstring(response.content)

udata/harvest/commands.py

@@ -6,155 +6,151 @@ from udata.commands import cli
 
 from . import actions
 
-
 log = logging.getLogger(__name__)
 
 
-@cli.group('harvest')
+@cli.group("harvest")
 def grp():
-    '''Remote repositories harvesting operations'''
+    """Remote repositories harvesting operations"""
     pass
 
 
 @grp.command()
-@click.argument('backend')
-@click.argument('url')
-@click.argument('name')
-@click.option('-f', '--frequency', default=None)
-@click.option('-u', '--owner', default=None)
-@click.option('-o', '--org', default=None)
+@click.argument("backend")
+@click.argument("url")
+@click.argument("name")
+@click.option("-f", "--frequency", default=None)
+@click.option("-u", "--owner", default=None)
+@click.option("-o", "--org", default=None)
 def create(name, url, backend, frequency=None, owner=None, org=None):
-    '''Create a new harvest source'''
+    """Create a new harvest source"""
     log.info('Creating a new Harvest source "%s"', name)
-    source = actions.create_source(name, url, backend,
-                                   frequency=frequency,
-                                   owner=owner,
-                                   organization=org)
-    log.info('''Created a new Harvest source:
+    source = actions.create_source(
+        name, url, backend, frequency=frequency, owner=owner, organization=org
+    )
+    log.info(
+        """Created a new Harvest source:
     name: {0.name},
     slug: {0.slug},
     url: {0.url},
     backend: {0.backend},
     frequency: {0.frequency},
     owner: {0.owner},
-    organization: {0.organization}'''.format(source))
+    organization: {0.organization}""".format(source)
+    )
 
 
 @grp.command()
-@click.argument('identifier')
+@click.argument("identifier")
 def validate(identifier):
-    '''Validate a source given its identifier'''
+    """Validate a source given its identifier"""
     source = actions.validate_source(identifier)
-    log.info('Source %s (%s) has been validated', source.slug, str(source.id))
+    log.info("Source %s (%s) has been validated", source.slug, str(source.id))
 
 
 @grp.command()
 def delete(identifier):
-    '''Delete a harvest source'''
+    """Delete a harvest source"""
     log.info('Deleting source "%s"', identifier)
     actions.delete_source(identifier)
     log.info('Deleted source "%s"', identifier)
 
 
 @grp.command()
-@click.argument('identifier')
+@click.argument("identifier")
 def clean(identifier):
-    '''Delete all datasets linked to a harvest source'''
+    """Delete all datasets linked to a harvest source"""
     log.info(f'Cleaning source "{identifier}"')
     num_of_datasets = actions.clean_source(identifier)
     log.info(f'Cleaned source "{identifier}" - deleted {num_of_datasets} dataset(s)')
 
 
 @grp.command()
-@click.option('-s', '--scheduled', is_flag=True,
-              help='list only scheduled source')
+@click.option("-s", "--scheduled", is_flag=True, help="list only scheduled source")
 def sources(scheduled=False):
-    '''List all harvest sources'''
+    """List all harvest sources"""
     sources = actions.list_sources()
     if scheduled:
         sources = [s for s in sources if s.periodic_task]
     if sources:
         for source in sources:
-            msg = '{source.name} ({source.backend}): {cron}'
+            msg = "{source.name} ({source.backend}): {cron}"
             if source.periodic_task:
                 cron = source.periodic_task.schedule_display
             else:
-                cron = 'not scheduled'
+                cron = "not scheduled"
             log.info(msg.format(source=source, cron=cron))
     elif scheduled:
-        log.info('No sources scheduled yet')
+        log.info("No sources scheduled yet")
     else:
-        log.info('No sources defined yet')
+        log.info("No sources defined yet")
 
 
 @grp.command()
 def backends():
-    '''List available backends'''
-    log.info('Available backends:')
+    """List available backends"""
+    log.info("Available backends:")
     for backend in actions.list_backends():
-        log.info('%s (%s)', backend.name, backend.display_name or backend.name)
+        log.info("%s (%s)", backend.name, backend.display_name or backend.name)
 
 
 @grp.command()
-@click.argument('identifier')
+@click.argument("identifier")
 def launch(identifier):
-    '''Launch a source harvesting on the workers'''
+    """Launch a source harvesting on the workers"""
     log.info('Launching harvest job for source "%s"', identifier)
     actions.launch(identifier)
 
 
 @grp.command()
-@click.argument('identifier')
+@click.argument("identifier")
 def run(identifier):
-    '''Run a harvester synchronously'''
+    """Run a harvester synchronously"""
     log.info('Harvesting source "%s"', identifier)
     actions.run(identifier)
 
 
 @grp.command()
-@click.argument('identifier')
-@click.option('-m', '--minute', default='*',
-              help='The crontab expression for minute')
-@click.option('-h', '--hour', default='*',
-              help='The crontab expression for hour')
-@click.option('-d', '--day', 'day_of_week', default='*',
-              help='The crontab expression for day of week')
-@click.option('-D', '--day-of-month', default='*',
-              help='The crontab expression for day of month')
-@click.option('-M', '--month-of-year', default='*',
-              help='The crontab expression for month of year')
+@click.argument("identifier")
+@click.option("-m", "--minute", default="*", help="The crontab expression for minute")
+@click.option("-h", "--hour", default="*", help="The crontab expression for hour")
+@click.option(
+    "-d", "--day", "day_of_week", default="*", help="The crontab expression for day of week"
+)
+@click.option("-D", "--day-of-month", default="*", help="The crontab expression for day of month")
+@click.option("-M", "--month-of-year", default="*", help="The crontab expression for month of year")
 def schedule(identifier, **kwargs):
-    '''Schedule a harvest job to run periodically'''
+    """Schedule a harvest job to run periodically"""
     source = actions.schedule(identifier, **kwargs)
-    msg = 'Scheduled {source.name} with the following crontab: {cron}'
+    msg = "Scheduled {source.name} with the following crontab: {cron}"
     log.info(msg.format(source=source, cron=source.periodic_task.crontab))
 
 
 @grp.command()
-@click.argument('identifier')
+@click.argument("identifier")
 def unschedule(identifier):
-    '''Unschedule a periodical harvest job'''
+    """Unschedule a periodical harvest job"""
     source = actions.unschedule(identifier)
     log.info('Unscheduled harvest source "%s"', source.name)
 
 
 @grp.command()
 def purge():
-    '''Permanently remove deleted harvest sources'''
-    log.info('Purging deleted harvest sources')
+    """Permanently remove deleted harvest sources"""
+    log.info("Purging deleted harvest sources")
     count = actions.purge_sources()
-    log.info('Purged %s source(s)', count)
+    log.info("Purged %s source(s)", count)
 
 
 @grp.command()
-@click.argument('filename')
-@click.argument('domain')
+@click.argument("filename")
+@click.argument("domain")
 def attach(domain, filename):
-    '''
+    """
     Attach existing datasets to their harvest remote id
 
     Mapping between identifiers should be in FILENAME CSV file.
-    '''
-    log.info('Attaching datasets for domain %s', domain)
+    """
+    log.info("Attaching datasets for domain %s", domain)
     result = actions.attach(domain, filename)
-    log.info('Attached %s datasets to %s', result.success, domain)
+    log.info("Attached %s datasets to %s", result.success, domain)

udata/harvest/csv.py

@@ -6,12 +6,12 @@ from .models import HarvestSource
 @csv.adapter(HarvestSource)
 class HarvestSourceCsvAdapter(csv.Adapter):
     fields = (
-        'id',
-        'name',
-        'url',
-        ('organization', 'organization.name'),
-        ('organization_id', 'organization.id'),
-        'backend',
-        'created_at',
-        ('validation', lambda o: o.validation.state),
+        "id",
+        "name",
+        "url",
+        ("organization", "organization.name"),
+        ("organization_id", "organization.id"),
+        "backend",
+        "created_at",
+        ("validation", lambda o: o.validation.state),
     )

udata/harvest/exceptions.py

@@ -1,13 +1,16 @@
 class HarvestException(Exception):
-    '''Base class for all harvest exception'''
+    """Base class for all harvest exception"""
+
     pass
 
 
 class HarvestSkipException(HarvestException):
-    '''Raised when an item is skipped'''
+    """Raised when an item is skipped"""
+
     pass
 
 
 class HarvestValidationError(HarvestException):
-    '''Raised when an harvested item is invalid'''
+    """Raised when an harvested item is invalid"""
+
     pass