udata 7.0.4.dev27782__py2.py3-none-any.whl → 7.0.5__py2.py3-none-any.whl

udata/harvest/api.py

@@ -25,7 +25,7 @@ def backends_ids():
 
 error_fields = api.model('HarvestError', {
     'created_at': fields.ISODateTime(description='The error creation date',
-                                     required=True),
+                                     required=True, readonly=True),
     'message': fields.String(description='The error short message',
                              required=True),
     'details': fields.String(description='Optional details (ie. stacktrace)'),
@@ -99,7 +99,7 @@ source_fields = api.model('HarvestSource', {
                              required=True),
     'config': fields.Raw(description='The configuration as key-value pairs'),
     'created_at': fields.ISODateTime(description='The source creation date',
-                                     required=True),
+                                     required=True, readonly=True),
     'active': fields.Boolean(description='Is this source active',
                              required=True, default=False),
     'autoarchive': fields.Boolean(
@@ -114,7 +114,7 @@ source_fields = api.model('HarvestSource', {
                            description='The owner information'),
     'organization': fields.Nested(org_ref_fields, allow_null=True,
                                   description='The producer organization'),
-    'deleted': fields.ISODateTime(description='The source deletion date'),
+    'deleted': fields.ISODateTime(description='The source deletion date', readonly=True),
     'schedule': fields.String(description='The source schedule (interval or cron expression)',
                               readonly=True),
 })

udata/harvest/backends/dcat.py

@@ -3,12 +3,17 @@ import logging
 from rdflib import Graph, URIRef
 from rdflib.namespace import RDF
 import xml.etree.ElementTree as ET
+import boto3
+from flask import current_app
+from datetime import date
+import json
 from typing import List
 
 from udata.rdf import (
     DCAT, DCT, HYDRA, SPDX, namespace_manager, guess_format, url_from_rdf
 )
 from udata.core.dataset.rdf import dataset_from_rdf
+from udata.storage.s3 import store_as_json, get_from_json
 
 from .base import BaseBackend
 
@@ -58,10 +63,30 @@ class DcatBackend(BaseBackend):
         '''List all datasets for a given ...'''
         fmt = self.get_format()
         graphs = self.parse_graph(self.source.url, fmt)
-        self.job.data = {
-            'graphs': [graph.serialize(format=fmt, indent=None) for graph in graphs],
-            'format': fmt,
-        }
+
+        self.job.data = { 'format': fmt }
+
+        serialized_graphs = [graph.serialize(format=fmt, indent=None) for graph in graphs]
+
+        # The official MongoDB document size in 16MB. The default value here is 15MB to account for other fields in the document (and for difference between * 1024 vs * 1000).
+        max_harvest_graph_size_in_mongo = current_app.config.get('HARVEST_MAX_CATALOG_SIZE_IN_MONGO')
+        if max_harvest_graph_size_in_mongo is None:
+            max_harvest_graph_size_in_mongo = 15 * 1000 * 1000
+
+        bucket = current_app.config.get('HARVEST_GRAPHS_S3_BUCKET')
+
+        if bucket is not None and sum([len(g.encode('utf-8')) for g in serialized_graphs]) >= max_harvest_graph_size_in_mongo:
+            prefix = current_app.config.get('HARVEST_GRAPHS_S3_FILENAME_PREFIX') or ''
+
+            # TODO: we could store each page in independant files to allow downloading only the require page in
+            # subsequent jobs. (less data to download in each job)
+            filename = f'{prefix}harvest_{self.job.id}_{date.today()}.json'
+
+            store_as_json(bucket, filename, serialized_graphs)
+
+            self.job.data['filename'] = filename
+        else:
+            self.job.data['graphs'] = serialized_graphs
 
     def get_format(self):
         fmt = guess_format(self.source.url)
@@ -127,7 +152,19 @@ class DcatBackend(BaseBackend):
         if item.remote_id == 'None':
             raise ValueError('The DCT.identifier is missing on this DCAT.Dataset record')
         graph = Graph(namespace_manager=namespace_manager)
-        data = self.job.data['graphs'][item.kwargs['page']]
+
+        if self.job.data.get('graphs') is not None:
+            graphs = self.job.data['graphs']
+        else:
+            bucket = current_app.config.get('HARVEST_GRAPHS_S3_BUCKET')
+            if bucket is None:
+                raise ValueError(f"No bucket configured but the harvest job item {item.id} on job {self.job.id} doesn't have a graph in MongoDB.")
+
+            graphs = get_from_json(bucket, self.job.data['filename'])
+            if graphs is None:
+                raise ValueError(f"The file '{self.job.data['filename']}' is missing in S3 bucket '{bucket}'")
+
+        data = graphs[item.kwargs['page']]
         format = self.job.data['format']
 
         graph.parse(data=bytes(data, encoding='utf8'), format=format)

udata/harvest/tests/dcat/bnodes.xml

@@ -5,6 +5,8 @@
   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
   xmlns:dcat="http://www.w3.org/ns/dcat#"
   xmlns:dct="http://purl.org/dc/terms/"
+  xmlns:ogc="http://www.opengis.net/ogc"
+  xmlns:geo="http://www.w3.org/2003/01/geo/wgs84_pos#"
   xmlns:dcterms="http://purl.org/dc/terms/"
   xmlns:vcard="http://www.w3.org/2006/vcard/ns#"
   xmlns:schema="http://schema.org/"
@@ -54,7 +56,13 @@
         <owl:versionInfo>1.0</owl:versionInfo>
         <dcat:distribution rdf:resource="http://data.test.org/datasets/1/resources/2"/>
         <dcat:keyword>Tag 4</dcat:keyword>
-        <dcterms:spatial rdf:resource="http://wuEurope.com/"/>
+        <dct:spatial>
+            <ogc:Polygon>
+                <geo:asWKT rdf:datatype="http://www.opengis.net/rdf#wktLiteral">
+                    wrong wkt
+                </geo:asWKT>
+            </ogc:Polygon>
+        </dct:spatial>
         <dcterms:modified rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2016-12-14T19:01:24.184120</dcterms:modified>
         <dcat:keyword>Tag 2</dcat:keyword>
         <dcat:keyword>Tag 1</dcat:keyword>
@@ -79,7 +87,13 @@
         <dcat:keyword>Tag 3</dcat:keyword>
         <dcat:distribution rdf:resource="http://data.test.org/datasets/2/resources/2"/>
         <dcterms:title>Dataset 2</dcterms:title>
-        <dcterms:spatial rdf:resource="http://wuEurope.com/"/>
+        <dct:spatial>
+            <ogc:Polygon>
+                <geo:asWKT rdf:datatype="http://www.opengis.net/rdf#wktLiteral">
+                    Polygon((4.44641288 45.54214467, 4.44641288 46.01316963, 4.75655252 46.01316963, 4.75655252 45.54214467, 4.44641288 45.54214467))
+                </geo:asWKT>
+            </ogc:Polygon>
+        </dct:spatial>
         <dcterms:identifier>2</dcterms:identifier>
         <dct:conformsTo rdf:resource="https://www.ecologie.gouv.fr/sites/default/files/R%C3%A9glementation%20IRVE.pdf" />
       </dcat:Dataset>

udata/harvest/tests/test_dcat_backend.py

@@ -1,14 +1,18 @@
 import logging
 import os
+from flask import current_app
 
 import pytest
 
 from datetime import date
+import boto3
 import xml.etree.ElementTree as ET
+from udata.harvest.models import HarvestJob
 
 from udata.models import Dataset
 from udata.core.organization.factories import OrganizationFactory
 from udata.core.dataset.factories import LicenseFactory, ResourceSchemaMockData
+from udata.storage.s3 import get_from_json
 
 from .factories import HarvestSourceFactory
 from ..backends.dcat import URIS_TO_REPLACE
@@ -136,7 +140,7 @@ class DcatBackendTest:
         assert datasets['1'].resources[0].format == 'json'
         assert datasets['1'].resources[0].mime == 'application/json'
 
-    @pytest.mark.options(SCHEMA_CATALOG_URL='https://example.com/schemas')
+    @pytest.mark.options(SCHEMA_CATALOG_URL='https://example.com/schemas', HARVEST_MAX_CATALOG_SIZE_IN_MONGO=None, HARVEST_GRAPHS_S3_BUCKET="test_bucket", S3_URL="https://example.org", S3_ACCESS_KEY_ID="myUser", S3_SECRET_ACCESS_KEY="password")
     def test_flat_with_blank_nodes_xml(self, rmock):
         rmock.get('https://example.com/schemas', json=ResourceSchemaMockData.get_mock_data())
 
@@ -156,6 +160,88 @@ class DcatBackendTest:
         assert len(datasets['1'].resources) == 2
         assert len(datasets['2'].resources) == 2
 
+    @pytest.mark.skip(reason="Mocking S3 requires `moto` which is not available for our current Python 3.7. We can manually test it.")
+    @pytest.mark.options(SCHEMA_CATALOG_URL='https://example.com/schemas', HARVEST_JOBS_RETENTION_DAYS=0)
+    # @mock_s3
+    # @pytest.mark.options(HARVEST_MAX_CATALOG_SIZE_IN_MONGO=15, HARVEST_GRAPHS_S3_BUCKET="test_bucket", S3_URL="https://example.org", S3_ACCESS_KEY_ID="myUser", S3_SECRET_ACCESS_KEY="password")
+    def test_harvest_big_catalog(self, rmock):
+        rmock.get('https://example.com/schemas', json=ResourceSchemaMockData.get_mock_data())
+
+        # We need to create the bucket since this is all in Moto's 'virtual' AWS account
+        # conn = boto3.resource(
+        #     "s3",
+        #     endpoint_url="https://example.org",
+        #     aws_access_key_id="myUser",
+        #     aws_secret_access_key="password",
+        # )
+        # conn.create_bucket(Bucket="test_bucket")
+
+        filename = 'bnodes.xml'
+        url = mock_dcat(rmock, filename)
+        org = OrganizationFactory()
+        source = HarvestSourceFactory(backend='dcat',
+                                      url=url,
+                                      organization=org)
+
+        actions.run(source.slug)
+
+        datasets = {d.harvest.dct_identifier: d for d in Dataset.objects}
+
+        assert datasets['1'].schema == None
+        resources_by_title = { resource['title']: resource for resource in datasets['1'].resources }
+
+        # Schema with wrong version are considered as external. Maybe we could change this in the future
+        assert resources_by_title['Resource 1-2'].schema.url == 'https://schema.data.gouv.fr/schemas/etalab/schema-irve-statique/1337.42.0/schema-statique.json'
+        assert resources_by_title['Resource 1-2'].schema.name == None
+        assert resources_by_title['Resource 1-2'].schema.version == None
+
+        assert datasets['2'].schema.name == None
+        assert datasets['2'].schema.url == 'https://www.ecologie.gouv.fr/sites/default/files/R%C3%A9glementation%20IRVE.pdf'
+        resources_by_title = { resource['title']: resource for resource in datasets['2'].resources }
+
+        # Unknown schema are kept as they were provided
+        assert resources_by_title['Resource 2-1'].schema.name == 'Example Schema'
+        assert resources_by_title['Resource 2-1'].schema.url == 'https://example.org/schema.json'
+        assert resources_by_title['Resource 2-1'].schema.version == None
+
+        assert resources_by_title['Resource 2-2'].schema == None
+
+        assert datasets['3'].schema == None
+        resources_by_title = { resource['title']: resource for resource in datasets['3'].resources }
+
+        # If there is just the URL, and it matches a known schema inside the catalog, only set the name and the version
+        # (discard the URL)
+        assert resources_by_title['Resource 3-1'].schema.name == 'etalab/schema-irve-statique'
+        assert resources_by_title['Resource 3-1'].schema.url == None
+        assert resources_by_title['Resource 3-1'].schema.version == '2.2.0'
+
+        job = HarvestJob.objects.order_by('-id').first()
+
+        assert job.source.slug == source.slug
+        assert get_from_json(current_app.config.get('HARVEST_GRAPHS_S3_BUCKET'), job.data['filename']) is not None
+
+        # Retention is 0 days in config
+        actions.purge_jobs()
+        assert get_from_json(current_app.config.get('HARVEST_GRAPHS_S3_BUCKET'), job.data['filename']) is None
+
+
+    @pytest.mark.options(SCHEMA_CATALOG_URL='https://example.com/schemas')
+    def test_harvest_spatial(self, rmock):
+        rmock.get('https://example.com/schemas', json=ResourceSchemaMockData.get_mock_data())
+
+        filename = 'bnodes.xml'
+        url = mock_dcat(rmock, filename)
+        org = OrganizationFactory()
+        source = HarvestSourceFactory(backend='dcat', url=url, organization=org)
+
+        actions.run(source.slug)
+
+        datasets = {d.harvest.dct_identifier: d for d in Dataset.objects}
+
+        assert datasets['1'].spatial == None
+        assert datasets['2'].spatial.geom == {'type': 'MultiPolygon', 'coordinates': [[[[4.44641288, 45.54214467], [4.44641288, 46.01316963], [4.75655252, 46.01316963], [4.75655252, 45.54214467], [4.44641288, 45.54214467]]]]}
+        assert datasets['3'].spatial == None
+
     @pytest.mark.options(SCHEMA_CATALOG_URL='https://example.com/schemas')
     def test_harvest_schemas(self, rmock):
         rmock.get('https://example.com/schemas', json=ResourceSchemaMockData.get_mock_data())

udata/settings.py

@@ -257,6 +257,15 @@ class Defaults(object):
 
     HARVEST_VALIDATION_CONTACT_FORM = None
 
+    HARVEST_MAX_CATALOG_SIZE_IN_MONGO = None # Defaults to the size of a MongoDB document 
+    HARVEST_GRAPHS_S3_BUCKET = None # If the catalog is bigger than `HARVEST_MAX_CATALOG_SIZE_IN_MONGO` store the graph inside S3 instead of MongoDB
+    HARVEST_GRAPHS_S3_FILENAME_PREFIX = '' # Useful to store the graphs inside a subfolder of the bucket. For example by setting `HARVEST_GRAPHS_S3_FILENAME_PREFIX = 'graphs/'`
+
+    # S3 connection details
+    S3_URL = None
+    S3_ACCESS_KEY_ID = None
+    S3_SECRET_ACCESS_KEY = None 
+
     ACTIVATE_TERRITORIES = False
     # The order is important to compute parents/children, smaller first.
     HANDLED_LEVELS = tuple()