udata 7.0.8.dev28841__py2.py3-none-any.whl → 9.0.1.dev29390__py2.py3-none-any.whl

udata/harvest/tests/factories.py

@@ -6,9 +6,10 @@ from flask.signals import Namespace
 
 from udata.factories import ModelFactory
 from udata.core.dataset.factories import DatasetFactory
+from udata.core.dataset.models import Dataset
 
 from .. import backends
-from ..models import HarvestSource, HarvestJob
+from ..models import HarvestItem, HarvestSource, HarvestJob
 
 
 def dtfactory(start, end):
@@ -55,14 +56,20 @@ class FactoryBackend(backends.BaseBackend):
         backends.HarvestFeature('toggled', 'Toggled', 'A togglable', True),
     )
 
-    def initialize(self):
+    def inner_harvest(self):
         mock_initialize.send(self)
         for i in range(self.config.get('count', DEFAULT_COUNT)):
-            self.add_item(i)
+            self.process_dataset(str(i))
+            if self.is_done():
+                return
 
-    def process(self, item):
-        mock_process.send(self, item=item)
-        return DatasetFactory.build(title='dataset-{0}'.format(item.remote_id))
+    def inner_process_dataset(self, item: HarvestItem):
+        mock_process.send(self, item=item.remote_id)
+
+        dataset = self.get_dataset(item.remote_id)
+        dataset.title = f'dataset-{item.remote_id}'
+
+        return dataset
 
 
 class MockBackendsMixin(object):

udata/harvest/tests/test_actions.py

@@ -580,7 +580,7 @@ class ExecutionTestMixin(MockBackendsMixin):
 
     def test_error_on_item(self):
         def process(self, item):
-            if item.remote_id == '1':
+            if item == '1':
                 raise ValueError('test')
 
         source = HarvestSourceFactory(backend='factory')
@@ -723,7 +723,7 @@ class HarvestPreviewTest(MockBackendsMixin):
 
     def test_preview_with_error_on_item(self):
         def process(self, item):
-            if item.remote_id == '1':
+            if item == '1':
                 raise ValueError('test')
 
         source = HarvestSourceFactory(backend='factory')

udata/harvest/tests/test_base_backend.py

@@ -5,6 +5,7 @@ from urllib.parse import urlparse
 from dateutil.parser import parse
 from voluptuous import Schema
 
+from udata.harvest.models import HarvestItem
 from udata.utils import faker
 from udata.core.dataset import tasks
 from udata.core.dataset.factories import DatasetFactory
@@ -31,12 +32,16 @@ class FakeBackend(BaseBackend):
         HarvestFeature('enabled', 'A test feature enabled by default', default=True),
     )
 
-    def initialize(self):
+    def inner_harvest(self):
         for i in range(self.source.config.get('nb_datasets', 3)):
-            self.add_item('fake-{0}'.format(i))
+            remote_id = f'fake-{i}'
+            self.process_dataset(remote_id)
+            if self.is_done():
+                return
 
-    def process(self, item):
+    def inner_process_dataset(self, item: HarvestItem):
         dataset = self.get_dataset(item.remote_id)
+
         for key, value in DatasetFactory.as_dict(visible=True).items():
             setattr(dataset, key, value)
         if self.source.config.get('last_modified'):
@@ -219,8 +224,7 @@ class BaseBackendTest:
         assert 'archived_at' not in dataset_no_arch.harvest
 
         # test unarchive: archive manually then relaunch harvest
-        q = {'harvest__remote_id': 'fake-1'}
-        dataset = Dataset.objects.get(**q)
+        dataset = Dataset.objects.get(**{'harvest__remote_id': 'fake-1'})
         dataset.archived = datetime.utcnow()
         dataset.harvest.archived = 'not-on-remote'
         dataset.harvest.archived_at = datetime.utcnow()

udata/harvest/tests/test_dcat_backend.py

@@ -240,6 +240,19 @@ class DcatBackendTest:
         actions.purge_jobs()
         assert get_from_json(current_app.config.get('HARVEST_GRAPHS_S3_BUCKET'), job.data['filename']) is None
 
+    @pytest.mark.options(SCHEMA_CATALOG_URL='https://example.com/schemas', HARVEST_MAX_ITEMS=2)
+    def test_harvest_max_items(self, rmock):
+        rmock.get('https://example.com/schemas', json=ResourceSchemaMockData.get_mock_data())
+
+        filename = 'bnodes.xml'
+        url = mock_dcat(rmock, filename)
+        org = OrganizationFactory()
+        source = HarvestSourceFactory(backend='dcat', url=url, organization=org)
+
+        actions.run(source.slug)
+
+        assert Dataset.objects.count() == 2
+        assert HarvestJob.objects.first().status == 'done'
 
     @pytest.mark.options(SCHEMA_CATALOG_URL='https://example.com/schemas')
     def test_harvest_spatial(self, rmock):
@@ -255,7 +268,7 @@ class DcatBackendTest:
         datasets = {d.harvest.dct_identifier: d for d in Dataset.objects}
 
         assert datasets['1'].spatial == None
-        assert datasets['2'].spatial.geom == {'type': 'MultiPolygon', 'coordinates': [[[[4.44641288, 45.54214467], [4.44641288, 46.01316963], [4.75655252, 46.01316963], [4.75655252, 45.54214467], [4.44641288, 45.54214467]]]]}
+        assert datasets['2'].spatial.geom == {'type': 'MultiPolygon', 'coordinates': [[[[-6,51],[10,51],[10,40],[-6,40],[-6,51]]], [[[4, 45], [4, 46], [4, 46], [4, 45], [4, 45]]], [[[159, -25.], [159, -11], [212, -11], [212, -25.], [159, -25.]]]]}
         assert datasets['3'].spatial == None
 
     @pytest.mark.options(SCHEMA_CATALOG_URL='https://example.com/schemas')
@@ -440,6 +453,9 @@ class DcatBackendTest:
         assert dataset.extras["harvest"]["dct:accessRights"] == "http://inspire.ec.europa.eu/metadata-codelist/LimitationsOnPublicAccess/INSPIRE_Directive_Article13_1e"
         assert dataset.extras["harvest"]["dct:provenance"] == ["Description de la provenance des données"]
 
+        assert 'observation-de-la-terre-et-environnement' in dataset.tags
+        assert 'hvd' in dataset.tags
+
         dataset = Dataset.objects.get(harvest__dct_identifier='1')
         # test html abstract description support
         assert dataset.description == '# h1 title\n\n## h2 title\n\n **and bold text**'

udata/rdf.py

@@ -21,6 +21,7 @@ log = logging.getLogger(__name__)
 # Extra Namespaces
 ADMS = Namespace('http://www.w3.org/ns/adms#')
 DCAT = Namespace('http://www.w3.org/ns/dcat#')
+DCATAP = Namespace('http://data.europa.eu/r5r/')
 HYDRA = Namespace('http://www.w3.org/ns/hydra/core#')
 SCHEMA = Namespace('http://schema.org/')
 SCV = Namespace('http://purl.org/NET/scovo#')
@@ -35,6 +36,7 @@ VCARD = Namespace('http://www.w3.org/2006/vcard/ns#')
 
 namespace_manager = NamespaceManager(Graph())
 namespace_manager.bind('dcat', DCAT)
+namespace_manager.bind('dcatap', DCATAP)
 namespace_manager.bind('dct', DCT)
 namespace_manager.bind('foaf', FOAF)
 namespace_manager.bind('foaf', FOAF)
@@ -98,6 +100,8 @@ RDF_EXTENSIONS = {
 # Includes control characters, unicode surrogate characters and unicode end-of-plane non-characters
 ILLEGAL_XML_CHARS = '[\x00-\x08\x0b\x0c\x0e-\x1F\uD800-\uDFFF\uFFFE\uFFFF]'
 
+HVD_LEGISLATION = 'http://data.europa.eu/eli/reg_impl/2023/138/oj'
+
 
 def guess_format(string):
     '''Guess format given an extension or a mime-type'''

udata/routing.py

@@ -10,6 +10,7 @@ from werkzeug.urls import url_quote
 from udata import models
 from udata.mongo import db
 from udata.core.spatial.models import GeoZone
+from udata.core.dataservices.models import Dataservice
 from udata.i18n import ISO_639_1_CODES
 
 
@@ -121,6 +122,10 @@ class DatasetConverter(ModelConverter):
     model = models.Dataset
 
 
+class DataserviceConverter(ModelConverter):
+    model = Dataservice
+
+
 class CommunityResourceConverter(ModelConverter):
     model = models.CommunityResource
 
@@ -222,6 +227,7 @@ def init_app(app):
     app.url_map.converters['pathlist'] = PathListConverter
     app.url_map.converters['uuid'] = UUIDConverter
     app.url_map.converters['dataset'] = DatasetConverter
+    app.url_map.converters['dataservice'] = DataserviceConverter
     app.url_map.converters['crid'] = CommunityResourceConverter
     app.url_map.converters['org'] = OrganizationConverter
     app.url_map.converters['reuse'] = ReuseConverter

udata/settings.py

@@ -269,7 +269,10 @@ class Defaults(object):
     # S3 connection details
     S3_URL = None
     S3_ACCESS_KEY_ID = None
-    S3_SECRET_ACCESS_KEY = None 
+    S3_SECRET_ACCESS_KEY = None
+
+    # Specific support for hvd (map HVD categories URIs to keywords)
+    HVD_SUPPORT = True
 
     ACTIVATE_TERRITORIES = False
     # The order is important to compute parents/children, smaller first.