udata 7.0.6__py2.py3-none-any.whl → 7.0.6.dev28209__py2.py3-none-any.whl

udata/__init__.py

@@ -4,5 +4,5 @@
 udata
 '''
 
-__version__ = '7.0.6'
+__version__ = '7.0.6.dev'
 __description__ = 'Open data portal'

udata/commands/__init__.py

@@ -145,6 +145,9 @@ def init_logging(app):
     handler.setFormatter(CliFormatter())
     handler.setLevel(log_level)
 
+    logger = logging.getLogger()
+    logger.addHandler(handler)
+
     logger = logging.getLogger('__main__')
     logger.setLevel(log_level)
     logger.handlers = []

udata/commands/dcat.py

@@ -8,7 +8,7 @@ from rdflib import Graph
 from udata.commands import cli, green, yellow, cyan, echo, magenta
 from udata.core.dataset.factories import DatasetFactory
 from udata.core.dataset.rdf import dataset_from_rdf
-from udata.harvest.backends.dcat import DcatBackend, CswDcatBackend, CswIso19139DcatBackend
+from udata.harvest.backends.dcat import DcatBackend, CswDcatBackend
 from udata.rdf import namespace_manager
 
 log = logging.getLogger(__name__)
@@ -23,10 +23,9 @@ def grp():
 @grp.command()
 @click.argument('url')
 @click.option('-q', '--quiet', is_flag=True, help='Ignore warnings')
-@click.option('-r', '--rid', help='Inspect specific remote id (contains)')
-@click.option('-c', '--csw', is_flag=True, help='The target is a CSW endpoint with DCAT output')
-@click.option('-i', '--iso', is_flag=True, help='The target is a CSW endpoint with ISO output')
-def parse_url(url, csw, iso, quiet=False, rid=''):
+@click.option('-i', '--rid', help='Inspect specific remote id (contains)')
+@click.option('-c', '--csw', is_flag=True, help='The target is a CSW endpoint')
+def parse_url(url, csw, quiet=False, rid=''):
     '''Parse the datasets in a DCAT format located at URL (debug)'''
     if quiet:
         verbose_loggers = ['rdflib', 'udata.core.dataset']
@@ -50,8 +49,6 @@ def parse_url(url, csw, iso, quiet=False, rid=''):
     source.url = url
     if csw:
         backend = CswDcatBackend(source, dryrun=True)
-    elif iso:
-        backend = CswIso19139DcatBackend(source, dryrun=True)
     else:
         backend = DcatBackend(source, dryrun=True)
     backend.job = MockJob()

udata/core/activity/api.py

@@ -37,7 +37,7 @@ activity_fields = api.model('Activity', {
         description='The key of the activity', required=True),
     'icon': fields.String(
         description='The icon of the activity', required=True),
-    'extras': fields.Raw(description='Extras attributes as key-value pairs'),
+    'kwargs': fields.Raw(description='Some action specific context'),
 })
 
 activity_page_fields = api.model('ActivityPage', fields.pager(activity_fields))

udata/core/activity/models.py

@@ -37,7 +37,7 @@ class Activity(db.Document, metaclass=EmitNewActivityMetaClass):
     related_to = db.ReferenceField(db.DomainModel, required=True)
     created_at = db.DateTimeField(default=datetime.utcnow, required=True)
 
-    extras = db.ExtrasField()
+    kwargs = db.DictField()
 
     on_new = Signal()
 
@@ -65,9 +65,8 @@ class Activity(db.Document, metaclass=EmitNewActivityMetaClass):
         return cls.on_new.connect(func, sender=cls)
 
     @classmethod
-    def emit(cls, related_to, organization=None, extras=None):
+    def emit(cls, related_to, organization=None, **kwargs):
         new_activity.send(cls,
                           related_to=related_to,
                           actor=current_user._get_current_object(),
-                          organization=organization,
-                          extras=extras)
+                          organization=organization)

udata/core/activity/tasks.py

@@ -9,23 +9,22 @@ log = logging.getLogger(__name__)
 
 
 @new_activity.connect
-def delay_activity(cls, related_to, actor, organization=None, extras=None):
+def delay_activity(cls, related_to, actor, organization=None):
     emit_activity.delay(
         cls.__name__,
         str(actor.id),
         related_to_cls=related_to.__class__.__name__,
         related_to_id=str(related_to.id),
         organization_id=str(organization.id) if organization else None,
-        extras=extras
     )
 
 
 @task
 def emit_activity(classname, actor_id, related_to_cls, related_to_id,
-                  organization_id=None, extras=None):
-    log.debug('Emit new activity: %s %s %s %s %s %s',
+                  organization_id=None):
+    log.debug('Emit new activity: %s %s %s %s %s',
               classname, actor_id, related_to_cls,
-              related_to_id, organization_id, extras)
+              related_to_id, organization_id)
     cls = db.resolve_model(classname)
     actor = User.objects.get(pk=actor_id)
     related_to = db.resolve_model(related_to_cls).objects.get(pk=related_to_id)
@@ -34,4 +33,4 @@ def emit_activity(classname, actor_id, related_to_cls, related_to_id,
     else:
         organization = None
     cls.objects.create(actor=actor, related_to=related_to,
-                       organization=organization, extras=extras)
+                       organization=organization)

udata/core/dataset/factories.py

@@ -34,8 +34,10 @@ class DatasetFactory(ModelFactory):
         nb_resources = 0
 
 
-class HiddenDatasetFactory(DatasetFactory):
-    private = True
+class VisibleDatasetFactory(DatasetFactory):
+    @factory.lazy_attribute
+    def resources(self):
+        return [ResourceFactory()]
 
 
 class ChecksumFactory(ModelFactory):

udata/core/dataset/models.py

@@ -342,10 +342,12 @@ class License(db.Document):
 
 class DatasetQuerySet(db.OwnedQuerySet):
     def visible(self):
-        return self(private__ne=True, deleted=None, archived=None)
+        return self(private__ne=True, resources__0__exists=True,
+                    deleted=None, archived=None)
 
     def hidden(self):
         return self(db.Q(private=True) |
+                    db.Q(resources__0__exists=False) |
                     db.Q(deleted__ne=None) |
                     db.Q(archived__ne=None))
 
@@ -675,7 +677,8 @@ class Dataset(WithMetrics, BadgeMixin, db.Owned, db.Document):
 
     @property
     def is_hidden(self):
-        return self.private or self.deleted or self.archived
+        return (len(self.resources) == 0 or self.private or self.deleted
+                or self.archived)
 
     @property
     def full_title(self):

udata/core/dataset/rdf.py

@@ -7,7 +7,6 @@ import logging
 
 from datetime import date
 from html.parser import HTMLParser
-from typing import Optional
 from dateutil.parser import parse as parse_dt
 from flask import current_app
 from geomet import wkt
@@ -340,51 +339,36 @@ def contact_point_from_rdf(rdf, dataset):
 
 
 def spatial_from_rdf(graph):
-    geojsons = []
     for term in graph.objects(DCT.spatial):
-        try:
-            # This may not be official in the norm but some ArcGis return 
-            # bbox as literal directly in DCT.spatial.
-            if isinstance(term, Literal):
-                geojson = bbox_to_geojson_multipolygon(term.toPython())
-                if geojson is not None:
-                    geojsons.append(geojson)
-                
-                continue
-
-            for object in term.objects():
-                if isinstance(object, Literal):
-                    if object.datatype.__str__() == 'https://www.iana.org/assignments/media-types/application/vnd.geo+json':
-                        try:
-                            geojson = json.loads(object.toPython())
-                        except ValueError as e:
-                            log.warning(f"Invalid JSON in spatial GeoJSON {object.toPython()} {e}")
-                            continue
-                    elif object.datatype.__str__() == 'http://www.opengis.net/rdf#wktLiteral':
-                        try:
-                            # .upper() si here because geomet doesn't support Polygon but only POLYGON
-                            geojson = wkt.loads(object.toPython().strip().upper())
-                        except ValueError as e:
-                            log.warning(f"Invalid JSON in spatial WKT {object.toPython()} {e}")
-                            continue
-                    else:
+        for object in term.objects():
+            if isinstance(object, Literal):
+                if object.datatype.__str__() == 'https://www.iana.org/assignments/media-types/application/vnd.geo+json':
+                    try:
+                        geojson = json.loads(object.toPython())
+                    except ValueError as e:
+                        log.warning(f"Invalid JSON in spatial GeoJSON {object.toPython()} {e}")
+                        continue
+                elif object.datatype.__str__() == 'http://www.opengis.net/rdf#wktLiteral':
+                    try:
+                        # .upper() si here because geomet doesn't support Polygon but only POLYGON
+                        geojson = wkt.loads(object.toPython().strip().upper())
+                    except ValueError as e:
+                        log.warning(f"Invalid JSON in spatial WKT {object.toPython()} {e}")
                         continue
+                else:
+                    continue
 
-                    if geojson['type'] == 'Polygon':
-                        geojson['type'] = 'MultiPolygon'
-                        geojson['coordinates'] = [geojson['coordinates']]
+                if geojson['type'] == 'Polygon':
+                    geojson['type'] = 'MultiPolygon'
+                    geojson['coordinates'] = [geojson['coordinates']]
 
-                    geojsons.append(geojson)
-        except Exception as e:
-            log.exception(f"Exception during `spatial_from_rdf` for term {term}: {e}", stack_info=True)
+                spatial_coverage = SpatialCoverage(geom=geojson)
 
-    for geojson in geojsons:
-        spatial_coverage = SpatialCoverage(geom=geojson)
-        try:
-            spatial_coverage.clean()
-            return spatial_coverage
-        except ValidationError:
-            continue
+                try:
+                    spatial_coverage.clean()
+                    return spatial_coverage
+                except ValidationError:
+                    continue
 
     return None
 
@@ -625,27 +609,3 @@ def dataset_from_rdf(graph: Graph, dataset=None, node=None):
     dataset.harvest.modified_at = modified_at
 
     return dataset
-
-def bbox_to_geojson_multipolygon(bbox_as_str: str) -> Optional[dict] : 
-    bbox = bbox_as_str.strip().split(',')
-    if len(bbox) != 4:
-        return None
-    
-    west = float(bbox[0])
-    south = float(bbox[1])
-    east = float(bbox[2])
-    north = float(bbox[3])
-
-    low_left = [west, south]
-    top_left = [west, north]
-    top_right = [east, north]
-    low_right = [east, south]
-
-    return {
-        'type': 'MultiPolygon',
-        'coordinates': [
-            [
-                [low_left, low_right, top_right, top_left, low_left],
-            ], 
-        ],
-    }

udata/core/dataset/search.py

@@ -47,6 +47,7 @@ class DatasetSearch(ModelSearchAdapter):
     @classmethod
     def is_indexable(cls, dataset):
         return (dataset.deleted is None and dataset.archived is None and
+                len(dataset.resources) > 0 and
                 not dataset.private)
 
     @classmethod

udata/core/spatial/tests/test_api.py

@@ -1,12 +1,15 @@
 from flask import url_for
 
+from udata.utils import get_by
+
 from udata.utils import faker
 from udata.tests.api import APITestCase
 from udata.tests.features.territories import (
     create_geozones_fixtures, TerritoriesSettings
 )
+from udata.tests.helpers import assert_json_equal
 from udata.core.organization.factories import OrganizationFactory
-from udata.core.dataset.factories import DatasetFactory
+from udata.core.dataset.factories import VisibleDatasetFactory
 from udata.core.spatial.factories import (
     SpatialCoverageFactory, GeoZoneFactory, GeoLevelFactory
 )
@@ -170,7 +173,7 @@ class SpatialApiTest(APITestCase):
         paca, bdr, arles = create_geozones_fixtures()
         organization = OrganizationFactory()
         for _ in range(3):
-            DatasetFactory(
+            VisibleDatasetFactory(
                 organization=organization,
                 spatial=SpatialCoverageFactory(zones=[paca.id]))
 
@@ -182,7 +185,7 @@ class SpatialApiTest(APITestCase):
         paca, bdr, arles = create_geozones_fixtures()
         organization = OrganizationFactory()
         for _ in range(3):
-            DatasetFactory(
+            VisibleDatasetFactory(
                 organization=organization,
                 spatial=SpatialCoverageFactory(zones=[paca.id]))
 
@@ -195,7 +198,7 @@ class SpatialApiTest(APITestCase):
         paca, bdr, arles = create_geozones_fixtures()
         organization = OrganizationFactory()
         for _ in range(3):
-            DatasetFactory(
+            VisibleDatasetFactory(
                 organization=organization,
                 spatial=SpatialCoverageFactory(zones=[paca.id]))
 
@@ -209,7 +212,7 @@ class SpatialApiTest(APITestCase):
         paca, bdr, arles = create_geozones_fixtures()
         organization = OrganizationFactory()
         for _ in range(3):
-            DatasetFactory(
+            VisibleDatasetFactory(
                 organization=organization,
                 spatial=SpatialCoverageFactory(zones=[paca.id]))
 
@@ -238,7 +241,7 @@ class SpatialTerritoriesApiTest(APITestCase):
         paca, bdr, arles = create_geozones_fixtures()
         organization = OrganizationFactory()
         for _ in range(3):
-            DatasetFactory(
+            VisibleDatasetFactory(
                 organization=organization,
                 spatial=SpatialCoverageFactory(zones=[paca.id]))
 
@@ -252,7 +255,7 @@ class SpatialTerritoriesApiTest(APITestCase):
         paca, bdr, arles = create_geozones_fixtures()
         organization = OrganizationFactory()
         for _ in range(3):
-            DatasetFactory(
+            VisibleDatasetFactory(
                 organization=organization,
                 spatial=SpatialCoverageFactory(zones=[paca.id]))
 

udata/core/topic/factories.py

@@ -1,7 +1,7 @@
 import factory
 
 from udata import utils
-from udata.core.dataset.factories import DatasetFactory
+from udata.core.dataset.factories import VisibleDatasetFactory
 from udata.core.reuse.factories import VisibleReuseFactory
 from udata.factories import ModelFactory
 
@@ -19,7 +19,7 @@ class TopicFactory(ModelFactory):
 
     @factory.lazy_attribute
     def datasets(self):
-        return DatasetFactory.create_batch(3)
+        return VisibleDatasetFactory.create_batch(3)
 
     @factory.lazy_attribute
     def reuses(self):

udata/harvest/backends/dcat.py

@@ -2,7 +2,7 @@ import logging
 
 from rdflib import Graph, URIRef
 from rdflib.namespace import RDF
-import lxml.etree as ET
+import xml.etree.ElementTree as ET
 import boto3
 from flask import current_app
 from datetime import date
@@ -173,36 +173,7 @@ class DcatBackend(BaseBackend):
         dataset = self.get_dataset(item.remote_id)
         dataset = dataset_from_rdf(graph, dataset, node=node)
         return dataset
-    
 
-    def next_record_if_should_continue(self, start, search_results):
-        next_record = int(search_results.attrib['nextRecord'])
-        matched_count = int(search_results.attrib['numberOfRecordsMatched'])
-        returned_count = int(search_results.attrib['numberOfRecordsReturned'])
-
-        # Break conditions copied gratefully from
-        # noqa https://github.com/geonetwork/core-geonetwork/blob/main/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Harvester.java#L338-L369
-        break_conditions = (
-            # standard CSW: A value of 0 means all records have been returned.
-            next_record == 0,
-
-            # Misbehaving CSW server returning a next record > matched count
-            next_record > matched_count,
-
-            # No results returned already
-            returned_count == 0,
-
-            # Current next record is lower than previous one
-            next_record < start,
-
-            # Enough items have been harvested already
-            self.max_items and len(self.job.items) >= self.max_items
-        )
-
-        if any(break_conditions):
-            return None
-        else:
-            return next_record
 
 class CswDcatBackend(DcatBackend):
     display_name = 'CSW-DCAT'
@@ -230,18 +201,17 @@ class CswDcatBackend(DcatBackend):
         graphs = []
         page = 0
         start = 1
-
         response = self.post(url, data=body.format(start=start, schema=self.DCAT_SCHEMA),
                              headers=headers)
         response.raise_for_status()
-        content = response.content
+        content = response.text
         tree = ET.fromstring(content)
         if tree.tag == '{' + OWS_NAMESPACE + '}ExceptionReport':
             raise ValueError(f'Failed to query CSW:\n{content}')
         while tree:
             graph = Graph(namespace_manager=namespace_manager)
             search_results = tree.find('csw:SearchResults', {'csw': CSW_NAMESPACE})
-            if search_results is None:
+            if not search_results:
                 log.error(f'No search results found for {url} on page {page}')
                 break
             for child in search_results:
@@ -255,111 +225,37 @@ class CswDcatBackend(DcatBackend):
                     kwargs['type'] = 'uriref' if isinstance(node, URIRef) else 'blank'
                     self.add_item(id, **kwargs)
             graphs.append(graph)
-
-            next_record = self.next_record_if_should_continue(start, search_results)
-            if not next_record:
-                break
-
-            start = next_record
             page += 1
 
-            tree = ET.fromstring(
-                self.post(url, data=body.format(start=start, schema=self.DCAT_SCHEMA),
-                          headers=headers).content)
-
-        return graphs
-    
-
-
-class CswIso19139DcatBackend(DcatBackend):
-    '''
-    An harvester that takes CSW ISO 19139 as input and transforms it to DCAT using SEMIC GeoDCAT-AP XSLT.
-    The parsing of items is then the same as for the DcatBackend.
-    '''
-
-    display_name = 'CSW-ISO-19139'
-
-    ISO_SCHEMA = 'http://www.isotc211.org/2005/gmd'
+            next_record = int(search_results.attrib['nextRecord'])
+            matched_count = int(search_results.attrib['numberOfRecordsMatched'])
+            returned_count = int(search_results.attrib['numberOfRecordsReturned'])
 
-    XSL_URL = "https://raw.githubusercontent.com/SEMICeu/iso-19139-to-dcat-ap/master/iso-19139-to-dcat-ap.xsl"
+            # Break conditions copied gratefully from
+            # noqa https://github.com/geonetwork/core-geonetwork/blob/main/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Harvester.java#L338-L369
+            break_conditions = (
+                # standard CSW: A value of 0 means all records have been returned.
+                next_record == 0,
 
-    def parse_graph(self, url: str, fmt: str) -> List[Graph]:
-        '''
-        Parse CSW graph querying ISO schema.
-        Use SEMIC GeoDCAT-AP XSLT to map it to a correct version.
-        See https://github.com/SEMICeu/iso-19139-to-dcat-ap for more information on the XSLT.
-        '''
-
-        # Load XSLT
-        xsl = ET.fromstring(self.get(self.XSL_URL).content)
-        transform = ET.XSLT(xsl)
-
-        # Start querying and parsing graph
-        body = '''<csw:GetRecords xmlns:csw="http://www.opengis.net/cat/csw/2.0.2"
-                                  xmlns:gmd="http://www.isotc211.org/2005/gmd"
-                                  service="CSW" version="2.0.2" resultType="results"
-                                  startPosition="{start}" maxPosition="10"
-                                  outputSchema="{schema}">
-                      <csw:Query typeNames="csw:Record">
-                        <csw:ElementSetName>full</csw:ElementSetName>
-                        <csw:Constraint version="1.1.0">
-                            <ogc:Filter xmlns:ogc="http://www.opengis.net/ogc">
-                                <ogc:PropertyIsEqualTo>
-                                    <ogc:PropertyName>dc:type</ogc:PropertyName>
-                                    <ogc:Literal>dataset</ogc:Literal>
-                                </ogc:PropertyIsEqualTo>
-                            </ogc:Filter>
-                        </csw:Constraint>
-                    </csw:Query>
-                </csw:GetRecords>'''
-        headers = {'Content-Type': 'application/xml'}
+                # Misbehaving CSW server returning a next record > matched count
+                next_record > matched_count,
 
-        graphs = []
-        page = 0
-        start = 1
+                # No results returned already
+                returned_count == 0,
 
-        response = self.post(url, data=body.format(start=start, schema=self.ISO_SCHEMA),
-                            headers=headers)
-        response.raise_for_status()
+                # Current next record is lower than previous one
+                next_record < start,
 
-        tree_before_transform = ET.fromstring(response.content)
-        # Disabling CoupledResourceLookUp to prevent failure on xlink:href
-        # https://github.com/SEMICeu/iso-19139-to-dcat-ap/blob/master/documentation/HowTo.md#parameter-coupledresourcelookup
-        tree = transform(tree_before_transform, CoupledResourceLookUp="'disabled'")
+                # Enough items have been harvested already
+                self.max_items and len(self.job.items) >= self.max_items
+            )
 
-        while tree:
-            # We query the tree before the transformation because the XSLT remove the search results
-            # infos (useful for pagination)
-            search_results = tree_before_transform.find('csw:SearchResults', {'csw': CSW_NAMESPACE})
-            if search_results is None:
-                log.error(f'No search results found for {url} on page {page}')
+            if any(break_conditions):
                 break
 
-            subgraph = Graph(namespace_manager=namespace_manager)
-            subgraph.parse(ET.tostring(tree), format=fmt)
-
-            if not subgraph.subjects(RDF.type, DCAT.Dataset):
-                raise ValueError("Failed to fetch CSW content")
-
-            for node in subgraph.subjects(RDF.type, DCAT.Dataset):
-                id = subgraph.value(node, DCT.identifier)
-                kwargs = {'nid': str(node), 'page': page}
-                kwargs['type'] = 'uriref' if isinstance(node, URIRef) else 'blank'
-                self.add_item(id, **kwargs)
-            graphs.append(subgraph)
-
-            next_record = self.next_record_if_should_continue(start, search_results)
-            if not next_record:
-                break
-            
             start = next_record
-            page += 1
-
-            response = self.post(url, data=body.format(start=start, schema=self.ISO_SCHEMA),
-                          headers=headers)
-            response.raise_for_status()
-
-            tree_before_transform = ET.fromstring(response.content)
-            tree = transform(tree_before_transform, CoupledResourceLookUp="'disabled'")
+            tree = ET.fromstring(
+                self.post(url, data=body.format(start=start, schema=self.DCAT_SCHEMA),
+                          headers=headers).text)
 
         return graphs