udata 8.0.2.dev29304__py2.py3-none-any.whl → 9.0.1.dev29390__py2.py3-none-any.whl

udata/__init__.py

@@ -4,5 +4,5 @@
 udata
 '''
 
-__version__ = '8.0.2.dev'
+__version__ = '9.0.1.dev'
 __description__ = 'Open data portal'

udata/core/dataset/csv.py

@@ -19,6 +19,9 @@ class DatasetCsvAdapter(csv.Adapter):
         ('url', 'external_url'),
         ('organization', 'organization.name'),
         ('organization_id', 'organization.id'),
+        ('owner', 'owner.slug'),  # in case it's owned by a user, or introduce 'owner_type'?
+        ('owner_id', 'owner.id'),
+        # 'contact_point', #  ?
         'description',
         'frequency',
         'license',
@@ -26,19 +29,20 @@ class DatasetCsvAdapter(csv.Adapter):
         'temporal_coverage.end',
         'spatial.granularity',
         ('spatial.zones', serialize_spatial_zones),
-        'private',
         ('featured', lambda o: o.featured or False),
         'created_at',
         'last_modified',
         ('tags', lambda o: ','.join(o.tags)),
         ('archived', lambda o: o.archived or False),
         ('resources_count', lambda o: len(o.resources)),
+        ('main_resources_count', lambda o: len([r for r in o.resources if r.type == 'main'])),
         'downloads',
         ('harvest.backend', lambda r: r.harvest and r.harvest.backend),
         ('harvest.domain', lambda r: r.harvest and r.harvest.domain),
         ('harvest.created_at', lambda r: r.harvest and r.harvest.created_at),
         ('harvest.modified_at', lambda r: r.harvest and r.harvest.modified_at),
         ('quality_score', lambda o: format(o.quality['score'], '.2f')),
+        # schema? what is the schema of a dataset?
     )
 
     def dynamic_fields(self):
@@ -85,6 +89,9 @@ class ResourcesCsvAdapter(csv.NestedAdapter):
         ('downloads', lambda o: int(o.metrics.get('views', 0))),
         ('harvest.created_at', lambda o: o.harvest and o.harvest.created_at),
         ('harvest.modified_at', lambda o: o.harvest and o.harvest.modified_at),
+        ('schema_name', 'schema.name'),
+        ('schema_version', 'schema.version'),
+        ('preview_url', lambda o: o.preview_url or False),
     )
     attribute = 'resources'
 

udata/core/dataset/rdf.py

@@ -22,9 +22,10 @@ from udata.frontend.markdown import parse_html
 from udata.core.dataset.models import HarvestDatasetMetadata, HarvestResourceMetadata
 from udata.models import db, ContactPoint
 from udata.rdf import (
-    DCAT, DCT, FREQ, SCV, SKOS, SPDX, SCHEMA, EUFREQ, EUFORMAT, IANAFORMAT, VCARD, RDFS,
-    namespace_manager, schema_from_rdf, url_from_rdf
+    DCAT, DCATAP, DCT, FREQ, SCV, SKOS, SPDX, SCHEMA, EUFREQ, EUFORMAT, IANAFORMAT, VCARD, RDFS,
+    HVD_LEGISLATION, namespace_manager, schema_from_rdf, url_from_rdf
 )
+from udata.tags import slug as slugify_tag
 from udata.utils import get_by, safe_unicode
 from udata.uris import endpoint_for
 
@@ -85,6 +86,7 @@ EU_HVD_CATEGORIES = {
     "http://data.europa.eu/bna/c_dd313021": "Observation de la terre et environnement",
     "http://data.europa.eu/bna/c_e1da4e07": "Statistiques"
 }
+TAG_TO_EU_HVD_CATEGORIES = {slugify_tag(EU_HVD_CATEGORIES[uri]): uri for uri in EU_HVD_CATEGORIES}
 
 
 class HTMLDetector(HTMLParser):
@@ -141,7 +143,7 @@ def owner_to_rdf(dataset, graph=None):
     return
 
 
-def resource_to_rdf(resource, dataset=None, graph=None):
+def resource_to_rdf(resource, dataset=None, graph=None, is_hvd=False):
     '''
     Map a Resource domain model to a DCAT/RDF graph
     '''
@@ -180,6 +182,9 @@ def resource_to_rdf(resource, dataset=None, graph=None):
         checksum.add(SPDX.algorithm, getattr(SPDX, algorithm))
         checksum.add(SPDX.checksumValue, Literal(resource.checksum.value))
         r.add(SPDX.checksum, checksum)
+    if is_hvd:
+        # DCAT-AP HVD applicable legislation is also expected at the distribution level
+        r.add(DCATAP.applicableLegislation, URIRef(HVD_LEGISLATION))
     return r
 
 
@@ -214,11 +219,20 @@ def dataset_to_rdf(dataset, graph=None):
     if dataset.acronym:
         d.set(SKOS.altLabel, Literal(dataset.acronym))
 
+    # Add DCAT-AP HVD properties if the dataset is tagged hvd.
+    # See https://semiceu.github.io/DCAT-AP/releases/2.2.0-hvd/
+    is_hvd = current_app.config['HVD_SUPPORT'] and 'hvd' in dataset.tags
+    if is_hvd:
+        d.add(DCATAP.applicableLegislation, URIRef(HVD_LEGISLATION))
+
     for tag in dataset.tags:
         d.add(DCAT.keyword, Literal(tag))
+        # Add HVD category if this dataset is tagged HVD
+        if is_hvd and tag in TAG_TO_EU_HVD_CATEGORIES:
+            d.add(DCATAP.hvdCategory, URIRef(TAG_TO_EU_HVD_CATEGORIES[tag]))
 
     for resource in dataset.resources:
-        d.add(DCAT.distribution, resource_to_rdf(resource, dataset, graph))
+        d.add(DCAT.distribution, resource_to_rdf(resource, dataset, graph, is_hvd))
 
     if dataset.temporal_coverage:
         d.set(DCT.temporal, temporal_to_rdf(dataset.temporal_coverage, graph))

udata/core/organization/csv.py

@@ -15,18 +15,20 @@ class OrganizationCsvAdapter(csv.Adapter):
         ('url', 'external_url'),
         'description',
         ('logo', lambda o: o.logo(external=True)),
-        ('badges', lambda o: [badge.kind for badge in o.badges]),
+        ('badges', lambda o: ','.join([badge.kind for badge in o.badges])),
         'created_at',
         'last_modified',
+        'business_number_id',
+        ('members_count', lambda o: len(o.members)),
     )
 
     def dynamic_fields(self):
         return csv.metric_fields(Organization) + self.get_dynamic_field_downloads()
-    
+
     def get_dynamic_field_downloads(self):
         downloads_counts = self.get_downloads_counts()
         return [('downloads', lambda o: downloads_counts.get(str(o.id), 0))]
-    
+
     def get_downloads_counts(self):
         '''
         Prefetch all the resources' downloads for all selected organization into memory

udata/core/reuse/csv.py

@@ -15,10 +15,13 @@ class ReuseCsvAdapter(csv.Adapter):
         ('remote_url', 'url'),
         ('organization', 'organization.name'),
         ('organization_id', 'organization.id'),
+        ('owner', 'owner.slug'),  # in case it's owned by a user
+        ('owner_id', 'owner.id'),
         ('image', lambda r: r.image(external=True)),
         ('featured', lambda r: r.featured or False),
         'created_at',
         'last_modified',
+        'topic',
         ('tags', lambda r: ','.join(r.tags)),
         ('datasets', lambda r: ','.join([str(d.id) for d in r.datasets])),
     )

udata/core/site/api.py

@@ -105,7 +105,10 @@ class SiteRdfCatalogFormat(API):
         params = multi_to_dict(request.args)
         page = int(params.get('page', 1))
         page_size = int(params.get('page_size', 100))
-        datasets = Dataset.objects.visible().paginate(page, page_size)
+        datasets = Dataset.objects.visible()
+        if 'tag' in params:
+            datasets = datasets.filter(tags=params.get('tag', ''))
+        datasets = datasets.paginate(page, page_size)
         catalog = build_catalog(current_site, datasets, format=format)
         # bypass flask-restplus make_response, since graph_response
         # is handling the content negociation directly

udata/rdf.py

@@ -21,6 +21,7 @@ log = logging.getLogger(__name__)
 # Extra Namespaces
 ADMS = Namespace('http://www.w3.org/ns/adms#')
 DCAT = Namespace('http://www.w3.org/ns/dcat#')
+DCATAP = Namespace('http://data.europa.eu/r5r/')
 HYDRA = Namespace('http://www.w3.org/ns/hydra/core#')
 SCHEMA = Namespace('http://schema.org/')
 SCV = Namespace('http://purl.org/NET/scovo#')
@@ -35,6 +36,7 @@ VCARD = Namespace('http://www.w3.org/2006/vcard/ns#')
 
 namespace_manager = NamespaceManager(Graph())
 namespace_manager.bind('dcat', DCAT)
+namespace_manager.bind('dcatap', DCATAP)
 namespace_manager.bind('dct', DCT)
 namespace_manager.bind('foaf', FOAF)
 namespace_manager.bind('foaf', FOAF)
@@ -98,6 +100,8 @@ RDF_EXTENSIONS = {
 # Includes control characters, unicode surrogate characters and unicode end-of-plane non-characters
 ILLEGAL_XML_CHARS = '[\x00-\x08\x0b\x0c\x0e-\x1F\uD800-\uDFFF\uFFFE\uFFFF]'
 
+HVD_LEGISLATION = 'http://data.europa.eu/eli/reg_impl/2023/138/oj'
+
 
 def guess_format(string):
     '''Guess format given an extension or a mime-type'''