udata 7.0.4.dev27777__py2.py3-none-any.whl → 7.0.5__py2.py3-none-any.whl

udata/__init__.py

@@ -4,5 +4,5 @@
 udata
 '''
 
-__version__ = '7.0.4.dev'
+__version__ = '7.0.5'
 __description__ = 'Open data portal'

udata/api/__init__.py

@@ -147,7 +147,7 @@ class UDataApi(Api):
 
     def validate(self, form_cls, obj=None):
         '''Validate a form from the request and handle errors'''
-        if 'application/json' not in request.headers.get('Content-Type'):
+        if 'application/json' not in request.headers.get('Content-Type', ''):
             errors = {'Content-Type': 'expecting application/json'}
             self.abort(400, errors=errors)
         form = form_cls.from_json(request.json, obj=obj, instance=obj,

udata/core/dataset/api.py

@@ -197,7 +197,7 @@ class DatasetListAPI(API):
     @api.secure
     @api.doc('create_dataset', responses={400: 'Validation error'})
     @api.expect(dataset_fields)
-    @api.marshal_with(dataset_fields)
+    @api.marshal_with(dataset_fields, code=201)
     def post(self):
         '''Create a new dataset'''
         form = api.validate(DatasetForm)
@@ -344,9 +344,9 @@ class ResourceRedirectAPI(API):
 @ns.route('/<dataset:dataset>/resources/', endpoint='resources')
 class ResourcesAPI(API):
     @api.secure
-    @api.doc('create_resource', **common_doc)
+    @api.doc('create_resource', **common_doc, responses={400: 'Validation error'})
     @api.expect(resource_fields)
-    @api.marshal_with(resource_fields)
+    @api.marshal_with(resource_fields, code=201)
     def post(self, dataset):
         '''Create a new resource for a given dataset'''
         ResourceEditPermission(dataset).test()
@@ -361,7 +361,7 @@ class ResourcesAPI(API):
         return resource, 201
 
     @api.secure
-    @api.doc('update_resources', **common_doc)
+    @api.doc('update_resources', **common_doc, responses={400: 'Validation error'})
     @api.expect([resource_fields])
     @api.marshal_list_with(resource_fields)
     def put(self, dataset):
@@ -397,9 +397,9 @@ class UploadMixin(object):
 @api.doc(**common_doc)
 class UploadNewDatasetResource(UploadMixin, API):
     @api.secure
-    @api.doc('upload_new_dataset_resource')
+    @api.doc('upload_new_dataset_resource', responses={415: 'Incorrect file content type', 400: 'Upload error'})
     @api.expect(upload_parser)
-    @api.marshal_with(upload_fields)
+    @api.marshal_with(upload_fields, code=201)
     def post(self, dataset):
         '''Upload a new dataset resource'''
         ResourceEditPermission(dataset).test()
@@ -416,9 +416,9 @@ class UploadNewDatasetResource(UploadMixin, API):
 @api.doc(**common_doc)
 class UploadNewCommunityResources(UploadMixin, API):
     @api.secure
-    @api.doc('upload_new_community_resource')
+    @api.doc('upload_new_community_resource', responses={415: 'Incorrect file content type', 400: 'Upload error'})
     @api.expect(upload_parser)
-    @api.marshal_with(upload_fields)
+    @api.marshal_with(upload_fields, code=201)
     def post(self, dataset):
         '''Upload a new community resource'''
         infos = self.handle_upload(dataset)
@@ -442,7 +442,7 @@ class ResourceMixin(object):
 @api.param('rid', 'The resource unique identifier')
 class UploadDatasetResource(ResourceMixin, UploadMixin, API):
     @api.secure
-    @api.doc('upload_dataset_resource')
+    @api.doc('upload_dataset_resource', responses={415: 'Incorrect file content type', 400: 'Upload error'})
     @api.marshal_with(upload_fields)
     def post(self, dataset, rid):
         '''Upload a file related to a given resource on a given dataset'''
@@ -465,7 +465,7 @@ class UploadDatasetResource(ResourceMixin, UploadMixin, API):
 @api.param('community', 'The community resource unique identifier')
 class ReuploadCommunityResource(ResourceMixin, UploadMixin, API):
     @api.secure
-    @api.doc('upload_community_resource')
+    @api.doc('upload_community_resource', responses={415: 'Incorrect file content type', 400: 'Upload error'})
     @api.marshal_with(upload_fields)
     def post(self, community):
         '''Update the file related to a given community resource'''
@@ -493,7 +493,7 @@ class ResourceAPI(ResourceMixin, API):
         return resource
 
     @api.secure
-    @api.doc('update_resource')
+    @api.doc('update_resource', responses={400: 'Validation error'})
     @api.expect(resource_fields)
     @api.marshal_with(resource_fields)
     def put(self, dataset, rid):
@@ -546,9 +546,9 @@ class CommunityResourcesAPI(API):
                                    .paginate(args['page'], args['page_size']))
 
     @api.secure
-    @api.doc('create_community_resource')
+    @api.doc('create_community_resource', responses={400: 'Validation error'})
     @api.expect(community_resource_fields)
-    @api.marshal_with(community_resource_fields)
+    @api.marshal_with(community_resource_fields, code=201)
     def post(self):
         '''Create a new community resource'''
         form = api.validate(CommunityResourceForm)
@@ -578,7 +578,7 @@ class CommunityResourceAPI(API):
         return community
 
     @api.secure
-    @api.doc('update_community_resource')
+    @api.doc('update_community_resource', responses={400: 'Validation error'})
     @api.expect(community_resource_fields)
     @api.marshal_with(community_resource_fields)
     def put(self, community):

udata/core/dataset/api_fields.py

@@ -33,9 +33,9 @@ schema_fields = api.model('Schema', {
 dataset_harvest_fields = api.model('HarvestDatasetMetadata', {
     'backend': fields.String(description='Harvest backend used', allow_null=True),
     'created_at': fields.ISODateTime(description='The dataset harvested creation date',
-                                     allow_null=True),
+                                     allow_null=True, readonly=True),
     'modified_at': fields.ISODateTime(description='The dataset harvest last modification date',
-                                      allow_null=True),
+                                      allow_null=True, readonly=True),
     'source_id': fields.String(description='The harvester id', allow_null=True),
     'remote_id': fields.String(description='The dataset remote id on the source portal',
                                allow_null=True),
@@ -54,9 +54,9 @@ dataset_harvest_fields = api.model('HarvestDatasetMetadata', {
 
 resource_harvest_fields = api.model('HarvestResourceMetadata', {
     'created_at': fields.ISODateTime(description='The resource harvested creation date',
-                                     allow_null=True),
+                                     allow_null=True, readonly=True),
     'modified_at': fields.ISODateTime(description='The resource harvest last modification date',
-                                      allow_null=True),
+                                      allow_null=True, readonly=True),
     'uri': fields.String(description='The resource harvest uri', allow_null=True)
 })
 
@@ -199,10 +199,10 @@ dataset_fields = api.model('Dataset', {
     'description': fields.Markdown(
         description='The dataset description in markdown', required=True),
     'created_at': fields.ISODateTime(
-        description='This date is computed between harvested creation date if any and site\'s internal creation date' , required=True),
+        description='This date is computed between harvested creation date if any and site\'s internal creation date' , required=True, readonly=True),
     'last_modified': fields.ISODateTime(
-        description='The dataset last modification date', required=True),
-    'deleted': fields.ISODateTime(description='The deletion date if deleted'),
+        description='The dataset last modification date', required=True, readonly=True),
+    'deleted': fields.ISODateTime(description='The deletion date if deleted', readonly=True),
     'archived': fields.ISODateTime(description='The archival date if archived'),
     'featured': fields.Boolean(description='Is the dataset featured'),
     'private': fields.Boolean(

udata/core/dataset/apiv2.py

@@ -73,10 +73,10 @@ dataset_fields = apiv2.model('Dataset', {
     'description': fields.Markdown(
         description='The dataset description in markdown', required=True),
     'created_at': fields.ISODateTime(
-        description='The dataset creation date', required=True),
+        description='The dataset creation date', required=True, readonly=True),
     'last_modified': fields.ISODateTime(
-        description='The dataset last modification date', required=True),
-    'deleted': fields.ISODateTime(description='The deletion date if deleted'),
+        description='The dataset last modification date', required=True, readonly=True),
+    'deleted': fields.ISODateTime(description='The deletion date if deleted', readonly=True),
     'archived': fields.ISODateTime(description='The archival date if archived'),
     'featured': fields.Boolean(description='Is the dataset featured'),
     'private': fields.Boolean(

udata/core/dataset/rdf.py

@@ -2,17 +2,21 @@
 This module centralize dataset helpers for RDF/DCAT serialization and parsing
 '''
 import calendar
+import json
 import logging
 
 from datetime import date
 from html.parser import HTMLParser
 from dateutil.parser import parse as parse_dt
 from flask import current_app
+from geomet import wkt
 from rdflib import Graph, URIRef, Literal, BNode
 from rdflib.resource import Resource as RdfResource
 from rdflib.namespace import RDF
+from mongoengine.errors import ValidationError
 
 from udata import i18n, uris
+from udata.core.spatial.models import SpatialCoverage
 from udata.frontend.markdown import parse_html
 from udata.core.dataset.models import HarvestDatasetMetadata, HarvestResourceMetadata
 from udata.models import db, ContactPoint
@@ -334,6 +338,40 @@ def contact_point_from_rdf(rdf, dataset):
                     ContactPoint(name=name, email=email, owner=dataset.owner).save())
 
 
+def spatial_from_rdf(graph):
+    for term in graph.objects(DCT.spatial):
+        for object in term.objects():
+            if isinstance(object, Literal):
+                if object.datatype.__str__() == 'https://www.iana.org/assignments/media-types/application/vnd.geo+json':
+                    try:
+                        geojson = json.loads(object.toPython())
+                    except ValueError as e:
+                        log.warning(f"Invalid JSON in spatial GeoJSON {object.toPython()} {e}")
+                        continue
+                elif object.datatype.__str__() == 'http://www.opengis.net/rdf#wktLiteral':
+                    try:
+                        # .upper() si here because geomet doesn't support Polygon but only POLYGON
+                        geojson = wkt.loads(object.toPython().strip().upper())
+                    except ValueError as e:
+                        log.warning(f"Invalid JSON in spatial WKT {object.toPython()} {e}")
+                        continue
+                else:
+                    continue
+
+                if geojson['type'] == 'Polygon':
+                    geojson['type'] = 'MultiPolygon'
+                    geojson['coordinates'] = [geojson['coordinates']]
+
+                spatial_coverage = SpatialCoverage(geom=geojson)
+
+                try:
+                    spatial_coverage.clean()
+                    return spatial_coverage
+                except ValidationError:
+                    continue
+
+    return None
+
 def frequency_from_rdf(term):
     if isinstance(term, str):
         try:
@@ -488,7 +526,7 @@ def resource_from_rdf(graph_or_distrib, dataset=None, is_additionnal=False):
     return resource
 
 
-def dataset_from_rdf(graph, dataset=None, node=None):
+def dataset_from_rdf(graph: Graph, dataset=None, node=None):
     '''
     Create or update a dataset from a RDF/DCAT graph
     '''
@@ -509,6 +547,10 @@ def dataset_from_rdf(graph, dataset=None, node=None):
     if schema:
         dataset.schema = schema
 
+    spatial_coverage = spatial_from_rdf(d)
+    if spatial_coverage:
+        dataset.spatial = spatial_coverage
+
     acronym = rdf_value(d, SKOS.altLabel)
     if acronym:
         dataset.acronym = acronym

udata/core/organization/csv.py

@@ -1,3 +1,4 @@
+from udata.core.dataset.models import Dataset
 from udata.frontend import csv
 
 from .models import Organization
@@ -5,6 +6,8 @@ from .models import Organization
 
 @csv.adapter(Organization)
 class OrganizationCsvAdapter(csv.Adapter):
+    downloads_counts = None
+
     fields = (
         'id',
         'name',
@@ -18,4 +21,27 @@ class OrganizationCsvAdapter(csv.Adapter):
     )
 
     def dynamic_fields(self):
-        return csv.metric_fields(Organization)
+        return csv.metric_fields(Organization) + self.get_dynamic_field_downloads()
+    
+    def get_dynamic_field_downloads(self):
+        downloads_counts = self.get_downloads_counts()
+        return [('downloads', lambda o: downloads_counts.get(str(o.id), 0))]
+    
+    def get_downloads_counts(self):
+        '''
+        Prefetch all the resources' downloads for all selected organization into memory
+        '''
+        if self.downloads_counts is not None:
+            return self.downloads_counts
+
+        self.downloads_counts = {}
+
+        ids = [o.id for o in self.queryset]
+        for dataset in Dataset.objects(organization__in=ids):
+            org_id = str(dataset.organization.id)
+            if self.downloads_counts.get(org_id) is None:
+                self.downloads_counts[org_id] = 0
+
+            self.downloads_counts[org_id] += sum(resource.metrics.get('views', 0) for resource in dataset.resources)
+
+        return self.downloads_counts

udata/core/organization/models.py

@@ -14,7 +14,8 @@ from udata.uris import endpoint_for
 
 __all__ = (
     'Organization', 'Team', 'Member', 'MembershipRequest',
-    'ORG_ROLES', 'MEMBERSHIP_STATUS', 'PUBLIC_SERVICE', 'CERTIFIED'
+    'ORG_ROLES', 'MEMBERSHIP_STATUS', 'PUBLIC_SERVICE', 'CERTIFIED',
+    'ASSOCIATION', 'COMPANY', 'LOCAL_AUTHORITY'
 )
 
 
@@ -36,6 +37,9 @@ LOGO_SIZES = [100, 60, 25]
 
 PUBLIC_SERVICE = 'public-service'
 CERTIFIED = 'certified'
+ASSOCIATION = 'Association'
+COMPANY = 'Company'
+LOCAL_AUTHORITY = 'Local authority'
 
 TITLE_SIZE_LIMIT = 350
 DESCRIPTION_SIZE_LIMIT = 100000
@@ -140,6 +144,9 @@ class Organization(WithMetrics, BadgeMixin, db.Datetimed, db.Document):
     __badges__ = {
         PUBLIC_SERVICE: _('Public Service'),
         CERTIFIED: _('Certified'),
+        ASSOCIATION: _('Association'),
+        COMPANY: _('Company'),
+        LOCAL_AUTHORITY: _('Local authority'),
     }
 
     __metrics_keys__ = [
@@ -199,6 +206,18 @@ class Organization(WithMetrics, BadgeMixin, db.Datetimed, db.Document):
         is_public_service = any(b.kind == PUBLIC_SERVICE for b in self.badges)
         return self.certified and is_public_service
 
+    @property
+    def company(self):
+        return any(b.kind == COMPANY for b in self.badges)
+
+    @property
+    def association(self):
+        return any(b.kind == ASSOCIATION for b in self.badges)
+
+    @property
+    def local_authority(self):
+        return any(b.kind == LOCAL_AUTHORITY for b in self.badges)
+
     def member(self, user):
         for member in self.members:
             if member.user == user:

udata/core/organization/tasks.py

@@ -7,7 +7,7 @@ from udata.tasks import job, task, get_logger
 
 from udata.core.badges.tasks import notify_new_badge
 
-from .models import Organization, CERTIFIED, PUBLIC_SERVICE
+from .models import Organization, CERTIFIED, PUBLIC_SERVICE, COMPANY, ASSOCIATION, LOCAL_AUTHORITY
 
 log = get_logger(__name__)
 
@@ -114,3 +114,63 @@ def notify_badge_public_service(org_id):
         organization=org,
         badge=org.get_badge(PUBLIC_SERVICE)
     )
+
+
+@notify_new_badge(Organization, COMPANY)
+def notify_badge_company(org_id):
+    '''
+    Send an email when a `COMPANY` badge is added to an `Organization`
+    '''
+    org = Organization.objects.get(pk=org_id)
+    recipients = [member.user for member in org.members]
+    subject = _(
+        'Your organization "%(name)s" has been identified as a company',
+        name=org.name
+    )
+    mail.send(
+        subject,
+        recipients,
+        'badge_added_company',
+        organization=org,
+        badge=org.get_badge(COMPANY)
+    )
+
+
+@notify_new_badge(Organization, ASSOCIATION)
+def notify_badge_association(org_id):
+    '''
+    Send an email when a `ASSOCIATION` badge is added to an `Organization`
+    '''
+    org = Organization.objects.get(pk=org_id)
+    recipients = [member.user for member in org.members]
+    subject = _(
+        'Your organization "%(name)s" has been identified as an association',
+        name=org.name
+    )
+    mail.send(
+        subject,
+        recipients,
+        'badge_added_association',
+        organization=org,
+        badge=org.get_badge(ASSOCIATION)
+    )
+
+
+@notify_new_badge(Organization, LOCAL_AUTHORITY)
+def notify_badge_local_authority(org_id):
+    '''
+    Send an email when a `LOCAL_AUTHORITY` badge is added to an `Organization`
+    '''
+    org = Organization.objects.get(pk=org_id)
+    recipients = [member.user for member in org.members]
+    subject = _(
+        'Your organization "%(name)s" has been identified as a local authority',
+        name=org.name
+    )
+    mail.send(
+        subject,
+        recipients,
+        'badge_added_local_authority',
+        organization=org,
+        badge=org.get_badge(LOCAL_AUTHORITY)
+    )

udata/core/spatial/commands.py

@@ -44,7 +44,10 @@ def load_levels(col, json_levels):
 
 
 def load_zones(col, json_geozones):
-    for i, geozone in enumerate(json_geozones):
+    loaded_geozones = 0
+    for _, geozone in enumerate(json_geozones):
+        if geozone.get('is_deleted', False):
+            continue
         params = {
             'slug': slugify.slugify(geozone['nom'], separator='-'),
             'level': str(geozone['level']),
@@ -56,11 +59,12 @@ def load_zones(col, json_geozones):
             col.objects(id=geozone['_id']).modify(upsert=True, **{
                 'set__{0}'.format(k): v for k, v in params.items()
             })
+            loaded_geozones += 1
         except errors.ValidationError as e:
             log.warning('Validation error (%s) for %s with %s',
                         e, geozone['nom'], params)
             continue
-    return i
+    return loaded_geozones
 
 
 @contextmanager
@@ -137,6 +141,10 @@ def load(geozones_file, levels_file, drop=False):
             total = load_zones(GeoZone, json_geozones)
     log.info('Loaded {total} zones'.format(total=total))
 
+    log.info('Clean removed geozones in datasets')
+    count = fixup_removed_geozone()
+    log.info(f'{count} geozones removed from datasets')
+
 
 @grp.command()
 def migrate():
@@ -184,3 +192,19 @@ def migrate():
     '''.format(level_summary, **counter)), level_summary])
     log.info(summary)
     log.info('Done')
+
+def fixup_removed_geozone():
+    count = 0
+    all_datasets = Dataset.objects(spatial__zones__0__exists=True).timeout(False)
+    for dataset in all_datasets:
+        zones = dataset.spatial.zones
+        new_zones = [z for z in zones if getattr(z, 'name', None) is not None]
+
+        if len(new_zones) < len(zones):
+            log.debug(f"Removing deleted zones from dataset '{dataset.title}'")
+            count += len(zones) - len(new_zones)
+            dataset.spatial.zones = new_zones
+            dataset.save()
+
+    return count
+        

udata/core/topic/api.py

@@ -3,6 +3,7 @@ from udata.core.dataset.api_fields import dataset_fields
 from udata.core.discussions.models import Discussion
 from udata.core.organization.api_fields import org_ref_fields
 from udata.core.reuse.api_fields import reuse_fields
+from udata.core.spatial.api_fields import spatial_coverage_fields
 from udata.core.topic.permissions import TopicEditPermission
 from udata.core.topic.parsers import TopicApiParser
 from udata.core.user.api_fields import user_ref_fields
@@ -37,6 +38,11 @@ topic_fields = api.model('Topic', {
     'private': fields.Boolean(description='Is the topic private'),
     'created_at': fields.ISODateTime(
         description='The topic creation date', readonly=True),
+    'spatial': fields.Nested(
+        spatial_coverage_fields, allow_null=True,
+        description='The spatial coverage'),
+    'last_modified': fields.ISODateTime(
+        description='The topic last modification date', readonly=True),
     'organization': fields.Nested(
         org_ref_fields, allow_null=True,
         description='The publishing organization', readonly=True),

udata/core/topic/apiv2.py

@@ -13,6 +13,7 @@ from udata.core.organization.api_fields import org_ref_fields
 from udata.core.reuse.api import ReuseApiParser
 from udata.core.reuse.apiv2 import reuse_page_fields
 from udata.core.reuse.models import Reuse
+from udata.core.spatial.api_fields import spatial_coverage_fields
 from udata.core.topic.models import Topic
 from udata.core.topic.parsers import TopicApiParser
 from udata.core.topic.permissions import TopicEditPermission
@@ -63,6 +64,11 @@ topic_fields = apiv2.model('Topic', {
     'private': fields.Boolean(description='Is the topic private'),
     'created_at': fields.ISODateTime(
         description='The topic creation date', readonly=True),
+    'spatial': fields.Nested(
+        spatial_coverage_fields, allow_null=True,
+        description='The spatial coverage'),
+    'last_modified': fields.ISODateTime(
+        description='The topic last modification date', readonly=True),
     'organization': fields.Nested(
         org_ref_fields, allow_null=True,
         description='The publishing organization', readonly=True),

udata/core/topic/forms.py

@@ -1,4 +1,5 @@
 from udata.forms import ModelForm, fields, validators
+from udata.core.spatial.forms import SpatialCoverageField
 from udata.i18n import lazy_gettext as _
 
 from .models import Topic
@@ -20,6 +21,10 @@ class TopicForm(ModelForm):
     datasets = fields.DatasetListField(_('Associated datasets'))
     reuses = fields.ReuseListField(_('Associated reuses'))
 
+    spatial = SpatialCoverageField(
+        _('Spatial coverage'),
+        description=_('The geographical area covered by the data.'))
+
     tags = fields.TagField(_('Tags'), [validators.DataRequired()])
     private = fields.BooleanField(_('Private'))
     featured = fields.BooleanField(_('Featured'))

udata/core/topic/models.py

@@ -1,8 +1,6 @@
-from datetime import datetime
 from flask import url_for
-from mongoengine.fields import DateTimeField
 from mongoengine.signals import pre_save
-from udata.models import db
+from udata.models import db, SpatialCoverage
 from udata.search import reindex
 from udata.tasks import as_task_param
 
@@ -10,7 +8,7 @@ from udata.tasks import as_task_param
 __all__ = ('Topic', )
 
 
-class Topic(db.Document, db.Owned):
+class Topic(db.Document, db.Owned, db.Datetimed):
     name = db.StringField(required=True)
     slug = db.SlugField(max_length=255, required=True, populate_from='name',
                         update=True, follow=True)
@@ -28,7 +26,7 @@ class Topic(db.Document, db.Owned):
     private = db.BooleanField()
     extras = db.ExtrasField()
 
-    created_at = DateTimeField(default=datetime.utcnow, required=True)
+    spatial = db.EmbeddedDocumentField(SpatialCoverage)
 
     meta = {
         'indexes': [

udata/forms/fields.py

@@ -180,6 +180,16 @@ class BooleanField(FieldHelper, fields.BooleanField):
         self.stacked = kwargs.pop('stacked', False)
         super(BooleanField, self).__init__(*args, **kwargs)
 
+    def process_formdata(self, valuelist):
+        # We override this so that when no value is provided
+        # the form doesn't think the value is `False` instead 
+        # the value is not present and the model can keep the
+        # existing value
+        if not valuelist:
+            return 
+
+        super().process_formdata(valuelist)
+
 
 class RadioField(FieldHelper, fields.RadioField):
     def __init__(self, *args, **kwargs):

udata/frontend/csv.py

@@ -58,8 +58,8 @@ class Adapter(object):
                     else:
                         field_tuple = (name, self.getter(*field))
                 except Exception as e:  # Catch all errors intentionally.
-                    log.error('Error exporting CSV for {name}: {error}'.format(
-                        name=self.__class__.__name__, error=e))
+                    log.exception('Error exporting CSV for {name}: {error_class} {error}'.format(
+                        name=self.__class__.__name__, error_class=e.__class__.__name__, error=e), stack_info=True)
                 self._fields.append(field_tuple)
         return self._fields
 
@@ -89,8 +89,8 @@ class Adapter(object):
                 try:
                     content = safestr(getter(obj))
                 except Exception as e:  # Catch all errors intentionally.
-                    log.error('Error exporting CSV for {name}: {error}'.format(
-                        name=self.__class__.__name__, error=e))
+                    log.exception('Error exporting CSV for {name}: {error_class} {error}'.format(
+                        name=self.__class__.__name__, error_class=e.__class__.__name__, error=e), stack_info=True)
             row.append(content)
         return row
 
@@ -130,8 +130,8 @@ class NestedAdapter(Adapter):
                     else:
                         field_tuple = (name, self.getter(*field))
                 except Exception as e:  # Catch all errors intentionally.
-                    log.error('Error exporting CSV for {name}: {error}'.format(
-                        name=self.__class__.__name__, error=e))
+                    log.exception('Error exporting CSV for {name}: {error_class} {error}'.format(
+                        name=self.__class__.__name__, error_class=e.__class__.__name__, error=e), stack_info=True)
                 self._nested_fields.append(field_tuple)
         return self._nested_fields
 
@@ -155,8 +155,8 @@ class NestedAdapter(Adapter):
                 try:
                     content = safestr(getter(nested))
                 except Exception as e:  # Catch all errors intentionally.
-                    log.error('Error exporting CSV for {name}: {error}'.format(
-                        name=self.__class__.__name__, error=e))
+                    log.exception('Error exporting CSV for {name}: {error_class} {error}'.format(
+                        name=self.__class__.__name__, error_class=e.__class__.__name__, error=e), stack_info=True)
             row.append(content)
         return row
 

udata/harvest/actions.py

@@ -10,6 +10,7 @@ from flask import current_app
 from udata.auth import current_user
 from udata.core.dataset.models import HarvestDatasetMetadata
 from udata.models import User, Organization, PeriodicTask, Dataset
+from udata.storage.s3 import delete_file
 
 from . import backends, signals
 from .models import (
@@ -162,6 +163,16 @@ def purge_jobs():
     '''Delete jobs older than retention policy'''
     retention = current_app.config['HARVEST_JOBS_RETENTION_DAYS']
     expiration = datetime.utcnow() - timedelta(days=retention)
+
+    jobs_with_external_files = HarvestJob.objects(data__filename__exists=True, created__lt=expiration)
+    for job in jobs_with_external_files:
+        bucket = current_app.config.get('HARVEST_GRAPHS_S3_BUCKET')
+        if bucket is None:
+            log.error(f"Bucket isn't configured anymore, but jobs still exist with external filenames. Could not delete them.")
+            break
+
+        delete_file(bucket, job.data['filename'])
+
     return HarvestJob.objects(created__lt=expiration).delete()