udata 9.1.2.dev30355__py2.py3-none-any.whl → 9.1.2.dev30454__py2.py3-none-any.whl

udata/harvest/filters.py

@@ -1,18 +1,17 @@
 import dateutil.parser
-
 from voluptuous import Invalid
 
 from udata import tags, uris
 
 
 def boolean(value):
-    '''
+    """
     Convert the content of a string (or a number) to a boolean.
     Do nothing when input value is already a boolean.
 
     This filter accepts usual values for ``True`` and ``False``:
     "0", "f", "false", "n", etc.
-    '''
+    """
     if value is None or isinstance(value, bool):
         return value
 
@@ -22,29 +21,29 @@ def boolean(value):
         lower_value = value.strip().lower()
         if not lower_value:
             return None
-        if lower_value in ('f', 'false', 'n', 'no', 'off'):
+        if lower_value in ("f", "false", "n", "no", "off"):
             return False
-        if lower_value in ('on', 't', 'true', 'y', 'yes'):
+        if lower_value in ("on", "t", "true", "y", "yes"):
             return True
-        raise Invalid('Unable to parse boolean {0}'.format(value))
+        raise Invalid("Unable to parse boolean {0}".format(value))
 
 
 def to_date(value):
-    '''Parse a date'''
+    """Parse a date"""
     return dateutil.parser.parse(value).date()
 
 
 def email(value):
-    '''Validate an email'''
-    if '@' not in value:
-        raise Invalid('This email is invalid.')
+    """Validate an email"""
+    if "@" not in value:
+        raise Invalid("This email is invalid.")
     return value
 
 
 def force_list(value):
-    '''
+    """
     Ensure single elements are wrapped into list
-    '''
+    """
     if not isinstance(value, (list, tuple)):
         return [value]
     return value
@@ -63,12 +62,12 @@ def taglist(value):
 
 
 def empty_none(value):
-    '''Replace falsy values with None'''
+    """Replace falsy values with None"""
     return value if value else None
 
 
 def strip(value):
-    '''Strip spaces from a string and remove it when empty.'''
+    """Strip spaces from a string and remove it when empty."""
     return empty_none(value.strip())
 
 
@@ -76,37 +75,39 @@ def line_endings(value):
     """Replaces CR + LF or CR to LF in a string,
     then strip spaces and remove it when empty.
     """
-    return value.replace('\r\n', '\n').replace('\r', '\n')
+    return value.replace("\r\n", "\n").replace("\r", "\n")
 
 
 def normalize_string(value):
     return strip(line_endings(value))
 
 
-def is_url(default_scheme='http', **kwargs):
+def is_url(default_scheme="http", **kwargs):
     """Return a converter that converts a clean string to an URL."""
+
     def converter(value):
         if value is None:
             return value
-        if '://' not in value and default_scheme:
-            value = '://'.join((default_scheme, value.strip()))
+        if "://" not in value and default_scheme:
+            value = "://".join((default_scheme, value.strip()))
         try:
             return uris.validate(value)
         except uris.ValidationError as e:
             raise Invalid(str(e))
+
     return converter
 
 
 def hash(value):
-    '''Detect an hash type'''
+    """Detect an hash type"""
     if not value:
         return
     elif len(value) == 32:
-        type = 'md5'
+        type = "md5"
     elif len(value) == 40:
-        type = 'sha1'
+        type = "sha1"
     elif len(value) == 64:
-        type = 'sha256'
+        type = "sha256"
     else:
         return None
-    return {'type': type, 'value': value}
+    return {"type": type, "value": value}

udata/harvest/forms.py

@@ -1,18 +1,18 @@
-from udata.utils import safe_unicode
 from udata.forms import Form, fields, validators
 from udata.i18n import lazy_gettext as _
-
+from udata.utils import safe_unicode
 
 from .actions import list_backends
-from .models import VALIDATION_STATES, VALIDATION_REFUSED
+from .models import VALIDATION_REFUSED, VALIDATION_STATES
 
-__all__ = 'HarvestSourceForm', 'HarvestSourceValidationForm'
+__all__ = "HarvestSourceForm", "HarvestSourceValidationForm"
 
 
 class HarvestConfigField(fields.DictField):
-    '''
+    """
     A DictField with extras validations on known configurations
-    '''
+    """
+
     def get_backend(self, form):
         return next(b for b in list_backends() if b.name == form.backend.data)
 
@@ -28,27 +28,27 @@ class HarvestConfigField(fields.DictField):
         if self.data:
             backend = self.get_backend(form)
             # Validate filters
-            for f in (self.data.get('filters') or []):
-                if not ('key' in f and 'value' in f):
-                    msg = 'A field should have both key and value properties'
+            for f in self.data.get("filters") or []:
+                if not ("key" in f and "value" in f):
+                    msg = "A field should have both key and value properties"
                     raise validators.ValidationError(msg)
-                specs = self.get_filter_specs(backend, f['key'])
+                specs = self.get_filter_specs(backend, f["key"])
                 if not specs:
                     msg = 'Unknown filter key "{0}" for "{1}" backend'
-                    msg = msg.format(f['key'], backend.name)
+                    msg = msg.format(f["key"], backend.name)
                     raise validators.ValidationError(msg)
 
-                if isinstance(f['value'], str):
-                    f['value'] = safe_unicode(f['value'])  # Fix encoding error
+                if isinstance(f["value"], str):
+                    f["value"] = safe_unicode(f["value"])  # Fix encoding error
 
-                if not isinstance(f['value'], specs.type):
+                if not isinstance(f["value"], specs.type):
                     msg = '"{0}" filter should of type "{1}"'
                     msg = msg.format(specs.key, specs.type.__name__)
                     raise validators.ValidationError(msg)
             # Validate features
-            for key, value in (self.data.get('features') or {}).items():
+            for key, value in (self.data.get("features") or {}).items():
                 if not isinstance(value, bool):
-                    msg = 'A feature should be a boolean'
+                    msg = "A feature should be a boolean"
                     raise validators.ValidationError(msg)
                 if not self.get_feature_specs(backend, key):
                     msg = 'Unknown feature "{0}" for "{1}" backend'
@@ -57,16 +57,16 @@ class HarvestConfigField(fields.DictField):
 
 
 class HarvestSourceForm(Form):
-    name = fields.StringField(_('Name'), [validators.DataRequired()])
+    name = fields.StringField(_("Name"), [validators.DataRequired()])
     description = fields.MarkdownField(
-        _('Description'),
-        description=_('Some optional details about this harvester'))
-    url = fields.URLField(_('URL'), [validators.DataRequired()])
-    backend = fields.SelectField(_('Backend'), choices=lambda: [
-        (b.name, b.display_name) for b in list_backends()
-    ])
+        _("Description"), description=_("Some optional details about this harvester")
+    )
+    url = fields.URLField(_("URL"), [validators.DataRequired()])
+    backend = fields.SelectField(
+        _("Backend"), choices=lambda: [(b.name, b.display_name) for b in list_backends()]
+    )
     owner = fields.CurrentUserField()
-    organization = fields.PublishAsField(_('Publish as'))
+    organization = fields.PublishAsField(_("Publish as"))
     active = fields.BooleanField()
     autoarchive = fields.BooleanField()
 
@@ -75,7 +75,6 @@ class HarvestSourceForm(Form):
 
 class HarvestSourceValidationForm(Form):
     state = fields.SelectField(choices=list(VALIDATION_STATES.items()))
-    comment = fields.StringField(_('Comment'),
-                                 [validators.RequiredIfVal('state',
-                                                           VALIDATION_REFUSED
-                                                           )])
+    comment = fields.StringField(
+        _("Comment"), [validators.RequiredIfVal("state", VALIDATION_REFUSED)]
+    )

udata/harvest/models.py

@@ -1,65 +1,75 @@
+import logging
 from collections import OrderedDict
 from datetime import datetime
-import logging
 from urllib.parse import urlparse
 
-from udata.core.dataservices.models import Dataservice
 from werkzeug.utils import cached_property
 
+from udata.core.dataservices.models import Dataservice
 from udata.core.dataset.models import HarvestDatasetMetadata
-from udata.models import db, Dataset
-from udata.i18n import lazy_gettext as _
 from udata.core.owned import Owned, OwnedQuerySet
+from udata.i18n import lazy_gettext as _
+from udata.models import Dataset, db
 
 log = logging.getLogger(__name__)
 
-HARVEST_FREQUENCIES = OrderedDict((
-    ('manual', _('Manual')),
-    ('monthly', _('Monthly')),
-    ('weekly', _('Weekly')),
-    ('daily', _('Daily')),
-))
-
-HARVEST_JOB_STATUS = OrderedDict((
-    ('pending', _('Pending')),
-    ('initializing', _('Initializing')),
-    ('initialized', _('Initialized')),
-    ('processing', _('Processing')),
-    ('done', _('Done')),
-    ('done-errors', _('Done with errors')),
-    ('failed', _('Failed')),
-))
-
-HARVEST_ITEM_STATUS = OrderedDict((
-    ('pending', _('Pending')),
-    ('started', _('Started')),
-    ('done', _('Done')),
-    ('failed', _('Failed')),
-    ('skipped', _('Skipped')),
-    ('archived', _('Archived'))
-))
-
-DEFAULT_HARVEST_FREQUENCY = 'manual'
-DEFAULT_HARVEST_JOB_STATUS = 'pending'
-DEFAULT_HARVEST_ITEM_STATUS = 'pending'
+HARVEST_FREQUENCIES = OrderedDict(
+    (
+        ("manual", _("Manual")),
+        ("monthly", _("Monthly")),
+        ("weekly", _("Weekly")),
+        ("daily", _("Daily")),
+    )
+)
+
+HARVEST_JOB_STATUS = OrderedDict(
+    (
+        ("pending", _("Pending")),
+        ("initializing", _("Initializing")),
+        ("initialized", _("Initialized")),
+        ("processing", _("Processing")),
+        ("done", _("Done")),
+        ("done-errors", _("Done with errors")),
+        ("failed", _("Failed")),
+    )
+)
+
+HARVEST_ITEM_STATUS = OrderedDict(
+    (
+        ("pending", _("Pending")),
+        ("started", _("Started")),
+        ("done", _("Done")),
+        ("failed", _("Failed")),
+        ("skipped", _("Skipped")),
+        ("archived", _("Archived")),
+    )
+)
+
+DEFAULT_HARVEST_FREQUENCY = "manual"
+DEFAULT_HARVEST_JOB_STATUS = "pending"
+DEFAULT_HARVEST_ITEM_STATUS = "pending"
 
 
 class HarvestError(db.EmbeddedDocument):
-    '''Store harvesting errors'''
+    """Store harvesting errors"""
+
     created_at = db.DateTimeField(default=datetime.utcnow, required=True)
     message = db.StringField()
     details = db.StringField()
 
+
 class HarvestLog(db.EmbeddedDocument):
     level = db.StringField()
     message = db.StringField()
 
+
 class HarvestItem(db.EmbeddedDocument):
     remote_id = db.StringField()
     dataset = db.ReferenceField(Dataset)
     dataservice = db.ReferenceField(Dataservice)
-    status = db.StringField(choices=list(HARVEST_ITEM_STATUS),
-                            default=DEFAULT_HARVEST_ITEM_STATUS, required=True)
+    status = db.StringField(
+        choices=list(HARVEST_ITEM_STATUS), default=DEFAULT_HARVEST_ITEM_STATUS, required=True
+    )
     created = db.DateTimeField(default=datetime.utcnow, required=True)
     started = db.DateTimeField()
     ended = db.DateTimeField()
@@ -69,23 +79,24 @@ class HarvestItem(db.EmbeddedDocument):
     kwargs = db.DictField()
 
 
-VALIDATION_ACCEPTED = 'accepted'
-VALIDATION_REFUSED = 'refused'
-VALIDATION_PENDING = 'pending'
+VALIDATION_ACCEPTED = "accepted"
+VALIDATION_REFUSED = "refused"
+VALIDATION_PENDING = "pending"
 
 VALIDATION_STATES = {
-    VALIDATION_PENDING: _('Pending'),
-    VALIDATION_ACCEPTED: _('Accepted'),
-    VALIDATION_REFUSED: _('Refused'),
+    VALIDATION_PENDING: _("Pending"),
+    VALIDATION_ACCEPTED: _("Accepted"),
+    VALIDATION_REFUSED: _("Refused"),
 }
 
 
 class HarvestSourceValidation(db.EmbeddedDocument):
-    '''Store harvest source validation details'''
-    state = db.StringField(choices=list(VALIDATION_STATES),
-                           default=VALIDATION_PENDING,
-                           required=True)
-    by = db.ReferenceField('User')
+    """Store harvest source validation details"""
+
+    state = db.StringField(
+        choices=list(VALIDATION_STATES), default=VALIDATION_PENDING, required=True
+    )
+    by = db.ReferenceField("User")
     on = db.DateTimeField()
     comment = db.StringField()
 
@@ -97,29 +108,28 @@ class HarvestSourceQuerySet(OwnedQuerySet):
 
 class HarvestSource(Owned, db.Document):
     name = db.StringField(max_length=255)
-    slug = db.SlugField(max_length=255, required=True, unique=True,
-                        populate_from='name', update=True)
+    slug = db.SlugField(
+        max_length=255, required=True, unique=True, populate_from="name", update=True
+    )
     description = db.StringField()
     url = db.StringField(required=True)
     backend = db.StringField(required=True)
     config = db.DictField()
-    periodic_task = db.ReferenceField('PeriodicTask',
-                                      reverse_delete_rule=db.NULLIFY)
+    periodic_task = db.ReferenceField("PeriodicTask", reverse_delete_rule=db.NULLIFY)
     created_at = db.DateTimeField(default=datetime.utcnow, required=True)
-    frequency = db.StringField(choices=list(HARVEST_FREQUENCIES),
-                               default=DEFAULT_HARVEST_FREQUENCY,
-                               required=True)
+    frequency = db.StringField(
+        choices=list(HARVEST_FREQUENCIES), default=DEFAULT_HARVEST_FREQUENCY, required=True
+    )
     active = db.BooleanField(default=True)
     autoarchive = db.BooleanField(default=True)
-    validation = db.EmbeddedDocumentField(HarvestSourceValidation,
-                                          default=HarvestSourceValidation)
+    validation = db.EmbeddedDocumentField(HarvestSourceValidation, default=HarvestSourceValidation)
 
     deleted = db.DateTimeField()
 
     @property
     def domain(self):
         parsed = urlparse(self.url)
-        return parsed.netloc.split(':')[0]
+        return parsed.netloc.split(":")[0]
 
     @classmethod
     def get(cls, ident):
@@ -128,9 +138,9 @@ class HarvestSource(Owned, db.Document):
     def get_last_job(self, reduced=False):
         qs = HarvestJob.objects(source=self)
         if reduced:
-            qs = qs.exclude('source', 'items', 'errors', 'data')
+            qs = qs.exclude("source", "items", "errors", "data")
             qs = qs.no_dereference()
-        return qs.order_by('-created').first()
+        return qs.order_by("-created").first()
 
     @cached_property
     def last_job(self):
@@ -143,48 +153,46 @@ class HarvestSource(Owned, db.Document):
         return self.periodic_task.schedule_display
 
     meta = {
-        'indexes': [
-            '-created_at',
-            'slug',
-            ('deleted', '-created_at'),
-        ] + Owned.meta['indexes'],
-        'ordering': ['-created_at'],
-        'queryset_class': HarvestSourceQuerySet,
+        "indexes": [
+            "-created_at",
+            "slug",
+            ("deleted", "-created_at"),
+        ]
+        + Owned.meta["indexes"],
+        "ordering": ["-created_at"],
+        "queryset_class": HarvestSourceQuerySet,
     }
 
     def __str__(self):
-        return self.name or ''
+        return self.name or ""
 
 
 class HarvestJob(db.Document):
-    '''Keep track of harvestings'''
+    """Keep track of harvestings"""
+
     created = db.DateTimeField(default=datetime.utcnow, required=True)
     started = db.DateTimeField()
     ended = db.DateTimeField()
-    status = db.StringField(choices=list(HARVEST_JOB_STATUS),
-                            default=DEFAULT_HARVEST_JOB_STATUS, required=True)
+    status = db.StringField(
+        choices=list(HARVEST_JOB_STATUS), default=DEFAULT_HARVEST_JOB_STATUS, required=True
+    )
     errors = db.ListField(db.EmbeddedDocumentField(HarvestError))
     items = db.ListField(db.EmbeddedDocumentField(HarvestItem))
     source = db.ReferenceField(HarvestSource, reverse_delete_rule=db.CASCADE)
     data = db.DictField()
 
     meta = {
-        'indexes': [
-            '-created',
-            'source',
-            ('source', '-created'),
-            'items.dataset'
-        ],
-        'ordering': ['-created'],
+        "indexes": ["-created", "source", ("source", "-created"), "items.dataset"],
+        "ordering": ["-created"],
     }
 
 
 def archive_harvested_dataset(dataset, reason, dryrun=False):
-    '''
+    """
     Archive an harvested dataset, setting extras accordingly.
     If `dryrun` is True, the dataset is not saved but validated only.
-    '''
-    log.debug('Archiving dataset %s', dataset.id)
+    """
+    log.debug("Archiving dataset %s", dataset.id)
     archival_date = datetime.utcnow()
     dataset.archived = archival_date
     if not dataset.harvest:

udata/harvest/notifications.py

@@ -1,15 +1,15 @@
-from udata.features.notifications.actions import notifier
+import logging
 
-from .models import HarvestSource, VALIDATION_PENDING
+from udata.features.notifications.actions import notifier
 
-import logging
+from .models import VALIDATION_PENDING, HarvestSource
 
 log = logging.getLogger(__name__)
 
 
-@notifier('validate_harvester')
+@notifier("validate_harvester")
 def validate_harvester_notifications(user):
-    '''Notify admins about pending harvester validation'''
+    """Notify admins about pending harvester validation"""
     if not user.sysadmin:
         return []
 
@@ -18,12 +18,17 @@ def validate_harvester_notifications(user):
     # Only fetch required fields for notification serialization
     # Greatly improve performances and memory usage
     qs = HarvestSource.objects(validation__state=VALIDATION_PENDING)
-    qs = qs.only('id', 'created_at', 'name')
+    qs = qs.only("id", "created_at", "name")
 
     for source in qs:
-        notifications.append((source.created_at, {
-            'id': source.id,
-            'name': source.name,
-        }))
+        notifications.append(
+            (
+                source.created_at,
+                {
+                    "id": source.id,
+                    "name": source.name,
+                },
+            )
+        )
 
     return notifications

udata/harvest/signals.py

@@ -7,40 +7,40 @@ log = logging.getLogger(__name__)
 ns = Namespace()
 
 #: Sent when a new HarvestSource is created
-harvest_source_created = ns.signal('harvest:source-created')
+harvest_source_created = ns.signal("harvest:source-created")
 
 #: Sent when a HarvestSource is updated
-harvest_source_updated = ns.signal('harvest:source-updated')
+harvest_source_updated = ns.signal("harvest:source-updated")
 
 #: Sent when a HarvestSource is deleted
-harvest_source_deleted = ns.signal('harvest:source-deleted')
+harvest_source_deleted = ns.signal("harvest:source-deleted")
 
 #: Run before each harvest job
-before_harvest_job = ns.signal('harvest:before-job')
+before_harvest_job = ns.signal("harvest:before-job")
 
 #: Run before each harvest job
-after_harvest_job = ns.signal('harvest:after-job')
+after_harvest_job = ns.signal("harvest:after-job")
 
 #: Sent when a new HarvestJob started
-harvest_job_started = ns.signal('harvest:job-started')
+harvest_job_started = ns.signal("harvest:job-started")
 
 #: Sent when a HarvestJob is done
-harvest_job_done = ns.signal('harvest:job-done')
+harvest_job_done = ns.signal("harvest:job-done")
 
 #: Sent when a HarvestJob failed
-harvest_job_failed = ns.signal('harvest:job-failed')
+harvest_job_failed = ns.signal("harvest:job-failed")
 
 #: Sent when a HarvestItem start
-harvest_item_started = ns.signal('harvest:item-started')
+harvest_item_started = ns.signal("harvest:item-started")
 
 #: Sent when a HarvestItem is done
-harvest_item_done = ns.signal('harvest:item-done')
+harvest_item_done = ns.signal("harvest:item-done")
 
 #: Sent when a HarvestItem failed
-harvest_item_failed = ns.signal('harvest:item-failed')
+harvest_item_failed = ns.signal("harvest:item-failed")
 
 #: Sent when a HarvestSource is scheduled
-harvest_source_scheduled = ns.signal('harvest:source-scheduled')
+harvest_source_scheduled = ns.signal("harvest:source-scheduled")
 
 #: Sent when a HarvestSource is unscheduled
-harvest_source_unscheduled = ns.signal('harvest:source-unscheduled')
+harvest_source_unscheduled = ns.signal("harvest:source-unscheduled")

udata/harvest/tasks.py

@@ -1,15 +1,15 @@
 from celery import chord
 from flask import current_app
 
-from udata.tasks import job, get_logger, task
+from udata.tasks import get_logger, job, task
 
 from . import backends
-from .models import HarvestSource, HarvestJob
+from .models import HarvestJob, HarvestSource
 
 log = get_logger(__name__)
 
 
-@job('harvest', route='low.harvest')
+@job("harvest", route="low.harvest")
 def harvest(self, ident):
     log.info('Launching harvest job for source "%s"', ident)
 
@@ -21,9 +21,8 @@ def harvest(self, ident):
 
     backend.harvest()
 
-    
 
-@task(ignore_result=False, route='low.harvest')
+@task(ignore_result=False, route="low.harvest")
 def harvest_job_item(job_id, item_id):
     log.info('Harvesting item %s for job "%s"', item_id, job_id)
 
@@ -37,7 +36,7 @@ def harvest_job_item(job_id, item_id):
     return item_id
 
 
-@task(ignore_result=False, route='low.harvest')
+@task(ignore_result=False, route="low.harvest")
 def harvest_job_finalize(results, job_id):
     log.info('Finalize harvesting for job "%s"', job_id)
     job = HarvestJob.objects.get(pk=job_id)
@@ -46,15 +45,17 @@ def harvest_job_finalize(results, job_id):
     backend.finalize()
 
 
-@job('purge-harvesters', route='low.harvest')
+@job("purge-harvesters", route="low.harvest")
 def purge_harvest_sources(self):
-    log.info('Purging HarvestSources flagged as deleted')
+    log.info("Purging HarvestSources flagged as deleted")
     from .actions import purge_sources
+
     purge_sources()
 
 
-@job('purge-harvest-jobs', route='low.harvest')
+@job("purge-harvest-jobs", route="low.harvest")
 def purge_harvest_jobs(self):
-    log.info('Purging HarvestJobs older than retention policy')
+    log.info("Purging HarvestJobs older than retention policy")
     from .actions import purge_jobs
+
     purge_jobs()