udata 9.1.2.dev30355__py2.py3-none-any.whl → 9.1.2.dev30382__py2.py3-none-any.whl

udata/core/dataset/commands.py

@@ -1,42 +1,44 @@
-import click
 import json
 import logging
-import requests
 
+import click
+import requests
 from bson import ObjectId
 
-from udata.commands import cli, success, exit_with_error
-from udata.models import License, Dataset
+from udata.commands import cli, exit_with_error, success
 from udata.core.dataset.constants import DEFAULT_LICENSE
-from .tasks import send_frequency_reminder
+from udata.models import Dataset, License
+
 from . import actions
+from .tasks import send_frequency_reminder
 
 log = logging.getLogger(__name__)
 
 # Use CKAN license group from opendefinition as default license list
-DEFAULT_LICENSE_FILE = 'http://licenses.opendefinition.org/licenses/groups/ckan.json'  # noqa
+DEFAULT_LICENSE_FILE = "http://licenses.opendefinition.org/licenses/groups/ckan.json"  # noqa
 
 FLAGS_MAP = {
-    'domain_content': 'domain_content',
-    'domain_data': 'domain_data',
-    'domain_software': 'domain_software',
-    'is_generic': 'generic',
-    'is_okd_compliant': 'okd_compliant',
-    'is_osi_compliant': 'osi_compliant',
+    "domain_content": "domain_content",
+    "domain_data": "domain_data",
+    "domain_software": "domain_software",
+    "is_generic": "generic",
+    "is_okd_compliant": "okd_compliant",
+    "is_osi_compliant": "osi_compliant",
 }
 
+
 @cli.command()
-@click.argument('source', default=DEFAULT_LICENSE_FILE)
+@click.argument("source", default=DEFAULT_LICENSE_FILE)
 def licenses(source=DEFAULT_LICENSE_FILE):
-    '''Feed the licenses from a JSON file'''
-    if source.startswith('http'):
+    """Feed the licenses from a JSON file"""
+    if source.startswith("http"):
         json_licenses = requests.get(source).json()
     else:
         with open(source) as fp:
             json_licenses = json.load(fp)
 
     if len(json_licenses):
-        log.info('Dropping existing licenses')
+        log.info("Dropping existing licenses")
         License.drop_collection()
 
     for json_license in json_licenses:
@@ -46,22 +48,22 @@ def licenses(source=DEFAULT_LICENSE_FILE):
                 flags.append(flag)
 
         license = License.objects.create(
-            id=json_license['id'],
-            title=json_license['title'],
-            url=json_license['url'] or None,
-            maintainer=json_license['maintainer'] or None,
+            id=json_license["id"],
+            title=json_license["title"],
+            url=json_license["url"] or None,
+            maintainer=json_license["maintainer"] or None,
             flags=flags,
-            active=json_license.get('active', False),
-            alternate_urls=json_license.get('alternate_urls', []),
-            alternate_titles=json_license.get('alternate_titles', []),
+            active=json_license.get("active", False),
+            alternate_urls=json_license.get("alternate_urls", []),
+            alternate_titles=json_license.get("alternate_titles", []),
         )
         log.info('Added license "%s"', license.title)
     try:
-        License.objects.get(id=DEFAULT_LICENSE['id'])
+        License.objects.get(id=DEFAULT_LICENSE["id"])
     except License.DoesNotExist:
         License.objects.create(**DEFAULT_LICENSE)
-        log.info('Added license "%s"', DEFAULT_LICENSE['title'])
-    success('Done')
+        log.info('Added license "%s"', DEFAULT_LICENSE["title"])
+    success("Done")
 
 
 @cli.command()
@@ -73,33 +75,33 @@ def frequency_reminder():
     send_frequency_reminder()
 
 
-@cli.group('dataset')
+@cli.group("dataset")
 def grp():
-    '''Dataset related operations'''
+    """Dataset related operations"""
     pass
 
 
 @grp.command()
-@click.argument('dataset_id')
-@click.option('-c', '--comment', is_flag=True, help='Post a comment when archiving')
+@click.argument("dataset_id")
+@click.option("-c", "--comment", is_flag=True, help="Post a comment when archiving")
 def archive_one(dataset_id, comment):
     """Archive one dataset"""
     try:
         dataset = Dataset.objects.get(id=dataset_id)
     except Dataset.DoesNotExist:
-        exit_with_error('Cannot find a dataset with id %s' % dataset_id)
+        exit_with_error("Cannot find a dataset with id %s" % dataset_id)
     else:
         actions.archive(dataset, comment)
 
 
 @grp.command()
-@click.argument('filepath')
-@click.option('-c', '--comment', is_flag=True, help='Post a comment when archiving')
+@click.argument("filepath")
+@click.option("-c", "--comment", is_flag=True, help="Post a comment when archiving")
 def archive(filepath, comment):
     """Archive multiple datasets from a list in a file (one id per line)"""
     count = 0
     errors = 0
-    log.info('Archiving datasets...')
+    log.info("Archiving datasets...")
     with open(filepath) as inputfile:
         for line in inputfile.readlines():
             line = line.rstrip()
@@ -108,10 +110,10 @@ def archive(filepath, comment):
             try:
                 dataset = Dataset.objects.get(id=ObjectId(line))
             except Exception as e:  # noqa  (Never stop on failure)
-                log.error('Unable to archive dataset %s: %s', line, e)
+                log.error("Unable to archive dataset %s: %s", line, e)
                 errors += 1
                 continue
             else:
                 actions.archive(dataset, comment)
                 count += 1
-    log.info('Archived %s datasets, %s failed', count, errors)
+    log.info("Archived %s datasets, %s failed", count, errors)

udata/core/dataset/constants.py

@@ -1,75 +1,82 @@
 from collections import OrderedDict
+
 from udata.i18n import lazy_gettext as _
 
 #: Udata frequencies with their labels
 #:
 #: See: http://dublincore.org/groups/collections/frequency/
-UPDATE_FREQUENCIES = OrderedDict([                              # Dublin core equivalent
-    ('unknown', _('Unknown')),                        # N/A
-    ('punctual', _('Punctual')),                      # N/A
-    ('continuous', _('Real time')),                   # freq:continuous
-    ('hourly', _('Hourly')),                          # N/A
-    ('fourTimesADay', _('Four times a day')),         # N/A
-    ('threeTimesADay', _('Three times a day')),       # N/A
-    ('semidaily', _('Semidaily')),                    # N/A
-    ('daily', _('Daily')),                            # freq:daily
-    ('fourTimesAWeek', _('Four times a week')),       # N/A
-    ('threeTimesAWeek', _('Three times a week')),     # freq:threeTimesAWeek
-    ('semiweekly', _('Semiweekly')),                  # freq:semiweekly
-    ('weekly', _('Weekly')),                          # freq:weekly
-    ('biweekly', _('Biweekly')),                      # freq:bimonthly
-    ('threeTimesAMonth', _('Three times a month')),   # freq:threeTimesAMonth
-    ('semimonthly', _('Semimonthly')),                # freq:semimonthly
-    ('monthly', _('Monthly')),                        # freq:monthly
-    ('bimonthly', _('Bimonthly')),                    # freq:bimonthly
-    ('quarterly', _('Quarterly')),                    # freq:quarterly
-    ('threeTimesAYear', _('Three times a year')),     # freq:threeTimesAYear
-    ('semiannual', _('Biannual')),                    # freq:semiannual
-    ('annual', _('Annual')),                          # freq:annual
-    ('biennial', _('Biennial')),                      # freq:biennial
-    ('triennial', _('Triennial')),                    # freq:triennial
-    ('quinquennial', _('Quinquennial')),              # N/A
-    ('irregular', _('Irregular')),                    # freq:irregular
-])
+UPDATE_FREQUENCIES = OrderedDict(
+    [  # Dublin core equivalent
+        ("unknown", _("Unknown")),  # N/A
+        ("punctual", _("Punctual")),  # N/A
+        ("continuous", _("Real time")),  # freq:continuous
+        ("hourly", _("Hourly")),  # N/A
+        ("fourTimesADay", _("Four times a day")),  # N/A
+        ("threeTimesADay", _("Three times a day")),  # N/A
+        ("semidaily", _("Semidaily")),  # N/A
+        ("daily", _("Daily")),  # freq:daily
+        ("fourTimesAWeek", _("Four times a week")),  # N/A
+        ("threeTimesAWeek", _("Three times a week")),  # freq:threeTimesAWeek
+        ("semiweekly", _("Semiweekly")),  # freq:semiweekly
+        ("weekly", _("Weekly")),  # freq:weekly
+        ("biweekly", _("Biweekly")),  # freq:bimonthly
+        ("threeTimesAMonth", _("Three times a month")),  # freq:threeTimesAMonth
+        ("semimonthly", _("Semimonthly")),  # freq:semimonthly
+        ("monthly", _("Monthly")),  # freq:monthly
+        ("bimonthly", _("Bimonthly")),  # freq:bimonthly
+        ("quarterly", _("Quarterly")),  # freq:quarterly
+        ("threeTimesAYear", _("Three times a year")),  # freq:threeTimesAYear
+        ("semiannual", _("Biannual")),  # freq:semiannual
+        ("annual", _("Annual")),  # freq:annual
+        ("biennial", _("Biennial")),  # freq:biennial
+        ("triennial", _("Triennial")),  # freq:triennial
+        ("quinquennial", _("Quinquennial")),  # N/A
+        ("irregular", _("Irregular")),  # freq:irregular
+    ]
+)
 
 #: Map legacy frequencies to currents
 LEGACY_FREQUENCIES = {
-    'fortnighly': 'biweekly',
-    'biannual': 'semiannual',
-    'realtime': 'continuous',
+    "fortnighly": "biweekly",
+    "biannual": "semiannual",
+    "realtime": "continuous",
 }
 
-DEFAULT_FREQUENCY = 'unknown'
+DEFAULT_FREQUENCY = "unknown"
 
 DEFAULT_LICENSE = {
-    'id': 'notspecified',
-    'title': "License Not Specified",
-    'flags': ["generic"],
-    'maintainer': None,
-    'url': None,
-    'active': True,
+    "id": "notspecified",
+    "title": "License Not Specified",
+    "flags": ["generic"],
+    "maintainer": None,
+    "url": None,
+    "active": True,
 }
 
-RESOURCE_TYPES = OrderedDict([
-    ('main', _('Main file')),
-    ('documentation', _('Documentation')),
-    ('update', _('Update')),
-    ('api', _('API')),
-    ('code', _('Code repository')),
-    ('other', _('Other')),
-])
+RESOURCE_TYPES = OrderedDict(
+    [
+        ("main", _("Main file")),
+        ("documentation", _("Documentation")),
+        ("update", _("Update")),
+        ("api", _("API")),
+        ("code", _("Code repository")),
+        ("other", _("Other")),
+    ]
+)
 
-RESOURCE_FILETYPE_FILE = 'file'
-RESOURCE_FILETYPES = OrderedDict([
-    (RESOURCE_FILETYPE_FILE, _('Uploaded file')),
-    ('remote', _('Remote file')),
-])
+RESOURCE_FILETYPE_FILE = "file"
+RESOURCE_FILETYPES = OrderedDict(
+    [
+        (RESOURCE_FILETYPE_FILE, _("Uploaded file")),
+        ("remote", _("Remote file")),
+    ]
+)
 
-CHECKSUM_TYPES = ('sha1', 'sha2', 'sha256', 'md5', 'crc')
-DEFAULT_CHECKSUM_TYPE = 'sha1'
+CHECKSUM_TYPES = ("sha1", "sha2", "sha256", "md5", "crc")
+DEFAULT_CHECKSUM_TYPE = "sha1"
 
-PIVOTAL_DATA = 'pivotal-data'
-CLOSED_FORMATS = ('pdf', 'doc', 'docx', 'word', 'xls', 'excel', 'xlsx')
+PIVOTAL_DATA = "pivotal-data"
+CLOSED_FORMATS = ("pdf", "doc", "docx", "word", "xls", "excel", "xlsx")
 
 # Maximum acceptable Damerau-Levenshtein distance
 # used to guess license

udata/core/dataset/csv.py

@@ -6,42 +6,42 @@ from .models import Dataset, Resource
 
 def serialize_spatial_zones(dataset):
     if dataset.spatial and dataset.spatial.zones:
-        return ','.join(z.name for z in dataset.spatial.zones)
+        return ",".join(z.name for z in dataset.spatial.zones)
 
 
 @csv.adapter(Dataset)
 class DatasetCsvAdapter(csv.Adapter):
     fields = (
-        'id',
-        'title',
-        'slug',
-        'acronym',
-        ('url', 'external_url'),
-        ('organization', 'organization.name'),
-        ('organization_id', 'organization.id'),
-        ('owner', 'owner.slug'),  # in case it's owned by a user, or introduce 'owner_type'?
-        ('owner_id', 'owner.id'),
+        "id",
+        "title",
+        "slug",
+        "acronym",
+        ("url", "external_url"),
+        ("organization", "organization.name"),
+        ("organization_id", "organization.id"),
+        ("owner", "owner.slug"),  # in case it's owned by a user, or introduce 'owner_type'?
+        ("owner_id", "owner.id"),
         # 'contact_point', #  ?
-        'description',
-        'frequency',
-        'license',
-        'temporal_coverage.start',
-        'temporal_coverage.end',
-        'spatial.granularity',
-        ('spatial.zones', serialize_spatial_zones),
-        ('featured', lambda o: o.featured or False),
-        'created_at',
-        'last_modified',
-        ('tags', lambda o: ','.join(o.tags)),
-        ('archived', lambda o: o.archived or False),
-        ('resources_count', lambda o: len(o.resources)),
-        ('main_resources_count', lambda o: len([r for r in o.resources if r.type == 'main'])),
-        'downloads',
-        ('harvest.backend', lambda r: r.harvest and r.harvest.backend),
-        ('harvest.domain', lambda r: r.harvest and r.harvest.domain),
-        ('harvest.created_at', lambda r: r.harvest and r.harvest.created_at),
-        ('harvest.modified_at', lambda r: r.harvest and r.harvest.modified_at),
-        ('quality_score', lambda o: format(o.quality['score'], '.2f')),
+        "description",
+        "frequency",
+        "license",
+        "temporal_coverage.start",
+        "temporal_coverage.end",
+        "spatial.granularity",
+        ("spatial.zones", serialize_spatial_zones),
+        ("featured", lambda o: o.featured or False),
+        "created_at",
+        "last_modified",
+        ("tags", lambda o: ",".join(o.tags)),
+        ("archived", lambda o: o.archived or False),
+        ("resources_count", lambda o: len(o.resources)),
+        ("main_resources_count", lambda o: len([r for r in o.resources if r.type == "main"])),
+        "downloads",
+        ("harvest.backend", lambda r: r.harvest and r.harvest.backend),
+        ("harvest.domain", lambda r: r.harvest and r.harvest.domain),
+        ("harvest.created_at", lambda r: r.harvest and r.harvest.created_at),
+        ("harvest.modified_at", lambda r: r.harvest and r.harvest.modified_at),
+        ("quality_score", lambda o: format(o.quality["score"], ".2f")),
         # schema? what is the schema of a dataset?
     )
 
@@ -50,62 +50,58 @@ class DatasetCsvAdapter(csv.Adapter):
 
 
 def dataset_field(name, getter=None):
-    return ('dataset.{0}'.format(name), getter or name)
+    return ("dataset.{0}".format(name), getter or name)
 
 
 @csv.adapter(Resource)
 class ResourcesCsvAdapter(csv.NestedAdapter):
     fields = (
-        dataset_field('id'),
-        dataset_field('title'),
-        dataset_field('slug'),
-        dataset_field('url', 'external_url'),
+        dataset_field("id"),
+        dataset_field("title"),
+        dataset_field("slug"),
+        dataset_field("url", "external_url"),
+        dataset_field("organization", lambda r: r.organization.name if r.organization else None),
         dataset_field(
-            'organization',
-            lambda r: r.organization.name if r.organization else None),
-        dataset_field(
-            'organization_id',
-            lambda r: str(r.organization.id) if r.organization else None),
-        dataset_field('license'),
-        dataset_field('private'),
-        dataset_field(
-            'archived',
-            lambda r: r.archived or False),
+            "organization_id", lambda r: str(r.organization.id) if r.organization else None
+        ),
+        dataset_field("license"),
+        dataset_field("private"),
+        dataset_field("archived", lambda r: r.archived or False),
     )
     nested_fields = (
-        'id',
-        'url',
-        'title',
-        'description',
-        'filetype',
-        'type',
-        'format',
-        'mime',
-        'filesize',
-        ('checksum.type', lambda o: getattr(o.checksum, 'type', None)),
-        ('checksum.value', lambda o: getattr(o.checksum, 'value', None)),
-        'created_at',
-        ('modified', lambda o: o.last_modified),
-        ('downloads', lambda o: int(o.metrics.get('views', 0))),
-        ('harvest.created_at', lambda o: o.harvest and o.harvest.created_at),
-        ('harvest.modified_at', lambda o: o.harvest and o.harvest.modified_at),
-        ('schema_name', 'schema.name'),
-        ('schema_version', 'schema.version'),
-        ('preview_url', lambda o: o.preview_url or False),
+        "id",
+        "url",
+        "title",
+        "description",
+        "filetype",
+        "type",
+        "format",
+        "mime",
+        "filesize",
+        ("checksum.type", lambda o: getattr(o.checksum, "type", None)),
+        ("checksum.value", lambda o: getattr(o.checksum, "value", None)),
+        "created_at",
+        ("modified", lambda o: o.last_modified),
+        ("downloads", lambda o: int(o.metrics.get("views", 0))),
+        ("harvest.created_at", lambda o: o.harvest and o.harvest.created_at),
+        ("harvest.modified_at", lambda o: o.harvest and o.harvest.modified_at),
+        ("schema_name", "schema.name"),
+        ("schema_version", "schema.version"),
+        ("preview_url", lambda o: o.preview_url or False),
     )
-    attribute = 'resources'
+    attribute = "resources"
 
 
 @csv.adapter(Discussion)
 class DiscussionCsvAdapter(csv.Adapter):
     fields = (
-        'id',
-        'user',
-        'subject',
-        'title',
-        ('size', lambda o: len(o.discussion)),
-        ('messages', lambda o: '\n'.join(msg.content for msg in o.discussion)),
-        'created',
-        'closed',
-        'closed_by',
+        "id",
+        "user",
+        "subject",
+        "title",
+        ("size", lambda o: len(o.discussion)),
+        ("messages", lambda o: "\n".join(msg.content for msg in o.discussion)),
+        "created",
+        "closed",
+        "closed_by",
     )

udata/core/dataset/events.py

@@ -1,52 +1,53 @@
 import datetime
+
 import requests
 from flask import current_app
 
-from udata.utils import to_iso_datetime, get_by
+from udata.event.values import EventMessageType
 from udata.models import Dataset
 from udata.tasks import task
-from udata.event.values import EventMessageType
+from udata.utils import get_by, to_iso_datetime
 
 
 def serialize_resource_for_event(resource):
     resource_dict = {
-        'id': str(resource.id),
-        'url': resource.url,
-        'format': resource.format,
-        'title': resource.title,
-        'schema': resource.schema.to_dict() if resource.schema else None,
-        'description': resource.description,
-        'filetype': resource.filetype,
-        'type': resource.type,
-        'mime': resource.mime,
-        'filesize': resource.filesize,
-        'checksum_type': resource.checksum.type if resource.checksum else None,
-        'checksum_value': resource.checksum.value if resource.checksum else None,
-        'created_at': to_iso_datetime(resource.created_at),
-        'last_modified': to_iso_datetime(resource.last_modified)
+        "id": str(resource.id),
+        "url": resource.url,
+        "format": resource.format,
+        "title": resource.title,
+        "schema": resource.schema.to_dict() if resource.schema else None,
+        "description": resource.description,
+        "filetype": resource.filetype,
+        "type": resource.type,
+        "mime": resource.mime,
+        "filesize": resource.filesize,
+        "checksum_type": resource.checksum.type if resource.checksum else None,
+        "checksum_value": resource.checksum.value if resource.checksum else None,
+        "created_at": to_iso_datetime(resource.created_at),
+        "last_modified": to_iso_datetime(resource.last_modified),
     }
     extras = {}
     for key, value in resource.extras.items():
         extras[key] = to_iso_datetime(value) if isinstance(value, datetime.datetime) else value
-    resource_dict.update({'extras': extras})
+    resource_dict.update({"extras": extras})
     if resource.harvest:
         harvest = {}
         for key, value in resource.harvest._data.items():
             harvest[key] = to_iso_datetime(value) if isinstance(value, datetime.datetime) else value
-        resource_dict.update({'harvest': harvest})
+        resource_dict.update({"harvest": harvest})
     return resource_dict
 
 
-@task(route='high.resource')
+@task(route="high.resource")
 def publish(url, document, resource_id, action):
     if action == EventMessageType.DELETED:
         resource = None
     else:
-        resource = serialize_resource_for_event(get_by(document.resources, 'id', resource_id))
+        resource = serialize_resource_for_event(get_by(document.resources, "id", resource_id))
     payload = {
-        'resource_id': str(resource_id),
-        'dataset_id': str(document.id),
-        'document': resource
+        "resource_id": str(resource_id),
+        "dataset_id": str(document.id),
+        "document": resource,
     }
     r = requests.post(url, json=payload)
     r.raise_for_status()
@@ -54,32 +55,38 @@ def publish(url, document, resource_id, action):
 
 @Dataset.on_resource_added.connect
 def publish_added_resource_message(sender, document, **kwargs):
-    if current_app.config.get('PUBLISH_ON_RESOURCE_EVENTS') and current_app.config.get('RESOURCES_ANALYSER_URI'):
+    if current_app.config.get("PUBLISH_ON_RESOURCE_EVENTS") and current_app.config.get(
+        "RESOURCES_ANALYSER_URI"
+    ):
         publish.delay(
             f"{current_app.config.get('RESOURCES_ANALYSER_URI')}/api/resource/created/",
             document,
-            kwargs['resource_id'],
-            EventMessageType.CREATED
+            kwargs["resource_id"],
+            EventMessageType.CREATED,
         )
 
 
 @Dataset.on_resource_updated.connect
 def publish_updated_resource_message(sender, document, **kwargs):
-    if current_app.config.get('PUBLISH_ON_RESOURCE_EVENTS') and current_app.config.get('RESOURCES_ANALYSER_URI'):
+    if current_app.config.get("PUBLISH_ON_RESOURCE_EVENTS") and current_app.config.get(
+        "RESOURCES_ANALYSER_URI"
+    ):
         publish.delay(
             f"{current_app.config.get('RESOURCES_ANALYSER_URI')}/api/resource/updated/",
             document,
-            kwargs['resource_id'],
-            EventMessageType.MODIFIED
+            kwargs["resource_id"],
+            EventMessageType.MODIFIED,
         )
 
 
 @Dataset.on_resource_removed.connect
 def publish_removed_resource_message(sender, document, **kwargs):
-    if current_app.config.get('PUBLISH_ON_RESOURCE_EVENTS') and current_app.config.get('RESOURCES_ANALYSER_URI'):
+    if current_app.config.get("PUBLISH_ON_RESOURCE_EVENTS") and current_app.config.get(
+        "RESOURCES_ANALYSER_URI"
+    ):
         publish.delay(
             f"{current_app.config.get('RESOURCES_ANALYSER_URI')}/api/resource/deleted/",
             document,
-            kwargs['resource_id'],
-            EventMessageType.DELETED
+            kwargs["resource_id"],
+            EventMessageType.DELETED,
         )

udata/core/dataset/exceptions.py

@@ -1,18 +1,22 @@
 class DatasetException(Exception):
-    '''Base class for all dataset exceptions'''
+    """Base class for all dataset exceptions"""
+
     pass
 
 
 class ResourceSchemaException(DatasetException):
-    '''Raised for resources' schema related exceptions'''
+    """Raised for resources' schema related exceptions"""
+
     pass
 
 
 class SchemasCatalogNotFoundException(ResourceSchemaException):
-    '''Raised when the schema catalog cannot be found'''
+    """Raised when the schema catalog cannot be found"""
+
     pass
 
 
 class SchemasCacheUnavailableException(ResourceSchemaException):
-    '''Raised when the schema catalog cache is not available or is empty'''
+    """Raised when the schema catalog cache is not available or is empty"""
+
     pass