udata 9.1.2.dev30355__py2.py3-none-any.whl → 9.1.2.dev30382__py2.py3-none-any.whl

udata/core/storages/api.py

@@ -1,5 +1,4 @@
 import os
-
 from datetime import datetime
 
 from flask import json
@@ -7,40 +6,39 @@ from werkzeug.datastructures import FileStorage
 
 from udata.api import api, fields
 
-from . import utils, chunks
-
+from . import chunks, utils
 
-META = 'meta.json'
+META = "meta.json"
 
-IMAGES_MIMETYPES = ('image/jpeg', 'image/png', 'image/webp')
+IMAGES_MIMETYPES = ("image/jpeg", "image/png", "image/webp")
 
 
-uploaded_image_fields = api.model('UploadedImage', {
-    'success': fields.Boolean(
-        description='Whether the upload succeeded or not.',
-        readonly=True, default=True),
-    'image': fields.ImageField(),
-})
+uploaded_image_fields = api.model(
+    "UploadedImage",
+    {
+        "success": fields.Boolean(
+            description="Whether the upload succeeded or not.", readonly=True, default=True
+        ),
+        "image": fields.ImageField(),
+    },
+)
 
-chunk_status_fields = api.model('UploadStatus', {
-    'success': fields.Boolean,
-    'error': fields.String
-})
+chunk_status_fields = api.model("UploadStatus", {"success": fields.Boolean, "error": fields.String})
 
 
 image_parser = api.parser()
-image_parser.add_argument('file', type=FileStorage, location='files')
-image_parser.add_argument('bbox', type=str, location='form')
+image_parser.add_argument("file", type=FileStorage, location="files")
+image_parser.add_argument("bbox", type=str, location="form")
 
 
 upload_parser = api.parser()
-upload_parser.add_argument('file', type=FileStorage, location='files')
-upload_parser.add_argument('uuid', type=str, location='form')
-upload_parser.add_argument('filename', type=str, location='form')
-upload_parser.add_argument('partindex', type=int, location='form')
-upload_parser.add_argument('partbyteoffset', type=int, location='form')
-upload_parser.add_argument('totalparts', type=int, location='form')
-upload_parser.add_argument('chunksize', type=int, location='form')
+upload_parser.add_argument("file", type=FileStorage, location="files")
+upload_parser.add_argument("uuid", type=str, location="form")
+upload_parser.add_argument("filename", type=str, location="form")
+upload_parser.add_argument("partindex", type=int, location="form")
+upload_parser.add_argument("partbyteoffset", type=int, location="form")
+upload_parser.add_argument("totalparts", type=int, location="form")
+upload_parser.add_argument("chunksize", type=int, location="form")
 
 
 class UploadStatus(Exception):
@@ -51,22 +49,24 @@ class UploadStatus(Exception):
 
 
 class UploadProgress(UploadStatus):
-    '''Raised on successful chunk uploaded'''
+    """Raised on successful chunk uploaded"""
+
     pass
 
 
 class UploadError(UploadStatus):
-    '''Raised on any upload error'''
+    """Raised on any upload error"""
+
     def __init__(self, error=None):
         super(UploadError, self).__init__(ok=False, error=error)
 
 
 def on_upload_status(status):
-    '''Not an error, just raised when chunk is processed'''
+    """Not an error, just raised when chunk is processed"""
     if status.ok:
-        return {'success': True}, 200
+        return {"success": True}, 200
     else:
-        return {'success': False, 'error': status.error}, 400
+        return {"success": False, "error": status.error}, 400
 
 
 @api.errorhandler(UploadStatus)
@@ -74,7 +74,7 @@ def on_upload_status(status):
 @api.errorhandler(UploadProgress)
 @api.marshal_with(chunk_status_fields, code=200)
 def api_upload_status(status):
-    '''API Upload response handler'''
+    """API Upload response handler"""
     return on_upload_status(status)
 
 
@@ -91,34 +91,40 @@ def get_file_size(file):
 
 def save_chunk(file, args):
     # Check file size
-    if get_file_size(file) != args['chunksize']:
-        raise UploadProgress(ok=False, error='Chunk size mismatch')
-    filename = chunk_filename(args['uuid'], args['partindex'])
+    if get_file_size(file) != args["chunksize"]:
+        raise UploadProgress(ok=False, error="Chunk size mismatch")
+    filename = chunk_filename(args["uuid"], args["partindex"])
     chunks.save(file, filename=filename)
-    meta_filename = chunk_filename(args['uuid'], META)
-    chunks.write(meta_filename, json.dumps({
-        'uuid': str(args['uuid']),
-        'filename': args['filename'],
-        'totalparts': args['totalparts'],
-        'lastchunk': datetime.utcnow(),
-    }), overwrite=True)
+    meta_filename = chunk_filename(args["uuid"], META)
+    chunks.write(
+        meta_filename,
+        json.dumps(
+            {
+                "uuid": str(args["uuid"]),
+                "filename": args["filename"],
+                "totalparts": args["totalparts"],
+                "lastchunk": datetime.utcnow(),
+            }
+        ),
+        overwrite=True,
+    )
     raise UploadProgress()
 
 
 def combine_chunks(storage, args, prefix=None):
-    '''
+    """
     Combine a chunked file into a whole file again.
     Goes through each part, in order,
     and appends that part's bytes to another destination file.
     Chunks are stored in the chunks storage.
-    '''
-    uuid = args['uuid']
+    """
+    uuid = args["uuid"]
     # Normalize filename including extension
-    target = utils.normalize(args['filename'])
+    target = utils.normalize(args["filename"])
     if prefix:
         target = os.path.join(prefix, target)
-    with storage.open(target, 'wb') as out:
-        for i in range(args['totalparts']):
+    with storage.open(target, "wb") as out:
+        for i in range(args["totalparts"]):
             partname = chunk_filename(uuid, i)
             out.write(chunks.read(partname))
             chunks.delete(partname)
@@ -128,8 +134,8 @@ def combine_chunks(storage, args, prefix=None):
 
 def handle_upload(storage, prefix=None):
     args = upload_parser.parse_args()
-    is_chunk = args['totalparts'] and args['totalparts'] > 1
-    uploaded_file = args['file']
+    is_chunk = args["totalparts"] and args["totalparts"] > 1
+    uploaded_file = args["file"]
 
     if is_chunk:
         if uploaded_file:
@@ -137,34 +143,30 @@ def handle_upload(storage, prefix=None):
         else:
             fs_filename = combine_chunks(storage, args, prefix=prefix)
     elif not uploaded_file:
-        raise UploadError('Missing file parameter')
+        raise UploadError("Missing file parameter")
     else:
         # Normalize filename including extension
         filename = utils.normalize(uploaded_file.filename)
-        fs_filename = storage.save(
-            uploaded_file,
-            prefix=prefix,
-            filename=filename
-        )
+        fs_filename = storage.save(uploaded_file, prefix=prefix, filename=filename)
 
     metadata = storage.metadata(fs_filename)
-    metadata['last_modified_internal'] = metadata.pop('modified')
-    metadata['fs_filename'] = fs_filename
-    checksum = metadata.pop('checksum')
-    algo, checksum = checksum.split(':', 1)
+    metadata["last_modified_internal"] = metadata.pop("modified")
+    metadata["fs_filename"] = fs_filename
+    checksum = metadata.pop("checksum")
+    algo, checksum = checksum.split(":", 1)
     metadata[algo] = checksum
-    metadata['format'] = utils.extension(fs_filename)
+    metadata["format"] = utils.extension(fs_filename)
     return metadata
 
 
 def parse_uploaded_image(field):
-    '''Parse an uploaded image and save into a db.ImageField()'''
+    """Parse an uploaded image and save into a db.ImageField()"""
     args = image_parser.parse_args()
 
-    image = args['file']
+    image = args["file"]
     if image.mimetype not in IMAGES_MIMETYPES:
-        api.abort(400, 'Unsupported image format')
-    bbox = args.get('bbox', None)
+        api.abort(400, "Unsupported image format")
+    bbox = args.get("bbox", None)
     if bbox:
-        bbox = [int(float(c)) for c in bbox.split(',')]
+        bbox = [int(float(c)) for c in bbox.split(",")]
     field.save(image, bbox=bbox)

udata/core/storages/tasks.py

@@ -1,6 +1,6 @@
 from datetime import datetime, timedelta
-from dateutil.parser import parse
 
+from dateutil.parser import parse
 from flask import current_app, json
 
 from udata.core.storages import chunks
@@ -10,14 +10,14 @@ from udata.tasks import get_logger, job
 log = get_logger(__name__)
 
 
-@job('purge-chunks')
+@job("purge-chunks")
 def purge_chunks(self):
-    log.info('Purging uploaded chunks')
-    max_retention = timedelta(seconds=current_app.config['UPLOAD_MAX_RETENTION'])
+    log.info("Purging uploaded chunks")
+    max_retention = timedelta(seconds=current_app.config["UPLOAD_MAX_RETENTION"])
     meta_files = (f for f in chunks.list_files() if f.endswith(META))
     for filename in meta_files:
         metadata = json.loads(chunks.read(filename))
-        if datetime.utcnow() - parse(metadata['lastchunk']) >= max_retention:
-            uuid = metadata['uuid']
-            log.info('Removing %s expired chunks', uuid)
+        if datetime.utcnow() - parse(metadata["lastchunk"]) >= max_retention:
+            uuid = metadata["uuid"]
+            log.info("Removing %s expired chunks", uuid)
             chunks.delete(uuid)

udata/core/storages/utils.py

@@ -6,15 +6,15 @@ import zlib
 from flask import current_app
 from slugify import Slugify
 
-CHUNK_SIZE = 2 ** 16
+CHUNK_SIZE = 2**16
 
 
-slugify = Slugify(separator='-', to_lower=True, safe_chars='.')
+slugify = Slugify(separator="-", to_lower=True, safe_chars=".")
 
 
 def hash(file, hasher):
     blk_size_to_read = hasher.block_size * CHUNK_SIZE
-    while (True):
+    while True:
         read_data = file.read(blk_size_to_read)
         if not read_data:
             break
@@ -23,28 +23,28 @@ def hash(file, hasher):
 
 
 def sha1(file):
-    '''Perform a SHA1 digest on file'''
+    """Perform a SHA1 digest on file"""
     return hash(file, hashlib.sha1())
 
 
 def md5(file):
-    '''Perform a MD5 digest on a file'''
+    """Perform a MD5 digest on a file"""
     return hash(file, hashlib.md5())
 
 
 def crc32(file):
-    '''Perform a CRC digest on a file'''
+    """Perform a CRC digest on a file"""
     value = zlib.crc32(file.read())
-    return '%08X' % (value & 0xFFFFFFFF)
+    return "%08X" % (value & 0xFFFFFFFF)
 
 
 def mime(url):
-    '''Get the mimetype from an url or a filename'''
+    """Get the mimetype from an url or a filename"""
     return mimetypes.guess_type(url)[0]
 
 
 def extension(filename):
-    '''
+    """
     Properly extract the extension from filename.
     We keep the last extension except for archive extensions, where we check
     previous extensions as well.
@@ -55,23 +55,23 @@ def extension(filename):
     - test.2022.zip -> zip
     - test.2022.csv.tar.gz -> csv.tar.gz
     - test.geojson.csv -> csv
-    '''
+    """
     filename = os.path.basename(filename)
     extension = None
 
-    while '.' in filename:
+    while "." in filename:
         filename, ext = os.path.splitext(filename)
-        if ext.startswith('.'):
+        if ext.startswith("."):
             ext = ext[1:]
 
-        if extension and ext not in current_app.config['ALLOWED_RESOURCES_EXTENSIONS']:
+        if extension and ext not in current_app.config["ALLOWED_RESOURCES_EXTENSIONS"]:
             # We don't want to add this extension if one has already been detected
             # and this one is not in the allowed resources extensions list.
             break
 
-        extension = ext if not extension else ext + '.' + extension
+        extension = ext if not extension else ext + "." + extension
 
-        if ext not in current_app.config['ALLOWED_ARCHIVE_EXTENSIONS']:
+        if ext not in current_app.config["ALLOWED_ARCHIVE_EXTENSIONS"]:
             # We don't want to continue the loop if this ext is not an allowed archived extension
             break
 

udata/core/storages/views.py

@@ -1,11 +1,10 @@
+import flask_storage as fs
 from flask import Blueprint, jsonify
 from flask_security import login_required
-import flask_storage as fs
-
-from .api import handle_upload, on_upload_status, UploadStatus
 
+from .api import UploadStatus, handle_upload, on_upload_status
 
-blueprint = Blueprint('storage', __name__)
+blueprint = Blueprint("storage", __name__)
 
 
 @blueprint.app_errorhandler(UploadStatus)
@@ -15,8 +14,8 @@ def jsonify_upload_status(status):
 
 
 @login_required
-@blueprint.route('/upload/<name>/', methods=['POST'])
+@blueprint.route("/upload/<name>/", methods=["POST"])
 def upload(name):
-    '''Handle upload on POST if authorized.'''
+    """Handle upload on POST if authorized."""
     storage = fs.by_name(name)
     return jsonify(success=True, **handle_upload(storage))

udata/core/tags/api.py

@@ -1,28 +1,31 @@
-from udata.api import api, API
-
-from udata.tags import slug  # TODO: merge this into this package
+from udata.api import API, api
 from udata.models import Tag
+from udata.tags import slug  # TODO: merge this into this package
 
 DEFAULT_SIZE = 8
 
-ns = api.namespace('tags', 'Tags related operations')
+ns = api.namespace("tags", "Tags related operations")
 
 parser = api.parser()
 parser.add_argument(
-    'q', type=str, help='The string to autocomplete/suggest',
-    location='args', required=True)
+    "q", type=str, help="The string to autocomplete/suggest", location="args", required=True
+)
 parser.add_argument(
-    'size', type=int, help='The amount of suggestion to fetch',
-    location='args', default=DEFAULT_SIZE)
+    "size",
+    type=int,
+    help="The amount of suggestion to fetch",
+    location="args",
+    default=DEFAULT_SIZE,
+)
 
 
-@ns.route('/suggest/', endpoint='suggest_tags')
+@ns.route("/suggest/", endpoint="suggest_tags")
 class SuggestTagsAPI(API):
-    @api.doc('suggest_tags')
+    @api.doc("suggest_tags")
     @api.expect(parser)
     def get(self):
-        '''Suggest tags'''
+        """Suggest tags"""
         args = parser.parse_args()
-        q = slug(args['q'])
-        results = [{'text': i.name} for i in Tag.objects(name__icontains=q).limit(args['size'])]
-        return sorted(results, key=lambda o: len(o['text']))
+        q = slug(args["q"])
+        results = [{"text": i.name} for i in Tag.objects(name__icontains=q).limit(args["size"])]
+        return sorted(results, key=lambda o: len(o["text"]))

udata/core/tags/csv.py

@@ -10,8 +10,8 @@ def counts(name):
 @csv.adapter(Tag)
 class TagCsvAdapter(csv.Adapter):
     fields = (
-        'name',
-        counts('datasets'),
-        counts('reuses'),
-        'total',
+        "name",
+        counts("datasets"),
+        counts("reuses"),
+        "total",
     )

udata/core/tags/models.py

@@ -5,21 +5,24 @@ from udata.mongo import db
 log = logging.getLogger(__name__)
 
 
-__all__ = ('Tag', )
+__all__ = ("Tag",)
 
 
 class Tag(db.Document):
-    '''
+    """
     This collection is auto-populated every hour map-reducing tag properties
     from Datasets dans Reuses.
-    '''
+    """
+
     name = db.StringField(required=True, unique=True)
     counts = db.DictField()
     total = db.IntField(default=0)
 
     meta = {
-        'indexes': ['name', '-total'],
-        'ordering': ['-total', ],
+        "indexes": ["name", "-total"],
+        "ordering": [
+            "-total",
+        ],
     }
 
     def clean(self):

udata/core/tags/tasks.py

@@ -7,15 +7,15 @@ from .models import Tag
 
 log = logging.getLogger(__name__)
 
-map_tags = '''
+map_tags = """
 function() {
     this.tags.forEach(function(tag) {
         emit(tag, 1);
     });
 }
-'''
+"""
 
-reduce_tags = '''
+reduce_tags = """
 function(key, values) {
     var total = 0;
     for(var i = 0; i < values.length; i++) {
@@ -23,23 +23,21 @@ function(key, values) {
     }
     return total;
 }
-'''
+"""
 
 TAGGED = {
-    'datasets': Dataset,
-    'reuses': Reuse,
+    "datasets": Dataset,
+    "reuses": Reuse,
 }
 
 
-@job('count-tags')
+@job("count-tags")
 def count_tags(self):
-    '''Count tag occurences by type and update the tag collection'''
+    """Count tag occurences by type and update the tag collection"""
     for key, model in TAGGED.items():
-        collection = '{0}_tags'.format(key)
-        results = (model.objects(tags__exists=True)
-                        .map_reduce(map_tags, reduce_tags, collection))
+        collection = "{0}_tags".format(key)
+        results = model.objects(tags__exists=True).map_reduce(map_tags, reduce_tags, collection)
         for result in results:
-            tag, created = Tag.objects.get_or_create(name=result.key,
-                                                     auto_save=False)
+            tag, created = Tag.objects.get_or_create(name=result.key, auto_save=False)
             tag.counts[key] = int(result.value) if result.value else 0
             tag.save()

udata/core/tags/views.py

@@ -9,10 +9,10 @@ from .models import Tag
 log = logging.getLogger(__name__)
 
 
-blueprint = I18nBlueprint('tags', __name__)
+blueprint = I18nBlueprint("tags", __name__)
 
 
-@blueprint.route('/tags.csv', endpoint='csv')
+@blueprint.route("/tags.csv", endpoint="csv")
 def tags_csv():
-    adapter = TagCsvAdapter(Tag.objects.order_by('-total'))
-    return csv.stream(adapter, 'tags')
+    adapter = TagCsvAdapter(Tag.objects.order_by("-total"))
+    return csv.stream(adapter, "tags")