udata 9.1.2.dev30355__py2.py3-none-any.whl → 9.1.2.dev30382__py2.py3-none-any.whl

udata/frontend/markdown.py

@@ -1,33 +1,29 @@
+import re
 from functools import partial
 from urllib.parse import urlparse
 
 import bleach
 import html2text
 import mistune
-import re
-
 from bleach.linkifier import LinkifyFilter
-from flask import current_app, Markup, request
+from flask import Markup, current_app, request
+from jinja2.filters import do_striptags, do_truncate
 from werkzeug.local import LocalProxy
-from jinja2.filters import do_truncate, do_striptags
 
 from udata.i18n import _
 
+md = LocalProxy(lambda: current_app.extensions["markdown"])
 
-md = LocalProxy(lambda: current_app.extensions['markdown'])
-
-EXCERPT_TOKEN = '<!--- --- -->'
+EXCERPT_TOKEN = "<!--- --- -->"
 
-RE_AUTOLINK = re.compile(
-    r'<([A-Za-z][A-Za-z0-9.+-]{1,31}:[^<>\x00-\x20]*)>',
-    re.IGNORECASE)
+RE_AUTOLINK = re.compile(r"<([A-Za-z][A-Za-z0-9.+-]{1,31}:[^<>\x00-\x20]*)>", re.IGNORECASE)
 
 
 def source_tooltip_callback(attrs, new=False):
     """
     Add a `data-tooltip` attribute with `Source` content for embeds.
     """
-    attrs[(None, 'data-tooltip')] = _('Source')
+    attrs[(None, "data-tooltip")] = _("Source")
     return attrs
 
 
@@ -39,31 +35,29 @@ def nofollow_callback(attrs, new=False):
     That callback is not splitted in order to parse the URL only once.
     """
 
-    if (None, 'href') not in attrs:
+    if (None, "href") not in attrs:
         return attrs
-    parsed_url = urlparse(attrs[(None, 'href')])
-    if parsed_url.netloc in ('', current_app.config['SERVER_NAME']):
+    parsed_url = urlparse(attrs[(None, "href")])
+    if parsed_url.netloc in ("", current_app.config["SERVER_NAME"]):
         path = parsed_url.path
-        attrs[(None, 'href')] = '{scheme}://{netloc}{path}{query}'.format(
-            scheme='https' if request.is_secure else 'http',
-            netloc=current_app.config['SERVER_NAME'],
-            path=path if path.startswith('/') else f'/{path}',
-            query='?' + parsed_url.query if parsed_url.query else '')
+        attrs[(None, "href")] = "{scheme}://{netloc}{path}{query}".format(
+            scheme="https" if request.is_secure else "http",
+            netloc=current_app.config["SERVER_NAME"],
+            path=path if path.startswith("/") else f"/{path}",
+            query="?" + parsed_url.query if parsed_url.query else "",
+        )
         return attrs
     else:
-        rel = [x for x in attrs.get((None, 'rel'), '').split(' ') if x]
-        if 'nofollow' not in [x.lower() for x in rel]:
-            rel.append('nofollow')
-        attrs[(None, 'rel')] = ' '.join(rel)
+        rel = [x for x in attrs.get((None, "rel"), "").split(" ") if x]
+        if "nofollow" not in [x.lower() for x in rel]:
+            rel.append("nofollow")
+        attrs[(None, "rel")] = " ".join(rel)
         return attrs
 
 
 class Renderer(mistune.Renderer):
     def table(self, header, body):
-        return (
-            '<table>\n<thead>\n%s</thead>\n'
-            '<tbody>\n%s</tbody>\n</table>\n'
-        ) % (header, body)
+        return ("<table>\n<thead>\n%s</thead>\n" "<tbody>\n%s</tbody>\n</table>\n") % (header, body)
 
 
 class UdataCleaner(bleach.Cleaner):
@@ -73,29 +67,31 @@ class UdataCleaner(bleach.Cleaner):
             callbacks.append(source_tooltip_callback)
 
         super().__init__(
-            tags=current_app.config['MD_ALLOWED_TAGS'],
-            attributes=current_app.config['MD_ALLOWED_ATTRIBUTES'],
-            styles=current_app.config['MD_ALLOWED_STYLES'],
-            protocols=current_app.config['MD_ALLOWED_PROTOCOLS'],
+            tags=current_app.config["MD_ALLOWED_TAGS"],
+            attributes=current_app.config["MD_ALLOWED_ATTRIBUTES"],
+            styles=current_app.config["MD_ALLOWED_STYLES"],
+            protocols=current_app.config["MD_ALLOWED_PROTOCOLS"],
             strip_comments=False,
-            filters=[partial(LinkifyFilter, skip_tags=['pre'], parse_email=False,
-                             callbacks=callbacks)])
+            filters=[
+                partial(LinkifyFilter, skip_tags=["pre"], parse_email=False, callbacks=callbacks)
+            ],
+        )
 
 
 class UDataMarkdown(object):
     """Consistent with Flask's extensions signature."""
 
     def __init__(self, app):
-        app.jinja_env.filters.setdefault('markdown', self.__call__)
+        app.jinja_env.filters.setdefault("markdown", self.__call__)
         renderer = Renderer(escape=False, hard_wrap=True)
         self.markdown = mistune.Markdown(renderer=renderer)
 
     def __call__(self, stream, source_tooltip=False, wrap=True):
         if not stream:
-            return ''
+            return ""
 
         # Prepare angle bracket autolinks to avoid bleach treating them as tag
-        stream = RE_AUTOLINK.sub(r'[\g<1>](\g<1>)', stream)
+        stream = RE_AUTOLINK.sub(r"[\g<1>](\g<1>)", stream)
         # Turn markdown to HTML.
         html = self.markdown(stream)
 
@@ -108,15 +104,15 @@ class UDataMarkdown(object):
         return Markup(html)
 
 
-def mdstrip(value, length=None, end='…'):
-    '''
+def mdstrip(value, length=None, end="…"):
+    """
     Truncate and strip tags from a markdown source
 
     The markdown source is truncated at the excerpt if present and
     smaller than the required length. Then, all html tags are stripped.
-    '''
+    """
     if not value:
-        return ''
+        return ""
     if EXCERPT_TOKEN in value:
         value = value.split(EXCERPT_TOKEN, 1)[0]
     rendered = md(value, wrap=False)
@@ -127,14 +123,14 @@ def mdstrip(value, length=None, end='…'):
 
 
 def parse_html(html):
-    '''
+    """
     Parse HTML and convert it into a udata-compatible markdown string
-    '''
+    """
     if not html:
-        return ''
+        return ""
     return html2text.html2text(html.strip(), bodywidth=0).strip()
 
 
 def init_app(app):
-    app.extensions['markdown'] = UDataMarkdown(app)
+    app.extensions["markdown"] = UDataMarkdown(app)
     app.add_template_filter(mdstrip)

udata/harvest/actions.py

@@ -1,6 +1,5 @@
-import logging
 import csv
-
+import logging
 from collections import namedtuple
 from datetime import datetime, timedelta
 
@@ -9,13 +8,17 @@ from flask import current_app
 
 from udata.auth import current_user
 from udata.core.dataset.models import HarvestDatasetMetadata
-from udata.models import User, Organization, PeriodicTask, Dataset
+from udata.models import Dataset, Organization, PeriodicTask, User
 from udata.storage.s3 import delete_file
 
 from . import backends, signals
 from .models import (
-    HarvestSource, HarvestJob, DEFAULT_HARVEST_FREQUENCY,
-    VALIDATION_ACCEPTED, VALIDATION_REFUSED, archive_harvested_dataset
+    DEFAULT_HARVEST_FREQUENCY,
+    VALIDATION_ACCEPTED,
+    VALIDATION_REFUSED,
+    HarvestJob,
+    HarvestSource,
+    archive_harvested_dataset,
 )
 from .tasks import harvest
 
@@ -25,7 +28,7 @@ DEFAULT_PAGE_SIZE = 10
 
 
 def list_backends():
-    '''List all available backends'''
+    """List all available backends"""
     return backends.get_all(current_app).values()
 
 
@@ -39,37 +42,40 @@ def _sources_queryset(owner=None, deleted=False):
 
 
 def list_sources(owner=None, deleted=False):
-    '''List all harvest sources'''
+    """List all harvest sources"""
     return list(_sources_queryset(owner=owner, deleted=deleted))
 
 
 def paginate_sources(owner=None, page=1, page_size=DEFAULT_PAGE_SIZE, deleted=False):
-    '''Paginate harvest sources'''
+    """Paginate harvest sources"""
     sources = _sources_queryset(owner=owner, deleted=deleted)
     page = max(page or 1, 1)
     return sources.paginate(page, page_size)
 
 
 def get_source(ident):
-    '''Get an harvest source given its ID or its slug'''
+    """Get an harvest source given its ID or its slug"""
     return HarvestSource.get(ident)
 
 
 def get_job(ident):
-    '''Get an harvest job given its ID'''
+    """Get an harvest job given its ID"""
     return HarvestJob.objects.get(id=ident)
 
 
-def create_source(name, url, backend,
-                  description=None,
-                  frequency=DEFAULT_HARVEST_FREQUENCY,
-                  owner=None,
-                  organization=None,
-                  config=None,
-                  active=None,
-                  autoarchive=None,
-                  ):
-    '''Create a new harvest source'''
+def create_source(
+    name,
+    url,
+    backend,
+    description=None,
+    frequency=DEFAULT_HARVEST_FREQUENCY,
+    owner=None,
+    organization=None,
+    config=None,
+    active=None,
+    autoarchive=None,
+):
+    """Create a new harvest source"""
     if owner and not isinstance(owner, User):
         owner = User.get(owner)
 
@@ -93,7 +99,7 @@ def create_source(name, url, backend,
 
 
 def update_source(ident, data):
-    '''Update an harvest source'''
+    """Update an harvest source"""
     source = get_source(ident)
     source.modify(**data)
     signals.harvest_source_updated.send(source)
@@ -101,7 +107,7 @@ def update_source(ident, data):
 
 
 def validate_source(ident, comment=None):
-    '''Validate a source for automatic harvesting'''
+    """Validate a source for automatic harvesting"""
     source = get_source(ident)
     source.validation.on = datetime.utcnow()
     source.validation.comment = comment
@@ -109,13 +115,13 @@ def validate_source(ident, comment=None):
     if current_user.is_authenticated:
         source.validation.by = current_user._get_current_object()
     source.save()
-    schedule(ident, cron=current_app.config['HARVEST_DEFAULT_SCHEDULE'])
+    schedule(ident, cron=current_app.config["HARVEST_DEFAULT_SCHEDULE"])
     launch(ident)
     return source
 
 
 def reject_source(ident, comment):
-    '''Reject a source for automatic harvesting'''
+    """Reject a source for automatic harvesting"""
     source = get_source(ident)
     source.validation.on = datetime.utcnow()
     source.validation.comment = comment
@@ -127,7 +133,7 @@ def reject_source(ident, comment):
 
 
 def delete_source(ident):
-    '''Delete an harvest source'''
+    """Delete an harvest source"""
     source = get_source(ident)
     source.deleted = datetime.utcnow()
     source.save()
@@ -136,7 +142,7 @@ def delete_source(ident):
 
 
 def clean_source(ident):
-    '''Deletes all datasets linked to a harvest source'''
+    """Deletes all datasets linked to a harvest source"""
     source = get_source(ident)
     datasets = Dataset.objects.filter(harvest__source_id=str(source.id))
     for dataset in datasets:
@@ -146,7 +152,7 @@ def clean_source(ident):
 
 
 def purge_sources():
-    '''Permanently remove sources flagged as deleted'''
+    """Permanently remove sources flagged as deleted"""
     sources = HarvestSource.objects(deleted__exists=True)
     count = sources.count()
     for source in sources:
@@ -154,30 +160,34 @@ def purge_sources():
             source.periodic_task.delete()
         datasets = Dataset.objects.filter(harvest__source_id=str(source.id))
         for dataset in datasets:
-            archive_harvested_dataset(dataset, reason='harvester-deleted', dryrun=False)
+            archive_harvested_dataset(dataset, reason="harvester-deleted", dryrun=False)
         source.delete()
     return count
 
 
 def purge_jobs():
-    '''Delete jobs older than retention policy'''
-    retention = current_app.config['HARVEST_JOBS_RETENTION_DAYS']
+    """Delete jobs older than retention policy"""
+    retention = current_app.config["HARVEST_JOBS_RETENTION_DAYS"]
     expiration = datetime.utcnow() - timedelta(days=retention)
 
-    jobs_with_external_files = HarvestJob.objects(data__filename__exists=True, created__lt=expiration)
+    jobs_with_external_files = HarvestJob.objects(
+        data__filename__exists=True, created__lt=expiration
+    )
     for job in jobs_with_external_files:
-        bucket = current_app.config.get('HARVEST_GRAPHS_S3_BUCKET')
+        bucket = current_app.config.get("HARVEST_GRAPHS_S3_BUCKET")
         if bucket is None:
-            log.error(f"Bucket isn't configured anymore, but jobs still exist with external filenames. Could not delete them.")
+            log.error(
+                f"Bucket isn't configured anymore, but jobs still exist with external filenames. Could not delete them."
+            )
             break
 
-        delete_file(bucket, job.data['filename'])
+        delete_file(bucket, job.data["filename"])
 
     return HarvestJob.objects(created__lt=expiration).delete()
 
 
 def run(ident):
-    '''Launch or resume an harvesting for a given source if none is running'''
+    """Launch or resume an harvesting for a given source if none is running"""
     source = get_source(ident)
     cls = backends.get(current_app, source.backend)
     backend = cls(source)
@@ -185,29 +195,32 @@ def run(ident):
 
 
 def launch(ident):
-    '''Launch or resume an harvesting for a given source if none is running'''
+    """Launch or resume an harvesting for a given source if none is running"""
     return harvest.delay(ident)
 
 
 def preview(ident):
-    '''Preview an harvesting for a given source'''
+    """Preview an harvesting for a given source"""
     source = get_source(ident)
     cls = backends.get(current_app, source.backend)
-    max_items = current_app.config['HARVEST_PREVIEW_MAX_ITEMS']
+    max_items = current_app.config["HARVEST_PREVIEW_MAX_ITEMS"]
     backend = cls(source, dryrun=True, max_items=max_items)
     return backend.harvest()
 
 
-def preview_from_config(name, url, backend,
-                        description=None,
-                        frequency=DEFAULT_HARVEST_FREQUENCY,
-                        owner=None,
-                        organization=None,
-                        config=None,
-                        active=None,
-                        autoarchive=None,
-                        ):
-    '''Preview an harvesting from a source created with the given parameters'''
+def preview_from_config(
+    name,
+    url,
+    backend,
+    description=None,
+    frequency=DEFAULT_HARVEST_FREQUENCY,
+    owner=None,
+    organization=None,
+    config=None,
+    active=None,
+    autoarchive=None,
+):
+    """Preview an harvesting from a source created with the given parameters"""
     if owner and not isinstance(owner, User):
         owner = User.get(owner)
 
@@ -227,14 +240,15 @@ def preview_from_config(name, url, backend,
         autoarchive=autoarchive,
     )
     cls = backends.get(current_app, source.backend)
-    max_items = current_app.config['HARVEST_PREVIEW_MAX_ITEMS']
+    max_items = current_app.config["HARVEST_PREVIEW_MAX_ITEMS"]
     backend = cls(source, dryrun=True, max_items=max_items)
     return backend.harvest()
 
 
-def schedule(ident, cron=None, minute='*', hour='*',
-             day_of_week='*', day_of_month='*', month_of_year='*'):
-    '''Schedule an harvesting on a source given a crontab'''
+def schedule(
+    ident, cron=None, minute="*", hour="*", day_of_week="*", day_of_month="*", month_of_year="*"
+):
+    """Schedule an harvesting on a source given a crontab"""
     source = get_source(ident)
 
     if cron:
@@ -245,29 +259,31 @@ def schedule(ident, cron=None, minute='*', hour='*',
         hour=str(hour),
         day_of_week=str(day_of_week),
         day_of_month=str(day_of_month),
-        month_of_year=str(month_of_year)
+        month_of_year=str(month_of_year),
     )
 
     if source.periodic_task:
         source.periodic_task.modify(crontab=crontab)
     else:
-        source.modify(periodic_task=PeriodicTask.objects.create(
-            task='harvest',
-            name='Harvest {0}'.format(source.name),
-            description='Periodic Harvesting',
-            enabled=True,
-            args=[str(source.id)],
-            crontab=crontab,
-        ))
+        source.modify(
+            periodic_task=PeriodicTask.objects.create(
+                task="harvest",
+                name="Harvest {0}".format(source.name),
+                description="Periodic Harvesting",
+                enabled=True,
+                args=[str(source.id)],
+                crontab=crontab,
+            )
+        )
     signals.harvest_source_scheduled.send(source)
     return source
 
 
 def unschedule(ident):
-    '''Unschedule an harvesting on a source'''
+    """Unschedule an harvesting on a source"""
     source = get_source(ident)
     if not source.periodic_task:
-        msg = 'Harvesting on source {0} is ot scheduled'.format(source.name)
+        msg = "Harvesting on source {0} is ot scheduled".format(source.name)
         raise ValueError(msg)
 
     source.periodic_task.delete()
@@ -275,11 +291,11 @@ def unschedule(ident):
     return source
 
 
-AttachResult = namedtuple('AttachResult', ['success', 'errors'])
+AttachResult = namedtuple("AttachResult", ["success", "errors"])
 
 
 def attach(domain, filename):
-    '''Attach existing dataset to their harvest remote id before harvesting.
+    """Attach existing dataset to their harvest remote id before harvesting.
 
     The expected csv file format is the following:
 
@@ -288,32 +304,28 @@ def attach(domain, filename):
 
     The delimiter should be ";". columns order
     and extras columns does not matter
-    '''
+    """
     count = 0
     errors = 0
     with open(filename) as csvfile:
-        reader = csv.DictReader(csvfile, delimiter=';', quotechar='"')
+        reader = csv.DictReader(csvfile, delimiter=";", quotechar='"')
         for row in reader:
             try:
-                dataset = Dataset.objects.get(id=ObjectId(row['local']))
+                dataset = Dataset.objects.get(id=ObjectId(row["local"]))
             except:  # noqa  (Never stop on failure)
-                log.warning('Unable to attach dataset : %s', row['local'])
+                log.warning("Unable to attach dataset : %s", row["local"])
                 errors += 1
                 continue
 
             # Detach previously attached dataset
-            Dataset.objects(**{
-                'harvest__domain': domain,
-                'harvest__remote_id': row['remote']
-            }).update(**{
-                'unset__harvest__domain': True,
-                'unset__harvest__remote_id': True
-            })
+            Dataset.objects(
+                **{"harvest__domain": domain, "harvest__remote_id": row["remote"]}
+            ).update(**{"unset__harvest__domain": True, "unset__harvest__remote_id": True})
 
             if not dataset.harvest:
                 dataset.harvest = HarvestDatasetMetadata()
             dataset.harvest.domain = domain
-            dataset.harvest.remote_id = row['remote']
+            dataset.harvest.remote_id = row["remote"]
 
             dataset.last_modified_internal = datetime.utcnow()
             dataset.save()