ckanext-validation-schema-generator 2.0.2__py3-none-any.whl

bin/extract-id.py

@@ -0,0 +1,5 @@
+# encoding: utf-8
+import json
+import sys
+
+print(json.loads(sys.stdin.read())['result']['id'])

ckanext/__init__.py

@@ -0,0 +1,9 @@
+# encoding: utf-8
+
+# this is a namespace package
+try:
+    import pkg_resources
+    pkg_resources.declare_namespace(__name__)
+except ImportError:
+    import pkgutil
+    __path__ = pkgutil.extend_path(__path__, __name__)

ckanext/validation_schema_generator/assets/css/validation_schema_generator.css

@@ -0,0 +1,27 @@
+.schema-generate .alert {
+    margin-bottom: 20px;
+}
+
+.generate-schema-form .btn-generate {
+    margin-bottom: 20px;
+}
+
+.generate-schema-form .table-schema {
+    margin-bottom: 30px;
+}
+
+.generate-schema-form .with-textarea .form-group {
+    margin: 0;
+}
+
+.generate-schema-form .with-textarea .form-group.control-select {
+    margin-bottom: 20px;
+}
+
+.generate-schema-form .with-textarea .form-group textarea {
+    min-height: 300px;
+}
+
+.generate-schema-form .with-textarea .control-label {
+    display: none;
+}

ckanext/validation_schema_generator/assets/webassets.yml

@@ -0,0 +1,14 @@
+vsg_js:
+  filter: rjsmin
+  output: ckanext-validation_schema_generator/%(version)s-validation_schema_generator.js
+  contents:
+    - js/validation_schema_generator.js
+  extra:
+    preload:
+      - base/main
+
+vsg_css:
+  filter: cssrewrite
+  output: ckanext-validation_schema_generator/%(version)s-validation_schema_generator.css
+  contents:
+    - css/validation_schema_generator.css

ckanext/validation_schema_generator/ckan_dataset.json

@@ -0,0 +1,144 @@
+{
+    "scheming_version": 1,
+    "dataset_type": "dataset",
+    "about": "A reimplementation of the default CKAN dataset schema",
+    "about_url": "http://github.com/ckan/ckanext-scheming",
+    "dataset_fields": [
+        {
+            "field_name": "title",
+            "label": "Title",
+            "preset": "title",
+            "form_placeholder": "eg. A descriptive title"
+        },
+        {
+            "field_name": "name",
+            "label": "URL",
+            "preset": "dataset_slug",
+            "form_placeholder": "eg. my-dataset"
+        },
+        {
+            "field_name": "notes",
+            "label": "Description",
+            "form_snippet": "markdown.html",
+            "form_placeholder": "eg. Some useful notes about the data",
+            "required": true
+        },
+        {
+            "field_name": "tag_string",
+            "label": "Tags",
+            "preset": "tag_string_autocomplete",
+            "form_placeholder": "eg. economy, mental health, government"
+        },
+        {
+            "field_name": "license_id",
+            "label": "License",
+            "form_snippet": "license.html",
+            "help_text": "License definitions and additional information can be found at http://opendefinition.org/",
+            "required": true
+        },
+        {
+            "field_name": "owner_org",
+            "label": "Organization",
+            "preset": "dataset_organization"
+        },
+        {
+            "field_name": "url",
+            "label": "Source",
+            "form_placeholder": "http://example.com/dataset.json",
+            "display_property": "foaf:homepage",
+            "display_snippet": "link.html"
+        },
+        {
+            "field_name": "version",
+            "label": "Version",
+            "validators": "scheming_required unicode_safe package_version_validator",
+            "form_placeholder": "1.0",
+            "required": true
+        },
+        {
+            "field_name": "author",
+            "label": "Author",
+            "form_placeholder": "Joe Bloggs",
+            "display_property": "dc:creator",
+            "display_snippet": null,
+            "form_snippet": null
+        },
+        {
+            "field_name": "author_email",
+            "label": "Email",
+            "form_placeholder": "joe@example.com",
+            "display_property": "dc:creator",
+            "display_snippet": "email.html",
+            "display_email_name_field": "author",
+            "required": true
+        },
+        {
+            "field_name": "maintainer",
+            "label": "Maintainer",
+            "form_placeholder": "Joe Bloggs",
+            "display_property": "dc:contributor",
+            "display_snippet": null,
+            "form_snippet": null
+        },
+        {
+            "field_name": "maintainer_email",
+            "label": "Maintainer Email",
+            "form_placeholder": "joe@example.com",
+            "display_property": "dc:contributor",
+            "display_email_name_field": "maintainer",
+            "display_snippet": null,
+            "form_snippet": null
+        },
+        {
+            "field_name": "schema_upload",
+            "form_snippet": null,
+            "display_snippet": null,
+            "validators": "ignore_missing"
+        },
+        {
+            "field_name": "schema_json",
+            "form_snippet": null,
+            "display_snippet": null,
+            "validators": "ignore_missing"
+        },
+        {
+            "field_name": "schema_url",
+            "form_snippet": null,
+            "display_snippet": null,
+            "validators": "ignore_missing"
+        }
+    ],
+    "resource_fields": [
+        {
+            "field_name": "url",
+            "label": "URL",
+            "preset": "resource_url_upload",
+            "required": true
+        },
+        {
+            "field_name": "name",
+            "label": "Name",
+            "form_placeholder": "eg. January 2011 Gold Prices",
+            "required": true
+        },
+        {
+            "field_name": "format",
+            "label": "Format",
+            "preset": "resource_format_autocomplete",
+            "required": true
+        },
+        {
+            "field_name": "description",
+            "label": "Description",
+            "form_snippet": "markdown.html",
+            "form_placeholder": "Some useful notes about the data",
+            "required": true
+        },
+        {
+            "field_name": "last_modified",
+            "display_snippet": null,
+            "form_snippet": null,
+            "validators": "ignore_empty"
+        }
+    ]
+}

ckanext/validation_schema_generator/constants.py

@@ -0,0 +1,30 @@
+import ckan.plugins.toolkit as tk
+
+TASK_TYPE = u"generate"
+TASK_KEY = u"validation_schema_generator"
+TASK_STATE_PENDING = u"Pending"
+TASK_STATE_FINISHED = u"Complete"
+TASK_STATE_NOT_GENERATED = u"Not generated"
+TASK_STATE_ERROR = u"Failed"
+VALID_STATES = [
+    TASK_STATE_ERROR,
+    TASK_STATE_FINISHED,
+    TASK_STATE_PENDING,
+]
+
+CF_JOB_TIMEOUT = u"ckanext.validation_schema_generator.job_timeout"
+CF_JOB_TIMEOUT_DF = 3600
+CF_API_KEY = u"ckanext.validation_schema_generator.api_key"
+
+APPLY_FOR_OPTIONS = [u"dataset", u"resource"]
+APPLY_FOR_DATASET = APPLY_FOR_OPTIONS[0]
+APPLY_FOR_RESOURCE = APPLY_FOR_OPTIONS[1]
+APPLY_FOR_FIELD = u"apply_for"
+
+RES_SCHEMA_FIELD = tk.config.get(
+    u"ckanext.validation_schema_generator.resource_schema_field_name", u"schema")
+PKG_SCHEMA_FIELD = tk.config.get(
+    u"ckanext.validation_schema_generator.package_schema_field_name", u"default_data_schema")
+
+EMPTY_SCHEMA = u""
+UNAPPLIED = u""

ckanext/validation_schema_generator/controller.py

@@ -0,0 +1,24 @@
+# encoding: utf-8
+
+from ckan.lib.base import BaseController
+import ckan.plugins.toolkit as tk
+
+from ckanext.validation_schema_generator.views import VSGIndexView
+
+
+class VSGController(BaseController):
+    def _check_access(self, resource_id):
+        try:
+            tk.check_access('vsg_generate', {}, {"id": resource_id})
+        except tk.NotAuthorized:
+            tk.abort(403, tk._(u'You are not allowed to generate resource schema'))
+
+    def index(self, dataset_id, resource_id):
+        self._check_access(resource_id)
+
+        view = VSGIndexView()
+
+        if tk.request.method == 'GET':
+            return view.get(dataset_id, resource_id)
+
+        return view.post(dataset_id, resource_id)

ckanext/validation_schema_generator/helpers.py

@@ -0,0 +1,14 @@
+# encoding: utf-8
+
+def _get_helpers():
+    return {
+        "vsg_get_apply_for_options": vsg_get_apply_for_options
+    }
+
+
+def vsg_get_apply_for_options():
+    return [
+        {'value': '', 'text': ''},
+        {'value': 'dataset', 'text': 'Dataset default'},
+        {'value': 'resource', 'text': 'Resource'}
+    ]

ckanext/validation_schema_generator/jobs.py

@@ -0,0 +1,87 @@
+# encoding: utf-8
+
+import requests
+
+from frictionless import describe
+from frictionless.exception import FrictionlessException
+
+import ckan.model as model
+import ckan.plugins.toolkit as tk
+
+from ckanext.validation_schema_generator.constants import (
+    CF_API_KEY,
+    TASK_STATE_FINISHED,
+    TASK_STATE_ERROR,
+)
+
+
+def generate_schema_from_resource(input):
+    context = _make_context()
+
+    resource = tk.get_action(u'resource_show')(context, {
+        'id': input[u'resource_id']
+    })
+
+    errors = {}
+    options = {}
+    source = None
+    schema = None
+
+    if not source:
+        source = resource[u'url']
+
+    try:
+        schema = describe(source, type='schema', **options)
+    except FrictionlessException as e:
+        errors[u'schema'] = str(e)
+    except Exception as e:
+        errors[u'undefined'] = str(e)
+    finally:
+        _update_task(input, errors, schema)
+
+
+def _make_context():
+    user = _get_site_user()
+
+    return {
+        'model': model,
+        'session': model.Session,
+        'ignore_auth': True,
+        'user': user[u'name'],
+        'auth_user_obj': None
+    }
+
+
+def _get_site_user():
+    return tk.get_action(u'get_site_user')({
+        'model': model,
+        'ignore_auth': True
+    }, {})
+
+
+def _make_session():
+    s = requests.Session()
+    s.headers.update({u'Authorization': _get_api_key()})
+    return s
+
+
+def _get_api_key():
+    return tk.config.get(CF_API_KEY, _get_site_user_api_key())
+
+
+def _get_site_user_api_key():
+    user = _get_site_user()
+    return user['apikey']
+
+
+def _update_task(input, errors, schema):
+    context = _make_context()
+
+    data_dict = {
+        'id': input[u'resource_id'],
+        'status': TASK_STATE_ERROR if errors else TASK_STATE_FINISHED,
+        'error': errors,
+        'schema': schema.to_json() if schema else ''
+    }
+
+    tk.get_action('vsg_update')(context, data_dict)

ckanext/validation_schema_generator/logic/action.py

@@ -0,0 +1,183 @@
+# encoding: utf-8
+
+import json
+import logging
+
+import ckan.plugins.toolkit as tk
+from ckan.logic import validate
+from ckan.lib.jobs import enqueue as enqueue_job
+
+import ckanext.validation_schema_generator.logic.schema as vsg_schema
+from ckanext.validation_schema_generator import jobs, utils as vsg_utils, constants as const
+
+log = logging.getLogger(__name__)
+
+
+def _get_actions():
+    return {
+        'vsg_generate': vsg_generate,
+        'vsg_status': vsg_status,
+        'vsg_update': vsg_update,
+        'vsg_apply': vsg_apply,
+        'vsg_unapply': vsg_unapply
+    }
+
+
+@validate(vsg_schema.vsg_generate_schema)
+def vsg_generate(context, data_dict):
+    """Generate a schema for a dataset
+
+    :param id: resource ID
+    :type id: string
+    """
+    tk.check_access('vsg_generate', context, data_dict)
+
+    task = vsg_utils.update_task(
+        context,
+        vsg_utils.generate_task_data(data_dict["id"],
+                                     const.TASK_STATE_PENDING))
+
+    data = {"resource_id": data_dict["id"], "task_id": task["id"]}
+
+    timeout = tk.asint(tk.config.get(
+        const.CF_JOB_TIMEOUT, const.CF_JOB_TIMEOUT_DF))
+    job = enqueue_job(jobs.generate_schema_from_resource, [data],
+                      rq_kwargs={"timeout": timeout})
+
+    task['value'] = {'job_id': job.id}
+    task['error'] = {}
+
+    return vsg_utils.update_task(context, task)
+
+
+@validate(vsg_schema.vsg_default_schema)
+def vsg_status(context, data_dict):
+    """Return a status of schema generation for a specific resource
+
+    :param id: resource ID
+    :type id: string
+    """
+    tk.check_access('vsg_generate', context, data_dict)
+
+    try:
+        task = tk.get_action('task_status_show')(context, {
+            'entity_id': data_dict["id"],
+            'task_type': const.TASK_TYPE,
+            'key': const.TASK_KEY
+        })
+    except tk.ObjectNotFound:
+        return {
+            'state': const.TASK_STATE_NOT_GENERATED,
+            'last_updated': None,
+            'value': None,
+            'error': None
+        }
+
+    task['value'] = json.loads(task['value'])
+    task['error'] = json.loads(task['error'])
+    return task
+
+
+@validate(vsg_schema.vsg_hook_schema)
+def vsg_update(context, data_dict):
+    """Hook to update vsg task from job"""
+    id, error, status = tk.get_or_bust(data_dict, ['id', 'error', 'status'])
+
+    tk.check_access('vsg_generate', context, {'id': id})
+    task = tk.get_action('vsg_status')(context, {'id': id})
+
+    task['state'] = status
+    task['last_updated'] = vsg_utils.get_current_time()
+    task['error'] = error
+
+    if data_dict.get("schema"):
+        task['value']['schema'] = json.loads(data_dict["schema"])
+
+    return vsg_utils.update_task(context, task)
+
+
+@validate(vsg_schema.vsg_apply_schema)
+def vsg_apply(context, data_dict):
+    """Apply a generated schema for a resource or dataset. The schema can be applied only
+    if the generation process is successfully completed
+
+    :param id: resource ID
+    :type id: string
+    :param apply_for: apply schema agaisnt the whole `dataset` or `resource` only.
+    :type apply_for: string
+    """
+    tk.check_access('vsg_generate', context, data_dict)
+
+    apply_for = data_dict.get(const.APPLY_FOR_FIELD)
+
+    task = tk.get_action('vsg_status')(context, data_dict)
+
+    if task['state'] in (const.TASK_STATE_NOT_GENERATED,
+                         const.TASK_STATE_PENDING):
+        raise tk.ValidationError(tk._(u"Schema is not generated yet."))
+    elif task['state'] == const.TASK_STATE_ERROR:
+        raise tk.ValidationError(
+            tk._(u"Schema generation failed. Check status."))
+
+    task['last_updated'] = vsg_utils.get_current_time()
+    task['value'][const.APPLY_FOR_FIELD] = apply_for
+    schema = vsg_utils.dump_schema(task['value']['schema'])
+
+    log.debug("Applying generated schema to %s: [%s]", apply_for, schema)
+    if apply_for == const.APPLY_FOR_DATASET:
+        _apply_pkg_schema(schema, data_dict['id'])
+    else:
+        _apply_res_schema(schema, data_dict['id'])
+
+    return vsg_utils.update_task(context, task)
+
+
+def _apply_res_schema(schema, resource_id):
+    """Apply the generated schema for resource and unapply for package"""
+    context = {"user": "", "ignore_auth": True}
+
+    res = tk.get_action(u'resource_show')(context, {u'id': resource_id})
+    res[const.RES_SCHEMA_FIELD] = schema
+
+    tk.get_action(u'resource_update')(context, res)
+
+
+def _apply_pkg_schema(schema, resource_id):
+    """Apply the generated schema for package and resource"""
+    context = {"user": "", "ignore_auth": True}
+
+    res = tk.get_action(u'resource_show')(context, {u'id': resource_id})
+    pkg = tk.get_action(u'package_show')(context, {u'id': res['package_id']})
+
+    pkg[const.PKG_SCHEMA_FIELD] = schema
+
+    if schema:
+        for resource in pkg.get('resources', []):
+            if resource['id'] == resource_id:
+                resource[const.RES_SCHEMA_FIELD] = schema
+
+    tk.get_action(u'package_update')(context, pkg)
+
+
+@validate(vsg_schema.vsg_default_schema)
+def vsg_unapply(context, data_dict):
+    """Unapply the schema
+    """
+    tk.check_access('vsg_generate', context, data_dict)
+
+    task = tk.get_action('vsg_status')(context, data_dict)
+
+    if task['state'] in (const.TASK_STATE_NOT_GENERATED,
+                         const.TASK_STATE_PENDING):
+        raise tk.ValidationError(u"Schema is not generated yet")
+    elif task['state'] == const.TASK_STATE_ERROR:
+        raise tk.ValidationError(u"Schema generation failed. Check status.")
+
+    applied_for = task['value'].get(const.APPLY_FOR_FIELD)
+
+    if not applied_for:
+        raise tk.ValidationError(u"The schema is not applied yet")
+
+    task['value'][const.APPLY_FOR_FIELD] = ''
+
+    return vsg_utils.update_task(context, task)

ckanext/validation_schema_generator/logic/auth.py

@@ -0,0 +1,28 @@
+# encoding: utf-8
+
+import ckan.plugins.toolkit as tk
+import ckan.authz as authz
+
+
+def _get_auth_functions():
+    return {
+        "vsg_generate": vsg_generate,
+    }
+
+
+def vsg_generate(context, data_dict):
+    """Check if user is allowed to generate schema
+
+    :param id: the id of the resource to generate schema for
+    :type id: string
+    """
+    authorized = authz.is_authorized('resource_update', context,
+                                     data_dict).get('success')
+
+    if authorized:
+        return {'success': True}
+
+    msg = tk._('You are not authorized to generate schema for resource {}').format(
+        data_dict['id']
+    )
+    return {'success': False, 'msg': msg}

ckanext/validation_schema_generator/logic/schema.py

@@ -0,0 +1,43 @@
+# encoding: utf-8
+
+from ckan.logic.schema import validator_args
+
+import ckanext.validation_schema_generator.constants as const
+
+
+@validator_args
+def vsg_default_schema(not_missing, vsg_resource_not_missing):
+    return {"id": [not_missing, vsg_resource_not_missing]}
+
+
+@validator_args
+def vsg_generate_schema(not_missing, vsg_is_resource_supportable,
+                        vsg_resource_not_missing):
+    return {
+        "id":
+        [not_missing, vsg_resource_not_missing, vsg_is_resource_supportable]
+    }
+
+
+@validator_args
+def vsg_apply_schema(not_missing, resource_id_exists, one_of):
+    return {
+        "id": [not_missing, resource_id_exists],
+        "apply_for": [not_missing, one_of(const.APPLY_FOR_OPTIONS)]
+    }
+
+
+@validator_args
+def vsg_hook_schema(not_missing, resource_id_exists, one_of, ignore_missing,
+                    vsg_validate_schema, unicode_safe, dict_only,
+                    vsg_is_resource_supportable, vsg_generation_started):
+
+    return {
+        "id": [
+            not_missing, resource_id_exists, vsg_is_resource_supportable,
+            vsg_generation_started
+        ],
+        "error": [not_missing, dict_only],
+        "status": [not_missing, one_of(const.VALID_STATES)],
+        "schema": [ignore_missing, unicode_safe, vsg_validate_schema]
+    }