ckanext-csvwmapandtransform 0.0.1__py3-none-any.whl

ckanext/csvwmapandtransform/mapper.py

@@ -0,0 +1,133 @@
+import json
+
+import ckan.plugins.toolkit as toolkit
+import requests
+
+log = __import__("logging").getLogger(__name__)
+
+
+def post_request(url, headers, data, files=None):
+    ssl_verify = toolkit.config.get("ckanext.csvwmapandtransform.ssl_verify")
+    if not ssl_verify:
+        requests.packages.urllib3.disable_warnings()
+
+    try:
+        if files:
+            # should create a multipart form upload
+            response = requests.post(
+                url, data=data, headers=headers, files=files, verify=ssl_verify
+            )
+        else:
+            # a application json post request
+            response = requests.post(
+                url, data=json.dumps(data), headers=headers, verify=ssl_verify
+            )
+        
+        # Log response details before raising for non-OK responses
+        if not response.ok:
+            error_body = response.text[:500] if response.text else "No response body"
+            log.error(
+                f"HTTP {response.status_code} from {url}: {error_body}"
+            )
+        
+        response.raise_for_status()
+        return response
+
+    except requests.exceptions.HTTPError as e:
+        error_body = e.response.text[:500] if e.response and e.response.text else "No response body"
+        log.error(
+            f"HTTP Error {e.response.status_code if e.response else 'Unknown'} calling {url}: {error_body}"
+        )
+        return None
+    except requests.exceptions.ConnectionError as e:
+        log.error(f"Connection Error calling {url}: {str(e)}")
+        return None
+    except requests.exceptions.Timeout as e:
+        log.error(f"Timeout calling {url}: {str(e)}")
+        return None
+    except Exception as e:
+        log.error(f"Unexpected error calling {url}: {type(e).__name__}: {str(e)}")
+        return None
+
+
+def check_mapping(map_url: str, data_url: str, authorization: None):
+    rdfconverter_url = toolkit.config.get(
+        "ckanext.csvwmapandtransform.rdfconverter_url"
+    )
+    log.debug("checking mapping at: {} with data url: {}".format(map_url, data_url))
+    # curl -X 'POST' 'http://docker-dev.iwm.fraunhofer.de:5003/api/checkmapping' -H 'accept: application/json' -H 'Content-Type: application/json' -d '{"data_url": "https://raw.githubusercontent.com/Mat-O-Lab/CSVToCSVW/main/examples/example-metadata.json", "mapping_url": "https://github.com/Mat-O-Lab/MapToMethod/raw/main/examples/example-map.yaml"}'
+    url = rdfconverter_url + "/api/checkmapping"
+    log.debug("rdf converter api call: {}".format(url))
+    data = {"mapping_url": map_url, "data_url": data_url}
+    headers = {"Content-Type": "application/json"}
+    if authorization:
+        headers["Authorization"] = authorization
+    r = post_request(url, headers, data)
+    # r=requests.get(rdfconverter_url+"/info")
+    # log.debug(r)
+    if r and r.status_code == 200:
+        res = r.json()
+        log.debug("map check results: {}".format(res))
+        return res
+    else:
+        log.debug("map check error: {}".format(r))
+        return None
+
+
+def get_joined_rdf(map_url: str, data_url: str, authorization: None):
+    log.info(f"Creating joined RDF with mapping: {map_url} and data: {data_url}")
+    rdfconverter_url = toolkit.config.get(
+        "ckanext.csvwmapandtransform.rdfconverter_url"
+    )
+    url = rdfconverter_url + "/api/createrdf?return_type=turtle"
+    data = {"mapping_url": map_url, "data_url": data_url}
+    headers = {"Content-type": "application/json", "Accept": "application/json"}
+    if authorization:
+        headers["Authorization"] = authorization
+    log.debug(f"Request headers: {headers}")
+    log.debug(f"Request data: {data}")
+
+    r = post_request(url, headers, data)
+    
+    if r is None:
+        log.error(
+            f"Failed to get response from RDF converter at {url}. "
+            f"Mapping: {map_url}, Data: {data_url}"
+        )
+        return (None, None, None, None)
+    
+    if r.status_code == 200:
+        try:
+            response_json = r.json()
+            filename = response_json.get("filename")
+            graph = response_json.get("graph")
+            num_applied = response_json.get("num_mappings_applied")
+            num_skipped = response_json.get("num_mappings_skipped")
+            
+            if not filename or not graph:
+                log.error(
+                    f"RDF converter returned incomplete response. "
+                    f"Filename: {filename}, Graph present: {bool(graph)}"
+                )
+                return (None, None, None, None)
+            
+            log.info(
+                f"Successfully created RDF: {filename} "
+                f"(applied {num_applied} rules, skipped {num_skipped})"
+            )
+            return (filename, graph, num_applied, num_skipped)
+            
+        except (ValueError, KeyError) as e:
+            log.error(
+                f"Failed to parse RDF converter response: {type(e).__name__}: {str(e)}. "
+                f"Response: {r.text[:500]}"
+            )
+            return (None, None, None, None)
+    else:
+        error_msg = r.text[:500] if r.text else "No error message"
+        log.error(
+            f"RDF converter returned status {r.status_code}. "
+            f"Mapping: {map_url}, Data: {data_url}. "
+            f"Error: {error_msg}"
+        )
+        return (None, None, None, None)

ckanext/csvwmapandtransform/plugin.py

@@ -0,0 +1,140 @@
+import os
+import re
+
+import ckan.plugins as plugins
+import ckan.plugins.toolkit as toolkit
+from ckan import model
+from ckan.config.declaration import Declaration, Key
+from ckan.lib.plugins import DefaultTranslation
+
+if toolkit.check_ckan_version("2.10"):
+    from ckan.types import Context
+else:
+
+    class Context(dict):
+        def __init__(self, **kwargs):
+            super().__init__(**kwargs)
+
+
+from typing import Any
+
+from ckanext.csvwmapandtransform import action, auth, helpers, views
+
+log = __import__("logging").getLogger(__name__)
+
+
+class CsvwMapAndTransformPlugin(plugins.SingletonPlugin, DefaultTranslation):
+    plugins.implements(plugins.ITranslation)
+    plugins.implements(plugins.IConfigurer)
+    plugins.implements(plugins.IConfigDeclaration)
+    plugins.implements(plugins.ITemplateHelpers)
+    plugins.implements(plugins.IResourceUrlChange)
+    plugins.implements(plugins.IResourceController, inherit=True)
+    plugins.implements(plugins.IActions)
+    plugins.implements(plugins.IAuthFunctions)
+    plugins.implements(plugins.IBlueprint)
+
+    # IConfigurer
+
+    def update_config(self, config_):
+        toolkit.add_template_directory(config_, "templates")
+        toolkit.add_public_directory(config_, "public")
+        toolkit.add_resource("assets", "csvwmapandtransform")
+
+    # IConfigDeclaration
+
+    def declare_config_options(self, declaration: Declaration, key: Key):
+
+        declaration.annotate("csvwmapandtransform")
+        group = key.ckanext.csvwmapandtransform
+        declaration.declare_bool(group.ssl_verify, True)
+        declaration.declare(group.db_url, plugins.toolkit.config.get("sqlalchemy.url"))
+        declaration.declare(group.maptomethod_url, "https://maptomethod.matolab.org")
+        declaration.declare(group.rdfconverter_url, "https://rdfconverter.matolab.org")
+        declaration.declare(group.ckan_token, "")
+        declaration.declare(
+            group.formats, "json json-ld turtle n3 nt hext trig longturtle xml ld+json"
+        )
+
+    # IResourceUrlChange
+
+    def notify(self, resource: model.Resource):
+        context: Context = {"ignore_auth": True}
+        resource_dict = toolkit.get_action("resource_show")(
+            context,
+            {
+                "id": resource.id,
+            },
+        )
+        self._sumbit_transform(resource_dict)
+
+    # IResourceController
+
+    if not toolkit.check_ckan_version("2.10") or toolkit.check_ckan_version("2.11"):
+
+        def after_create(self, context, resource_dict):
+            self.after_resource_create(context, resource_dict)
+
+        # def before_show(self, resource_dict):
+        #     self.before_resource_show(resource_dict)
+
+        def after_update(self, context: Context, resource_dict: dict[str, Any]):
+            self._sumbit_transform(resource_dict)
+
+    def after_resource_create(self, context: Context, resource_dict: dict[str, Any]):
+        self._sumbit_transform(resource_dict)
+
+    def _sumbit_transform(self, resource_dict: dict[str, Any]):
+        context = {"model": model, "ignore_auth": True, "defer_commit": True}
+        formats = toolkit.config.get("ckanext.csvwmapandtransform.formats")
+        format = resource_dict.get("format", None)
+        submit = (
+            format
+            and format.lower() in formats
+            and "-joined" not in resource_dict["url"]
+        )
+        log.debug(
+            "Submitting resource {0} with format {1}".format(
+                resource_dict["id"], format
+            )
+            + " to csvwmapandtransform_transform"
+        )
+
+        if not submit:
+            return
+
+        try:
+            log.debug(
+                "Submitting resource {0}".format(resource_dict["id"])
+                + " to csvwmapandtransform_transform"
+            )
+            toolkit.get_action("csvwmapandtransform_transform")(
+                context, {"id": resource_dict["id"]}
+            )
+
+        except toolkit.ValidationError as e:
+            # If RDFConverter is offline want to catch error instead
+            # of raising otherwise resource save will fail with 500
+            log.critical(e)
+            pass
+
+    # ITemplateHelpers
+
+    def get_helpers(self):
+        return helpers.get_helpers()
+
+    # IActions
+
+    def get_actions(self):
+        actions = action.get_actions()
+        return actions
+
+    # IBlueprint
+
+    def get_blueprint(self):
+        return views.get_blueprint()
+
+    # IAuthFunctions
+
+    def get_auth_functions(self):
+        return auth.get_auth_functions()

ckanext/csvwmapandtransform/tasks.py

@@ -0,0 +1,262 @@
+import datetime
+import json
+import tempfile
+
+import ckanapi
+import ckanapi.datapackage
+import requests
+from ckan import model
+from ckan.plugins.toolkit import asbool, config, get_action
+
+# from ckanext.csvtocsvw.annotate import annotate_csv_upload
+from ckanext.csvwmapandtransform import db, mapper
+
+try:
+    from urllib.parse import urlsplit
+except ImportError:
+    from urlparse import urlsplit
+
+# log = __import__("logging").getLogger(__name__)
+
+CHUNK_INSERT_ROWS = 250
+
+from rq import get_current_job
+from werkzeug.datastructures import FileStorage as FlaskFileStorage
+
+
+def transform(
+    res_url, res_id, dataset_id, callback_url, last_updated, skip_if_no_changes=True
+):
+    # url = '{ckan}/dataset/{pkg}/resource/{res_id}/download/{filename}'.format(
+    #         ckan=CKAN_URL, pkg=dataset_id, res_id=res_id, filename=res_url)
+    tomap_res = get_action("resource_show")({"ignore_auth": True}, {"id": res_id})
+    context = {"session": model.meta.create_local_session(), "ignore_auth": True}
+    metadata = {
+        "ckan_url": config.get("ckan.site_url"),
+        "resource_id": res_id,
+        "task_created": last_updated,
+        "original_url": res_url,
+    }
+    token = config.get("ckanext.csvwmapandtransform.ckan_token")
+    job_info = dict()
+    job_dict = dict(metadata=metadata, status="running", job_info=job_info)
+    job_id = get_current_job().id
+    errored = False
+    db.init()
+
+    # Set-up logging to the db
+    handler = StoringHandler(job_id, job_dict)
+    level = logging.DEBUG
+    handler.setLevel(level)
+    logger = logging.getLogger(job_id)
+    # logger = logging.getLogger()
+    handler.setFormatter(logging.Formatter("%(message)s"))
+    logger.addHandler(handler)
+    # also show logs on stderr
+    logger.addHandler(logging.StreamHandler())
+    logger.setLevel(logging.DEBUG)
+
+    callback_csvwmapandtransform_hook(callback_url, api_key=token, job_dict=job_dict)
+    logger.info("Trying to find fitting mapping for: {}".format(tomap_res["url"]))
+    # need to get it as string, casue url annotation doesnt work with private datasets
+    # filename,filedata=annotate_csv_uri(csv_res['url'])
+    mappings = get_action("csvwmapandtransform_find_mappings")({}, {})
+    mapping_urls = [res["url"] for res in mappings]
+    logger.info("Mappings found: {}".format(mapping_urls))
+    # tests=get_action(u'csvwmapandtransform_test_mappings')(
+    #             {}, {
+    #                 u'data_url': resource['url'],
+    #                 u'map_urls': [res['url'] for res in mapping_resources]
+    #             }
+    #         )
+    logger.info("testing mappings with: {}".format(tomap_res["url"]))
+    # tests=get_action(u'csvwmapandtransform_test_map
+    res = [
+        {
+            "mapping": map_url,
+            "test": mapper.check_mapping(
+                map_url=map_url,
+                data_url=tomap_res["url"],
+                authorization=token,
+            ),
+        }
+        for map_url in mapping_urls
+    ]
+    # remove None resulting test Items
+    valid_items = [item for item in res if item["test"]]
+    for item in valid_items:
+        if item["test"]:
+            # the more rules can be applied and the more are not skipped the better the mapping
+            item["rating"] = (
+                item["test"]["rules_applicable"] - item["test"]["rules_skipped"]
+            )
+    # sort by rating
+    sorted_list = sorted(valid_items, key=lambda x: x["rating"], reverse=True)
+    logger.info("Rated mappings: {}".format(sorted_list))
+    callback_csvwmapandtransform_hook(callback_url, api_key=token, job_dict=job_dict)
+    # best cnadidate is sorted_list[0]
+    if sorted_list and sorted_list[0]["rating"] > 0:
+        best_condidate = sorted_list[0]["mapping"]
+    else:
+        best_condidate = None
+    # run mapping and join data
+    if best_condidate:
+        logger.info(f"Applying best mapping candidate: {best_condidate}")
+        filename, graph_data, num_applied, num_skipped = mapper.get_joined_rdf(
+            map_url=best_condidate,
+            data_url=tomap_res["url"],
+            authorization=token,
+        )
+        if not filename:
+            errored = True
+            logger.error(
+                f"Failed to generate RDF from mapping {best_condidate} for resource {tomap_res['url']}. "
+                "Check previous error messages for details."
+            )
+        else:
+            s = requests.Session()
+            s.headers.update({"Authorization": token})
+            prefix, suffix = filename.rsplit(".", 1)
+            if not prefix:
+                prefix = "unnamed"
+            if not suffix:
+                suffix = "ttl"
+            # log.debug(csv_data)
+            # # Upload resource to CKAN as a new/updated resource
+            ressouce_existing = resource_search(dataset_id, filename)
+            with tempfile.NamedTemporaryFile(
+                prefix=prefix, suffix="." + suffix
+            ) as graph_file:
+                graph_file.write(graph_data.encode("utf-8"))
+                graph_file.seek(0)
+                tmp_filename = graph_file.name
+                upload = FlaskFileStorage(open(tmp_filename, "rb"), filename)
+                resource = dict(
+                    package_id=dataset_id,
+                    # url='dummy-value',
+                    upload=upload,
+                    name=filename,
+                    format="text/turtle; charset=utf-8",
+                )
+                if not ressouce_existing:
+                    logger.info(
+                        "Writing new resource {} to dataset {}".format(
+                            filename, dataset_id
+                        )
+                    )
+                    # local_ckan.action.resource_create(**resource)
+                    metadata_res = get_action("resource_create")(
+                        {"ignore_auth": True}, resource
+                    )
+                else:
+                    logger.info(
+                        "Updating resource - {}".format(ressouce_existing["url"])
+                    )
+                    # local_ckan.action.resource_patch(
+                    #     id=res['id'],
+                    #     **resource)
+                    resource["id"] = ressouce_existing["id"]
+                    metadata_res = get_action("resource_update")(
+                        {"ignore_auth": True}, resource
+                    )
+            logger.info("job completed results at {}".format(metadata_res["url"]))
+    else:
+        logger.warning(
+            "found no mapping candidate for resource {}".format(tomap_res["url"])
+        )
+    # all is done update job status
+    job_dict["status"] = "complete"
+    callback_csvwmapandtransform_hook(callback_url, api_key=token, job_dict=job_dict)
+    return "error" if errored else None
+
+
+def get_resource(id):
+    local_ckan = ckanapi.LocalCKAN()
+    try:
+        res = local_ckan.action.resource_show(id=id)
+    except:
+        return False
+    else:
+        return res
+
+
+def resource_search(dataset_id, res_name):
+    local_ckan = ckanapi.LocalCKAN()
+    dataset = local_ckan.action.package_show(id=dataset_id)
+    for res in dataset["resources"]:
+        if res["name"] == res_name:
+            return res
+    return None
+
+
+def callback_csvwmapandtransform_hook(result_url, api_key, job_dict):
+    """Tells CKAN about the result of the csvwmapandtransform (i.e. calls the callback
+    function 'csvwmapandtransform_hook'). Usually called by the csvwmapandtransform queue job.
+    """
+    headers = {"Content-Type": "application/json"}
+    if api_key:
+        if ":" in api_key:
+            header, key = api_key.split(":")
+        else:
+            header, key = "Authorization", api_key
+        headers[header] = key
+    ssl_verify = config.get("ckanext.csvwmapandtransform.ssl_verify")
+    if not ssl_verify:
+        requests.packages.urllib3.disable_warnings()
+    try:
+        result = requests.post(
+            result_url,
+            data=json.dumps(job_dict, cls=DatetimeJsonEncoder),
+            verify=ssl_verify,
+            headers=headers,
+        )
+    except requests.ConnectionError:
+        return False
+
+    return result.status_code == requests.codes.ok
+
+
+import logging
+
+
+class StoringHandler(logging.Handler):
+    """A handler that stores the logging records in a database."""
+
+    def __init__(self, task_id, input):
+        logging.Handler.__init__(self)
+        self.task_id = task_id
+        self.input = input
+
+    def emit(self, record):
+        conn = db.ENGINE.connect()
+        try:
+            # Turn strings into unicode to stop SQLAlchemy
+            # "Unicode type received non-unicode bind param value" warnings.
+            message = str(record.getMessage())
+            level = str(record.levelname)
+            module = str(record.module)
+            funcName = str(record.funcName)
+
+            conn.execute(
+                db.LOGS_TABLE.insert().values(
+                    job_id=self.task_id,
+                    timestamp=datetime.datetime.utcnow(),
+                    message=message,
+                    level=level,
+                    module=module,
+                    funcName=funcName,
+                    lineno=record.lineno,
+                )
+            )
+        except:
+            pass
+        finally:
+            conn.close()
+
+
+class DatetimeJsonEncoder(json.JSONEncoder):
+    # Custom JSON encoder
+    def default(self, obj):
+        if isinstance(obj, datetime.datetime):
+            return obj.isoformat()
+        return json.JSONEncoder.default(self, obj)

ckanext/csvwmapandtransform/templates/csvwmapandtransform/create_mapping.html

@@ -0,0 +1,56 @@
+{% extends "package/base.html" %}
+
+{% set logged_in = true if c.userobj else false %}
+{% set res = resource %}
+
+{% block breadcrumb_content_selected %}{% endblock %}
+
+{% block breadcrumb_content %}
+{{ super() }}
+{% if res %}
+<li>{% link_for h.resource_display_name(res)|truncate(30), named_route=pkg.type ~ '_resource.read', id=pkg.name,
+  resource_id=res.id %}</li>
+<li{% block breadcrumb_edit_selected %} class="active" {% endblock %}><a href="">{{ _('Map') }}</a></li>
+  {% endif %}
+  {% endblock %}
+
+  {% block content_action %}
+  {% if res %}
+  {% link_for _('View resource'), named_route=pkg.type ~ '_resource.read', id=pkg.name, resource_id=res.id, class_='btn
+  btn-default', icon='eye' %}
+  {% endif %}
+  {% endblock %}
+
+  {% block content_primary_nav %}
+  {% endblock %}
+
+  {% block primary_content_inner %}
+  <h1>{% block form_title %}{{ _('Map resource') }}{% endblock %}</h1>
+  <p>Create a rule bases mapping by filling the form below. It will query the given metadata file and the graph template
+    by the Class IRI set for subjects and objects. When clicking "Start Mapping" select widgets will spawn allowing you
+    to map a subject to an object. The resulting YAML file will contain a ruleset for each of the assertions made, and
+    when run create triples connecting the subject meeting the condition by the predicate IRI given. Download the file,
+    make changes if needed and upload it to the "mappings" group here in CKAN if you want to make use of the automated
+    mapping process applying the mapping.</p>
+  {% block form %}
+  {% endblock %}
+  <iframe class="col-12" name="my-iframe"
+    src="{{iframe_url}}"" onload='javascript:(function(o){o.style.height=o.contentWindow.document.body.scrollHeight+"
+    px";}(this));' style="height:1100px;width:100%;border:none;overflow:hidden;"></iframe>
+
+  {% endblock %}
+
+  {% block secondary_content %}
+  {% snippet 'package/snippets/resource_info.html', res=res %}
+  {% snippet 'package/snippets/resources.html', pkg=pkg, active=res.id %}
+  {% endblock %}
+
+  {% block scripts %}
+  {{ super() }}
+  <!-- <script>
+    document.addEventListener("DOMContentLoaded", function() {
+      document.getElementById("my-form").submit();
+    });
+  </script> -->
+  {% asset 'vendor/fileupload' %}
+  {% endblock %}