ckanext-csvwmapandtransform 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. ckanext/csvwmapandtransform/__init__.py +0 -0
  2. ckanext/csvwmapandtransform/action.py +405 -0
  3. ckanext/csvwmapandtransform/assets/script.js +81 -0
  4. ckanext/csvwmapandtransform/assets/style.css +124 -0
  5. ckanext/csvwmapandtransform/assets/webassets.yml +13 -0
  6. ckanext/csvwmapandtransform/auth.py +23 -0
  7. ckanext/csvwmapandtransform/cli.py +18 -0
  8. ckanext/csvwmapandtransform/db.py +397 -0
  9. ckanext/csvwmapandtransform/helpers.py +67 -0
  10. ckanext/csvwmapandtransform/mapper.py +92 -0
  11. ckanext/csvwmapandtransform/plugin.py +140 -0
  12. ckanext/csvwmapandtransform/tasks.py +257 -0
  13. ckanext/csvwmapandtransform/templates/csvwmapandtransform/create_mapping.html +56 -0
  14. ckanext/csvwmapandtransform/templates/csvwmapandtransform/transform.html +108 -0
  15. ckanext/csvwmapandtransform/templates/package/resource_read.html +8 -0
  16. ckanext/csvwmapandtransform/templates/package/snippets/resource_item.html +23 -0
  17. ckanext/csvwmapandtransform/tests/__init__.py +0 -0
  18. ckanext/csvwmapandtransform/views.py +205 -0
  19. ckanext_csvwmapandtransform-1.0.0-py3.13-nspkg.pth +1 -0
  20. ckanext_csvwmapandtransform-1.0.0.dist-info/METADATA +121 -0
  21. ckanext_csvwmapandtransform-1.0.0.dist-info/RECORD +26 -0
  22. ckanext_csvwmapandtransform-1.0.0.dist-info/WHEEL +5 -0
  23. ckanext_csvwmapandtransform-1.0.0.dist-info/entry_points.txt +2 -0
  24. ckanext_csvwmapandtransform-1.0.0.dist-info/licenses/LICENSE +661 -0
  25. ckanext_csvwmapandtransform-1.0.0.dist-info/namespace_packages.txt +1 -0
  26. ckanext_csvwmapandtransform-1.0.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,257 @@
1
+ import datetime
2
+ import json
3
+ import tempfile
4
+
5
+ import ckanapi
6
+ import ckanapi.datapackage
7
+ import requests
8
+ from ckan import model
9
+ from ckan.plugins.toolkit import asbool, config, get_action
10
+
11
+ # from ckanext.csvtocsvw.annotate import annotate_csv_upload
12
+ from ckanext.csvwmapandtransform import db, mapper
13
+
14
+ try:
15
+ from urllib.parse import urlsplit
16
+ except ImportError:
17
+ from urlparse import urlsplit
18
+
19
+ # log = __import__("logging").getLogger(__name__)
20
+
21
+ CHUNK_INSERT_ROWS = 250
22
+
23
+ from rq import get_current_job
24
+ from werkzeug.datastructures import FileStorage as FlaskFileStorage
25
+
26
+
27
+ def transform(
28
+ res_url, res_id, dataset_id, callback_url, last_updated, skip_if_no_changes=True
29
+ ):
30
+ # url = '{ckan}/dataset/{pkg}/resource/{res_id}/download/{filename}'.format(
31
+ # ckan=CKAN_URL, pkg=dataset_id, res_id=res_id, filename=res_url)
32
+ tomap_res = get_action("resource_show")({"ignore_auth": True}, {"id": res_id})
33
+ context = {"session": model.meta.create_local_session(), "ignore_auth": True}
34
+ metadata = {
35
+ "ckan_url": config.get("ckan.site_url"),
36
+ "resource_id": res_id,
37
+ "task_created": last_updated,
38
+ "original_url": res_url,
39
+ }
40
+ token = config.get("ckanext.csvwmapandtransform.ckan_token")
41
+ job_info = dict()
42
+ job_dict = dict(metadata=metadata, status="running", job_info=job_info)
43
+ job_id = get_current_job().id
44
+ errored = False
45
+ db.init()
46
+
47
+ # Set-up logging to the db
48
+ handler = StoringHandler(job_id, job_dict)
49
+ level = logging.DEBUG
50
+ handler.setLevel(level)
51
+ logger = logging.getLogger(job_id)
52
+ # logger = logging.getLogger()
53
+ handler.setFormatter(logging.Formatter("%(message)s"))
54
+ logger.addHandler(handler)
55
+ # also show logs on stderr
56
+ logger.addHandler(logging.StreamHandler())
57
+ logger.setLevel(logging.DEBUG)
58
+
59
+ callback_csvwmapandtransform_hook(callback_url, api_key=token, job_dict=job_dict)
60
+ logger.info("Trying to find fitting mapping for: {}".format(tomap_res["url"]))
61
+ # need to get it as string, casue url annotation doesnt work with private datasets
62
+ # filename,filedata=annotate_csv_uri(csv_res['url'])
63
+ mappings = get_action("csvwmapandtransform_find_mappings")({}, {})
64
+ mapping_urls = [res["url"] for res in mappings]
65
+ logger.info("Mappings found: {}".format(mapping_urls))
66
+ # tests=get_action(u'csvwmapandtransform_test_mappings')(
67
+ # {}, {
68
+ # u'data_url': resource['url'],
69
+ # u'map_urls': [res['url'] for res in mapping_resources]
70
+ # }
71
+ # )
72
+ logger.info("testing mappings with: {}".format(tomap_res["url"]))
73
+ # tests=get_action(u'csvwmapandtransform_test_map
74
+ res = [
75
+ {
76
+ "mapping": map_url,
77
+ "test": mapper.check_mapping(
78
+ map_url=map_url,
79
+ data_url=tomap_res["url"],
80
+ authorization=token,
81
+ ),
82
+ }
83
+ for map_url in mapping_urls
84
+ ]
85
+ # remove None resulting test Items
86
+ valid_items = [item for item in res if item["test"]]
87
+ for item in valid_items:
88
+ if item["test"]:
89
+ # the more rules can be applied and the more are not skipped the better the mapping
90
+ item["rating"] = (
91
+ item["test"]["rules_applicable"] - item["test"]["rules_skipped"]
92
+ )
93
+ # sort by rating
94
+ sorted_list = sorted(valid_items, key=lambda x: x["rating"], reverse=True)
95
+ logger.info("Rated mappings: {}".format(sorted_list))
96
+ callback_csvwmapandtransform_hook(callback_url, api_key=token, job_dict=job_dict)
97
+ # best cnadidate is sorted_list[0]
98
+ if sorted_list and sorted_list[0]["rating"] > 0:
99
+ best_condidate = sorted_list[0]["mapping"]
100
+ else:
101
+ best_condidate = None
102
+ # run mapping and join data
103
+ if best_condidate:
104
+ filename, graph_data, num_applied, num_skipped = mapper.get_joined_rdf(
105
+ map_url=best_condidate,
106
+ data_url=tomap_res["url"],
107
+ authorization=token,
108
+ )
109
+ if not filename:
110
+ errored = True
111
+ else:
112
+ s = requests.Session()
113
+ s.headers.update({"Authorization": token})
114
+ prefix, suffix = filename.rsplit(".", 1)
115
+ if not prefix:
116
+ prefix = "unnamed"
117
+ if not suffix:
118
+ suffix = "ttl"
119
+ # log.debug(csv_data)
120
+ # # Upload resource to CKAN as a new/updated resource
121
+ ressouce_existing = resource_search(dataset_id, filename)
122
+ with tempfile.NamedTemporaryFile(
123
+ prefix=prefix, suffix="." + suffix
124
+ ) as graph_file:
125
+ graph_file.write(graph_data.encode("utf-8"))
126
+ graph_file.seek(0)
127
+ tmp_filename = graph_file.name
128
+ upload = FlaskFileStorage(open(tmp_filename, "rb"), filename)
129
+ resource = dict(
130
+ package_id=dataset_id,
131
+ # url='dummy-value',
132
+ upload=upload,
133
+ name=filename,
134
+ format="text/turtle; charset=utf-8",
135
+ )
136
+ if not ressouce_existing:
137
+ logger.info(
138
+ "Writing new resource {} to dataset {}".format(
139
+ filename, dataset_id
140
+ )
141
+ )
142
+ # local_ckan.action.resource_create(**resource)
143
+ metadata_res = get_action("resource_create")(
144
+ {"ignore_auth": True}, resource
145
+ )
146
+ else:
147
+ logger.info(
148
+ "Updating resource - {}".format(ressouce_existing["url"])
149
+ )
150
+ # local_ckan.action.resource_patch(
151
+ # id=res['id'],
152
+ # **resource)
153
+ resource["id"] = ressouce_existing["id"]
154
+ metadata_res = get_action("resource_update")(
155
+ {"ignore_auth": True}, resource
156
+ )
157
+ logger.info("job completed results at {}".format(metadata_res["url"]))
158
+ else:
159
+ logger.warning(
160
+ "found no mapping candidate for resource {}".format(tomap_res["url"])
161
+ )
162
+ # all is done update job status
163
+ job_dict["status"] = "complete"
164
+ callback_csvwmapandtransform_hook(callback_url, api_key=token, job_dict=job_dict)
165
+ return "error" if errored else None
166
+
167
+
168
+ def get_resource(id):
169
+ local_ckan = ckanapi.LocalCKAN()
170
+ try:
171
+ res = local_ckan.action.resource_show(id=id)
172
+ except:
173
+ return False
174
+ else:
175
+ return res
176
+
177
+
178
+ def resource_search(dataset_id, res_name):
179
+ local_ckan = ckanapi.LocalCKAN()
180
+ dataset = local_ckan.action.package_show(id=dataset_id)
181
+ for res in dataset["resources"]:
182
+ if res["name"] == res_name:
183
+ return res
184
+ return None
185
+
186
+
187
+ def callback_csvwmapandtransform_hook(result_url, api_key, job_dict):
188
+ """Tells CKAN about the result of the csvwmapandtransform (i.e. calls the callback
189
+ function 'csvwmapandtransform_hook'). Usually called by the csvwmapandtransform queue job.
190
+ """
191
+ headers = {"Content-Type": "application/json"}
192
+ if api_key:
193
+ if ":" in api_key:
194
+ header, key = api_key.split(":")
195
+ else:
196
+ header, key = "Authorization", api_key
197
+ headers[header] = key
198
+ ssl_verify = config.get("ckanext.csvwmapandtransform.ssl_verify")
199
+ if not ssl_verify:
200
+ requests.packages.urllib3.disable_warnings()
201
+ try:
202
+ result = requests.post(
203
+ result_url,
204
+ data=json.dumps(job_dict, cls=DatetimeJsonEncoder),
205
+ verify=ssl_verify,
206
+ headers=headers,
207
+ )
208
+ except requests.ConnectionError:
209
+ return False
210
+
211
+ return result.status_code == requests.codes.ok
212
+
213
+
214
+ import logging
215
+
216
+
217
+ class StoringHandler(logging.Handler):
218
+ """A handler that stores the logging records in a database."""
219
+
220
+ def __init__(self, task_id, input):
221
+ logging.Handler.__init__(self)
222
+ self.task_id = task_id
223
+ self.input = input
224
+
225
+ def emit(self, record):
226
+ conn = db.ENGINE.connect()
227
+ try:
228
+ # Turn strings into unicode to stop SQLAlchemy
229
+ # "Unicode type received non-unicode bind param value" warnings.
230
+ message = str(record.getMessage())
231
+ level = str(record.levelname)
232
+ module = str(record.module)
233
+ funcName = str(record.funcName)
234
+
235
+ conn.execute(
236
+ db.LOGS_TABLE.insert().values(
237
+ job_id=self.task_id,
238
+ timestamp=datetime.datetime.utcnow(),
239
+ message=message,
240
+ level=level,
241
+ module=module,
242
+ funcName=funcName,
243
+ lineno=record.lineno,
244
+ )
245
+ )
246
+ except:
247
+ pass
248
+ finally:
249
+ conn.close()
250
+
251
+
252
+ class DatetimeJsonEncoder(json.JSONEncoder):
253
+ # Custom JSON encoder
254
+ def default(self, obj):
255
+ if isinstance(obj, datetime.datetime):
256
+ return obj.isoformat()
257
+ return json.JSONEncoder.default(self, obj)
@@ -0,0 +1,56 @@
1
+ {% extends "package/base.html" %}
2
+
3
+ {% set logged_in = true if c.userobj else false %}
4
+ {% set res = resource %}
5
+
6
+ {% block breadcrumb_content_selected %}{% endblock %}
7
+
8
+ {% block breadcrumb_content %}
9
+ {{ super() }}
10
+ {% if res %}
11
+ <li>{% link_for h.resource_display_name(res)|truncate(30), named_route=pkg.type ~ '_resource.read', id=pkg.name,
12
+ resource_id=res.id %}</li>
13
+ <li{% block breadcrumb_edit_selected %} class="active" {% endblock %}><a href="">{{ _('Map') }}</a></li>
14
+ {% endif %}
15
+ {% endblock %}
16
+
17
+ {% block content_action %}
18
+ {% if res %}
19
+ {% link_for _('View resource'), named_route=pkg.type ~ '_resource.read', id=pkg.name, resource_id=res.id, class_='btn
20
+ btn-default', icon='eye' %}
21
+ {% endif %}
22
+ {% endblock %}
23
+
24
+ {% block content_primary_nav %}
25
+ {% endblock %}
26
+
27
+ {% block primary_content_inner %}
28
+ <h1>{% block form_title %}{{ _('Map resource') }}{% endblock %}</h1>
29
+ <p>Create a rule bases mapping by filling the form below. It will query the given metadata file and the graph template
30
+ by the Class IRI set for subjects and objects. When clicking "Start Mapping" select widgets will spawn allowing you
31
+ to map a subject to an object. The resulting YAML file will contain a ruleset for each of the assertions made, and
32
+ when run create triples connecting the subject meeting the condition by the predicate IRI given. Download the file,
33
+ make changes if needed and upload it to the "mappings" group here in CKAN if you want to make use of the automated
34
+ mapping process applying the mapping.</p>
35
+ {% block form %}
36
+ {% endblock %}
37
+ <iframe class="col-12" name="my-iframe"
38
+ src="{{iframe_url}}"" onload='javascript:(function(o){o.style.height=o.contentWindow.document.body.scrollHeight+"
39
+ px";}(this));' style="height:1100px;width:100%;border:none;overflow:hidden;"></iframe>
40
+
41
+ {% endblock %}
42
+
43
+ {% block secondary_content %}
44
+ {% snippet 'package/snippets/resource_info.html', res=res %}
45
+ {% snippet 'package/snippets/resources.html', pkg=pkg, active=res.id %}
46
+ {% endblock %}
47
+
48
+ {% block scripts %}
49
+ {{ super() }}
50
+ <!-- <script>
51
+ document.addEventListener("DOMContentLoaded", function() {
52
+ document.getElementById("my-form").submit();
53
+ });
54
+ </script> -->
55
+ {% asset 'vendor/fileupload' %}
56
+ {% endblock %}
@@ -0,0 +1,108 @@
1
+ {% extends "package/base.html" %}
2
+
3
+ {% set logged_in = true if c.userobj else false %}
4
+ {% set res = resource %}
5
+
6
+ {%- block styles %}
7
+ {{ super() }}
8
+ {% asset 'csvwmapandtransform/style' %}
9
+ {% endblock %}
10
+
11
+ {% block breadcrumb_content_selected %}{% endblock %}
12
+
13
+ {% block breadcrumb_content %}
14
+ {{ super() }}
15
+ {% if res %}
16
+ <li>{% link_for h.resource_display_name(res)|truncate(30), named_route=pkg.type ~ '_resource.read', id=pkg.name,
17
+ resource_id=res.id %}</li>
18
+ <li{% block breadcrumb_edit_selected %} class="active" {% endblock %}><a href="">{{ _('Transform') }}</a></li>
19
+ {% endif %}
20
+ {% endblock %}
21
+
22
+ {% block content_action %}
23
+ {% if res %}
24
+ {% link_for _('View resource'), named_route=pkg.type ~ '_resource.read', id=pkg.name, resource_id=res.id, class_='btn
25
+ btn-default', icon='eye' %}
26
+ {% endif %}
27
+ {% endblock %}
28
+
29
+ {% block content_primary_nav %}
30
+ <!-- {{ h.build_nav_icon(pkg.type ~ '_resource.edit', _('Edit resource'), id=pkg.name, resource_id=res.id, icon='pencil') }}
31
+ {% block inner_primary_nav %}{% endblock %}
32
+ {{ h.build_nav_icon(pkg.type ~ '_resource.views', _('Views'), id=pkg.name, resource_id=res.id, icon='chart-bar') }} -->
33
+ {% endblock %}
34
+
35
+ {% block primary_content_inner %}
36
+ <h1>{% block form_title %}{{ _('Transform Status') }}{% endblock %}</h1>
37
+ <div id="ajax-status" data-module="csvwmapandtransform" data-api-url="{{ status_url }}">
38
+ {% block form %}
39
+ <form class="add-to-group" method="post">
40
+ {{ h.csrf_input() }}
41
+ <div class="col-12 d-flex align-items-center justify-content-between">
42
+ <div class="d-flex align-items-center">
43
+ <button class="btn btn-secondary me-2 {% if not
44
+ service_status %}disabled{% endif %}" name="extract/update" type="submit">
45
+ <i class="fa fa-refresh"></i> {{ _('Run Transformation') }}
46
+ </button>
47
+ <!-- <button class="btn btn-danger invisible" name="delete" type="submit">
48
+ <i class="fa fa-trash"></i> {{ _('Delete') }}
49
+ </button> -->
50
+ </div>
51
+ <!-- Status Indicator Section -->
52
+ <style>
53
+ .indicator {
54
+ width: 20px;
55
+ height: 20px;
56
+ border-radius: 50%;
57
+ background-color: red;
58
+ /* Default: Service unavailable */
59
+ }
60
+ </style>
61
+ <div class="d-flex align-items-center">
62
+ <div id="service-indicator" class="indicator" data-bs-toggle="tooltip"
63
+ title="{{ _('The status of the service (Green means available, Red means unavailable)') }}" {% if
64
+ service_status %} style="background-color: green;" {% endif %}>
65
+ </div>
66
+ </div>
67
+ </div>
68
+ </form>
69
+ {% endblock %}
70
+ <hr class="mt-0">
71
+ <div class="invisible" name=status>
72
+ <table class="table table-bordered">
73
+ <colgroup>
74
+ <col width="150">
75
+ <col>
76
+ </colgroup>
77
+ <tr>
78
+ <th>{{ _('Status') }}</th>
79
+ <td>{{status.status}}</td>
80
+ </tr>
81
+ <tr>
82
+ <th>{{ _('Last updated') }}</th>
83
+ {% if status.status %}
84
+ <td><span class="date" title="{{ h.render_datetime(status.last_updated, with_hours=True) }}">{{
85
+ h.time_ago_from_timestamp(status.last_updated) }}</span></td>
86
+ {% else %}
87
+ <td>{{ _('Never') }}</td>
88
+ {% endif %}
89
+ </tr>
90
+ </table>
91
+
92
+ <h3>{{ _('Graph Update Log') }}</h3>
93
+ <ul class="activity" name="log">
94
+ </ul>
95
+ </div>
96
+ </div>
97
+ {% endblock %}
98
+
99
+ {% block secondary_content %}
100
+ {% snippet 'package/snippets/resource_info.html', res=res %}
101
+ {% snippet 'package/snippets/resources.html', pkg=pkg, active=res.id %}
102
+ {% endblock %}
103
+
104
+ csvwmapandtransform
105
+ {%- block scripts %}
106
+ {{ super() }}
107
+ {% asset 'csvwmapandtransform/js' %}
108
+ {% endblock %}
@@ -0,0 +1,8 @@
1
+ {% ckan_extends %}
2
+
3
+ {% block action_manage %}
4
+ {{ super() }}
5
+ {% if h.csvwmapandtransform_show_tools(res) %}
6
+ <li>{% link_for _('Transform'), named_route='csvwmapandtransform.transform', id=pkg.name, resource_id=res.id, class_='btn btn-secondary', icon='play' %}</li>
7
+ {% endif %}
8
+ {% endblock action_manage %}
@@ -0,0 +1,23 @@
1
+ {% ckan_extends %}
2
+
3
+ {% block resource_item_explore_links %}
4
+ {{ super() }}
5
+ {% if not url_is_edit and h.csvwmapandtransform_show_tools(res) %}
6
+ {% if res.url and h.is_url(res.url) %}
7
+ <li>
8
+ <a href="{{ url }}/map" class="dropdown-item resource-url-maptransform-create" target="_blank" rel="noreferrer">
9
+ <i class="fa fa-external-link"></i>
10
+ {{ _('Create Mapping') }}
11
+ </a>
12
+ </li>
13
+ {% if h.check_access('csvwmapandtransform_transform',res) %}
14
+ <li>
15
+ <a href="{{ url }}/transform" class="dropdown-item resource-url-maptransform-run" target="_blank" rel="noreferrer">
16
+ <i class="fa fa-external-link"></i>
17
+ {{ _('Transform') }}
18
+ </a>
19
+ </li>
20
+ {% endif %}
21
+ {% endif %}
22
+ {% endif %}
23
+ {% endblock %}
File without changes
@@ -0,0 +1,205 @@
1
+ import json
2
+ import os
3
+
4
+ import ckan.lib.base as base
5
+ import ckan.lib.helpers as core_helpers
6
+ import ckan.plugins.toolkit as toolkit
7
+ import requests
8
+ from ckan.common import _
9
+ from flask import Blueprint, Request, request
10
+ from flask.views import MethodView
11
+
12
+ from ckanext.csvwmapandtransform.helpers import csvwmapandtransform_service_available
13
+
14
+ log = __import__("logging").getLogger(__name__)
15
+
16
+
17
+ blueprint = Blueprint("csvwmapandtransform", __name__)
18
+
19
+
20
+ class TransformView(MethodView):
21
+ def post(self, id: str, resource_id: str):
22
+ try:
23
+ toolkit.get_action("csvwmapandtransform_transform")(
24
+ {}, {"resource_id": resource_id}
25
+ )
26
+ except toolkit.ObjectNotFound:
27
+ base.abort(404, "Dataset not found")
28
+ except toolkit.NotAuthorized:
29
+ base.abort(403, _("Not authorized to see this page"))
30
+ except toolkit.ValidationError:
31
+ log.debug(toolkit.ValidationError)
32
+
33
+ return core_helpers.redirect_to(
34
+ "csvwmapandtransform.transform", id=id, resource_id=resource_id
35
+ )
36
+
37
+ def get(self, id: str, resource_id: str):
38
+ try:
39
+ pkg_dict = toolkit.get_action("package_show")({}, {"id": id})
40
+ resource = toolkit.get_action("resource_show")({}, {"id": resource_id})
41
+
42
+ # backward compatibility with old templates
43
+ toolkit.g.pkg_dict = pkg_dict
44
+ toolkit.g.resource = resource
45
+
46
+ status = toolkit.get_action("csvwmapandtransform_transform_status")(
47
+ {}, {"resource_id": resource_id}
48
+ )
49
+ except toolkit.ObjectNotFound:
50
+ base.abort(404, "Resource not found")
51
+ except toolkit.NotAuthorized:
52
+ base.abort(403, _("Not authorized to see this page"))
53
+
54
+ return base.render(
55
+ "csvwmapandtransform/transform.html",
56
+ extra_vars={
57
+ "pkg_dict": pkg_dict,
58
+ "resource": resource,
59
+ "status": status,
60
+ "status_url": toolkit.url_for(
61
+ "csvwmapandtransform.status",
62
+ id=id,
63
+ resource_id=resource_id,
64
+ qualified=True,
65
+ ),
66
+ "service_status": csvwmapandtransform_service_available(),
67
+ "refresh_rate": 10,
68
+ },
69
+ )
70
+
71
+
72
+ class CreateMapView(MethodView):
73
+ def post(self, id: str, resource_id: str):
74
+ return core_helpers.redirect_to(
75
+ "csvwmapandtransform.map", id=id, resource_id=resource_id
76
+ )
77
+
78
+ def get(self, id: str, resource_id: str):
79
+ try:
80
+ pkg_dict = toolkit.get_action("package_show")({}, {"id": id})
81
+ resource = toolkit.get_action("resource_show")({}, {"id": resource_id})
82
+
83
+ # backward compatibility with old templates
84
+ toolkit.g.pkg_dict = pkg_dict
85
+ toolkit.g.resource = resource
86
+ except toolkit.ObjectNotFound:
87
+ base.abort(404, "Dataset not found")
88
+ except toolkit.NotAuthorized:
89
+ base.abort(403, _("Not authorized to see this page"))
90
+ # iframe_url = toolkit.url_for(
91
+ # "api.action",
92
+ # ver=3,
93
+ # logic_function="csvwmapandtransform_map",
94
+ # qualified=True
95
+ # )
96
+ # iframe_url='http://docker-dev.iwm.fraunhofer.de:6002/'
97
+ iframe_url = toolkit.url_for(
98
+ "csvwmapandtransform.iframe_maptomethod",
99
+ id=id,
100
+ resource_id=resource_id,
101
+ qualified=True,
102
+ )
103
+ return base.render(
104
+ "csvwmapandtransform/create_mapping.html",
105
+ extra_vars={
106
+ "pkg_dict": pkg_dict,
107
+ "resource": resource,
108
+ "iframe_url": iframe_url,
109
+ },
110
+ )
111
+
112
+
113
+ def iframe_maptomethod(id, resource_id):
114
+ # extra_vars['q'] = q = request.args.get('q', '')
115
+ # if 'data_url' in data_dict:
116
+ # url = data_dict['data_url']
117
+ headers = {
118
+ "Content-Type": "application/json",
119
+ "Authorization": toolkit.config.get("ckanext.csvwmapandtransform.ckan_token"),
120
+ "Accept": "text/html",
121
+ }
122
+ resource_dict = toolkit.get_action("resource_show")({}, {"id": resource_id})
123
+
124
+ # log.debug(request.values)
125
+ data = {
126
+ "data_url": resource_dict["url"],
127
+ "method_url": "https://github.com/Mat-O-Lab/MSEO/raw/main/methods/DIN_EN_ISO_527-3.drawio.ttl",
128
+ "advanced-data_subject_super_class_uris-0": "http://www.w3.org/ns/csvw#Column",
129
+ "advanced-data_subject_super_class_uris-1": "http://www.w3.org/ns/oa#Annotation",
130
+ "advanced-data_mapping_predicate_uri": "http://purl.obolibrary.org/obo/RO_0010002",
131
+ "advanced-method_object_super_class_uris-0": "https://spec.industrialontologies.org/ontology/core/Core/InformationContentEntity",
132
+ "advanced-method_object_super_class_uris-1": "http://purl.obolibrary.org/obo/BFO_0000008",
133
+ }
134
+ ssl_verify = toolkit.config.get("ckanext.csvwmapandtransform.ssl_verify")
135
+ if not ssl_verify:
136
+ requests.packages.urllib3.disable_warnings()
137
+ maptomethod_url = toolkit.config.get("ckanext.csvwmapandtransform.maptomethod_url")
138
+ html = requests.post(
139
+ url=maptomethod_url + "/create_mapper",
140
+ headers=headers,
141
+ data=json.dumps(data),
142
+ verify=ssl_verify,
143
+ )
144
+ # html=requests.post(url="http://docker-dev.iwm.fraunhofer.de:6002"+"/create_mapper", headers=headers, data=json.dumps(data))
145
+ html.raise_for_status()
146
+ result = html.text
147
+ # log.debug("Response from MapToMethod: {}".format(result))
148
+ return result
149
+
150
+
151
+ class StatusView(MethodView):
152
+
153
+ def get(self, id: str, resource_id: str):
154
+ pkg_dict = {}
155
+ try:
156
+ pkg_dict = toolkit.get_action("package_show")({}, {"id": id})
157
+ status = toolkit.get_action("csvwmapandtransform_transform_status")(
158
+ {}, {"resource_id": resource_id}
159
+ )
160
+ except toolkit.ObjectNotFound:
161
+ base.abort(404, "Resource not found")
162
+ except toolkit.NotAuthorized:
163
+ base.abort(403, _("Not authorized to see this page"))
164
+
165
+ if status and "logs" in status.keys():
166
+ for index, item in enumerate(status["logs"]):
167
+ status["logs"][index]["timestamp"] = (
168
+ core_helpers.time_ago_from_timestamp(item["timestamp"])
169
+ )
170
+ if item["level"] == "DEBUG":
171
+ status["logs"][index]["alertlevel"] = "info"
172
+ status["logs"][index]["icon"] = "bug-slash"
173
+ status["logs"][index]["class"] = "success"
174
+ elif item["level"] == "INFO":
175
+ status["logs"][index]["alertlevel"] = "info"
176
+ status["logs"][index]["icon"] = "check"
177
+ status["logs"][index]["class"] = "success"
178
+ else:
179
+ status["logs"][index]["alertlevel"] = "error"
180
+ status["logs"][index]["icon"] = "exclamation"
181
+ status["logs"][index]["class"] = "failure"
182
+ return {"pkg_dict": pkg_dict, "status": status}
183
+
184
+
185
+ blueprint.add_url_rule(
186
+ "/dataset/<id>/resource/<resource_id>/transform/status",
187
+ view_func=StatusView.as_view(str("status")),
188
+ )
189
+
190
+ blueprint.add_url_rule(
191
+ "/dataset/<id>/resource/<resource_id>/transform",
192
+ view_func=TransformView.as_view(str("transform")),
193
+ )
194
+ blueprint.add_url_rule(
195
+ "/dataset/<id>/resource/<resource_id>/map",
196
+ view_func=CreateMapView.as_view(str("map")),
197
+ )
198
+
199
+ blueprint.add_url_rule(
200
+ "/dataset/<id>/resource/<resource_id>/maptomethod", view_func=iframe_maptomethod
201
+ )
202
+
203
+
204
+ def get_blueprint():
205
+ return blueprint
@@ -0,0 +1 @@
1
+ import sys, types, os;p = os.path.join(sys._getframe(1).f_locals['sitedir'], *('ckanext',));importlib = __import__('importlib.util');__import__('importlib.machinery');m = sys.modules.setdefault('ckanext', importlib.util.module_from_spec(importlib.machinery.PathFinder.find_spec('ckanext', [os.path.dirname(p)])));m = m or sys.modules.setdefault('ckanext', types.ModuleType('ckanext'));mp = (m or []) and m.__dict__.setdefault('__path__',[]);(p not in mp) and mp.append(p)