ckanext-csvwmapandtransform 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ckanext/csvwmapandtransform/__init__.py +0 -0
- ckanext/csvwmapandtransform/action.py +405 -0
- ckanext/csvwmapandtransform/assets/script.js +81 -0
- ckanext/csvwmapandtransform/assets/style.css +124 -0
- ckanext/csvwmapandtransform/assets/webassets.yml +13 -0
- ckanext/csvwmapandtransform/auth.py +23 -0
- ckanext/csvwmapandtransform/cli.py +18 -0
- ckanext/csvwmapandtransform/db.py +397 -0
- ckanext/csvwmapandtransform/helpers.py +67 -0
- ckanext/csvwmapandtransform/mapper.py +92 -0
- ckanext/csvwmapandtransform/plugin.py +140 -0
- ckanext/csvwmapandtransform/tasks.py +257 -0
- ckanext/csvwmapandtransform/templates/csvwmapandtransform/create_mapping.html +56 -0
- ckanext/csvwmapandtransform/templates/csvwmapandtransform/transform.html +108 -0
- ckanext/csvwmapandtransform/templates/package/resource_read.html +8 -0
- ckanext/csvwmapandtransform/templates/package/snippets/resource_item.html +23 -0
- ckanext/csvwmapandtransform/tests/__init__.py +0 -0
- ckanext/csvwmapandtransform/views.py +205 -0
- ckanext_csvwmapandtransform-1.0.0-py3.13-nspkg.pth +1 -0
- ckanext_csvwmapandtransform-1.0.0.dist-info/METADATA +121 -0
- ckanext_csvwmapandtransform-1.0.0.dist-info/RECORD +26 -0
- ckanext_csvwmapandtransform-1.0.0.dist-info/WHEEL +5 -0
- ckanext_csvwmapandtransform-1.0.0.dist-info/entry_points.txt +2 -0
- ckanext_csvwmapandtransform-1.0.0.dist-info/licenses/LICENSE +661 -0
- ckanext_csvwmapandtransform-1.0.0.dist-info/namespace_packages.txt +1 -0
- ckanext_csvwmapandtransform-1.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
import datetime
|
|
2
|
+
import json
|
|
3
|
+
import tempfile
|
|
4
|
+
|
|
5
|
+
import ckanapi
|
|
6
|
+
import ckanapi.datapackage
|
|
7
|
+
import requests
|
|
8
|
+
from ckan import model
|
|
9
|
+
from ckan.plugins.toolkit import asbool, config, get_action
|
|
10
|
+
|
|
11
|
+
# from ckanext.csvtocsvw.annotate import annotate_csv_upload
|
|
12
|
+
from ckanext.csvwmapandtransform import db, mapper
|
|
13
|
+
|
|
14
|
+
try:
|
|
15
|
+
from urllib.parse import urlsplit
|
|
16
|
+
except ImportError:
|
|
17
|
+
from urlparse import urlsplit
|
|
18
|
+
|
|
19
|
+
# log = __import__("logging").getLogger(__name__)
|
|
20
|
+
|
|
21
|
+
CHUNK_INSERT_ROWS = 250
|
|
22
|
+
|
|
23
|
+
from rq import get_current_job
|
|
24
|
+
from werkzeug.datastructures import FileStorage as FlaskFileStorage
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def transform(
|
|
28
|
+
res_url, res_id, dataset_id, callback_url, last_updated, skip_if_no_changes=True
|
|
29
|
+
):
|
|
30
|
+
# url = '{ckan}/dataset/{pkg}/resource/{res_id}/download/{filename}'.format(
|
|
31
|
+
# ckan=CKAN_URL, pkg=dataset_id, res_id=res_id, filename=res_url)
|
|
32
|
+
tomap_res = get_action("resource_show")({"ignore_auth": True}, {"id": res_id})
|
|
33
|
+
context = {"session": model.meta.create_local_session(), "ignore_auth": True}
|
|
34
|
+
metadata = {
|
|
35
|
+
"ckan_url": config.get("ckan.site_url"),
|
|
36
|
+
"resource_id": res_id,
|
|
37
|
+
"task_created": last_updated,
|
|
38
|
+
"original_url": res_url,
|
|
39
|
+
}
|
|
40
|
+
token = config.get("ckanext.csvwmapandtransform.ckan_token")
|
|
41
|
+
job_info = dict()
|
|
42
|
+
job_dict = dict(metadata=metadata, status="running", job_info=job_info)
|
|
43
|
+
job_id = get_current_job().id
|
|
44
|
+
errored = False
|
|
45
|
+
db.init()
|
|
46
|
+
|
|
47
|
+
# Set-up logging to the db
|
|
48
|
+
handler = StoringHandler(job_id, job_dict)
|
|
49
|
+
level = logging.DEBUG
|
|
50
|
+
handler.setLevel(level)
|
|
51
|
+
logger = logging.getLogger(job_id)
|
|
52
|
+
# logger = logging.getLogger()
|
|
53
|
+
handler.setFormatter(logging.Formatter("%(message)s"))
|
|
54
|
+
logger.addHandler(handler)
|
|
55
|
+
# also show logs on stderr
|
|
56
|
+
logger.addHandler(logging.StreamHandler())
|
|
57
|
+
logger.setLevel(logging.DEBUG)
|
|
58
|
+
|
|
59
|
+
callback_csvwmapandtransform_hook(callback_url, api_key=token, job_dict=job_dict)
|
|
60
|
+
logger.info("Trying to find fitting mapping for: {}".format(tomap_res["url"]))
|
|
61
|
+
# need to get it as string, casue url annotation doesnt work with private datasets
|
|
62
|
+
# filename,filedata=annotate_csv_uri(csv_res['url'])
|
|
63
|
+
mappings = get_action("csvwmapandtransform_find_mappings")({}, {})
|
|
64
|
+
mapping_urls = [res["url"] for res in mappings]
|
|
65
|
+
logger.info("Mappings found: {}".format(mapping_urls))
|
|
66
|
+
# tests=get_action(u'csvwmapandtransform_test_mappings')(
|
|
67
|
+
# {}, {
|
|
68
|
+
# u'data_url': resource['url'],
|
|
69
|
+
# u'map_urls': [res['url'] for res in mapping_resources]
|
|
70
|
+
# }
|
|
71
|
+
# )
|
|
72
|
+
logger.info("testing mappings with: {}".format(tomap_res["url"]))
|
|
73
|
+
# tests=get_action(u'csvwmapandtransform_test_map
|
|
74
|
+
res = [
|
|
75
|
+
{
|
|
76
|
+
"mapping": map_url,
|
|
77
|
+
"test": mapper.check_mapping(
|
|
78
|
+
map_url=map_url,
|
|
79
|
+
data_url=tomap_res["url"],
|
|
80
|
+
authorization=token,
|
|
81
|
+
),
|
|
82
|
+
}
|
|
83
|
+
for map_url in mapping_urls
|
|
84
|
+
]
|
|
85
|
+
# remove None resulting test Items
|
|
86
|
+
valid_items = [item for item in res if item["test"]]
|
|
87
|
+
for item in valid_items:
|
|
88
|
+
if item["test"]:
|
|
89
|
+
# the more rules can be applied and the more are not skipped the better the mapping
|
|
90
|
+
item["rating"] = (
|
|
91
|
+
item["test"]["rules_applicable"] - item["test"]["rules_skipped"]
|
|
92
|
+
)
|
|
93
|
+
# sort by rating
|
|
94
|
+
sorted_list = sorted(valid_items, key=lambda x: x["rating"], reverse=True)
|
|
95
|
+
logger.info("Rated mappings: {}".format(sorted_list))
|
|
96
|
+
callback_csvwmapandtransform_hook(callback_url, api_key=token, job_dict=job_dict)
|
|
97
|
+
# best cnadidate is sorted_list[0]
|
|
98
|
+
if sorted_list and sorted_list[0]["rating"] > 0:
|
|
99
|
+
best_condidate = sorted_list[0]["mapping"]
|
|
100
|
+
else:
|
|
101
|
+
best_condidate = None
|
|
102
|
+
# run mapping and join data
|
|
103
|
+
if best_condidate:
|
|
104
|
+
filename, graph_data, num_applied, num_skipped = mapper.get_joined_rdf(
|
|
105
|
+
map_url=best_condidate,
|
|
106
|
+
data_url=tomap_res["url"],
|
|
107
|
+
authorization=token,
|
|
108
|
+
)
|
|
109
|
+
if not filename:
|
|
110
|
+
errored = True
|
|
111
|
+
else:
|
|
112
|
+
s = requests.Session()
|
|
113
|
+
s.headers.update({"Authorization": token})
|
|
114
|
+
prefix, suffix = filename.rsplit(".", 1)
|
|
115
|
+
if not prefix:
|
|
116
|
+
prefix = "unnamed"
|
|
117
|
+
if not suffix:
|
|
118
|
+
suffix = "ttl"
|
|
119
|
+
# log.debug(csv_data)
|
|
120
|
+
# # Upload resource to CKAN as a new/updated resource
|
|
121
|
+
ressouce_existing = resource_search(dataset_id, filename)
|
|
122
|
+
with tempfile.NamedTemporaryFile(
|
|
123
|
+
prefix=prefix, suffix="." + suffix
|
|
124
|
+
) as graph_file:
|
|
125
|
+
graph_file.write(graph_data.encode("utf-8"))
|
|
126
|
+
graph_file.seek(0)
|
|
127
|
+
tmp_filename = graph_file.name
|
|
128
|
+
upload = FlaskFileStorage(open(tmp_filename, "rb"), filename)
|
|
129
|
+
resource = dict(
|
|
130
|
+
package_id=dataset_id,
|
|
131
|
+
# url='dummy-value',
|
|
132
|
+
upload=upload,
|
|
133
|
+
name=filename,
|
|
134
|
+
format="text/turtle; charset=utf-8",
|
|
135
|
+
)
|
|
136
|
+
if not ressouce_existing:
|
|
137
|
+
logger.info(
|
|
138
|
+
"Writing new resource {} to dataset {}".format(
|
|
139
|
+
filename, dataset_id
|
|
140
|
+
)
|
|
141
|
+
)
|
|
142
|
+
# local_ckan.action.resource_create(**resource)
|
|
143
|
+
metadata_res = get_action("resource_create")(
|
|
144
|
+
{"ignore_auth": True}, resource
|
|
145
|
+
)
|
|
146
|
+
else:
|
|
147
|
+
logger.info(
|
|
148
|
+
"Updating resource - {}".format(ressouce_existing["url"])
|
|
149
|
+
)
|
|
150
|
+
# local_ckan.action.resource_patch(
|
|
151
|
+
# id=res['id'],
|
|
152
|
+
# **resource)
|
|
153
|
+
resource["id"] = ressouce_existing["id"]
|
|
154
|
+
metadata_res = get_action("resource_update")(
|
|
155
|
+
{"ignore_auth": True}, resource
|
|
156
|
+
)
|
|
157
|
+
logger.info("job completed results at {}".format(metadata_res["url"]))
|
|
158
|
+
else:
|
|
159
|
+
logger.warning(
|
|
160
|
+
"found no mapping candidate for resource {}".format(tomap_res["url"])
|
|
161
|
+
)
|
|
162
|
+
# all is done update job status
|
|
163
|
+
job_dict["status"] = "complete"
|
|
164
|
+
callback_csvwmapandtransform_hook(callback_url, api_key=token, job_dict=job_dict)
|
|
165
|
+
return "error" if errored else None
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def get_resource(id):
|
|
169
|
+
local_ckan = ckanapi.LocalCKAN()
|
|
170
|
+
try:
|
|
171
|
+
res = local_ckan.action.resource_show(id=id)
|
|
172
|
+
except:
|
|
173
|
+
return False
|
|
174
|
+
else:
|
|
175
|
+
return res
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def resource_search(dataset_id, res_name):
|
|
179
|
+
local_ckan = ckanapi.LocalCKAN()
|
|
180
|
+
dataset = local_ckan.action.package_show(id=dataset_id)
|
|
181
|
+
for res in dataset["resources"]:
|
|
182
|
+
if res["name"] == res_name:
|
|
183
|
+
return res
|
|
184
|
+
return None
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def callback_csvwmapandtransform_hook(result_url, api_key, job_dict):
|
|
188
|
+
"""Tells CKAN about the result of the csvwmapandtransform (i.e. calls the callback
|
|
189
|
+
function 'csvwmapandtransform_hook'). Usually called by the csvwmapandtransform queue job.
|
|
190
|
+
"""
|
|
191
|
+
headers = {"Content-Type": "application/json"}
|
|
192
|
+
if api_key:
|
|
193
|
+
if ":" in api_key:
|
|
194
|
+
header, key = api_key.split(":")
|
|
195
|
+
else:
|
|
196
|
+
header, key = "Authorization", api_key
|
|
197
|
+
headers[header] = key
|
|
198
|
+
ssl_verify = config.get("ckanext.csvwmapandtransform.ssl_verify")
|
|
199
|
+
if not ssl_verify:
|
|
200
|
+
requests.packages.urllib3.disable_warnings()
|
|
201
|
+
try:
|
|
202
|
+
result = requests.post(
|
|
203
|
+
result_url,
|
|
204
|
+
data=json.dumps(job_dict, cls=DatetimeJsonEncoder),
|
|
205
|
+
verify=ssl_verify,
|
|
206
|
+
headers=headers,
|
|
207
|
+
)
|
|
208
|
+
except requests.ConnectionError:
|
|
209
|
+
return False
|
|
210
|
+
|
|
211
|
+
return result.status_code == requests.codes.ok
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
import logging
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
class StoringHandler(logging.Handler):
|
|
218
|
+
"""A handler that stores the logging records in a database."""
|
|
219
|
+
|
|
220
|
+
def __init__(self, task_id, input):
|
|
221
|
+
logging.Handler.__init__(self)
|
|
222
|
+
self.task_id = task_id
|
|
223
|
+
self.input = input
|
|
224
|
+
|
|
225
|
+
def emit(self, record):
|
|
226
|
+
conn = db.ENGINE.connect()
|
|
227
|
+
try:
|
|
228
|
+
# Turn strings into unicode to stop SQLAlchemy
|
|
229
|
+
# "Unicode type received non-unicode bind param value" warnings.
|
|
230
|
+
message = str(record.getMessage())
|
|
231
|
+
level = str(record.levelname)
|
|
232
|
+
module = str(record.module)
|
|
233
|
+
funcName = str(record.funcName)
|
|
234
|
+
|
|
235
|
+
conn.execute(
|
|
236
|
+
db.LOGS_TABLE.insert().values(
|
|
237
|
+
job_id=self.task_id,
|
|
238
|
+
timestamp=datetime.datetime.utcnow(),
|
|
239
|
+
message=message,
|
|
240
|
+
level=level,
|
|
241
|
+
module=module,
|
|
242
|
+
funcName=funcName,
|
|
243
|
+
lineno=record.lineno,
|
|
244
|
+
)
|
|
245
|
+
)
|
|
246
|
+
except:
|
|
247
|
+
pass
|
|
248
|
+
finally:
|
|
249
|
+
conn.close()
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
class DatetimeJsonEncoder(json.JSONEncoder):
|
|
253
|
+
# Custom JSON encoder
|
|
254
|
+
def default(self, obj):
|
|
255
|
+
if isinstance(obj, datetime.datetime):
|
|
256
|
+
return obj.isoformat()
|
|
257
|
+
return json.JSONEncoder.default(self, obj)
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
{% extends "package/base.html" %}
|
|
2
|
+
|
|
3
|
+
{% set logged_in = true if c.userobj else false %}
|
|
4
|
+
{% set res = resource %}
|
|
5
|
+
|
|
6
|
+
{% block breadcrumb_content_selected %}{% endblock %}
|
|
7
|
+
|
|
8
|
+
{% block breadcrumb_content %}
|
|
9
|
+
{{ super() }}
|
|
10
|
+
{% if res %}
|
|
11
|
+
<li>{% link_for h.resource_display_name(res)|truncate(30), named_route=pkg.type ~ '_resource.read', id=pkg.name,
|
|
12
|
+
resource_id=res.id %}</li>
|
|
13
|
+
<li{% block breadcrumb_edit_selected %} class="active" {% endblock %}><a href="">{{ _('Map') }}</a></li>
|
|
14
|
+
{% endif %}
|
|
15
|
+
{% endblock %}
|
|
16
|
+
|
|
17
|
+
{% block content_action %}
|
|
18
|
+
{% if res %}
|
|
19
|
+
{% link_for _('View resource'), named_route=pkg.type ~ '_resource.read', id=pkg.name, resource_id=res.id, class_='btn
|
|
20
|
+
btn-default', icon='eye' %}
|
|
21
|
+
{% endif %}
|
|
22
|
+
{% endblock %}
|
|
23
|
+
|
|
24
|
+
{% block content_primary_nav %}
|
|
25
|
+
{% endblock %}
|
|
26
|
+
|
|
27
|
+
{% block primary_content_inner %}
|
|
28
|
+
<h1>{% block form_title %}{{ _('Map resource') }}{% endblock %}</h1>
|
|
29
|
+
<p>Create a rule bases mapping by filling the form below. It will query the given metadata file and the graph template
|
|
30
|
+
by the Class IRI set for subjects and objects. When clicking "Start Mapping" select widgets will spawn allowing you
|
|
31
|
+
to map a subject to an object. The resulting YAML file will contain a ruleset for each of the assertions made, and
|
|
32
|
+
when run create triples connecting the subject meeting the condition by the predicate IRI given. Download the file,
|
|
33
|
+
make changes if needed and upload it to the "mappings" group here in CKAN if you want to make use of the automated
|
|
34
|
+
mapping process applying the mapping.</p>
|
|
35
|
+
{% block form %}
|
|
36
|
+
{% endblock %}
|
|
37
|
+
<iframe class="col-12" name="my-iframe"
|
|
38
|
+
src="{{iframe_url}}"" onload='javascript:(function(o){o.style.height=o.contentWindow.document.body.scrollHeight+"
|
|
39
|
+
px";}(this));' style="height:1100px;width:100%;border:none;overflow:hidden;"></iframe>
|
|
40
|
+
|
|
41
|
+
{% endblock %}
|
|
42
|
+
|
|
43
|
+
{% block secondary_content %}
|
|
44
|
+
{% snippet 'package/snippets/resource_info.html', res=res %}
|
|
45
|
+
{% snippet 'package/snippets/resources.html', pkg=pkg, active=res.id %}
|
|
46
|
+
{% endblock %}
|
|
47
|
+
|
|
48
|
+
{% block scripts %}
|
|
49
|
+
{{ super() }}
|
|
50
|
+
<!-- <script>
|
|
51
|
+
document.addEventListener("DOMContentLoaded", function() {
|
|
52
|
+
document.getElementById("my-form").submit();
|
|
53
|
+
});
|
|
54
|
+
</script> -->
|
|
55
|
+
{% asset 'vendor/fileupload' %}
|
|
56
|
+
{% endblock %}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
{% extends "package/base.html" %}
|
|
2
|
+
|
|
3
|
+
{% set logged_in = true if c.userobj else false %}
|
|
4
|
+
{% set res = resource %}
|
|
5
|
+
|
|
6
|
+
{%- block styles %}
|
|
7
|
+
{{ super() }}
|
|
8
|
+
{% asset 'csvwmapandtransform/style' %}
|
|
9
|
+
{% endblock %}
|
|
10
|
+
|
|
11
|
+
{% block breadcrumb_content_selected %}{% endblock %}
|
|
12
|
+
|
|
13
|
+
{% block breadcrumb_content %}
|
|
14
|
+
{{ super() }}
|
|
15
|
+
{% if res %}
|
|
16
|
+
<li>{% link_for h.resource_display_name(res)|truncate(30), named_route=pkg.type ~ '_resource.read', id=pkg.name,
|
|
17
|
+
resource_id=res.id %}</li>
|
|
18
|
+
<li{% block breadcrumb_edit_selected %} class="active" {% endblock %}><a href="">{{ _('Transform') }}</a></li>
|
|
19
|
+
{% endif %}
|
|
20
|
+
{% endblock %}
|
|
21
|
+
|
|
22
|
+
{% block content_action %}
|
|
23
|
+
{% if res %}
|
|
24
|
+
{% link_for _('View resource'), named_route=pkg.type ~ '_resource.read', id=pkg.name, resource_id=res.id, class_='btn
|
|
25
|
+
btn-default', icon='eye' %}
|
|
26
|
+
{% endif %}
|
|
27
|
+
{% endblock %}
|
|
28
|
+
|
|
29
|
+
{% block content_primary_nav %}
|
|
30
|
+
<!-- {{ h.build_nav_icon(pkg.type ~ '_resource.edit', _('Edit resource'), id=pkg.name, resource_id=res.id, icon='pencil') }}
|
|
31
|
+
{% block inner_primary_nav %}{% endblock %}
|
|
32
|
+
{{ h.build_nav_icon(pkg.type ~ '_resource.views', _('Views'), id=pkg.name, resource_id=res.id, icon='chart-bar') }} -->
|
|
33
|
+
{% endblock %}
|
|
34
|
+
|
|
35
|
+
{% block primary_content_inner %}
|
|
36
|
+
<h1>{% block form_title %}{{ _('Transform Status') }}{% endblock %}</h1>
|
|
37
|
+
<div id="ajax-status" data-module="csvwmapandtransform" data-api-url="{{ status_url }}">
|
|
38
|
+
{% block form %}
|
|
39
|
+
<form class="add-to-group" method="post">
|
|
40
|
+
{{ h.csrf_input() }}
|
|
41
|
+
<div class="col-12 d-flex align-items-center justify-content-between">
|
|
42
|
+
<div class="d-flex align-items-center">
|
|
43
|
+
<button class="btn btn-secondary me-2 {% if not
|
|
44
|
+
service_status %}disabled{% endif %}" name="extract/update" type="submit">
|
|
45
|
+
<i class="fa fa-refresh"></i> {{ _('Run Transformation') }}
|
|
46
|
+
</button>
|
|
47
|
+
<!-- <button class="btn btn-danger invisible" name="delete" type="submit">
|
|
48
|
+
<i class="fa fa-trash"></i> {{ _('Delete') }}
|
|
49
|
+
</button> -->
|
|
50
|
+
</div>
|
|
51
|
+
<!-- Status Indicator Section -->
|
|
52
|
+
<style>
|
|
53
|
+
.indicator {
|
|
54
|
+
width: 20px;
|
|
55
|
+
height: 20px;
|
|
56
|
+
border-radius: 50%;
|
|
57
|
+
background-color: red;
|
|
58
|
+
/* Default: Service unavailable */
|
|
59
|
+
}
|
|
60
|
+
</style>
|
|
61
|
+
<div class="d-flex align-items-center">
|
|
62
|
+
<div id="service-indicator" class="indicator" data-bs-toggle="tooltip"
|
|
63
|
+
title="{{ _('The status of the service (Green means available, Red means unavailable)') }}" {% if
|
|
64
|
+
service_status %} style="background-color: green;" {% endif %}>
|
|
65
|
+
</div>
|
|
66
|
+
</div>
|
|
67
|
+
</div>
|
|
68
|
+
</form>
|
|
69
|
+
{% endblock %}
|
|
70
|
+
<hr class="mt-0">
|
|
71
|
+
<div class="invisible" name=status>
|
|
72
|
+
<table class="table table-bordered">
|
|
73
|
+
<colgroup>
|
|
74
|
+
<col width="150">
|
|
75
|
+
<col>
|
|
76
|
+
</colgroup>
|
|
77
|
+
<tr>
|
|
78
|
+
<th>{{ _('Status') }}</th>
|
|
79
|
+
<td>{{status.status}}</td>
|
|
80
|
+
</tr>
|
|
81
|
+
<tr>
|
|
82
|
+
<th>{{ _('Last updated') }}</th>
|
|
83
|
+
{% if status.status %}
|
|
84
|
+
<td><span class="date" title="{{ h.render_datetime(status.last_updated, with_hours=True) }}">{{
|
|
85
|
+
h.time_ago_from_timestamp(status.last_updated) }}</span></td>
|
|
86
|
+
{% else %}
|
|
87
|
+
<td>{{ _('Never') }}</td>
|
|
88
|
+
{% endif %}
|
|
89
|
+
</tr>
|
|
90
|
+
</table>
|
|
91
|
+
|
|
92
|
+
<h3>{{ _('Graph Update Log') }}</h3>
|
|
93
|
+
<ul class="activity" name="log">
|
|
94
|
+
</ul>
|
|
95
|
+
</div>
|
|
96
|
+
</div>
|
|
97
|
+
{% endblock %}
|
|
98
|
+
|
|
99
|
+
{% block secondary_content %}
|
|
100
|
+
{% snippet 'package/snippets/resource_info.html', res=res %}
|
|
101
|
+
{% snippet 'package/snippets/resources.html', pkg=pkg, active=res.id %}
|
|
102
|
+
{% endblock %}
|
|
103
|
+
|
|
104
|
+
csvwmapandtransform
|
|
105
|
+
{%- block scripts %}
|
|
106
|
+
{{ super() }}
|
|
107
|
+
{% asset 'csvwmapandtransform/js' %}
|
|
108
|
+
{% endblock %}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
{% ckan_extends %}
|
|
2
|
+
|
|
3
|
+
{% block action_manage %}
|
|
4
|
+
{{ super() }}
|
|
5
|
+
{% if h.csvwmapandtransform_show_tools(res) %}
|
|
6
|
+
<li>{% link_for _('Transform'), named_route='csvwmapandtransform.transform', id=pkg.name, resource_id=res.id, class_='btn btn-secondary', icon='play' %}</li>
|
|
7
|
+
{% endif %}
|
|
8
|
+
{% endblock action_manage %}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
{% ckan_extends %}
|
|
2
|
+
|
|
3
|
+
{% block resource_item_explore_links %}
|
|
4
|
+
{{ super() }}
|
|
5
|
+
{% if not url_is_edit and h.csvwmapandtransform_show_tools(res) %}
|
|
6
|
+
{% if res.url and h.is_url(res.url) %}
|
|
7
|
+
<li>
|
|
8
|
+
<a href="{{ url }}/map" class="dropdown-item resource-url-maptransform-create" target="_blank" rel="noreferrer">
|
|
9
|
+
<i class="fa fa-external-link"></i>
|
|
10
|
+
{{ _('Create Mapping') }}
|
|
11
|
+
</a>
|
|
12
|
+
</li>
|
|
13
|
+
{% if h.check_access('csvwmapandtransform_transform',res) %}
|
|
14
|
+
<li>
|
|
15
|
+
<a href="{{ url }}/transform" class="dropdown-item resource-url-maptransform-run" target="_blank" rel="noreferrer">
|
|
16
|
+
<i class="fa fa-external-link"></i>
|
|
17
|
+
{{ _('Transform') }}
|
|
18
|
+
</a>
|
|
19
|
+
</li>
|
|
20
|
+
{% endif %}
|
|
21
|
+
{% endif %}
|
|
22
|
+
{% endif %}
|
|
23
|
+
{% endblock %}
|
|
File without changes
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
import ckan.lib.base as base
|
|
5
|
+
import ckan.lib.helpers as core_helpers
|
|
6
|
+
import ckan.plugins.toolkit as toolkit
|
|
7
|
+
import requests
|
|
8
|
+
from ckan.common import _
|
|
9
|
+
from flask import Blueprint, Request, request
|
|
10
|
+
from flask.views import MethodView
|
|
11
|
+
|
|
12
|
+
from ckanext.csvwmapandtransform.helpers import csvwmapandtransform_service_available
|
|
13
|
+
|
|
14
|
+
log = __import__("logging").getLogger(__name__)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
blueprint = Blueprint("csvwmapandtransform", __name__)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class TransformView(MethodView):
|
|
21
|
+
def post(self, id: str, resource_id: str):
|
|
22
|
+
try:
|
|
23
|
+
toolkit.get_action("csvwmapandtransform_transform")(
|
|
24
|
+
{}, {"resource_id": resource_id}
|
|
25
|
+
)
|
|
26
|
+
except toolkit.ObjectNotFound:
|
|
27
|
+
base.abort(404, "Dataset not found")
|
|
28
|
+
except toolkit.NotAuthorized:
|
|
29
|
+
base.abort(403, _("Not authorized to see this page"))
|
|
30
|
+
except toolkit.ValidationError:
|
|
31
|
+
log.debug(toolkit.ValidationError)
|
|
32
|
+
|
|
33
|
+
return core_helpers.redirect_to(
|
|
34
|
+
"csvwmapandtransform.transform", id=id, resource_id=resource_id
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
def get(self, id: str, resource_id: str):
|
|
38
|
+
try:
|
|
39
|
+
pkg_dict = toolkit.get_action("package_show")({}, {"id": id})
|
|
40
|
+
resource = toolkit.get_action("resource_show")({}, {"id": resource_id})
|
|
41
|
+
|
|
42
|
+
# backward compatibility with old templates
|
|
43
|
+
toolkit.g.pkg_dict = pkg_dict
|
|
44
|
+
toolkit.g.resource = resource
|
|
45
|
+
|
|
46
|
+
status = toolkit.get_action("csvwmapandtransform_transform_status")(
|
|
47
|
+
{}, {"resource_id": resource_id}
|
|
48
|
+
)
|
|
49
|
+
except toolkit.ObjectNotFound:
|
|
50
|
+
base.abort(404, "Resource not found")
|
|
51
|
+
except toolkit.NotAuthorized:
|
|
52
|
+
base.abort(403, _("Not authorized to see this page"))
|
|
53
|
+
|
|
54
|
+
return base.render(
|
|
55
|
+
"csvwmapandtransform/transform.html",
|
|
56
|
+
extra_vars={
|
|
57
|
+
"pkg_dict": pkg_dict,
|
|
58
|
+
"resource": resource,
|
|
59
|
+
"status": status,
|
|
60
|
+
"status_url": toolkit.url_for(
|
|
61
|
+
"csvwmapandtransform.status",
|
|
62
|
+
id=id,
|
|
63
|
+
resource_id=resource_id,
|
|
64
|
+
qualified=True,
|
|
65
|
+
),
|
|
66
|
+
"service_status": csvwmapandtransform_service_available(),
|
|
67
|
+
"refresh_rate": 10,
|
|
68
|
+
},
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class CreateMapView(MethodView):
|
|
73
|
+
def post(self, id: str, resource_id: str):
|
|
74
|
+
return core_helpers.redirect_to(
|
|
75
|
+
"csvwmapandtransform.map", id=id, resource_id=resource_id
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
def get(self, id: str, resource_id: str):
|
|
79
|
+
try:
|
|
80
|
+
pkg_dict = toolkit.get_action("package_show")({}, {"id": id})
|
|
81
|
+
resource = toolkit.get_action("resource_show")({}, {"id": resource_id})
|
|
82
|
+
|
|
83
|
+
# backward compatibility with old templates
|
|
84
|
+
toolkit.g.pkg_dict = pkg_dict
|
|
85
|
+
toolkit.g.resource = resource
|
|
86
|
+
except toolkit.ObjectNotFound:
|
|
87
|
+
base.abort(404, "Dataset not found")
|
|
88
|
+
except toolkit.NotAuthorized:
|
|
89
|
+
base.abort(403, _("Not authorized to see this page"))
|
|
90
|
+
# iframe_url = toolkit.url_for(
|
|
91
|
+
# "api.action",
|
|
92
|
+
# ver=3,
|
|
93
|
+
# logic_function="csvwmapandtransform_map",
|
|
94
|
+
# qualified=True
|
|
95
|
+
# )
|
|
96
|
+
# iframe_url='http://docker-dev.iwm.fraunhofer.de:6002/'
|
|
97
|
+
iframe_url = toolkit.url_for(
|
|
98
|
+
"csvwmapandtransform.iframe_maptomethod",
|
|
99
|
+
id=id,
|
|
100
|
+
resource_id=resource_id,
|
|
101
|
+
qualified=True,
|
|
102
|
+
)
|
|
103
|
+
return base.render(
|
|
104
|
+
"csvwmapandtransform/create_mapping.html",
|
|
105
|
+
extra_vars={
|
|
106
|
+
"pkg_dict": pkg_dict,
|
|
107
|
+
"resource": resource,
|
|
108
|
+
"iframe_url": iframe_url,
|
|
109
|
+
},
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def iframe_maptomethod(id, resource_id):
|
|
114
|
+
# extra_vars['q'] = q = request.args.get('q', '')
|
|
115
|
+
# if 'data_url' in data_dict:
|
|
116
|
+
# url = data_dict['data_url']
|
|
117
|
+
headers = {
|
|
118
|
+
"Content-Type": "application/json",
|
|
119
|
+
"Authorization": toolkit.config.get("ckanext.csvwmapandtransform.ckan_token"),
|
|
120
|
+
"Accept": "text/html",
|
|
121
|
+
}
|
|
122
|
+
resource_dict = toolkit.get_action("resource_show")({}, {"id": resource_id})
|
|
123
|
+
|
|
124
|
+
# log.debug(request.values)
|
|
125
|
+
data = {
|
|
126
|
+
"data_url": resource_dict["url"],
|
|
127
|
+
"method_url": "https://github.com/Mat-O-Lab/MSEO/raw/main/methods/DIN_EN_ISO_527-3.drawio.ttl",
|
|
128
|
+
"advanced-data_subject_super_class_uris-0": "http://www.w3.org/ns/csvw#Column",
|
|
129
|
+
"advanced-data_subject_super_class_uris-1": "http://www.w3.org/ns/oa#Annotation",
|
|
130
|
+
"advanced-data_mapping_predicate_uri": "http://purl.obolibrary.org/obo/RO_0010002",
|
|
131
|
+
"advanced-method_object_super_class_uris-0": "https://spec.industrialontologies.org/ontology/core/Core/InformationContentEntity",
|
|
132
|
+
"advanced-method_object_super_class_uris-1": "http://purl.obolibrary.org/obo/BFO_0000008",
|
|
133
|
+
}
|
|
134
|
+
ssl_verify = toolkit.config.get("ckanext.csvwmapandtransform.ssl_verify")
|
|
135
|
+
if not ssl_verify:
|
|
136
|
+
requests.packages.urllib3.disable_warnings()
|
|
137
|
+
maptomethod_url = toolkit.config.get("ckanext.csvwmapandtransform.maptomethod_url")
|
|
138
|
+
html = requests.post(
|
|
139
|
+
url=maptomethod_url + "/create_mapper",
|
|
140
|
+
headers=headers,
|
|
141
|
+
data=json.dumps(data),
|
|
142
|
+
verify=ssl_verify,
|
|
143
|
+
)
|
|
144
|
+
# html=requests.post(url="http://docker-dev.iwm.fraunhofer.de:6002"+"/create_mapper", headers=headers, data=json.dumps(data))
|
|
145
|
+
html.raise_for_status()
|
|
146
|
+
result = html.text
|
|
147
|
+
# log.debug("Response from MapToMethod: {}".format(result))
|
|
148
|
+
return result
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
class StatusView(MethodView):
|
|
152
|
+
|
|
153
|
+
def get(self, id: str, resource_id: str):
|
|
154
|
+
pkg_dict = {}
|
|
155
|
+
try:
|
|
156
|
+
pkg_dict = toolkit.get_action("package_show")({}, {"id": id})
|
|
157
|
+
status = toolkit.get_action("csvwmapandtransform_transform_status")(
|
|
158
|
+
{}, {"resource_id": resource_id}
|
|
159
|
+
)
|
|
160
|
+
except toolkit.ObjectNotFound:
|
|
161
|
+
base.abort(404, "Resource not found")
|
|
162
|
+
except toolkit.NotAuthorized:
|
|
163
|
+
base.abort(403, _("Not authorized to see this page"))
|
|
164
|
+
|
|
165
|
+
if status and "logs" in status.keys():
|
|
166
|
+
for index, item in enumerate(status["logs"]):
|
|
167
|
+
status["logs"][index]["timestamp"] = (
|
|
168
|
+
core_helpers.time_ago_from_timestamp(item["timestamp"])
|
|
169
|
+
)
|
|
170
|
+
if item["level"] == "DEBUG":
|
|
171
|
+
status["logs"][index]["alertlevel"] = "info"
|
|
172
|
+
status["logs"][index]["icon"] = "bug-slash"
|
|
173
|
+
status["logs"][index]["class"] = "success"
|
|
174
|
+
elif item["level"] == "INFO":
|
|
175
|
+
status["logs"][index]["alertlevel"] = "info"
|
|
176
|
+
status["logs"][index]["icon"] = "check"
|
|
177
|
+
status["logs"][index]["class"] = "success"
|
|
178
|
+
else:
|
|
179
|
+
status["logs"][index]["alertlevel"] = "error"
|
|
180
|
+
status["logs"][index]["icon"] = "exclamation"
|
|
181
|
+
status["logs"][index]["class"] = "failure"
|
|
182
|
+
return {"pkg_dict": pkg_dict, "status": status}
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
blueprint.add_url_rule(
|
|
186
|
+
"/dataset/<id>/resource/<resource_id>/transform/status",
|
|
187
|
+
view_func=StatusView.as_view(str("status")),
|
|
188
|
+
)
|
|
189
|
+
|
|
190
|
+
blueprint.add_url_rule(
|
|
191
|
+
"/dataset/<id>/resource/<resource_id>/transform",
|
|
192
|
+
view_func=TransformView.as_view(str("transform")),
|
|
193
|
+
)
|
|
194
|
+
blueprint.add_url_rule(
|
|
195
|
+
"/dataset/<id>/resource/<resource_id>/map",
|
|
196
|
+
view_func=CreateMapView.as_view(str("map")),
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
blueprint.add_url_rule(
|
|
200
|
+
"/dataset/<id>/resource/<resource_id>/maptomethod", view_func=iframe_maptomethod
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def get_blueprint():
|
|
205
|
+
return blueprint
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import sys, types, os;p = os.path.join(sys._getframe(1).f_locals['sitedir'], *('ckanext',));importlib = __import__('importlib.util');__import__('importlib.machinery');m = sys.modules.setdefault('ckanext', importlib.util.module_from_spec(importlib.machinery.PathFinder.find_spec('ckanext', [os.path.dirname(p)])));m = m or sys.modules.setdefault('ckanext', types.ModuleType('ckanext'));mp = (m or []) and m.__dict__.setdefault('__path__',[]);(p not in mp) and mp.append(p)
|