ckanext-csvwmapandtransform 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ckanext/csvwmapandtransform/__init__.py +22 -0
- ckanext/csvwmapandtransform/action.py +405 -0
- ckanext/csvwmapandtransform/assets/.gitignore +0 -0
- ckanext/csvwmapandtransform/assets/script.js +81 -0
- ckanext/csvwmapandtransform/assets/style.css +124 -0
- ckanext/csvwmapandtransform/assets/webassets.yml +13 -0
- ckanext/csvwmapandtransform/auth.py +23 -0
- ckanext/csvwmapandtransform/cli.py +18 -0
- ckanext/csvwmapandtransform/db.py +397 -0
- ckanext/csvwmapandtransform/helpers.py +67 -0
- ckanext/csvwmapandtransform/i18n/.gitignore +0 -0
- ckanext/csvwmapandtransform/i18n/ckanext-csvwmapandtransform.pot +108 -0
- ckanext/csvwmapandtransform/i18n/de/LC_MESSAGES/ckanext-csvwmapandtransform.mo +0 -0
- ckanext/csvwmapandtransform/i18n/de/LC_MESSAGES/ckanext-csvwmapandtransform.po +113 -0
- ckanext/csvwmapandtransform/mapper.py +133 -0
- ckanext/csvwmapandtransform/plugin.py +140 -0
- ckanext/csvwmapandtransform/public/.gitignore +0 -0
- ckanext/csvwmapandtransform/public/dotted.png +0 -0
- ckanext/csvwmapandtransform/tasks.py +262 -0
- ckanext/csvwmapandtransform/templates/.gitignore +0 -0
- ckanext/csvwmapandtransform/templates/csvwmapandtransform/create_mapping.html +56 -0
- ckanext/csvwmapandtransform/templates/csvwmapandtransform/transform.html +108 -0
- ckanext/csvwmapandtransform/templates/package/resource_read.html +8 -0
- ckanext/csvwmapandtransform/templates/package/snippets/resource_item.html +23 -0
- ckanext/csvwmapandtransform/tests/__init__.py +0 -0
- ckanext/csvwmapandtransform/views.py +205 -0
- ckanext_csvwmapandtransform-0.0.1-py3.14-nspkg.pth +1 -0
- ckanext_csvwmapandtransform-0.0.1.dist-info/METADATA +121 -0
- ckanext_csvwmapandtransform-0.0.1.dist-info/RECORD +34 -0
- ckanext_csvwmapandtransform-0.0.1.dist-info/WHEEL +5 -0
- ckanext_csvwmapandtransform-0.0.1.dist-info/entry_points.txt +5 -0
- ckanext_csvwmapandtransform-0.0.1.dist-info/licenses/LICENSE +661 -0
- ckanext_csvwmapandtransform-0.0.1.dist-info/namespace_packages.txt +1 -0
- ckanext_csvwmapandtransform-0.0.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
{% extends "package/base.html" %}
|
|
2
|
+
|
|
3
|
+
{% set logged_in = true if c.userobj else false %}
|
|
4
|
+
{% set res = resource %}
|
|
5
|
+
|
|
6
|
+
{%- block styles %}
|
|
7
|
+
{{ super() }}
|
|
8
|
+
{% asset 'csvwmapandtransform/style' %}
|
|
9
|
+
{% endblock %}
|
|
10
|
+
|
|
11
|
+
{% block breadcrumb_content_selected %}{% endblock %}
|
|
12
|
+
|
|
13
|
+
{% block breadcrumb_content %}
|
|
14
|
+
{{ super() }}
|
|
15
|
+
{% if res %}
|
|
16
|
+
<li>{% link_for h.resource_display_name(res)|truncate(30), named_route=pkg.type ~ '_resource.read', id=pkg.name,
|
|
17
|
+
resource_id=res.id %}</li>
|
|
18
|
+
<li{% block breadcrumb_edit_selected %} class="active" {% endblock %}><a href="">{{ _('Transform') }}</a></li>
|
|
19
|
+
{% endif %}
|
|
20
|
+
{% endblock %}
|
|
21
|
+
|
|
22
|
+
{% block content_action %}
|
|
23
|
+
{% if res %}
|
|
24
|
+
{% link_for _('View resource'), named_route=pkg.type ~ '_resource.read', id=pkg.name, resource_id=res.id, class_='btn
|
|
25
|
+
btn-default', icon='eye' %}
|
|
26
|
+
{% endif %}
|
|
27
|
+
{% endblock %}
|
|
28
|
+
|
|
29
|
+
{% block content_primary_nav %}
|
|
30
|
+
<!-- {{ h.build_nav_icon(pkg.type ~ '_resource.edit', _('Edit resource'), id=pkg.name, resource_id=res.id, icon='pencil') }}
|
|
31
|
+
{% block inner_primary_nav %}{% endblock %}
|
|
32
|
+
{{ h.build_nav_icon(pkg.type ~ '_resource.views', _('Views'), id=pkg.name, resource_id=res.id, icon='chart-bar') }} -->
|
|
33
|
+
{% endblock %}
|
|
34
|
+
|
|
35
|
+
{% block primary_content_inner %}
|
|
36
|
+
<h1>{% block form_title %}{{ _('Transform Status') }}{% endblock %}</h1>
|
|
37
|
+
<div id="ajax-status" data-module="csvwmapandtransform" data-api-url="{{ status_url }}">
|
|
38
|
+
{% block form %}
|
|
39
|
+
<form class="add-to-group" method="post">
|
|
40
|
+
{{ h.csrf_input() }}
|
|
41
|
+
<div class="col-12 d-flex align-items-center justify-content-between">
|
|
42
|
+
<div class="d-flex align-items-center">
|
|
43
|
+
<button class="btn btn-secondary me-2 {% if not
|
|
44
|
+
service_status %}disabled{% endif %}" name="extract/update" type="submit">
|
|
45
|
+
<i class="fa fa-refresh"></i> {{ _('Run Transformation') }}
|
|
46
|
+
</button>
|
|
47
|
+
<!-- <button class="btn btn-danger invisible" name="delete" type="submit">
|
|
48
|
+
<i class="fa fa-trash"></i> {{ _('Delete') }}
|
|
49
|
+
</button> -->
|
|
50
|
+
</div>
|
|
51
|
+
<!-- Status Indicator Section -->
|
|
52
|
+
<style>
|
|
53
|
+
.indicator {
|
|
54
|
+
width: 20px;
|
|
55
|
+
height: 20px;
|
|
56
|
+
border-radius: 50%;
|
|
57
|
+
background-color: red;
|
|
58
|
+
/* Default: Service unavailable */
|
|
59
|
+
}
|
|
60
|
+
</style>
|
|
61
|
+
<div class="d-flex align-items-center">
|
|
62
|
+
<div id="service-indicator" class="indicator" data-bs-toggle="tooltip"
|
|
63
|
+
title="{{ _('The status of the service (Green means available, Red means unavailable)') }}" {% if
|
|
64
|
+
service_status %} style="background-color: green;" {% endif %}>
|
|
65
|
+
</div>
|
|
66
|
+
</div>
|
|
67
|
+
</div>
|
|
68
|
+
</form>
|
|
69
|
+
{% endblock %}
|
|
70
|
+
<hr class="mt-0">
|
|
71
|
+
<div class="invisible" name=status>
|
|
72
|
+
<table class="table table-bordered">
|
|
73
|
+
<colgroup>
|
|
74
|
+
<col width="150">
|
|
75
|
+
<col>
|
|
76
|
+
</colgroup>
|
|
77
|
+
<tr>
|
|
78
|
+
<th>{{ _('Status') }}</th>
|
|
79
|
+
<td>{{status.status}}</td>
|
|
80
|
+
</tr>
|
|
81
|
+
<tr>
|
|
82
|
+
<th>{{ _('Last updated') }}</th>
|
|
83
|
+
{% if status.status %}
|
|
84
|
+
<td><span class="date" title="{{ h.render_datetime(status.last_updated, with_hours=True) }}">{{
|
|
85
|
+
h.time_ago_from_timestamp(status.last_updated) }}</span></td>
|
|
86
|
+
{% else %}
|
|
87
|
+
<td>{{ _('Never') }}</td>
|
|
88
|
+
{% endif %}
|
|
89
|
+
</tr>
|
|
90
|
+
</table>
|
|
91
|
+
|
|
92
|
+
<h3>{{ _('Graph Update Log') }}</h3>
|
|
93
|
+
<ul class="activity" name="log">
|
|
94
|
+
</ul>
|
|
95
|
+
</div>
|
|
96
|
+
</div>
|
|
97
|
+
{% endblock %}
|
|
98
|
+
|
|
99
|
+
{% block secondary_content %}
|
|
100
|
+
{% snippet 'package/snippets/resource_info.html', res=res %}
|
|
101
|
+
{% snippet 'package/snippets/resources.html', pkg=pkg, active=res.id %}
|
|
102
|
+
{% endblock %}
|
|
103
|
+
|
|
104
|
+
csvwmapandtransform
|
|
105
|
+
{%- block scripts %}
|
|
106
|
+
{{ super() }}
|
|
107
|
+
{% asset 'csvwmapandtransform/js' %}
|
|
108
|
+
{% endblock %}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
{% ckan_extends %}
|
|
2
|
+
|
|
3
|
+
{% block action_manage %}
|
|
4
|
+
{{ super() }}
|
|
5
|
+
{% if h.csvwmapandtransform_show_tools(res) %}
|
|
6
|
+
<li>{% link_for _('Transform'), named_route='csvwmapandtransform.transform', id=pkg.name, resource_id=res.id, class_='btn btn-secondary', icon='play' %}</li>
|
|
7
|
+
{% endif %}
|
|
8
|
+
{% endblock action_manage %}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
{% ckan_extends %}
|
|
2
|
+
|
|
3
|
+
{% block resource_item_explore_links %}
|
|
4
|
+
{{ super() }}
|
|
5
|
+
{% if not url_is_edit and h.csvwmapandtransform_show_tools(res) %}
|
|
6
|
+
{% if res.url and h.is_url(res.url) %}
|
|
7
|
+
<li>
|
|
8
|
+
<a href="{{ url }}/map" class="dropdown-item resource-url-maptransform-create" target="_blank" rel="noreferrer">
|
|
9
|
+
<i class="fa fa-external-link"></i>
|
|
10
|
+
{{ _('Create Mapping') }}
|
|
11
|
+
</a>
|
|
12
|
+
</li>
|
|
13
|
+
{% if h.check_access('csvwmapandtransform_transform',res) %}
|
|
14
|
+
<li>
|
|
15
|
+
<a href="{{ url }}/transform" class="dropdown-item resource-url-maptransform-run" target="_blank" rel="noreferrer">
|
|
16
|
+
<i class="fa fa-external-link"></i>
|
|
17
|
+
{{ _('Transform') }}
|
|
18
|
+
</a>
|
|
19
|
+
</li>
|
|
20
|
+
{% endif %}
|
|
21
|
+
{% endif %}
|
|
22
|
+
{% endif %}
|
|
23
|
+
{% endblock %}
|
|
File without changes
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
import ckan.lib.base as base
|
|
5
|
+
import ckan.lib.helpers as core_helpers
|
|
6
|
+
import ckan.plugins.toolkit as toolkit
|
|
7
|
+
import requests
|
|
8
|
+
from ckan.common import _
|
|
9
|
+
from flask import Blueprint, Request, request
|
|
10
|
+
from flask.views import MethodView
|
|
11
|
+
|
|
12
|
+
from ckanext.csvwmapandtransform.helpers import csvwmapandtransform_service_available
|
|
13
|
+
|
|
14
|
+
log = __import__("logging").getLogger(__name__)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
blueprint = Blueprint("csvwmapandtransform", __name__)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class TransformView(MethodView):
|
|
21
|
+
def post(self, id: str, resource_id: str):
|
|
22
|
+
try:
|
|
23
|
+
toolkit.get_action("csvwmapandtransform_transform")(
|
|
24
|
+
{}, {"resource_id": resource_id}
|
|
25
|
+
)
|
|
26
|
+
except toolkit.ObjectNotFound:
|
|
27
|
+
base.abort(404, "Dataset not found")
|
|
28
|
+
except toolkit.NotAuthorized:
|
|
29
|
+
base.abort(403, _("Not authorized to see this page"))
|
|
30
|
+
except toolkit.ValidationError:
|
|
31
|
+
log.debug(toolkit.ValidationError)
|
|
32
|
+
|
|
33
|
+
return core_helpers.redirect_to(
|
|
34
|
+
"csvwmapandtransform.transform", id=id, resource_id=resource_id
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
def get(self, id: str, resource_id: str):
|
|
38
|
+
try:
|
|
39
|
+
pkg_dict = toolkit.get_action("package_show")({}, {"id": id})
|
|
40
|
+
resource = toolkit.get_action("resource_show")({}, {"id": resource_id})
|
|
41
|
+
|
|
42
|
+
# backward compatibility with old templates
|
|
43
|
+
toolkit.g.pkg_dict = pkg_dict
|
|
44
|
+
toolkit.g.resource = resource
|
|
45
|
+
|
|
46
|
+
status = toolkit.get_action("csvwmapandtransform_transform_status")(
|
|
47
|
+
{}, {"resource_id": resource_id}
|
|
48
|
+
)
|
|
49
|
+
except toolkit.ObjectNotFound:
|
|
50
|
+
base.abort(404, "Resource not found")
|
|
51
|
+
except toolkit.NotAuthorized:
|
|
52
|
+
base.abort(403, _("Not authorized to see this page"))
|
|
53
|
+
|
|
54
|
+
return base.render(
|
|
55
|
+
"csvwmapandtransform/transform.html",
|
|
56
|
+
extra_vars={
|
|
57
|
+
"pkg_dict": pkg_dict,
|
|
58
|
+
"resource": resource,
|
|
59
|
+
"status": status,
|
|
60
|
+
"status_url": toolkit.url_for(
|
|
61
|
+
"csvwmapandtransform.status",
|
|
62
|
+
id=id,
|
|
63
|
+
resource_id=resource_id,
|
|
64
|
+
qualified=True,
|
|
65
|
+
),
|
|
66
|
+
"service_status": csvwmapandtransform_service_available(),
|
|
67
|
+
"refresh_rate": 10,
|
|
68
|
+
},
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class CreateMapView(MethodView):
|
|
73
|
+
def post(self, id: str, resource_id: str):
|
|
74
|
+
return core_helpers.redirect_to(
|
|
75
|
+
"csvwmapandtransform.map", id=id, resource_id=resource_id
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
def get(self, id: str, resource_id: str):
|
|
79
|
+
try:
|
|
80
|
+
pkg_dict = toolkit.get_action("package_show")({}, {"id": id})
|
|
81
|
+
resource = toolkit.get_action("resource_show")({}, {"id": resource_id})
|
|
82
|
+
|
|
83
|
+
# backward compatibility with old templates
|
|
84
|
+
toolkit.g.pkg_dict = pkg_dict
|
|
85
|
+
toolkit.g.resource = resource
|
|
86
|
+
except toolkit.ObjectNotFound:
|
|
87
|
+
base.abort(404, "Dataset not found")
|
|
88
|
+
except toolkit.NotAuthorized:
|
|
89
|
+
base.abort(403, _("Not authorized to see this page"))
|
|
90
|
+
# iframe_url = toolkit.url_for(
|
|
91
|
+
# "api.action",
|
|
92
|
+
# ver=3,
|
|
93
|
+
# logic_function="csvwmapandtransform_map",
|
|
94
|
+
# qualified=True
|
|
95
|
+
# )
|
|
96
|
+
# iframe_url='http://docker-dev.iwm.fraunhofer.de:6002/'
|
|
97
|
+
iframe_url = toolkit.url_for(
|
|
98
|
+
"csvwmapandtransform.iframe_maptomethod",
|
|
99
|
+
id=id,
|
|
100
|
+
resource_id=resource_id,
|
|
101
|
+
qualified=True,
|
|
102
|
+
)
|
|
103
|
+
return base.render(
|
|
104
|
+
"csvwmapandtransform/create_mapping.html",
|
|
105
|
+
extra_vars={
|
|
106
|
+
"pkg_dict": pkg_dict,
|
|
107
|
+
"resource": resource,
|
|
108
|
+
"iframe_url": iframe_url,
|
|
109
|
+
},
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def iframe_maptomethod(id, resource_id):
|
|
114
|
+
# extra_vars['q'] = q = request.args.get('q', '')
|
|
115
|
+
# if 'data_url' in data_dict:
|
|
116
|
+
# url = data_dict['data_url']
|
|
117
|
+
headers = {
|
|
118
|
+
"Content-Type": "application/json",
|
|
119
|
+
"Authorization": toolkit.config.get("ckanext.csvwmapandtransform.ckan_token"),
|
|
120
|
+
"Accept": "text/html",
|
|
121
|
+
}
|
|
122
|
+
resource_dict = toolkit.get_action("resource_show")({}, {"id": resource_id})
|
|
123
|
+
|
|
124
|
+
# log.debug(request.values)
|
|
125
|
+
data = {
|
|
126
|
+
"data_url": resource_dict["url"],
|
|
127
|
+
"method_url": "https://github.com/Mat-O-Lab/MSEO/raw/main/methods/DIN_EN_ISO_527-3.drawio.ttl",
|
|
128
|
+
"advanced-data_subject_super_class_uris-0": "http://www.w3.org/ns/csvw#Column",
|
|
129
|
+
"advanced-data_subject_super_class_uris-1": "http://www.w3.org/ns/oa#Annotation",
|
|
130
|
+
"advanced-data_mapping_predicate_uri": "http://purl.obolibrary.org/obo/RO_0010002",
|
|
131
|
+
"advanced-method_object_super_class_uris-0": "https://spec.industrialontologies.org/ontology/core/Core/InformationContentEntity",
|
|
132
|
+
"advanced-method_object_super_class_uris-1": "http://purl.obolibrary.org/obo/BFO_0000008",
|
|
133
|
+
}
|
|
134
|
+
ssl_verify = toolkit.config.get("ckanext.csvwmapandtransform.ssl_verify")
|
|
135
|
+
if not ssl_verify:
|
|
136
|
+
requests.packages.urllib3.disable_warnings()
|
|
137
|
+
maptomethod_url = toolkit.config.get("ckanext.csvwmapandtransform.maptomethod_url")
|
|
138
|
+
html = requests.post(
|
|
139
|
+
url=maptomethod_url + "/create_mapper",
|
|
140
|
+
headers=headers,
|
|
141
|
+
data=json.dumps(data),
|
|
142
|
+
verify=ssl_verify,
|
|
143
|
+
)
|
|
144
|
+
# html=requests.post(url="http://docker-dev.iwm.fraunhofer.de:6002"+"/create_mapper", headers=headers, data=json.dumps(data))
|
|
145
|
+
html.raise_for_status()
|
|
146
|
+
result = html.text
|
|
147
|
+
# log.debug("Response from MapToMethod: {}".format(result))
|
|
148
|
+
return result
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
class StatusView(MethodView):
|
|
152
|
+
|
|
153
|
+
def get(self, id: str, resource_id: str):
|
|
154
|
+
pkg_dict = {}
|
|
155
|
+
try:
|
|
156
|
+
pkg_dict = toolkit.get_action("package_show")({}, {"id": id})
|
|
157
|
+
status = toolkit.get_action("csvwmapandtransform_transform_status")(
|
|
158
|
+
{}, {"resource_id": resource_id}
|
|
159
|
+
)
|
|
160
|
+
except toolkit.ObjectNotFound:
|
|
161
|
+
base.abort(404, "Resource not found")
|
|
162
|
+
except toolkit.NotAuthorized:
|
|
163
|
+
base.abort(403, _("Not authorized to see this page"))
|
|
164
|
+
|
|
165
|
+
if status and "logs" in status.keys():
|
|
166
|
+
for index, item in enumerate(status["logs"]):
|
|
167
|
+
status["logs"][index]["timestamp"] = (
|
|
168
|
+
core_helpers.time_ago_from_timestamp(item["timestamp"])
|
|
169
|
+
)
|
|
170
|
+
if item["level"] == "DEBUG":
|
|
171
|
+
status["logs"][index]["alertlevel"] = "info"
|
|
172
|
+
status["logs"][index]["icon"] = "bug-slash"
|
|
173
|
+
status["logs"][index]["class"] = "success"
|
|
174
|
+
elif item["level"] == "INFO":
|
|
175
|
+
status["logs"][index]["alertlevel"] = "info"
|
|
176
|
+
status["logs"][index]["icon"] = "check"
|
|
177
|
+
status["logs"][index]["class"] = "success"
|
|
178
|
+
else:
|
|
179
|
+
status["logs"][index]["alertlevel"] = "error"
|
|
180
|
+
status["logs"][index]["icon"] = "exclamation"
|
|
181
|
+
status["logs"][index]["class"] = "failure"
|
|
182
|
+
return {"pkg_dict": pkg_dict, "status": status}
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
blueprint.add_url_rule(
|
|
186
|
+
"/dataset/<id>/resource/<resource_id>/transform/status",
|
|
187
|
+
view_func=StatusView.as_view(str("status")),
|
|
188
|
+
)
|
|
189
|
+
|
|
190
|
+
blueprint.add_url_rule(
|
|
191
|
+
"/dataset/<id>/resource/<resource_id>/transform",
|
|
192
|
+
view_func=TransformView.as_view(str("transform")),
|
|
193
|
+
)
|
|
194
|
+
blueprint.add_url_rule(
|
|
195
|
+
"/dataset/<id>/resource/<resource_id>/map",
|
|
196
|
+
view_func=CreateMapView.as_view(str("map")),
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
blueprint.add_url_rule(
|
|
200
|
+
"/dataset/<id>/resource/<resource_id>/maptomethod", view_func=iframe_maptomethod
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def get_blueprint():
|
|
205
|
+
return blueprint
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import sys, types, os;p = os.path.join(sys._getframe(1).f_locals['sitedir'], *('ckanext',));importlib = __import__('importlib.util');__import__('importlib.machinery');m = sys.modules.setdefault('ckanext', importlib.util.module_from_spec(importlib.machinery.PathFinder.find_spec('ckanext', [os.path.dirname(p)])));m = m or sys.modules.setdefault('ckanext', types.ModuleType('ckanext'));mp = (m or []) and m.__dict__.setdefault('__path__',[]);(p not in mp) and mp.append(p)
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ckanext-csvwmapandtransform
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: CKAN extension facilitating YARRRML mappings in conjunction with RDFConverter to run automatic transforms matching best fitting maps as a background job.
|
|
5
|
+
Home-page: https://github.com/Mat-O-Lab/ckanext-csvwmapandtransform
|
|
6
|
+
Author: Thomas Hanke
|
|
7
|
+
Author-email: Thomas Hanke <thomas.hanke@iwm.fraunhofer.de>
|
|
8
|
+
License: AGPL
|
|
9
|
+
Project-URL: Homepage, https://github.com/Mat-O-Lab/ckanext-csvwmapandtransform
|
|
10
|
+
Keywords: CKAN,CSVW,CSV,JSON-LD
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.7
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Requires-Python: >=3.7
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
License-File: LICENSE
|
|
20
|
+
Requires-Dist: ckanapi
|
|
21
|
+
Requires-Dist: requests>=2.4.2
|
|
22
|
+
Provides-Extra: dev
|
|
23
|
+
Requires-Dist: pytest; extra == "dev"
|
|
24
|
+
Requires-Dist: pytest-ckan; extra == "dev"
|
|
25
|
+
Dynamic: license-file
|
|
26
|
+
|
|
27
|
+
[](https://github.com/Mat-O-Lab/ckanext-csvwmapandtransform/actions/workflows/test.yml)
|
|
28
|
+
|
|
29
|
+
# ckanext-csvwmapandtransform
|
|
30
|
+
|
|
31
|
+
Extension automatically generating csvw metadata for uploaded textual tabular data. It uploads the data of the first table documented into a datastore for the source csv file.
|
|
32
|
+
|
|
33
|
+
## Requirements
|
|
34
|
+
Needs a running instance of the [MapToMethod Application](https://github.com/Mat-O-Lab/MapToMethod) and [RDFConverter Application](https://github.com/Mat-O-Lab/RDFConverter)
|
|
35
|
+
Point at it through env variables.
|
|
36
|
+
Also needed is a Api Token for an account with the right privaledges to make the background job work on private datasets and ressources.
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
CSVWMAPANDTRANSFORM_TOKEN=${CKAN_API_TOKEN}
|
|
40
|
+
MAPTOMETHOD_CONTAINER_NAME="ckan_maptomethod"
|
|
41
|
+
MAPTOMETHOD_APP_PORT=5002
|
|
42
|
+
# must be reachable from outside container net or iframe wil not work
|
|
43
|
+
CKAN_MAPTOMETHOD_URL=http://<CKAN_HOST>:${MAPTOMETHOD_APP_PORT}
|
|
44
|
+
RDFCONVERTER_CONTAINER_NAME="ckan_rdfconverter"
|
|
45
|
+
RDFCONVERTER_APP_PORT=5003
|
|
46
|
+
CKAN_RDFCONVERTER_URL=http://${RDFCONVERTER_CONTAINER_NAME}:${RDFCONVERTER_APP_PORT}
|
|
47
|
+
CSVWMAPANDTRANSFORM_SQLALCHEMY_URL=postgresql://<ckandbuser>:<ckandbpassword>@<db>/ckandb
|
|
48
|
+
PARSER_PORT=3001
|
|
49
|
+
MAPPER_PORT=4000
|
|
50
|
+
CONVERTER_PORT=5000
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
You can set the default formats to run trusformation on by setting the env variable CSVWMAPANDTRANSFORM_FORMATS for example
|
|
54
|
+
```bash
|
|
55
|
+
CSVWMAPANDTRANSFORM_FORMATS="json-ld turtle n3 nt hext trig longturtle xml"
|
|
56
|
+
```
|
|
57
|
+
else it will react to the following formats: "json json-ld turtle n3 nt hext trig longturtle xml"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
CSVWMAPANDTRANSFORM_FORMATS="json json-ld turtle n3 nt hext trig longturtle xml"
|
|
61
|
+
|
|
62
|
+
Compatibility with core CKAN versions:
|
|
63
|
+
|
|
64
|
+
| CKAN version | Compatible? |
|
|
65
|
+
| --------------- | ------------- |
|
|
66
|
+
| 2.9 and arlier | not tested |
|
|
67
|
+
| 2.10 | yes |
|
|
68
|
+
| 2.11 | yes |
|
|
69
|
+
|
|
70
|
+
* "yes"
|
|
71
|
+
* "not tested" - I can't think of a reason why it wouldn't work
|
|
72
|
+
* "not yet" - there is an intention to get it working
|
|
73
|
+
* "no"
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
## Installation
|
|
77
|
+
|
|
78
|
+
To install the extension:
|
|
79
|
+
|
|
80
|
+
1. Activate your CKAN virtual environment, for example:
|
|
81
|
+
```bash
|
|
82
|
+
. /usr/lib/ckan/default/bin/activate
|
|
83
|
+
```
|
|
84
|
+
2. Use pip to install package
|
|
85
|
+
```bash
|
|
86
|
+
pip install ckanext-csvwmapandtransform
|
|
87
|
+
```
|
|
88
|
+
3. Add `csvwmapandtransform` to the `ckan.plugins` setting in your CKAN
|
|
89
|
+
config file (by default the config file is located at
|
|
90
|
+
`/etc/ckan/default/ckan.ini`).
|
|
91
|
+
|
|
92
|
+
4. Restart CKAN. For example, if you've deployed CKAN with Apache on Ubuntu:
|
|
93
|
+
```bash
|
|
94
|
+
sudo service apache2 reload
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
## Developer installation
|
|
98
|
+
|
|
99
|
+
To install ckanext-csvtocsvw for development, activate your CKAN virtualenv and
|
|
100
|
+
do:
|
|
101
|
+
```bash
|
|
102
|
+
git clone https://github.com/Mat-O-Lab/ckanext-csvtocsvw.git
|
|
103
|
+
cd ckanext-csvtocsvw
|
|
104
|
+
python setup.py develop
|
|
105
|
+
pip install -r dev-requirements.txt
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
## Tests
|
|
109
|
+
|
|
110
|
+
To run the tests, do:
|
|
111
|
+
```bash
|
|
112
|
+
pytest --ckan-ini=test.ini
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
## License
|
|
116
|
+
|
|
117
|
+
[AGPL](https://www.gnu.org/licenses/agpl-3.0.en.html)
|
|
118
|
+
|
|
119
|
+
# Acknowledgments
|
|
120
|
+
The authors would like to thank the Federal Government and the Heads of Government of the Länder for their funding and support within the framework of the [Platform Material Digital](https://www.materialdigital.de) consortium. Funded by the German [Federal Ministry of Education and Research (BMBF)](https://www.bmbf.de/bmbf/en/) through the [MaterialDigital](https://www.bmbf.de/SharedDocs/Publikationen/de/bmbf/5/31701_MaterialDigital.pdf?__blob=publicationFile&v=5) Call in Project [KupferDigital](https://www.materialdigital.de/project/1) - project id 13XP5119.
|
|
121
|
+
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
ckanext_csvwmapandtransform-0.0.1-py3.14-nspkg.pth,sha256=NXfLjB2L7-vMVBgj11KMWlHly-aPPPj1kuWUW7A1iMM,472
|
|
2
|
+
ckanext/csvwmapandtransform/__init__.py,sha256=45GqWeH6dqfasWe8njaOdZUCjfADMLW2aurlRaVVzRQ,575
|
|
3
|
+
ckanext/csvwmapandtransform/action.py,sha256=9dVUJ4MBQ0hbUbRPLXBbYlcBnhqWASos-wFVFQ3Vyrg,14534
|
|
4
|
+
ckanext/csvwmapandtransform/auth.py,sha256=RlZkmr579ilbeV6r74kYnZo15KZdgaNs52rMa8mAN8E,767
|
|
5
|
+
ckanext/csvwmapandtransform/cli.py,sha256=Fts-S1FktmmvsThWdCRVETH4CTEBUgHL932sV2e2LFs,367
|
|
6
|
+
ckanext/csvwmapandtransform/db.py,sha256=z11lJSh1y1f31BKWWAcQ8mpaQAAhNie2rd_BhHrPTqw,11602
|
|
7
|
+
ckanext/csvwmapandtransform/helpers.py,sha256=zN1PYewUZITaaogh6AJYYkQd4bMg2zI89eo5tV7t1Eo,2167
|
|
8
|
+
ckanext/csvwmapandtransform/mapper.py,sha256=KEfzIqt4De_o3UPU3L2OGDCMoQhhejLrSPg2io3C_cY,5293
|
|
9
|
+
ckanext/csvwmapandtransform/plugin.py,sha256=EWlX958yL78jcKDmFVL4s5MeQ86K7YvDb09wjSD68G4,4492
|
|
10
|
+
ckanext/csvwmapandtransform/tasks.py,sha256=z4uFJmDJMXbm3_6bgve0GSbBdNaXyryfyG_TyCkLVwM,9505
|
|
11
|
+
ckanext/csvwmapandtransform/views.py,sha256=hR_I7HRbgyag81ZUxXOoNEb7fhGecDCgaoVrF6D4dC8,7674
|
|
12
|
+
ckanext/csvwmapandtransform/assets/.gitignore,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
13
|
+
ckanext/csvwmapandtransform/assets/script.js,sha256=iuPqQExUeAdedPNdgstzqwZm0-d_2IRE_wc1EzFCZzA,3306
|
|
14
|
+
ckanext/csvwmapandtransform/assets/style.css,sha256=ENKH6-gHFt7S6AExhyAPVlyni3fPWAALdcklSqmjbA0,1888
|
|
15
|
+
ckanext/csvwmapandtransform/assets/webassets.yml,sha256=OjnUVX9TGimTLdg1C4mnDHIvZmmzPVISvz3e-yHrrms,232
|
|
16
|
+
ckanext/csvwmapandtransform/i18n/.gitignore,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
17
|
+
ckanext/csvwmapandtransform/i18n/ckanext-csvwmapandtransform.pot,sha256=iLncmy-DJCgnJOPCmrkvVp4wGWaCwMyTKZsMgTTEJXE,3202
|
|
18
|
+
ckanext/csvwmapandtransform/i18n/de/LC_MESSAGES/ckanext-csvwmapandtransform.mo,sha256=TEGIWw5ZT0MJcSJNYUHB9ki5HbxNELTo-QTPZ0d0O0M,1509
|
|
19
|
+
ckanext/csvwmapandtransform/i18n/de/LC_MESSAGES/ckanext-csvwmapandtransform.po,sha256=-kopC1jWqOKB18eSbcGRo3bbCQWA9AG4gpMmfVEpv_c,3628
|
|
20
|
+
ckanext/csvwmapandtransform/public/.gitignore,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
21
|
+
ckanext/csvwmapandtransform/public/dotted.png,sha256=TOPWzPIWZ7wIdMvJLhTO8tKaBnPgq0hYJrtr-NXx7BE,74
|
|
22
|
+
ckanext/csvwmapandtransform/templates/.gitignore,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
23
|
+
ckanext/csvwmapandtransform/templates/csvwmapandtransform/create_mapping.html,sha256=Pc400O-bM7GM-bcw-O-rgPOoj0dvhCKBUMY60hT343A,2269
|
|
24
|
+
ckanext/csvwmapandtransform/templates/csvwmapandtransform/transform.html,sha256=Cl8HnMifTdlEGKm8J8agAkTB5Hxgb_7-zoP-ednXXlU,3798
|
|
25
|
+
ckanext/csvwmapandtransform/templates/package/resource_read.html,sha256=b4Lj42qABgibIcVvDmU3CRTf708sQItvut8JDGAkB-8,305
|
|
26
|
+
ckanext/csvwmapandtransform/templates/package/snippets/resource_item.html,sha256=ke8ZUk2EtwmF2QAuGA9EGzR164qyLhS_wW4riEkLY28,697
|
|
27
|
+
ckanext/csvwmapandtransform/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
28
|
+
ckanext_csvwmapandtransform-0.0.1.dist-info/licenses/LICENSE,sha256=2lWcRAHjsQhqavGNnR30Ymxq3GJ9BaYL_dnfGO_-WFA,34500
|
|
29
|
+
ckanext_csvwmapandtransform-0.0.1.dist-info/METADATA,sha256=vwkP49brN9L6f2ETrYQzus35UAosm5eZhJQhfPctDEg,4688
|
|
30
|
+
ckanext_csvwmapandtransform-0.0.1.dist-info/WHEEL,sha256=YCfwYGOYMi5Jhw2fU4yNgwErybb2IX5PEwBKV4ZbdBo,91
|
|
31
|
+
ckanext_csvwmapandtransform-0.0.1.dist-info/entry_points.txt,sha256=T-gUgTQoMeBkVeVM91x0mryuaZhHMWDPJ2v-z5-Bm_s,155
|
|
32
|
+
ckanext_csvwmapandtransform-0.0.1.dist-info/namespace_packages.txt,sha256=5yjNwq-s42weaiMMUuA5lZ45g99ANsfcRBCvac1JMS4,8
|
|
33
|
+
ckanext_csvwmapandtransform-0.0.1.dist-info/top_level.txt,sha256=5yjNwq-s42weaiMMUuA5lZ45g99ANsfcRBCvac1JMS4,8
|
|
34
|
+
ckanext_csvwmapandtransform-0.0.1.dist-info/RECORD,,
|