otlmow-template 0.11rc3__py3-none-any.whl → 1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- otlmow_template/SubsetTemplateCreator.py +504 -462
- {otlmow_template-0.11rc3.dist-info → otlmow_template-1.0.dist-info}/METADATA +4 -4
- otlmow_template-1.0.dist-info/RECORD +7 -0
- {otlmow_template-0.11rc3.dist-info → otlmow_template-1.0.dist-info}/WHEEL +1 -1
- otlmow_template/CsvTemplateCreator.py +0 -155
- otlmow_template/ExcelTemplateCreator.py +0 -199
- otlmow_template/__init__.py +0 -0
- otlmow_template-0.11rc3.dist-info/RECORD +0 -10
- {otlmow_template-0.11rc3.dist-info → otlmow_template-1.0.dist-info/licenses}/LICENSE +0 -0
- {otlmow_template-0.11rc3.dist-info → otlmow_template-1.0.dist-info}/top_level.txt +0 -0
|
@@ -1,28 +1,30 @@
|
|
|
1
|
-
import
|
|
1
|
+
import contextlib
|
|
2
2
|
import csv
|
|
3
3
|
import logging
|
|
4
|
-
import ntpath
|
|
5
4
|
import os
|
|
6
|
-
import
|
|
5
|
+
import shutil
|
|
7
6
|
import tempfile
|
|
8
7
|
from asyncio import sleep
|
|
8
|
+
from collections import defaultdict
|
|
9
9
|
from pathlib import Path
|
|
10
10
|
|
|
11
|
-
|
|
12
11
|
from openpyxl.reader.excel import load_workbook
|
|
13
12
|
from openpyxl.styles import PatternFill, Alignment
|
|
14
13
|
from openpyxl.utils import get_column_letter
|
|
14
|
+
from openpyxl.workbook import Workbook
|
|
15
15
|
from openpyxl.worksheet.datavalidation import DataValidation
|
|
16
16
|
from openpyxl.worksheet.dimensions import DimensionHolder, ColumnDimension
|
|
17
|
+
from openpyxl.worksheet.worksheet import Worksheet
|
|
17
18
|
from otlmow_converter.DotnotationHelper import DotnotationHelper
|
|
18
19
|
from otlmow_converter.OtlmowConverter import OtlmowConverter
|
|
19
20
|
from otlmow_model.OtlmowModel.BaseClasses.BooleanField import BooleanField
|
|
20
21
|
from otlmow_model.OtlmowModel.BaseClasses.KeuzelijstField import KeuzelijstField
|
|
21
|
-
from otlmow_model.OtlmowModel.BaseClasses.OTLObject import dynamic_create_instance_from_uri
|
|
22
|
+
from otlmow_model.OtlmowModel.BaseClasses.OTLObject import dynamic_create_instance_from_uri, OTLObject, \
|
|
23
|
+
get_attribute_by_name
|
|
22
24
|
from otlmow_model.OtlmowModel.Helpers.generated_lists import get_hardcoded_relation_dict
|
|
25
|
+
from otlmow_modelbuilder.HelperFunctions import get_ns_and_name_from_uri
|
|
23
26
|
from otlmow_modelbuilder.OSLOCollector import OSLOCollector
|
|
24
27
|
from otlmow_modelbuilder.SQLDataClasses.OSLOClass import OSLOClass
|
|
25
|
-
from universalasync import async_to_sync_wraps
|
|
26
28
|
|
|
27
29
|
ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
|
|
28
30
|
|
|
@@ -32,176 +34,475 @@ enumeration_validation_rules = {
|
|
|
32
34
|
"^https://wegenenverkeer.data.vlaanderen.be/ns/.+"]
|
|
33
35
|
}
|
|
34
36
|
|
|
37
|
+
short_to_long_ns = {
|
|
38
|
+
'ond': 'https://wegenenverkeer.data.vlaanderen.be/ns/onderdeel#',
|
|
39
|
+
'onderdeel': 'https://wegenenverkeer.data.vlaanderen.be/ns/onderdeel#',
|
|
40
|
+
'ins': 'https://wegenenverkeer.data.vlaanderen.be/ns/installatie#',
|
|
41
|
+
'installatie': 'https://wegenenverkeer.data.vlaanderen.be/ns/installatie#',
|
|
42
|
+
'imp': 'https://wegenenverkeer.data.vlaanderen.be/ns/implementatieelement#',
|
|
43
|
+
'implementatieelement': 'https://wegenenverkeer.data.vlaanderen.be/ns/implementatieelement#',
|
|
44
|
+
'proefenmeting': 'https://wegenenverkeer.data.vlaanderen.be/ns/proefenmeting#',
|
|
45
|
+
'pro': 'https://wegenenverkeer.data.vlaanderen.be/ns/proefenmeting#',
|
|
46
|
+
'lev': 'https://wegenenverkeer.data.vlaanderen.be/ns/levenscyclus#',
|
|
47
|
+
'levenscyclus': 'https://wegenenverkeer.data.vlaanderen.be/ns/levenscyclus#',
|
|
35
48
|
|
|
36
|
-
|
|
37
|
-
def __init__(self):
|
|
38
|
-
pass
|
|
49
|
+
}
|
|
39
50
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
51
|
+
|
|
52
|
+
class SubsetTemplateCreator:
|
|
53
|
+
@classmethod
|
|
54
|
+
def _load_collector_from_subset_path(cls, subset_path: Path) -> OSLOCollector:
|
|
55
|
+
collector = OSLOCollector(subset_path)
|
|
43
56
|
collector.collect_all(include_abstract=True)
|
|
44
57
|
return collector
|
|
45
58
|
|
|
46
|
-
@
|
|
47
|
-
async def
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
59
|
+
@classmethod
|
|
60
|
+
async def generate_template_from_subset_async(
|
|
61
|
+
cls,
|
|
62
|
+
subset_path: Path,
|
|
63
|
+
template_file_path: Path,
|
|
64
|
+
ignore_relations: bool = True,
|
|
65
|
+
filter_attributes_by_subset: bool = True,
|
|
66
|
+
class_uris_filter: [str] = None,
|
|
67
|
+
dummy_data_rows: int = 1,
|
|
68
|
+
add_geometry: bool = True,
|
|
69
|
+
add_attribute_info: bool = False,
|
|
70
|
+
add_deprecated: bool = False,
|
|
71
|
+
generate_choice_list: bool = True,
|
|
72
|
+
split_per_type: bool = True,
|
|
73
|
+
model_directory: Path = None):
|
|
74
|
+
"""
|
|
75
|
+
Generate a template from a subset file, async version.
|
|
76
|
+
Await this function!
|
|
77
|
+
|
|
78
|
+
:param subset_path: Path to the subset file
|
|
79
|
+
:param template_file_path: Path to where the template file should be created
|
|
80
|
+
:param ignore_relations: Whether to ignore relations when creating the template, defaults to True
|
|
81
|
+
:param filter_attributes_by_subset: Whether to filter by the attributes in the subset, defaults to True
|
|
82
|
+
:param class_uris_filter: List of class URIs to filter by. If not None, only classes with these URIs will be included, defaults to None
|
|
83
|
+
:param dummy_data_rows: Amount of dummy data rows to add to the template, defaults to 1
|
|
84
|
+
:param add_geometry: Whether to include the geometry attribute in the template, defaults to True
|
|
85
|
+
:param add_attribute_info: Whether to add attribute information to the template (colored grey in Excel), defaults to False
|
|
86
|
+
:param add_deprecated: Whether to add a deprecated row to the template (colored red in Excel), defaults to False
|
|
87
|
+
:param generate_choice_list: Whether to generate a choice list in the template (only for Excel), defaults to True
|
|
88
|
+
:param split_per_type: Whether to split the template into a file per type (only for CSV), defaults to True
|
|
89
|
+
:param model_directory: Path to the model directory, defaults to None
|
|
90
|
+
|
|
91
|
+
:return: None
|
|
92
|
+
"""
|
|
93
|
+
# generate objects to write to file
|
|
94
|
+
objects = await cls.generate_objects_for_template_async(
|
|
95
|
+
subset_path=subset_path, ignore_relations=ignore_relations, class_uris_filter=class_uris_filter,
|
|
96
|
+
add_geometry=add_geometry, filter_attributes_by_subset=filter_attributes_by_subset,
|
|
97
|
+
dummy_data_rows=dummy_data_rows, model_directory=model_directory)
|
|
98
|
+
|
|
99
|
+
# write the file
|
|
100
|
+
await OtlmowConverter.from_objects_to_file_async(
|
|
101
|
+
file_path=template_file_path, sequence_of_objects=objects, split_per_type=split_per_type)
|
|
102
|
+
|
|
103
|
+
# alter the file if needed
|
|
104
|
+
extension = template_file_path.suffix.lower()
|
|
105
|
+
if extension == '.xlsx':
|
|
106
|
+
await cls.alter_excel_template_async(
|
|
107
|
+
generate_choice_list=generate_choice_list, file_path=template_file_path, dummy_data_rows=dummy_data_rows,
|
|
108
|
+
instances=objects, add_deprecated=add_deprecated, add_attribute_info=add_attribute_info)
|
|
109
|
+
|
|
110
|
+
elif extension == '.csv':
|
|
111
|
+
await cls.alter_csv_template_async(
|
|
112
|
+
split_per_type=split_per_type, file_path=template_file_path, dummy_data_rows=dummy_data_rows,
|
|
113
|
+
instances=objects, add_deprecated=add_deprecated, add_attribute_info=add_attribute_info)
|
|
114
|
+
|
|
115
|
+
@classmethod
|
|
116
|
+
def generate_template_from_subset(
|
|
117
|
+
cls,
|
|
118
|
+
subset_path: Path,
|
|
119
|
+
template_file_path: Path,
|
|
120
|
+
ignore_relations: bool = True,
|
|
121
|
+
filter_attributes_by_subset: bool = True,
|
|
122
|
+
class_uris_filter: [str] = None,
|
|
123
|
+
dummy_data_rows: int = 1,
|
|
124
|
+
add_geometry: bool = True,
|
|
125
|
+
add_attribute_info: bool = False,
|
|
126
|
+
add_deprecated: bool = False,
|
|
127
|
+
generate_choice_list: bool = True,
|
|
128
|
+
split_per_type: bool = True,
|
|
129
|
+
model_directory: Path = None):
|
|
130
|
+
"""
|
|
131
|
+
Generate a template from a subset file.
|
|
132
|
+
|
|
133
|
+
:param subset_path: Path to the subset file
|
|
134
|
+
:param template_file_path: Path to where the template file should be created
|
|
135
|
+
:param ignore_relations: Whether to ignore relations when creating the template, defaults to True
|
|
136
|
+
:param filter_attributes_by_subset: Whether to filter by the attributes in the subset, defaults to True
|
|
137
|
+
:param class_uris_filter: List of class URIs to filter by. If not None, only classes with these URIs will be included, defaults to None
|
|
138
|
+
:param dummy_data_rows: Amount of dummy data rows to add to the template, defaults to 1
|
|
139
|
+
:param add_geometry: Whether to include the geometry attribute in the template, defaults to True
|
|
140
|
+
:param add_attribute_info: Whether to add attribute information to the template (colored grey in Excel), defaults to False
|
|
141
|
+
:param add_deprecated: Whether to tag deprecated attributes in the template, defaults to False
|
|
142
|
+
:param generate_choice_list: Whether to generate a choice list in the template (only for Excel), defaults to True
|
|
143
|
+
:param split_per_type: Whether to split the template into a file per type (only for CSV), defaults to True
|
|
144
|
+
:param model_directory: Path to the model directory, defaults to None
|
|
145
|
+
|
|
146
|
+
:return: None
|
|
147
|
+
"""
|
|
148
|
+
# generate objects to write to file
|
|
149
|
+
objects = cls.generate_objects_for_template(
|
|
150
|
+
subset_path=subset_path, ignore_relations=ignore_relations, class_uris_filter=class_uris_filter,
|
|
151
|
+
add_geometry=add_geometry, filter_attributes_by_subset=filter_attributes_by_subset,
|
|
152
|
+
dummy_data_rows=dummy_data_rows, model_directory=model_directory)
|
|
153
|
+
|
|
154
|
+
# write the file
|
|
155
|
+
OtlmowConverter.from_objects_to_file(
|
|
156
|
+
file_path=template_file_path, sequence_of_objects=objects, split_per_type=split_per_type,
|
|
157
|
+
model_directory=model_directory)
|
|
158
|
+
|
|
159
|
+
# alter the file if needed
|
|
160
|
+
extension = template_file_path.suffix.lower()
|
|
62
161
|
if extension == '.xlsx':
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
**kwargs)
|
|
162
|
+
cls.alter_excel_template(
|
|
163
|
+
generate_choice_list=generate_choice_list, file_path=template_file_path, dummy_data_rows=dummy_data_rows,
|
|
164
|
+
instances=objects, add_deprecated=add_deprecated, add_attribute_info=add_attribute_info)
|
|
67
165
|
elif extension == '.csv':
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
166
|
+
cls.alter_csv_template(
|
|
167
|
+
split_per_type=split_per_type, file_path=template_file_path, dummy_data_rows=dummy_data_rows,
|
|
168
|
+
instances=objects, add_deprecated=add_deprecated, add_attribute_info=add_attribute_info)
|
|
169
|
+
|
|
170
|
+
@classmethod
|
|
171
|
+
def generate_objects_for_template(
|
|
172
|
+
cls, subset_path: Path, class_uris_filter: [str], filter_attributes_by_subset: bool,
|
|
173
|
+
dummy_data_rows: int, add_geometry: bool, ignore_relations: bool, model_directory: Path = None
|
|
174
|
+
) -> [OTLObject]:
|
|
175
|
+
"""
|
|
176
|
+
This method is used to generate objects for the template. It will generate objects based on the subset file
|
|
177
|
+
"""
|
|
178
|
+
collector = cls._load_collector_from_subset_path(subset_path=subset_path)
|
|
179
|
+
filtered_class_list = cls.filters_classes_by_subset(collector=collector, class_uris_filter=class_uris_filter)
|
|
180
|
+
relation_dict = get_hardcoded_relation_dict(model_directory=model_directory)
|
|
181
|
+
|
|
182
|
+
amount_objects_to_create = max(1, dummy_data_rows)
|
|
84
183
|
otl_objects = []
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
184
|
+
|
|
185
|
+
while True:
|
|
186
|
+
for oslo_class in [cl for cl in filtered_class_list if cl.abstract == 0]:
|
|
187
|
+
if ignore_relations and oslo_class.objectUri in relation_dict:
|
|
188
|
+
continue
|
|
189
|
+
|
|
190
|
+
for _ in range(amount_objects_to_create):
|
|
191
|
+
otl_object = cls.generate_object_from_oslo_class(
|
|
192
|
+
oslo_class=oslo_class, add_geometry=add_geometry, collector=collector,
|
|
193
|
+
filter_attributes_by_subset=filter_attributes_by_subset, model_directory=model_directory)
|
|
194
|
+
if otl_object is not None:
|
|
195
|
+
otl_objects.append(otl_object)
|
|
196
|
+
created = len(otl_objects)
|
|
197
|
+
unique_ids = len({obj.assetId.identificator if hasattr(obj, 'assetId') else obj.agentId.identificator
|
|
198
|
+
for obj in otl_objects})
|
|
199
|
+
if created == unique_ids:
|
|
200
|
+
break
|
|
201
|
+
otl_objects = []
|
|
202
|
+
|
|
203
|
+
return otl_objects
|
|
204
|
+
|
|
205
|
+
@classmethod
|
|
206
|
+
async def generate_objects_for_template_async(
|
|
207
|
+
cls, subset_path: Path, class_uris_filter: [str], filter_attributes_by_subset: bool,
|
|
208
|
+
dummy_data_rows: int, add_geometry: bool, ignore_relations: bool, model_directory: Path = None
|
|
209
|
+
) -> [OTLObject]:
|
|
210
|
+
"""
|
|
211
|
+
This method is used to generate objects for the template. It will generate objects based on the subset file
|
|
212
|
+
"""
|
|
213
|
+
await sleep(0)
|
|
214
|
+
collector = cls._load_collector_from_subset_path(subset_path=subset_path)
|
|
215
|
+
await sleep(0)
|
|
216
|
+
filtered_class_list = cls.filters_classes_by_subset(collector=collector, class_uris_filter=class_uris_filter)
|
|
217
|
+
await sleep(0)
|
|
89
218
|
relation_dict = get_hardcoded_relation_dict(model_directory=model_directory)
|
|
90
219
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
generate_dummy_records = amount_of_examples
|
|
220
|
+
amount_objects_to_create = max(1, dummy_data_rows)
|
|
221
|
+
otl_objects = []
|
|
94
222
|
|
|
95
|
-
for
|
|
96
|
-
|
|
223
|
+
for oslo_class in [cl for cl in filtered_class_list if cl.abstract == 0]:
|
|
224
|
+
await sleep(0)
|
|
225
|
+
if ignore_relations and oslo_class.objectUri in relation_dict:
|
|
97
226
|
continue
|
|
98
|
-
|
|
99
|
-
|
|
227
|
+
|
|
228
|
+
for _ in range(amount_objects_to_create):
|
|
229
|
+
otl_object = cls.generate_object_from_oslo_class(
|
|
230
|
+
oslo_class=oslo_class, add_geometry=add_geometry, collector=collector,
|
|
231
|
+
filter_attributes_by_subset=filter_attributes_by_subset, model_directory=model_directory)
|
|
100
232
|
await sleep(0)
|
|
101
|
-
if
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
233
|
+
if otl_object is not None:
|
|
234
|
+
otl_objects.append(otl_object)
|
|
235
|
+
|
|
236
|
+
return otl_objects
|
|
237
|
+
|
|
238
|
+
@classmethod
|
|
239
|
+
def generate_object_from_oslo_class(
|
|
240
|
+
cls, oslo_class: OSLOClass, add_geometry: bool,
|
|
241
|
+
filter_attributes_by_subset: bool, collector: OSLOCollector, model_directory: Path = None) -> [OTLObject]:
|
|
242
|
+
"""
|
|
243
|
+
Generate an object from a given OSLO class
|
|
244
|
+
"""
|
|
245
|
+
instance = dynamic_create_instance_from_uri(oslo_class.objectUri, model_directory=model_directory)
|
|
246
|
+
if instance is None:
|
|
247
|
+
return
|
|
248
|
+
|
|
249
|
+
if filter_attributes_by_subset:
|
|
250
|
+
for attribute_object in collector.find_attributes_by_class(oslo_class):
|
|
251
|
+
attr = get_attribute_by_name(instance, attribute_object.name)
|
|
252
|
+
if attr is not None:
|
|
253
|
+
attr.fill_with_dummy_data()
|
|
254
|
+
else:
|
|
255
|
+
logging.warning(f'Attribute {attribute_object.name} not found in class {oslo_class.objectUri}')
|
|
256
|
+
else:
|
|
257
|
+
for attr in instance:
|
|
258
|
+
if attr.naam != 'geometry':
|
|
106
259
|
attr.fill_with_dummy_data()
|
|
107
|
-
|
|
108
|
-
|
|
260
|
+
with contextlib.suppress(AttributeError):
|
|
261
|
+
if add_geometry:
|
|
262
|
+
geo_attr = get_attribute_by_name(instance, 'geometry')
|
|
263
|
+
if geo_attr is not None:
|
|
109
264
|
geo_attr.fill_with_dummy_data()
|
|
110
|
-
except AttributeError:
|
|
111
|
-
pass
|
|
112
|
-
otl_objects.append(instance)
|
|
113
265
|
|
|
114
|
-
|
|
266
|
+
asset_versie = get_attribute_by_name(instance, 'assetVersie')
|
|
267
|
+
if asset_versie is not None:
|
|
268
|
+
asset_versie.set_waarde(None)
|
|
115
269
|
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
sequence_of_objects=otl_objects, **kwargs)
|
|
120
|
-
path_is_split = kwargs.get('split_per_type', True)
|
|
121
|
-
extension = os.path.splitext(path_to_template_file_and_extension)[-1].lower()
|
|
122
|
-
instantiated_attributes = []
|
|
123
|
-
if path_is_split is False or extension == '.xlsx':
|
|
124
|
-
instantiated_attributes = await converter.from_file_to_objects(file_path=temporary_path,
|
|
125
|
-
path_to_subset=path_to_subset)
|
|
126
|
-
return list(instantiated_attributes)
|
|
270
|
+
DotnotationHelper.clear_list_of_list_attributes(instance)
|
|
271
|
+
|
|
272
|
+
return instance
|
|
127
273
|
|
|
128
274
|
@classmethod
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
await sleep(0)
|
|
133
|
-
generate_choice_list = kwargs.get('generate_choice_list', False)
|
|
134
|
-
add_geo_artefact = kwargs.get('add_geo_artefact', False)
|
|
135
|
-
add_attribute_info = kwargs.get('add_attribute_info', False)
|
|
136
|
-
highlight_deprecated_attributes = kwargs.get('highlight_deprecated_attributes', False)
|
|
137
|
-
amount_of_examples = kwargs.get('amount_of_examples', 0)
|
|
138
|
-
if add_attribute_info and amount_of_examples == 0:
|
|
139
|
-
amount_of_examples = 1
|
|
140
|
-
await sleep(0)
|
|
141
|
-
wb = load_workbook(temporary_path)
|
|
275
|
+
def alter_excel_template(cls, instances: list, file_path: Path, add_attribute_info: bool,
|
|
276
|
+
generate_choice_list: bool, dummy_data_rows: int, add_deprecated: bool):
|
|
277
|
+
wb = load_workbook(file_path)
|
|
142
278
|
wb.create_sheet('Keuzelijsten')
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
279
|
+
|
|
280
|
+
choice_list_dict = {}
|
|
281
|
+
for sheet in wb:
|
|
282
|
+
if sheet.title == 'Keuzelijsten':
|
|
283
|
+
break
|
|
284
|
+
|
|
285
|
+
cls.alter_excel_sheet(add_attribute_info=add_attribute_info, choice_list_dict=choice_list_dict,
|
|
286
|
+
generate_choice_list=generate_choice_list, dummy_data_rows=dummy_data_rows,
|
|
287
|
+
instances=instances, sheet=sheet, add_deprecated=add_deprecated, workbook=wb)
|
|
288
|
+
|
|
289
|
+
wb.save(file_path)
|
|
290
|
+
wb.close()
|
|
291
|
+
|
|
292
|
+
@classmethod
|
|
293
|
+
def fill_class_dict(cls, instances: list) -> dict:
|
|
294
|
+
class_dict = defaultdict(list)
|
|
295
|
+
for instance in instances:
|
|
296
|
+
class_dict[instance.typeURI].append(instance)
|
|
297
|
+
return class_dict
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
@classmethod
|
|
301
|
+
def alter_csv_template(cls, instances: list, file_path: Path, add_attribute_info: bool,
|
|
302
|
+
split_per_type: bool, dummy_data_rows: int, add_deprecated: bool):
|
|
303
|
+
classes_dict = cls.fill_class_dict(instances)
|
|
304
|
+
if split_per_type:
|
|
305
|
+
for type_uri, typed_instances in classes_dict.items():
|
|
306
|
+
ns, name = get_ns_and_name_from_uri(type_uri)
|
|
307
|
+
class_file_path = file_path.parent / f'{file_path.stem}_{ns}_{name}.csv'
|
|
308
|
+
cls.alter_csv_file(add_attribute_info=add_attribute_info, add_deprecated=add_deprecated,
|
|
309
|
+
dummy_data_rows=dummy_data_rows, instances=typed_instances, file_path=class_file_path)
|
|
310
|
+
else:
|
|
311
|
+
cls.alter_csv_file(add_attribute_info=add_attribute_info, add_deprecated=add_deprecated,
|
|
312
|
+
dummy_data_rows=dummy_data_rows, instances=instances, file_path=file_path)
|
|
313
|
+
|
|
314
|
+
@classmethod
|
|
315
|
+
async def alter_csv_template_async(cls, instances: list, file_path: Path, add_deprecated: bool,
|
|
316
|
+
add_attribute_info: bool, split_per_type: bool, dummy_data_rows: int):
|
|
317
|
+
classes_dict = cls.fill_class_dict(instances)
|
|
318
|
+
if split_per_type:
|
|
319
|
+
for type_uri, typed_instances in classes_dict.items():
|
|
320
|
+
await sleep(0)
|
|
321
|
+
ns, name = get_ns_and_name_from_uri(type_uri)
|
|
322
|
+
class_file_path = file_path.parent / f'{file_path.stem}_{ns}_{name}.csv'
|
|
323
|
+
cls.alter_csv_file(add_attribute_info=add_attribute_info,
|
|
324
|
+
dummy_data_rows=dummy_data_rows, instances=typed_instances, add_deprecated=add_deprecated,
|
|
325
|
+
file_path=class_file_path)
|
|
326
|
+
else:
|
|
149
327
|
await sleep(0)
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
cls.add_mock_data_excel(workbook=wb, rows_of_examples=amount_of_examples) # remove dummy rows if needed
|
|
328
|
+
cls.alter_csv_file(add_attribute_info=add_attribute_info,
|
|
329
|
+
dummy_data_rows=dummy_data_rows, instances=instances, add_deprecated=add_deprecated,
|
|
330
|
+
file_path=file_path)
|
|
154
331
|
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
332
|
+
@classmethod
|
|
333
|
+
def alter_csv_file(cls, add_attribute_info: bool, instances: [OTLObject], add_deprecated: bool, file_path: Path,
|
|
334
|
+
dummy_data_rows: int):
|
|
335
|
+
collected_attribute_info_row = []
|
|
336
|
+
deprecated_attributes_row = []
|
|
337
|
+
instance = instances[0]
|
|
338
|
+
quote_char = '"'
|
|
339
|
+
|
|
340
|
+
with open(file_path, encoding='utf-8') as file:
|
|
341
|
+
csv_reader = csv.reader(file, delimiter=';', quotechar=quote_char)
|
|
342
|
+
header_row = next(csv_reader)
|
|
343
|
+
csv_data = list(csv_reader)
|
|
344
|
+
|
|
345
|
+
for index, header in enumerate(header_row):
|
|
346
|
+
if header is None or header == '':
|
|
347
|
+
continue
|
|
348
|
+
|
|
349
|
+
if header == 'typeURI':
|
|
350
|
+
if add_attribute_info:
|
|
351
|
+
collected_attribute_info_row.append(
|
|
352
|
+
'De URI van het object volgens https://www.w3.org/2001/XMLSchema#anyURI .')
|
|
353
|
+
if add_deprecated:
|
|
354
|
+
deprecated_attributes_row.append('')
|
|
355
|
+
continue
|
|
356
|
+
|
|
357
|
+
attribute = DotnotationHelper.get_attribute_by_dotnotation(instance, header)
|
|
358
|
+
|
|
359
|
+
if add_attribute_info:
|
|
360
|
+
collected_attribute_info_row.append(attribute.definition)
|
|
361
|
+
|
|
362
|
+
if add_deprecated:
|
|
363
|
+
deprecated_attributes_row.append('DEPRECATED' if attribute.deprecated_version else '')
|
|
364
|
+
|
|
365
|
+
with open(file_path, 'w') as file:
|
|
366
|
+
csv_writer = csv.writer(file, delimiter=';', quotechar=quote_char, quoting=csv.QUOTE_MINIMAL)
|
|
367
|
+
if add_attribute_info:
|
|
368
|
+
csv_writer.writerow(collected_attribute_info_row)
|
|
369
|
+
if add_deprecated and any(deprecated_attributes_row):
|
|
370
|
+
csv_writer.writerow(deprecated_attributes_row)
|
|
371
|
+
csv_writer.writerow(header_row)
|
|
372
|
+
if dummy_data_rows != 0:
|
|
373
|
+
for line in csv_data:
|
|
374
|
+
csv_writer.writerow(line)
|
|
375
|
+
|
|
376
|
+
@classmethod
|
|
377
|
+
async def alter_excel_template_async(cls, instances: list, file_path: Path, add_attribute_info: bool,
|
|
378
|
+
generate_choice_list: bool, dummy_data_rows: int, add_deprecated: bool):
|
|
379
|
+
wb = load_workbook(file_path)
|
|
380
|
+
wb.create_sheet('Keuzelijsten')
|
|
381
|
+
|
|
382
|
+
choice_list_dict = {}
|
|
383
|
+
for sheet in wb:
|
|
384
|
+
if sheet.title == 'Keuzelijsten':
|
|
385
|
+
break
|
|
386
|
+
|
|
387
|
+
cls.alter_excel_sheet(add_attribute_info=add_attribute_info, choice_list_dict=choice_list_dict,
|
|
388
|
+
generate_choice_list=generate_choice_list, dummy_data_rows=dummy_data_rows,
|
|
389
|
+
instances=instances, sheet=sheet, add_deprecated=add_deprecated, workbook=wb)
|
|
159
390
|
await sleep(0)
|
|
160
|
-
|
|
391
|
+
|
|
392
|
+
wb.save(file_path)
|
|
393
|
+
wb.close()
|
|
394
|
+
|
|
395
|
+
@classmethod
|
|
396
|
+
def alter_excel_sheet(cls, add_attribute_info: bool, choice_list_dict: dict, generate_choice_list: bool,
|
|
397
|
+
instances: [OTLObject], sheet: Worksheet, add_deprecated: bool, workbook: Workbook,
|
|
398
|
+
dummy_data_rows: int):
|
|
399
|
+
type_uri = cls.get_uri_from_sheet_name(sheet.title)
|
|
400
|
+
instance = next(x for x in instances if x.typeURI == type_uri)
|
|
401
|
+
|
|
402
|
+
boolean_validation = DataValidation(type="list", formula1='"TRUE,FALSE,"', allow_blank=True)
|
|
403
|
+
sheet.add_data_validation(boolean_validation)
|
|
404
|
+
collected_attribute_info = []
|
|
405
|
+
deprecated_attributes_row = []
|
|
406
|
+
header_row = next(sheet.iter_rows(min_row=1, max_row=1))
|
|
407
|
+
for index, header_cell in enumerate(header_row):
|
|
408
|
+
header = header_cell.value
|
|
409
|
+
if header is None or header == '':
|
|
410
|
+
continue
|
|
411
|
+
|
|
412
|
+
if header == 'typeURI':
|
|
413
|
+
data_validation = DataValidation(type="list", formula1=f'"{type_uri}"', allow_blank=True)
|
|
414
|
+
sheet.add_data_validation(data_validation)
|
|
415
|
+
data_validation.add(f'{header_cell.column_letter}2:{header_cell.column_letter}1000')
|
|
416
|
+
if add_attribute_info:
|
|
417
|
+
collected_attribute_info.append('De URI van het object volgens https://www.w3.org/2001/XMLSchema#anyURI .')
|
|
418
|
+
if add_deprecated:
|
|
419
|
+
deprecated_attributes_row.append('')
|
|
420
|
+
continue
|
|
421
|
+
|
|
422
|
+
if type_uri == 'http://purl.org/dc/terms/Agent' and header.startswith('assetId.'):
|
|
423
|
+
continue
|
|
424
|
+
|
|
425
|
+
attribute = DotnotationHelper.get_attribute_by_dotnotation(instance, header)
|
|
426
|
+
|
|
427
|
+
if add_attribute_info:
|
|
428
|
+
collected_attribute_info.append(attribute.definition)
|
|
429
|
+
|
|
430
|
+
if add_deprecated:
|
|
431
|
+
deprecated_attributes_row.append('DEPRECATED' if attribute.deprecated_version else '')
|
|
432
|
+
|
|
433
|
+
if generate_choice_list:
|
|
434
|
+
if issubclass(attribute.field, BooleanField):
|
|
435
|
+
boolean_validation.add(f'{header_cell.column_letter}{2}:{header_cell.column_letter}1000')
|
|
436
|
+
continue
|
|
437
|
+
|
|
438
|
+
if issubclass(attribute.field, KeuzelijstField):
|
|
439
|
+
cls.generate_choice_list_in_excel(
|
|
440
|
+
attribute=attribute, choice_list_dict=choice_list_dict, column=header_cell.column,
|
|
441
|
+
row_nr=1, sheet=sheet, workbook=workbook)
|
|
442
|
+
|
|
443
|
+
if dummy_data_rows == 0:
|
|
444
|
+
sheet.delete_rows(idx=2)
|
|
445
|
+
|
|
446
|
+
if add_deprecated and any(deprecated_attributes_row):
|
|
447
|
+
cls.add_deprecated_row_to_sheet(deprecated_attributes_row, sheet)
|
|
448
|
+
|
|
161
449
|
if add_attribute_info:
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
wb.save(path_to_template_file_and_extension)
|
|
166
|
-
file_location = os.path.dirname(temporary_path)
|
|
167
|
-
[f.unlink() for f in Path(file_location).glob("*") if f.is_file()]
|
|
168
|
-
|
|
169
|
-
@async_to_sync_wraps
|
|
170
|
-
async def determine_multiplicity_csv(self, path_to_template_file_and_extension: Path, path_to_subset: Path,
|
|
171
|
-
instantiated_attributes: list, temporary_path: Path, **kwargs):
|
|
172
|
-
path_is_split = kwargs.get('split_per_type', True)
|
|
173
|
-
await sleep(0)
|
|
174
|
-
if path_is_split is False:
|
|
175
|
-
self.alter_csv_template(path_to_template_file_and_extension=path_to_template_file_and_extension,
|
|
176
|
-
temporary_path=temporary_path, path_to_subset=path_to_subset, **kwargs)
|
|
177
|
-
else:
|
|
178
|
-
self.multiple_csv_template(path_to_template_file_and_extension=path_to_template_file_and_extension,
|
|
179
|
-
temporary_path=temporary_path,
|
|
180
|
-
path_to_subset=path_to_subset, instantiated_attributes=instantiated_attributes,
|
|
181
|
-
**kwargs)
|
|
182
|
-
file_location = os.path.dirname(temporary_path)
|
|
183
|
-
[f.unlink() for f in Path(file_location).glob("*") if f.is_file()]
|
|
450
|
+
cls.add_attribute_info_to_sheet(collected_attribute_info, sheet)
|
|
451
|
+
|
|
452
|
+
cls.set_fixed_column_width(sheet=sheet, width=25)
|
|
184
453
|
|
|
185
454
|
@classmethod
|
|
186
|
-
def
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
455
|
+
def add_deprecated_row_to_sheet(cls, deprecated_attributes_row, sheet):
|
|
456
|
+
sheet.insert_rows(idx=1)
|
|
457
|
+
for index, depr_info in enumerate(deprecated_attributes_row, start=1):
|
|
458
|
+
if depr_info != '':
|
|
459
|
+
cell = sheet.cell(row=1, column=index)
|
|
460
|
+
cell.value = depr_info
|
|
461
|
+
cell.fill = PatternFill(start_color="FF7276", end_color="FF7276", fill_type="solid")
|
|
462
|
+
|
|
463
|
+
@classmethod
|
|
464
|
+
def add_attribute_info_to_sheet(cls, collected_attribute_info, sheet):
|
|
465
|
+
sheet.insert_rows(idx=1)
|
|
466
|
+
for index, attr_info in enumerate(collected_attribute_info, start=1):
|
|
467
|
+
cell = sheet.cell(row=1, column=index)
|
|
468
|
+
cell.value = attr_info
|
|
469
|
+
cell.alignment = Alignment(wrapText=True, vertical='top')
|
|
470
|
+
cell.fill = PatternFill(start_color="808080", end_color="808080", fill_type="solid")
|
|
471
|
+
|
|
472
|
+
@classmethod
|
|
473
|
+
def generate_choice_list_in_excel(cls, attribute, choice_list_dict, column, row_nr, sheet: Worksheet,
|
|
474
|
+
workbook: Workbook):
|
|
475
|
+
choice_list_values = [cv for cv in attribute.field.options.values()
|
|
476
|
+
if cv.status != 'verwijderd']
|
|
477
|
+
if attribute.field.naam not in choice_list_dict:
|
|
478
|
+
cls.add_choice_list_to_sheet(workbook=workbook, name=attribute.field.naam,
|
|
479
|
+
options=choice_list_values, choice_list_dict=choice_list_dict)
|
|
480
|
+
column_in_choice_sheet = choice_list_dict[attribute.field.naam]
|
|
481
|
+
start_range = f"${column_in_choice_sheet}$2"
|
|
482
|
+
end_range = f"${column_in_choice_sheet}${len(choice_list_values) + 1}"
|
|
483
|
+
data_val = DataValidation(type="list", formula1=f"Keuzelijsten!{start_range}:{end_range}",
|
|
484
|
+
allowBlank=True)
|
|
485
|
+
sheet.add_data_validation(data_val)
|
|
486
|
+
data_val.add(f'{get_column_letter(column)}{row_nr + 1}:'
|
|
487
|
+
f'{get_column_letter(column)}1000')
|
|
190
488
|
|
|
191
|
-
if list_of_otl_objectUri == []:
|
|
192
|
-
return collector.classes
|
|
193
|
-
return [x for x in collector.classes if x.objectUri in list_of_otl_objectUri]
|
|
194
489
|
|
|
195
|
-
@
|
|
196
|
-
def
|
|
197
|
-
|
|
198
|
-
|
|
490
|
+
@classmethod
|
|
491
|
+
def determine_multiplicity_csv(cls, template_file_path: Path, subset_path: Path,
|
|
492
|
+
instances: list, temporary_path: Path, **kwargs):
|
|
493
|
+
pass
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
@classmethod
|
|
497
|
+
def filters_classes_by_subset(cls, collector: OSLOCollector,
|
|
498
|
+
class_uris_filter: [str] = None) -> list[OSLOClass]:
|
|
499
|
+
if class_uris_filter is None:
|
|
500
|
+
return collector.classes
|
|
501
|
+
return [x for x in collector.classes if x.objectUri in class_uris_filter]
|
|
199
502
|
|
|
200
503
|
@classmethod
|
|
201
|
-
|
|
202
|
-
async def add_type_uri_choice_list_in_excel(cls, sheet, instantiated_attributes, add_attribute_info: bool):
|
|
504
|
+
def add_type_uri_choice_list_in_excel(cls, sheet, instances, add_attribute_info: bool):
|
|
203
505
|
starting_row = '3' if add_attribute_info else '2'
|
|
204
|
-
await sleep(0)
|
|
205
506
|
if sheet.title == 'Keuzelijsten':
|
|
206
507
|
return
|
|
207
508
|
type_uri_found = False
|
|
@@ -215,7 +516,6 @@ class SubsetTemplateCreator:
|
|
|
215
516
|
if not type_uri_found:
|
|
216
517
|
return
|
|
217
518
|
|
|
218
|
-
await sleep(0)
|
|
219
519
|
sheet_name = sheet.title
|
|
220
520
|
type_uri = ''
|
|
221
521
|
if sheet_name.startswith('http'):
|
|
@@ -224,7 +524,7 @@ class SubsetTemplateCreator:
|
|
|
224
524
|
split_name = sheet_name.split("#")
|
|
225
525
|
subclass_name = split_name[1]
|
|
226
526
|
|
|
227
|
-
possible_classes = [x for x in
|
|
527
|
+
possible_classes = [x for x in instances if x.typeURI.endswith(subclass_name)]
|
|
228
528
|
if len(possible_classes) == 1:
|
|
229
529
|
type_uri = possible_classes[0].typeURI
|
|
230
530
|
|
|
@@ -232,339 +532,81 @@ class SubsetTemplateCreator:
|
|
|
232
532
|
return
|
|
233
533
|
|
|
234
534
|
data_validation = DataValidation(type="list", formula1=f'"{type_uri}"', allow_blank=True)
|
|
235
|
-
await sleep(0)
|
|
236
535
|
for rows in sheet.iter_rows(min_row=1, max_row=1, min_col=1, max_col=1):
|
|
237
536
|
for cell in rows:
|
|
238
|
-
await sleep(0)
|
|
239
537
|
column = cell.column
|
|
240
538
|
sheet.add_data_validation(data_validation)
|
|
241
539
|
data_validation.add(f'{get_column_letter(column)}{starting_row}:{get_column_letter(column)}1000')
|
|
242
540
|
|
|
243
541
|
@classmethod
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
add_attribute_info=add_attribute_info)
|
|
251
|
-
await cls.remove_asset_versie(sheet=sheet)
|
|
252
|
-
|
|
253
|
-
@classmethod
|
|
254
|
-
@async_to_sync_wraps
|
|
255
|
-
async def remove_asset_versie(cls, sheet):
|
|
256
|
-
for row in sheet.iter_rows(min_row=1, max_row=1, min_col=4):
|
|
257
|
-
for cell in row:
|
|
258
|
-
await sleep(0)
|
|
259
|
-
if cell.value is None or not cell.value.startswith('assetVersie'):
|
|
260
|
-
continue
|
|
261
|
-
for rows in sheet.iter_rows(min_row=row, max_row=row, min_col=2, max_col=1000):
|
|
262
|
-
for c in rows:
|
|
263
|
-
await sleep(0)
|
|
264
|
-
c.value = ''
|
|
265
|
-
|
|
266
|
-
@classmethod
|
|
267
|
-
@async_to_sync_wraps
|
|
268
|
-
async def set_fixed_column_width(cls, sheet, width: int):
|
|
269
|
-
dim_holder = DimensionHolder(worksheet=sheet)
|
|
270
|
-
for col in range(sheet.min_column, sheet.max_column + 1):
|
|
271
|
-
await sleep(0)
|
|
272
|
-
dim_holder[get_column_letter(col)] = ColumnDimension(sheet, min=col, max=col, width=width)
|
|
273
|
-
sheet.column_dimensions = dim_holder
|
|
274
|
-
|
|
275
|
-
@classmethod
|
|
276
|
-
@async_to_sync_wraps
|
|
277
|
-
async def add_attribute_info_excel(cls, workbook, instantiated_attributes: list):
|
|
278
|
-
dotnotation_module = DotnotationHelper()
|
|
279
|
-
for sheet in workbook:
|
|
280
|
-
if sheet == workbook['Keuzelijsten']:
|
|
281
|
-
break
|
|
282
|
-
filter_uri = SubsetTemplateCreator.find_uri_in_sheet(sheet)
|
|
283
|
-
await sleep(0)
|
|
284
|
-
single_attribute = next(x for x in instantiated_attributes if x.typeURI == filter_uri)
|
|
285
|
-
sheet.insert_rows(1)
|
|
286
|
-
for rows in sheet.iter_rows(min_row=2, max_row=2, min_col=1):
|
|
287
|
-
for cell in rows:
|
|
288
|
-
await sleep(0)
|
|
289
|
-
if cell.value == 'typeURI':
|
|
290
|
-
value = 'De URI van het object volgens https://www.w3.org/2001/XMLSchema#anyURI .'
|
|
291
|
-
elif cell.value.find('[DEPRECATED]') != -1:
|
|
292
|
-
strip = cell.value.split(' ')
|
|
293
|
-
dotnotation_attribute = dotnotation_module.get_attribute_by_dotnotation(single_attribute,
|
|
294
|
-
strip[1])
|
|
295
|
-
value = dotnotation_attribute.definition
|
|
296
|
-
else:
|
|
297
|
-
dotnotation_attribute = dotnotation_module.get_attribute_by_dotnotation(single_attribute,
|
|
298
|
-
cell.value)
|
|
299
|
-
value = dotnotation_attribute.definition
|
|
300
|
-
newcell = sheet.cell(row=1, column=cell.column, value=value)
|
|
301
|
-
newcell.alignment = Alignment(wrapText=True, vertical='top')
|
|
302
|
-
newcell.fill = PatternFill(start_color="808080", end_color="808080",
|
|
303
|
-
fill_type="solid")
|
|
304
|
-
|
|
305
|
-
@classmethod
|
|
306
|
-
def check_for_deprecated_attributes(cls, workbook, instantiated_attributes: list):
|
|
307
|
-
dotnotation_module = DotnotationHelper()
|
|
308
|
-
for sheet in workbook:
|
|
309
|
-
if sheet == workbook['Keuzelijsten']:
|
|
310
|
-
break
|
|
311
|
-
filter_uri = SubsetTemplateCreator.find_uri_in_sheet(sheet)
|
|
312
|
-
single_attribute = next(x for x in instantiated_attributes if x.typeURI == filter_uri)
|
|
313
|
-
for rows in sheet.iter_rows(min_row=1, max_row=1, min_col=2):
|
|
314
|
-
for cell in rows:
|
|
315
|
-
is_deprecated = False
|
|
316
|
-
if cell.value.count('.') == 1:
|
|
317
|
-
dot_split = cell.value.split('.')
|
|
318
|
-
attribute = dotnotation_module.get_attribute_by_dotnotation(single_attribute,
|
|
319
|
-
dot_split[0])
|
|
320
|
-
|
|
321
|
-
if len(attribute.deprecated_version) > 0:
|
|
322
|
-
is_deprecated = True
|
|
323
|
-
dotnotation_attribute = dotnotation_module.get_attribute_by_dotnotation(single_attribute,
|
|
324
|
-
cell.value)
|
|
325
|
-
if len(dotnotation_attribute.deprecated_version) > 0:
|
|
326
|
-
is_deprecated = True
|
|
327
|
-
|
|
328
|
-
if is_deprecated:
|
|
329
|
-
cell.value = f'[DEPRECATED] {cell.value}'
|
|
330
|
-
|
|
331
|
-
@classmethod
|
|
332
|
-
def find_uri_in_sheet(cls, sheet):
|
|
333
|
-
filter_uri = None
|
|
542
|
+
async def add_type_uri_choice_list_in_excel_async(cls, sheet, instances, add_attribute_info: bool):
|
|
543
|
+
starting_row = '3' if add_attribute_info else '2'
|
|
544
|
+
await sleep(0)
|
|
545
|
+
if sheet.title == 'Keuzelijsten':
|
|
546
|
+
return
|
|
547
|
+
type_uri_found = False
|
|
334
548
|
for row in sheet.iter_rows(min_row=1, max_row=1):
|
|
335
549
|
for cell in row:
|
|
336
550
|
if cell.value == 'typeURI':
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
return filter_uri
|
|
341
|
-
|
|
342
|
-
@classmethod
|
|
343
|
-
def remove_geo_artefact_excel(cls, workbook):
|
|
344
|
-
for sheet in workbook:
|
|
345
|
-
for row in sheet.iter_rows(min_row=1, max_row=1):
|
|
346
|
-
for cell in row:
|
|
347
|
-
if cell.value == 'geometry':
|
|
348
|
-
sheet.delete_cols(cell.column)
|
|
349
|
-
|
|
350
|
-
@classmethod
|
|
351
|
-
@async_to_sync_wraps
|
|
352
|
-
async def add_choice_list_excel(cls, workbook, instantiated_attributes: list, path_to_subset: Path,
|
|
353
|
-
add_attribute_info: bool=False):
|
|
354
|
-
choice_list_dict = {}
|
|
355
|
-
starting_row = '3' if add_attribute_info else '2'
|
|
356
|
-
dotnotation_module = DotnotationHelper()
|
|
357
|
-
for sheet in workbook:
|
|
358
|
-
await sleep(0)
|
|
359
|
-
if sheet == workbook['Keuzelijsten']:
|
|
360
|
-
break
|
|
361
|
-
filter_uri = SubsetTemplateCreator.find_uri_in_sheet(sheet)
|
|
362
|
-
single_attribute = next(x for x in instantiated_attributes if x.typeURI == filter_uri)
|
|
363
|
-
for rows in sheet.iter_rows(min_row=1, max_row=1, min_col=2):
|
|
364
|
-
for cell in rows:
|
|
365
|
-
await sleep(0)
|
|
366
|
-
if cell.value.find('[DEPRECATED]') != -1:
|
|
367
|
-
strip = cell.value.split(' ')
|
|
368
|
-
dotnotation_attribute = dotnotation_module.get_attribute_by_dotnotation(single_attribute,
|
|
369
|
-
strip[1])
|
|
370
|
-
else:
|
|
371
|
-
dotnotation_attribute = dotnotation_module.get_attribute_by_dotnotation(single_attribute,
|
|
372
|
-
cell.value)
|
|
373
|
-
await sleep(0)
|
|
374
|
-
if issubclass(dotnotation_attribute.field, KeuzelijstField):
|
|
375
|
-
name = dotnotation_attribute.field.naam
|
|
376
|
-
valid_options = [v.invulwaarde for k, v in dotnotation_attribute.field.options.items()
|
|
377
|
-
if v.status != 'verwijderd']
|
|
378
|
-
if (dotnotation_attribute.field.naam not in choice_list_dict):
|
|
379
|
-
choice_list_dict = cls.add_choice_list_to_sheet(
|
|
380
|
-
workbook=workbook, name=name, options=valid_options, choice_list_dict=choice_list_dict)
|
|
381
|
-
column = choice_list_dict[dotnotation_attribute.field.naam]
|
|
382
|
-
start_range = f"${column}$2"
|
|
383
|
-
end_range = f"${column}${len(valid_options) + 1}"
|
|
384
|
-
data_val = DataValidation(type="list", formula1=f"Keuzelijsten!{start_range}:{end_range}",
|
|
385
|
-
allowBlank=True)
|
|
386
|
-
sheet.add_data_validation(data_val)
|
|
387
|
-
data_val.add(f'{get_column_letter(cell.column)}{starting_row}:'
|
|
388
|
-
f'{get_column_letter(cell.column)}1000')
|
|
389
|
-
|
|
390
|
-
await sleep(0)
|
|
391
|
-
if issubclass(dotnotation_attribute.field, BooleanField):
|
|
392
|
-
data_validation = DataValidation(type="list", formula1='"TRUE,FALSE,"', allow_blank=True)
|
|
393
|
-
column = cell.column
|
|
394
|
-
sheet.add_data_validation(data_validation)
|
|
395
|
-
data_validation.add(f'{get_column_letter(column)}{starting_row}:'
|
|
396
|
-
f'{get_column_letter(column)}1000')
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
@classmethod
|
|
400
|
-
def add_mock_data_excel(cls, workbook, rows_of_examples: int):
|
|
401
|
-
for sheet in workbook:
|
|
402
|
-
if sheet == workbook["Keuzelijsten"]:
|
|
551
|
+
type_uri_found = True
|
|
552
|
+
break
|
|
553
|
+
if type_uri_found:
|
|
403
554
|
break
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
for cell in rows:
|
|
407
|
-
cell.value = ''
|
|
408
|
-
|
|
409
|
-
@classmethod
|
|
410
|
-
def remove_geo_artefact_csv(cls, header, data):
|
|
411
|
-
if 'geometry' in header:
|
|
412
|
-
deletion_index = header.index('geometry')
|
|
413
|
-
header.remove('geometry')
|
|
414
|
-
for d in data:
|
|
415
|
-
d.pop(deletion_index)
|
|
416
|
-
return [header, data]
|
|
417
|
-
|
|
418
|
-
@classmethod
|
|
419
|
-
def multiple_csv_template(cls, path_to_template_file_and_extension, path_to_subset, temporary_path,
|
|
420
|
-
instantiated_attributes, **kwargs):
|
|
421
|
-
file_location = os.path.dirname(path_to_template_file_and_extension)
|
|
422
|
-
tempdir = Path(tempfile.gettempdir()) / 'temp-otlmow'
|
|
423
|
-
logging.debug(file_location)
|
|
424
|
-
file_name = ntpath.basename(path_to_template_file_and_extension)
|
|
425
|
-
split_file_name = file_name.split('.')
|
|
426
|
-
things_in_there = os.listdir(tempdir)
|
|
427
|
-
csv_templates = [x for x in things_in_there if x.startswith(f'{split_file_name[0]}_')]
|
|
428
|
-
for file in csv_templates:
|
|
429
|
-
test_template_loc = Path(os.path.dirname(path_to_template_file_and_extension)) / file
|
|
430
|
-
temp_loc = Path(tempdir) / file
|
|
431
|
-
cls.alter_csv_template(path_to_template_file_and_extension=test_template_loc, temporary_path=temp_loc,
|
|
432
|
-
path_to_subset=path_to_subset, **kwargs)
|
|
433
|
-
|
|
434
|
-
@classmethod
|
|
435
|
-
def alter_csv_template(cls, path_to_template_file_and_extension, path_to_subset, temporary_path,
|
|
436
|
-
**kwargs):
|
|
437
|
-
converter = OtlmowConverter()
|
|
438
|
-
instantiated_attributes = converter.from_file_to_objects(file_path=temporary_path,
|
|
439
|
-
path_to_subset=path_to_subset)
|
|
440
|
-
header = []
|
|
441
|
-
data = []
|
|
442
|
-
delimiter = ';'
|
|
443
|
-
add_geo_artefact = kwargs.get('add_geo_artefact', False)
|
|
444
|
-
add_attribute_info = kwargs.get('add_attribute_info', False)
|
|
445
|
-
highlight_deprecated_attributes = kwargs.get('highlight_deprecated_attributes', False)
|
|
446
|
-
amount_of_examples = kwargs.get('amount_of_examples', 0)
|
|
447
|
-
quote_char = '"'
|
|
448
|
-
with open(temporary_path, 'r+', encoding='utf-8') as csvfile:
|
|
449
|
-
with open(path_to_template_file_and_extension, 'w', encoding='utf-8') as new_file:
|
|
450
|
-
reader = csv.reader(csvfile, delimiter=delimiter, quotechar=quote_char)
|
|
451
|
-
for row_nr, row in enumerate(reader):
|
|
452
|
-
if row_nr == 0:
|
|
453
|
-
header = row
|
|
454
|
-
else:
|
|
455
|
-
data.append(row)
|
|
456
|
-
if add_geo_artefact is False:
|
|
457
|
-
[header, data] = cls.remove_geo_artefact_csv(header=header, data=data)
|
|
458
|
-
if add_attribute_info:
|
|
459
|
-
[info, header] = cls.add_attribute_info_csv(header=header, data=data,
|
|
460
|
-
instantiated_attributes=instantiated_attributes)
|
|
461
|
-
new_file.write(delimiter.join(info) + '\n')
|
|
462
|
-
data = cls.add_mock_data_csv(header=header, data=data, rows_of_examples=amount_of_examples)
|
|
463
|
-
if highlight_deprecated_attributes:
|
|
464
|
-
header = cls.highlight_deprecated_attributes_csv(header=header, data=data,
|
|
465
|
-
instantiated_attributes=instantiated_attributes)
|
|
466
|
-
new_file.write(delimiter.join(header) + '\n')
|
|
467
|
-
for d in data:
|
|
468
|
-
new_file.write(delimiter.join(d) + '\n')
|
|
555
|
+
if not type_uri_found:
|
|
556
|
+
return
|
|
469
557
|
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
if d[uri_index] not in found_uri:
|
|
479
|
-
found_uri.append(d[uri_index])
|
|
480
|
-
for uri in found_uri:
|
|
481
|
-
single_object = next(x for x in instantiated_attributes if x.typeURI == uri)
|
|
482
|
-
for dotnototation_title in info_data:
|
|
483
|
-
if dotnototation_title == 'typeURI':
|
|
484
|
-
index = info_data.index(dotnototation_title)
|
|
485
|
-
info_data[index] = 'De URI van het object volgens https://www.w3.org/2001/XMLSchema#anyURI .'
|
|
486
|
-
else:
|
|
487
|
-
index = info_data.index(dotnototation_title)
|
|
488
|
-
try:
|
|
489
|
-
dotnotation_attribute = dotnotation_module.get_attribute_by_dotnotation(
|
|
490
|
-
single_object, dotnototation_title)
|
|
491
|
-
except AttributeError as e:
|
|
492
|
-
continue
|
|
493
|
-
info_data[index] = dotnotation_attribute.definition
|
|
494
|
-
return [info_data, header]
|
|
558
|
+
await sleep(0)
|
|
559
|
+
sheet_name = sheet.title
|
|
560
|
+
type_uri = ''
|
|
561
|
+
if sheet_name.startswith('http'):
|
|
562
|
+
type_uri = sheet_name
|
|
563
|
+
else:
|
|
564
|
+
split_name = sheet_name.split("#")
|
|
565
|
+
subclass_name = split_name[1]
|
|
495
566
|
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
data = []
|
|
500
|
-
return data
|
|
567
|
+
possible_classes = [x for x in instances if x.typeURI.endswith(subclass_name)]
|
|
568
|
+
if len(possible_classes) == 1:
|
|
569
|
+
type_uri = possible_classes[0].typeURI
|
|
501
570
|
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
found_uri = []
|
|
505
|
-
dotnotation_module = DotnotationHelper()
|
|
506
|
-
uri_index = cls.find_uri_in_csv(header)
|
|
507
|
-
for d in data:
|
|
508
|
-
if d[uri_index] not in found_uri:
|
|
509
|
-
found_uri.append(d[uri_index])
|
|
510
|
-
for uri in found_uri:
|
|
511
|
-
single_object = next(x for x in instantiated_attributes if x.typeURI == uri)
|
|
512
|
-
for dotnototation_title in header:
|
|
513
|
-
if dotnototation_title == 'typeURI':
|
|
514
|
-
continue
|
|
571
|
+
if type_uri == '':
|
|
572
|
+
return
|
|
515
573
|
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
if len(attribute.deprecated_version) > 0:
|
|
526
|
-
is_deprecated = True
|
|
527
|
-
dotnotation_attribute = dotnotation_module.get_attribute_by_dotnotation(single_object,
|
|
528
|
-
dotnototation_title)
|
|
529
|
-
if len(dotnotation_attribute.deprecated_version) > 0:
|
|
530
|
-
is_deprecated = True
|
|
531
|
-
except AttributeError:
|
|
532
|
-
continue
|
|
533
|
-
if is_deprecated:
|
|
534
|
-
header[index] = f"[DEPRECATED] {value}"
|
|
535
|
-
return header
|
|
574
|
+
data_validation = DataValidation(type="list", formula1=f'"{type_uri}"', allow_blank=True)
|
|
575
|
+
await sleep(0)
|
|
576
|
+
for rows in sheet.iter_rows(min_row=1, max_row=1, min_col=1, max_col=1):
|
|
577
|
+
for cell in rows:
|
|
578
|
+
await sleep(0)
|
|
579
|
+
column = cell.column
|
|
580
|
+
sheet.add_data_validation(data_validation)
|
|
581
|
+
data_validation.add(f'{get_column_letter(column)}{starting_row}:{get_column_letter(column)}1000')
|
|
536
582
|
|
|
537
583
|
@classmethod
|
|
538
|
-
def
|
|
539
|
-
|
|
584
|
+
def set_fixed_column_width(cls, sheet, width: int):
|
|
585
|
+
dim_holder = DimensionHolder(worksheet=sheet)
|
|
586
|
+
for col in range(sheet.min_column, sheet.max_column + 1):
|
|
587
|
+
dim_holder[get_column_letter(col)] = ColumnDimension(sheet, min=col, max=col, width=width)
|
|
588
|
+
sheet.column_dimensions = dim_holder
|
|
540
589
|
|
|
541
590
|
@classmethod
|
|
542
591
|
def add_choice_list_to_sheet(cls, workbook, name, options, choice_list_dict):
|
|
543
592
|
active_sheet = workbook['Keuzelijsten']
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
if
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
xls_location = Path(ROOT_DIR) / 'UnitTests' / 'Subset' / 'testFileStorage' / 'template_file.csv'
|
|
565
|
-
subset_tool.generate_template_from_subset(path_to_subset=subset_location,
|
|
566
|
-
path_to_template_file_and_extension=xls_location, add_attribute_info=True,
|
|
567
|
-
highlight_deprecated_attributes=True,
|
|
568
|
-
amount_of_examples=5,
|
|
569
|
-
generate_choice_list=True,
|
|
570
|
-
split_per_type=False)
|
|
593
|
+
column_nr = choice_list_dict.keys().__len__() + 1
|
|
594
|
+
row_nr = 1
|
|
595
|
+
new_header = active_sheet.cell(row=row_nr, column=column_nr)
|
|
596
|
+
if new_header.value is not None:
|
|
597
|
+
raise ValueError(f'Header already exists at column {column_nr}: {new_header.value}')
|
|
598
|
+
new_header.value = name
|
|
599
|
+
for index, option in enumerate(options, start=1):
|
|
600
|
+
cell = active_sheet.cell(row=row_nr + index, column=column_nr)
|
|
601
|
+
cell.value = option.invulwaarde
|
|
602
|
+
|
|
603
|
+
choice_list_dict[name] = new_header.column_letter
|
|
604
|
+
|
|
605
|
+
@classmethod
|
|
606
|
+
def get_uri_from_sheet_name(cls, title: str) -> str:
|
|
607
|
+
if title == 'Agent':
|
|
608
|
+
return 'http://purl.org/dc/terms/Agent'
|
|
609
|
+
if '#' not in title:
|
|
610
|
+
raise ValueError('Sheet title does not contain a #')
|
|
611
|
+
class_ns, class_name = title.split('#', maxsplit=1)
|
|
612
|
+
return short_to_long_ns.get(class_ns, class_ns) + class_name
|