otlmow-template 1.2__tar.gz → 1.4__tar.gz

{otlmow_template-1.2 → otlmow_template-1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: otlmow_template
-Version: 1.2
+Version: 1.4
 Author-email: David Vlaminck <david.vlaminck@mow.vlaanderen.be>, Jasper Berton <jasperberton1@telenet.be>
 License:                     GNU GENERAL PUBLIC LICENSE
                                Version 3, 29 June 2007
@@ -697,7 +697,7 @@ Classifier: Topic :: Software Development :: Quality Assurance
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: otlmow-converter>=1.11
+Requires-Dist: otlmow-converter>=1.14
 Requires-Dist: otlmow-modelbuilder>=0.27
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"

{otlmow_template-1.2 → otlmow_template-1.4}/otlmow_template/SubsetTemplateCreator.py

@@ -1,11 +1,12 @@
+import asyncio
+import concurrent
 import contextlib
 import csv
 import logging
 import os
-import shutil
-import tempfile
 from asyncio import sleep
 from collections import defaultdict
+from concurrent.futures import ThreadPoolExecutor, as_completed, ALL_COMPLETED
 from pathlib import Path
 
 from openpyxl.reader.excel import load_workbook
@@ -16,13 +17,15 @@ from openpyxl.worksheet.datavalidation import DataValidation
 from openpyxl.worksheet.dimensions import DimensionHolder, ColumnDimension
 from openpyxl.worksheet.worksheet import Worksheet
 from otlmow_converter.DotnotationHelper import DotnotationHelper
+from otlmow_converter.Exceptions.UnknownExcelError import UnknownExcelError
 from otlmow_converter.OtlmowConverter import OtlmowConverter
 from otlmow_model.OtlmowModel.BaseClasses.BooleanField import BooleanField
 from otlmow_model.OtlmowModel.BaseClasses.KeuzelijstField import KeuzelijstField
 from otlmow_model.OtlmowModel.BaseClasses.OTLObject import dynamic_create_instance_from_uri, OTLObject, \
-    get_attribute_by_name
+    get_attribute_by_name, dynamic_create_type_from_uri
+from otlmow_model.OtlmowModel.Helpers.GenericHelper import get_ns_and_name_from_uri
+from otlmow_model.OtlmowModel.Helpers.RelationCreator import create_betrokkenerelation, create_relation
 from otlmow_model.OtlmowModel.Helpers.generated_lists import get_hardcoded_relation_dict
-from otlmow_modelbuilder.HelperFunctions import get_ns_and_name_from_uri
 from otlmow_modelbuilder.OSLOCollector import OSLOCollector
 from otlmow_modelbuilder.SQLDataClasses.OSLOClass import OSLOClass
 
@@ -167,6 +170,34 @@ class SubsetTemplateCreator:
                 split_per_type=split_per_type, file_path=template_file_path, dummy_data_rows=dummy_data_rows,
                 instances=objects, add_deprecated=add_deprecated, add_attribute_info=add_attribute_info)
 
+
+    @classmethod
+    def create_x_objects(cls, oslo_class, add_geometry, collector, filter_attributes_by_subset, model_directory,
+                         amount_objects_to_create):
+        if oslo_class.objectUri in cls.relation_dict:
+            return []
+
+        otl_objects = []
+        for _ in range(amount_objects_to_create):
+            otl_object = cls.generate_object_from_oslo_class(
+                oslo_class=oslo_class, add_geometry=add_geometry, collector=collector,
+                filter_attributes_by_subset=filter_attributes_by_subset, model_directory=model_directory)
+            if otl_object is not None:
+                otl_objects.append(otl_object)
+        return otl_objects
+
+    @classmethod
+    def get_number_of_cpus(cls) -> int:
+        import multiprocessing
+        cpu_count = multiprocessing.cpu_count()
+        if cpu_count is None or cpu_count == 0:
+            import os
+            cpu_count = os.cpu_count()
+        if cpu_count is None or cpu_count == 0:
+            cpu_count = 8
+        return cpu_count
+
+
     @classmethod
     def generate_objects_for_template(
             cls, subset_path: Path, class_uris_filter: [str], filter_attributes_by_subset: bool,
@@ -177,31 +208,60 @@ class SubsetTemplateCreator:
         """
         collector = cls._load_collector_from_subset_path(subset_path=subset_path)
         filtered_class_list = cls.filters_classes_by_subset(collector=collector, class_uris_filter=class_uris_filter)
-        relation_dict = get_hardcoded_relation_dict(model_directory=model_directory)
+        cls.relation_dict = get_hardcoded_relation_dict(model_directory=model_directory)
 
         amount_objects_to_create = max(1, dummy_data_rows)
         otl_objects = []
 
         while True:
-            for oslo_class in [cl for cl in filtered_class_list if cl.abstract == 0]:
-                if ignore_relations and oslo_class.objectUri in relation_dict:
-                    continue
+            with ThreadPoolExecutor(max_workers=cls.get_number_of_cpus()) as executor:
+                futures = [
+                    executor.submit(cls.create_x_objects, oslo_class, add_geometry, collector,
+                                    filter_attributes_by_subset, model_directory, amount_objects_to_create)
+                    for oslo_class in [cl for cl in filtered_class_list if cl.abstract == 0]
+                ]
+                while futures:
+                    done, not_done = concurrent.futures.wait(futures, return_when=ALL_COMPLETED, timeout=60)
+                    for future in as_completed(done):
+                        otl_objects.extend(future.result())
+                    futures = not_done
+                    print(f'when using multiprocessing: {len(done)} done, {len(not_done)} not done')
+                    if len(not_done) > 0:
+                        print('restarting the loop with the not done ones')
 
-                for _ in range(amount_objects_to_create):
-                    otl_object = cls.generate_object_from_oslo_class(
-                        oslo_class=oslo_class, add_geometry=add_geometry, collector=collector,
-                        filter_attributes_by_subset=filter_attributes_by_subset, model_directory=model_directory)
-                    if otl_object is not None:
-                        otl_objects.append(otl_object)
             created = len(otl_objects)
-            unique_ids = len({obj.assetId.identificator if hasattr(obj, 'assetId') else obj.agentId.identificator
+            unique_ids = len({obj.assetId.identificator if obj.typeURI != 'http://purl.org/dc/terms/Agent' else obj.agentId.identificator
                               for obj in otl_objects})
             if created == unique_ids:
                 break
             otl_objects = []
 
+        if not ignore_relations:
+            non_relations_class_uris = [cl.objectUri for cl in filtered_class_list
+                                        if cl.abstract == 0 and cl.objectUri not in cls.relation_dict]
+            cls.append_relations_to_objects(otl_objects=otl_objects, collector=collector,
+                                            class_uris_filter=non_relations_class_uris, model_directory=model_directory)
+
         return otl_objects
 
+    @classmethod
+    async def create_x_objects_async(cls, oslo_class, add_geometry, collector, filter_attributes_by_subset,
+                                  model_directory,
+                         amount_objects_to_create):
+        if oslo_class.objectUri in cls.relation_dict:
+            return []
+
+        otl_objects = []
+        for _ in range(amount_objects_to_create):
+            await sleep(0)
+            otl_object = cls.generate_object_from_oslo_class(
+                oslo_class=oslo_class, add_geometry=add_geometry, collector=collector,
+                filter_attributes_by_subset=filter_attributes_by_subset, model_directory=model_directory)
+            if otl_object is not None:
+                otl_objects.append(otl_object)
+        return otl_objects
+
+
     @classmethod
     async def generate_objects_for_template_async(
             cls, subset_path: Path, class_uris_filter: [str], filter_attributes_by_subset: bool,
@@ -215,23 +275,33 @@ class SubsetTemplateCreator:
         await sleep(0)
         filtered_class_list = cls.filters_classes_by_subset(collector=collector, class_uris_filter=class_uris_filter)
         await sleep(0)
-        relation_dict = get_hardcoded_relation_dict(model_directory=model_directory)
+        cls.relation_dict = get_hardcoded_relation_dict(model_directory=model_directory)
 
         amount_objects_to_create = max(1, dummy_data_rows)
         otl_objects = []
 
-        for oslo_class in [cl for cl in filtered_class_list if cl.abstract == 0]:
-            await sleep(0)
-            if ignore_relations and oslo_class.objectUri in relation_dict:
-                continue
+        while True:
+            tasks = [
+                cls.create_x_objects_async(oslo_class, add_geometry, collector,
+                                           filter_attributes_by_subset, model_directory, amount_objects_to_create)
+                for oslo_class in [cl for cl in filtered_class_list if cl.abstract == 0]
+            ]
+            results = await asyncio.gather(*tasks)
+            otl_objects.extend([item for sublist in results for item in sublist])
 
-            for _ in range(amount_objects_to_create):
-                otl_object = cls.generate_object_from_oslo_class(
-                    oslo_class=oslo_class, add_geometry=add_geometry, collector=collector,
-                    filter_attributes_by_subset=filter_attributes_by_subset, model_directory=model_directory)
-                await sleep(0)
-                if otl_object is not None:
-                    otl_objects.append(otl_object)
+            created = len(otl_objects)
+            unique_ids = len({
+                                 obj.assetId.identificator if obj.typeURI != 'http://purl.org/dc/terms/Agent' else obj.agentId.identificator
+                                 for obj in otl_objects})
+            if created == unique_ids:
+                break
+            otl_objects = []
+
+        if not ignore_relations:
+            non_relations_class_uris = [cl.objectUri for cl in filtered_class_list
+                                        if cl.abstract == 0 and cl.objectUri not in cls.relation_dict]
+            cls.append_relations_to_objects(otl_objects=otl_objects, collector=collector,
+                                            class_uris_filter=non_relations_class_uris, model_directory=model_directory)
 
         return otl_objects
 
@@ -249,15 +319,23 @@ class SubsetTemplateCreator:
         if filter_attributes_by_subset:
             for attribute_object in collector.find_attributes_by_class(oslo_class):
                 attr = get_attribute_by_name(instance, attribute_object.name)
-                if attr is not None:
-                    attr.fill_with_dummy_data()
-                else:
+                if attr is None:
                     logging.warning(f'Attribute {attribute_object.name} not found in class {oslo_class.objectUri}')
+                elif attr.naam == 'isActief':
+                    attr.set_waarde(True)
+                else:
+                    attr.fill_with_dummy_data()
         else:
             for attr in instance:
-                if attr.naam != 'geometry':
+                if attr.naam == 'isActief':
+                    attr.set_waarde(True)
+                elif attr.naam != 'geometry':
                     attr.fill_with_dummy_data()
+
         with contextlib.suppress(AttributeError):
+            assetId_attr = get_attribute_by_name(instance, 'assetId')
+            if assetId_attr is not None:
+                assetId_attr.fill_with_dummy_data()
             if add_geometry:
                 geo_attr = get_attribute_by_name(instance, 'geometry')
                 if geo_attr is not None:
@@ -304,7 +382,10 @@ class SubsetTemplateCreator:
         if split_per_type:
             for type_uri, typed_instances in classes_dict.items():
                 ns, name = get_ns_and_name_from_uri(type_uri)
-                class_file_path = file_path.parent / f'{file_path.stem}_{ns}_{name}.csv'
+                ns_name = f'{ns}_{name}'
+                if name == 'Agent':
+                    ns_name = 'Agent'
+                class_file_path = file_path.parent / f'{file_path.stem}_{ns_name}.csv'
                 cls.alter_csv_file(add_attribute_info=add_attribute_info, add_deprecated=add_deprecated,
                                    dummy_data_rows=dummy_data_rows, instances=typed_instances, file_path=class_file_path)
         else:
@@ -319,7 +400,10 @@ class SubsetTemplateCreator:
             for type_uri, typed_instances in classes_dict.items():
                 await sleep(0)
                 ns, name = get_ns_and_name_from_uri(type_uri)
-                class_file_path = file_path.parent / f'{file_path.stem}_{ns}_{name}.csv'
+                ns_name = f'{ns}_{name}'
+                if name == 'Agent':
+                    ns_name = 'Agent'
+                class_file_path = file_path.parent / f'{file_path.stem}_{ns_name}.csv'
                 cls.alter_csv_file(add_attribute_info=add_attribute_info,
                                    dummy_data_rows=dummy_data_rows, instances=typed_instances, add_deprecated=add_deprecated,
                                    file_path=class_file_path)
@@ -354,7 +438,14 @@ class SubsetTemplateCreator:
                     deprecated_attributes_row.append('')
                 continue
 
-            attribute = DotnotationHelper.get_attribute_by_dotnotation(instance, header)
+            try:
+                attribute = DotnotationHelper.get_attribute_by_dotnotation(instance, header)
+            except AttributeError:
+                if add_attribute_info:
+                    collected_attribute_info_row.append('')
+                if add_deprecated:
+                    deprecated_attributes_row.append('')
+                continue
 
             if add_attribute_info:
                 collected_attribute_info_row.append(attribute.definition)
@@ -362,7 +453,7 @@ class SubsetTemplateCreator:
             if add_deprecated:
                 deprecated_attributes_row.append('DEPRECATED' if attribute.deprecated_version else '')
 
-        with open(file_path, 'w') as file:
+        with open(file_path, 'w', newline='', encoding='utf-8') as file:
             csv_writer = csv.writer(file, delimiter=';', quotechar=quote_char, quoting=csv.QUOTE_MINIMAL)
             if add_attribute_info:
                 csv_writer.writerow(collected_attribute_info_row)
@@ -397,7 +488,11 @@ class SubsetTemplateCreator:
                           instances: [OTLObject], sheet: Worksheet, add_deprecated: bool, workbook: Workbook,
                           dummy_data_rows: int):
         type_uri = cls.get_uri_from_sheet_name(sheet.title)
-        instance = next(x for x in instances if x.typeURI == type_uri)
+        instance = next((x for x in instances if x.typeURI == type_uri), None)
+        if instance is None:
+            instance = next((x for x in instances if x.typeURI.startswith(type_uri)), None)
+        if instance is None:
+            raise UnknownExcelError(f'When creating a template, no instance could be created for {type_uri}')
 
         boolean_validation = DataValidation(type="list", formula1='"TRUE,FALSE,"', allow_blank=True)
         sheet.add_data_validation(boolean_validation)
@@ -461,13 +556,14 @@ class SubsetTemplateCreator:
                 cell.fill = PatternFill(start_color="FF7276", end_color="FF7276", fill_type="solid")
 
     @classmethod
-    def add_attribute_info_to_sheet(cls, collected_attribute_info, sheet):
+    def add_attribute_info_to_sheet(cls, collected_attribute_info: list[str], sheet: Worksheet):
         sheet.insert_rows(idx=1)
+        fill = PatternFill(start_color="808080", fill_type="solid")
+        alignment = Alignment(wrapText=True, vertical='top')
         for index, attr_info in enumerate(collected_attribute_info, start=1):
-            cell = sheet.cell(row=1, column=index)
-            cell.value = attr_info
-            cell.alignment = Alignment(wrapText=True, vertical='top')
-            cell.fill = PatternFill(start_color="808080", end_color="808080", fill_type="solid")
+            cell = sheet.cell(row=1, column=index, value=attr_info)
+            cell.alignment = alignment
+            cell.fill = fill
 
     @classmethod
     def generate_choice_list_in_excel(cls, attribute, choice_list_dict, column, row_nr, sheet: Worksheet,
@@ -610,3 +706,28 @@ class SubsetTemplateCreator:
             raise ValueError('Sheet title does not contain a #')
         class_ns, class_name = title.split('#', maxsplit=1)
         return short_to_long_ns.get(class_ns, class_ns) + class_name
+
+    @classmethod
+    def append_relations_to_objects(cls, otl_objects: [OTLObject], collector: OSLOCollector, class_uris_filter: [str],
+                                    model_directory: Path):
+        class_dict = defaultdict(list)
+        for instance in otl_objects:
+            class_dict[instance.typeURI].append(instance)
+
+        for class_uri in class_uris_filter:
+            for relation in collector.find_all_concrete_relations(objectUri=class_uri):
+                if class_uri != relation.bron_uri:
+                    continue
+                for i, bron_instance in enumerate(class_dict[relation.bron_uri]):
+                    doel_instance = class_dict[relation.doel_uri][i]
+                    if relation.objectUri == 'https://wegenenverkeer.data.vlaanderen.be/ns/onderdeel#HeeftBetrokkene':
+                        relation_instance = create_betrokkenerelation(rol='toezichter', source=bron_instance,
+                                                            target=doel_instance, model_directory=model_directory)
+                    else:
+                        if relation.richting == 'Unspecified' and bron_instance.assetId.identificator > doel_instance.assetId.identificator:
+                            continue
+                        relation_type = dynamic_create_type_from_uri(class_uri=relation.objectUri, model_directory=model_directory)
+                        relation_instance = create_relation(relation_type=relation_type, source=bron_instance,
+                                                            target=doel_instance, model_directory=model_directory)
+
+                    otl_objects.append(relation_instance)

{otlmow_template-1.2 → otlmow_template-1.4}/otlmow_template.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: otlmow_template
-Version: 1.2
+Version: 1.4
 Author-email: David Vlaminck <david.vlaminck@mow.vlaanderen.be>, Jasper Berton <jasperberton1@telenet.be>
 License:                     GNU GENERAL PUBLIC LICENSE
                                Version 3, 29 June 2007
@@ -697,7 +697,7 @@ Classifier: Topic :: Software Development :: Quality Assurance
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: otlmow-converter>=1.11
+Requires-Dist: otlmow-converter>=1.14
 Requires-Dist: otlmow-modelbuilder>=0.27
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"

{otlmow_template-1.2 → otlmow_template-1.4}/otlmow_template.egg-info/requires.txt

@@ -1,4 +1,4 @@
-otlmow-converter>=1.11
+otlmow-converter>=1.14
 otlmow-modelbuilder>=0.27
 
 [test]

{otlmow_template-1.2 → otlmow_template-1.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "otlmow_template"
-version = "1.2"
+version = "1.4"
 authors = [{name = "David Vlaminck", email = "david.vlaminck@mow.vlaanderen.be"},
     {name = "Jasper Berton", email = "jasperberton1@telenet.be"},]
 readme = "README.md"
@@ -29,7 +29,7 @@ classifiers = [
 ]
 requires-python = ">=3.9"
 dependencies = [
-  'otlmow-converter >= 1.11',
+  'otlmow-converter >= 1.14',
   'otlmow-modelbuilder >= 0.27'
 ]