otlmow-template 1.3__py3-none-any.whl → 1.4__py3-none-any.whl

otlmow_template/SubsetTemplateCreator.py

@@ -1,11 +1,12 @@
+import asyncio
+import concurrent
 import contextlib
 import csv
 import logging
 import os
-import shutil
-import tempfile
 from asyncio import sleep
 from collections import defaultdict
+from concurrent.futures import ThreadPoolExecutor, as_completed, ALL_COMPLETED
 from pathlib import Path
 
 from openpyxl.reader.excel import load_workbook
@@ -169,6 +170,34 @@ class SubsetTemplateCreator:
                 split_per_type=split_per_type, file_path=template_file_path, dummy_data_rows=dummy_data_rows,
                 instances=objects, add_deprecated=add_deprecated, add_attribute_info=add_attribute_info)
 
+
+    @classmethod
+    def create_x_objects(cls, oslo_class, add_geometry, collector, filter_attributes_by_subset, model_directory,
+                         amount_objects_to_create):
+        if oslo_class.objectUri in cls.relation_dict:
+            return []
+
+        otl_objects = []
+        for _ in range(amount_objects_to_create):
+            otl_object = cls.generate_object_from_oslo_class(
+                oslo_class=oslo_class, add_geometry=add_geometry, collector=collector,
+                filter_attributes_by_subset=filter_attributes_by_subset, model_directory=model_directory)
+            if otl_object is not None:
+                otl_objects.append(otl_object)
+        return otl_objects
+
+    @classmethod
+    def get_number_of_cpus(cls) -> int:
+        import multiprocessing
+        cpu_count = multiprocessing.cpu_count()
+        if cpu_count is None or cpu_count == 0:
+            import os
+            cpu_count = os.cpu_count()
+        if cpu_count is None or cpu_count == 0:
+            cpu_count = 8
+        return cpu_count
+
+
     @classmethod
     def generate_objects_for_template(
             cls, subset_path: Path, class_uris_filter: [str], filter_attributes_by_subset: bool,
@@ -179,22 +208,27 @@ class SubsetTemplateCreator:
         """
         collector = cls._load_collector_from_subset_path(subset_path=subset_path)
         filtered_class_list = cls.filters_classes_by_subset(collector=collector, class_uris_filter=class_uris_filter)
-        relation_dict = get_hardcoded_relation_dict(model_directory=model_directory)
+        cls.relation_dict = get_hardcoded_relation_dict(model_directory=model_directory)
 
         amount_objects_to_create = max(1, dummy_data_rows)
         otl_objects = []
 
         while True:
-            for oslo_class in [cl for cl in filtered_class_list if cl.abstract == 0]:
-                if oslo_class.objectUri in relation_dict:
-                    continue
+            with ThreadPoolExecutor(max_workers=cls.get_number_of_cpus()) as executor:
+                futures = [
+                    executor.submit(cls.create_x_objects, oslo_class, add_geometry, collector,
+                                    filter_attributes_by_subset, model_directory, amount_objects_to_create)
+                    for oslo_class in [cl for cl in filtered_class_list if cl.abstract == 0]
+                ]
+                while futures:
+                    done, not_done = concurrent.futures.wait(futures, return_when=ALL_COMPLETED, timeout=60)
+                    for future in as_completed(done):
+                        otl_objects.extend(future.result())
+                    futures = not_done
+                    print(f'when using multiprocessing: {len(done)} done, {len(not_done)} not done')
+                    if len(not_done) > 0:
+                        print('restarting the loop with the not done ones')
 
-                for _ in range(amount_objects_to_create):
-                    otl_object = cls.generate_object_from_oslo_class(
-                        oslo_class=oslo_class, add_geometry=add_geometry, collector=collector,
-                        filter_attributes_by_subset=filter_attributes_by_subset, model_directory=model_directory)
-                    if otl_object is not None:
-                        otl_objects.append(otl_object)
             created = len(otl_objects)
             unique_ids = len({obj.assetId.identificator if obj.typeURI != 'http://purl.org/dc/terms/Agent' else obj.agentId.identificator
                               for obj in otl_objects})
@@ -204,12 +238,30 @@ class SubsetTemplateCreator:
 
         if not ignore_relations:
             non_relations_class_uris = [cl.objectUri for cl in filtered_class_list
-                                        if cl.abstract == 0 and cl.objectUri not in relation_dict]
+                                        if cl.abstract == 0 and cl.objectUri not in cls.relation_dict]
             cls.append_relations_to_objects(otl_objects=otl_objects, collector=collector,
                                             class_uris_filter=non_relations_class_uris, model_directory=model_directory)
 
         return otl_objects
 
+    @classmethod
+    async def create_x_objects_async(cls, oslo_class, add_geometry, collector, filter_attributes_by_subset,
+                                  model_directory,
+                         amount_objects_to_create):
+        if oslo_class.objectUri in cls.relation_dict:
+            return []
+
+        otl_objects = []
+        for _ in range(amount_objects_to_create):
+            await sleep(0)
+            otl_object = cls.generate_object_from_oslo_class(
+                oslo_class=oslo_class, add_geometry=add_geometry, collector=collector,
+                filter_attributes_by_subset=filter_attributes_by_subset, model_directory=model_directory)
+            if otl_object is not None:
+                otl_objects.append(otl_object)
+        return otl_objects
+
+
     @classmethod
     async def generate_objects_for_template_async(
             cls, subset_path: Path, class_uris_filter: [str], filter_attributes_by_subset: bool,
@@ -223,27 +275,31 @@ class SubsetTemplateCreator:
         await sleep(0)
         filtered_class_list = cls.filters_classes_by_subset(collector=collector, class_uris_filter=class_uris_filter)
         await sleep(0)
-        relation_dict = get_hardcoded_relation_dict(model_directory=model_directory)
+        cls.relation_dict = get_hardcoded_relation_dict(model_directory=model_directory)
 
         amount_objects_to_create = max(1, dummy_data_rows)
         otl_objects = []
 
-        for oslo_class in [cl for cl in filtered_class_list if cl.abstract == 0]:
-            await sleep(0)
-            if oslo_class.objectUri in relation_dict:
-                continue
+        while True:
+            tasks = [
+                cls.create_x_objects_async(oslo_class, add_geometry, collector,
+                                           filter_attributes_by_subset, model_directory, amount_objects_to_create)
+                for oslo_class in [cl for cl in filtered_class_list if cl.abstract == 0]
+            ]
+            results = await asyncio.gather(*tasks)
+            otl_objects.extend([item for sublist in results for item in sublist])
 
-            for _ in range(amount_objects_to_create):
-                otl_object = cls.generate_object_from_oslo_class(
-                    oslo_class=oslo_class, add_geometry=add_geometry, collector=collector,
-                    filter_attributes_by_subset=filter_attributes_by_subset, model_directory=model_directory)
-                await sleep(0)
-                if otl_object is not None:
-                    otl_objects.append(otl_object)
+            created = len(otl_objects)
+            unique_ids = len({
+                                 obj.assetId.identificator if obj.typeURI != 'http://purl.org/dc/terms/Agent' else obj.agentId.identificator
+                                 for obj in otl_objects})
+            if created == unique_ids:
+                break
+            otl_objects = []
 
         if not ignore_relations:
             non_relations_class_uris = [cl.objectUri for cl in filtered_class_list
-                                        if cl.abstract == 0 and cl.objectUri not in relation_dict]
+                                        if cl.abstract == 0 and cl.objectUri not in cls.relation_dict]
             cls.append_relations_to_objects(otl_objects=otl_objects, collector=collector,
                                             class_uris_filter=non_relations_class_uris, model_directory=model_directory)
 
@@ -500,13 +556,14 @@ class SubsetTemplateCreator:
                 cell.fill = PatternFill(start_color="FF7276", end_color="FF7276", fill_type="solid")
 
     @classmethod
-    def add_attribute_info_to_sheet(cls, collected_attribute_info, sheet):
+    def add_attribute_info_to_sheet(cls, collected_attribute_info: list[str], sheet: Worksheet):
         sheet.insert_rows(idx=1)
+        fill = PatternFill(start_color="808080", fill_type="solid")
+        alignment = Alignment(wrapText=True, vertical='top')
         for index, attr_info in enumerate(collected_attribute_info, start=1):
-            cell = sheet.cell(row=1, column=index)
-            cell.value = attr_info
-            cell.alignment = Alignment(wrapText=True, vertical='top')
-            cell.fill = PatternFill(start_color="808080", end_color="808080", fill_type="solid")
+            cell = sheet.cell(row=1, column=index, value=attr_info)
+            cell.alignment = alignment
+            cell.fill = fill
 
     @classmethod
     def generate_choice_list_in_excel(cls, attribute, choice_list_dict, column, row_nr, sheet: Worksheet,

{otlmow_template-1.3.dist-info → otlmow_template-1.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: otlmow_template
-Version: 1.3
+Version: 1.4
 Author-email: David Vlaminck <david.vlaminck@mow.vlaanderen.be>, Jasper Berton <jasperberton1@telenet.be>
 License:                     GNU GENERAL PUBLIC LICENSE
                                Version 3, 29 June 2007

otlmow_template-1.4.dist-info/RECORD

@@ -0,0 +1,7 @@
+otlmow_template/SubsetTemplateCreator.py,sha256=iZD2kjhOnymQ9tgqU6zq2ZCD8KVIT232mbrxTSR4fQ8,35845
+otlmow_template/Exceptions/MissingTypeUriException.py,sha256=DSKwywmP9Bq8n7rzBoDcEPlxvC1IChx18QIHFUCTtdA,51
+otlmow_template-1.4.dist-info/licenses/LICENSE,sha256=ixuiBLtpoK3iv89l7ylKkg9rs2GzF9ukPH7ynZYzK5s,35148
+otlmow_template-1.4.dist-info/METADATA,sha256=MqaUUEubb-DhKErKteUljO6Z3lxjGlaJoDL4a0cam-k,44220
+otlmow_template-1.4.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+otlmow_template-1.4.dist-info/top_level.txt,sha256=zPgBoaTLG-avoOLySlwOUEtHaFyA5Vc5wJqkSeX1l6A,16
+otlmow_template-1.4.dist-info/RECORD,,

otlmow_template-1.3.dist-info/RECORD

@@ -1,7 +0,0 @@
-otlmow_template/SubsetTemplateCreator.py,sha256=SNXGHR5zMTSTs_xzWWDZ_183oV-D46SN6PQ-EPMikKs,33360
-otlmow_template/Exceptions/MissingTypeUriException.py,sha256=DSKwywmP9Bq8n7rzBoDcEPlxvC1IChx18QIHFUCTtdA,51
-otlmow_template-1.3.dist-info/licenses/LICENSE,sha256=ixuiBLtpoK3iv89l7ylKkg9rs2GzF9ukPH7ynZYzK5s,35148
-otlmow_template-1.3.dist-info/METADATA,sha256=BzvQ3dSl1uof3tLpGWJt6sHHAxEp0PAwv0Bz8iC50Pc,44220
-otlmow_template-1.3.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-otlmow_template-1.3.dist-info/top_level.txt,sha256=zPgBoaTLG-avoOLySlwOUEtHaFyA5Vc5wJqkSeX1l6A,16
-otlmow_template-1.3.dist-info/RECORD,,