python-materialsdb 0.3.0__py3-none-any.whl → 0.3.1__py3-none-any.whl

materialsdb/cache.py

@@ -2,6 +2,7 @@ import os
 import pathlib
 import urllib.request
 from collections import namedtuple
+from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
 
 from lxml import etree
 
@@ -85,25 +86,77 @@ def _plan_index(index):
     return _IndexPlan(index, new_index, unchanged, todos)
 
 
-def update_producers_data(url_list=MATERIALSDBINDEXURLLIST, on_progress=None):
-    """on_progress(done, total, name) is called after each producer download
-    with a global running total; returning a truthy value stops the update
-    between two downloads (the partial Report is returned as-is)."""
+def update_producers_data(url_list=MATERIALSDBINDEXURLLIST, on_progress=None, max_workers: int = 8):
+    """Update the cached producer files from the index URLs.
+
+    Downloads run on a bounded thread pool (max_workers). on_progress(done,
+    total, name) is called once per completed download from the calling
+    thread with a monotonic running total; returning a truthy value stops
+    NEW submissions (queued jobs are abandoned, in-flight ones finish but
+    no longer count toward the partial Report). A download error stops the
+    run and the first exception is re-raised after the pool drains.
+    """
     plans = [_plan_index(index) for index in url_list]
-    total = sum(len(plan.todos) for plan in plans)
-    existing, updated, deleted = [], [], []
+    jobs = [
+        (plan, company, cached_producer, producer_path)
+        for plan in plans
+        for (company, cached_producer, producer_path) in plan.todos
+    ]
+    total = len(jobs)
+    workers = max(1, min(int(max_workers), total or 1))
+
+    existing: list = []
+    updated: list = []
+    deleted: list = []
     done = 0
+    cancel_requested = False
+    failure = None
+
+    def _download(job):
+        _, company, cached_producer, producer_path = job
+        deleted_path = None
+        if cached_producer is not None:
+            deleted_path = get_producers_dir() / pathlib.Path(cached_producer.get("href")).name
+            deleted_path.unlink(True)
+        urllib.request.urlretrieve(company.get("href"), producer_path)
+        return deleted_path
+
+    pool = ThreadPoolExecutor(max_workers=workers)
+    pending = list(jobs)  # pops from the FRONT, submission order preserved
+    futures: dict = {}
+    try:
+        while futures or pending:
+            while pending and len(futures) < workers:
+                job = pending.pop(0)
+                futures[pool.submit(_download, job)] = job
+            if not futures:
+                break
+            ready, _ = wait(set(futures), return_when=FIRST_COMPLETED)
+            for future in ready:
+                job = futures.pop(future)
+                try:
+                    deleted_path = future.result()
+                except Exception as err:  # noqa: BLE001 - re-raised below, caller decides
+                    if failure is None:
+                        failure = err
+                    break
+                if deleted_path is not None:
+                    deleted.append(deleted_path)
+                updated.append(job[3])
+                done += 1
+                if on_progress is not None and on_progress(done, total, pathlib.Path(job[3]).name):
+                    cancel_requested = True
+                    break
+            if cancel_requested or failure is not None:
+                break
+    finally:
+        pool.shutdown(wait=True, cancel_futures=True)
+
+    if failure is not None:
+        raise failure
+    if cancel_requested:
+        return Report(existing, updated, deleted)
     for plan in plans:
-        for company, cached_producer, producer_path in plan.todos:
-            if cached_producer is not None:
-                cached_path = get_producers_dir() / pathlib.Path(cached_producer.get("href")).name
-                deleted.append(cached_path)
-                cached_path.unlink(True)
-            urllib.request.urlretrieve(company.get("href"), producer_path)
-            updated.append(producer_path)
-            done += 1
-            if on_progress is not None and on_progress(done, total, pathlib.Path(company.get("href")).name):
-                return Report(existing, updated, deleted)
         if plan.todos:
             plan.new_index.write(str(get_cached_index_path(plan.index)))
         existing.extend(plan.unchanged)

materialsdb/serialiser.py

@@ -25,9 +25,31 @@ def get_xml_schema() -> str:
     return str(Path(__file__).parent / "schema/materialsdb103.xsd")
 
 
+@cache
+def cached_type_hints(cls) -> dict:
+    return typing.get_type_hints(cls)
+
+
+@cache
+def _tag_local_name(tag: str) -> str:
+    m = re.search("{.*}(.*)", tag)
+    return m.group(1) if m else tag
+
+
 def get_element_name(element: objectify.ObjectifiedElement) -> str:
-    m = re.search("{.*}(.*)", element.tag)
-    return m.group(1) if m else element.tag
+    return _tag_local_name(element.tag)
+
+
+@cache
+def _is_optional(type_hint):
+    return typing.get_origin(type_hint) is typing.Union and typing.get_args(type_hint)[1] is type(None)
+
+
+@cache
+def _strip_optional(type_hint):
+    if _is_optional(type_hint):
+        return typing.get_args(type_hint)[0]
+    return type_hint
 
 
 def create_element_maker():
@@ -49,11 +71,6 @@ def get_valid_root(tree: objectify.ObjectifiedElement) -> objectify.ObjectifiedE
     return root
 
 
-@cache
-def cached_type_hints(cls) -> dict:
-    return typing.get_type_hints(cls)
-
-
 class XmlDeserialiser:
     def __init__(self):
         self.schema = etree.XMLSchema(file=get_xml_schema())
@@ -93,7 +110,7 @@ class XmlDeserialiser:
             value = element.get(attrib)
             if value is None:
                 continue
-            base_class = self.strip_optional(type_hints[attrib])
+            base_class = _strip_optional(type_hints[attrib])
 
             kwargs[attrib] = base_class(value)
         for child_name in getattr(element_class, "xml_elements", ()):
@@ -127,12 +144,10 @@ class XmlDeserialiser:
         return instance
 
     def strip_optional(self, type_hint):
-        if self.is_optional(type_hint):
-            return typing.get_args(type_hint)[0]
-        return type_hint
+        return _strip_optional(type_hint)
 
     def is_optional(self, type_hint):
-        return typing.get_origin(type_hint) is typing.Union and typing.get_args(type_hint)[1] is type(None)
+        return _is_optional(type_hint)
 
     def cls_name(self, name: str) -> str:
         return name if name[0].isupper() else name.title()

{python_materialsdb-0.3.0.dist-info → python_materialsdb-0.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-materialsdb
-Version: 0.3.0
+Version: 0.3.1
 Summary: A library to work with materialsdb.org open standard for building materials.
 Author-email: Cyril Waechter <cyrwae@hotmail.com>
 License-Expression: GPL-3.0-or-later

{python_materialsdb-0.3.0.dist-info → python_materialsdb-0.3.1.dist-info}/RECORD

@@ -1,10 +1,10 @@
 materialsdb/__init__.py,sha256=lml7ffWFF3-0MD-Z7YbPZNireUpDO7tR9B3KNamrI0k,127
-materialsdb/cache.py,sha256=9XpJ7I5mLNpQZsVZSggIdUwS3votombJ9PfjbVbl-OA,4751
+materialsdb/cache.py,sha256=wPCksig6hidUssjRx_KaPk9V5ZeWbHfbs0Qt2XTuLPk,6594
 materialsdb/classes.py,sha256=WdXNiCqFcevgWrqDZKLR8HPio3BwcmOJ1NyVGArQ0hw,21549
 materialsdb/config.py,sha256=Tp14G_Y_XEkYshwrp4gd9XjGb5e_mO-hxYOevCTVnmI,1291
 materialsdb/construction.py,sha256=29H5gt3EX0LFNJ7ftE0Ob7huQgyZzx3ygW7nNcUSo5s,17817
 materialsdb/query.py,sha256=h9kP3Zc41l4h9bcxjwp-UUb8ezsG0A3ZkraJjaQZVVI,620
-materialsdb/serialiser.py,sha256=GL_e1FW6inuvfLvqQzp81L8XgoByRtVtd7nWtrYKWmY,7056
+materialsdb/serialiser.py,sha256=J6bUc7eKYD569TrOWVAK5ZDeb7cdD2YbF0nD6jf3iPY,7263
 materialsdb/store.py,sha256=_Iu425X7-sYE7A2vvQVrR1R6ju2R63m_BK43c9y3mhM,11000
 materialsdb/summary.py,sha256=vl6LhVIfN4KHN-KZxU6oVBYut0FPA3E_baGp_fegnBU,2558
 materialsdb/utils.py,sha256=2xGwr6b4NgOtjT5lxpFVN9jnOSlDsmZPnelSGEs5MDs,3068
@@ -25,9 +25,9 @@ materialsdb/ifc/project_library.py,sha256=hOvirB73ZiTYiP6rOu6eN1dU2ZXFRgSOGpQrH2
 materialsdb/schema/MaterialsDBIndex100.xsd,sha256=oK81n2b3_82Mt5G31yEm9vwkMjUmNyM6jglt_zpUQZc,2199
 materialsdb/schema/materialsdb102.xsd,sha256=IaC6E_B3N1VaU9lhrl6ZGwi8m2rk2p7HTEZKN_Yj2AY,32880
 materialsdb/schema/materialsdb103.xsd,sha256=iX7Q1ejWEN_ZSuRVCX40zjYt5UraY3f_JxBzqZUFu7Q,37762
-python_materialsdb-0.3.0.dist-info/licenses/LICENSE.md,sha256=M7wm1EmMGDtwPRdg7kW4d00h1uAXjKOT3HFScYQMeiE,34916
-python_materialsdb-0.3.0.dist-info/METADATA,sha256=dugVsiclmutXpwQj_CI8tkEe0oGNus2Amc8vnZqit_w,7673
-python_materialsdb-0.3.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-python_materialsdb-0.3.0.dist-info/entry_points.txt,sha256=7bCySVv6oV9hiva7Fn5Dm9oboPRIC6DU6DRlsjCLO6k,66
-python_materialsdb-0.3.0.dist-info/top_level.txt,sha256=5ZHbF8Oj1W24bifG65Fl3hCaivZpkO6EyOHHaRavZxE,12
-python_materialsdb-0.3.0.dist-info/RECORD,,
+python_materialsdb-0.3.1.dist-info/licenses/LICENSE.md,sha256=M7wm1EmMGDtwPRdg7kW4d00h1uAXjKOT3HFScYQMeiE,34916
+python_materialsdb-0.3.1.dist-info/METADATA,sha256=y09iI0wUVkcDhSEnA3YwIEB4S-CXQx4Ms9IIfMiYvrU,7673
+python_materialsdb-0.3.1.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+python_materialsdb-0.3.1.dist-info/entry_points.txt,sha256=7bCySVv6oV9hiva7Fn5Dm9oboPRIC6DU6DRlsjCLO6k,66
+python_materialsdb-0.3.1.dist-info/top_level.txt,sha256=5ZHbF8Oj1W24bifG65Fl3hCaivZpkO6EyOHHaRavZxE,12
+python_materialsdb-0.3.1.dist-info/RECORD,,