python-materialsdb 0.0.2__py3-none-any.whl → 0.1.0__py3-none-any.whl

materialsdb/cache.py

@@ -75,7 +75,8 @@ def update_producers_data(url_list=MATERIALSDBINDEXURLLIST):
 def update_producers_from_index(index):
     cached_index = parse_cached_index(index)
     cached_root = cached_index.getroot()
-    new_index = etree.parse(index)
+    with urllib.request.urlopen(index) as response:
+        new_index = etree.parse(response)
     new_root = new_index.getroot()
     producers_dir = get_producers_dir()
     has_index_update = False

materialsdb/ifc/project_library.py

@@ -209,7 +209,7 @@ class ProjectLibrary:
                     ifcopenshell.api.run(
                         "material.assign_material",
                         file,
-                        product=wall,
+                        products=[wall],
                         material=assigned_material,
                     )
                 if material.information.roof:
@@ -221,7 +221,7 @@ class ProjectLibrary:
                     ifcopenshell.api.run(
                         "material.assign_material",
                         file,
-                        product=roof,
+                        products=[roof],
                         material=assigned_material,
                     )
                 if material.information.floor:
@@ -233,7 +233,7 @@ class ProjectLibrary:
                     ifcopenshell.api.run(
                         "material.assign_material",
                         file,
-                        product=slab,
+                        products=[slab],
                         material=assigned_material,
                     )
                 if material.information.door:
@@ -245,7 +245,7 @@ class ProjectLibrary:
                     ifcopenshell.api.run(
                         "material.assign_material",
                         file,
-                        product=door,
+                        products=[door],
                         material=assigned_material,
                     )
 

materialsdb/query.py

@@ -0,0 +1,24 @@
+"""Convenience facade over the SQLite material store."""
+from functools import lru_cache
+from typing import List, Optional
+
+from materialsdb.classes import Material
+from materialsdb.store import MaterialStore, Report
+from materialsdb.summary import MaterialSummary
+
+
+@lru_cache(maxsize=1)
+def get_store() -> MaterialStore:
+    return MaterialStore()
+
+
+def get_material(material_id: str) -> Optional[Material]:
+    return get_store().get(material_id)
+
+
+def search(text: str, **filters) -> List[MaterialSummary]:
+    return get_store().summaries(text=text, **filters)
+
+
+def refresh(force: bool = False) -> Report:
+    return get_store().refresh(force=force)

materialsdb/serialiser.py

@@ -9,6 +9,8 @@ See the LICENSE.md file for more details.
 Author : Cyril Waechter
 """
 import re
+from concurrent.futures import ThreadPoolExecutor
+from functools import lru_cache
 from pathlib import Path
 import typing
 from typing import Protocol, Tuple, Dict, Type, Optional, Any, Union
@@ -50,6 +52,11 @@ def get_valid_root(tree: objectify.ObjectifiedElement) -> str:
     return root
 
 
+@lru_cache(maxsize=None)
+def cached_type_hints(cls) -> dict:
+    return typing.get_type_hints(cls)
+
+
 class XmlDeserialiser:
     def __init__(self):
         self.schema = etree.XMLSchema(file=get_xml_schema())
@@ -62,13 +69,29 @@ class XmlDeserialiser:
             tree = objectify.parse(xml_path)
         return self.from_element(get_valid_root(tree))
 
+    def from_xml_files(self, paths, max_workers=None):
+        """Parse multiple producer XML files concurrently.
+
+        Yields (path, Materials) per successfully parsed file and
+        (path, None) for files that could not be parsed."""
+
+        def load(path):
+            try:
+                return Path(path), self.from_xml(str(path))
+            except Exception as err:
+                print(f"{Path(path).name}: could not parse file:\n\t{err}")
+                return Path(path), None
+
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            yield from executor.map(load, paths)
+
     def from_element(self, element=None, base_class=None):
         element_name = get_element_name(element)
         element_class = base_class or getattr(classes, self.cls_name(element_name))
         kwargs: Dict[str, Any] = {}
         if element_class.xs_type != "element":
             kwargs["object"] = element.text or ""
-        type_hints = typing.get_type_hints(element_class)
+        type_hints = cached_type_hints(element_class)
         for attrib in getattr(element_class, "xml_attributes", ()):
             value = element.get(attrib)
             if value is None:

materialsdb/store.py

@@ -0,0 +1,290 @@
+import datetime
+import hashlib
+import json
+import sqlite3
+from collections import namedtuple
+from pathlib import Path
+from typing import List, Optional
+
+from lxml import etree, objectify
+
+from materialsdb import cache, config
+from materialsdb.classes import Material
+from materialsdb.serialiser import XmlDeserialiser, get_valid_root
+from materialsdb.summary import MaterialSummary, summarize_material
+
+Report = namedtuple("Report", ["existing", "updated", "deleted", "skipped"])
+
+SCHEMA_VERSION = "1"
+
+_SCHEMA = """
+CREATE TABLE IF NOT EXISTS materials (
+    id TEXT PRIMARY KEY, company_id TEXT, company TEXT, category TEXT,
+    names TEXT, descriptions TEXT,
+    lambda_min REAL, lambda_max REAL, thick_min REAL, thick_max REAL,
+    usage TEXT, source_file TEXT, xml BLOB);
+CREATE INDEX IF NOT EXISTS idx_company ON materials(company);
+CREATE INDEX IF NOT EXISTS idx_category ON materials(category);
+CREATE INDEX IF NOT EXISTS idx_lambda ON materials(lambda_min);
+CREATE TABLE IF NOT EXISTS producer_files (
+    path TEXT PRIMARY KEY, sha256 TEXT, built_at REAL);
+CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT);
+"""
+
+_MATERIAL_COLUMNS = (
+    "id",
+    "company_id",
+    "company",
+    "category",
+    "names",
+    "descriptions",
+    "lambda_min",
+    "lambda_max",
+    "thick_min",
+    "thick_max",
+    "usage",
+    "source_file",
+    "xml",
+)
+_COLUMN_LIST = ", ".join(_MATERIAL_COLUMNS)
+
+_NUMERIC_SORTS = {"lambda": "lambda_min", "thick": "thick_min"}
+_STRING_SORTS = {"company": "company", "category": "category"}
+
+
+def _sha256(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+class MaterialStore:
+    SCHEMA_VERSION = SCHEMA_VERSION
+
+    def __init__(self, db_path: Optional[Path] = None):
+        self.db_path = (
+            Path(db_path) if db_path else cache.get_cache_folder() / "materials.db"
+        )
+        self.db_path.parent.mkdir(parents=True, exist_ok=True)
+        self.connection = sqlite3.connect(str(self.db_path))
+        self.connection.execute("PRAGMA journal_mode=WAL")
+        self.connection.executescript(_SCHEMA)
+        self._ensure_schema_version()
+
+    # ---------- meta / lifecycle ----------
+
+    def _ensure_schema_version(self):
+        row = self.connection.execute(
+            "SELECT value FROM meta WHERE key='schema_version'"
+        ).fetchone()
+        stored = row[0] if row else None
+        if stored != SCHEMA_VERSION:
+            self.connection.execute("DROP TABLE IF EXISTS materials")
+            self.connection.execute("DROP TABLE IF EXISTS producer_files")
+            self.connection.executescript(_SCHEMA)
+            self.connection.execute(
+                "INSERT INTO meta(key, value) VALUES ('schema_version', ?) "
+                "ON CONFLICT(key) DO UPDATE SET value=excluded.value",
+                (SCHEMA_VERSION,),
+            )
+            self.connection.commit()
+
+    def close(self):
+        self.connection.close()
+
+    # ---------- build / refresh ----------
+
+    def refresh(self, force=False, paths=None) -> Report:
+        if paths is None:
+            paths = list(cache.producers())
+        else:
+            paths = [Path(p) for p in paths]
+
+        kept = {str(p) for p in paths}
+        deleted = []
+        for (stored_path,) in self.connection.execute(
+            "SELECT path FROM producer_files"
+        ).fetchall():
+            if stored_path not in kept:
+                self.connection.execute(
+                    "DELETE FROM materials WHERE source_file=?", (stored_path,)
+                )
+                self.connection.execute(
+                    "DELETE FROM producer_files WHERE path=?", (stored_path,)
+                )
+                deleted.append(Path(stored_path))
+
+        existing, updated, skipped = [], [], []
+        deserialiser = XmlDeserialiser()
+        for path in paths:
+            row = self.connection.execute(
+                "SELECT sha256 FROM producer_files WHERE path=?", (str(path),)
+            ).fetchone()
+            try:
+                digest = _sha256(path)
+                if not force and row and row[0] == digest:
+                    existing.append(path)
+                    continue
+                self._upsert_file(deserialiser, path)
+            except Exception as err:
+                print(f"{path.name}: skipped during store refresh:\n\t{err}")
+                skipped.append(path)
+                continue
+            self.connection.execute(
+                "INSERT INTO producer_files(path, sha256, built_at) VALUES (?, ?, ?) "
+                "ON CONFLICT(path) DO UPDATE SET sha256=excluded.sha256, "
+                "built_at=excluded.built_at",
+                (str(path), digest, datetime.datetime.now().timestamp()),
+            )
+            updated.append(path)
+
+        self.connection.commit()
+        return Report(existing, updated, deleted, skipped)
+
+    def _upsert_file(self, deserialiser: XmlDeserialiser, path: Path):
+        tree = objectify.parse(str(path))
+        root = get_valid_root(tree)
+        source = deserialiser.from_element(root)
+        self.connection.execute(
+            "DELETE FROM materials WHERE source_file=?", (str(path),)
+        )
+        for element in root.material:
+            material = deserialiser.from_element(element)
+            summary = summarize_material(
+                material, company_id=str(source.companyid), company=source.company
+            )
+            self.connection.execute(
+                f"INSERT INTO materials ({_COLUMN_LIST}) "
+                "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                (
+                    summary.id,
+                    summary.company_id,
+                    summary.company,
+                    summary.category,
+                    json.dumps(summary.names),
+                    json.dumps(summary.descriptions),
+                    summary.lambda_min,
+                    summary.lambda_max,
+                    summary.thick_min,
+                    summary.thick_max,
+                    json.dumps(summary.usage),
+                    str(path),
+                    sqlite3.Binary(etree.tostring(element)),
+                ),
+            )
+
+    # ---------- queries ----------
+
+    def summaries(
+        self,
+        company=None,
+        category=None,
+        min_lambda=None,
+        max_lambda=None,
+        min_thick=None,
+        max_thick=None,
+        usage=None,
+        text=None,
+        sort="company",
+        ascending=True,
+        lang=None,
+    ) -> List[MaterialSummary]:
+        where, params = [], []
+
+        def add(condition, value):
+            where.append(condition)
+            params.append(value)
+
+        if company:
+            add("company=?", company)
+        if category:
+            add("category=?", category)
+        if min_lambda is not None:
+            add("lambda_max>=?", min_lambda)
+        if max_lambda is not None:
+            add("lambda_min<=?", max_lambda)
+        if min_thick is not None:
+            add("thick_max>=?", min_thick)
+        if max_thick is not None:
+            add("thick_min<=?", max_thick)
+
+        query = f"SELECT {_COLUMN_LIST} FROM materials"
+        if where:
+            query += " WHERE " + " AND ".join(where)
+        rows = self.connection.execute(query, params).fetchall()
+
+        lang = lang or config.get_lang()
+        results = [self._row_to_summary(row) for row in rows]
+
+        if usage:
+            results = [r for r in results if r.usage.get(usage)]
+        if text:
+            needle = text.lower()
+            results = [
+                r
+                for r in results
+                if needle in (r.names.get(lang) or r.names.get("") or "").lower()
+            ]
+        return self._sorted(results, sort, ascending)
+
+    @staticmethod
+    def _sorted(
+        results: List[MaterialSummary], sort: str, ascending: bool
+    ) -> List[MaterialSummary]:
+        reverse = not ascending
+        if sort == "name":
+            return sorted(
+                results,
+                key=lambda r: r.names.get("") or "",
+                reverse=reverse,
+            )
+        if sort in _NUMERIC_SORTS:
+            attr = _NUMERIC_SORTS[sort]
+            return sorted(
+                results,
+                key=lambda r: (
+                    (getattr(r, attr) is None) != reverse,
+                    getattr(r, attr) if getattr(r, attr) is not None else 0,
+                ),
+                reverse=reverse,
+            )
+        attr = _STRING_SORTS.get(sort, "company")
+        return sorted(results, key=lambda r: str(getattr(r, attr)), reverse=reverse)
+
+    @staticmethod
+    def _row_to_summary(row) -> MaterialSummary:
+        (
+            id_,
+            company_id,
+            company,
+            category,
+            names,
+            descriptions,
+            lambda_min,
+            lambda_max,
+            thick_min,
+            thick_max,
+            usage,
+            _source_file,
+            _xml,
+        ) = row
+        return MaterialSummary(
+            id=id_,
+            company_id=company_id,
+            company=company,
+            category=category,
+            names=json.loads(names),
+            descriptions=json.loads(descriptions),
+            lambda_min=lambda_min,
+            lambda_max=lambda_max,
+            thick_min=thick_min,
+            thick_max=thick_max,
+            usage=json.loads(usage),
+        )
+
+    def get(self, material_id: str) -> Optional[Material]:
+        row = self.connection.execute(
+            "SELECT xml FROM materials WHERE id=?", (material_id,)
+        ).fetchone()
+        if row is None:
+            return None
+        element = objectify.fromstring(bytes(row[0]))
+        return XmlDeserialiser().from_element(element)

materialsdb/summary.py

@@ -0,0 +1,75 @@
+from dataclasses import dataclass
+from typing import Dict, Optional, Tuple
+
+from materialsdb import config, utils
+from materialsdb.classes import Material
+
+USAGE_FLAGS = ("wall", "roof", "floor", "door")
+
+
+@dataclass
+class MaterialSummary:
+    id: str
+    company_id: str
+    company: str
+    category: str
+    names: Dict[str, str]
+    descriptions: Dict[str, str]
+    lambda_min: Optional[float]
+    lambda_max: Optional[float]
+    thick_min: Optional[float]
+    thick_max: Optional[float]
+    usage: Dict[str, bool]
+
+
+def _localized_dict(items) -> Dict[str, str]:
+    result: Dict[str, str] = {}
+    for item in items or ():
+        result[str(item.lang or "")] = str(item)
+    return result
+
+
+def _min_max(values) -> Tuple[Optional[float], Optional[float]]:
+    values = [v for v in values if v is not None]
+    if not values:
+        return None, None
+    return min(values), max(values)
+
+
+def summarize_material(
+    material: Material,
+    company_id: str = "",
+    company: str = "",
+    country: Optional[str] = None,
+) -> MaterialSummary:
+    country = country or config.get_country()
+    information = material.information
+
+    lambdas = []
+    thicks = []
+    for layer in utils.get_material_layers(material):
+        thermal = utils.get_by_country(layer.thermal or (), country)
+        geometry = utils.get_by_country(layer.geometry or (), country)
+        if thermal is not None:
+            lambdas.append(thermal.lambda_value)
+        if geometry is not None:
+            thicks.append(geometry.thick)
+
+    lambda_min, lambda_max = _min_max(lambdas)
+    thick_min, thick_max = _min_max(thicks)
+
+    return MaterialSummary(
+        id=str(material.id),
+        company_id=str(company_id),
+        company=str(company),
+        category=str(information.group or ""),
+        names=_localized_dict(getattr(information.names, "name", ())),
+        descriptions=_localized_dict(
+            getattr(getattr(information, "explanations", None), "explanation", ())
+        ),
+        lambda_min=lambda_min,
+        lambda_max=lambda_max,
+        thick_min=thick_min,
+        thick_max=thick_max,
+        usage={flag: str(getattr(information, flag)) == "1" for flag in USAGE_FLAGS},
+    )

{python_materialsdb-0.0.2.dist-info → python_materialsdb-0.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: python-materialsdb
-Version: 0.0.2
+Version: 0.1.0
 Summary: A library to work with materialsdb.org open standard for building materials.
 Home-page: https://github.com/CyrilWaechter/python-materialsdb
 Author: Cyril Waechter
@@ -18,7 +18,10 @@ Description-Content-Type: text/markdown
 License-File: LICENSE.md
 Requires-Dist: lxml
 Provides-Extra: ifc
-Requires-Dist: ifcopenshell ; extra == 'ifc'
+Requires-Dist: ifcopenshell; extra == "ifc"
+Provides-Extra: dev
+Requires-Dist: pytest-benchmark; extra == "dev"
+Dynamic: license-file
 
 python-materialsdb is an unofficial python library for [materialsdb.org][1] an open format and database for building materials.
 
@@ -47,9 +50,21 @@ Note: in materialsdb standard languages are [ISO 639-1](https://en.wikipedia.org
 
 # Usage examples :
 Check out some [examples](examples):
-* [Convert lastest materials data to ifc](examples/generate_ifc_project_libraries.py)
+* [Convert latest materials data to ifc](examples/generate_ifc_project_libraries.py)
 * [Create your own materialsdb.org compliant XML](examples/create_layers.py)
 
+# Querying materials :
+The library keeps an sqlite index of the cached materials data for fast
+filtering and single-material access:
+
+```python
+from materialsdb import query
+
+query.refresh()                                   # incremental update from cached xml
+rows = query.search("isolant", sort="lambda")     # filtered, sorted summaries
+material = query.get_material(rows[0].id)         # full material dataclass
+```
+
 # How to install
 ## Using pip
 ```bash
@@ -57,8 +72,8 @@ pip install python-materialsdb
 ```
 
 # Dependencies
-* [lxml][2] (BSD) : xml parser
-* [ifcopenshell][3] (LGPL) : ifc read/write
+* [lxml][2] (BSD) : xml parser (tested with version 6.1.1)
+* [ifcopenshell][3] (LGPL) : ifc read/write (tested with version 0.8.5)
 
 # Third parties :
 * [materialsdb.org][1] (GPL) : materials schema

python_materialsdb-0.1.0.dist-info/RECORD

@@ -0,0 +1,17 @@
+materialsdb/__init__.py,sha256=2UuaCCmONhxIvpxB1eXHh88eb05i2ilR0-F0o6rKlCQ,63
+materialsdb/cache.py,sha256=ags7_jiAwq5q2nKZZxoA7ybwF5sMVLSKpNe8Ps-iU0o,3386
+materialsdb/classes.py,sha256=obuS_JRfdvqKJ8RMOx65X7AzI2J5W0xNgIXJ2y7evbM,21194
+materialsdb/config.py,sha256=Bz16QTIl8u81g07www_zGdA-Ilsii_kbi3NYuSXlrlQ,1331
+materialsdb/query.py,sha256=xoClMj0Xl7nQ_6DjETTc59gUYYaNH75lDje0BDwrxQQ,656
+materialsdb/serialiser.py,sha256=_QENTdjoX1yeDFMfpUWtrC1-sN-Vusi5QpJL5l-Evck,6677
+materialsdb/store.py,sha256=B5j99dqr5H1_dnTGceowAoLpFRq6CGG5w_fL_o-XGig,9734
+materialsdb/summary.py,sha256=hBL8l27vKF7qgZXtYgA9O3WQRH3Ai-VZ43_PTTe-mjQ,2196
+materialsdb/utils.py,sha256=jwJETjvBuHb6_K_7STsNaM-bE2Oc1X-th1xTrllW-Go,3081
+materialsdb/ifc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+materialsdb/ifc/material_psets.json,sha256=g4Z-wOsnOP-TZR2R_9ikDpa9GalfyA8o3gDHvQtMAgg,15472
+materialsdb/ifc/project_library.py,sha256=yZtI3qRHHDYib61CEVQb0G_X63_Ot_THqqljqUJY3DY,12202
+python_materialsdb-0.1.0.dist-info/licenses/LICENSE.md,sha256=M7wm1EmMGDtwPRdg7kW4d00h1uAXjKOT3HFScYQMeiE,34916
+python_materialsdb-0.1.0.dist-info/METADATA,sha256=t-77t35f4VValTki4NbypXfZzcl-Ccli4aSACONPOO8,3112
+python_materialsdb-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+python_materialsdb-0.1.0.dist-info/top_level.txt,sha256=5ZHbF8Oj1W24bifG65Fl3hCaivZpkO6EyOHHaRavZxE,12
+python_materialsdb-0.1.0.dist-info/RECORD,,

{python_materialsdb-0.0.2.dist-info → python_materialsdb-0.1.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.40.0)
+Generator: setuptools (84.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

python_materialsdb-0.0.2.dist-info/RECORD

@@ -1,14 +0,0 @@
-materialsdb/__init__.py,sha256=2UuaCCmONhxIvpxB1eXHh88eb05i2ilR0-F0o6rKlCQ,63
-materialsdb/cache.py,sha256=4YYaX4cyp-zahhdd5ejppk8_GZ3exwEb6GUeww2wmpQ,3327
-materialsdb/classes.py,sha256=obuS_JRfdvqKJ8RMOx65X7AzI2J5W0xNgIXJ2y7evbM,21194
-materialsdb/config.py,sha256=Bz16QTIl8u81g07www_zGdA-Ilsii_kbi3NYuSXlrlQ,1331
-materialsdb/serialiser.py,sha256=27W9W2ux_00F5QPCUWFVwNcQJnrGy3loCRMXjm2x1Qk,5881
-materialsdb/utils.py,sha256=jwJETjvBuHb6_K_7STsNaM-bE2Oc1X-th1xTrllW-Go,3081
-materialsdb/ifc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-materialsdb/ifc/material_psets.json,sha256=g4Z-wOsnOP-TZR2R_9ikDpa9GalfyA8o3gDHvQtMAgg,15472
-materialsdb/ifc/project_library.py,sha256=3H9LzIIpJEiygtx5ar8BFA-MkJy4n8F-TQ_uXX6PlK4,12190
-python_materialsdb-0.0.2.dist-info/LICENSE.md,sha256=M7wm1EmMGDtwPRdg7kW4d00h1uAXjKOT3HFScYQMeiE,34916
-python_materialsdb-0.0.2.dist-info/METADATA,sha256=MneyvAMo67vbqky_7fwLrFvc4KU91vGArYg7SjxNRo0,2546
-python_materialsdb-0.0.2.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-python_materialsdb-0.0.2.dist-info/top_level.txt,sha256=5ZHbF8Oj1W24bifG65Fl3hCaivZpkO6EyOHHaRavZxE,12
-python_materialsdb-0.0.2.dist-info/RECORD,,