openepd 2.0.0__py3-none-any.whl → 3.1.0__py3-none-any.whl

openepd/api/common.py

@@ -0,0 +1,239 @@
+#
+#  Copyright 2024 by C Change Labs Inc. www.c-change-labs.com
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#  This software was developed with support from the Skanska USA,
+#  Charles Pankow Foundation, Microsoft Sustainability Fund, Interface, MKA Foundation, and others.
+#  Find out more at www.BuildingTransparency.org
+#
+from collections.abc import Iterable
+from contextlib import contextmanager
+from datetime import datetime, timedelta
+import threading
+from time import sleep
+from typing import Callable, Generic, Iterator, cast
+
+from openepd.api.dto.common import DEFAULT_PAGE_SIZE, MetaCollectionDto, OpenEpdApiResponse
+from openepd.api.dto.meta import PagingMeta, PagingMetaMixin
+from openepd.model.base import TOpenEpdObject
+
+
+class Throttler:
+    """Throttle calls to a function to a certain rate."""
+
+    def __init__(self, rate_per_sec: float) -> None:
+        """
+        Construct a throttler.
+
+        :param rate_per_sec: number of calls to throttle per second
+        """
+        super().__init__()
+
+        self.rate: float = rate_per_sec
+
+        self.__count_lock = threading.Lock()
+        self.__time_lock = threading.Lock()
+
+        self.__count = 0
+        self.__start = datetime.now()
+        self.__time_limit = timedelta(seconds=1)
+
+    @contextmanager
+    def throttle(self):
+        """Create context manager which throttles inside the context."""
+        with self.__count_lock:
+            count = self.__count
+
+        with self.__time_lock:
+            diff = datetime.now() - self.__start
+            if diff < self.__time_limit and count >= self.rate:
+                seconds = (self.__time_limit - diff).total_seconds()
+                sleep(seconds)
+                self.__start = datetime.now()
+                self.__count = count - self.rate
+
+        with self.__count_lock:
+            self.__count += 1
+
+        yield
+
+
+class StreamingListResponse(Iterable[TOpenEpdObject], Generic[TOpenEpdObject]):
+    """
+    Iterator over a list of objects which could be from remote API in batches by given fetch function.
+
+    This class could work only with OpenEpdApiResponse which:
+    a) contains a list of objects in payload
+    b) contains PagingMeta in meta
+
+    Typical use case:
+
+        stream: StreamingListResponse[Epd] # Assume we collected this from somewhere (e.g. from API Client)
+
+        # Iterate over all EPDs (this will fetch the next page when needed, one single page is buffered)
+        for epd in stream:
+            print(epd)
+            print("Current page: ", stream.current_page)
+
+        # Get specific page
+        page = stream.goto_page(4)
+
+        # Get paging information
+        stream.get_paging_meta()
+        # or
+        stream.get_get_total_pages(), stream.get_total_items()
+
+        # Iterate starting from specific page
+        for epd in stream.iterator(4):
+            print(epd)
+        # Alternative approach
+        stream.goto_page(4)
+        for epd in stream:
+            print(epd)
+    """
+
+    def __init__(
+        self,
+        fetch_handler: Callable[[int, int], OpenEpdApiResponse[list[TOpenEpdObject], MetaCollectionDto]],
+        auto_init: bool = True,
+        page_size: int | None = None,
+    ):
+        self.__fetch_handler = fetch_handler
+        self.__page_size = page_size or DEFAULT_PAGE_SIZE
+        self.__current_page = 0
+        self.__recent_response: OpenEpdApiResponse[list[TOpenEpdObject], MetaCollectionDto] | None = None
+        if auto_init:
+            self.goto_page(1)
+
+    def goto_page(self, page_num: int, force_reload: bool = False) -> list[TOpenEpdObject]:
+        """
+        Go to the given page and return items as result.
+
+        This will query remote server to retrieve given page, the state of iterator will be updated accordingly.
+        All information from the internal buffer will be overridden.
+
+        :param page_num: page number to retrieve
+        :param force_reload: if True, force reload of the page even if it was already loaded
+        :return: list of items on the page
+        """
+        if page_num <= 0:
+            raise ValueError("Page number must be positive")
+        if self.__current_page != page_num or force_reload:
+            self.__recent_response = self.__fetch_handler(page_num, self.__page_size)
+            self.__current_page = page_num
+        if self.__recent_response is None:
+            raise RuntimeError("Response is empty, this should not happen, check if fetch_handler is compatible")
+        if self.__recent_response.payload is None:
+            raise ValueError("Response does not contain payload")
+        if not isinstance(self.__recent_response.payload, list):
+            raise ValueError("Response does not contain a list")
+        if self.__recent_response.meta is None:
+            raise ValueError("Response does not contain meta")
+        if not isinstance(self.__recent_response.meta, PagingMetaMixin):
+            raise ValueError("Response does not contain paging meta")
+        return self.__recent_response.payload
+
+    def get_paging_meta(self) -> PagingMeta:
+        """
+        Get paging meta from the most recent response.
+
+        If object is not initialized (no pages were retrieved yet), the first page will be fetched automatically.
+        See also: get_meta()
+        :return: paging meta
+        """
+        paging_meta = cast(PagingMetaMixin, self.get_meta()).paging
+        if paging_meta is None:
+            raise ValueError("Response does not contain paging meta")
+        return paging_meta
+
+    def get_meta(self) -> MetaCollectionDto:
+        """
+        Get meta from the most recent response.
+
+        If object is not initialized (no pages were retrieved yet), the first page will be fetched automatically.
+        :return: paging meta
+        """
+        self._ensure_initialized()
+        return self.__recent_response.meta  # type: ignore
+
+    @property
+    def current_page(self) -> int:
+        """Get current page number (numbering is 1-based)."""
+        return self.__current_page
+
+    def get_total_pages(self) -> int:
+        """Get total number of pages."""
+        return self.get_paging_meta().total_pages
+
+    def get_total_count(self) -> int:
+        """Get total number of items."""
+        return self.get_paging_meta().total_count
+
+    def has_next_page(self) -> bool:
+        """Check if there is a next page."""
+        return self.current_page < self.get_total_pages()
+
+    def reset(self):
+        """Reset iterator to the very beginning, cleanup internal buffers."""
+        self.__current_page = 0
+        self.__recent_response = None
+
+    def __iter__(self) -> Iterator[TOpenEpdObject]:
+        """
+        Iterate over all items, when needed the new pages from server will be requested.
+
+        Iteration from starts from the current page.
+        """
+        return self.iterator(self.current_page)
+
+    def iterator(self, start_from_page: int = 1) -> Iterator[TOpenEpdObject]:
+        """
+        Iterate over all items, when needed the new pages from server will be requested.
+
+        :param start_from_page: page number to start from (1-based)
+        """
+        if start_from_page <= 0:
+            start_from_page = 1
+        self.goto_page(start_from_page)
+        while True:
+            items = self.goto_page(self.current_page)
+            for x in items:
+                yield x
+            if not self.has_next_page():
+                return  # no more pages
+            else:
+                self.goto_page(self.current_page + 1)
+
+    def __len__(self):
+        return self.get_total_count()
+
+    def _is_initialized(self) -> bool:
+        """Check if object is initialized (at least one page was retrieved)."""
+        return self.__recent_response is not None
+
+    def _ensure_initialized(self):
+        """Ensure that object is initialized, if not - request the first page."""
+        if not self._is_initialized():
+            self.goto_page(1)
+
+
+def no_trailing_slash(val: str) -> str:
+    """
+    Remove all trailing slashes from the given string. Might be useful to normalize URLs.
+
+    Non-string input is considered as error.
+    """
+    while val.endswith("/"):
+        val = val[:-1]
+    return val

openepd/api/dto/__init__.py

@@ -0,0 +1,19 @@
+#
+#  Copyright 2024 by C Change Labs Inc. www.c-change-labs.com
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#  This software was developed with support from the Skanska USA,
+#  Charles Pankow Foundation, Microsoft Sustainability Fund, Interface, MKA Foundation, and others.
+#  Find out more at www.BuildingTransparency.org
+#

openepd/api/dto/base.py

@@ -0,0 +1,41 @@
+#
+#  Copyright 2024 by C Change Labs Inc. www.c-change-labs.com
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#  This software was developed with support from the Skanska USA,
+#  Charles Pankow Foundation, Microsoft Sustainability Fund, Interface, MKA Foundation, and others.
+#  Find out more at www.BuildingTransparency.org
+#
+import abc
+
+from openepd.compat.pydantic import pyd
+
+
+class BaseOpenEpdApiModel(pyd.BaseModel):
+    """Base class for OpenEPD API DTOs."""
+
+    class Config:
+        extra = pyd.Extra.ignore
+
+
+class BaseMetaDto(BaseOpenEpdApiModel, metaclass=abc.ABCMeta):
+    """Base class for all meta DTOs."""
+
+    pass
+
+
+class MetaExtensionBase(BaseOpenEpdApiModel):
+    """Base class for custom meta extensions."""
+
+    pass

openepd/api/dto/common.py

@@ -0,0 +1,115 @@
+#
+#  Copyright 2024 by C Change Labs Inc. www.c-change-labs.com
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#  This software was developed with support from the Skanska USA,
+#  Charles Pankow Foundation, Microsoft Sustainability Fund, Interface, MKA Foundation, and others.
+#  Find out more at www.BuildingTransparency.org
+#
+import abc
+import datetime
+from typing import Final, Generic, TypeAlias, TypeVar
+
+from openepd.api.dto.base import BaseMetaDto, BaseOpenEpdApiModel, MetaExtensionBase
+from openepd.api.dto.meta import PerformanceMetaMixin
+from openepd.compat.pydantic import pyd, pyd_generics
+from openepd.model.base import AnySerializable, BaseOpenEpdSchema
+
+DEFAULT_PAGE_SIZE: Final[int] = 100
+MAX_PAGE_SIZE: Final[int] = 250
+
+
+class AuditableDto(BaseOpenEpdApiModel, metaclass=abc.ABCMeta):
+    """Base class for all DTOs that hold audit information."""
+
+    created_by: str = pyd.Field(
+        title="Created By",
+        example="johnsmith@cqd.io",
+        description="User's email or script name that created this list.",
+    )
+    updated_by: str = pyd.Field(
+        title="Updated By",
+        example="bobbuilder@buildingtransparency.org",
+        description="User's email or script name that updated this list last time.",
+    )
+    created_on: datetime.datetime = pyd.Field(
+        title="Created On",
+        example="2019-06-13T13:17:09+00:00",
+        description="A timestamp when this object has been created "
+        "in [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601) format.",
+    )
+    updated_on: datetime.datetime = pyd.Field(
+        title="Updated On",
+        example="2020-07-13T13:17:09+00:00",
+        description="A timestamp when this object has been updated last time "
+        "in [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601) format.",
+    )
+
+
+Payload: TypeAlias = AnySerializable
+TPayload = TypeVar("TPayload", bound=Payload)
+TMetaDto = TypeVar("TMetaDto", bound=BaseMetaDto)
+TMetaExtension = TypeVar("TMetaExtension", bound=MetaExtensionBase)
+
+
+class MetaCollectionDto(BaseOpenEpdApiModel, pyd_generics.GenericModel, Generic[TMetaExtension]):
+    """
+    This class is intended to be used as a container for different meta objects.
+
+    From a specific controller, you should return a specific subclass of MetaCollectionDto and appropriate mixins. This
+    would allow to retain type information to generate schema for meta section of response.
+
+    For example, EPD Search method, which should return some metadata about performance, material filter, and paging,
+    would return a concrete subclass of MetaCollectionDto like this one:
+
+    `class EpdSearchViewMeta(MaterialFilterMetaMixin, PagingMetaMixin, WarningMetaMixin, MetaCollectionDto)`
+
+    Mixins are used to define the resulting meta's dictionary-like structure, this would result in a json like:
+    ```
+    {
+    "paging": {"page_size": 100, "total_count": 1, "total_pages": 1},
+    "ec3_warnings": null,
+    "material_filter": {...}
+    }
+    ```
+
+    It is generified by the extension type of similar structure (key-dict value), which should not be a part of OpenEPD
+    spec. This allows to add custom meta objects to the response, which would be ignored by OpenEPD implementors.
+    """
+
+    ext: TMetaExtension | None = None
+
+    class Config(BaseOpenEpdSchema.Config):
+        schema_extra = {
+            "description": "Base structure of the response meta section",
+        }
+
+
+class BaseMeta(PerformanceMetaMixin, MetaCollectionDto[TMetaExtension], Generic[TMetaExtension], metaclass=abc.ABCMeta):
+    """Base class for creating meta objects specific to a controller."""
+
+    pass
+
+
+TMeta = TypeVar("TMeta", bound=MetaCollectionDto, covariant=True)
+
+
+class OpenEpdApiResponse(pyd_generics.GenericModel, BaseOpenEpdApiModel, Generic[TPayload, TMeta]):
+    """Standard DTO representing response from OpenEPD API server."""
+
+    payload: TPayload
+    # there is an issue with using covariant type variables as parameters when used for container mutable types,
+    # as described in https://github.com/python/mypy/issues/7049
+    # However, in our workflow TMeta is not a container type which is further returned back to caller.
+    meta: TMeta  # type: ignore[misc]

openepd/api/dto/meta.py

@@ -0,0 +1,69 @@
+#
+#  Copyright 2024 by C Change Labs Inc. www.c-change-labs.com
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#  This software was developed with support from the Skanska USA,
+#  Charles Pankow Foundation, Microsoft Sustainability Fund, Interface, MKA Foundation, and others.
+#  Find out more at www.BuildingTransparency.org
+#
+from openepd.api.dto.base import BaseMetaDto, BaseOpenEpdApiModel
+from openepd.compat.pydantic import pyd
+
+
+class PerformanceMeta(BaseMetaDto):
+    """Meta for performance information."""
+
+    execution_time_ms: int
+
+
+class PerformanceMetaMixin(BaseMetaDto):
+    """Mixin for adding performance meta to MetaCollection."""
+
+    performance: PerformanceMeta | None = None
+
+
+class PagingMeta(BaseMetaDto):
+    """Meta for paging information."""
+
+    total_count: int = pyd.Field(
+        title="Total results", example=1233, description="Total number of records for the search"
+    )
+    total_pages: int = pyd.Field(title="Total pages", example=20, description="Total pages available")
+    page_size: int = pyd.Field(title="Page size", example=150, description="Number of records in page")
+
+
+class PagingMetaMixin(BaseOpenEpdApiModel):
+    """Mixin for adding paging meta to MetaCollection."""
+
+    paging: PagingMeta | None
+
+
+class WarningMessageDto(BaseOpenEpdApiModel):
+    """DTO for warning messages."""
+
+    message: str = pyd.Field(
+        title="Warning message", example="Categories limited during search, see effective_omf in meta"
+    )
+    code: str = pyd.Field(title="Warning code", example="CATEGORIES_LIMITED")
+    field: str | None = pyd.Field(
+        title="Field",
+        description="Field to which the warning relates",
+        example="subcategories",
+    )
+
+
+class WarningMetaMixin(BaseOpenEpdApiModel):
+    """Mixin for adding warnings meta to MetaCollection."""
+
+    warnings: list[WarningMessageDto] | None = None

openepd/api/dto/mf.py

@@ -0,0 +1,59 @@
+#
+#  Copyright 2024 by C Change Labs Inc. www.c-change-labs.com
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#  This software was developed with support from the Skanska USA,
+#  Charles Pankow Foundation, Microsoft Sustainability Fund, Interface, MKA Foundation, and others.
+#  Find out more at www.BuildingTransparency.org
+#
+from openepd.api.dto.base import BaseOpenEpdApiModel
+from openepd.compat.pydantic import pyd
+
+
+class MaterialFilterDefinition(BaseOpenEpdApiModel):
+    """
+    Material filter definition.
+
+    This object includes material filter itself as well as its hash.
+    """
+
+    mf: str = pyd.Field(
+        title="MaterialFilter",
+        example='!EC3 search("AluminiumBillets") !pragma oMF("1.0/1")',
+        description="MaterialFilter in string format",
+    )
+    mf_hash: str = pyd.Field(
+        title="MaterialFilter hash",
+        example="22bf5b78cee5b79e1c76e818873d521c3972688b",
+        description="MaterialFilter hash. Can be used to compare filters for equality, put to cache etc.",
+    )
+
+
+class MaterialFilterMeta(BaseOpenEpdApiModel):
+    """Meta holding supplementary information about OMF query execution."""
+
+    excluded_fields: list[str] | None = pyd.Field(
+        example=["building_jurisdiction", "jurisdiction"],
+        description="list of fields excluded by server process for any reason",
+        default=None,
+    )
+    effective_omf: MaterialFilterDefinition = pyd.Field(
+        description="Effective OpenMaterialFilter as applied to search, after transformations if any"
+    )
+
+
+class MaterialFilterMetaMixin(BaseOpenEpdApiModel):
+    """Mixing for adding MaterialFilterMeta to MetaCollection."""
+
+    material_filter: MaterialFilterMeta | None

openepd/api/dto/params.py

@@ -0,0 +1,19 @@
+#
+#  Copyright 2024 by C Change Labs Inc. www.c-change-labs.com
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#  This software was developed with support from the Skanska USA,
+#  Charles Pankow Foundation, Microsoft Sustainability Fund, Interface, MKA Foundation, and others.
+#  Find out more at www.BuildingTransparency.org
+#

openepd/api/epd/__init__.py

@@ -0,0 +1,19 @@
+#
+#  Copyright 2024 by C Change Labs Inc. www.c-change-labs.com
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#  This software was developed with support from the Skanska USA,
+#  Charles Pankow Foundation, Microsoft Sustainability Fund, Interface, MKA Foundation, and others.
+#  Find out more at www.BuildingTransparency.org
+#