zepben.ewb 1.0.0__py3-none-any.whl

zepben/ewb/database/sqlite/common/base_cim_writer.py

@@ -0,0 +1,159 @@
+#  Copyright 2024 Zeppelin Bend Pty Ltd
+#  This Source Code Form is subject to the terms of the Mozilla Public
+#  License, v. 2.0. If a copy of the MPL was not distributed with this
+#  file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+__all__ = ["BaseCimWriter"]
+
+from abc import ABC
+from typing import Optional
+
+from zepben.ewb.database.sqlite.common.base_database_tables import BaseDatabaseTables
+from zepben.ewb.database.sqlite.common.base_entry_writer import BaseEntryWriter
+from zepben.ewb.database.sqlite.extensions.prepared_statement import PreparedStatement
+from zepben.ewb.database.sqlite.tables.iec61968.common.table_documents import TableDocuments
+from zepben.ewb.database.sqlite.tables.iec61968.common.table_organisation_roles import TableOrganisationRoles
+from zepben.ewb.database.sqlite.tables.iec61968.common.table_organisations import TableOrganisations
+from zepben.ewb.database.sqlite.tables.iec61970.base.core.table_identified_objects import TableIdentifiedObjects
+from zepben.ewb.database.sqlite.tables.iec61970.base.core.table_name_types import TableNameTypes
+from zepben.ewb.database.sqlite.tables.iec61970.base.core.table_names import TableNames
+from zepben.ewb.model.cim.iec61968.common.document import Document
+from zepben.ewb.model.cim.iec61968.common.organisation import Organisation
+from zepben.ewb.model.cim.iec61968.common.organisation_role import OrganisationRole
+from zepben.ewb.model.cim.iec61970.base.core.identified_object import IdentifiedObject
+from zepben.ewb.model.cim.iec61970.base.core.name import Name
+from zepben.ewb.model.cim.iec61970.base.core.name_type import NameType
+
+
+class BaseCimWriter(BaseEntryWriter, ABC):
+    """
+    A base class for writing CIM objects to a database.
+    """
+
+    def __init__(self, database_tables: BaseDatabaseTables):
+        super().__init__()
+        self._database_tables = database_tables
+
+    ###################
+    # IEC61968 Common #
+    ###################
+
+    def _save_document(self, table: TableDocuments, insert: PreparedStatement, document: Document, description: str) -> bool:
+        """
+        Save the `Document` fields to `TableDocuments`.
+
+        :param table: The database table to write the `Document` fields to.
+        :param insert: The `PreparedStatement` to bind the field values to.
+        :param document: The `Document` instance to write to the database.
+        :param description: A readable version of the type of object being written for logging purposes.
+
+        :return: True if the `Document` was successfully written to the database, otherwise false.
+        :raises SQLException: For any errors encountered writing to the database.
+        """
+        insert.add_value(table.title.query_index, document.title)
+        # The timestamp in the database uses Z for UTC while python uses +HH:mm format, so convert between them.
+        insert.add_value(table.created_date_time.query_index, f"{document.created_date_time.isoformat()}Z" if document.created_date_time else None)
+        insert.add_value(table.author_name.query_index, document.author_name)
+        insert.add_value(table.type.query_index, document.type)
+        insert.add_value(table.status.query_index, document.status)
+        insert.add_value(table.comment.query_index, document.comment)
+
+        return self._save_identified_object(table, insert, document, description)
+
+    def save_organisation(self, organisation: Organisation) -> bool:
+        """
+        Save the `Organisation` fields to `TableOrganisations`.
+
+        :param organisation: The `Organisation` instance to write to the database.
+
+        :return: True if the `Organisation` was successfully written to the database, otherwise false.
+        :raises SQLException: For any errors encountered writing to the database.
+        """
+        table = self._database_tables.get_table(TableOrganisations)
+        insert = self._database_tables.get_insert(TableOrganisations)
+
+        return self._save_identified_object(table, insert, organisation, "organisation")
+
+    def _save_organisation_role(self, table: TableOrganisationRoles, insert: PreparedStatement, organisation_role: OrganisationRole, description: str) -> bool:
+        """
+        Save the `OrganisationRole` fields to `TableOrganisationRoles`.
+
+        :param table: The database table to write the `OrganisationRole` fields to.
+        :param insert: The `PreparedStatement` to bind the field values to.
+        :param organisation_role: The `OrganisationRole` instance to write to the database.
+        :param description: A readable version of the type of object being written for logging purposes.
+
+        :return: True if the `OrganisationRole` was successfully written to the database, otherwise false.
+        :raises SQLException: For any errors encountered writing to the database.
+        """
+        insert.add_value(table.organisation_mrid.query_index, self._mrid_or_none(organisation_role.organisation))
+
+        return self._save_identified_object(table, insert, organisation_role, description)
+
+    ######################
+    # IEC61970 Base Core #
+    ######################
+
+    def _save_identified_object(
+        self,
+        table: TableIdentifiedObjects,
+        insert: PreparedStatement,
+        identified_object: IdentifiedObject,
+        description: str
+    ) -> bool:
+        """
+        Save the `IdentifiedObject` fields to `TableIdentifiedObjects`.
+
+        :param table: The database table to write the `IdentifiedObject` fields to.
+        :param insert: The `PreparedStatement` to bind the field values to.
+        :param identified_object: The `IdentifiedObject` instance to write to the database.
+        :param description: A readable version of the type of object being written for logging purposes.
+
+        :return: True if the `IdentifiedObject` was successfully written to the database, otherwise false.
+        :raises SQLException: For any errors encountered writing to the database.
+        """
+        insert.add_value(table.mrid.query_index, identified_object.mrid)
+        insert.add_value(table.name_.query_index, identified_object.name)
+        insert.add_value(table.description.query_index, identified_object.description)
+        insert.add_value(table.num_diagram_objects.query_index, 0)  # Currently unused
+
+        return self._try_execute_single_update(insert, description)
+
+    def save_name_type(self, name_type: NameType) -> bool:
+        """
+        Save the `NameType` fields to `TableNameTypes`.
+
+        :param name_type: The `NameType` instance to write to the database.
+
+        :return: True if the `NameType` was successfully written to the database, otherwise false.
+        :raises SQLException: For any errors encountered writing to the database.
+        """
+        table = self._database_tables.get_table(TableNameTypes)
+        insert = self._database_tables.get_insert(TableNameTypes)
+
+        insert.add_value(table.name_.query_index, name_type.name)
+        insert.add_value(table.description.query_index, name_type.description)
+
+        return self._try_execute_single_update(insert, "name type")
+
+    def save_name(self, name: Name) -> bool:
+        """
+        Save the `Name` fields to `TableNames`.
+
+        :param name: The `Name` instance to write to the database.
+
+        :return: True if the `Name` was successfully written to the database, otherwise false.
+        :raises SQLException: For any errors encountered writing to the database.
+        """
+        table = self._database_tables.get_table(TableNames)
+        insert = self._database_tables.get_insert(TableNames)
+
+        insert.add_value(table.name_.query_index, name.name)
+        insert.add_value(table.name_type_name.query_index, name.type.name)
+        insert.add_value(table.identified_object_mrid.query_index, name.identified_object.mrid)
+
+        return self._try_execute_single_update(insert, "name")
+
+    @staticmethod
+    def _mrid_or_none(io: Optional[IdentifiedObject]) -> Optional[str]:
+        return io.mrid if io is not None else None

zepben/ewb/database/sqlite/common/base_collection_reader.py

@@ -0,0 +1,96 @@
+#  Copyright 2024 Zeppelin Bend Pty Ltd
+#  This Source Code Form is subject to the terms of the Mozilla Public
+#  License, v. 2.0. If a copy of the MPL was not distributed with this
+#  file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+__all__ = ["BaseCollectionReader"]
+
+import logging
+from abc import ABC, abstractmethod
+from contextlib import closing
+from sqlite3 import Connection
+from typing import Callable, Type, Optional
+
+from zepben.ewb.database.sqlite.common.base_database_tables import BaseDatabaseTables, TSqliteTable
+from zepben.ewb.database.sqlite.common.reader_exceptions import MRIDLookupException, DuplicateMRIDException
+from zepben.ewb.database.sqlite.extensions.prepared_statement import SqlException
+from zepben.ewb.database.sqlite.extensions.result_set import ResultSet
+
+
+class BaseCollectionReader(ABC):
+    """
+    A base class for reading collections of object collections from a database.
+    """
+
+    def __init__(self, tables: BaseDatabaseTables, connection: Connection):
+        super().__init__()
+        self._logger: logging.Logger = logging.getLogger(self.__class__.__name__)
+        """
+        The logger to use for this collection reader.
+        """
+
+        self.base_database_tables: BaseDatabaseTables = tables
+        """
+        The tables that are available in the database.
+        """
+
+        self._connection: Connection = connection
+        """
+        The connection to the database to read.
+        """
+
+    @abstractmethod
+    def load(self) -> bool:
+        """
+        Load all the objects for the available collections.
+
+        :return: True if the load was successful, otherwise False.
+        """
+        pass
+
+    def _load_each(self, type_: Type[TSqliteTable], process_row: Callable[[TSqliteTable, ResultSet, Callable[[str], str]], bool]) -> bool:
+        """
+        Load each row of a table.
+
+        :param type_: The `SqliteTable` to read the objects from.
+        :param process_row: A callback for processing each row in the table. The callback will be provided with the table, the results for the row and a
+          callback to set the identifier for the row, which returns the same value, so it can be used fluently.
+        """
+        table = self.base_database_tables.get_table(type_)
+
+        def process_rows(results: ResultSet):
+            last_identifier: Optional[str] = None
+
+            def set_identifier(identifier: str) -> str:
+                nonlocal last_identifier
+                last_identifier = identifier
+                return identifier
+
+            try:
+                count = 0
+                while results.next():
+                    if process_row(table, results, set_identifier):
+                        count = count + 1
+
+                return count
+            except SqlException as e:
+                self._logger.error(f"Failed to load '{last_identifier}' from '{table.name}': {e}")
+                raise e
+
+        return self._load_all(table, process_rows)
+
+    def _load_all(self, table: TSqliteTable, process_rows: Callable[[ResultSet], int]) -> bool:
+        """
+        You really shouldn't need to use this function directly, use `load_each` instead.
+        """
+        self._logger.info(f"Loading {table.describe()}...")
+
+        try:
+            with closing(self._connection.cursor()) as cur:
+                cur.execute(table.select_sql)
+                count = process_rows(ResultSet(cur.fetchall()))
+            self._logger.info(f"Successfully loaded {count} {table.describe()}.")
+            return True
+        except (SqlException, ValueError, MRIDLookupException, DuplicateMRIDException) as ex:
+            self._logger.exception(f"Failed to read the {table.describe()} from '{table.name}': {ex}")
+            return False

zepben/ewb/database/sqlite/common/base_collection_writer.py

@@ -0,0 +1,73 @@
+#  Copyright 2024 Zeppelin Bend Pty Ltd
+#  This Source Code Form is subject to the terms of the Mozilla Public
+#  License, v. 2.0. If a copy of the MPL was not distributed with this
+#  file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+__all__ = ["BaseCollectionWriter"]
+
+import logging
+from abc import ABC, abstractmethod
+from typing import Callable, TypeVar, Iterable
+
+from zepben.ewb.database.sqlite.extensions.prepared_statement import SqlException
+
+T = TypeVar("T")
+
+
+class BaseCollectionWriter(ABC):
+    """
+    A base class for writing collections of object collections to a database.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self._logger: logging.Logger = logging.getLogger(self.__class__.__name__)
+        """
+        The logger to use for this collection writer.
+        """
+
+    @abstractmethod
+    def save(self) -> bool:
+        """
+        Save all the objects for the available collections.
+
+        :return: True if the save was successful, otherwise False.
+        """
+        pass
+
+    def _save_each(self, items: Iterable[T], saver: Callable[[T], bool], on_save_failure: Callable[[T, Exception], None]) -> bool:
+        """
+        Save each of the [items] to the database.
+
+        NOTE: This function does not short circuit and all items will be attempted, even if one fails.
+
+        :param items: The collection of items to save.
+        :param saver: A callback for saving each object to the database.
+        :param on_save_failure: A callback that should be used to indicate there was a failure in the `saver`. You should pass the object that was being saved,
+          and the exception that caused the failure.
+
+        :return: True if all `items` were successfully saved, otherwise False.
+        """
+        status = True
+
+        for it in items:
+            status = self._validate_save(it, saver, lambda e: on_save_failure(it, e)) and status
+
+        return status
+
+    @staticmethod
+    def _validate_save(it: T, saver: [[T], bool], on_save_failure: [[Exception], None]) -> bool:
+        """
+        Validate that a save actually works, and convert all exceptions into failures with a callback.
+
+        :param it: The object instance to save.
+        :param saver: The callback that will save the object to the database.
+        :param on_save_failure: A callback if an exception was thrown by the [saver].
+
+        :return: True if the saver successfully saved the object to the database, otherwise False.
+        """
+        try:
+            return saver(it)
+        except SqlException as e:
+            on_save_failure(e)
+            return False

zepben/ewb/database/sqlite/common/base_database_reader.py

@@ -0,0 +1,127 @@
+#  Copyright 2024 Zeppelin Bend Pty Ltd
+#  This Source Code Form is subject to the terms of the Mozilla Public
+#  License, v. 2.0. If a copy of the MPL was not distributed with this
+#  file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+__all__ = ["BaseDatabaseReader"]
+
+import logging
+from abc import ABC
+from contextlib import closing
+from sqlite3 import Connection
+from typing import Optional
+
+from zepben.ewb.database.sqlite.common.base_service_reader import BaseServiceReader
+from zepben.ewb.database.sqlite.common.metadata_collection_reader import MetadataCollectionReader
+from zepben.ewb.database.sqlite.tables.table_version import TableVersion
+from zepben.ewb.services.common.base_service import BaseService
+
+
+class BaseDatabaseReader(ABC):
+    """
+    A base class for reading objects from one of our databases.
+    """
+
+    def __init__(
+        self,
+        connection: Connection,
+        metadata_reader: MetadataCollectionReader,
+        service_reader: BaseServiceReader,
+        service: BaseService,
+        database_description: str,
+        table_version: TableVersion
+    ):
+        super().__init__()
+        self._logger: logging.Logger = logging.getLogger(self.__class__.__name__)
+        """
+        The logger to use for this reader.
+        """
+
+        self._connection: Connection = connection
+        """
+        The connection to the database.
+        """
+
+        self._metadata_reader: MetadataCollectionReader = metadata_reader
+        """
+        The reader for the [MetadataCollection] included in the database.
+        """
+
+        self._service_reader: BaseServiceReader = service_reader
+        """
+        The reader for the [BaseService] supported by the database.
+        """
+
+        self._service: BaseService = service
+        """
+        The [BaseService] that will be populated by the [BaseServiceReader]. Used for post-processing.
+        """
+
+        self._database_description: str = database_description
+        """
+        The description of the database for logging (e.g. filename).
+        """
+
+        self._table_version: TableVersion = table_version
+        """
+        The version table object for checking the database version number.
+        """
+
+        self._has_been_used: bool = False
+        self._supported_version: int = table_version.SUPPORTED_VERSION
+
+    async def _post_load(self) -> bool:
+        """
+        Customisable function for performing actions after the database has been loaded.
+        """
+        self._logger.info("Ensuring all references resolved...")
+        for it in self._service.unresolved_references():
+            raise ValueError(
+                f"Unresolved references found in {self._service.name} service after load - this should not occur. Failing reference was from " +
+                f"{it.from_ref} resolving {it.resolver.to_class.__name__} {it.to_mrid}"
+            )
+        self._logger.info("Unresolved references were all resolved during load.")
+
+        return True
+
+    async def load(self) -> bool:
+        """
+        Load the database.
+        """
+        try:
+            if self._has_been_used:
+                raise ValueError("You can only use the database reader once.")
+            self._has_been_used = True
+
+            return self._pre_load() and self._load_from_readers() and await self._post_load()
+        except Exception as e:
+            self._logger.exception(f"Unable to load database: {e}")
+            return False
+
+    def _pre_load(self) -> bool:
+        try:
+            with closing(self._connection.cursor()) as cur:
+                version = self._table_version.get_version(cur)
+                if version == self._supported_version:
+                    self._logger.info(f"Loading from database version v{version}")
+                    return True
+                else:
+                    self._logger.error(self._format_version_error(version))
+                    return False
+        except Exception as e:
+            self._logger.exception(f"Failed to connect to the database for reading: {e}")
+            return False
+
+    def _load_from_readers(self):
+        return self._metadata_reader.load() and self._service_reader.load()
+
+    def _format_version_error(self, version: Optional[int]) -> str:
+        if version is None:
+            return "Failed to read the version number form the selected database. Are you sure it is a EWB database?"
+        elif version < self._supported_version:
+            return self._unexpected_version(version, "Consider using the UpgradeRunner if you wish to support this database.")
+        else:
+            return self._unexpected_version(version, "You need to use a newer version of the SDK to load this database.")
+
+    def _unexpected_version(self, version: Optional[int], action: str):
+        return f"Unable to load from database {self._database_description} [found v{version}, expected v{self._supported_version}]. {action}"

zepben/ewb/database/sqlite/common/base_database_tables.py

@@ -0,0 +1,137 @@
+#  Copyright 2024 Zeppelin Bend Pty Ltd
+#  This Source Code Form is subject to the terms of the Mozilla Public
+#  License, v. 2.0. If a copy of the MPL was not distributed with this
+#  file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+__all__ = ["TSqliteTable", "BaseDatabaseTables"]
+
+from abc import ABC
+from sqlite3 import Connection, Cursor, ProgrammingError
+from typing import Dict, TypeVar, Type, Generator, Callable, Optional
+
+from zepben.ewb.database.sqlite.extensions.prepared_statement import PreparedStatement
+from zepben.ewb.database.sqlite.tables.exceptions import MissingTableConfigException
+from zepben.ewb.database.sqlite.tables.iec61970.base.core.table_name_types import *
+from zepben.ewb.database.sqlite.tables.iec61970.base.core.table_names import *
+from zepben.ewb.database.sqlite.tables.sqlite_table import *
+from zepben.ewb.database.sqlite.tables.table_metadata_data_sources import TableMetadataDataSources
+from zepben.ewb.database.sqlite.tables.table_version import TableVersion
+
+TSqliteTable = TypeVar("TSqliteTable", bound=SqliteTable)
+
+
+class BaseDatabaseTables(ABC):
+    """
+    The base collection of tables for all our databases.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self._tables: Optional[Dict[Type[TSqliteTable], SqliteTable]] = None
+        self._insert_statements: Optional[Dict[Type[TSqliteTable], PreparedStatement]] = None
+        self._insert_cursor: Optional[Cursor] = None
+
+    @property
+    def tables(self) -> Generator[SqliteTable, None, None]:
+        """
+        The tables that are available in this database, keyed on the table class. You should use `get_table` to access individual tables.
+        """
+        self._ensure_tables()
+        for t in self._tables.values():
+            yield t
+
+    @property
+    def insert_statements(self) -> Generator[PreparedStatement, None, None]:
+        """
+        A collection of `PreparedStatement` for each table. You should use `get_insert` to access individual inserts.
+        """
+        if self._insert_statements is not None:
+            for s in self._insert_statements.values():
+                yield s
+
+    @property
+    def _included_tables(self) -> Generator[SqliteTable, None, None]:
+        """
+        A sequence of `SqliteTable` indicating which tables are included in this database, which will be consumed to build the `tables` collection.
+
+        NOTE: You should always append your tables to `super()._included_tables` when overriding.
+        """
+        yield TableMetadataDataSources()
+        yield TableVersion()
+        yield TableNameTypes()
+        yield TableNames()
+
+    def get_table(self, type_: Type[TSqliteTable]) -> TSqliteTable:
+        """
+        Helper function for getting the table of the specified type.
+
+        :param type_: The type of table to get.
+        :return: The requested table if it belongs to this database.
+        :raises MissingTableConfigException: If the requested table doesn't belong to this database.
+        """
+        self._ensure_tables()
+        try:
+            return self._tables[type_]
+        except KeyError:
+            raise MissingTableConfigException(f"INTERNAL ERROR: No table has been registered for {type_}. You might want to consider fixing that.")
+
+    def get_insert(self, type_: Type[TSqliteTable]) -> PreparedStatement:
+        """
+        Helper function for getting the insert statement for the specified table.
+
+        :param type_: The type of table to get.
+        :return: The insert statement for the requested table if it belongs to this database.
+        :raises MissingTableConfigException: If the requested table doesn't belong to this database.
+        """
+        try:
+            return self._insert_statements[type_]
+        except KeyError:
+            raise MissingTableConfigException(f"INTERNAL ERROR: No prepared statement has been registered for {type_}. You might want to consider fixing that.")
+        except TypeError:
+            raise MissingTableConfigException("INTERNAL ERROR: Statements have not been prepared. You must call `prepare_insert_statements` first.")
+
+    def for_each_table(self, action: Callable[[SqliteTable], None]):
+        """
+        Call the `action` on each table.
+
+        :param action: A callback invoked on each table.
+        """
+        for t in self.tables:
+            action(t)
+
+    def prepare_insert_statements(self, connection: Connection):
+        """
+        Create a `PreparedStatement` for inserting into each table.
+
+        :param connection: The `Connection` to prepare the statements on.
+        """
+        self._close_insert_statements()
+
+        # Make sure the underlying collection has been populated.
+        self._ensure_tables()
+
+        self._insert_cursor = connection.cursor()
+        self._insert_statements = dict()
+        for t, table in self._tables.items():
+            self._insert_statements[t] = PreparedStatement(table.prepared_insert_sql, self._insert_cursor)
+
+    def close(self):
+        self._close_insert_statements()
+
+    def _close_insert_statements(self):
+        try:
+            if self._insert_cursor:
+                self._insert_cursor.close()
+            self._insert_cursor = None
+        except ProgrammingError:
+            # Can be thrown when the database is closed
+            pass
+
+        if self.insert_statements:
+            for s in self.insert_statements:
+                s.close()
+        self._insert_statements = None
+
+    def _ensure_tables(self):
+        if not self._tables:
+            self._tables = {type(it): it for it in self._included_tables}