zepben.ewb 1.0.0b1__py3-none-any.whl

zepben/ewb/database/sqlite/common/base_collection_reader.py

@@ -0,0 +1,96 @@
+#  Copyright 2024 Zeppelin Bend Pty Ltd
+#  This Source Code Form is subject to the terms of the Mozilla Public
+#  License, v. 2.0. If a copy of the MPL was not distributed with this
+#  file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+__all__ = ["BaseCollectionReader"]
+
+import logging
+from abc import ABC, abstractmethod
+from contextlib import closing
+from sqlite3 import Connection
+from typing import Callable, Type, Optional
+
+from zepben.ewb.database.sqlite.common.base_database_tables import BaseDatabaseTables, TSqliteTable
+from zepben.ewb.database.sqlite.common.reader_exceptions import MRIDLookupException, DuplicateMRIDException
+from zepben.ewb.database.sqlite.extensions.prepared_statement import SqlException
+from zepben.ewb.database.sqlite.extensions.result_set import ResultSet
+
+
+class BaseCollectionReader(ABC):
+    """
+    A base class for reading collections of object collections from a database.
+    """
+
+    def __init__(self, tables: BaseDatabaseTables, connection: Connection):
+        super().__init__()
+        self._logger: logging.Logger = logging.getLogger(self.__class__.__name__)
+        """
+        The logger to use for this collection reader.
+        """
+
+        self.base_database_tables: BaseDatabaseTables = tables
+        """
+        The tables that are available in the database.
+        """
+
+        self._connection: Connection = connection
+        """
+        The connection to the database to read.
+        """
+
+    @abstractmethod
+    def load(self) -> bool:
+        """
+        Load all the objects for the available collections.
+
+        :return: True if the load was successful, otherwise False.
+        """
+        pass
+
+    def _load_each(self, type_: Type[TSqliteTable], process_row: Callable[[TSqliteTable, ResultSet, Callable[[str], str]], bool]) -> bool:
+        """
+        Load each row of a table.
+
+        :param type_: The `SqliteTable` to read the objects from.
+        :param process_row: A callback for processing each row in the table. The callback will be provided with the table, the results for the row and a
+          callback to set the identifier for the row, which returns the same value, so it can be used fluently.
+        """
+        table = self.base_database_tables.get_table(type_)
+
+        def process_rows(results: ResultSet):
+            last_identifier: Optional[str] = None
+
+            def set_identifier(identifier: str) -> str:
+                nonlocal last_identifier
+                last_identifier = identifier
+                return identifier
+
+            try:
+                count = 0
+                while results.next():
+                    if process_row(table, results, set_identifier):
+                        count = count + 1
+
+                return count
+            except SqlException as e:
+                self._logger.error(f"Failed to load '{last_identifier}' from '{table.name}': {e}")
+                raise e
+
+        return self._load_all(table, process_rows)
+
+    def _load_all(self, table: TSqliteTable, process_rows: Callable[[ResultSet], int]) -> bool:
+        """
+        You really shouldn't need to use this function directly, use `load_each` instead.
+        """
+        self._logger.info(f"Loading {table.describe()}...")
+
+        try:
+            with closing(self._connection.cursor()) as cur:
+                cur.execute(table.select_sql)
+                count = process_rows(ResultSet(cur.fetchall()))
+            self._logger.info(f"Successfully loaded {count} {table.describe()}.")
+            return True
+        except (SqlException, ValueError, MRIDLookupException, DuplicateMRIDException) as ex:
+            self._logger.exception(f"Failed to read the {table.describe()} from '{table.name}': {ex}")
+            return False

zepben/ewb/database/sqlite/common/base_collection_writer.py

@@ -0,0 +1,73 @@
+#  Copyright 2024 Zeppelin Bend Pty Ltd
+#  This Source Code Form is subject to the terms of the Mozilla Public
+#  License, v. 2.0. If a copy of the MPL was not distributed with this
+#  file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+__all__ = ["BaseCollectionWriter"]
+
+import logging
+from abc import ABC, abstractmethod
+from typing import Callable, TypeVar, Iterable
+
+from zepben.ewb.database.sqlite.extensions.prepared_statement import SqlException
+
+T = TypeVar("T")
+
+
+class BaseCollectionWriter(ABC):
+    """
+    A base class for writing collections of object collections to a database.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self._logger: logging.Logger = logging.getLogger(self.__class__.__name__)
+        """
+        The logger to use for this collection writer.
+        """
+
+    @abstractmethod
+    def save(self) -> bool:
+        """
+        Save all the objects for the available collections.
+
+        :return: True if the save was successful, otherwise False.
+        """
+        pass
+
+    def _save_each(self, items: Iterable[T], saver: Callable[[T], bool], on_save_failure: Callable[[T, Exception], None]) -> bool:
+        """
+        Save each of the [items] to the database.
+
+        NOTE: This function does not short circuit and all items will be attempted, even if one fails.
+
+        :param items: The collection of items to save.
+        :param saver: A callback for saving each object to the database.
+        :param on_save_failure: A callback that should be used to indicate there was a failure in the `saver`. You should pass the object that was being saved,
+          and the exception that caused the failure.
+
+        :return: True if all `items` were successfully saved, otherwise False.
+        """
+        status = True
+
+        for it in items:
+            status = self._validate_save(it, saver, lambda e: on_save_failure(it, e)) and status
+
+        return status
+
+    @staticmethod
+    def _validate_save(it: T, saver: [[T], bool], on_save_failure: [[Exception], None]) -> bool:
+        """
+        Validate that a save actually works, and convert all exceptions into failures with a callback.
+
+        :param it: The object instance to save.
+        :param saver: The callback that will save the object to the database.
+        :param on_save_failure: A callback if an exception was thrown by the [saver].
+
+        :return: True if the saver successfully saved the object to the database, otherwise False.
+        """
+        try:
+            return saver(it)
+        except SqlException as e:
+            on_save_failure(e)
+            return False

zepben/ewb/database/sqlite/common/base_database_reader.py

@@ -0,0 +1,127 @@
+#  Copyright 2024 Zeppelin Bend Pty Ltd
+#  This Source Code Form is subject to the terms of the Mozilla Public
+#  License, v. 2.0. If a copy of the MPL was not distributed with this
+#  file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+__all__ = ["BaseDatabaseReader"]
+
+import logging
+from abc import ABC
+from contextlib import closing
+from sqlite3 import Connection
+from typing import Optional
+
+from zepben.ewb.database.sqlite.common.base_service_reader import BaseServiceReader
+from zepben.ewb.database.sqlite.common.metadata_collection_reader import MetadataCollectionReader
+from zepben.ewb.database.sqlite.tables.table_version import TableVersion
+from zepben.ewb.services.common.base_service import BaseService
+
+
+class BaseDatabaseReader(ABC):
+    """
+    A base class for reading objects from one of our databases.
+    """
+
+    def __init__(
+        self,
+        connection: Connection,
+        metadata_reader: MetadataCollectionReader,
+        service_reader: BaseServiceReader,
+        service: BaseService,
+        database_description: str,
+        table_version: TableVersion
+    ):
+        super().__init__()
+        self._logger: logging.Logger = logging.getLogger(self.__class__.__name__)
+        """
+        The logger to use for this reader.
+        """
+
+        self._connection: Connection = connection
+        """
+        The connection to the database.
+        """
+
+        self._metadata_reader: MetadataCollectionReader = metadata_reader
+        """
+        The reader for the [MetadataCollection] included in the database.
+        """
+
+        self._service_reader: BaseServiceReader = service_reader
+        """
+        The reader for the [BaseService] supported by the database.
+        """
+
+        self._service: BaseService = service
+        """
+        The [BaseService] that will be populated by the [BaseServiceReader]. Used for post-processing.
+        """
+
+        self._database_description: str = database_description
+        """
+        The description of the database for logging (e.g. filename).
+        """
+
+        self._table_version: TableVersion = table_version
+        """
+        The version table object for checking the database version number.
+        """
+
+        self._has_been_used: bool = False
+        self._supported_version: int = table_version.SUPPORTED_VERSION
+
+    async def _post_load(self) -> bool:
+        """
+        Customisable function for performing actions after the database has been loaded.
+        """
+        self._logger.info("Ensuring all references resolved...")
+        for it in self._service.unresolved_references():
+            raise ValueError(
+                f"Unresolved references found in {self._service.name} service after load - this should not occur. Failing reference was from " +
+                f"{it.from_ref} resolving {it.resolver.to_class.__name__} {it.to_mrid}"
+            )
+        self._logger.info("Unresolved references were all resolved during load.")
+
+        return True
+
+    async def load(self) -> bool:
+        """
+        Load the database.
+        """
+        try:
+            if self._has_been_used:
+                raise ValueError("You can only use the database reader once.")
+            self._has_been_used = True
+
+            return self._pre_load() and self._load_from_readers() and await self._post_load()
+        except Exception as e:
+            self._logger.exception(f"Unable to load database: {e}")
+            return False
+
+    def _pre_load(self) -> bool:
+        try:
+            with closing(self._connection.cursor()) as cur:
+                version = self._table_version.get_version(cur)
+                if version == self._supported_version:
+                    self._logger.info(f"Loading from database version v{version}")
+                    return True
+                else:
+                    self._logger.error(self._format_version_error(version))
+                    return False
+        except Exception as e:
+            self._logger.exception(f"Failed to connect to the database for reading: {e}")
+            return False
+
+    def _load_from_readers(self):
+        return self._metadata_reader.load() and self._service_reader.load()
+
+    def _format_version_error(self, version: Optional[int]) -> str:
+        if version is None:
+            return "Failed to read the version number form the selected database. Are you sure it is a EWB database?"
+        elif version < self._supported_version:
+            return self._unexpected_version(version, "Consider using the UpgradeRunner if you wish to support this database.")
+        else:
+            return self._unexpected_version(version, "You need to use a newer version of the SDK to load this database.")
+
+    def _unexpected_version(self, version: Optional[int], action: str):
+        return f"Unable to load from database {self._database_description} [found v{version}, expected v{self._supported_version}]. {action}"

zepben/ewb/database/sqlite/common/base_database_tables.py

@@ -0,0 +1,137 @@
+#  Copyright 2024 Zeppelin Bend Pty Ltd
+#  This Source Code Form is subject to the terms of the Mozilla Public
+#  License, v. 2.0. If a copy of the MPL was not distributed with this
+#  file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+__all__ = ["TSqliteTable", "BaseDatabaseTables"]
+
+from abc import ABC
+from sqlite3 import Connection, Cursor, ProgrammingError
+from typing import Dict, TypeVar, Type, Generator, Callable, Optional
+
+from zepben.ewb.database.sqlite.extensions.prepared_statement import PreparedStatement
+from zepben.ewb.database.sqlite.tables.exceptions import MissingTableConfigException
+from zepben.ewb.database.sqlite.tables.iec61970.base.core.table_name_types import *
+from zepben.ewb.database.sqlite.tables.iec61970.base.core.table_names import *
+from zepben.ewb.database.sqlite.tables.sqlite_table import *
+from zepben.ewb.database.sqlite.tables.table_metadata_data_sources import TableMetadataDataSources
+from zepben.ewb.database.sqlite.tables.table_version import TableVersion
+
+TSqliteTable = TypeVar("TSqliteTable", bound=SqliteTable)
+
+
+class BaseDatabaseTables(ABC):
+    """
+    The base collection of tables for all our databases.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self._tables: Optional[Dict[Type[TSqliteTable], SqliteTable]] = None
+        self._insert_statements: Optional[Dict[Type[TSqliteTable], PreparedStatement]] = None
+        self._insert_cursor: Optional[Cursor] = None
+
+    @property
+    def tables(self) -> Generator[SqliteTable, None, None]:
+        """
+        The tables that are available in this database, keyed on the table class. You should use `get_table` to access individual tables.
+        """
+        self._ensure_tables()
+        for t in self._tables.values():
+            yield t
+
+    @property
+    def insert_statements(self) -> Generator[PreparedStatement, None, None]:
+        """
+        A collection of `PreparedStatement` for each table. You should use `get_insert` to access individual inserts.
+        """
+        if self._insert_statements is not None:
+            for s in self._insert_statements.values():
+                yield s
+
+    @property
+    def _included_tables(self) -> Generator[SqliteTable, None, None]:
+        """
+        A sequence of `SqliteTable` indicating which tables are included in this database, which will be consumed to build the `tables` collection.
+
+        NOTE: You should always append your tables to `super()._included_tables` when overriding.
+        """
+        yield TableMetadataDataSources()
+        yield TableVersion()
+        yield TableNameTypes()
+        yield TableNames()
+
+    def get_table(self, type_: Type[TSqliteTable]) -> TSqliteTable:
+        """
+        Helper function for getting the table of the specified type.
+
+        :param type_: The type of table to get.
+        :return: The requested table if it belongs to this database.
+        :raises MissingTableConfigException: If the requested table doesn't belong to this database.
+        """
+        self._ensure_tables()
+        try:
+            return self._tables[type_]
+        except KeyError:
+            raise MissingTableConfigException(f"INTERNAL ERROR: No table has been registered for {type_}. You might want to consider fixing that.")
+
+    def get_insert(self, type_: Type[TSqliteTable]) -> PreparedStatement:
+        """
+        Helper function for getting the insert statement for the specified table.
+
+        :param type_: The type of table to get.
+        :return: The insert statement for the requested table if it belongs to this database.
+        :raises MissingTableConfigException: If the requested table doesn't belong to this database.
+        """
+        try:
+            return self._insert_statements[type_]
+        except KeyError:
+            raise MissingTableConfigException(f"INTERNAL ERROR: No prepared statement has been registered for {type_}. You might want to consider fixing that.")
+        except TypeError:
+            raise MissingTableConfigException("INTERNAL ERROR: Statements have not been prepared. You must call `prepare_insert_statements` first.")
+
+    def for_each_table(self, action: Callable[[SqliteTable], None]):
+        """
+        Call the `action` on each table.
+
+        :param action: A callback invoked on each table.
+        """
+        for t in self.tables:
+            action(t)
+
+    def prepare_insert_statements(self, connection: Connection):
+        """
+        Create a `PreparedStatement` for inserting into each table.
+
+        :param connection: The `Connection` to prepare the statements on.
+        """
+        self._close_insert_statements()
+
+        # Make sure the underlying collection has been populated.
+        self._ensure_tables()
+
+        self._insert_cursor = connection.cursor()
+        self._insert_statements = dict()
+        for t, table in self._tables.items():
+            self._insert_statements[t] = PreparedStatement(table.prepared_insert_sql, self._insert_cursor)
+
+    def close(self):
+        self._close_insert_statements()
+
+    def _close_insert_statements(self):
+        try:
+            if self._insert_cursor:
+                self._insert_cursor.close()
+            self._insert_cursor = None
+        except ProgrammingError:
+            # Can be thrown when the database is closed
+            pass
+
+        if self.insert_statements:
+            for s in self.insert_statements:
+                s.close()
+        self._insert_statements = None
+
+    def _ensure_tables(self):
+        if not self._tables:
+            self._tables = {type(it): it for it in self._included_tables}

zepben/ewb/database/sqlite/common/base_database_writer.py

@@ -0,0 +1,195 @@
+#  Copyright 2024 Zeppelin Bend Pty Ltd
+#  This Source Code Form is subject to the terms of the Mozilla Public
+#  License, v. 2.0. If a copy of the MPL was not distributed with this
+#  file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+__all__ = ["BaseDatabaseWriter"]
+
+import logging
+import os
+from abc import ABC
+from contextlib import closing
+from pathlib import Path
+from sqlite3 import Connection, Cursor, OperationalError
+from typing import Callable, Union
+
+from zepben.ewb.database.sqlite.common.base_database_tables import BaseDatabaseTables
+from zepben.ewb.database.sqlite.common.base_service_writer import BaseServiceWriter
+from zepben.ewb.database.sqlite.common.metadata_collection_writer import MetadataCollectionWriter
+from zepben.ewb.database.sqlite.extensions.prepared_statement import SqlException, PreparedStatement
+from zepben.ewb.database.sqlite.tables.exceptions import MissingTableConfigException
+from zepben.ewb.database.sqlite.tables.sqlite_table import SqliteTable
+from zepben.ewb.database.sqlite.tables.table_version import TableVersion
+
+
+class BaseDatabaseWriter(ABC):
+    """
+    A base class for writing objects to one of our databases.
+    """
+
+    def __init__(
+        self,
+        database_file: Union[Path, str],
+        database_tables: BaseDatabaseTables,
+        create_metadata_writer: Callable[[], MetadataCollectionWriter],
+        create_service_writer: Callable[[], BaseServiceWriter],
+        get_connection: Callable[[str], Connection]
+    ):
+        super().__init__()
+        self._logger: logging.Logger = logging.getLogger(self.__class__.__name__)
+        """
+        The logger to use for this database writer.
+        """
+
+        self._database_file: str = str(database_file)
+        """
+        The filename of the database to write.
+        """
+
+        self._database_tables: BaseDatabaseTables = database_tables
+        """
+        The tables to create in the database.
+        """
+
+        self._create_metadata_writer: Callable[[], MetadataCollectionWriter] = create_metadata_writer
+        """
+        Create a [MetadataCollectionWriter] that uses the provided [Connection].
+        """
+
+        self._create_service_writer: Callable[[], BaseServiceWriter] = create_service_writer
+        """
+        Create a [BaseServiceWriter] that uses the provided [Connection].
+        """
+
+        self._get_connection: Callable[[str], Connection] = get_connection
+        """
+        Provider of the connection to the specified database.
+        """
+
+        self._save_connection: Connection
+        self._has_been_used: bool = False
+
+    def save(self) -> bool:
+        """
+        Save the database using the [MetadataCollectionWriter] and [BaseServiceWriter].
+
+        @return True if the database was successfully saved, otherwise False.
+        """
+        if self._has_been_used:
+            self._logger.error(f"You can only use the database writer once.")
+            return False
+        self._has_been_used = True
+
+        if not self._pre_save():
+            self._close_connection()
+            return False
+
+        status: bool
+        try:
+            status = all([
+                self._create_metadata_writer().save(),
+                self._create_service_writer().save()
+            ])
+        except MissingTableConfigException as e:
+            self._logger.exception(f"Unable to save database: {e}")
+            status = False
+
+        return all([status, self._post_save()])
+
+    def _pre_save(self) -> bool:
+        return (self._remove_existing()
+                and self._connect()
+                and self._create()
+                and self._prepare_insert_statements())
+
+    def _remove_existing(self) -> bool:
+        try:
+            if os.path.isfile(self._database_file):
+                os.remove(self._database_file)
+            return True
+        except Exception as e:
+            self._logger.exception(f"Unable to save database, failed to remove previous instance: {e}")
+            return False
+
+    def _connect(self) -> bool:
+        try:
+            self._save_connection = self._get_connection(self._database_file)
+
+            cur: Cursor
+            with closing(self._save_connection.cursor()) as cur:
+                cur.execute("PRAGMA journal_mode = OFF")
+                cur.execute("PRAGMA synchronous = OFF")
+
+            self._save_connection.isolation_level = None  # autocommit off
+
+            return True
+        except (SqlException, OperationalError) as e:
+            self._logger.exception(f"Failed to connect to the database for saving: {e}")
+            self._close_connection()
+            return False
+
+    def _prepare_insert_statements(self) -> bool:
+        try:
+            self._database_tables.prepare_insert_statements(self._save_connection)
+            return True
+        except SqlException as e:
+            self._logger.exception(f"Failed to prepare insert statements: {e}")
+            self._close_connection()
+            return False
+
+    def _create(self) -> bool:
+        try:
+            version_table = self._database_tables.get_table(TableVersion)
+            self._logger.info(f"Creating database schema v{version_table.SUPPORTED_VERSION}...")
+
+            cur: Cursor
+            with closing(self._save_connection.cursor()) as cur:
+                def create_table(it: SqliteTable):
+                    cur.execute(it.create_table_sql)
+
+                self._database_tables.for_each_table(create_table)
+
+                # Add the version number to the database.
+                insert = PreparedStatement(version_table.prepared_insert_sql, cur)
+                insert.add_value(version_table.version.query_index, version_table.SUPPORTED_VERSION)
+                insert.execute()
+
+                self._save_connection.commit()
+                self._logger.info("Schema created.")
+
+            return True
+        except SqlException as e:
+            self._logger.exception(f"Failed to create database schema: {e}")
+            return False
+
+    def _close_connection(self):
+        try:
+            if self._save_connection is not None:
+                self._save_connection.close()
+        except SqlException as e:
+            self._logger.exception(f"Failed to close connection to database: {e}")
+
+    def _post_save(self) -> bool:
+        try:
+            self._logger.info("Adding indexes...")
+
+            cur: Cursor
+            with closing(self._save_connection.cursor()) as cur:
+                def create_indexes(it: SqliteTable):
+                    for sql in it.create_indexes_sql:
+                        cur.execute(sql)
+
+                self._database_tables.for_each_table(create_indexes)
+
+            self._logger.info("Indexes added.")
+            self._logger.info("Committing...")
+
+            self._save_connection.commit()
+
+            self._logger.info("Done.")
+            return True
+        except SqlException as e:
+            self._logger.exception(f"Failed to finalise the database: {e}")
+            return False
+        finally:
+            self._close_connection()

zepben/ewb/database/sqlite/common/base_entry_writer.py

@@ -0,0 +1,34 @@
+#  Copyright 2024 Zeppelin Bend Pty Ltd
+#  This Source Code Form is subject to the terms of the Mozilla Public
+#  License, v. 2.0. If a copy of the MPL was not distributed with this
+#  file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+__all__ = ["BaseEntryWriter"]
+
+import logging
+from abc import ABC
+
+from zepben.ewb.database.sqlite.extensions.prepared_statement import PreparedStatement
+
+
+class BaseEntryWriter(ABC):
+    """
+    A base class for writing entries into tables of the database.
+    """
+
+    def __init__(self):
+        self._logger = logging.getLogger(self.__class__.__name__)
+
+    def _try_execute_single_update(self, prepared_statement: PreparedStatement, description: str) -> bool:
+        """
+        Helper function for writing the entry to the database and logging any failures.
+
+        :param prepared_statement: The `PreparedStatement` to execute.
+        :param description: A description of the object being written to use for logging of failures.
+        :return: True if the entry was successfully written to the database, otherwise False.
+        """
+
+        def log_failure(ex: Exception):
+            prepared_statement.log_failure(self._logger, description, ex)
+
+        return prepared_statement.try_execute_single_update(log_failure)