zepben.ewb 1.0.0__py3-none-any.whl

zepben/ewb/database/sqlite/common/base_database_writer.py

@@ -0,0 +1,195 @@
+#  Copyright 2024 Zeppelin Bend Pty Ltd
+#  This Source Code Form is subject to the terms of the Mozilla Public
+#  License, v. 2.0. If a copy of the MPL was not distributed with this
+#  file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+__all__ = ["BaseDatabaseWriter"]
+
+import logging
+import os
+from abc import ABC
+from contextlib import closing
+from pathlib import Path
+from sqlite3 import Connection, Cursor, OperationalError
+from typing import Callable, Union, Optional
+
+from zepben.ewb.database.sqlite.common.base_database_tables import BaseDatabaseTables
+from zepben.ewb.database.sqlite.common.base_service_writer import BaseServiceWriter
+from zepben.ewb.database.sqlite.common.metadata_collection_writer import MetadataCollectionWriter
+from zepben.ewb.database.sqlite.extensions.prepared_statement import SqlException, PreparedStatement
+from zepben.ewb.database.sqlite.tables.exceptions import MissingTableConfigException
+from zepben.ewb.database.sqlite.tables.sqlite_table import SqliteTable
+from zepben.ewb.database.sqlite.tables.table_version import TableVersion
+
+
+class BaseDatabaseWriter(ABC):
+    """
+    A base class for writing objects to one of our databases.
+    """
+
+    def __init__(
+        self,
+        database_file: Union[Path, str],
+        database_tables: BaseDatabaseTables,
+        create_metadata_writer: Callable[[], MetadataCollectionWriter],
+        create_service_writer: Callable[[], BaseServiceWriter],
+        get_connection: Callable[[str], Connection]
+    ):
+        super().__init__()
+        self._logger: logging.Logger = logging.getLogger(self.__class__.__name__)
+        """
+        The logger to use for this database writer.
+        """
+
+        self._database_file: str = str(database_file)
+        """
+        The filename of the database to write.
+        """
+
+        self._database_tables: BaseDatabaseTables = database_tables
+        """
+        The tables to create in the database.
+        """
+
+        self._create_metadata_writer: Callable[[], MetadataCollectionWriter] = create_metadata_writer
+        """
+        Create a [MetadataCollectionWriter] that uses the provided [Connection].
+        """
+
+        self._create_service_writer: Callable[[], BaseServiceWriter] = create_service_writer
+        """
+        Create a [BaseServiceWriter] that uses the provided [Connection].
+        """
+
+        self._get_connection: Callable[[str], Connection] = get_connection
+        """
+        Provider of the connection to the specified database.
+        """
+
+        self._save_connection: Optional[Connection] = None
+        self._has_been_used: bool = False
+
+    def save(self) -> bool:
+        """
+        Save the database using the [MetadataCollectionWriter] and [BaseServiceWriter].
+
+        @return True if the database was successfully saved, otherwise False.
+        """
+        if self._has_been_used:
+            self._logger.error(f"You can only use the database writer once.")
+            return False
+        self._has_been_used = True
+
+        if not self._pre_save():
+            self._close_connection()
+            return False
+
+        status: bool
+        try:
+            status = all([
+                self._create_metadata_writer().save(),
+                self._create_service_writer().save()
+            ])
+        except MissingTableConfigException as e:
+            self._logger.exception(f"Unable to save database: {e}")
+            status = False
+
+        return all([status, self._post_save()])
+
+    def _pre_save(self) -> bool:
+        return (self._remove_existing()
+                and self._connect()
+                and self._create()
+                and self._prepare_insert_statements())
+
+    def _remove_existing(self) -> bool:
+        try:
+            if os.path.isfile(self._database_file):
+                os.remove(self._database_file)
+            return True
+        except Exception as e:
+            self._logger.exception(f"Unable to save database, failed to remove previous instance: {e}")
+            return False
+
+    def _connect(self) -> bool:
+        try:
+            self._save_connection = self._get_connection(self._database_file)
+
+            cur: Cursor
+            with closing(self._save_connection.cursor()) as cur:
+                cur.execute("PRAGMA journal_mode = OFF")
+                cur.execute("PRAGMA synchronous = OFF")
+
+            self._save_connection.isolation_level = None  # autocommit off
+
+            return True
+        except (SqlException, OperationalError) as e:
+            self._logger.exception(f"Failed to connect to the database for saving: {e}")
+            self._close_connection()
+            return False
+
+    def _prepare_insert_statements(self) -> bool:
+        try:
+            self._database_tables.prepare_insert_statements(self._save_connection)
+            return True
+        except SqlException as e:
+            self._logger.exception(f"Failed to prepare insert statements: {e}")
+            self._close_connection()
+            return False
+
+    def _create(self) -> bool:
+        try:
+            version_table = self._database_tables.get_table(TableVersion)
+            self._logger.info(f"Creating database schema v{version_table.SUPPORTED_VERSION}...")
+
+            cur: Cursor
+            with closing(self._save_connection.cursor()) as cur:
+                def create_table(it: SqliteTable):
+                    cur.execute(it.create_table_sql)
+
+                self._database_tables.for_each_table(create_table)
+
+                # Add the version number to the database.
+                insert = PreparedStatement(version_table.prepared_insert_sql, cur)
+                insert.add_value(version_table.version.query_index, version_table.SUPPORTED_VERSION)
+                insert.execute()
+
+                self._save_connection.commit()
+                self._logger.info("Schema created.")
+
+            return True
+        except SqlException as e:
+            self._logger.exception(f"Failed to create database schema: {e}")
+            return False
+
+    def _close_connection(self):
+        try:
+            if self._save_connection is not None:
+                self._save_connection.close()
+        except SqlException as e:
+            self._logger.exception(f"Failed to close connection to database: {e}")
+
+    def _post_save(self) -> bool:
+        try:
+            self._logger.info("Adding indexes...")
+
+            cur: Cursor
+            with closing(self._save_connection.cursor()) as cur:
+                def create_indexes(it: SqliteTable):
+                    for sql in it.create_indexes_sql:
+                        cur.execute(sql)
+
+                self._database_tables.for_each_table(create_indexes)
+
+            self._logger.info("Indexes added.")
+            self._logger.info("Committing...")
+
+            self._save_connection.commit()
+
+            self._logger.info("Done.")
+            return True
+        except SqlException as e:
+            self._logger.exception(f"Failed to finalise the database: {e}")
+            return False
+        finally:
+            self._close_connection()

zepben/ewb/database/sqlite/common/base_entry_writer.py

@@ -0,0 +1,34 @@
+#  Copyright 2024 Zeppelin Bend Pty Ltd
+#  This Source Code Form is subject to the terms of the Mozilla Public
+#  License, v. 2.0. If a copy of the MPL was not distributed with this
+#  file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+__all__ = ["BaseEntryWriter"]
+
+import logging
+from abc import ABC
+
+from zepben.ewb.database.sqlite.extensions.prepared_statement import PreparedStatement
+
+
+class BaseEntryWriter(ABC):
+    """
+    A base class for writing entries into tables of the database.
+    """
+
+    def __init__(self):
+        self._logger = logging.getLogger(self.__class__.__name__)
+
+    def _try_execute_single_update(self, prepared_statement: PreparedStatement, description: str) -> bool:
+        """
+        Helper function for writing the entry to the database and logging any failures.
+
+        :param prepared_statement: The `PreparedStatement` to execute.
+        :param description: A description of the object being written to use for logging of failures.
+        :return: True if the entry was successfully written to the database, otherwise False.
+        """
+
+        def log_failure(ex: Exception):
+            prepared_statement.log_failure(self._logger, description, ex)
+
+        return prepared_statement.try_execute_single_update(log_failure)

zepben/ewb/database/sqlite/common/base_service_reader.py

@@ -0,0 +1,50 @@
+#  Copyright 2024 Zeppelin Bend Pty Ltd
+#  This Source Code Form is subject to the terms of the Mozilla Public
+#  License, v. 2.0. If a copy of the MPL was not distributed with this
+#  file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+__all__ = ["BaseServiceReader"]
+
+from abc import ABC, abstractmethod
+from sqlite3 import Connection
+
+from zepben.ewb.database.sqlite.common.base_cim_reader import BaseCimReader
+from zepben.ewb.database.sqlite.common.base_collection_reader import BaseCollectionReader
+from zepben.ewb.database.sqlite.common.base_database_tables import BaseDatabaseTables
+from zepben.ewb.database.sqlite.tables.iec61970.base.core.table_name_types import TableNameTypes
+from zepben.ewb.database.sqlite.tables.iec61970.base.core.table_names import TableNames
+
+
+class BaseServiceReader(BaseCollectionReader, ABC):
+    """
+    A base class for reading items stored in a `BaseService` from the database.
+
+    :param tables: The tables available in the database.
+    :param connection: A connection to the database.
+    :param reader: The `BaseCimReader` used to load the objects from the database.
+    """
+
+    def __init__(
+        self,
+        tables: BaseDatabaseTables,
+        connection: Connection,
+        reader: BaseCimReader
+    ):
+        super().__init__(tables, connection)
+        self._reader: BaseCimReader = reader
+        """
+        The `BaseCimReader` used to load the objects from the database.
+        """
+
+    def load(self) -> bool:
+        return (self._do_load()
+                and self._load_each(TableNameTypes, self._reader.load_name_types)
+                and self._load_each(TableNames, self._reader.load_names))
+
+    @abstractmethod
+    def _do_load(self) -> bool:
+        """
+        Load the service specific objects from the database.
+
+        :return: True if the objects were successfully loaded from the database, otherwise False
+        """

zepben/ewb/database/sqlite/common/base_service_writer.py

@@ -0,0 +1,104 @@
+#  Copyright 2024 Zeppelin Bend Pty Ltd
+#  This Source Code Form is subject to the terms of the Mozilla Public
+#  License, v. 2.0. If a copy of the MPL was not distributed with this
+#  file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+__all__ = ["BaseServiceWriter"]
+
+from abc import ABC, abstractmethod
+from typing import Callable, Type
+
+from zepben.ewb.database.sqlite.common.base_cim_writer import BaseCimWriter
+from zepben.ewb.database.sqlite.common.base_collection_writer import BaseCollectionWriter
+from zepben.ewb.model.cim.iec61970.base.core.identified_object import TIdentifiedObject
+from zepben.ewb.model.cim.iec61970.base.core.name import Name
+from zepben.ewb.model.cim.iec61970.base.core.name_type import NameType
+from zepben.ewb.services.common.base_service import BaseService
+
+
+class BaseServiceWriter(BaseCollectionWriter, ABC):
+    """
+    A base class for writing object from a [BaseService] into the database.
+
+    :param service: The `BaseService` to save to the database.
+    :param writer: The `BaseServiceWriter` used to actually write the objects to the database.
+    """
+
+    def __init__(self, service: BaseService, writer: BaseCimWriter):
+        super().__init__()
+        self._service: BaseService = service
+        """
+        The `BaseService` to save to the database.
+        """
+
+        self._writer: BaseCimWriter = writer
+        """
+        The `BaseServiceWriter` used to actually write the objects to the database.
+        """
+
+    def save(self) -> bool:
+        return all([self._save_name_types(), self._do_save()])
+
+    @abstractmethod
+    def _do_save(self) -> bool:
+        """
+        Save the service specific objects to the database.
+
+        :return: True if the objects were successfully saved to the database, otherwise False
+        """
+
+    """
+    Save each object of the specified type using the provided `saver`.
+    
+    @param T The type of object to save to the database.
+    @param saver The callback used to save the objects to the database. Will be called once for each object and should return True if the object is
+      successfully saved to the database.
+    
+    :return: True if all objects are successfully saved to the database, otherwise False.
+    """
+
+    def _save_each_object(self, type_: Type[TIdentifiedObject], saver: Callable[[TIdentifiedObject], bool]) -> bool:
+        status = True
+        for it in self._service.objects(type_):
+            status = status and self._validate_save_object(it, saver)
+
+        return status
+
+    """
+    Validate that an object is actually saved to the database, logging an error if anything goes wrong.
+    
+    @param T The type of object being saved.
+    @param it The object being saved.
+    @param saver The callback actually saving the object to the database.
+    
+    :return: True if the object is successfully saved to the database, otherwise False.
+    """
+
+    def _validate_save_object(self, it: TIdentifiedObject, saver: Callable[[TIdentifiedObject], bool]) -> bool:
+        def log_error(e: Exception):
+            self._logger.error(f"Failed to save {it.__class__.__name__} {it.name} [{it.mrid}]: {e}")
+
+        return self._validate_save(it, saver, log_error)
+
+    def _save_name_types(self) -> bool:
+        status = True
+
+        for it in self._service.name_types:
+            status = all([status, self._validate_save_name_type(it, self._writer.save_name_type)])
+
+            for name in it.names:
+                status = all([status, self._validate_save_name(name, self._writer.save_name)])
+
+        return status
+
+    def _validate_save_name_type(self, name_type: NameType, saver: Callable[[NameType], bool]) -> bool:
+        def log_error(e: Exception):
+            self._logger.error(f"Failed to save {name_type.__class__.__name__} {name_type.name}: {e}")
+
+        return self._validate_save(name_type, saver, log_error)
+
+    def _validate_save_name(self, name: Name, saver: Callable[[Name], bool]) -> bool:
+        def log_error(e: Exception):
+            self._logger.error(f"Failed to save {name.__class__.__name__} {name.name}: {e}")
+
+        return self._validate_save(name, saver, log_error)

zepben/ewb/database/sqlite/common/metadata_collection_reader.py

@@ -0,0 +1,39 @@
+#  Copyright 2024 Zeppelin Bend Pty Ltd
+#  This Source Code Form is subject to the terms of the Mozilla Public
+#  License, v. 2.0. If a copy of the MPL was not distributed with this
+#  file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+__all__ = ["MetadataCollectionReader"]
+
+from sqlite3 import Connection
+
+from zepben.ewb.database.sqlite.common.base_collection_reader import BaseCollectionReader
+from zepben.ewb.database.sqlite.common.base_database_tables import BaseDatabaseTables
+from zepben.ewb.database.sqlite.common.metadata_entry_reader import MetadataEntryReader
+from zepben.ewb.database.sqlite.tables.table_metadata_data_sources import TableMetadataDataSources
+from zepben.ewb.services.common.base_service import BaseService
+
+
+class MetadataCollectionReader(BaseCollectionReader):
+    """
+    Class for reading the `MetadataCollection` from the database.
+
+    :param service: The `BaseService` containing the `MetadataCollection` to populate from the database.
+    :param tables: The tables available in the database.
+    :param connection: The `Connection` to the database.
+    """
+
+    def __init__(
+        self,
+        service: BaseService,
+        tables: BaseDatabaseTables,
+        connection: Connection,
+        reader: MetadataEntryReader = None
+    ):
+        super().__init__(tables, connection)
+        self._reader: MetadataEntryReader = reader if reader is not None else MetadataEntryReader(service)
+
+    def load(self) -> bool:
+        return all([
+            self._load_each(TableMetadataDataSources, self._reader.load_metadata)
+        ])

zepben/ewb/database/sqlite/common/metadata_collection_writer.py

@@ -0,0 +1,38 @@
+#  Copyright 2024 Zeppelin Bend Pty Ltd
+#  This Source Code Form is subject to the terms of the Mozilla Public
+#  License, v. 2.0. If a copy of the MPL was not distributed with this
+#  file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+__all__ = ["MetadataCollectionWriter"]
+
+from zepben.ewb.database.sqlite.common.base_collection_writer import BaseCollectionWriter
+from zepben.ewb.database.sqlite.common.base_database_tables import BaseDatabaseTables
+from zepben.ewb.database.sqlite.common.metadata_entry_writer import MetadataEntryWriter
+from zepben.ewb.services.common.base_service import BaseService
+from zepben.ewb.services.common.meta.data_source import DataSource
+
+
+class MetadataCollectionWriter(BaseCollectionWriter):
+    """
+    Class for writing the `MetadataCollection` to the database.
+
+    :param service: The `BaseService` containing the `MetadataCollection` to save to the database.
+    :param database_tables: The tables available in the database.
+    """
+
+    def __init__(
+        self,
+        service: BaseService,
+        database_tables: BaseDatabaseTables,
+        writer: MetadataEntryWriter = None
+    ):
+        super().__init__()
+        self._service: BaseService = service
+        self._writer: MetadataEntryWriter = writer if writer is not None else MetadataEntryWriter(database_tables)
+
+
+    def save(self) -> bool:
+        def log_error(it: DataSource, e: Exception):
+            self._logger.error(f"Failed to save {it}: {e}")
+
+        return self._save_each(self._service.metadata.data_sources, self._writer.save_data_source, log_error)

zepben/ewb/database/sqlite/common/metadata_entry_reader.py

@@ -0,0 +1,45 @@
+#  Copyright 2024 Zeppelin Bend Pty Ltd
+#  This Source Code Form is subject to the terms of the Mozilla Public
+#  License, v. 2.0. If a copy of the MPL was not distributed with this
+#  file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+__all__ = ["MetadataEntryReader"]
+
+import datetime
+from typing import Callable
+
+from zepben.ewb.database.sqlite.extensions.result_set import ResultSet
+from zepben.ewb.database.sqlite.tables.table_metadata_data_sources import TableMetadataDataSources
+from zepben.ewb.services.common.base_service import BaseService
+from zepben.ewb.services.common.meta.data_source import DataSource
+
+
+class MetadataEntryReader:
+    """
+    A class for reading the `MetadataCollection` entries from the database.
+
+    :param service: The `BaseService` containing the `MetadataCollection` to populate from the database.
+    """
+
+    def __init__(self, service: BaseService):
+        super().__init__()
+        self._service: BaseService = service
+
+    def load_metadata(self, table: TableMetadataDataSources, result_set: ResultSet, set_identifier: Callable[[str], str]) -> bool:
+        """
+        Populate the `DataSource` fields from `TableMetadataDataSources`.
+    
+        :param table: The database table to read the `DataSource` fields from.
+        :param result_set: The record in the database table containing the fields for this `DataSource`.
+        :param set_identifier: A callback to set the identifier of the current row for logging purposes, which returns a copy of the provided string for
+          fluent use.
+    
+        :return: True if the `DataSource` is successfully loaded from the database, otherwise False.
+        """
+        data_source = DataSource(
+            set_identifier(result_set.get_string(table.source.query_index)),
+            result_set.get_string(table.version.query_index),
+            result_set.get_instant(table.timestamp.query_index, datetime.datetime(1970, 1, 1))
+        )
+
+        return self._service.metadata.add(data_source)

zepben/ewb/database/sqlite/common/metadata_entry_writer.py

@@ -0,0 +1,41 @@
+#  Copyright 2024 Zeppelin Bend Pty Ltd
+#  This Source Code Form is subject to the terms of the Mozilla Public
+#  License, v. 2.0. If a copy of the MPL was not distributed with this
+#  file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+__all__ = ["MetadataEntryWriter"]
+
+from zepben.ewb.database.sqlite.common.base_database_tables import BaseDatabaseTables
+from zepben.ewb.database.sqlite.common.base_entry_writer import BaseEntryWriter
+from zepben.ewb.database.sqlite.tables.table_metadata_data_sources import TableMetadataDataSources
+from zepben.ewb.services.common.meta.data_source import DataSource
+
+class MetadataEntryWriter(BaseEntryWriter):
+    """
+    A class for reading the `MetadataCollection` entries from the database.
+
+    @param database_tables The tables that are available in the database.
+    """
+
+    def __init__(self, database_tables: BaseDatabaseTables):
+        super().__init__()
+        self._database_tables: BaseDatabaseTables = database_tables
+
+
+    def save_data_source(self, data_source: DataSource) -> bool:
+        """
+        Save the `DataSource` fields to `TableMetadataDataSources`.
+
+        @param data_source The `DataSource` to save to the database.
+
+        @return true if the `DataSource` is successfully written to the database, otherwise false.
+        """
+        table = self._database_tables.get_table(TableMetadataDataSources)
+        insert = self._database_tables.get_insert(TableMetadataDataSources)
+
+        insert.add_value(table.source.query_index, data_source.source)
+        insert.add_value(table.version.query_index, data_source.version)
+        # The timestamp in the database uses Z for UTC while python uses +HH:mm format, so convert between them.
+        insert.add_value(table.timestamp.query_index, f"{data_source.timestamp.isoformat()}Z")
+
+        return insert.try_execute_single_update(lambda ex: insert.log_failure(self._logger, "data source", ex))

zepben/ewb/database/sqlite/common/reader_exceptions.py

@@ -0,0 +1,30 @@
+#  Copyright 2024 Zeppelin Bend Pty Ltd
+#  This Source Code Form is subject to the terms of the Mozilla Public
+#  License, v. 2.0. If a copy of the MPL was not distributed with this
+#  file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+__all__ = ["MRIDLookupException", "NameTypeLookupException", "DuplicateMRIDException", "DuplicateNameTypeException"]
+
+
+class MRIDLookupException(Exception):
+    """
+    An exception indicating that an mRID could not be found in a [BaseService].
+    """
+
+
+class NameTypeLookupException(Exception):
+    """
+    An exception indicating that a [NameType] could not be found in a [BaseService].
+    """
+
+
+class DuplicateMRIDException(Exception):
+    """
+    An exception indicating that an mRID has already been used by a different objects in a [BaseService].
+    """
+
+
+class DuplicateNameTypeException(Exception):
+    """
+    An exception indicating that n [NameType] has already been used in a [BaseService].
+    """

zepben/ewb/database/sqlite/customer/__init__.py

@@ -0,0 +1,4 @@
+#  Copyright 2024 Zeppelin Bend Pty Ltd
+#  This Source Code Form is subject to the terms of the Mozilla Public
+#  License, v. 2.0. If a copy of the MPL was not distributed with this
+#  file, You can obtain one at https://mozilla.org/MPL/2.0/.