pychestrate-core 0.0.1__py3-none-any.whl

ccai/pychestrate/core/examples/create_database.py

@@ -0,0 +1,52 @@
+
+__author__ = "Myron Walker"
+__copyright__ = "Copyright 2025, Automation Mojo LLC"
+__credits__ = []
+__version__ = "1.0.0"
+
+from ccai.runtime.activation import activate_runtime, ActivationProfile
+
+from ccai.configuration.configurationmaps import resolve_configuration_maps
+from ccai.configuration.configurationsettings import establish_configuration_settings
+from ccai.configuration.configurationvariables import resolve_configuration_variables
+
+
+from ccai.credentials.wellknown import CredentialManagerSingleton
+
+from ccai.dataprofiles.wellknown import DataProfileManagerSingleton
+
+
+from ccai.pychestrate.core.storage.dbconnection import create_database_connection_factory
+from ccai.pychestrate.core.storage.dbio import database_exists, database_create
+from ccai.pychestrate.core.storage.orm import PychestrateModelBase
+
+
+def create_database_main():
+
+    establish_configuration_settings(name="pychestrate", presence_dir="~/pychestrate")
+
+    resolve_configuration_variables()
+
+    activate_runtime(profile=ActivationProfile.Command)
+
+    resolve_configuration_maps(use_credentials=True, use_landscape=True)
+
+    profile_manager = DataProfileManagerSingleton()
+    dbprofile = profile_manager.lookup_profile("software-pipeline-admin")
+
+    cred_manager = CredentialManagerSingleton()
+
+    dbfactory = create_database_connection_factory(dbprofile=dbprofile, cred_manager=cred_manager)
+
+    dbname = "pipeline"
+
+    if not database_exists(dbfactory, dbname):
+        database_create(dbfactory, dbname=dbname, dbmetadata=PychestrateModelBase.metadata)
+
+    return
+
+
+if __name__ == "__main__":
+
+    create_database_main()
+

ccai/pychestrate/core/storage/dbconnection.py

@@ -0,0 +1,269 @@
+"""
+.. module:: dbconnection
+    :platform: Darwin, Linux, Unix, Windows
+    :synopsis: Contains helper functions for working with specific databases
+
+.. moduleauthor:: Myron Walker <myron.walker@gmail.com>
+"""
+
+__author__ = "Myron Walker"
+__copyright__ = "Copyright 2025, Automation Mojo LLC"
+__credits__ = []
+__version__ = "1.0.0"
+
+from typing import Optional
+
+import logging
+import os
+
+from sqlalchemy import create_engine
+from sqlalchemy.engine.base import Engine as SqlEngine
+
+from ccai.errors.exceptions import ConfigurationError
+
+from ccai.credentials.basecredential import BaseCredential
+from ccai.credentials.basiccredential import BasicCredential
+from ccai.credentials.credentialmanager import CredentialManager
+
+from ccai.dataprofiles.databasebasicsocketprofile import DatabaseBasicSocketProfile
+from ccai.dataprofiles.databasebasictcpprofile import DatabaseBasicTcpProfile
+
+
+logger = logging.getLogger()
+
+
+class DatabaseConnectionFactory:
+
+    def __init__(self, profile_name: str, *, dbprofile: DatabaseBasicSocketProfile, credential: BaseCredential):
+        self._profile_name = profile_name
+        self._dbprofile = dbprofile
+        self._credential = credential
+        return
+    
+    @property
+    def credential(self) -> BaseCredential:
+        return self._credential
+    
+    @property
+    def dbprofile(self) -> DatabaseBasicSocketProfile:
+        return self._dbprofile
+    
+    @property
+    def profile_name(self) -> str:
+        return self._profile_name
+
+    def create_engine(self, *, dbname=None, echo=True) -> SqlEngine:
+        errmsg = "The 'create_engine' method must be overloaded by 'DatabaseConnectionFactory' derived types."
+        raise NotImplementedError(errmsg)
+    
+    def format_connection_string() -> str:
+        errmsg = "The 'format_connection_string' method must be overloaded by 'DatabaseConnectionFactory' derived types."
+        raise NotImplementedError(errmsg)
+
+
+class BasicSocketDatabaseConnectionFactory(DatabaseConnectionFactory):
+
+    def __init__(self, profile_name: str, *, dbprofile: DatabaseBasicSocketProfile, credential: BaseCredential):
+        super().__init__(profile_name, dbprofile=dbprofile, credential=credential)
+        return
+
+    def create_engine(self, *, dbname=None, echo=True) -> SqlEngine:
+        """Create a SQLAlchemy engine using pipe based connectivity."""
+
+        socket_path: str = self.dbprofile.socket
+
+        if dbname is None and self.dbprofile.dbname is not None:
+            dbname = self.dbprofile.dbname
+
+        if dbname is None:
+            warnmsg = "BasicSocketDatabaseConnectionFactory.create_engine: dbname was None."
+            logger.warn(warnmsg)
+
+        if isinstance(self.credential, BasicCredential):
+            connstr = self.format_connection_string(dbname=dbname)
+            dbengine = create_engine(connstr, echo=echo)
+        else:
+            errmsg = (
+                "BasicSocketDatabaseConnectionFactory requires a BasicCredential to "
+                "create an engine."
+            )
+            raise ConfigurationError(errmsg)
+
+        return dbengine
+
+    def get_connection_scheme(self):
+        """Return the SQLAlchemy connection scheme for the profile."""
+
+        conn_scheme = None
+
+        if self.dbprofile.dbtype == "postgresql":
+            try:
+                import psycopg2  # noqa: F401
+            except ImportError:
+                errmsg_lines = [
+                    "In order to create a postgresql database engine, you must install the following dependencies.",
+                    "DEPENDENCIES",
+                    "    psycopg2",
+                ]
+                errmsg = os.linesep.join(errmsg_lines)
+                raise ConfigurationError(errmsg)
+
+            conn_scheme = "postgresql+psycopg2"
+
+        return conn_scheme
+
+    def format_connection_string(self, dbname: Optional[str] = None) -> str:
+        """Build a connection string for pipe based access."""
+
+        conn_scheme: str = self.get_connection_scheme()
+
+        socket_path: str = self.dbprofile.socket
+
+        if not isinstance(self.credential, BasicCredential):
+            errmsg = (
+                "BasicSocketDatabaseConnectionFactory requires a BasicCredential to "
+                "format the connection string."
+            )
+            raise ConfigurationError(errmsg)
+
+        credobj: BasicCredential = self.credential
+
+        username = credobj.username
+        password = credobj.password
+
+        if dbname is None:
+            dbname = self.dbprofile.dbname
+
+        conn_str = f"{conn_scheme}://{username}:{password}@/{dbname}?host={socket_path}"
+
+        return conn_str
+
+
+class BasicTcpDatabaseConnectionFactory(DatabaseConnectionFactory):
+
+    def __init__(self, profile_name: str, *, dbprofile: DatabaseBasicTcpProfile, credential: BaseCredential):
+        super().__init__(profile_name, dbprofile=dbprofile, credential=credential)
+        return
+
+    @property
+    def dbprofile(self) -> DatabaseBasicTcpProfile:
+        return self._dbprofile
+
+    def create_engine(self, *, dbname=None, echo=True) -> SqlEngine:
+
+        conn_scheme = None
+
+        # If the database type we are creating an engine for is postgresql, make sure
+        # we can import all of the postgresql dependencies.
+        if self.dbprofile.dbtype == 'postgresql':
+            try:
+                import psycopg2
+            except ImportError:
+                errmsg_lines = [
+                    "In order to create a postgresql database engine, you must install the following dependencies.",
+                    "DEPENDENCIES",
+                    "    psycopg2"
+                ]
+                errmsg = os.linesep.join(errmsg_lines)
+                raise ConfigurationError(errmsg)
+            
+            conn_scheme = "postgresql+psycopg2"
+
+        if dbname is None and self.dbprofile.dbname is not None:
+            dbname = self.dbprofile.dbname
+
+        if dbname is None:
+            warnmsg = "BasicTcpDatabaseConnectionFactory.create_engine: dbname was None."
+            logger.warn(warnmsg)
+
+        if isinstance(self.credential, BasicCredential):
+            credobj: BasicCredential = self.credential
+
+            passwd = credobj.password
+
+            connstr = self.format_connection_string(dbname=dbname)
+            
+            dbengine = create_engine(connstr, echo=echo)
+
+        return dbengine
+    
+    def get_connection_scheme(self):
+
+        conn_scheme = None
+
+        # If the database type we are creating an engine for is postgresql, make sure
+        # we can import all of the postgresql dependencies.
+        if self.dbprofile.dbtype == 'postgresql':
+            try:
+                import psycopg2
+            except ImportError:
+                errmsg_lines = [
+                    "In order to create a postgresql database engine, you must install the following dependencies.",
+                    "DEPENDENCIES",
+                    "    psycopg2"
+                ]
+                errmsg = os.linesep.join(errmsg_lines)
+                raise ConfigurationError(errmsg)
+            
+            conn_scheme = "postgresql+psycopg2"
+
+        return conn_scheme
+
+    def format_connection_string(self, dbname: Optional[str]=None) -> str:
+        
+        conn_str = None
+
+        conn_scheme: str = self.get_connection_scheme()
+        host: str = self.dbprofile.host
+        port: str = self.dbprofile.port
+        
+        if isinstance(self.credential, BasicCredential):
+            credobj: BasicCredential = self.credential
+
+            username = credobj.username
+            password = credobj.password
+
+            conn_str = f"{conn_scheme}://{username}:{password}@{host}:{port}"
+
+            if dbname is not None:
+                conn_str += f"/{dbname}"
+            else:
+                conn_str += f"/{self.dbprofile.dbname}"
+
+        return conn_str
+
+
+
+DATABASE_CONNECTION_FACTORIES = {}
+
+
+def create_database_connection_factory(dbprofile: DatabaseBasicSocketProfile, cred_manager: CredentialManager) -> DatabaseConnectionFactory:
+
+    global DATABASE_CONNECTION_FACTORIES
+
+    conn_factory = None
+
+    profile_name = dbprofile.identifier
+
+    if profile_name in DATABASE_CONNECTION_FACTORIES:
+        conn_factory = DATABASE_CONNECTION_FACTORIES[profile_name]
+    
+    else:
+
+        credential_name = dbprofile.credential
+
+        credential: BaseCredential = cred_manager.credentials[credential_name]
+
+        category = dbprofile.category
+        if category == "basic-socket":
+            conn_factory = BasicSocketDatabaseConnectionFactory(profile_name, dbprofile=dbprofile, credential=credential)
+        elif category == "basic-tcp":
+            conn_factory = BasicTcpDatabaseConnectionFactory(profile_name, dbprofile=dbprofile, credential=credential)
+        else:
+            errmsg = "Unknown database connection type. profile_name={} category={}".format(
+                profile_name, category)
+            raise ConfigurationError(errmsg)
+
+        DATABASE_CONNECTION_FACTORIES[profile_name] = conn_factory
+
+    return conn_factory

ccai/pychestrate/core/storage/dbio.py

@@ -0,0 +1,158 @@
+"""
+.. module:: dbio
+    :platform: Darwin, Linux, Unix, Windows
+    :synopsis: Contains helper functions for working with specific databases
+
+.. moduleauthor:: Myron Walker <myron.walker@gmail.com>
+"""
+
+__author__ = "Myron Walker"
+__copyright__ = "Copyright 2025, Automation Mojo LLC"
+__credits__ = []
+__version__ = "1.0.0"
+
+
+from typing import Optional
+
+
+import contextlib
+import logging
+
+from sqlalchemy.engine.base import Engine as SqlEngine
+from sqlalchemy.sql.schema import MetaData
+
+from sqlalchemy_utils import database_exists as sqlutils_database_exists
+from sqlalchemy_utils import create_database as sqlutils_create_database
+from sqlalchemy_utils import drop_database as sqlutils_drop_database
+
+from ccai.errors.exceptions import ConfigurationError
+
+from ccai.credentials.credentialmanager import CredentialManager
+
+from ccai.dataprofiles.databasebasicsocketprofile import DatabaseBasicSocketProfile
+
+from ccai.pychestrate.core.storage.orm import PychestrateModelBase
+
+from ccai.pychestrate.core.storage.dbconnection import DatabaseConnectionFactory, create_database_connection_factory
+
+
+logger = logging.getLogger()
+
+
+def database_exists(conn_factory: DatabaseConnectionFactory, dbname: Optional[str]=None) -> bool:
+    """
+        Checks to see if the specified database exists.
+    """
+    conn_str = conn_factory.format_connection_string(dbname=dbname)
+
+    exists = sqlutils_database_exists(conn_str)
+
+    return exists
+
+
+def database_create(conn_factory: DatabaseConnectionFactory, dbname: Optional[str]=None, dbmetadata: Optional[MetaData]=None):
+    """ 
+        Creates the specified database.
+    """
+
+    conn_str = conn_factory.format_connection_string(dbname=dbname)
+
+    sqlutils_create_database(conn_str)
+
+    if dbmetadata is not None:
+        engine = conn_factory.create_engine(dbname=dbname)
+        dbmetadata.create_all(engine)
+
+    return
+
+
+def create_network_attached_database(db_profile: DatabaseBasicSocketProfile, cred_manager: CredentialManager, orm_metadata: MetaData, *, reset: bool = False):
+    """
+        Creates the 'pychestrate' database.
+    
+        :param db_profile: The database profile configuration.
+        :param cred_manager: Credential manager instance for obtaining credentials.
+        :param orm_metadata: The Object Relational Model to use when creating the database. 
+        :param reset: When ``True`` delete the existing database before creation.
+    """
+    db_name = db_profile.dbname
+
+    conn_factory = create_database_connection_factory(db_profile, cred_manager)
+    if conn_factory is None:
+        db_profile_name = db_profile["name"]
+        errmsg = "'create_network_attached_database' could not get a connection factory for profile={}".format(
+            db_profile_name
+        )
+        raise ConfigurationError(errmsg)
+
+    if reset and database_exists(conn_factory, db_name):
+        conn_str = conn_factory.format_connection_string(dbname=db_name)
+        sqlutils_drop_database(conn_str)
+
+    if not database_exists(conn_factory, db_name):
+        database_create(conn_factory, db_name)
+
+    engine = conn_factory.create_engine(dbname=db_name, echo=True)
+
+    orm_metadata.create_all(engine, checkfirst=True)
+
+    return
+
+
+def open_network_attached_database(db_profile: DatabaseBasicSocketProfile, cred_manager: CredentialManager) -> SqlEngine:
+    """
+        Opens the postgresql database.
+
+        :param db_profile: The database profile configuration.
+        :param cred_manager: Credential manager instance for obtaining credentials.
+        :param orm_metadata: The Object Relational Model to use when creating the database. 
+        :param reset: When ``True`` delete the existing database before creation.
+
+        :returns: A :class:`SqlEngine` object that can be used to interoperate with the specified database.
+    """
+    dbname = db_profile.dbname
+
+    conn_factory = create_database_connection_factory(db_profile, cred_manager)
+    if conn_factory is None:
+        db_profile_name = db_profile["name"]
+        errmsg = "'open_pychestrate_database' could not get a connection factory for profile={}".format(
+            db_profile_name
+        )
+        raise ConfigurationError(errmsg)
+
+    engine = conn_factory.create_engine(dbname=dbname, echo=True)
+
+    return engine
+
+
+def reset_network_attached_database(db_profile: DatabaseBasicSocketProfile, cred_manager: CredentialManager, orm_metadata: MetaData):
+    """
+        Reset a network attached database database.
+
+        :param db_profile: The database profile configuration.
+        :param cred_manager: Credential manager instance for obtaining credentials.
+        :param orm_metadata: The Object Relational Model to use when creating the database. 
+        :param reset: When ``True`` delete the existing database before creation.
+    """
+    dbname = db_profile.dbname
+
+    conn_factory = create_database_connection_factory(db_profile, cred_manager)
+    if conn_factory is None:
+        db_profile_name = db_profile["name"]
+        errmsg = "'reset_pychestrate_database' could not get a connection factory for profile={}".format(
+            db_profile_name
+        )
+        raise ConfigurationError(errmsg)
+
+    if database_exists(conn_factory, dbname):
+    
+        engine = conn_factory.create_engine(dbname=dbname, echo=True)
+
+        with contextlib.closing(engine.connect()) as con:
+            trans = con.begin()
+            con.execute('TRUNCATE {} RESTART IDENTITY;'.format(
+                ','.join(table.name 
+                        for table in reversed(orm_metadata.sorted_tables))))
+            trans.commit()
+
+    return

ccai/pychestrate/core/storage/defaults.py

@@ -0,0 +1,29 @@
+__author__ = "Myron Walker"
+__copyright__ = "Copyright 2025, Automation Mojo LLC"
+__credits__ = []
+
+from uuid import uuid4
+
+from datetime import datetime, timezone, timedelta
+
+def new_uuid() -> str:
+    rtnval = str(uuid4())
+    return rtnval
+
+def utc_now() -> datetime:
+    """
+        Provides the current UTC time. This is used as a default value for datetime fields in models.
+        
+        :return: datetime
+    """
+    now = datetime.now(timezone.utc)
+    return now
+
+def reset_token_expiration() -> datetime:
+    """
+        Provides the expiration time for reset tokens. This is set to 24 hours from now.
+        
+        :return: datetime
+    """
+    expiration = utc_now() + timedelta(days=1)
+    return expiration