bdrc-db-lib2 2.0.1__py3-none-any.whl

BdrcDbLib/DBConfig.py

@@ -0,0 +1,80 @@
+"""
+Created on Mar 7, 2018
+
+@author: jsk
+"""
+import configparser
+import os
+from pathlib import Path
+
+class DBConfig:
+    """
+    :summary: Prepares database connection options from a single config file and environment variables.
+    The config file path is obtained from the BDRC_DB_CNF environment variable with no fallback.
+    Passwords are NEVER extracted from the config file and must be provided via environment variables.
+    """
+
+    def __init__(self, dbName, configFileName=None):
+        """
+        :param dbName: section in the config file. Saved in db_alias
+        :param configFileName: Ignored. Path is taken from BDRC_DB_CNF environment variable.
+        """
+        self.db_alias = dbName
+        cnf_path = os.getenv('BDRC_DB_CNF')
+        if not cnf_path:
+            raise OSError("Environment variable BDRC_DB_CNF is not set. It must point to the database configuration file.")
+        
+        self._configFQPath = os.path.expanduser(cnf_path)
+        self._configParser = configparser.ConfigParser()
+        if not self._configParser.read(self._configFQPath):
+            raise FileNotFoundError(f"Database configuration file not found at {self._configFQPath}")
+
+    @property
+    def db_alias(self):
+        return self._serverSection
+
+    @db_alias.setter
+    def db_alias(self, value):
+        self._serverSection = value
+
+    @property
+    def db_host(self):
+        """Returns the section name in the config file (for compatibility)."""
+        return self.db_alias
+
+    @property
+    def db_cnf(self):
+        """Returns the path to the config file."""
+        return self._configFQPath
+
+    def get_value(self, section: str, key: str):
+        """
+        Gets a value from the config, with environment variable overrides.
+        'section' is ignored in favor of the instance's db_alias.
+        """
+        key = key.lower()
+        
+        if key == 'password':
+            # Always from environment
+            val = os.getenv(f"BDRC_{self.db_alias.upper()}_PASSWORD")
+            if not val:
+                val = os.getenv("BDRC_DB_PASSWORD")
+            if not val:
+                raise ValueError(f"Password for section [{self.db_alias}] not found in environment (checked BDRC_{self.db_alias.upper()}_PASSWORD and BDRC_DB_PASSWORD)")
+            return val
+
+        if not self._configParser.has_section(self.db_alias):
+            raise ValueError(f"Section [{self.db_alias}] not found in config file {self._configFQPath}")
+        
+        if self._configParser.has_option(self.db_alias, key):
+            return self._configParser.get(self.db_alias, key)
+            
+        # Default for port, overridden by file value
+        if key == 'port':
+            return '3306'
+            
+        raise ValueError(f"Key '{key}' not found in section [{self.db_alias}] and no environment override found.")
+
+    @property
+    def config_file_name(self): 
+        return self._configFQPath

BdrcDbLib/DbApp.py

@@ -0,0 +1,262 @@
+"""
+Created 2018-VIII-24
+@author: jimk
+"""
+import logging
+import os
+import random
+import sys
+import time
+
+import pymysql
+import pymysql as mysql
+
+from BdrcDbLib.DBConfig import DBConfig
+from BdrcDbLib.SprocColumnError import SprocColumnError
+
+
+# create
+def retry(max_retries=3, sleep_range=(7, 15)):
+    """
+    a python decorator that calls a function and retries
+    it a set number of times, sleeping a random number of seconds
+    between 7 and 15 seconds, until a retry limit is reached or the
+    operation succeeds. On success, return the argument's results'
+    :param max_retries: number of retries
+    :param sleep_range: range of seconds to sleep between retries
+    :return: the function's return value
+    """
+
+    def decorator(func):
+        def wrapper(*args, **kwargs):
+            log:logging = logging.getLogger(__name__)
+            for i in range(max_retries):
+                try:
+                    log.info(f"Attempt {i + 1} of {max_retries} for {func.__name__}")
+                    return func(*args, **kwargs)
+                except Exception as e:
+                    log.error(f"Attempt {i + 1} of {max_retries} failed: {e}")
+                    if i < max_retries - 1:
+                        time.sleep(random.randint(*sleep_range))
+                    else:
+                        raise e
+        return wrapper
+    return decorator
+
+
+class DbApp:
+    """
+    Base class for database applications
+    """
+    _invoked_object: str
+    _dbConfig: DBConfig
+    _cn: mysql.Connection
+    _dbConnection: mysql.Connection
+    _expectedColumns: list = None
+    _log: logging
+
+    def __init__(self, db_config: str):
+        self.dbConfig = db_config
+        self.connection = None
+        self.ExpectedColumns = []
+        self._log = logging.getLogger(__name__)
+        logging.basicConfig(datefmt='[ %Y-%m-%d %X ]', format='%(asctime)s%(levelname)s:%(message)s',
+                            level=logging.INFO)
+
+    def start_connect(self) -> None:
+        """
+        Opens a database connection using the DBConfig
+        :return: Nothing. Sets class connection property
+        """
+        user = self.dbConfig.get_value(None, "user")
+        password = self.dbConfig.get_value(None, "password")
+        host = self.dbConfig.get_value(None, "host")
+        port = self.dbConfig.get_value(None, "port")
+        database = self.dbConfig.get_value(None, "database")
+
+        self.connection = mysql.connect(
+            user=user,
+            password=password,
+            host=host,
+            port=int(port),
+            database=database,
+            charset='utf8'
+        )
+
+    @property
+    def ExpectedColumns(self) -> list:
+        """
+        If a subclass returns a result set from a query, you may only want some of the query columns
+        If this list is empty, all the data set columns are returned
+        :return:
+        """
+        return self._expectedColumns
+
+    @ExpectedColumns.setter
+    def ExpectedColumns(self, value: list):
+        assert isinstance(value, list)
+        self._expectedColumns = value
+
+    @property
+    def dbConfig(self) -> DBConfig:
+        return self._dbConfig
+
+    @dbConfig.setter
+    def dbConfig(self, drs_db_config: str):
+        """
+        gets dbConfig values for setup
+        :param drs_db_config: the database alias (optionally alias:ignored_file)
+        :return:
+        """
+        if drs_db_config is None:
+            # noinspection PyTypeChecker
+            self._dbConfig = None
+            return
+
+        db_name = drs_db_config.split(':')[0]
+        self._dbConfig = DBConfig(db_name)
+
+    @property
+    def connection(self) -> mysql.Connection:
+        return self._cn
+
+    @connection.setter
+    def connection(self, value):
+        self._cn = value
+
+    def validateExpectedColumns(self, cursor_description: list) -> None:
+        """
+        Validates the cursor after a call to the database. Checks for
+        the required columns (from member ExpectedColumns) in the output
+
+        :param cursor_description: tuple of tuples
+        :return: Throws ValueError on fail
+        """
+        found = False
+        found_columns: list = []
+
+        # Data expected but not returned
+        if cursor_description is None and len(self.ExpectedColumns) > 0:
+            raise SprocColumnError(f'Invoked object {self._invoked_object} returned no expected data.')
+
+        for expected_column in self.ExpectedColumns:
+            found = False
+            for cursor_tuple in cursor_description:
+                if cursor_tuple[0] == expected_column:
+                    found = True
+                    found_columns.append(cursor_tuple[0])
+                    break
+            # each expected column must be in the list
+            if not found:
+                break
+        if not found:
+            raise SprocColumnError(
+                f'Invoked object {self._invoked_object} Expected to return columns {self.ExpectedColumns}. Only '
+                f'returned {found_columns}.')
+
+        # desc = queryCursor.description
+        # hope something's here
+
+    def GetSprocResults(self, sproc: str, max_works: int = 200) -> list:
+        """
+        call a sproc using the internal connection,
+        validate the result columns with the internal member.
+
+        :rtype: list of dictionary objects of results. Caller decodes format
+        :param sproc: routine to call
+        :param max_works: limit of return rows
+        :returns: a list of dictionary items, each item is a return row
+        """
+
+        self._invoked_object = sproc
+        self.start_connect()
+
+        rl: list[dict] = []
+
+        has_next: bool = True
+        with self.connection:
+            try:
+                # jimk #drs-deposit 76. Dont use unbuffered cursor, which blocks access to the db
+                # until it's done or cleared. (e.g. pyCharm queries
+                work_cursor: mysql.Connection.Cursor = self.connection.cursor(mysql.cursors.DictCursor)
+                print(f'Calling {sproc} for n = {max_works} ')
+                work_cursor.callproc(f'{sproc}', (max_works,))
+                self.validateExpectedColumns(work_cursor.description)
+
+                while has_next:
+                    result_rows = work_cursor.fetchall()
+                    rl.append(result_rows)
+                    has_next = work_cursor.nextset()
+            finally:
+                # have to drain result sets if there was an exception (if
+                while has_next:
+                    work_cursor.fetchall()
+                    has_next = work_cursor.nextset()
+        return rl
+
+    @retry(max_retries=3, sleep_range=(15, 40))
+    def CallAnySproc(self, sproc: str, *args) -> []:
+        """
+        Calls a routine without analyzing the result
+        :param sproc: routine name
+        :param out_arg_index:  If  there is an out arg, its index in the tuple of args
+        :param args: arguments
+        :return: true if there are any results, throws exception otherwise.
+        Caller handles
+        """
+        self.start_connect()
+
+        rl: [] = []
+
+        with self.connection:
+
+            work_cursor: mysql.Connection.Cursor = self.connection.cursor(mysql.cursors.DictCursor)
+
+            sql_args = tuple(arg for arg in args)
+            self._log.debug(f"Calling {sproc} args {':'.join(str(sql_arg) for sql_arg in sql_args)}")
+            try:
+                work_cursor.callproc(f'{sproc}', sql_args)
+                has_next: bool = True
+                while has_next:
+                    result_rows = work_cursor.fetchall()
+                    rl.append(result_rows)
+                    has_next = work_cursor.nextset()
+                self.connection.commit()
+            except pymysql.Error as e:
+                self._log.error("Error calling", exc_info=sys.exc_info())
+                self.connection.rollback()
+                raise e
+        return rl
+
+    def ExecQuery(self, query: str, *args) -> []:
+        """
+        Calls a routine without analyzing the result
+        :param sproc: routine name
+        :param out_arg_index:  If  there is an out arg, its index in the tuple of args
+        :param args: arguments
+        :return: true if there are any results, throws exception otherwise.
+        Caller handles
+        """
+        self.start_connect()
+
+        rl: [] = []
+
+        with self.connection:
+
+            work_cursor: mysql.Connection.Cursor = self.connection.cursor(mysql.cursors.DictCursor)
+
+            sql_args = tuple(arg for arg in args)
+            self._log.debug(f"Calling query args {':'.join(str(sql_arg) for sql_arg in sql_args)}")
+            try:
+                work_cursor.execute(query, sql_args)
+                has_next: bool = True
+                while has_next:
+                    result_rows = work_cursor.fetchall()
+                    rl.append(result_rows)
+                    has_next = work_cursor.nextset()
+                self.connection.commit()
+            except pymysql.Error as e:
+                self._log.error("Error invoking", exc_info=sys.exc_info())
+                self.connection.rollback()
+                raise e
+        return rl

BdrcDbLib/DbAppParser.py

@@ -0,0 +1,118 @@
+"""
+Base class for all parsers for Db Applications
+"""
+import argparse
+import pathlib
+import datetime
+import os
+
+
+class DbArgNamespace:
+    """
+    Empty arguments, holds output of arg parsing
+    """
+    pass
+
+
+class DbAppParser:
+    """
+    Base class for database arguments. When a subclass calls
+    argparse.parseArguments, this class returns a structure containing
+    a member drsDbConfig: str
+    """
+
+    _default_config: str = 'prod'
+    _parser: argparse.ArgumentParser = None
+
+    _args: DbArgNamespace = None
+
+    def __init__(self, description: str, usage: str):
+        self._parser = argparse.ArgumentParser(description=description,
+                                               usage="%(prog)s | -d DBAppSection " + usage)
+        self._parser.add_argument('-d', '--drsDbConfig', help='specify database alias (section in config file)', required=False,
+                                  default=self._default_config)
+
+    @property
+    def parsedArgs(self) -> DbArgNamespace:
+        """
+        Readonly, calc once
+        parses the classes arguments, and returns the namespace
+        :return:
+        """
+        # Enforce once only
+        if self._args is None:
+            self._args = DbArgNamespace()
+            # noinspection PyTypeChecker
+            self._parser.parse_args(namespace=self._args)
+        return self._args
+
+
+# section parser validations and utilities
+
+
+def str2date(arg: str) -> datetime.datetime:
+    """
+    parses date given in yyyy-mm-dd
+    """
+    return datetime.datetime.strptime(arg, "%Y-%m-%d")
+
+
+def str2datetime(arg: str) -> datetime.datetime:
+    """
+    parses date given as in bash  date +"%Y-%m-%d %R:%S",
+    or 2021-05-24 5:3.22
+    """
+    return datetime.datetime.strptime(arg, "%Y-%m-%d %H:%M:%S")
+
+
+def X_writableExpandoFile(path: str):
+    """
+    argparse type for a file in a writable directory
+    :param path:
+    :return:
+    """
+
+    os_path = os.path.expanduser(path)
+    p = pathlib.Path(os_path)
+    if os.path.isdir(os_path):
+        raise argparse.ArgumentTypeError(f"{os_path} is a directory. A file name is required.")
+
+    # Is the parent writable?
+    p_dir = p.parent
+    if not os.access(str(p_dir), os.W_OK):
+        raise argparse.ArgumentTypeError(f"{os_path} is in a readonly directory ")
+
+    return path
+
+
+# We don't need existing directories and files any
+def X_mustExistDirectory(path: str):
+    """
+    Argparse type specifying a string which represents
+    an existing file path
+    :param path:
+    :return:
+    """
+    if not os.path.isdir(path):
+        raise argparse.ArgumentTypeError(f"{path} not found")
+    for root, dirs, files in os.walk(path, True):
+        if len(dirs) == 0:
+            raise argparse.ArgumentTypeError
+        else:
+            return path
+
+
+def X_mustExistFile(path: str):
+    """
+    Common utility. Returns
+    :param path:
+    :return:
+    """
+    full_path = os.path.expanduser(path)
+    if not os.path.exists(full_path):
+        raise argparse.ArgumentTypeError(f"{full_path} not found")
+    else:
+        return full_path
+
+
+# end section parser validations and utilities