inoopa_utils 1.0.0__py3-none-any.whl

inoopa_utils-1.0.0.dist-info/METADATA

@@ -0,0 +1,15 @@
+Metadata-Version: 2.1
+Name: inoopa_utils
+Version: 1.0.0
+Summary: Collection of python utils we use at Inoopa.
+Project-URL: Homepage, https://bitbucket.org/inoopa/inoopa_utils/
+Project-URL: Bug Tracker, https://bitbucket.org/inoopa/inoopa_utils/jira
+Author-email: Inoopa <dev@inoopa.com>
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+
+# Inoopa's helpers
+
+This repo contains helper functions we use in all of our python projects.
+
+# This is pushed publicly to Pypi, so NEVER commit any secret here!

inoopa_utils-1.0.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+src/env_variables_helper.py,sha256=kBhK1uv5SGXprq6esCYms1QZRylxuTvgEOImvAQfGsY,537
+src/exceptions.py,sha256=8porK_kXp5c6eVOSPgrGyOFjC27eAz3R6dD1JkbGRQM,144
+src/inoopa_logging.py,sha256=iLAzPjjDUAB0L6l7UiI5lSoLCrK0y9BVGH-jRYrhgxU,1397
+src/rabbitmq_helpers.py,sha256=cq6LJtxcggSgVXLLrP1AhL6Xxu3C2PRWuqrqqGbHJXc,3179
+src/sql_helpers.py,sha256=9Q5pEtxXO61RfwFD030POkonOk-qstfZ5l6pmB2YK1I,4717
+inoopa_utils-1.0.0.dist-info/METADATA,sha256=yynkZ9eR4Yg7gkr3xIUKH2Nykcva6VCIUkPaEMJtVAc,515
+inoopa_utils-1.0.0.dist-info/WHEEL,sha256=9QBuHhg6FNW7lppboF2vKVbCGTVzsFykgRQjjlajrhA,87
+inoopa_utils-1.0.0.dist-info/RECORD,,

inoopa_utils-1.0.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.18.0
+Root-Is-Purelib: true
+Tag: py3-none-any

src/env_variables_helper.py

@@ -0,0 +1,19 @@
+import os
+from typing import Literal
+
+from exceptions import MissingEnvVariable
+
+
+EnvName = Literal["dev", "staging", "prod"]
+
+def get_env_name() -> EnvName:
+    env = os.getenv("ENV")
+    
+    if env is None:
+        raise MissingEnvVariable("Could not get 'ENV' env variable, therefore can't deduce logging_level.")
+    elif env in ["dev","staging", "prod"]:
+        return env
+    else:
+        raise ValueError(
+            f"'ENV' is set to: {env}. This is not a supported env! Please choose between: [dev, staging, prod]"
+        )

src/exceptions.py

@@ -0,0 +1,8 @@
+"""List all our custom exceptions."""
+
+
+class MissingEnvVariable(Exception):
+    """Raised if a critical env variable is missing."""
+
+    pass
+

src/inoopa_logging.py

@@ -0,0 +1,39 @@
+"""Base Inoopa configuration for python Logging."""
+import logging
+from typing import Literal
+from datetime import datetime
+
+from env_variables_helper import get_env_name
+
+
+LoggingLevel = Literal["CRITICAL","FATAL","ERROR","WARN","WARNING","INFO","DEBUG"]
+
+def create_logger(logger_name: str, logging_level: LoggingLevel | None = None) -> logging.Logger:
+    """
+    Configure how logging should be done.
+    
+    :param logger_name: The logger name to return.
+    :param logging_level: The level of logging to filter. If none, will deduce from "ENV" env variable:
+        'dev' will set logging_level to "DEBUG"
+        'staging' will set logging_level to "INFO"
+        'prod' will set logging_level to "INFO"
+    """
+    logging.basicConfig(
+        # Define the log level for externals (libs) loggers
+        level=logging.ERROR,
+        handlers=[
+            # Write logs to file
+            logging.FileHandler(f"logs/{datetime.now().strftime('%d-%m-%Y_%H:%M')}.log"),
+            # Allow the logger to also log in console
+            logging.StreamHandler(),
+        ],
+        format="%(asctime)s %(levelname)-8s %(name)-20s -> %(message)s",
+        datefmt="%d/%m/%Y %H:%M:%S",
+    )
+    
+    if logging_level is None:
+        logging_level = "DEBUG" if get_env_name() == "dev" else "INFO"
+
+    logger = logging.getLogger(logger_name)
+    logger.setLevel(logging_level)
+    return logger

src/rabbitmq_helpers.py

@@ -0,0 +1,75 @@
+
+import os
+import rabbitpy
+from typing import List, Tuple
+
+from inoopa_logging import create_logger
+from exceptions import MissingEnvVariable
+
+_logger = create_logger("INOOPA.UTILS.RABBITMQ_HELPERS")
+
+
+def _get_credentials_from_env() -> Tuple[str, str, str]:
+    env_variables = {
+        "queue_host": os.getenv("QUEUE_HOST"),
+        "queue_username": os.getenv("QUEUE_USERNAME"),
+        "queue_password": os.getenv("QUEUE_PASSWORD"),
+    }
+
+    missing_variables = []
+    for key, value in env_variables.items():
+        if value is None:
+            _logger.error(f"QUEUE env variable missing: {key}")
+            missing_variables.append(key)
+    if len(missing_variables) > 0:
+        raise MissingEnvVariable(f"Missing env variables: {missing_variables}")
+
+    return env_variables["queue_host"], env_variables['queue_username'], env_variables['queue_password']
+
+def _get_queue_name_with_env(queue_name: str) -> str:
+    """Helper function to add the env name to the queue name."""
+    env_name = os.getenv("ENV")
+    if env_name is None:
+        raise MissingEnvVariable("Missing env variable: ENV")
+    return f"{queue_name}_{env_name}"
+
+def get_messages_from_queue(queue_name: str, queue_batch_size: int = 1) -> List[str]:
+    """Helper function to get X messages from a rabbitMQ queue."""
+    host, username, password = _get_credentials_from_env()
+    queue_name_with_env = _get_queue_name_with_env(queue_name)
+    messages_body = []
+    _logger.info(f"Connecting to queue: {queue_name_with_env}...")
+    with rabbitpy.Connection(f"amqp://{username}:{password}@{host}/%2f") as conn:
+        with conn.channel() as channel:
+            queue = rabbitpy.Queue(channel, queue_name_with_env, durable=True)
+            queue.declare()
+            _logger.info("Connected to queue, reading messages...")
+            for i in range(queue_batch_size):
+                message = queue.get(acknowledge=True)
+                if message:
+                    messages_body.append(message.body.decode())
+                    # Tell the queue that this message has been read and should be removed from queue
+                    message.ack()
+                    _logger.info(f"Message {i+1}/{queue_batch_size} read")
+                else:
+                    _logger.info("Queue empty, stoping...")
+                    break
+    return messages_body
+
+def push_to_queue(queue_name: str, message: str) -> None:
+    """Helper function to push a message to a rabbitMQ queue."""
+    host, username, password = _get_credentials_from_env()
+    queue_name_with_env = _get_queue_name_with_env(queue_name)
+
+    _logger.info(f"Connecting to queue: {queue_name_with_env}...")
+    with rabbitpy.Connection(f"amqp://{username}:{password}@{host}/%2f") as conn:
+        with conn.channel() as channel:
+            queue = rabbitpy.Queue(channel, queue_name_with_env, durable=True)
+            queue.declare()
+            
+            # Create a new message
+            msg = rabbitpy.Message(channel, message)
+            
+            # Publish the message to the specified queue
+            msg.publish('', queue_name_with_env)
+            _logger.info(f"Message pushed to {queue_name_with_env}.")

src/sql_helpers.py

@@ -0,0 +1,100 @@
+import os
+from urllib.parse import quote_plus
+from sqlalchemy import Engine, create_engine, text
+
+from exceptions import MissingEnvVariable
+from inoopa_logging import create_logger
+from pandas import DataFrame
+
+
+_logger = create_logger("INOOPA.UTILS.SQL_HELPERS")
+
+def get_env_name():
+    return os.getenv("ENV", "dev").lower()
+
+
+def create_sql_engine(mysql_ca_cert_path: str = "/tmp/mysql_ca_cert.cert") -> Engine:
+    """
+    Create a sqlalchemy engine to use to send SQL requests.
+
+    :param mysql_database: A string with the name of the database to connect to. Default is 'master' the prod database.
+    :param mysql_ca_certificate_path: Path where the sql ca certificate should be written to as a file.
+    :return: A sqlalchemy engine.
+    """
+    connection_string = _get_sql_connection_string(mysql_ca_cert_path)
+    return create_engine(connection_string, connect_args={'ssl_ca': mysql_ca_cert_path})
+
+def _get_sql_connection_string(mysql_ca_certificate_path: str) -> str:
+    """
+    Create a connection string from env variables to connect to the MYSQL database.
+    Also create a mysql ca certificate file at specified path.
+
+    :param mysql_database: A string with the name of the database to connect to.
+    :param mysql_ca_certificate_path: Path where the sql ca certificate should be written to as a file.
+    :return: A formatted connection string for sql database
+    """
+    env_variables = {
+        "mysql_host": os.getenv("MYSQL_HOST"),
+        "mysql_username": os.getenv("MYSQL_USERNAME"),
+        "mysql_password": os.getenv("MYSQL_PASSWORD"),
+        "mysql_database_name": os.getenv(f"MYSQL_DATABASE_NAME"),
+        "mysql_ca_certificate": os.getenv("MYSQL_CA_CERTIFICATE"),
+    }
+
+    missing_variables = []
+    for key, value in env_variables.items():
+        if value is None:
+            _logger.error(f"MYSQL env variable missing: {key}")
+            missing_variables.append(key)
+    if len(missing_variables) > 0:
+        raise MissingEnvVariable(f"Missing env variables: {missing_variables}")
+
+    # Create a ca certificate file to use to create your sql engine
+    # This is required with our DB security config
+    with open(mysql_ca_certificate_path, "w") as file:
+        # Convert the literal '\n' to the actual newline character
+        env_variables["mysql_ca_certificate"] = env_variables["mysql_ca_certificate"].replace('\\n', '\n')
+        file.write(env_variables["mysql_ca_certificate"])
+    _logger.info(f"MYSQL_CA_CERTIFICATE written to: {mysql_ca_certificate_path}")
+
+    return f"mysql+pymysql://{env_variables['mysql_username']}:{quote_plus(env_variables['mysql_password'])}@{env_variables['mysql_host']}/{env_variables['mysql_database_name']}"
+
+def update_df_rows_in_db(df: DataFrame, table_name: str, engine: Engine, primary_key: str = "id") -> None:
+    """
+    !!! READ THE FULL DOCSTRING BEFORE USAGE !!!
+    Update rows in a given SQL table based on the contents of a pandas DataFrame.
+
+    This function will update rows in the specified table based on the DataFrame's 
+    rows using the "id" column as the primary key. Only the columns present in 
+    the DataFrame will be updated, leaving other columns in the database untouched.
+
+    :param df: (pandas.DataFrame): The DataFrame containing the data to update. 
+      It should have an "id" column which acts as the primary key.
+    :param table_name: The name of the SQL table to update.
+
+    Notes:
+    - We only use this because we don't manipulate much the data in the Dataframe. we follows this pattern: 
+        - Get the data from db pd.Dataframe.read_sql(query)
+        - Update a few fields
+        - Push back the data to DB
+    - It is assumed that "id" is the primary key in both the DataFrame and the table.
+    - Be cautious when using this function to ensure that the DataFrame's structure 
+      matches the table's structure, especially in terms of column data types.
+    """
+    with engine.begin() as conn:
+        for _, row in df.iterrows():
+            # Construct an update statement
+            update_query = f"UPDATE {table_name} SET "
+
+            # Add each column to the update statement
+            update_parts = [f"`{col}` = :{col}" for col in df.columns if col != primary_key]
+            update_query += ", ".join(update_parts)
+            update_query += f" WHERE {primary_key} = :{primary_key}"
+
+            # Prepare the dictionary with the values from the row
+            values = {col: row[col] for col in df.columns if col != primary_key}
+            values[primary_key] = row[primary_key]
+
+            prepared_query = text(update_query).bindparams(**values)
+            _logger.debug(f"Update query: {prepared_query}")
+            conn.execute(prepared_query)