inoopa_utils 1.0.0__tar.gz

inoopa_utils-1.0.0/.gitignore

@@ -0,0 +1,53 @@
+# These are some examples of commonly ignored file patterns.
+# You should customize this list as applicable to your project.
+# Learn more about .gitignore:
+#     https://www.atlassian.com/git/tutorials/saving-changes/gitignore
+
+# Node artifact files
+node_modules/
+dist/
+*__pycache__*
+.env
+venv
+
+# Compiled Java class files
+*.class
+
+# Compiled Python bytecode
+*.py[cod]
+
+# Log files
+*.log
+
+# Package files
+*.jar
+
+# Maven
+target/
+dist/
+
+# JetBrains IDE
+.idea/
+
+# Unit test reports
+TEST*.xml
+
+# Generated by MacOS
+.DS_Store
+
+# Generated by Windows
+Thumbs.db
+
+# Applications
+*.app
+*.exe
+*.war
+
+# Large media files
+*.mp4
+*.tiff
+*.avi
+*.flv
+*.mov
+*.wmv
+

inoopa_utils-1.0.0/PKG-INFO

@@ -0,0 +1,15 @@
+Metadata-Version: 2.1
+Name: inoopa_utils
+Version: 1.0.0
+Summary: Collection of python utils we use at Inoopa.
+Project-URL: Homepage, https://bitbucket.org/inoopa/inoopa_utils/
+Project-URL: Bug Tracker, https://bitbucket.org/inoopa/inoopa_utils/jira
+Author-email: Inoopa <dev@inoopa.com>
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+
+# Inoopa's helpers
+
+This repo contains helper functions we use in all of our python projects.
+
+# This is pushed publicly to Pypi, so NEVER commit any secret here!

inoopa_utils-1.0.0/README.md

@@ -0,0 +1,5 @@
+# Inoopa's helpers
+
+This repo contains helper functions we use in all of our python projects.
+
+# This is pushed publicly to Pypi, so NEVER commit any secret here!

inoopa_utils-1.0.0/pyproject.toml

@@ -0,0 +1,18 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "inoopa_utils"
+version = "1.0.0"
+authors = [
+  { name="Inoopa", email="dev@inoopa.com" },
+]
+description = "Collection of python utils we use at Inoopa."
+readme = "README.md"
+requires-python = ">=3.11"
+
+
+[project.urls]
+"Homepage" = "https://bitbucket.org/inoopa/inoopa_utils/"
+"Bug Tracker" = "https://bitbucket.org/inoopa/inoopa_utils/jira"

inoopa_utils-1.0.0/requirements.txt

@@ -0,0 +1,15 @@
+iniconfig==2.0.0
+numpy==1.26.1
+packaging==23.2
+pamqp==2.3.0
+pandas==2.1.2
+pluggy==1.3.0
+PyMySQL==1.1.0
+pytest==7.4.3
+python-dateutil==2.8.2
+pytz==2023.3.post1
+rabbitpy==2.0.1
+six==1.16.0
+SQLAlchemy==2.0.22
+typing_extensions==4.8.0
+tzdata==2023.3

inoopa_utils-1.0.0/src/env_variables_helper.py

@@ -0,0 +1,19 @@
+import os
+from typing import Literal
+
+from exceptions import MissingEnvVariable
+
+
+EnvName = Literal["dev", "staging", "prod"]
+
+def get_env_name() -> EnvName:
+    env = os.getenv("ENV")
+    
+    if env is None:
+        raise MissingEnvVariable("Could not get 'ENV' env variable, therefore can't deduce logging_level.")
+    elif env in ["dev","staging", "prod"]:
+        return env
+    else:
+        raise ValueError(
+            f"'ENV' is set to: {env}. This is not a supported env! Please choose between: [dev, staging, prod]"
+        )

inoopa_utils-1.0.0/src/exceptions.py

@@ -0,0 +1,8 @@
+"""List all our custom exceptions."""
+
+
+class MissingEnvVariable(Exception):
+    """Raised if a critical env variable is missing."""
+
+    pass
+

inoopa_utils-1.0.0/src/inoopa_logging.py

@@ -0,0 +1,39 @@
+"""Base Inoopa configuration for python Logging."""
+import logging
+from typing import Literal
+from datetime import datetime
+
+from env_variables_helper import get_env_name
+
+
+LoggingLevel = Literal["CRITICAL","FATAL","ERROR","WARN","WARNING","INFO","DEBUG"]
+
+def create_logger(logger_name: str, logging_level: LoggingLevel | None = None) -> logging.Logger:
+    """
+    Configure how logging should be done.
+    
+    :param logger_name: The logger name to return.
+    :param logging_level: The level of logging to filter. If none, will deduce from "ENV" env variable:
+        'dev' will set logging_level to "DEBUG"
+        'staging' will set logging_level to "INFO"
+        'prod' will set logging_level to "INFO"
+    """
+    logging.basicConfig(
+        # Define the log level for externals (libs) loggers
+        level=logging.ERROR,
+        handlers=[
+            # Write logs to file
+            logging.FileHandler(f"logs/{datetime.now().strftime('%d-%m-%Y_%H:%M')}.log"),
+            # Allow the logger to also log in console
+            logging.StreamHandler(),
+        ],
+        format="%(asctime)s %(levelname)-8s %(name)-20s -> %(message)s",
+        datefmt="%d/%m/%Y %H:%M:%S",
+    )
+    
+    if logging_level is None:
+        logging_level = "DEBUG" if get_env_name() == "dev" else "INFO"
+
+    logger = logging.getLogger(logger_name)
+    logger.setLevel(logging_level)
+    return logger

inoopa_utils-1.0.0/src/rabbitmq_helpers.py

@@ -0,0 +1,75 @@
+
+import os
+import rabbitpy
+from typing import List, Tuple
+
+from inoopa_logging import create_logger
+from exceptions import MissingEnvVariable
+
+_logger = create_logger("INOOPA.UTILS.RABBITMQ_HELPERS")
+
+
+def _get_credentials_from_env() -> Tuple[str, str, str]:
+    env_variables = {
+        "queue_host": os.getenv("QUEUE_HOST"),
+        "queue_username": os.getenv("QUEUE_USERNAME"),
+        "queue_password": os.getenv("QUEUE_PASSWORD"),
+    }
+
+    missing_variables = []
+    for key, value in env_variables.items():
+        if value is None:
+            _logger.error(f"QUEUE env variable missing: {key}")
+            missing_variables.append(key)
+    if len(missing_variables) > 0:
+        raise MissingEnvVariable(f"Missing env variables: {missing_variables}")
+
+    return env_variables["queue_host"], env_variables['queue_username'], env_variables['queue_password']
+
+def _get_queue_name_with_env(queue_name: str) -> str:
+    """Helper function to add the env name to the queue name."""
+    env_name = os.getenv("ENV")
+    if env_name is None:
+        raise MissingEnvVariable("Missing env variable: ENV")
+    return f"{queue_name}_{env_name}"
+
+def get_messages_from_queue(queue_name: str, queue_batch_size: int = 1) -> List[str]:
+    """Helper function to get X messages from a rabbitMQ queue."""
+    host, username, password = _get_credentials_from_env()
+    queue_name_with_env = _get_queue_name_with_env(queue_name)
+    messages_body = []
+    _logger.info(f"Connecting to queue: {queue_name_with_env}...")
+    with rabbitpy.Connection(f"amqp://{username}:{password}@{host}/%2f") as conn:
+        with conn.channel() as channel:
+            queue = rabbitpy.Queue(channel, queue_name_with_env, durable=True)
+            queue.declare()
+            _logger.info("Connected to queue, reading messages...")
+            for i in range(queue_batch_size):
+                message = queue.get(acknowledge=True)
+                if message:
+                    messages_body.append(message.body.decode())
+                    # Tell the queue that this message has been read and should be removed from queue
+                    message.ack()
+                    _logger.info(f"Message {i+1}/{queue_batch_size} read")
+                else:
+                    _logger.info("Queue empty, stoping...")
+                    break
+    return messages_body
+
+def push_to_queue(queue_name: str, message: str) -> None:
+    """Helper function to push a message to a rabbitMQ queue."""
+    host, username, password = _get_credentials_from_env()
+    queue_name_with_env = _get_queue_name_with_env(queue_name)
+
+    _logger.info(f"Connecting to queue: {queue_name_with_env}...")
+    with rabbitpy.Connection(f"amqp://{username}:{password}@{host}/%2f") as conn:
+        with conn.channel() as channel:
+            queue = rabbitpy.Queue(channel, queue_name_with_env, durable=True)
+            queue.declare()
+            
+            # Create a new message
+            msg = rabbitpy.Message(channel, message)
+            
+            # Publish the message to the specified queue
+            msg.publish('', queue_name_with_env)
+            _logger.info(f"Message pushed to {queue_name_with_env}.")

inoopa_utils-1.0.0/src/sql_helpers.py

@@ -0,0 +1,100 @@
+import os
+from urllib.parse import quote_plus
+from sqlalchemy import Engine, create_engine, text
+
+from exceptions import MissingEnvVariable
+from inoopa_logging import create_logger
+from pandas import DataFrame
+
+
+_logger = create_logger("INOOPA.UTILS.SQL_HELPERS")
+
+def get_env_name():
+    return os.getenv("ENV", "dev").lower()
+
+
+def create_sql_engine(mysql_ca_cert_path: str = "/tmp/mysql_ca_cert.cert") -> Engine:
+    """
+    Create a sqlalchemy engine to use to send SQL requests.
+
+    :param mysql_database: A string with the name of the database to connect to. Default is 'master' the prod database.
+    :param mysql_ca_certificate_path: Path where the sql ca certificate should be written to as a file.
+    :return: A sqlalchemy engine.
+    """
+    connection_string = _get_sql_connection_string(mysql_ca_cert_path)
+    return create_engine(connection_string, connect_args={'ssl_ca': mysql_ca_cert_path})
+
+def _get_sql_connection_string(mysql_ca_certificate_path: str) -> str:
+    """
+    Create a connection string from env variables to connect to the MYSQL database.
+    Also create a mysql ca certificate file at specified path.
+
+    :param mysql_database: A string with the name of the database to connect to.
+    :param mysql_ca_certificate_path: Path where the sql ca certificate should be written to as a file.
+    :return: A formatted connection string for sql database
+    """
+    env_variables = {
+        "mysql_host": os.getenv("MYSQL_HOST"),
+        "mysql_username": os.getenv("MYSQL_USERNAME"),
+        "mysql_password": os.getenv("MYSQL_PASSWORD"),
+        "mysql_database_name": os.getenv(f"MYSQL_DATABASE_NAME"),
+        "mysql_ca_certificate": os.getenv("MYSQL_CA_CERTIFICATE"),
+    }
+
+    missing_variables = []
+    for key, value in env_variables.items():
+        if value is None:
+            _logger.error(f"MYSQL env variable missing: {key}")
+            missing_variables.append(key)
+    if len(missing_variables) > 0:
+        raise MissingEnvVariable(f"Missing env variables: {missing_variables}")
+
+    # Create a ca certificate file to use to create your sql engine
+    # This is required with our DB security config
+    with open(mysql_ca_certificate_path, "w") as file:
+        # Convert the literal '\n' to the actual newline character
+        env_variables["mysql_ca_certificate"] = env_variables["mysql_ca_certificate"].replace('\\n', '\n')
+        file.write(env_variables["mysql_ca_certificate"])
+    _logger.info(f"MYSQL_CA_CERTIFICATE written to: {mysql_ca_certificate_path}")
+
+    return f"mysql+pymysql://{env_variables['mysql_username']}:{quote_plus(env_variables['mysql_password'])}@{env_variables['mysql_host']}/{env_variables['mysql_database_name']}"
+
+def update_df_rows_in_db(df: DataFrame, table_name: str, engine: Engine, primary_key: str = "id") -> None:
+    """
+    !!! READ THE FULL DOCSTRING BEFORE USAGE !!!
+    Update rows in a given SQL table based on the contents of a pandas DataFrame.
+
+    This function will update rows in the specified table based on the DataFrame's 
+    rows using the "id" column as the primary key. Only the columns present in 
+    the DataFrame will be updated, leaving other columns in the database untouched.
+
+    :param df: (pandas.DataFrame): The DataFrame containing the data to update. 
+      It should have an "id" column which acts as the primary key.
+    :param table_name: The name of the SQL table to update.
+
+    Notes:
+    - We only use this because we don't manipulate much the data in the Dataframe. we follows this pattern: 
+        - Get the data from db pd.Dataframe.read_sql(query)
+        - Update a few fields
+        - Push back the data to DB
+    - It is assumed that "id" is the primary key in both the DataFrame and the table.
+    - Be cautious when using this function to ensure that the DataFrame's structure 
+      matches the table's structure, especially in terms of column data types.
+    """
+    with engine.begin() as conn:
+        for _, row in df.iterrows():
+            # Construct an update statement
+            update_query = f"UPDATE {table_name} SET "
+
+            # Add each column to the update statement
+            update_parts = [f"`{col}` = :{col}" for col in df.columns if col != primary_key]
+            update_query += ", ".join(update_parts)
+            update_query += f" WHERE {primary_key} = :{primary_key}"
+
+            # Prepare the dictionary with the values from the row
+            values = {col: row[col] for col in df.columns if col != primary_key}
+            values[primary_key] = row[primary_key]
+
+            prepared_query = text(update_query).bindparams(**values)
+            _logger.debug(f"Update query: {prepared_query}")
+            conn.execute(prepared_query)

inoopa_utils-1.0.0/src/tests/test_rabbitmq.py

@@ -0,0 +1,35 @@
+import unittest
+
+from rabbitmq_helpers import get_messages_from_queue, push_to_queue
+
+
+class IntegrationTestRabbitMQHelpers(unittest.TestCase):
+    """Test class to ensure pushing & reading from queue works as expected."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Setup once for all tests"""
+        cls.queue_name = "updaters_integration_tests"
+
+    def test_push_and_get_message(self):
+        """Push a message and then get it from the queue"""
+
+        # Given a message
+        test_message = "Integration test message"
+
+        # When pushing the message to the queue
+        push_to_queue(self.queue_name, test_message)
+
+        # Then it should be retrieved from the queue
+        messages = get_messages_from_queue(self.queue_name, queue_batch_size=1)
+        self.assertIn(test_message, messages)
+
+    @classmethod
+    def tearDownClass(cls):
+        """Cleanup once after all tests"""
+        # Optionally, you can clear the queue if required
+        while get_messages_from_queue(cls.queue_name, queue_batch_size=1):
+            pass
+
+if __name__ == '__main__':
+    unittest.main()