DataOpsHub 6.2.0__py3-none-any.whl

DataOpsHub/databases/upload_data.py

@@ -0,0 +1,118 @@
+"""
+**Note**
+
+This program is an application of the main module 'database_handler'
+in the sub-package 'databases', and it uses some of its attributes and/or functions.
+YOU MAY REDISTRIBUTE this program along any other directory,
+but keep in mind that the module is designed to work with absolute paths.
+
+Purpose
+-------
+- This program loads data from various file types (CSV, Excel, ODS) into a MySQL database.
+- The function 'upload_file_data' from 'database_handler' is utilised for this task.
+- Parameters can be modified to suit different data files and database configurations.
+
+For 'dtype_dict_obj' Usage, please refer the docs of parameter 'dtype_dict'
+at 'upload_file_data' function
+
+Data Type Options
+-----------------
+Refer to the following table for mapping Pandas data types to MySQL data types:
+
++--------------------------+---------------------+
+|     Pandas Data Type     |   MySQL Data Type   |
++--------------------------+---------------------+
+| `int64`                  | `BIGINT`            |
+| `int32`                  | `INTEGER`           |
+| `float64`                | `DOUBLE`            |
+| `float32`                | `FLOAT`             |
+| `bool`                   | `BOOLEAN`           |
+| `datetime64`             | `DATETIME`          |
+| `timedelta[{time_unit}]` | `TIME`              |
+| `object`                 | `TEXT` or `VARCHAR` |
+| `category`               | `VARCHAR`           |
++--------------------------+---------------------+
+
+Data Files
+----------
+- The 'input_file_obj' variable can be assigned a single file path or a list of paths.
+- Alternatively, 'find_files' from the 'path_utils' module can be used to search for files,
+  either for glob strings or for extensions.
+"""
+
+#------------------------#
+# Import project modules #
+#------------------------#
+
+from DataOpsHub.databases.database_handler import upload_file_data
+from filewise.file_operations.path_utils import find_files 
+from paramlib.config_params import DATABASE_CREDENTIALS
+
+#-------------------#
+# Define parameters #
+#-------------------#
+
+# Database type #
+#---------------#
+
+DATABASE_TYPE = "mysql"
+
+# Table parameters #
+#------------------#
+
+TABLE_PARAM_DICT = dict(
+    if_exists="replace",
+    import_index=False,
+    separator="\t", 
+    header=0,
+    parser_engine=None,
+    decimal=","
+    )
+
+# Data type dictionary to apply for one or more sheets #
+#------------------------------------------------------#
+
+DTYPE_DICT_OBJ = {
+    'column1': 'int64',
+    'column2': 'float64',
+    'column3': 'object'
+}
+
+# DTYPE_DICT_OBJ = [
+#     {
+#         'file1_column1': 'int64',
+#         'file1_column2': 'float64',
+#         'file1_column3': 'object'
+#     },
+#     {
+#         'file2_column1': 'int64',
+#         'file2_column2': 'float64',
+#         'file2_column3': 'object'
+#     },
+# ]
+
+# DTYPE_DICT_OBJ = None
+
+# Data files #
+#------------#
+
+INPUT_FILE_OBJ = "/home/jonander/Documents/gordetegiak/pygenutils.databases/test.csv"
+
+# INPUT_FILE_OBJ = [
+#     "/home/jonander/Documents/gordetegiak/pygenutils/databases/test1.csv",
+#     "/home/jonander/Documents/gordetegiak/pygenutils/databases/test2.csv",
+#     "/home/jonander/Documents/gordetegiak/pygenutils/databases/test3.csv"
+#     ]
+
+# INPUT_FILE_OBJ = find_files("test*", search_path=".", match_type="glob", top_path_only=True)
+# INPUT_FILE_OBJ = find_files(".csv", search_path=".", match_type="ext", top_path_only=True)
+
+#-------------#
+# Run program #
+#-------------#
+
+upload_file_data(INPUT_FILE_OBJ,
+                 DATABASE_CREDENTIALS, 
+                 database_type=DATABASE_TYPE, 
+                 **TABLE_PARAM_DICT,
+                 dtype_dict=DTYPE_DICT_OBJ)

DataOpsHub/models/__init__.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+Models package for DataOpsHub.
+
+This package contains all the data models used in the application.
+"""
+
+# Define what should be available when using 'from DataOpsHub.models import *'
+__all__ = [
+    'BaseModel',
+    'LabResults',
+    'MedicationsHistory',
+    'User',
+    'VitalSigns'
+] 

DataOpsHub/models/base.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+Base Model Module
+
+This module defines the BaseModel class for the DataOpsHub application.
+"""
+
+# Import modules #
+#----------------#
+
+from sqlalchemy import Column, String
+from sqlalchemy.orm import declarative_base
+
+# Declare objects #
+#-----------------#
+
+Base = declarative_base()
+
+# Define classes #
+#----------------#
+
+class BaseModel(Base):
+    """
+    Base model class with common fields and methods for all models.
+    
+    This class serves as a base for all other model classes in the application.
+    It provides common fields and methods that are shared across all models.
+    """
+    __abstract__ = True
+    
+    # Common fields that all models should have
+    hl7_v2 = Column(String, nullable=True)
+    
+    @classmethod
+    def get_user_id_field(cls):
+        """
+        Get the user ID field for this model.
+        
+        Returns
+        -------
+        sqlalchemy.Column
+            The user ID column for this model.
+        """
+        from ..constants.table_mappings import TABLE_FIELD_MAPPING
+        if cls.__tablename__ in TABLE_FIELD_MAPPING:
+            field_name = TABLE_FIELD_MAPPING[cls.__tablename__]['id_field']
+            if field_name:
+                return getattr(cls, field_name)
+        return None
+    
+    @classmethod
+    def get_date_field(cls):
+        """
+        Get the date field for this model.
+        
+        Returns
+        -------
+        sqlalchemy.Column
+            The date column for this model.
+        """
+        from ..constants.table_mappings import TABLE_FIELD_MAPPING
+        if cls.__tablename__ in TABLE_FIELD_MAPPING:
+            field_name = TABLE_FIELD_MAPPING[cls.__tablename__]['date_field']
+            if field_name:
+                return getattr(cls, field_name)
+        return None
+
+    @classmethod
+    def register_model(cls, table_name: str):
+        """
+        Register the model in the MODEL_REGISTRY.
+        
+        Parameters
+        ----------
+        table_name : str
+            The name of the table this model represents.
+        """
+        from ..databases.database_handler import MODEL_REGISTRY
+        MODEL_REGISTRY[table_name] = cls
+        return cls  # Return the class to allow for decorator-style usage if needed 

DataOpsHub/models/user.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+User Model Module
+
+This module defines the User model for the DataOpsHub application.
+"""
+
+# Import modules #
+#----------------#
+
+from datetime import datetime
+
+from sqlalchemy import Column, DateTime, Float, ForeignKey, Integer, String, Text
+from sqlalchemy.orm import relationship
+
+# Import project modules #
+#------------------------#
+
+from DataOpsHub.models.base import BaseModel
+
+# Define classes #
+#----------------#
+
+class User(BaseModel):
+    """
+    User model class representing a user in the system.
+    """
+    __tablename__ = 'users'
+
+    user_id = Column(String(10), primary_key=True)
+    first_name = Column(String(50), nullable=False)
+    last_name = Column(String(50), nullable=False)
+    date_of_birth = Column(DateTime, nullable=False)
+    gender = Column(String(10), nullable=False)
+    email = Column(String(100), unique=True, nullable=False)
+    phone = Column(String(15))
+    address = Column(Text)
+    created_at = Column(DateTime, default=datetime.utcnow)
+    updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
+
+    # Relationships
+    vital_signs = relationship("VitalSigns", back_populates="user")
+    lab_results = relationship("LabResults", back_populates="user")
+    medications_history = relationship("MedicationsHistory", back_populates="user")
+
+    def __repr__(self):
+        return f"<User(user_id='{self.user_id}', name='{self.first_name} {self.last_name}')>"
+
+# Register the model
+User.register_model('users')
+
+class VitalSigns(BaseModel):
+    """
+    Model for storing user vital signs measurements.
+    """
+    __tablename__ = 'vital_signs'
+
+    id = Column(Integer, primary_key=True)
+    user_id = Column(String(10), ForeignKey('users.user_id'))
+    timestamp = Column(DateTime, default=datetime.utcnow)
+    heart_rate = Column(Integer)
+    blood_pressure = Column(String(20))
+    temperature = Column(Float)
+    oxygen_saturation = Column(Integer)
+
+    user = relationship("User", back_populates="vital_signs")
+
+# Register the model
+VitalSigns.register_model('vital_signs')
+
+class LabResults(BaseModel):
+    """
+    Model for storing user laboratory test results.
+    """
+    __tablename__ = 'lab_results'
+
+    id = Column(Integer, primary_key=True)
+    user_id = Column(String(10), ForeignKey('users.user_id'))
+    timestamp = Column(DateTime, default=datetime.utcnow)
+    test_name = Column(String(100))
+    result = Column(String(100))
+    unit = Column(String(20))
+    reference_range = Column(String(50))
+
+    user = relationship("User", back_populates="lab_results")
+
+# Register the model
+LabResults.register_model('lab_results')
+
+class MedicationsHistory(BaseModel):
+    """
+    Model for tracking user medication history.
+    """
+    __tablename__ = 'medications_history'
+
+    id = Column(Integer, primary_key=True)
+    user_id = Column(String(10), ForeignKey('users.user_id'))
+    timestamp = Column(DateTime, default=datetime.utcnow)
+    medication_name = Column(String(100))
+    dosage = Column(String(50))
+    frequency = Column(String(50))
+    start_date = Column(DateTime)
+    end_date = Column(DateTime)
+
+    user = relationship("User", back_populates="medications_history")
+
+# Register the model
+MedicationsHistory.register_model('medications_history') 

DataOpsHub/security/__init__.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+Security package for DataOpsHub.
+
+This package contains security-related functionality.
+"""
+
+# Define what should be available when using 'from DataOpsHub.security import *'
+__all__ = [
+    'SecurityConfig',
+    'SecurityError',
+    'generate_password'
+]

DataOpsHub/security/automated_passwords.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+#----------------#
+# Import modules #
+#----------------#
+
+import numpy as np
+
+#------------------#
+# Define functions #
+#------------------#
+
+def generate_automated_password(lower_char_idx,
+                                upper_char_idx,
+                                password_length):
+
+    # Parameters #
+    #------------#
+    
+    # Output info elements #
+    key_expl = "Automatically generated password"
+    lke = len(key_expl)
+    hashtag_underliner = '#' * lke
+    
+    # List delimiter #
+    delim = ""
+    
+    # Construct the password #
+    #------------------------#
+    
+    # Get the corresponding unicode string list #
+    chars_glob = [chr(i) for i in range(lower_char_idx, upper_char_idx)]
+    lcg = len(chars_glob)
+    
+    # Define random numbers ranging from 0 to the length of the list #
+    random_idx = np.random.randint(lcg, size=password_length)
+    
+    # Select unicode strings according to these random numbers #
+    char_randomsel_list = [chars_glob[j] for j in random_idx]
+    
+    # Convert the list to string #
+    passwd = delim.join(char_randomsel_list)
+    
+    # Output the password with formatted printing #
+    passwd_str = f"{key_expl}\n{hashtag_underliner}\n{passwd}"
+    return passwd_str
+    
+#--------------------#
+# Call the functions #
+#--------------------#
+
+automated_passwd = generate_automated_password(31, 127, 16)
+print(automated_passwd)

DataOpsHub/security/config.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+Configuration Module
+
+This module provides configuration settings for the DataOpsHub application.
+"""
+
+import os
+from datetime import timedelta
+
+# Database configuration
+DATABASE_TYPE = os.getenv('DATABASE_TYPE', 'mysql')
+DATABASE_HOST = os.getenv('DATABASE_HOST', 'localhost')
+DATABASE_PORT = os.getenv('DATABASE_PORT', '3306')
+DATABASE_NAME = os.getenv('DATABASE_NAME', 'dataopshub')
+DATABASE_USER = os.getenv('DATABASE_USER', 'root')
+DATABASE_PASSWORD = os.getenv('DATABASE_PASSWORD', '')
+
+# Security configuration
+SECRET_KEY = os.getenv('SECRET_KEY', 'your-secret-key-here')
+JWT_SECRET_KEY = os.getenv('JWT_SECRET_KEY', 'your-jwt-secret-key-here')
+JWT_ACCESS_TOKEN_EXPIRES = timedelta(hours=1)
+JWT_REFRESH_TOKEN_EXPIRES = timedelta(days=30)
+PASSWORD_RESET_TOKEN_EXPIRES = timedelta(hours=1)
+
+# Application configuration
+DEBUG = os.getenv('DEBUG', 'False').lower() == 'true'
+TESTING = os.getenv('TESTING', 'False').lower() == 'true'
+ENVIRONMENT = os.getenv('ENVIRONMENT', 'development')
+
+# API configuration
+API_PREFIX = '/api/v1'
+API_TITLE = 'DataOpsHub API'
+API_VERSION = '1.0.0'
+API_DESCRIPTION = 'API for the DataOpsHub application'
+
+# CORS configuration
+CORS_ORIGINS = os.getenv('CORS_ORIGINS', '*').split(',')
+
+# Logging configuration
+LOG_LEVEL = os.getenv('LOG_LEVEL', 'INFO')
+LOG_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+LOG_FILE = os.getenv('LOG_FILE', 'dataopshub.log')
+
+# Email configuration
+SMTP_HOST = os.getenv('SMTP_HOST', 'smtp.gmail.com')
+SMTP_PORT = int(os.getenv('SMTP_PORT', '587'))
+SMTP_USER = os.getenv('SMTP_USER', '')
+SMTP_PASSWORD = os.getenv('SMTP_PASSWORD', '')
+SMTP_USE_TLS = os.getenv('SMTP_USE_TLS', 'True').lower() == 'true'
+EMAIL_FROM = os.getenv('EMAIL_FROM', 'noreply@dataopshub.com')
+
+# File upload configuration
+UPLOAD_FOLDER = os.getenv('UPLOAD_FOLDER', 'uploads')
+MAX_CONTENT_LENGTH = int(os.getenv('MAX_CONTENT_LENGTH', '16777216'))  # 16MB
+ALLOWED_EXTENSIONS = {'txt', 'pdf', 'png', 'jpg', 'jpeg', 'gif'}
+
+# Cache configuration
+CACHE_TYPE = os.getenv('CACHE_TYPE', 'simple')
+CACHE_DEFAULT_TIMEOUT = int(os.getenv('CACHE_DEFAULT_TIMEOUT', '300'))  # 5 minutes
+CACHE_REDIS_HOST = os.getenv('CACHE_REDIS_HOST', 'localhost')
+CACHE_REDIS_PORT = int(os.getenv('CACHE_REDIS_PORT', '6379'))
+CACHE_REDIS_PASSWORD = os.getenv('CACHE_REDIS_PASSWORD', '')
+CACHE_REDIS_DB = int(os.getenv('CACHE_REDIS_DB', '0'))
+
+# Rate limiting configuration
+RATELIMIT_DEFAULT = os.getenv('RATELIMIT_DEFAULT', '200 per day')
+RATELIMIT_STORAGE_URL = os.getenv('RATELIMIT_STORAGE_URL', 'memory://')
+
+# Session configuration
+SESSION_TYPE = os.getenv('SESSION_TYPE', 'filesystem')
+SESSION_FILE_DIR = os.getenv('SESSION_FILE_DIR', 'flask_session')
+SESSION_PERMANENT = os.getenv('SESSION_PERMANENT', 'False').lower() == 'true'
+PERMANENT_SESSION_LIFETIME = timedelta(days=1)
+
+# Security headers configuration
+SECURITY_HEADERS = {
+    'X-Content-Type-Options': 'nosniff',
+    'X-Frame-Options': 'SAMEORIGIN',
+    'X-XSS-Protection': '1; mode=block',
+    'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
+    'Content-Security-Policy': "default-src 'self'",
+    'Referrer-Policy': 'strict-origin-when-cross-origin'
+} 

DataOpsHub/security/exceptions.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+Exceptions Module
+
+This module defines custom exceptions for the DataOpsHub application.
+"""
+
+class DataOpsHubError(Exception):
+    """
+    Base exception class for DataOpsHub.
+    """
+    pass
+
+class DatabaseError(DataOpsHubError):
+    """
+    Exception raised for database-related errors.
+    """
+    pass
+
+class ValidationError(DataOpsHubError):
+    """
+    Exception raised for validation errors.
+    """
+    pass
+
+class AuthenticationError(DataOpsHubError):
+    """
+    Exception raised for authentication errors.
+    """
+    pass
+
+class AuthorisationError(DataOpsHubError):
+    """
+    Exception raised for authorisation errors.
+    """
+    pass
+
+class NotFoundError(DataOpsHubError):
+    """
+    Exception raised when a requested resource is not found.
+    """
+    pass
+
+class ConflictError(DataOpsHubError):
+    """
+    Exception raised when there is a conflict with an existing resource.
+    """
+    pass
+
+class ServiceError(DataOpsHubError):
+    """
+    Exception raised for service-related errors.
+    """
+    pass 

DataOpsHub/services/__init__.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+Services package for DataOpsHub.
+
+This package contains all the service classes used in the application.
+"""
+
+# Define what should be available when using 'from DataOpsHub.services import *'
+__all__ = [
+    'DataRetrievalService',
+    'UserService'
+]