dataflow-core 1.0.0__tar.gz

dataflow-core-1.0.0/PKG-INFO

@@ -0,0 +1,10 @@
+Metadata-Version: 1.0
+Name: dataflow-core
+Version: 1.0.0
+Summary: Dataflow core package
+Home-page: UNKNOWN
+Author: Dataflow
+Author-email: UNKNOWN
+License: UNKNOWN
+Description: UNKNOWN
+Platform: UNKNOWN

dataflow-core-1.0.0/README.md

@@ -0,0 +1 @@
+# Dataflow-Core

dataflow-core-1.0.0/authenticator/dataflowairflowauthenticator.py

@@ -0,0 +1,121 @@
+from .package.configuration import ConfigurationManager
+from .package.models.database import DatabaseManager
+from .package.models import (
+    user as m_user,
+    session as m_session
+    )
+
+from typing import Any, Callable
+from airflow.www.security import FabAirflowSecurityManagerOverride
+
+class DataflowAirflowAuthenticator(FabAirflowSecurityManagerOverride):
+    def __init__(self, wsgi_app: Callable) -> None:
+        self.wsgi_app = wsgi_app
+
+        # Dataflow database configuration
+        self.dataflow_config = ConfigurationManager('/dataflow/app/config/dataflow.cfg')
+        self.dataflow_database_url = self.dataflow_config.get_config_value('database', 'database_url')
+
+        self.dataflow_db_instance = DatabaseManager(self.dataflow_database_url)
+        self.dataflow_db = next(self.dataflow_db_instance.get_session())
+
+        # Airflow database configuration
+        self.airflow_config = ConfigurationManager('airflow.cfg')
+        self.airflow_database_url = self.airflow_config.get_config_value('database', 'sql_alchemy_conn')
+        
+        self.airflow_db_instance = DatabaseManager(self.airflow_database_url)
+        self.airflow_db = next(self.airflow_db_instance.get_session())
+
+        m_user.Base.metadata.create_all(bind=self.dataflow_db_instance.get_engine())
+        m_session.Base.metadata.create_all(bind=self.dataflow_db_instance.get_engine())
+
+    def __call__(self, environ: dict, start_response: Callable) -> Any:
+
+        path = environ.get('PATH_INFO', '')
+        if not path == '/login/':
+            return self.wsgi_app(environ, start_response)
+
+        try:
+            # Extracting browser cookies
+            cookies = environ.get('HTTP_COOKIE', '')
+            user_session_id = None
+            parts = cookies.split('; ')
+            for part in parts:
+                if part.startswith('dataflow_session='):
+                    user_session_id = part
+                    break
+
+            if user_session_id is None:
+                raise Exception("No session id found")
+            
+            user_session_id = user_session_id.split('=')[1]
+
+            # Retrieving user details
+            user_data = self.find_dataflow_user(user_session_id)
+
+            if user_data is None:
+                raise Exception("No user found for the dataflow_session id")
+            
+            user = self.find_user(user_data.user_name)
+
+            if not user:
+                user_role = self.find_role(user_data.role.title())
+                user = self.add_user(username=user_data.user_name, first_name=self.not_none(user_data.first_name), last_name=self.not_none(user_data.last_name), email=self.not_none(user_data.email), role=user_role)
+
+            environ['REMOTE_USER'] = user.username
+            self.write_user_id(user_data.user_id)
+            return self.wsgi_app(environ, start_response)
+
+        except Exception as e:
+            return self.wsgi_app(environ, start_response)
+    
+    def not_none(self, value):
+        return value if value is not None else ""
+    
+    def find_dataflow_user(self, user_session_id):
+        """Find user by session_id in dataflow database."""
+        query = self.dataflow_db.query(m_session.Session_table)
+        session = query.filter(m_session.Session_table.session_id == user_session_id).first()
+        if session is None:
+            return None
+
+        user_data = self.dataflow_db.query(m_user.User).filter(m_user.User.user_id == session.user_id).first()
+        if user_data is None:
+            return None
+        
+        return user_data
+
+    def find_user(self, username=None):
+        """Find user by username or email."""
+        return self.airflow_db.query(self.user_model).filter_by(username=username).one_or_none()
+
+    def find_role(self, role):
+        """Find a role in the database."""
+        return self.airflow_db.query(self.role_model).filter_by(name=role).one_or_none()
+
+    def add_user(self, username, first_name, last_name, email, role, password=""):
+        """Create a user."""
+        user = self.user_model()
+        user.first_name = first_name
+        user.last_name = last_name
+        user.username = username
+        user.email = email
+        user.active = True
+        user.roles = role if isinstance(role, list) else [role]
+        user.password = password
+        self.airflow_db.add(user)
+        self.airflow_db.commit()
+        return user
+
+    def write_user_id(self, user_id):
+        """
+        Write the given user_id to a file named dataflow_user_id.txt.
+        
+        Args:
+            user_id (str): The user ID to be written to the file.
+        """
+        file_name = 'dataflow_user_id.txt'
+        with open(file_name, 'w') as file:
+            file.write(str(user_id))
+
+        

dataflow-core-1.0.0/authenticator/dataflowhubauthenticator.py

@@ -0,0 +1,66 @@
+from .package.configuration import ConfigurationManager
+from .package.models.database import DatabaseManager
+from .package.models import (
+    user as m_user,
+    session as m_session
+    )
+
+import uuid
+from jupyterhub.auth import Authenticator
+
+class DataflowHubAuthenticator(Authenticator):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.dataflow_config = ConfigurationManager('/dataflow/app/config/dataflow.cfg')
+        self.database_url = self.dataflow_config.get_config_value('database', 'database_url')
+
+        self.db_instance = DatabaseManager(self.database_url)
+        self.db = next(self.db_instance.get_session())
+
+        m_user.Base.metadata.create_all(bind=self.db_instance.get_engine())
+        m_session.Base.metadata.create_all(bind=self.db_instance.get_engine())
+    
+    def generate_session_id(self):
+        return str(uuid.uuid4())
+    
+    async def authenticate(self, handler, data):
+        # get username and password
+        username = data["username"]
+        password = data["password"]
+
+        try:
+            # check if user exists
+            query = self.db.query(m_user.User)
+            user = query.filter(m_user.User.user_name == username).first()
+
+            if user is not None:
+                # check if password is correct
+                if user.password != password:
+                    return None
+                
+                # generate session_id
+                session_id = self.generate_session_id()
+                query = self.db.query(m_session.Session_table)
+                isSession = query.filter(m_session.Session_table.session_id == session_id).first()
+
+                # if session_id is already exists in the database, generate a new one
+                while isSession is not None:
+                    session_id = self.generate_session_id()
+                    isSession = query.filter(m_session.Session_table.session_id == session_id).first()
+
+                # add session_id to the database
+                db_item = m_session.Session_table(user_id=user.user_id, session_id=session_id)
+                self.db.add(db_item)
+                self.db.commit()
+                self.db.refresh(db_item)
+
+                # return user dictionary and set cookie
+                handler.set_cookie("dataflow_session", session_id)
+                user_dict = {"name": username, "session_id": session_id}
+                return user_dict
+            else:
+                return None
+
+        except Exception as e:
+            return None

dataflow-core-1.0.0/authenticator/package/configuration.py

@@ -0,0 +1,27 @@
+"""configuration.py"""
+import configparser
+from configparser import NoOptionError, NoSectionError
+
+class ConfigurationManager:
+    """
+    Configuration Manager
+    """
+
+    def __init__(self, config_file):
+
+        self.config_file = config_file
+        self.config = configparser.ConfigParser()
+        try:
+            self.config.read(self.config_file)
+
+        except Exception as e:
+            return None
+        
+    def get_config_value(self, section, option):
+        """
+        Get configuration value
+        """
+        try:
+            return self.config.get(section, option)
+        except (NoOptionError, NoSectionError):
+            return None

dataflow-core-1.0.0/authenticator/package/models/database.py

@@ -0,0 +1,32 @@
+"""models/database.py"""
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+class DatabaseManager:
+    def __init__(self, db_url):
+        self.db_url = db_url
+
+    def get_engine(self):
+        try:
+            engine = create_engine(self.db_url)
+            return engine
+        except SQLAlchemyError as e:
+            raise e
+        
+    def get_session(self):
+        try:
+            engine = self.get_engine()
+            session = sessionmaker(autocommit=False, autoflush=False, bind=engine or create_engine(self.db_url))
+            db = session()
+            try:
+                yield db
+            finally:
+                db.close()
+
+        except SQLAlchemyError as e:
+            raise e
+    
+    def get_base(self):
+        return declarative_base()

dataflow-core-1.0.0/authenticator/package/models/session.py

@@ -0,0 +1,20 @@
+"""models.py"""
+from sqlalchemy import Column, Integer, String
+from sqlalchemy.ext.declarative import declarative_base
+
+#instance for create declarative base
+Base=declarative_base()
+
+class Session_table(Base):
+    """
+    Table SESSIONS
+    """
+
+    __tablename__='SESSION'
+
+    id = Column(Integer, primary_key=True, index=True, unique=True, nullable=False, autoincrement=True)
+    session_id = Column(String, unique=True, nullable=False)
+    user_id = Column(String, nullable=False)    
+
+
+    

dataflow-core-1.0.0/authenticator/package/models/user.py

@@ -0,0 +1,23 @@
+"""models.py"""
+from sqlalchemy import Column, Integer, String, LargeBinary, Enum
+from sqlalchemy.ext.declarative import declarative_base
+
+#instance for create declarative base
+Base=declarative_base()
+
+class User(Base):
+    """
+    Table USER
+    """
+
+    __tablename__='USER'
+
+    user_id = Column(Integer, primary_key=True, index=True, autoincrement=True, nullable=False)
+    user_name = Column(String, unique=True, nullable=False)
+    first_name = Column(String)
+    last_name = Column(String)
+    email = Column(String, unique=True)
+    role = Column(Enum('admin', 'user', name='role_field'), nullable=False)
+    image = Column(LargeBinary)
+    active = Column(Enum('N', 'Y', name='active_field'), nullable=False, server_default=str("N"))
+    password = Column(String, nullable=False)

dataflow-core-1.0.0/dataflow/configuration.py

@@ -0,0 +1,27 @@
+"""configuration.py"""
+import configparser
+from configparser import NoOptionError, NoSectionError
+
+class ConfigurationManager:
+    """
+    Configuration Manager
+    """
+
+    def __init__(self, config_file):
+
+        self.config_file = config_file
+        self.config = configparser.ConfigParser()
+        try:
+            self.config.read(self.config_file)
+
+        except Exception as e:
+            return None
+        
+    def get_config_value(self, section, option):
+        """
+        Get configuration value
+        """
+        try:
+            return self.config.get(section, option)
+        except (NoOptionError, NoSectionError):
+            return None

dataflow-core-1.0.0/dataflow/dataflow.py

@@ -0,0 +1,92 @@
+import os
+from .configuration import ConfigurationManager
+from .models import (
+    session as m_session,
+    user as m_user,
+)
+from .models.database import DatabaseManager
+from sqlalchemy.inspection import inspect
+from .utils.aws_secrets_manager import SecretsManagerClient
+import json
+
+
+class Dataflow:
+    def __init__(self):
+        self.dataflow_config = ConfigurationManager('/dataflow/app/config/dataflow.cfg')
+        self.dataflow_database_url = self.dataflow_config.get_config_value('database', 'database_url')
+
+        self.dataflow_db_instance = DatabaseManager(self.dataflow_database_url)
+        self.dataflow_db = next(self.dataflow_db_instance.get_session())
+
+        self.secrets_manager = SecretsManagerClient('us-east-1')
+
+        m_user.Base.metadata.create_all(bind=self.dataflow_db_instance.get_engine())
+        m_session.Base.metadata.create_all(bind=self.dataflow_db_instance.get_engine())
+
+    def auth(self, session_id: str):
+        """Find user by session_id in dataflow database."""
+        try:
+            query = self.dataflow_db.query(m_session.Session_table)
+            session = query.filter(m_session.Session_table.session_id == session_id).first()
+            if session is None:
+                return False
+
+            user_data = self.dataflow_db.query(m_user.User).filter(m_user.User.user_id == session.user_id).first()
+            if user_data is None:
+                return False
+            
+            user_dict = {"user_name": user_data.user_name, "name": f"{user_data.first_name} {user_data.last_name}", "email": user_data.email, "role": user_data.role}
+            return user_dict
+        
+        except Exception as e:
+            return False
+    
+    def variable(self, variable_name: str):
+        """Get variable value from secrets manager."""
+        try:
+            host_name = os.environ["HOSTNAME"]
+            user_name = host_name.replace("jupyter-","")
+            
+            vault_path = "variable"
+            return self.secrets_manager.get_secret_by_key(vault_path, user_name, variable_name)
+            
+        except Exception as e:
+            return None
+        
+    def connection(self, conn_id: str):
+        """Get connection details from secrets manager."""
+        try:
+            host_name = os.environ["HOSTNAME"]
+            user_name=host_name.replace("jupyter-","")
+            
+            vault_path = "connections"
+            secret = self.secrets_manager.get_secret_by_key(vault_path, user_name, conn_id)
+
+            conn_type = secret['conn_type'].lower()
+            username = secret['login']
+            password = secret.get('password', '')
+            host = secret['host']
+            port = secret['port']
+            database = secret.get('schemas', '')
+
+            user_info = f"{username}:{password}@" if password else f"{username}@"
+            db_info = f"/{database}" if database else ""
+
+            connection_string = f"{conn_type}://{user_info}{host}:{port}{db_info}"
+
+            extra = secret.get('extra', '')
+            if extra:
+                try:
+                    extra_params = json.loads(extra)
+                    if extra_params:
+                        extra_query = "&".join(f"{key}={value}" for key, value in extra_params.items())
+                        connection_string += f"?{extra_query}"
+                except json.JSONDecodeError:
+                    # If 'extra' is not valid JSON, skip adding extra parameters
+                    pass
+
+            connection_instance = DatabaseManager(connection_string)
+            return next(connection_instance.get_session())
+        
+        except Exception as e:
+            return None

dataflow-core-1.0.0/dataflow/models/database.py

@@ -0,0 +1,32 @@
+"""models/database.py"""
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+class DatabaseManager:
+    def __init__(self, db_url):
+        self.db_url = db_url
+
+    def get_engine(self):
+        try:
+            engine = create_engine(self.db_url)
+            return engine
+        except SQLAlchemyError as e:
+            raise e
+        
+    def get_session(self):
+        try:
+            engine = self.get_engine()
+            session = sessionmaker(autocommit=False, autoflush=False, bind=engine or create_engine(self.db_url))
+            db = session()
+            try:
+                yield db
+            finally:
+                db.close()
+
+        except SQLAlchemyError as e:
+            raise e
+    
+    def get_base(self):
+        return declarative_base()

dataflow-core-1.0.0/dataflow/models/session.py

@@ -0,0 +1,17 @@
+"""models.py"""
+from sqlalchemy import Column, Integer, String
+from sqlalchemy.ext.declarative import declarative_base
+
+#instance for create declarative base
+Base=declarative_base()
+
+class Session_table(Base):
+    """
+    Table SESSIONS
+    """
+
+    __tablename__='SESSION'
+
+    id = Column(Integer, primary_key=True, index=True, unique=True, nullable=False, autoincrement=True)
+    session_id = Column(String, unique=True, nullable=False)
+    user_id = Column(String, nullable=False)    

dataflow-core-1.0.0/dataflow/models/user.py

@@ -0,0 +1,23 @@
+"""models.py"""
+from sqlalchemy import Column, Integer, String, LargeBinary, Enum
+from sqlalchemy.ext.declarative import declarative_base
+
+#instance for create declarative base
+Base=declarative_base()
+
+class User(Base):
+    """
+    Table USER
+    """
+
+    __tablename__='USER'
+
+    user_id = Column(Integer, primary_key=True, index=True, autoincrement=True, nullable=False)
+    user_name = Column(String, unique=True, nullable=False)
+    first_name = Column(String)
+    last_name = Column(String)
+    email = Column(String, unique=True)
+    role = Column(Enum('admin', 'user', name='role_field'), nullable=False)
+    image = Column(LargeBinary)
+    active = Column(Enum('N', 'Y', name='active_field'), nullable=False, server_default=str("N"))
+    password = Column(String, nullable=False)

dataflow-core-1.0.0/dataflow/utils/aws_secrets_manager.py

@@ -0,0 +1,64 @@
+import boto3
+from botocore.exceptions import ClientError, EndpointConnectionError, NoCredentialsError
+import json
+
+class SecretsManagerClient:
+    """
+    A class to interact with AWS Secrets Manager for managing secrets.
+
+    Args:
+        region_name (str): The AWS region name.
+
+    Attributes:
+        client: The Boto3 client for Secrets Manager.
+        json_handler: An instance of JsonHandler for handling JSON operations.
+    """
+    def __init__(self, region_name: str):
+        try:
+            self.client = boto3.client('secretsmanager', region_name=region_name)
+        except EndpointConnectionError as e:
+            self.logger.error(f"Failed to initialize SecretsManagerClient: {e}")
+            raise Exception(f"Failed to initialize SecretsManagerClient: Unable to connect to the endpoint. {e}")
+        except NoCredentialsError as e:
+            self.logger.error(f"Failed to initialize SecretsManagerClient: {e}")
+            raise Exception(f"Failed to initialize SecretsManagerClient: No AWS credentials found. {e}")
+
+
+    def get_secret_by_key(self, vault_path, user_name, secret_key: str):
+        """
+        Get information about a specific secret.
+
+        Args:
+            vault_path (str): The vault path.
+            user_name (str): The user name.
+            secret_key (str): The key of the secret to retrieve.
+
+        Returns:
+            str: Information about the secret in JSON format.
+
+        Raises:
+            Exception: If the operation fails.
+        """
+        try:
+            if not user_name:
+                raise Exception("user_name is required when secret_key is provided")
+
+            secret_name = f"{user_name}/{vault_path}/{secret_key}"
+            response = self.client.get_secret_value(SecretId=secret_name)
+            secret_metadata = self.client.describe_secret(SecretId=secret_name)
+            secret_data = json.loads(response.get('SecretString'))
+
+            if secret_data.get('is_active') == 'Y':
+                secret_info={
+                    "Name": secret_key,
+                    "Description": secret_metadata.get('Description')
+                }
+                secret_info.update(secret_data)
+                return secret_info
+            else:
+                raise Exception(f"Secret named '{secret_key}' is not active")
+        except ClientError as e:
+            if e.response['Error']['Code'] == 'ResourceNotFoundException':
+                raise Exception(f"Secret named '{secret_key}' not found")
+            else:
+                raise Exception(f"Failed to get secret '{secret_key}': {e}")

dataflow-core-1.0.0/dataflow_core.egg-info/PKG-INFO

@@ -0,0 +1,10 @@
+Metadata-Version: 1.0
+Name: dataflow-core
+Version: 1.0.0
+Summary: Dataflow core package
+Home-page: UNKNOWN
+Author: Dataflow
+Author-email: UNKNOWN
+License: UNKNOWN
+Description: UNKNOWN
+Platform: UNKNOWN

dataflow-core-1.0.0/dataflow_core.egg-info/SOURCES.txt

@@ -0,0 +1,26 @@
+README.md
+setup.py
+authenticator/__init__.py
+authenticator/dataflowairflowauthenticator.py
+authenticator/dataflowhubauthenticator.py
+authenticator/package/__init__.py
+authenticator/package/configuration.py
+authenticator/package/models/__init__.py
+authenticator/package/models/database.py
+authenticator/package/models/session.py
+authenticator/package/models/user.py
+dataflow/__init__.py
+dataflow/configuration.py
+dataflow/dataflow.py
+dataflow/models/__init__.py
+dataflow/models/database.py
+dataflow/models/session.py
+dataflow/models/user.py
+dataflow/utils/__init__.py
+dataflow/utils/aws_secrets_manager.py
+dataflow_core.egg-info/PKG-INFO
+dataflow_core.egg-info/SOURCES.txt
+dataflow_core.egg-info/dependency_links.txt
+dataflow_core.egg-info/entry_points.txt
+dataflow_core.egg-info/requires.txt
+dataflow_core.egg-info/top_level.txt

dataflow-core-1.0.0/dataflow_core.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

dataflow-core-1.0.0/dataflow_core.egg-info/entry_points.txt

@@ -0,0 +1,3 @@
+[jupyterhub.authenticators]
+dataflow_authenticator = authenticator.dataflowhubauthenticator:DataflowHubAuthenticator
+

dataflow-core-1.0.0/dataflow_core.egg-info/requires.txt

@@ -0,0 +1,2 @@
+sqlalchemy
+boto3

dataflow-core-1.0.0/dataflow_core.egg-info/top_level.txt

@@ -0,0 +1,2 @@
+authenticator
+dataflow

dataflow-core-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

dataflow-core-1.0.0/setup.py

@@ -0,0 +1,21 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="dataflow-core",
+    version="1.0.0",
+    packages=find_packages(exclude=["tests", "tests.*"]),
+    include_package_data=True,
+    package_data={},
+    install_requires=[
+        'sqlalchemy',
+        'boto3',
+    ],
+    author="Dataflow",
+    author_email="",
+    description="Dataflow core package",
+    entry_points={
+        'jupyterhub.authenticators': [
+            'dataflow_authenticator = authenticator.dataflowhubauthenticator:DataflowHubAuthenticator',
+        ],
+    },
+)