dataflow-core 2.0.11__tar.gz → 2.1.0__tar.gz

{dataflow_core-2.0.11 → dataflow_core-2.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataflow-core
-Version: 2.0.11
+Version: 2.1.0
 Summary: Dataflow core package
 Author: Dataflow
 Author-email: 

dataflow_core-2.1.0/authenticator/dataflowairflowauthenticator.py

@@ -0,0 +1,58 @@
+from flask import redirect, request
+from flask_appbuilder.security.views import AuthDBView
+from flask_appbuilder.security.views import expose
+from flask_login import login_user
+from airflow.www.security import FabAirflowSecurityManagerOverride
+from dataflow.dataflow import Dataflow
+import logging
+
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    datefmt='%Y-%m-%d %H:%M:%S'
+)
+logger = logging.getLogger(__name__)
+
+dataflow = Dataflow()
+
+class DataflowAuthDBView(AuthDBView):    
+    @expose('/login/', methods=['GET', 'POST'])
+    def login(self):
+        """
+        Override the default login method to handle custom authentication
+        """
+        try:
+            session_id = request.cookies.get('dataflow_session')
+            if not session_id:
+                logger.info("No session cookie found, falling back to standard login.")
+                return super().login()
+            
+            user_details = dataflow.auth(session_id)
+            logger.info(f"User details retrieved for: {user_details['user_name']}")
+            user = self.appbuilder.sm.find_user(username=user_details['user_name'])
+            if user:
+                logger.info(f"User found: {user}")
+                login_user(user, remember=False)
+            else:
+                user = self.appbuilder.sm.add_user(
+                    username=user_details['user_name'], 
+                    first_name=user_details.get("first_name", ""),
+                    last_name=user_details.get("last_name", ""), 
+                    email=user_details.get("email", ""), 
+                    role=self.appbuilder.sm.find_role(user_details.get("base_role", "user").title())
+                )
+                logger.info(f"New user created: {user}")
+                if user:
+                    login_user(user, remember=False)
+            
+            return redirect(self.appbuilder.get_url_for_index)
+
+        except Exception as e:
+            logger.error(f"Login failed: {e}")
+            return super().login()
+
+class DataflowAirflowAuthenticator(FabAirflowSecurityManagerOverride):
+    authdbview = DataflowAuthDBView
+    
+    def __init__(self, appbuilder):
+        super().__init__(appbuilder)

{dataflow_core-2.0.11 → dataflow_core-2.1.0}/authenticator/dataflowhubauthenticator.py

@@ -1,6 +1,5 @@
-from .package.configuration import ConfigurationManager
-from .package.models.database import DatabaseManager
-from .package.models import (
+from dataflow.db import get_db
+from dataflow.models import (
     user as m_user,
     session as m_session
     )
@@ -11,15 +10,7 @@ from jupyterhub.auth import Authenticator
 class DataflowHubAuthenticator(Authenticator):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
-
-        self.dataflow_config = ConfigurationManager('/dataflow/app/config/dataflow.cfg')
-        self.database_url = self.dataflow_config.get_config_value('database', 'database_url')
-
-        self.db_instance = DatabaseManager(self.database_url)
-        self.db = next(self.db_instance.get_session())
-
-        m_user.Base.metadata.create_all(bind=self.db_instance.get_engine())
-        m_session.Base.metadata.create_all(bind=self.db_instance.get_engine())
+        self.db = next(get_db())
     
     def generate_session_id(self):
         return str(uuid.uuid4())
@@ -39,8 +30,8 @@ class DataflowHubAuthenticator(Authenticator):
 
             # Check if the user already has an existing session
             existing_session = (
-                self.db.query(m_session.Session_table)
-                .filter(m_session.Session_table.user_id == str(user.user_id))
+                self.db.query(m_session.Session)
+                .filter(m_session.Session.user_id == user.user_id)
                 .first()
             )
 
@@ -50,16 +41,16 @@ class DataflowHubAuthenticator(Authenticator):
             else:
                 # Generate a new session_id
                 session_id = self.generate_session_id()
-                query = self.db.query(m_session.Session_table)
-                isSession = query.filter(m_session.Session_table.session_id == session_id).first()
+                query = self.db.query(m_session.Session)
+                isSession = query.filter(m_session.Session.session_id == session_id).first()
 
                 # If session_id(uuid string) already exists in the database, generate a new one
                 while isSession is not None:
                     session_id = self.generate_session_id()
-                    isSession = query.filter(m_session.Session_table.session_id == session_id).first()
+                    isSession = query.filter(m_session.Session.session_id == session_id).first()
 
                 # add session_id to the database
-                db_item = m_session.Session_table(user_id=user.user_id, session_id=session_id)
+                db_item = m_session.Session(user_id=user.user_id, session_id=session_id)
                 self.db.add(db_item)
                 self.db.commit()
                 self.db.refresh(db_item)
@@ -86,4 +77,7 @@ class DataflowHubAuthenticator(Authenticator):
             return user_dict
 
         except Exception as e:
-            return None
+            return None
+        
+        finally:
+            self.db.close()

dataflow_core-2.0.11/dataflow/models/database.py → dataflow_core-2.1.0/dataflow/database_manager.py

@@ -7,6 +7,7 @@ from sqlalchemy.orm import sessionmaker
 class DatabaseManager:
     def __init__(self, db_url):
         self.db_url = db_url
+        self.engine = self.get_engine()
 
     def get_engine(self):
         try:
@@ -17,8 +18,8 @@ class DatabaseManager:
         
     def get_session(self):
         try:
-            engine = self.get_engine()
-            session = sessionmaker(autocommit=False, autoflush=False, bind=engine or create_engine(self.db_url))
+            engine = self.engine
+            session = sessionmaker(autocommit=False, autoflush=False, bind=engine)
             db = session()
             try:
                 yield db
@@ -29,4 +30,4 @@ class DatabaseManager:
             raise e
     
     def get_base(self):
-        return declarative_base()
+        return declarative_base()

{dataflow_core-2.0.11 → dataflow_core-2.1.0}/dataflow/dataflow.py

@@ -1,8 +1,8 @@
 import os, requests
-from .models.database import DatabaseManager
+from .database_manager import DatabaseManager
 from .utils.aws_secrets_manager import SecretsManagerClient
 import json
-from authenticator.package.configuration import ConfigurationManager
+from .configuration import ConfigurationManager
 
 
 class Dataflow:
@@ -10,7 +10,15 @@ class Dataflow:
         self.secrets_manager = SecretsManagerClient()
 
     def auth(self, session_id: str):
-        """Retrieve user information from the auth API."""
+        """
+        Retrieve and return user information using their session ID.
+        
+        Args:
+            session_id (str): User's session ID from cookies
+            
+        Returns:
+            dict: User information including username, name, email, and role
+        """
         try:
             dataflow_config = ConfigurationManager('/dataflow/app/auth_config/dataflow_auth.cfg')
             auth_api = dataflow_config.get_config_value('auth', 'ui_auth_api')
@@ -28,7 +36,7 @@ class Dataflow:
                 "first_name": user_data["first_name"],
                 "last_name": user_data["last_name"] if user_data.get("last_name") else "",
                 "email": user_data["email"],
-                "role": user_data["role"]
+                "role": user_data["base_role"]
             }
             return user_dict
                   
@@ -37,7 +45,13 @@ class Dataflow:
     
     def variable(self, variable_name: str):
         """
-        Get variable value from UI API.
+        Retrieve a Dataflow variable.
+        
+        Args:
+            variable_name (str): Name of the variable to retrieve
+            
+        Returns:
+            str or None: Variable value if found, None otherwise
         """
         try:
             host_name = os.environ.get("HOSTNAME", "")
@@ -87,7 +101,15 @@ class Dataflow:
             return None
         
     def secret(self, secret_name: str):
-        """Get secret value from secrets manager for Studio or Runtime."""
+        """
+        Retrieve a Dataflow secret value.
+        
+        Args:
+            secret_name (str): Name of the secret to retrieve
+            
+        Returns:
+            str or None: Secret value if found, None otherwise
+        """
         try:
             host_name = os.environ.get("HOSTNAME", "")
             user_name = host_name.replace("jupyter-", "") if host_name.startswith("jupyter-") else host_name
@@ -121,8 +143,17 @@ class Dataflow:
             print(f"[Dataflow.secret] Exception occurred: {e}")
             return None
 
-    def connection(self, conn_id: str):
-        """Get connection details from secrets manager."""
+    def connection(self, conn_id: str, mode="session"):
+        """
+        Connects with a Dataflow connection.
+        
+        Args:
+            conn_id (str): Connection identifier
+            mode (str): Return type - "session" (default) or "engine" or "url"
+            
+        Returns:
+            Session or Engine: SQLAlchemy session or engine based on mode
+        """
         try:
             host_name = os.environ["HOSTNAME"]
             user_name=host_name.replace("jupyter-","")
@@ -155,8 +186,14 @@ class Dataflow:
                     # If 'extra' is not valid JSON, skip adding extra parameters
                     pass
 
+            if mode == "url":
+                return connection_string
+    
             connection_instance = DatabaseManager(connection_string)
-            return next(connection_instance.get_session())
+            if mode == "engine":
+                return connection_instance.get_engine()
+            elif mode == "session":
+                return next(connection_instance.get_session())
         
         except Exception as e:
             return None

dataflow_core-2.1.0/dataflow/db.py

@@ -0,0 +1,51 @@
+from .database_manager import DatabaseManager
+from .configuration import ConfigurationManager
+from sqlalchemy.orm import declarative_base
+from sqlalchemy.exc import SQLAlchemyError
+from .utils.logger import CustomLogger
+
+logger = CustomLogger().get_logger(__name__)
+
+dataflow_config = ConfigurationManager('/dataflow/app/config/dataflow.cfg')
+db_url = dataflow_config.get_config_value('database', 'database_url')
+local_db_url = dataflow_config.get_config_value('database', 'single_user_db_url')
+
+db_manager = DatabaseManager(db_url)
+local_db_manager = None
+
+Base = declarative_base()
+Local_Base = declarative_base()
+
+def create_tables(local_db=False):
+    """
+    Create all tables in the database.
+    This is called at the start of the application.
+    """
+    try:
+        if local_db:
+            global local_db_manager
+            if local_db_manager is None:
+                local_db_manager = DatabaseManager(local_db_url) 
+            Local_Base.metadata.create_all(bind=local_db_manager.get_engine())
+        else:
+            Base.metadata.create_all(bind=db_manager.get_engine())
+        logger.info("Database tables created successfully")
+    except SQLAlchemyError as e:
+        error_message = f"Failed to create tables: {str(e)}"
+        logger.error(error_message)
+        raise e
+    
+def get_local_db():
+    """
+    Get a local database session.
+    
+    Yields:
+        Session: Local database session
+    """
+    global local_db_manager
+    if local_db_manager is None:
+        local_db_manager = DatabaseManager(local_db_url)
+    yield from local_db_manager.get_session()
+
+def get_db():
+    yield from db_manager.get_session()

{dataflow_core-2.0.11 → dataflow_core-2.1.0}/dataflow/environment.py

@@ -2,7 +2,7 @@ import os, shutil, subprocess, datetime
 from .models.environment import JobLogs, Environment
 import json, asyncio, pkg_resources
 from sqlalchemy.orm import Session
-from authenticator.package.configuration import ConfigurationManager
+from .configuration import ConfigurationManager
 
 class EnvironmentManager:
     def __init__(self):

dataflow_core-2.1.0/dataflow/models/__init__.py

@@ -0,0 +1,22 @@
+# init for loading models in the application
+
+from .role import Role
+from .user import User
+from .team import Team
+from .environment import Environment
+from .project_details import ProjectDetails
+from .recent_projects import RecentProjects
+from .pinned_projects import PinnedProject
+from .app_types import AppType
+from .blacklist_library import BlacklistedLibrary
+from .environment_status import EnvironmentStatus
+from .session import Session
+from .server_config import ServerConfig
+from .runtime import RuntimeZone
+from .environment_status import EnvironmentStatus
+from .user_team import UserTeam
+from .role_server import RoleServer
+from .variables import Variable
+from .recent_project_studio import RecentProjectStudio
+from .connection import Connection
+from .git_ssh import GitSSH

dataflow_core-2.1.0/dataflow/models/app_types.py

@@ -0,0 +1,10 @@
+from sqlalchemy import Column, Integer, String, Boolean
+from dataflow.db import Base
+
+class AppType(Base):
+    __tablename__ = "RUNTIME_APP_TYPE"
+    
+    id = Column(Integer, primary_key=True, autoincrement=True, unique=True)
+    name = Column(String, unique=True, nullable=True)
+    display_name = Column(String, nullable=False)
+    code_based = Column(Boolean, nullable=False)

dataflow_core-2.1.0/dataflow/models/blacklist_library.py

@@ -0,0 +1,26 @@
+# models/blacklist_library.py
+from sqlalchemy import Column, Integer, String, UniqueConstraint
+from dataflow.db import Base
+
+class BlacklistedLibrary(Base):
+    """
+    BlacklistedLibrary model represents a table for storing blacklisted libraries with their versions.
+
+    Attributes:
+        id (int): Primary key of the table, auto-incremented.
+        library_name (str): The name of the blacklisted library.
+        version (str): The version of the blacklisted library.
+
+    Unique constraint to ensure the combination of library_name and version is unique.
+    """
+
+    __tablename__ = "BLACKLISTED_LIBRARY"
+    
+    id = Column(Integer, primary_key=True, index=True, doc="Primary key for the library.")
+    library_name = Column(String, index=True, doc="The name of the blacklisted library.")
+    version = Column(String, doc="The version of the blacklisted library.")
+
+    __table_args__ = (
+        UniqueConstraint('library_name', 'version', name='uq_library_version'),
+    )
+    

dataflow_core-2.1.0/dataflow/models/connection.py

@@ -0,0 +1,25 @@
+from sqlalchemy import Column, String, Integer, Boolean, DateTime, UniqueConstraint
+from sqlalchemy.sql import func
+from dataflow.db import Base
+
+class Connection(Base):
+    """
+    Database model for storing non-sensitive connection metadata
+    """
+    __tablename__ = "CONNECTION"
+
+    id = Column(Integer, primary_key=True, index=True)
+    conn_id = Column(String, index=True, nullable=False)
+    description = Column(String, nullable=True)
+    conn_type = Column(String, nullable=False)
+    runtime = Column(String, nullable=True)
+    slug = Column(String, nullable=True)
+    status = Column(Boolean, default=False)
+    created_by = Column(String, nullable=True)
+    created_at = Column(DateTime(timezone=True), server_default=func.now())
+    updated_at = Column(DateTime(timezone=True), onupdate=func.now())
+    is_active = Column(Boolean, default=True)
+    
+    __table_args__ = (
+        UniqueConstraint('conn_id', 'runtime', 'slug', 'is_active', 'created_by', name='uq_active_conn_with_runtime_slug'),
+    )

{dataflow_core-2.0.11 → dataflow_core-2.1.0}/dataflow/models/environment.py

@@ -1,9 +1,8 @@
 from sqlalchemy import Column, Integer, String, Boolean, Text, ForeignKey, DateTime
-from sqlalchemy.orm import relationship, declarative_base
+from sqlalchemy.orm import relationship
 from sqlalchemy.sql import func
 from datetime import datetime
-
-Base = declarative_base()
+from dataflow.db import Base
 
 class EnvironmentAttributes(Base):
     """
@@ -53,7 +52,7 @@ class ArchivedEnvironment(EnvironmentAttributes):
 
 
 class JobLogs(Base):
-    __tablename__ = "JOB_LOGS"
+    __tablename__ = "JOB_LOG"
 
     id = Column(Integer, primary_key=True, index=True)
     created_at = Column(DateTime, default=datetime.now)
@@ -61,4 +60,5 @@ class JobLogs(Base):
     log_file_name = Column(String, unique=True, nullable=False)
     log_file_location = Column(String, nullable=False)
     status = Column(String)
-    created_by = Column(String)
+    created_by = Column(String)
+

dataflow_core-2.1.0/dataflow/models/environment_status.py

@@ -0,0 +1,17 @@
+"""models.py"""
+from sqlalchemy import Column, Integer, String, ForeignKey, DateTime
+from sqlalchemy.sql import func
+from dataflow.db import Base
+
+class EnvironmentStatus(Base):
+    """
+    Table ENVIRONMENT_STATUS
+    """
+
+    __tablename__='ENVIRONMENT_STATUS'
+
+    id = Column(Integer, ForeignKey('ENVIRONMENT.id'), primary_key=True, nullable=False)
+    status = Column(String, nullable=False)
+    comment = Column(String)
+    status_changed_date = Column(DateTime, server_default=func.now(), nullable=False)
+    

dataflow_core-2.1.0/dataflow/models/git_ssh.py

@@ -0,0 +1,18 @@
+"""models.py"""
+from sqlalchemy import Column, Integer, String, DateTime, ForeignKey, UniqueConstraint
+from sqlalchemy.sql import func
+from dataflow.db import Base
+
+class GitSSH(Base):
+    __tablename__ = 'GIT_SSH'
+
+    id = Column(Integer, primary_key=True, index=True, autoincrement=True)
+    user_name = Column(String, ForeignKey('USER.user_name', ondelete="CASCADE"), nullable=False)
+    description = Column(String)
+    key_name = Column(String, nullable=False)
+    created_date = Column(DateTime, server_default=func.now(), nullable=False)
+    last_used_date = Column(DateTime)
+
+    __table_args__ = (
+        UniqueConstraint(user_name, key_name, name='user_name_key_name_unique'),
+    )

dataflow_core-2.1.0/dataflow/models/pinned_projects.py

@@ -0,0 +1,15 @@
+from sqlalchemy import Column, Integer, ForeignKey, DateTime, UniqueConstraint
+from dataflow.db import Base
+from datetime import datetime
+
+class PinnedProject(Base):
+    __tablename__ = "PINNED_PROJECT"
+
+    id = Column(Integer, primary_key=True)
+    user_id = Column(Integer, ForeignKey('USER.user_id'), index=True)
+    project_id = Column(Integer, ForeignKey('PROJECT_DETAIL.project_id'), index=True)
+    pinned_at = Column(DateTime, default=datetime.utcnow)
+
+    __table_args__ = (
+        UniqueConstraint("user_id", "project_id", name="uix_user_project"),
+    )

dataflow_core-2.1.0/dataflow/models/project_details.py

@@ -0,0 +1,23 @@
+from sqlalchemy import Column, String, Enum, DateTime, Integer, func, ForeignKey
+from sqlalchemy.orm import relationship
+from dataflow.db import Base
+
+class ProjectDetails(Base):
+    __tablename__ = "PROJECT_DETAIL"
+    
+    project_id = Column(Integer, primary_key=True, autoincrement=True)
+    project_name = Column(String, nullable=False)
+    git_url = Column(String)
+    git_branch = Column(String, nullable=True)
+    git_folder = Column(String, nullable=True)
+    type = Column(String, ForeignKey('RUNTIME_APP_TYPE.name'), nullable=False)
+    slug = Column(String, nullable=False, unique=True)
+    runtime = Column(String, nullable=False)
+    py_env = Column(String, nullable=True)
+    launch_url = Column(String, nullable=True)  
+    status = Column(Enum("pending", "created" ,"deployed", "stopped", "failed", name="deployment_status"), default="created")
+    last_deployed = Column(DateTime, nullable=True)
+    created_at = Column(DateTime, nullable=False, server_default=func.now())
+    created_by = Column(String, nullable=False)
+
+    app_type = relationship("AppType")

dataflow_core-2.1.0/dataflow/models/recent_project_studio.py

@@ -0,0 +1,19 @@
+"""models.py"""
+from sqlalchemy import Column, Integer, String, DateTime, UniqueConstraint, Boolean
+from sqlalchemy.sql import func
+from dataflow.db import Local_Base as Base
+
+class RecentProjectStudio(Base):
+    __tablename__ = 'RECENT_PROJECT_STUDIO'
+
+    id = Column(Integer, primary_key=True, index=True, autoincrement=True)
+    user_name = Column(String, nullable=False, index=True)
+    app_name = Column(String, nullable=False, index=True)
+    project_name = Column(String, nullable=False)
+    project_path = Column(String, nullable=False)
+    last_opened_date = Column(DateTime, server_default=func.now(), nullable=False)
+    remember = Column(Boolean, default=False)
+
+    __table_args__ = (
+        UniqueConstraint(user_name, project_path, app_name, name='user_name_project_path_app_name_unique'),
+    )

dataflow_core-2.1.0/dataflow/models/recent_projects.py

@@ -0,0 +1,9 @@
+from sqlalchemy import Column, Integer, ForeignKey
+from dataflow.db import Base
+
+class RecentProjects(Base):
+    __tablename__ = "RECENT_PROJECT"
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    project_id = Column(Integer, ForeignKey('PROJECT_DETAIL.project_id'), nullable=False)
+

dataflow_core-2.1.0/dataflow/models/role.py

@@ -0,0 +1,19 @@
+"""models.py"""
+from sqlalchemy import Column, Integer, String, Enum
+from sqlalchemy.orm import relationship
+from dataflow.db import Base
+
+class Role(Base):
+    """
+    Table Role
+    """
+
+    __tablename__='ROLE'
+
+    id = Column(Integer, primary_key=True, index=True, autoincrement=True, nullable=False)
+    name = Column(String, unique=True, nullable=False)
+    description = Column(String, nullable=True)
+    base_role = Column(Enum('admin', 'user', name='base_role_field'), default='user', nullable=False)
+
+    users = relationship("User", back_populates="role_details", cascade="all, delete-orphan")
+    role_server_assocs = relationship("RoleServer", back_populates="role")

dataflow_core-2.1.0/dataflow/models/role_server.py

@@ -0,0 +1,14 @@
+# models/user_team.py
+from sqlalchemy import Column, Integer, ForeignKey, UniqueConstraint
+from sqlalchemy.orm import relationship
+from dataflow.db import Base
+
+class RoleServer(Base):
+    __tablename__ = 'ROLE_SERVER'
+    __table_args__ = (UniqueConstraint('role_id', 'server_id', name='_role_server_uc'),)
+
+    role_id = Column(Integer, ForeignKey('ROLE.id', ondelete="CASCADE"), nullable=False, primary_key=True)
+    server_id = Column(Integer, ForeignKey('CUSTOM_SERVER.id', ondelete="CASCADE"), nullable=False, primary_key=True)
+
+    role = relationship("Role", back_populates="role_server_assocs")
+    server = relationship("CustomServerConfig", back_populates="role_server_assocs")

dataflow_core-2.1.0/dataflow/models/runtime.py

@@ -0,0 +1,11 @@
+from sqlalchemy import Column, Integer, String, Boolean
+from dataflow.db import Base
+
+class RuntimeZone(Base):
+    __tablename__ = "RUNTIME_ZONE"
+
+    id = Column(Integer, primary_key=True, autoincrement=True, unique=True)
+    name = Column(String, unique=True, nullable=False)
+    display_name = Column(String, nullable=False)
+    display_order = Column(Integer)
+    spark_enabled = Column(Boolean, default=False)

dataflow_core-2.1.0/dataflow/models/server_config.py

@@ -0,0 +1,33 @@
+from sqlalchemy import Column, Integer, String, Boolean, Text, ForeignKey
+from sqlalchemy.dialects.postgresql import JSONB 
+from sqlalchemy.sql import func
+from sqlalchemy.orm import relationship
+from dataflow.db import Base
+
+class ServerConfig(Base):
+    __tablename__ = "SERVER_CONFIG"
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    display_name = Column(String, nullable=False, unique=True)
+    slug = Column(String, unique=True, nullable=False)
+    price = Column(String, nullable=False)
+    ram = Column(String, nullable=False)
+    cpu = Column(String, nullable=False)
+    gpu = Column(String)
+    default = Column(Boolean, default=False)
+    tags = Column(JSONB, default=func.json([]))
+    description = Column(Text, nullable=True)
+    kubespawner_override = Column(JSONB, default=func.json({}))
+
+
+class CustomServerConfig(Base):
+    __tablename__ = "CUSTOM_SERVER"
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    base_server_id = Column(Integer, ForeignKey(ServerConfig.id), nullable=False)
+    display_name = Column(String, nullable=False, unique=True, index=True)
+    description = Column(Text, nullable=True)
+    
+    # Relationship to the server_config table
+    server_config = relationship(ServerConfig)
+    role_server_assocs = relationship("RoleServer", back_populates="server", cascade="all, delete-orphan")

dataflow_core-2.1.0/dataflow/models/session.py

@@ -0,0 +1,17 @@
+"""models.py"""
+from sqlalchemy import Column, Integer, String, ForeignKey
+from dataflow.db import Base
+
+class Session(Base):
+    """
+    Table SESSION
+    """
+
+    __tablename__='SESSION'
+
+    id = Column(Integer, primary_key=True, index=True, unique=True, nullable=False, autoincrement=True)
+    session_id = Column(String, unique=True, nullable=False)
+    user_id = Column(Integer, ForeignKey('USER.user_id', ondelete="CASCADE"), nullable=False)    
+
+
+    

dataflow_core-2.1.0/dataflow/models/team.py

@@ -0,0 +1,17 @@
+"""models.py"""
+from sqlalchemy import Column, Integer, String
+from sqlalchemy.orm import relationship
+from dataflow.db import Base
+
+class Team(Base):
+    """
+    Table TEAM
+    """
+
+    __tablename__='TEAM'
+
+    team_id = Column(Integer, primary_key=True, index=True, autoincrement=True, nullable=False)
+    team_name = Column(String, unique=True, nullable=False)
+    description = Column(String, nullable=True)
+    
+    user_team_assocs = relationship("UserTeam", back_populates="team")

dataflow_core-2.1.0/dataflow/models/user.py

@@ -0,0 +1,29 @@
+"""models.py"""
+from sqlalchemy import Column, Integer, String, Boolean, LargeBinary, Enum, ForeignKey
+from sqlalchemy.orm import relationship
+from dataflow.db import Base
+
+class User(Base):
+    """
+    Table USER
+    """
+
+    __tablename__='USER'
+
+    user_id = Column(Integer, primary_key=True, index=True, autoincrement=True, nullable=False)
+    user_name = Column(String, unique=True, nullable=False)
+    first_name = Column(String)
+    last_name = Column(String)
+    email = Column(String, unique=True)
+    role_id = Column(Integer, ForeignKey('ROLE.id'), nullable=False)
+    image = Column(LargeBinary)
+    image_url = Column(String, nullable=True)
+    active = Column(Enum('N', 'Y', name='active_field'), nullable=False, server_default=str("N"))
+    password = Column(String, nullable=False)
+    active_env = Column(String, ForeignKey('ENVIRONMENT.short_name')) 
+    current_server = Column(String)
+    show_server_page = Column(Boolean, default = True)
+
+    role_details = relationship("Role")
+
+    user_team_assocs = relationship("UserTeam", back_populates="user")

dataflow_core-2.1.0/dataflow/models/user_environment.py

@@ -0,0 +1,16 @@
+from sqlalchemy import Column, Integer, String, DateTime
+from sqlalchemy.sql.schema import ForeignKey
+from sqlalchemy.sql.expression import func
+from dataflow.db import Base
+
+class UserEnvironment(Base):
+    """
+    Table USER_ENVIRONMENT
+    """
+
+    __tablename__ = 'USER_ENVIRONMENT'
+
+    id = Column(Integer, primary_key=True, index=True, autoincrement=True)
+    user_id = Column(Integer, ForeignKey('USER.user_id', ondelete="CASCADE"), nullable=False)
+    env_name = Column(String)
+    timestamp = Column(DateTime, server_default=func.now(), onupdate=func.now())

dataflow_core-2.1.0/dataflow/models/user_team.py

@@ -0,0 +1,14 @@
+# models/user_team.py
+from sqlalchemy import Column, Integer, ForeignKey, UniqueConstraint
+from sqlalchemy.orm import relationship
+from dataflow.db import Base
+
+class UserTeam(Base):
+    __tablename__ = 'USER_TEAM'
+    __table_args__ = (UniqueConstraint('user_id', 'team_id', name='_user_team_uc'),)
+
+    user_id = Column(Integer, ForeignKey('USER.user_id', ondelete="CASCADE"), nullable=False, primary_key=True)
+    team_id = Column(Integer, ForeignKey('TEAM.team_id', ondelete="CASCADE"), nullable=False, primary_key=True)
+
+    user = relationship("User", back_populates="user_team_assocs")
+    team = relationship("Team", back_populates="user_team_assocs")

dataflow_core-2.1.0/dataflow/models/variables.py

@@ -0,0 +1,27 @@
+from sqlalchemy import Column, Integer, String, ForeignKey, Text, DateTime, func, UniqueConstraint, CheckConstraint, Boolean
+from dataflow.db import Base
+
+class Variable(Base):
+    """
+    Unified VARIABLE table to support both Studio and Runtime environments.
+    """
+
+    __tablename__ = 'VARIABLE'
+
+    id = Column(Integer, primary_key=True, index=True, autoincrement=True, nullable=False)
+    key = Column(String, nullable=False)
+    value = Column(Text, nullable=False)
+    type = Column(String, nullable=False)
+    description = Column(Text, nullable=True)
+    runtime = Column(String, nullable=True)
+    slug = Column(String, nullable=True)
+    created_at = Column(DateTime, server_default=func.now())
+    updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
+    created_by = Column(String, ForeignKey('USER.user_name'), nullable=True)
+    is_active = Column(Boolean, default=True, nullable=False)
+
+
+    __table_args__ = (
+        CheckConstraint(type.in_(['variable', 'secret']), name='check_variable_type'),
+        UniqueConstraint('key', 'runtime', 'slug', 'created_by', name='unique_key'),
+    )

dataflow_core-2.1.0/dataflow/utils/get_current_user.py

@@ -0,0 +1,35 @@
+from fastapi import HTTPException, status
+from sqlalchemy.orm import Session
+from dataflow.models import user as m_user  
+from dataflow.models import session as m_session
+
+def get_user_from_session(session_id: str, db: Session):
+    """
+    Retrieve a user based on session ID.
+    
+    Args:
+        session_id (str): The unique session identifier
+        db (Session): Database session
+        
+    Returns:
+        User: User object if found
+        
+    Raises:
+        HTTPException: If session is invalid or user not found
+    """
+    session_record = db.query(m_session.Session).filter(m_session.Session.session_id == session_id).first()
+    if not session_record:
+        raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid session")
+
+    user = db.query(m_user.User).filter(m_user.User.user_id == session_record.user_id).first()
+    if not user:
+        raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="User not found")
+
+    base_role = user.role_details.base_role
+    role_name = user.role_details.name
+    user.base_role = base_role
+    user.role = role_name
+    
+    return user
+
+

dataflow_core-2.1.0/dataflow/utils/logger.py

@@ -0,0 +1,41 @@
+import logging
+import sys
+
+class CustomLogger:
+    """
+    Custom logger class for configuring and using console-based logging functionality.
+    """
+    def __init__(self):
+        self._configure_handler()
+
+    def _configure_handler(self):
+        """
+        Configure the console handler once.
+        """
+        self.formatter = logging.Formatter(
+            '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+        )
+
+        self.console_handler = logging.StreamHandler(sys.stdout)
+        self.console_handler.setLevel(logging.INFO)
+        self.console_handler.setFormatter(self.formatter)
+
+    def get_logger(self, name=None):
+        """
+        Retrieve a configured logger instance with a dynamic name.
+
+        Args:
+            name (str): The name of the logger. Defaults to 'dataflow_logger'.
+
+        Returns:
+            logging.Logger: Configured logger instance.
+        """
+        logger_name = name or "dataflow_logger"
+        logger = logging.getLogger(logger_name)
+        logger.setLevel(logging.INFO)
+
+        # Prevent duplicate handlers if logger already configured
+        if not logger.handlers:
+            logger.addHandler(self.console_handler)
+
+        return logger

{dataflow_core-2.0.11 → dataflow_core-2.1.0}/dataflow_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataflow-core
-Version: 2.0.11
+Version: 2.1.0
 Summary: Dataflow core package
 Author: Dataflow
 Author-email: 

dataflow_core-2.1.0/dataflow_core.egg-info/SOURCES.txt

@@ -0,0 +1,46 @@
+README.md
+setup.py
+authenticator/__init__.py
+authenticator/dataflowairflowauthenticator.py
+authenticator/dataflowhubauthenticator.py
+authenticator/dataflowsupersetauthenticator.py
+dataflow/__init__.py
+dataflow/configuration.py
+dataflow/database_manager.py
+dataflow/dataflow.py
+dataflow/db.py
+dataflow/environment.py
+dataflow/models/__init__.py
+dataflow/models/app_types.py
+dataflow/models/blacklist_library.py
+dataflow/models/connection.py
+dataflow/models/environment.py
+dataflow/models/environment_status.py
+dataflow/models/git_ssh.py
+dataflow/models/pinned_projects.py
+dataflow/models/project_details.py
+dataflow/models/recent_project_studio.py
+dataflow/models/recent_projects.py
+dataflow/models/role.py
+dataflow/models/role_server.py
+dataflow/models/runtime.py
+dataflow/models/server_config.py
+dataflow/models/session.py
+dataflow/models/team.py
+dataflow/models/user.py
+dataflow/models/user_environment.py
+dataflow/models/user_team.py
+dataflow/models/variables.py
+dataflow/scripts/clone_environment.sh
+dataflow/scripts/create_environment.sh
+dataflow/utils/__init__.py
+dataflow/utils/aws_secrets_manager.py
+dataflow/utils/get_current_user.py
+dataflow/utils/json_handler.py
+dataflow/utils/logger.py
+dataflow_core.egg-info/PKG-INFO
+dataflow_core.egg-info/SOURCES.txt
+dataflow_core.egg-info/dependency_links.txt
+dataflow_core.egg-info/entry_points.txt
+dataflow_core.egg-info/requires.txt
+dataflow_core.egg-info/top_level.txt

{dataflow_core-2.0.11 → dataflow_core-2.1.0}/setup.py

@@ -14,7 +14,7 @@ class PostInstall(install):
 
 setup(
     name="dataflow-core",
-    version="2.0.11",
+    version="2.1.0",
     packages=find_packages(include=["dataflow", "dataflow.*", "authenticator", "authenticator.*"]),
     include_package_data=True,
     package_data={

dataflow_core-2.0.11/authenticator/dataflowairflowauthenticator.py

@@ -1,83 +0,0 @@
-from .package.models.database import DatabaseManager
-from dataflow.dataflow import Dataflow
-
-from typing import Any, Callable
-from airflow.www.security import FabAirflowSecurityManagerOverride
-from airflow.configuration import conf
-
-class DataflowAirflowAuthenticator(FabAirflowSecurityManagerOverride):
-    def __init__(self, wsgi_app: Callable) -> None:
-        self.wsgi_app = wsgi_app
-        self.dataflow = Dataflow()
-        self.airflow_database_url = conf.get("database", "sql_alchemy_conn")
-
-        self.airflow_db_instance = DatabaseManager(self.airflow_database_url)
-        self.airflow_db = next(self.airflow_db_instance.get_session())
-
-    def __call__(self, environ: dict, start_response: Callable) -> Any:
-
-        path = environ.get('PATH_INFO', '')
-        if not path == '/login/':
-            return self.wsgi_app(environ, start_response)
-
-        try:
-            # Extracting browser cookies
-            cookies = environ.get('HTTP_COOKIE', '')
-            user_session_id = None
-            parts = cookies.split('; ')
-            for part in parts:
-                if part.startswith('dataflow_session='):
-                    user_session_id = part
-                    break
-
-            if user_session_id is None:
-                raise Exception("No session id found")
-            
-            user_session_id = user_session_id.split('=')[1]
-
-            # Retrieving user details
-            user_data = self.dataflow.auth(user_session_id)
-
-            if user_data is None:
-                raise Exception("No user found for the dataflow_session id")
-            
-            user = self.find_user(user_data["user_name"])
-
-            if not user:
-                user_role = self.find_role(user_data["role"].title())
-                user = self.add_user(
-                    username=user_data["user_name"],
-                    first_name=user_data.get("first_name", ""),
-                    last_name=user_data.get("last_name", ""),
-                    email=user_data.get("email", ""),
-                    role=user_role
-                )
-
-            environ['REMOTE_USER'] = user.username
-            return self.wsgi_app(environ, start_response)
-
-        except Exception as e:
-            return self.wsgi_app(environ, start_response)
-
-    def find_user(self, username=None):
-        """Find user by username or email."""
-        return self.airflow_db.query(self.user_model).filter_by(username=username).one_or_none()
-
-    def find_role(self, role):
-        """Find a role in the database."""
-        return self.airflow_db.query(self.role_model).filter_by(name=role).one_or_none()
-
-    def add_user(self, username, first_name, last_name, email, role, password=""):
-        """Create a user."""
-        user = self.user_model()
-        user.first_name = first_name
-        user.last_name = last_name
-        user.username = username
-        user.email = email
-        user.active = True
-        user.roles = role if isinstance(role, list) else [role]
-        user.password = password
-        self.airflow_db.add(user)
-        self.airflow_db.commit()
-        return user
-        

dataflow_core-2.0.11/authenticator/package/models/database.py

@@ -1,32 +0,0 @@
-"""models/database.py"""
-from sqlalchemy.exc import SQLAlchemyError
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy import create_engine
-from sqlalchemy.orm import sessionmaker
-
-class DatabaseManager:
-    def __init__(self, db_url):
-        self.db_url = db_url
-
-    def get_engine(self):
-        try:
-            engine = create_engine(self.db_url)
-            return engine
-        except SQLAlchemyError as e:
-            raise e
-        
-    def get_session(self):
-        try:
-            engine = self.get_engine()
-            session = sessionmaker(autocommit=False, autoflush=False, bind=engine or create_engine(self.db_url))
-            db = session()
-            try:
-                yield db
-            finally:
-                db.close()
-
-        except SQLAlchemyError as e:
-            raise e
-    
-    def get_base(self):
-        return declarative_base()

dataflow_core-2.0.11/authenticator/package/models/session.py

@@ -1,20 +0,0 @@
-"""models.py"""
-from sqlalchemy import Column, Integer, String
-from sqlalchemy.ext.declarative import declarative_base
-
-#instance for create declarative base
-Base=declarative_base()
-
-class Session_table(Base):
-    """
-    Table SESSIONS
-    """
-
-    __tablename__='SESSION'
-
-    id = Column(Integer, primary_key=True, index=True, unique=True, nullable=False, autoincrement=True)
-    session_id = Column(String, unique=True, nullable=False)
-    user_id = Column(String, nullable=False)    
-
-
-    

dataflow_core-2.0.11/authenticator/package/models/user.py

@@ -1,22 +0,0 @@
-"""models.py"""
-from sqlalchemy import Column, Integer, String, LargeBinary, Enum, Boolean
-from sqlalchemy.ext.declarative import declarative_base
-
-#instance for create declarative base
-Base=declarative_base()
-
-class User(Base):
-    """
-    Table USER
-    """
-
-    __tablename__='USER'
-
-    user_id = Column(Integer, primary_key=True, index=True, autoincrement=True, nullable=False)
-    user_name = Column(String, unique=True, nullable=False)
-    first_name = Column(String)
-    last_name = Column(String)
-    email = Column(String, unique=True)
-    image = Column(LargeBinary)
-    active = Column(Enum('N', 'Y', name='active_field'), nullable=False, server_default=str("N"))
-    password = Column(String, nullable=False)

dataflow_core-2.0.11/dataflow/configuration.py

@@ -1,27 +0,0 @@
-"""configuration.py"""
-import configparser
-from configparser import NoOptionError, NoSectionError
-
-class ConfigurationManager:
-    """
-    Configuration Manager
-    """
-
-    def __init__(self, config_file):
-
-        self.config_file = config_file
-        self.config = configparser.ConfigParser()
-        try:
-            self.config.read(self.config_file)
-
-        except Exception as e:
-            return None
-        
-    def get_config_value(self, section, option):
-        """
-        Get configuration value
-        """
-        try:
-            return self.config.get(section, option)
-        except (NoOptionError, NoSectionError):
-            return None

dataflow_core-2.0.11/dataflow_core.egg-info/SOURCES.txt

@@ -1,30 +0,0 @@
-README.md
-setup.py
-authenticator/__init__.py
-authenticator/dataflowairflowauthenticator.py
-authenticator/dataflowhubauthenticator.py
-authenticator/dataflowsupersetauthenticator.py
-authenticator/package/__init__.py
-authenticator/package/configuration.py
-authenticator/package/models/__init__.py
-authenticator/package/models/database.py
-authenticator/package/models/session.py
-authenticator/package/models/user.py
-dataflow/__init__.py
-dataflow/configuration.py
-dataflow/dataflow.py
-dataflow/environment.py
-dataflow/models/__init__.py
-dataflow/models/database.py
-dataflow/models/environment.py
-dataflow/scripts/clone_environment.sh
-dataflow/scripts/create_environment.sh
-dataflow/utils/__init__.py
-dataflow/utils/aws_secrets_manager.py
-dataflow/utils/json_handler.py
-dataflow_core.egg-info/PKG-INFO
-dataflow_core.egg-info/SOURCES.txt
-dataflow_core.egg-info/dependency_links.txt
-dataflow_core.egg-info/entry_points.txt
-dataflow_core.egg-info/requires.txt
-dataflow_core.egg-info/top_level.txt