dataflow-core 2.1.18rc2__tar.gz → 2.1.18rc3__tar.gz

{dataflow_core-2.1.18rc2 → dataflow_core-2.1.18rc3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataflow-core
-Version: 2.1.18rc2
+Version: 2.1.18rc3
 Summary: Dataflow core package
 Author: Dataflow
 Author-email: 

{dataflow_core-2.1.18rc2 → dataflow_core-2.1.18rc3}/authenticator/dataflowairflowauthenticator.py

@@ -18,9 +18,14 @@ dataflow = Dataflow()
 class DataflowAuthDBView(AuthDBView):    
     @expose('/login/', methods=['GET', 'POST'])
     def login(self):
+
+        """This method checks for a 'dataflow_session' cookie, retrieves user details from Dataflow,
+        and logs in or creates the user in Airflow accordingly. 
+        If the cookie is not present, it falls back to the standard login process.
+        
+        Overrides the default login method to integrate with Dataflow authentication.
         """
-        Override the default login method to handle custom authentication
-        """
+
         try:
             session_id = request.cookies.get('dataflow_session')
             if not session_id:
@@ -52,6 +57,9 @@ class DataflowAuthDBView(AuthDBView):
             return super().login()
 
 class DataflowAirflowAuthenticator(FabAirflowSecurityManagerOverride):
+    
+    """Custom Security Manager to integrate Airflow authentication with Dataflow."""
+    
     authdbview = DataflowAuthDBView
     
     def __init__(self, appbuilder):

{dataflow_core-2.1.18rc2 → dataflow_core-2.1.18rc3}/authenticator/dataflowhubauthenticator.py

@@ -10,6 +10,13 @@ from dataflow.models import user as m_user, session as m_session
 from sqlalchemy import or_
 
 class DataflowBaseAuthenticator(Authenticator):
+    
+    """Base Authenticator to handle Dataflow authentication and session management.
+    Provides methods to authenticate users via Dataflow credentials, manage sessions.
+    
+    Overrides JupyterHub's Authenticator class.
+    """
+    
     enable_dataflow_auth = Bool(True, config=True, help="Enable username/password authentication")
 
     def __init__(self, **kwargs):
@@ -24,9 +31,19 @@ class DataflowBaseAuthenticator(Authenticator):
             raise
 
     def generate_session_id(self):
+
+        """Generate and return a unique session ID using UUID4."""
+
         return str(uuid.uuid4())
 
     def set_session_cookie(self, handler, session_id):
+
+        """Set the dataflow_session cookie in the user's browser.
+        
+        Args:
+            handler: The request handler to set the cookie on.
+            session_id: The session ID to set in the cookie."""
+
         expires = datetime.now(ZoneInfo("UTC")) + timedelta(days=60)
         host = handler.request.host
         domain = '.'.join(host.split('.')[-2:]) if len(host.split('.')) >= 2 else host
@@ -43,6 +60,16 @@ class DataflowBaseAuthenticator(Authenticator):
         self.log.info(f"Set session cookie: dataflow_session={session_id} for host={host}")
 
     def get_or_create_session(self, user_id):
+
+        """Retrieve existing session ID for user or create a new one.
+        
+        Args:
+            user_id: The ID of the user to get or create a session for.
+        
+        Returns:
+            session_id (str): The existing or newly created session ID.
+        """
+
         session_id = self.generate_session_id()
         while self.db.query(m_session.Session).filter(
             m_session.Session.session_id == session_id
@@ -57,6 +84,16 @@ class DataflowBaseAuthenticator(Authenticator):
         return session_id
     
     def check_blocked_users(self, username, authenticated):
+        
+        """Check if the authenticated user is blocked based on allowed_users list.
+        
+        Args:
+            username (str): The username of the authenticated user.
+            authenticated (dict|None): The authentication data returned from authenticate method.
+        
+        Returns:
+            username (str|None): The username if not blocked, else None."""
+        
         self.log.info(f"Checking blocked users for {username}: authenticated={authenticated}, allowed_users={self.allowed_users}")
 
         if not authenticated:
@@ -70,20 +107,46 @@ class DataflowBaseAuthenticator(Authenticator):
         return super().check_blocked_users(username, authenticated)
 
     def extract_username_from_email(self, email):
-        """Extract username from email by removing domain"""
+
+        """Extract username from email by removing domain
+        
+        Args:
+            email (str): User's email address
+        
+        Returns:
+            username (str): Extracted username after removing domain
+        """
+        
         if '@' in email:
             return email.split('@')[0]
         return email
 
     def generate_secure_password(self):
-        """Generate a secure random password hash"""
+        
+        """Generate secure random password hash
+        
+        Returns:
+            password_hash (str): Securely hashed password
+        """
+        
         salt = secrets.token_hex(16)
         random_uuid = str(uuid.uuid4())
         hash_obj = hashlib.sha256((random_uuid + salt).encode())
         return hash_obj.hexdigest()
 
     def create_new_user(self, email, first_name=None, last_name=None):
-        """Create a new user with Applicant role"""
+        
+        """Create a new user with Applicant role
+        
+        Args:
+            email (str): User's email address
+            first_name (str): User's first name
+            last_name (str): User's last name
+        
+        Returns:
+            new_user (m_user.User|None): Created user object or None if creation failed
+        """
+        
         try:
             username = self.extract_username_from_email(email)
             username = re.sub(r'[^a-z0-9]', '', username.lower())
@@ -130,6 +193,17 @@ class DataflowBaseAuthenticator(Authenticator):
             return None
 
     async def authenticate_dataflow(self, handler, data):
+
+        """Authenticate user using Dataflow username/password.
+
+        Args:
+            handler: The request handler.
+            data: The authentication data containing username and password.
+        
+        Returns:
+            dict|None: Authentication result with username and session_id if successful, else None.
+        """
+
         if not (self.enable_dataflow_auth and isinstance(data, dict) and data.get("username") and data.get("password")):
             return None
         user_name_or_email = data["username"]
@@ -162,6 +236,18 @@ class DataflowBaseAuthenticator(Authenticator):
             self.db.close()
 
 class DataflowGoogleAuthenticator(DataflowBaseAuthenticator, GoogleOAuthenticator):
+
+    """Authenticator to handle Google OAuth authentication with Dataflow integration.
+    
+    Overrides
+      - DataflowBaseAuthenticator
+      - GoogleOAuthenticator
+    
+    Requires Google OAuth credentials.
+      - google_client_id
+      - google_client_secret
+    """
+
     dataflow_oauth_type = Unicode(
         default_value="google",
         config=True,
@@ -180,6 +266,17 @@ class DataflowGoogleAuthenticator(DataflowBaseAuthenticator, GoogleOAuthenticato
                       f"enable_dataflow_auth={self.enable_dataflow_auth}")
 
     async def authenticate(self, handler, data):
+
+        """Authenticate user using Google OAuth with Dataflow integration.
+
+        Args:
+            handler: The request handler.
+            data: The authentication data.
+        
+        Returns:
+            dict|None: Authentication result with username and session_id if successful, else None.
+        """
+
         self.log.info(f"Authenticate called with data: {data}, request_uri: {handler.request.uri}")
         result = await self.authenticate_dataflow(handler, data)
         if result:
@@ -228,6 +325,19 @@ class DataflowGoogleAuthenticator(DataflowBaseAuthenticator, GoogleOAuthenticato
             self.db.close()
 
 class DataflowAzureAuthenticator(DataflowBaseAuthenticator, AzureAdOAuthenticator):
+
+    """Authenticator to handle Azure AD OAuth authentication with Dataflow integration.
+
+    Overrides
+      - DataflowBaseAuthenticator
+      - AzureAdOAuthenticator
+
+    Requires Azure AD OAuth credentials.
+      - azure_client_id
+      - azure_client_secret
+      - azure_tenant_id
+    """
+
     azure_client_id = Unicode(config=True, help="Azure AD OAuth client ID")
     azure_client_secret = Unicode(config=True, help="Azure AD OAuth client secret")
     azure_tenant_id = Unicode(config=True, help="Azure AD tenant ID")
@@ -249,6 +359,17 @@ class DataflowAzureAuthenticator(DataflowBaseAuthenticator, AzureAdOAuthenticato
                       f"enable_dataflow_auth={self.enable_dataflow_auth}")
 
     async def authenticate(self, handler, data):
+
+        """Authenticate user using Azure AD OAuth with Dataflow integration.
+
+        Args:
+            handler: The request handler.
+            data: The authentication data.
+        
+        Returns:
+            dict|None: Authentication result with username and session_id if successful, else None.
+        """
+        
         result = await self.authenticate_dataflow(handler, data)
         if result:
             return result

{dataflow_core-2.1.18rc2 → dataflow_core-2.1.18rc3}/authenticator/dataflowsupersetauthenticator.py

@@ -12,6 +12,7 @@ from superset.security import SupersetSecurityManager
 from dataflow.dataflow import Dataflow
 
 class DataflowAuthDBView(AuthDBView):
+
     def __init__(self):
         self.dataflow = Dataflow()
 
@@ -33,14 +34,15 @@ class DataflowAuthDBView(AuthDBView):
     @expose('/login/', methods=['GET', "POST"])
     def login(self):
         """
-        Handles both GET and POST login requests.
+        This method handles authentication for superset in Dataflow.
 
-        - GET: Used for browser-based login. Authenticates using session cookie and redirects to home.
-        - POST: Used for API-based login. Returns JWT access token for programmatic access.
+        Methods:
+        - GET: 
+            Used for browser-based login. Authenticates using session cookie and redirects to home.
 
-        Returns:
-            - Redirect to home page (GET)
-            - JSON response with access token (POST)
+        - POST: 
+            Used for API-based login. Returns JWT access token for programmatic access.
+            Returns JSON response with access token
         """
         if request.method == "GET":
             session_id = request.cookies.get('dataflow_session')
@@ -71,6 +73,9 @@ class DataflowAuthDBView(AuthDBView):
             return jsonify(resp)
     
 class DataflowSecurityManager(SupersetSecurityManager):
+
+    """Custom Security Manager integrating Dataflow authentication with superset."""
+    
     authdbview = DataflowAuthDBView
     def __init__(self, appbuilder):
         super(DataflowSecurityManager, self).__init__(appbuilder)

{dataflow_core-2.1.18rc2 → dataflow_core-2.1.18rc3}/dataflow/configuration.py

@@ -20,6 +20,13 @@ class ConfigurationManager:
     def get_config_value(self, section, option):
         """
         Get configuration value
+
+        Args:
+            section (str): The section in the config file.
+            option (str): The option within the section.
+        
+        Returns:
+            str | None: The configuration value or None if not found.
         """
         try:
             return self.config.get(section, option)

{dataflow_core-2.1.18rc2 → dataflow_core-2.1.18rc3}/dataflow/database_manager.py

@@ -5,11 +5,22 @@ from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 
 class DatabaseManager:
+
+    """Manages database connections and sessions."""
+
     def __init__(self, db_url):
         self.db_url = db_url
         self.engine = self.get_engine()
 
     def get_engine(self):
+        
+        """
+        Create a new SQLAlchemy engine instance.
+
+        Returns:
+            Engine: The SQLAlchemy engine instance.
+        """
+        
         try:
             engine = create_engine(self.db_url)
             return engine
@@ -17,6 +28,12 @@ class DatabaseManager:
             raise e
         
     def get_session(self):
+        """
+        Create a new SQLAlchemy session.
+
+        Returns:
+            Session: The SQLAlchemy session instance.
+        """
         try:
             engine = self.engine
             session = sessionmaker(autocommit=False, autoflush=False, bind=engine)
@@ -30,4 +47,10 @@ class DatabaseManager:
             raise e
     
     def get_base(self):
+        """
+        Get the declarative base class for the ORM models.
+
+        Returns:
+            Base: The declarative base class.
+        """
         return declarative_base()

{dataflow_core-2.1.18rc2 → dataflow_core-2.1.18rc3}/dataflow/db.py

@@ -23,9 +23,10 @@ Base = declarative_base()
 Local_Base = declarative_base()
 
 def create_tables(local_db=False):
-    """
-    Create all tables in the database.
-    This is called at the start of the application.
+    """Create all tables in the database.
+    
+    Args:
+        local_db (bool): Flag indicating whether to create tables in the local database.
     """
     try:
         if local_db:

{dataflow_core-2.1.18rc2 → dataflow_core-2.1.18rc3}/dataflow/environment.py

@@ -19,8 +19,7 @@ class EnvironmentManager:
         self.logger = CustomLogger().get_logger(__name__)
     
     async def create_env(self, env_name, py_version, pip_libraries, conda_libraries, status, env_version='1', user_name=None, db:Session=None):
-        """
-        Creates a conda environment with specified Python version and packages.
+        """Creates a conda environment with specified Python version and packages.
         
         Args:
             env_name (str): Name of the environment
@@ -84,8 +83,7 @@ class EnvironmentManager:
             raise ValueError("Invalid status. Use 'draft' or 'published'.")
     
     async def clone_env(self, source_path, env_name, pip_libraries, conda_libraries, user_name, db=None, local_clone=False):
-        """
-        Clones an existing conda environment.
+        """Clones an existing conda environment.
         
         Args:
             source_path (str): Path to source environment
@@ -141,8 +139,7 @@ class EnvironmentManager:
         return clone_status
     
     async def revert_env(self, env_name, curr_version, revert_version, new_version, user_name, db: Session):
-        """
-        Reverts an environment to a previous version.
+        """Reverts an environment to a previous version.
         
         Args:
             env_name (str): Name of the environment
@@ -195,8 +192,7 @@ class EnvironmentManager:
         log_file_location=None,
         py_version=None
     ):
-        """
-        Executes environment operations (create or clone).
+        """Executes environment operations (create or clone).
         
         Args:
             env_name (str): Name of the environment
@@ -325,8 +321,7 @@ class EnvironmentManager:
             return "failed"
     
     def _setup_logging(self, env_name: str, env_version: str, user_name: str, db: Session):
-        """
-        Sets up logging for environment operations.
+        """Sets up logging for environment operations.
         
         Args:
             env_name (str): Name of the environment
@@ -352,8 +347,7 @@ class EnvironmentManager:
         return log_file_location
     
     async def _update_job_status(self, log_file_name: str, build_status: str, log_file_location: str, db: Session):
-        """
-        Updates job status with retry logic.
+        """Updates job status with retry logic.
         
         Args:
             db (Session): Database session
@@ -386,8 +380,7 @@ class EnvironmentManager:
                     self.logger.error(f"Failed to update job log after multiple attempts: {e}")
     
     def create_job_entry(self, user_name: str, db: Session, log_file_name: str, log_file_location: str):
-        """
-        Creates or updates a job entry for environment tracking.
+        """Creates or updates a job entry for environment tracking.
         
         Args:
             user_name (str): The user who initiated the job
@@ -428,8 +421,7 @@ class EnvironmentManager:
         return job
     
     def update_job_log(self, db, log_file_name, final_build_status):
-        """
-        Updates the JobLogs table with completion time and status.
+        """Updates the JobLogs table with completion time and status.
         
         Args:
             db (Session): Database session
@@ -460,8 +452,15 @@ class EnvironmentManager:
         return datetime.datetime.now().strftime("%b %d  %I:%M:%S %p")
     
     def update_environment_db(self, env_short_name, version, pip_libraries, conda_libraries, status, db: Session):
-        """
-        Updates the environment table with the new version and libraries.
+        """Updates the environment table with the new version and libraries.
+
+        Args:
+            env_short_name (str): Short name of the environment
+            version (str): Version of the environment
+            pip_libraries (list): List of pip libraries
+            conda_libraries (list): List of conda libraries
+            status (str): Build status ('success' or 'failed')
+            db (Session): Database session
         """
         try:
             if isinstance(pip_libraries, list):
@@ -561,6 +560,9 @@ class EnvironmentManager:
             conda_channels (list): List of conda channels
             conda_packages (list): List of conda packages to install
             pip_packages (list): List of pip packages to install
+        
+        Returns:
+            str: Path to the created YAML file
         """
         try:
             # Create the environment specification

dataflow_core-2.1.18rc3/dataflow/models/app_types.py

@@ -0,0 +1,31 @@
+from sqlalchemy import Column, Integer, String, Boolean
+from sqlalchemy.orm import relationship
+from dataflow.db import Base
+
+class AppType(Base):
+
+    """TABLE 'APP_TYPE'.
+    
+    Attributes:
+        id (int): Primary key for the app type.
+        name (str): Unique name identifier for the app type.
+        display_name (str): Human-readable name for the app type.
+        code_based (bool): Indicates if the app type is code-based.
+        studio (bool): Indicates if the app type is associated with Dataflow Studio.
+        runtime (bool): Indicates if the app type is a runtime application.
+        organizations (list): Relationship to organizations using this app type.
+    
+    Relationships:
+        organizations: Many-to-many relationship with Organization model via ORGANIZATION_APP_TYPE association table.
+    """
+    
+    __tablename__ = "APP_TYPE"
+    
+    id = Column(Integer, primary_key=True, autoincrement=True, unique=True)
+    name = Column(String, unique=True, nullable=False)
+    display_name = Column(String, nullable=False)
+    code_based = Column(Boolean, nullable=False)
+    studio = Column(Boolean, nullable=False, default=False, server_default='false')
+    runtime = Column(Boolean, nullable=False, default=False, server_default='false')
+
+    organizations = relationship("Organization", secondary="ORGANIZATION_APP_TYPE", back_populates="apps")

{dataflow_core-2.1.18rc2 → dataflow_core-2.1.18rc3}/dataflow/models/blacklist_library.py

@@ -3,15 +3,15 @@ from sqlalchemy import Column, Integer, String, UniqueConstraint
 from dataflow.db import Base
 
 class BlacklistedLibrary(Base):
-    """
-    BlacklistedLibrary model represents a table for storing blacklisted libraries with their versions.
+    """TABLE 'BLACKLISTED_LIBRARY'
 
     Attributes:
         id (int): Primary key of the table, auto-incremented.
         library_name (str): The name of the blacklisted library.
         version (str): The version of the blacklisted library.
 
-    Unique constraint to ensure the combination of library_name and version is unique.
+    Constraints:
+        Unique constraint to ensure unique combination of library_name and version.
     """
 
     __tablename__ = "BLACKLISTED_LIBRARY"

{dataflow_core-2.1.18rc2 → dataflow_core-2.1.18rc3}/dataflow/models/connection.py

@@ -3,9 +3,26 @@ from sqlalchemy.sql import func
 from dataflow.db import Base
 
 class Connection(Base):
+    """TABLE 'CONNECTION'
+
+    Attributes:
+        id (int): Primary key for the connection.
+        conn_id (str): Identifier for the connection.
+        org_id (int): Foreign key referencing the organization.
+        description (str): Description of the connection.
+        conn_type (str): Type of the connection.
+        runtime (str): Runtime environment for the connection.
+        slug (str): Slug identifier for the connection.
+        status (bool): Status of the connection (active/inactive).
+        created_by (str): User who created the connection.
+        created_at (datetime): Timestamp of when the connection was created.
+        updated_at (datetime): Timestamp of the last update to the connection.
+        is_active (bool): Indicates if the connection is active.
+    
+    Constraints:
+        UniqueConstraint: Ensures unique active connections based on conn_id, org_id, runtime, slug, is_active, and created_by.
     """
-    Database model for storing non-sensitive connection metadata
-    """
+
     __tablename__ = "CONNECTION"
 
     id = Column(Integer, primary_key=True, index=True)

{dataflow_core-2.1.18rc2 → dataflow_core-2.1.18rc3}/dataflow/models/dataflow_zone.py

@@ -3,6 +3,21 @@ from sqlalchemy.orm import relationship
 from dataflow.db import Base
 
 class DataflowZone(Base):
+
+    """TABLE 'DATAFLOW_ZONE'.
+    
+    Attributes:
+        id (int): Primary key for the dataflow zone.
+        slug (str): Unique slug identifier for the zone.
+        display_name (str): Human-readable name for the zone.
+        is_runtime (bool): Indicates if the zone is a runtime zone.
+        subdomain (str): Subdomain associated with the zone.
+        display_order (int): Order for displaying the zone in lists.
+    
+    Relationships:
+        role_zone_assocs: One-to-many relationship with RoleZone model.
+    """
+
     __tablename__ = "DATAFLOW_ZONE"
 
     id = Column(Integer, primary_key=True, autoincrement=True)