dataflow-core 2.1.1__py3-none-any.whl → 2.1.3__py3-none-any.whl

authenticator/dataflowhubauthenticator.py

@@ -1,83 +1,322 @@
-from dataflow.db import get_db
-from dataflow.models import (
-    user as m_user,
-    session as m_session
-    )
-from datetime import datetime, timedelta, timezone
+import os
 import uuid
+import re
+from datetime import datetime, timedelta
+from zoneinfo import ZoneInfo
+from traitlets import Bool, Unicode
 from jupyterhub.auth import Authenticator
+from oauthenticator.google import GoogleOAuthenticator
+from oauthenticator.azuread import AzureAdOAuthenticator
+from dataflow.db import get_db
+from dataflow.models import user as m_user, session as m_session, role as m_role
+
+class DataflowBaseAuthenticator(Authenticator):
+    enable_dataflow_auth = Bool(True, config=True, help="Enable username/password authentication")
 
-class DataflowHubAuthenticator(Authenticator):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
-        self.db = next(get_db())
-    
+        try:
+            self.db = next(get_db())
+            m_user.Base.metadata.create_all(bind=self.db.get_bind(), checkfirst=True)
+            m_session.Base.metadata.create_all(bind=self.db.get_bind(), checkfirst=True)
+            self.log.info("Dataflow database initialized successfully")
+        except Exception as e:
+            self.log.error(f"Failed to initialize Dataflow database: {str(e)}")
+            raise
+
     def generate_session_id(self):
         return str(uuid.uuid4())
+
+    def set_session_cookie(self, handler, session_id):
+        expires = datetime.now(ZoneInfo("UTC")) + timedelta(days=365)
+        host = handler.request.host
+        domain = '.'.join(host.split('.')[-2:]) if len(host.split('.')) >= 2 else host
+        handler.set_cookie(
+            "dataflow_session",
+            session_id,
+            domain=f".{domain}",
+            path="/",
+            expires=expires,
+            secure=True,
+            httponly=True,
+            samesite="None"
+        )
+        self.log.info(f"Set session cookie: dataflow_session={session_id} for host={host}")
+
+    def get_or_create_session(self, user_id):
+        existing_session = (
+            self.db.query(m_session.Session)
+            .filter(m_session.Session.user_id == str(user_id))
+            .first()
+        )
+        if existing_session:
+            self.log.info(f"Reusing existing session: {existing_session.session_id}")
+            return existing_session.session_id
+        session_id = self.generate_session_id()
+        while self.db.query(m_session.Session).filter(
+            m_session.Session.session_id == session_id
+        ).first():
+            session_id = self.generate_session_id()
+        db_item = m_session.Session(user_id=user_id, session_id=session_id)
+        self.db.add(db_item)
+        self.db.commit()
+        self.db.refresh(db_item)
+        self.log.info(f"Created new session: {session_id}")
+        return session_id
     
-    async def authenticate(self, handler, data):
-        # get username and password
-        username = data["username"]
-        password = data["password"]
+    def check_blocked_users(self, username, authenticated):
+        self.log.info(f"Checking blocked users for {username}: authenticated={authenticated}, allowed_users={self.allowed_users}")
+
+        if not authenticated:
+            self.log.warning(f"No authenticated data for user: {username}")
+            return None
+
+        if isinstance(authenticated, dict) and "session_id" in authenticated:
+            self.log.info(f"Allowing Dataflow authentication for user: {username}")
+            return username
+
+        return super().check_blocked_users(username, authenticated)
 
+    def get_applicant_role_id(self):
+        """Get the role ID for 'Applicant' role"""
         try:
-            # check if user exists
-            query = self.db.query(m_user.User)
-            user = query.filter(m_user.User.user_name == username).first()
+            applicant_role = (
+                self.db.query(m_role.Role)
+                .filter(m_role.Role.name == "Applicant")
+                .first()
+            )
+            if applicant_role:
+                return applicant_role.id
+            else:
+                self.log.warning("Applicant role not found in database")
+                return None
+        except Exception as e:
+            self.log.error(f"Error getting Applicant role: {str(e)}")
+            return None
+
+    def extract_username_from_email(self, email):
+        """Extract username from email by removing domain"""
+        if '@' in email:
+            return email.split('@')[0]
+        return email
 
-            if user is None or user.password != password:
+    def create_new_user(self, email, first_name=None, last_name=None):
+        """Create a new user with Applicant role"""
+        try:
+            role_id = self.get_applicant_role_id()
+            if not role_id:
+                self.log.error("Cannot create user: Applicant role not found")
                 return None
 
-            # Check if the user already has an existing session
-            existing_session = (
-                self.db.query(m_session.Session)
-                .filter(m_session.Session.user_id == user.user_id)
+            username = self.extract_username_from_email(email)
+            username = re.sub(r'[^A-Za-z0-9]', '', username)
+            if not username:
+                self.log.error("Cannot create user: Username is empty")
+                return None
+            existing_user = (
+                self.db.query(m_user.User)
+                .filter(m_user.User.user_name == username)
                 .first()
             )
+            if existing_user:
+                counter = 1
+                original_username = username
+                while existing_user:
+                    username = f"{original_username}_{counter}"
+                    existing_user = (
+                        self.db.query(m_user.User)
+                        .filter(m_user.User.user_name == username)
+                        .first()
+                    )
+                    counter += 1
 
-            if existing_session:
-                # Reuse the existing session_id
-                session_id = existing_session.session_id
-            else:
-                # Generate a new session_id
-                session_id = self.generate_session_id()
-                query = self.db.query(m_session.Session)
-                isSession = query.filter(m_session.Session.session_id == session_id).first()
-
-                # If session_id(uuid string) already exists in the database, generate a new one
-                while isSession is not None:
-                    session_id = self.generate_session_id()
-                    isSession = query.filter(m_session.Session.session_id == session_id).first()
-
-                # add session_id to the database
-                db_item = m_session.Session(user_id=user.user_id, session_id=session_id)
-                self.db.add(db_item)
-                self.db.commit()
-                self.db.refresh(db_item)
-
-            expires = datetime.now(timezone.utc) + timedelta(days=365)
-            host = handler.request.host
-            parts = host.split('.')
-            if len(parts) >= 2:
-                domain =  '.'.join(parts[-2:])
-            else:
-                domain =  host 
-            base_domain = f".{domain}" 
-            handler.set_cookie(
-                "dataflow_session",
-                session_id,
-                domain=base_domain, 
-                path="/",
-                expires=expires,
-                secure=True,            
-                httponly=True,
-                samesite="None"          
+            new_user = m_user.User(
+                user_name=username,
+                first_name=first_name or username,
+                last_name=last_name or "",
+                email=email,
+                role_id=role_id,
+                active='Y',
+                password='user@123',
+            )
+            
+            self.db.add(new_user)
+            self.db.commit()
+            self.db.refresh(new_user)
+            
+            self.log.info(f"Created new user: {username} with email: {email}")
+            return new_user
+            
+        except Exception as e:
+            self.log.error(f"Error creating new user: {str(e)}")
+            self.db.rollback()
+            return None
+
+    async def authenticate_dataflow(self, handler, data):
+        if not (self.enable_dataflow_auth and isinstance(data, dict) and data.get("username") and data.get("password")):
+            return None
+        username = data["username"]
+        password = data["password"]
+        self.log.info(f"Attempting Dataflow authentication for user: {username}")
+        try:
+            user = (
+                self.db.query(m_user.User)
+                .filter(m_user.User.user_name == username)
+                .first()
+            )
+            if not user or user.password != password:
+                self.log.warning(f"Dataflow authentication failed for user: {username}")
+                return None
+            session_id = self.get_or_create_session(user.user_id)
+            self.set_session_cookie(handler, session_id)
+            self.log.info(f"Dataflow authentication successful for user: {username}")
+            return {"name": username, "session_id": session_id, "auth_state": {}}
+        except Exception as e:
+            self.log.error(f"Dataflow authentication error: {str(e)}")
+            return None
+        finally:
+            self.db.close()
+
+class DataflowGoogleAuthenticator(DataflowBaseAuthenticator, GoogleOAuthenticator):
+    dataflow_oauth_type = Unicode(
+        default_value="google",
+        config=True,
+        help="The OAuth provider type for DataflowHub (e.g., github, google)"
+    )
+    google_client_id = Unicode(config=True, help="Google OAuth client ID")
+    google_client_secret = Unicode(config=True, help="Google OAuth client secret")
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.client_id = self.google_client_id
+        self.client_secret = self.google_client_secret
+        self.dataflow_oauth_type = self.dataflow_oauth_type
+        self.log.info(f"DataflowGoogleAuthenticator initialized with google_client_id={self.google_client_id}, "
+                      f"oauth_callback_url={self.oauth_callback_url}, "
+                      f"enable_dataflow_auth={self.enable_dataflow_auth}")
+
+    async def authenticate(self, handler, data):
+        self.log.info(f"Authenticate called with data: {data}, request_uri: {handler.request.uri}")
+        result = await self.authenticate_dataflow(handler, data)
+        if result:
+            return result
+        try:
+            user = await super().authenticate(handler, data)
+            self.log.info(f"Google OAuth authentication returned: {user}")
+            if not user:
+                self.log.warning("Google OAuth authentication failed: No user data returned")
+                return None
+            
+            email = user["name"]
+            db_user = (
+                self.db.query(m_user.User)
+                .filter(m_user.User.email == email)
+                .first()
             )
-            user_dict = {"name": username, "session_id": session_id}
-            return user_dict
+            
+            if not db_user:
+                self.log.info(f"User with email {email} not found in Dataflow database, creating new user")
+                # Extract additional info from user data if available
+                auth_state = user.get("auth_state", {})
+                user_info = auth_state.get("user", {}) if auth_state else {}
+                
+                first_name =  user_info.get("name")
+                last_name =  user_info.get("last_name")
+                
+                db_user = self.create_new_user(email, first_name, last_name)
+                if not db_user:
+                    self.log.error(f"Failed to create new user for email: {email}")
+                    return None
+            
+            username = db_user.user_name
+            session_id = self.get_or_create_session(db_user.user_id)
+            self.set_session_cookie(handler, session_id)
+            self.log.info(f"Google OAuth completed for user: {username}, session_id={session_id}")
+            return {
+                "name": username,
+                "session_id": session_id,
+                "auth_state": user.get("auth_state", {})
+            }
+        except Exception as e:
+            self.log.error(f"Google OAuth authentication error: {str(e)}", exc_info=True)
+            return None
+        finally:
+            self.db.close()
+
+class DataflowAzureAuthenticator(DataflowBaseAuthenticator, AzureAdOAuthenticator):
+    azure_client_id = Unicode(config=True, help="Azure AD OAuth client ID")
+    azure_client_secret = Unicode(config=True, help="Azure AD OAuth client secret")
+    azure_tenant_id = Unicode(config=True, help="Azure AD tenant ID")
+    azure_scope = Unicode("openid profile email", config=True, help="Azure AD OAuth scopes")
 
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.client_id = self.azure_client_id
+        self.client_secret = self.azure_client_secret
+        self.tenant_id = self.azure_tenant_id
+        self.scope = self.azure_scope.split()
+        self.dataflow_oauth_type = self.dataflow_oauth_type
+        self.log.info(f"DataflowAzureAuthenticator initialized with azure_client_id={self.azure_client_id}, "
+                      f"oauth_callback_url={self.oauth_callback_url}, "
+                      f"enable_dataflow_auth={self.enable_dataflow_auth}")
+
+    async def authenticate(self, handler, data):
+        result = await self.authenticate_dataflow(handler, data)
+        if result:
+            return result
+        try:
+            user = await super().authenticate(handler, data)
+            self.log.info(f"Azure AD OAuth authentication returned: {user}")
+            if not user:
+                self.log.warning("Azure AD OAuth authentication failed: No user data returned")
+                return None
+            
+            email = user.get("email") or user.get("preferred_username")
+            if not email:
+                self.log.warning("Azure AD OAuth authentication failed: No email in user data")
+                return None
+            
+            db_user = (
+                self.db.query(m_user.User)
+                .filter(m_user.User.email == email)
+                .first()
+            )
+            
+            if not db_user:
+                self.log.info(f"User with email {email} not found in Dataflow database, creating new user")
+                # Extract additional info from user data if available
+                auth_state = user.get("auth_state", {})
+                user_info = auth_state.get("user", {}) if auth_state else {}
+                
+                first_name = user_info.get("given_name") or user.get("given_name")
+                last_name = user_info.get("family_name") or user.get("family_name")
+                
+                db_user = self.create_new_user(email, first_name, last_name)
+                if not db_user:
+                    self.log.error(f"Failed to create new user for email: {email}")
+                    return None
+            
+            username = db_user.user_name
+            session_id = self.get_or_create_session(db_user.user_id)
+            self.set_session_cookie(handler, session_id)
+            self.log.info(f"Azure AD OAuth completed for user: {username}, session_id={session_id}")
+            return {
+                "name": username,
+                "session_id": session_id,
+                "auth_state": user.get("auth_state", {})
+            }
         except Exception as e:
+            self.log.error(f"Azure AD OAuth authentication error: {str(e)}", exc_info=True)
             return None
-        
         finally:
-            self.db.close()
+            self.db.close()
+
+auth_type = os.environ.get("DATAFLOW_AUTH_TYPE", "google")
+
+if auth_type == "google":
+    BaseAuthenticator = DataflowGoogleAuthenticator
+else:
+    BaseAuthenticator = DataflowAzureAuthenticator
+
+class DataflowHubAuthenticator(BaseAuthenticator):
+    pass

dataflow/environment.py

@@ -1,18 +1,21 @@
-import os, shutil, subprocess, datetime
+import os, shutil, subprocess, datetime, yaml, re
 from .models.environment import JobLogs, Environment
 import json, asyncio, pkg_resources
 from sqlalchemy.orm import Session
 from .configuration import ConfigurationManager
+from .utils.logger import CustomLogger
 
 class EnvironmentManager:
     def __init__(self):
         """Initialize the EnvironmentManager"""
         self.config = ConfigurationManager('/dataflow/app/config/dataflow.cfg')
-        self.published_env_path = self.config.get_config_value('paths', 'published_env_path')
-        self.draft_env_path = self.config.get_config_value('paths', 'drafts_env_path')
+        self.env_base_path = self.config.get_config_value('paths', 'env_path')
         self.env_logs_path = self.config.get_config_value('paths', 'env_logs_path')
+        self.env_version_path = self.config.get_config_value('paths', 'env_versions_path')
+        os.makedirs(self.env_version_path, exist_ok=True)
+        self.logger = CustomLogger().get_logger(__name__)
     
-    async def create_env(self, env_name, py_version, py_requirements, status, base_env_id, env_version=None, user_name=None, db:Session=None):
+    async def create_env(self, env_name, py_version, pip_libraries, conda_libraries, status, env_version='1', user_name=None, db:Session=None):
         """
         Creates a conda environment with specified Python version and packages.
         
@@ -26,47 +29,56 @@ class EnvironmentManager:
             db (Session): Database session (optional, will create if None)
             
         Returns:
-            str: Build status ('success' or 'fail')
+            str: Build status ('success' or 'failed')
         """
         # Set up logging
         log_file_location = None
         if db:
             log_file_location = self._setup_logging(env_name, env_version, user_name, db)
 
+        # Create the conda environment YAML file
+        yaml_path = os.path.join(self.env_version_path, f"{env_name}_v{env_version}.yaml")
+        self.create_conda_yaml(
+            yaml_path=yaml_path,
+            env_name=env_name,
+            python_version=py_version,
+            conda_packages=conda_libraries,
+            pip_packages=pip_libraries
+        )
+
         if status == "published":
             return await self._execute_env_operation(
                 env_name=env_name,
-                py_version=py_version,
-                py_requirements=py_requirements,
                 status="published",
-                env_version=None,
-                mode="create"
+                mode="create",
+                yaml_file_path=yaml_path,
+                version=int(env_version)
             )
         elif status == "draft":
-            # Build the environment
+            mode = "create" if env_version == '1' else "update"
             build_status = await self._execute_env_operation(
                 env_name=env_name,
-                py_version=py_version,
-                py_requirements=py_requirements,
                 status=status,
-                env_version=env_version,
+                mode=mode,
+                yaml_file_path=yaml_path,
                 log_file_location=log_file_location,
-                mode="create"
+                version=int(env_version)
             )
             
             # Update job log status if db was provided
             if db and log_file_location:
                 log_file_name = os.path.basename(log_file_location)
                 await self._update_job_status(log_file_name, build_status, log_file_location, db)
-                updated_py_requirements = self.update_library_versions(py_requirements, os.path.join(self.draft_env_path, env_name, f"{env_name}_v{env_version}"))
-                self.update_environment_db(env_name, env_version, updated_py_requirements, base_env_id, py_version, db)
+                pip_libraries, conda_libraries = self.update_library_versions(yaml_path)
+                self.update_environment_db(env_name, env_version, pip_libraries, conda_libraries, build_status, db)
 
             return build_status
         
         else:
+            self.logger.error(f"Invalid status '{status}' provided for environment creation.")
             raise ValueError("Invalid status. Use 'draft' or 'published'.")
     
-    async def clone_env(self, source_path, target_env_name, libraries, py_version, user_name=None, db: Session=None):
+    async def clone_env(self, source_path, env_name, pip_libraries, conda_libraries, user_name, db=None):
         """
         Clones an existing conda environment.
         
@@ -79,56 +91,95 @@ class EnvironmentManager:
             db (Session): Database session (optional, will create if None)
             
         Returns:
-            str: Build status ('success' or 'fail')
+            str: Build status ('success' or 'failed')
         """
         # Set up logging
         log_file_location = None
         if db:
-            log_file_location = self._setup_logging(target_env_name, "1", user_name, db)
+            log_file_location = self._setup_logging(env_name, "1", user_name, db)
+
+        yaml_path = f"{self.env_version_path}/{env_name}_v1.yaml"
         
         # Perform the clone operation
         clone_status = await self._execute_env_operation(
-            env_name=target_env_name,
-            source_path=source_path,
+            env_name=env_name,
             status="draft",
-            env_version="1",
+            mode="clone",
+            yaml_file_path=yaml_path,
+            source_path=source_path,
             log_file_location=log_file_location,
-            mode="clone"
+            version=1           
         )
         
         # Update job log status if db was provided
         if db and log_file_location:
             log_file_name = os.path.basename(log_file_location)
             await self._update_job_status(log_file_name, clone_status, log_file_location, db)
-            self.update_environment_db(env_short_name=target_env_name, version="1", libraries=libraries, base_env_id=None, py_version=py_version, db=db)
+            self.update_environment_db(
+                env_short_name=env_name, 
+                version="1", 
+                pip_libraries=pip_libraries, 
+                conda_libraries=conda_libraries,
+                status=clone_status, 
+                db=db
+            )
             
         return clone_status
     
-    async def create_published_env(self, env_name, py_version, py_requirements):
+    async def revert_env(self, env_name, curr_version, revert_version, new_version, user_name, db: Session):
         """
-        Creates a published conda environment.
+        Reverts an environment to a previous version.
         
         Args:
             env_name (str): Name of the environment
-            py_version (str): Python version to use
-            py_requirements (list): List of packages to install
-            status (str): Environment status ('draft' or 'published')
-            env_version (str): Version of the environment (for draft environments)
-            user_name (str): Username who initiated the creation
+            version (str): Version to revert to
+            db (Session): Database session
             
         Returns:
-            str: Build status ('success' or 'fail')
+            str: Build status ('success' or 'failed')
         """
-        return self._execute_env_operation(
-            env_name=env_name,
-            py_version=py_version,
-            py_requirements=py_requirements,
-            status="published",
-            env_version=None,
-            mode="create"
-        )
+        try:            
+            # Get the YAML file for the specified version
+            old_yaml_path = f"{self.env_version_path}/{env_name}_v{revert_version}.yaml"
+            new_yaml_path = f"{self.env_version_path}/{env_name}_v{new_version}.yaml"
+            if not os.path.exists(old_yaml_path):
+                raise FileNotFoundError(f"YAML file for version {revert_version} does not exist.")
+            
+            os.symlink(old_yaml_path, new_yaml_path)
+
+            log_file_location = None
+            if db:
+                log_file_location = self._setup_logging(env_name, new_version, user_name, db)
+            
+            # Execute the revert operation
+            revert_status = await self._execute_env_operation(
+                env_name=env_name,
+                status="draft",
+                mode="update",
+                yaml_file_path=new_yaml_path,
+                log_file_location=log_file_location,
+                version=int(new_version)
+            )
+            
+            log_file_name = os.path.basename(log_file_location)
+            await self._update_job_status(log_file_name, revert_status, log_file_location, db)
+            pip_libraries, conda_libraries = self.update_library_versions(new_yaml_path)
+            self.update_environment_db(env_name, new_version, pip_libraries, conda_libraries, revert_status, db)
         
-    async def _execute_env_operation(self, env_name: str, status: str, mode: str, env_version: str = None, py_version: str = None, py_requirements=None, source_path=None, log_file_location=None):
+        except Exception as e:
+            self.logger.error(f"Failed to revert environment {env_name}: {e}")
+            return "failed"
+        
+    async def _execute_env_operation(
+        self, 
+        env_name: str, 
+        status: str, 
+        mode: str, 
+        yaml_file_path: str, 
+        version: int,
+        source_path=None, 
+        log_file_location=None,
+    ):
         """
         Executes environment operations (create or clone).
         
@@ -143,34 +194,32 @@ class EnvironmentManager:
             log_file_location (str): Path to log file
             
         Returns:
-            str: Build status ('success' or 'fail')
+            str: Build status ('success' or 'failed')
         """
+        self.logger.info(f"Executing environment operation: {env_name}, Status: {status}, Mode: {mode}")
         status = status.lower()
-        if status == "published":
-            env_base_path = self.config.get_config_value('paths', 'published_env_path')
-            conda_env_path = os.path.join(env_base_path, env_name)
-        else:
-            env_base_path = self.config.get_config_value('paths', 'drafts_env_path')
-            conda_env_path = os.path.join(env_base_path, env_name, f"{env_name}_v{env_version}")
+        conda_env_path = os.path.join(self.env_base_path, env_name)
 
         try:
-            if not os.path.exists(conda_env_path):
-                os.makedirs(conda_env_path, exist_ok=True)
+            if os.path.exists(conda_env_path) and mode == "create":
+                raise FileExistsError(f"Environment '{env_name}' already exists at {conda_env_path}.")
+            
+            os.makedirs(conda_env_path, exist_ok=True)
 
-            if mode == "create":
-                # Convert requirements list to comma-separated string
-                if isinstance(py_requirements, list):
-                    py_requirements = ",".join(py_requirements)
-                
+            if mode == "create":                
                 create_env_script_path = pkg_resources.resource_filename('dataflow', 'scripts/create_environment.sh')
-                command = ["bash", create_env_script_path, py_requirements, conda_env_path, py_version]
-                
+                command = ["bash", create_env_script_path, yaml_file_path, conda_env_path]
+
+            elif mode == "update":
+                update_env_script_path = pkg_resources.resource_filename('dataflow', 'scripts/update_environment.sh')
+                command = ["bash", update_env_script_path, yaml_file_path, conda_env_path]
+            
             elif mode == "clone":
                 clone_env_script_path = pkg_resources.resource_filename('dataflow', 'scripts/clone_environment.sh')
                 command = ["bash", clone_env_script_path, source_path, conda_env_path]
                 
             else:
-                raise ValueError("Invalid mode. Use 'create' or 'clone'.")
+                raise ValueError("Invalid mode. Use 'create', 'update', or 'clone'.")
 
             process = await asyncio.create_subprocess_exec(
                 *command, 
@@ -184,6 +233,16 @@ class EnvironmentManager:
             with open(log_file_location, "a") as log_file:
                 success_detected = False
                 try:
+                    # Write an initial log entry to indicate the operation has started
+                    start_message = {
+                        "timestamp": self.format_timestamp(),
+                        "type": "log",
+                        "content": f"Starting environment {mode} operation for {env_name}"
+                    }
+                    log_file.write(json.dumps(start_message) + "\n")
+                    log_file.flush()
+                    
+                    # Process stdout line by line
                     while True:
                         line = await process.stdout.readline()
                         if not line:
@@ -213,7 +272,7 @@ class EnvironmentManager:
                         }
                         log_file.write(json.dumps(error_message_dict) + "\n")
 
-                    final_build_status = "fail" if process.returncode != 0 else "success"
+                    final_build_status = "failed" if process.returncode != 0 else "success"
 
                 except asyncio.CancelledError:
                     process.kill()
@@ -224,27 +283,24 @@ class EnvironmentManager:
                         "content": msg_content
                     }
                     log_file.write(json.dumps(cancellation_message) + "\n")
-                    final_build_status = "fail"
+                    final_build_status = "failed"
                 
                 finally:
-                    if final_build_status == "success" and status == "draft":
-                        symlink_path = os.path.join(env_base_path, env_name, "default")
-                        self.update_symlink(symlink_path, conda_env_path)
-                    elif final_build_status != "success":
+                    if final_build_status != "success" and version == 1:
                         if os.path.exists(conda_env_path):
                             shutil.rmtree(conda_env_path)
                 
             return final_build_status
         
         except OSError as e:
-            print(f"OS error while operating on {conda_env_path}: {e}")
-            return "fail"
+            self.logger.error(f"OS error while operating on {conda_env_path}: {e}")
+            return "failed"
         except subprocess.CalledProcessError as e:
-            print(f"Subprocess error during environment operation: {e}")
-            return "fail"
+            self.logger.error(f"Subprocess error during environment operation: {e}")
+            return "failed"
         except Exception as e:
-            print(f"Unexpected error during environment operation for {env_name}: {e}")
-            return "fail"
+            self.logger.error(f"Unexpected error during environment operation for {env_name}: {e}")
+            return "failed"
     
     def _setup_logging(self, env_name: str, env_version: str, user_name: str, db: Session):
         """
@@ -281,7 +337,7 @@ class EnvironmentManager:
         Args:
             db (Session): Database session
             log_file_name (str): Name of the log file
-            build_status (str): Build status ('success' or 'fail') 
+            build_status (str): Build status ('success' or 'failed') 
             log_file_location (str): Path to the log file
         """
         attempts = 3
@@ -306,7 +362,7 @@ class EnvironmentManager:
                 if attempts > 0:
                     await asyncio.sleep(retry_delay)
                 else:
-                    print(f"Failed to update job log after multiple attempts: {e}")
+                    self.logger.error(f"Failed to update job log after multiple attempts: {e}")
     
     def create_job_entry(self, user_name: str, db: Session, log_file_name: str, log_file_location: str):
         """
@@ -325,8 +381,9 @@ class EnvironmentManager:
 
         if job:
             if job.status == "success":
+                self.logger.error(f"Job with log_file_name '{log_file_name}' already completed successfully.")
                 raise ValueError(f"Job with log_file_name '{log_file_name}' already completed successfully.")
-            if job.status == "fail":
+            if job.status == "failed":
                 job.created_at = datetime.datetime.now() 
                 job.status = "in_progress"
         else:
@@ -349,7 +406,7 @@ class EnvironmentManager:
         Args:
             db (Session): Database session
             log_file_name (str): Name of the log file
-            final_build_status (str): Final status of the build ('success' or 'fail')
+            final_build_status (str): Final status of the build ('success' or 'failed')
         """
         try:
             job_record = db.query(JobLogs).filter(JobLogs.log_file_name == log_file_name).first()
@@ -358,24 +415,12 @@ class EnvironmentManager:
                 job_record.status = final_build_status
                 db.commit()
             else:
+                self.logger.error(f"No job log found for file: {log_file_name}")
                 raise ValueError(f"No job log found for file: {log_file_name}")
         except Exception as e:
+            self.logger.error(f"Failed to update job log for {log_file_name}: {e}")
             db.rollback()
             raise
-            
-    def update_symlink(self, symlink_path, conda_env_path):
-        """
-        Creates or updates the symlink to point to the default version.
-        """
-        symlink_dir = os.path.dirname(symlink_path)
-        if not os.path.exists(symlink_dir):
-            os.makedirs(symlink_dir, exist_ok=True)
-
-        # If symlink exists, remove it before updating
-        if os.path.islink(symlink_path):
-            os.remove(symlink_path)
-
-        subprocess.run(["ln", "-sf", conda_env_path, symlink_path], check=True)
 
     def format_timestamp(self):
         """
@@ -386,73 +431,214 @@ class EnvironmentManager:
         """
         return datetime.datetime.now().strftime("%b %d  %I:%M:%S %p")
     
-    def update_environment_db(self, env_short_name, version, libraries, base_env_id, py_version, db: Session):
+    def update_environment_db(self, env_short_name, version, pip_libraries, conda_libraries, status, db: Session):
         """
         Updates the environment table with the new version and libraries.
         """
         try:
-            if isinstance(libraries, list):
-                libraries = ", ".join(libraries)
+            if isinstance(pip_libraries, list):
+                pip_libraries = ", ".join(pip_libraries)
+            if isinstance(conda_libraries, list):
+                conda_libraries = ", ".join(conda_libraries)
             current_env = db.query(Environment).filter(Environment.short_name == env_short_name).first()
-            status = "Draft" if current_env and current_env.status == "Saved" else current_env.status
-            db.query(Environment).filter(Environment.short_name == env_short_name).update({"version": version, "py_requirements": libraries,"base_image_id": base_env_id,"py_version": py_version,"status": status})
+            if not current_env:
+                raise ValueError(f"Environment with short name '{env_short_name}' does not exist.")
+            
+            env_status = "Draft" if status == "success" else "Failed"
+
+            db.query(Environment).filter(
+                Environment.short_name == env_short_name
+            ).update({"version": version, "pip_libraries": pip_libraries, "conda_libraries": conda_libraries, "status": env_status})
             db.commit()
 
         except Exception as e:
+            self.logger.error(f"Failed to update environment {env_short_name} in database: {e}")
             db.rollback()
             raise
 
-    
-    def update_library_versions(self, libraries: list, conda_env_path: str) -> list:
+    def update_library_versions(self, yaml_path: str):
         """
-        Updates libraries without version specifications by getting their actual installed versions.
+        Updates libraries without version specifications by getting their actual installed versions from a conda YAML file.
         
         Args:
-            libraries (list): List of library requirements, some may not have version specs.
-            conda_env_path (str): Path to the conda environment where libraries are installed.
+            yaml_path (str): Path to the conda environment YAML file.
             
         Returns:
-            list: Updated list of libraries with version specifications.
+            tuple: Updated lists of (pip_libraries, conda_libraries) with version specifications.
         """
         try:
-            pip_freeze_cmd = f"{conda_env_path}/bin/pip freeze"
-            result = subprocess.run(
-                pip_freeze_cmd, 
-                shell=True, 
-                capture_output=True, 
-                text=True,
-                check=True
-            )
+            # Define default conda packages to ignore
+            default_conda_packages = {
+                "_libgcc_mutex", "_openmp_mutex", "bzip2", "ca-certificates", 
+                "ld_impl_linux-64", "libexpat", "libffi", "libgcc", "libgcc-ng", 
+                "libgomp", "liblzma", "libnsl", "libsqlite", "libuuid", "libxcrypt", 
+                "libzlib", "ncurses", "openssl", "readline", "setuptools", "tk", 
+                "tzdata", "wheel", "libstdcxx-ng", "python"
+            }
             
-            installed_versions = {}
-            for line in result.stdout.splitlines():
-                if "==" in line:
-                    lib_name, version = line.split("==", 1)
-                    installed_versions[lib_name.lower()] = version
+            # Read the YAML file
+            with open(yaml_path, 'r') as f:
+                yaml_content = yaml.safe_load(f)
             
-            # Update libraries without version specs
-            updated_libraries = []
-            for lib in libraries:
-                # Skip libraries that are python version specifications
-                if lib.lower().startswith("python=="):
-                    continue
-                    
-                if "==" not in lib:
-                    lib_name = lib.strip()
-                    lib_name_lower = lib_name.lower()
+            # Extract conda and pip dependencies
+            dependencies = yaml_content.get('dependencies', [])
+            
+            # Process conda libraries
+            conda_libraries = []
+            pip_libraries = []
+            
+            for dep in dependencies:
+                if isinstance(dep, str):
+                    if dep.startswith("python="):
+                        continue
                     
-                    if lib_name_lower in installed_versions:
-                        updated_libraries.append(f"{lib_name}=={installed_versions[lib_name_lower]}")
-                    else:
-                        updated_libraries.append(lib)
-                else:
-                    updated_libraries.append(lib)
+                    parts = dep.split('=')
+                    package_name = parts[0].strip()
                     
-            return updated_libraries
+                    if package_name.lower() not in default_conda_packages:
+                        if len(parts) >= 2:
+                            package_with_version = f"{package_name}={parts[1]}"
+                            conda_libraries.append(package_with_version)
+                        else:
+                            # No version specified, keep as is
+                            conda_libraries.append(dep)
+                
+                elif isinstance(dep, dict) and 'pip' in dep:
+                    # This is the pip section
+                    for pip_pkg in dep['pip']:
+                        pip_libraries.append(pip_pkg)
+            
+            return pip_libraries, conda_libraries
+            
+        except Exception as e:
+            self.logger.error(f"Error reading YAML file and extracting libraries: {str(e)}")
+            return [], []
             
         except subprocess.CalledProcessError as e:
-            print(f"Error running pip freeze: {e.stderr}")
-            return libraries
+            self.logger.error(f"Error running pip freeze: {e.stderr}")
+            return pip_libraries
+        except Exception as e:
+            self.logger.error(f"Error updating library versions: {str(e)}")
+            return pip_libraries
+
+    def create_conda_yaml(self, yaml_path, env_name, python_version, conda_packages, pip_packages):
+        """
+        Creates a conda environment YAML file with specified packages and channels.
+        
+        Args:
+            yaml_path (str): Path where to save the YAML file
+            env_name (str): Name of the conda environment
+            python_version (str): Python version to use
+            conda_channels (list): List of conda channels
+            conda_packages (list): List of conda packages to install
+            pip_packages (list): List of pip packages to install
+        """
+        try:
+            # Create the environment specification
+            env_spec = {
+                "name": env_name,
+                "channels": ["conda-forge", "defaults"],
+                "dependencies": [
+                    f"python={python_version}"
+                ]
+            }
+            
+            # Add conda packages
+            if conda_packages and len(conda_packages) > 0:
+                env_spec["dependencies"].extend(conda_packages)
+
+            pip_pattern = re.compile(r"^pip([=]{1,2}.*)?$")  # matches pip, pip=..., pip==...
+            pip_found = any(pip_pattern.match(pkg.strip()) for pkg in conda_packages)
+            
+            # if pip is not already included in conda packages, add it
+            if not pip_found:
+                env_spec["dependencies"].append("pip")
+        
+            # Add pip packages if any
+            if pip_packages and len(pip_packages) > 0:
+                pip_section = {
+                    "pip": pip_packages
+                }
+                env_spec["dependencies"].append(pip_section)
+                
+            with open(yaml_path, 'w') as yaml_file:
+                yaml.dump(env_spec, yaml_file, default_flow_style=False)
+                
+            return yaml_path
+            
         except Exception as e:
-            print(f"Error updating library versions: {str(e)}")
-            return libraries
+            self.logger.error(f"Failed to create conda environment YAML file: {str(e)}")
+            raise Exception(f"Failed to create conda environment YAML file: {str(e)}")
+       
+    def format_py_requirements(self, env):
+        """
+        Format pip and conda libraries into a standardized list of dictionaries
+        sorted alphabetically by library name.
+        
+        Args:
+            env: Environment object containing pip_libraries and conda_libraries strings
+            
+        Returns:
+            list: List of dictionaries with format [{"name":"lib_name", "version":"version", "manager":"pip|conda"}, ...]
+        """
+        py_requirements = []
+        
+        # process libraries, handle both '==' and '=' version specifications
+        if env.pip_libraries:
+            for lib in env.pip_libraries.split(','):
+                lib = lib.strip()
+                if not lib:
+                    continue
+                    
+                if '==' in lib:
+                    name, version = lib.split('==', 1)
+                    py_requirements.append({
+                        "name": name.strip(),
+                        "version": version.strip(),
+                        "manager": "pip"
+                    })
+                elif '=' in lib:
+                    name, version = lib.split('=', 1)
+                    py_requirements.append({
+                        "name": name.strip(),
+                        "version": version.strip(),
+                        "manager": "pip"
+                    })
+                else:
+                    py_requirements.append({
+                        "name": lib,
+                        "version": "",
+                        "manager": "pip"
+                    })
+        
+        if env.conda_libraries:
+            for lib in env.conda_libraries.split(','):
+                lib = lib.strip()
+                if not lib:
+                    continue
+                    
+                if '==' in lib:
+                    name, version = lib.split('==', 1)
+                    py_requirements.append({
+                        "name": name.strip(),
+                        "version": version.strip(),
+                        "manager": "conda"
+                    })
+                elif '=' in lib:
+                    name, version = lib.split('=', 1)
+                    py_requirements.append({
+                        "name": name.strip(),
+                        "version": version.strip(),
+                        "manager": "conda"
+                    })
+                else:
+                    py_requirements.append({
+                        "name": lib,
+                        "version": "",
+                        "manager": "conda"
+                    })
+
+        # sort the requirements list alphabetically by name
+        py_requirements.sort(key=lambda x: x["name"].lower())
+        
+        return py_requirements

dataflow/models/environment.py

@@ -15,22 +15,21 @@ class EnvironmentAttributes(Base):
     enabled = Column(Boolean, default=True)
     version = Column(String, default=0)
     is_latest = Column(Boolean, default=True)
-    base_image_id = Column(Integer, default=None)
+    base_env_id = Column(Integer, default=None)
     short_name = Column(String(5))
     status = Column(String, default="Saved")
     icon = Column(String)
     py_version = Column(String)
     r_version = Column(String)
-    py_requirements = Column(Text)
+    pip_libraries = Column(Text)
+    conda_libraries = Column(Text)
     r_requirements = Column(Text)
-    py_requirements_compiled = Column(Text)
-    r_requirements_compiled = Column(Text)
     created_date = Column(DateTime, server_default=func.now())
     created_by = Column(String)
 
 
 
-class Environment(EnvironmentAttributes):
+class Environment(EnvironmentAttributes): 
     __tablename__ = 'ENVIRONMENT'
 
     id = Column(Integer, primary_key=True, autoincrement=True)
@@ -43,7 +42,7 @@ class ArchivedEnvironment(EnvironmentAttributes):
     __tablename__ = 'ARCHIVED_ENVIRONMENT'
 
     id = Column(Integer, primary_key=True, autoincrement=True)
-    original_env_id = Column(Integer, ForeignKey('ENVIRONMENT.id'))
+    original_env_id = Column(Integer, ForeignKey('ENVIRONMENT.id', ondelete='CASCADE'))
     is_latest = Column(Boolean, default=False) 
 
     # Relationship with Environment

dataflow/models/environment_status.py

@@ -10,7 +10,7 @@ class EnvironmentStatus(Base):
 
     __tablename__='ENVIRONMENT_STATUS'
 
-    id = Column(Integer, ForeignKey('ENVIRONMENT.id'), primary_key=True, nullable=False)
+    id = Column(Integer, ForeignKey('ENVIRONMENT.id', ondelete='CASCADE'), primary_key=True, nullable=False)
     status = Column(String, nullable=False)
     comment = Column(String)
     status_changed_date = Column(DateTime, server_default=func.now(), nullable=False)

dataflow/models/role.py

@@ -13,7 +13,7 @@ class Role(Base):
     id = Column(Integer, primary_key=True, index=True, autoincrement=True, nullable=False)
     name = Column(String, unique=True, nullable=False)
     description = Column(String, nullable=True)
-    base_role = Column(Enum('admin', 'user', name='base_role_field'), default='user', nullable=False)
+    base_role = Column(Enum('admin', 'user', 'applicant', name='base_role_field'), default='user', nullable=False)
 
     users = relationship("User", back_populates="role_details", cascade="all, delete-orphan")
     role_server_assocs = relationship("RoleServer", back_populates="role")

dataflow/scripts/clone_environment.sh

@@ -12,4 +12,6 @@ trap 'rm -rf "$CONDA_PKGS_DIRS"' EXIT
 # 1. Cloning conda env
 conda create --clone ${source_env_name} --prefix ${target_env_path} --yes
 
+conda env export --prefix "$conda_env_path" > "$yaml_file_path"
+
 echo "Environment Creation Successful"

dataflow/scripts/create_environment.sh

@@ -1,17 +1,29 @@
 #!/bin/bash
+# filepath: /home/hari/dbo/dataflow-core/dataflow/scripts/create_environment.sh
 set -e
 
-IFS=',' read -r -a  libraries <<< $1
+# Accept new parameters
+yaml_file_path=$1
 conda_env_path=$2
-py_version=$3
+
+# Validate inputs
+if [ -z "$yaml_file_path" ] || [ -z "$conda_env_path" ]; then
+    echo "Error: Missing required parameters"
+    exit 1
+fi
+
+if [ ! -f "$yaml_file_path" ]; then
+    echo "Error: YAML file does not exist: $yaml_file_path"
+    exit 1
+fi
 
 # Use an isolated conda package cache to avoid concurrency issues
 export CONDA_PKGS_DIRS=$(mktemp -d)
+
 # to delete conda package cache after script finishes
 trap 'rm -rf "$CONDA_PKGS_DIRS"' EXIT
 
-# 1. Creating conda environment
-conda create --prefix ${conda_env_path} --yes python=${py_version} 
+# Create the conda environment from the YAML file
+conda env create --file "$yaml_file_path" --prefix "$conda_env_path" --yes
 
-# 2. Install user libraries
-${conda_env_path}/bin/pip install --root-user-action ignore ${libraries[@]}      
+conda env export --prefix "$conda_env_path" > "$yaml_file_path"

dataflow/scripts/update_environment.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+# filepath: /home/hari/dbo/dataflow-core/dataflow/scripts/update_environment.sh
+set -e
+
+# Accept parameters
+yaml_file_path=$1
+conda_env_path=$2
+
+# Validate inputs
+if [ -z "$yaml_file_path" ] || [ -z "$conda_env_path" ]; then
+    echo "Error: Missing required parameters"
+    exit 1
+fi
+
+if [ ! -f "$yaml_file_path" ]; then
+    echo "Error: YAML file does not exist: $yaml_file_path"
+    exit 1
+fi
+
+if [ ! -d "$conda_env_path" ]; then
+    echo "Error: Conda environment does not exist at: $conda_env_path"
+    exit 1
+fi
+
+# Use an isolated conda package cache to avoid concurrency issues
+export CONDA_PKGS_DIRS=$(mktemp -d)
+
+# to delete conda package cache after script finishes
+trap 'rm -rf "$CONDA_PKGS_DIRS"' EXIT
+
+# Update the conda environment using the YAML file
+conda env update --prefix "$conda_env_path" --file "$yaml_file_path" --prune
+
+if [ ! -L "$yaml_file_path" ]; then
+    conda env export --prefix "$conda_env_path" > "$yaml_file_path"
+fi

{dataflow_core-2.1.1.dist-info → dataflow_core-2.1.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataflow-core
-Version: 2.1.1
+Version: 2.1.3
 Summary: Dataflow core package
 Author: Dataflow
 Author-email: 

{dataflow_core-2.1.1.dist-info → dataflow_core-2.1.3.dist-info}/RECORD

@@ -1,25 +1,25 @@
 authenticator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 authenticator/dataflowairflowauthenticator.py,sha256=gEdCiL2yJQ7lYvAwbrjcAkccVMfehoMJldw9eU7cc2s,2243
-authenticator/dataflowhubauthenticator.py,sha256=A0S0_PJVCPiTfuoLormmO0xNDqjIh00Cf93P0-mF0cw,2913
+authenticator/dataflowhubauthenticator.py,sha256=-wFEPEQfgCgtghC0Eo9e18B-OU1JsPKam0tbQaYCg1s,13563
 authenticator/dataflowsupersetauthenticator.py,sha256=NkAmDaIc-ui-qEolu4xz_UY7P_2g8111hwNjPvAOW1Q,2839
 dataflow/__init__.py,sha256=WTRg8HMpMWSgxYJ9ZGVldx4k07fAbta3mBmZ1hG9mWE,30
 dataflow/configuration.py,sha256=7To6XwH1eESiYp39eqPcswXWwrdBUdPF6xN6WnazOF0,663
 dataflow/database_manager.py,sha256=tJHMuOZ9Muskrh9t4uLRlTuFU0VkHAzoHlGP5DORIC4,899
 dataflow/dataflow.py,sha256=-UYZst7EO1GgaOjlAkKu-tu7RC6XsgadGeDp1MOvZiA,7439
 dataflow/db.py,sha256=5UwE4w5Vn9RqFIhr8ARlu2haZX-PtHDLRPjmn5BG2m8,1649
-dataflow/environment.py,sha256=04LwJwAjciBOz-EJJGzBA_BSQRKf587TJ30R8cDzvhg,19582
+dataflow/environment.py,sha256=eAWYhQFM7CA3o-CA3qTWbkE8ZHZ-Jo2qvSkDQ4dKqmY,27238
 dataflow/models/__init__.py,sha256=QMLiKj8BMhfScWMm8kgHkMjwAlFeg5Cym3_AI1NvBUA,783
 dataflow/models/app_types.py,sha256=yE_ZB13lhpK7AZ7PyBwnQlf0RlIHYs_-vdMKx7_RMlY,379
 dataflow/models/blacklist_library.py,sha256=B2oi3Z8GcR_glhLAyinFk0W8c9txXvm3uOER6dY-q7I,991
 dataflow/models/connection.py,sha256=_VJL3KuIrm8t4lJmtunIL3-AXF9Yvi5wUolzdR3tE0E,1017
-dataflow/models/environment.py,sha256=HQW1L0qKksFR0qeQQbwcPjd-zcZwS1PEvzMOFrKgnpE,2142
-dataflow/models/environment_status.py,sha256=GnoAKI8GdCTqTq4HLhx16K2k--LMpjeJuRRRjGfAXoA,516
+dataflow/models/environment.py,sha256=Vg-4vQe_cHIfOX-kPJWd6SIEapPJcgxoCITgdRrqT_o,2107
+dataflow/models/environment_status.py,sha256=lvPDNUsUoTW9D97B07aKqJQHRKp4LvPM28pQDMPH1ac,536
 dataflow/models/git_ssh.py,sha256=W15SDypxzGOz_aZkHEnVZ6DIMVsjAsbSIXVIEt2mPYU,694
 dataflow/models/pinned_projects.py,sha256=rkpPX_f2U9HjmrRo7_K8rnZIeXuQKGq6hYTrtLmu21c,566
 dataflow/models/project_details.py,sha256=94wTygXv9iGB0w8g_6vtkB5ZqIzpEv1W9uWwCA4hM0Y,1078
 dataflow/models/recent_project_studio.py,sha256=m12KGCsv453C1ijHjfVD8E7cJ7Og_0N8uc7_9VlfkYw,812
 dataflow/models/recent_projects.py,sha256=QqDlk3ll7tBaQl5hqvRarlB9_SUBuN44muLIuTVbPe0,301
-dataflow/models/role.py,sha256=i068kzy1UYfl2XNPGl8O4h-1Dcpxb_zzrbI9IW2FAxI,680
+dataflow/models/role.py,sha256=_I5F2TFox_k2-LGgjVuO2PIW9-gpwpoX8Te7m024A8k,693
 dataflow/models/role_server.py,sha256=mMcfjsGX1cY8hOAOBBmrZgw8ozdfuvjKJoBlR6F0Kdc,689
 dataflow/models/runtime.py,sha256=OiuBfZTMg81U10GS00DxfhiAmHlcyQUw5LBR8RaPl7s,415
 dataflow/models/server_config.py,sha256=GTMtQfgtuvKUbxV16VhEpKGhYoNISFLRWdUPqBJmYbM,1365
@@ -29,15 +29,16 @@ dataflow/models/user.py,sha256=PT-zwZj7NWUubIj_7EY2EsjduMbI_42EyMclWMLESGk,1073
 dataflow/models/user_environment.py,sha256=yI9NutULcLiwlycuEin6ROe6o1Sjdv_sgw2MEkJFeYg,568
 dataflow/models/user_team.py,sha256=r_fmKvf6JuGgiiI9TXWjVG2QZ3WOvDrOwYWVQ3r8oWo,659
 dataflow/models/variables.py,sha256=Sinvv3zFYni5i_GrL69cVfhCh4tOOaIHiEzWYRJ-i10,1132
-dataflow/scripts/clone_environment.sh,sha256=PJfMWPgiWSkvY-x98WmjeAkRREjC9824JzTazHe2iQQ,390
-dataflow/scripts/create_environment.sh,sha256=ams50MD1r53cHRYDfpAvmEAMsCaCFvlL0cmnRVXhFgY,496
+dataflow/scripts/clone_environment.sh,sha256=dVs-NAGHtpYsk-OjoZ_gbYuZZoi3jIbEp_zXbDXEVbc,455
+dataflow/scripts/create_environment.sh,sha256=TLJ7FKYyhsLe0bqBy74FnpuvjFTSHiGXp1iLMcOaeJA,798
+dataflow/scripts/update_environment.sh,sha256=p8r2qV4blqLyC7eksHSkUDoXx_UL4Xc4NWmx8y0h_rc,962
 dataflow/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dataflow/utils/aws_secrets_manager.py,sha256=A_fNs9VNah9dDdl9NhqizJamYU7xr2v_GXlw9InEDFk,2380
 dataflow/utils/get_current_user.py,sha256=akjcUyTpmMdAZj9LFGSTs76hjBRjltNk9hLUqC_BdkA,1140
 dataflow/utils/json_handler.py,sha256=5_7WdypegRBDe2HSqBXyrJAdd92wsha8qRcmQvCj1TA,782
 dataflow/utils/logger.py,sha256=7BFrOq5Oiqn8P4XZbgJzMP5O07d2fpdECbbfsjrUuHw,1213
-dataflow_core-2.1.1.dist-info/METADATA,sha256=WdN8Xh9UGcvgYnv-i-CB9fANkjBlUjcw_NawSc2_EcY,301
-dataflow_core-2.1.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dataflow_core-2.1.1.dist-info/entry_points.txt,sha256=ppj_EIbYrJJwCPg1kfdsZk5q1N-Ejfis1neYrnjhO8o,117
-dataflow_core-2.1.1.dist-info/top_level.txt,sha256=SZsUOpSCK9ntUy-3Tusxzf5A2e8ebwD8vouPb1dPt_8,23
-dataflow_core-2.1.1.dist-info/RECORD,,
+dataflow_core-2.1.3.dist-info/METADATA,sha256=dRhOgxGmyMCLl9eJQ0XfExl5Fp0BYZzdJfYGgvVS3Mo,301
+dataflow_core-2.1.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dataflow_core-2.1.3.dist-info/entry_points.txt,sha256=ppj_EIbYrJJwCPg1kfdsZk5q1N-Ejfis1neYrnjhO8o,117
+dataflow_core-2.1.3.dist-info/top_level.txt,sha256=SZsUOpSCK9ntUy-3Tusxzf5A2e8ebwD8vouPb1dPt_8,23
+dataflow_core-2.1.3.dist-info/RECORD,,