dataflow-core 2.1.2__py3-none-any.whl → 2.1.4__py3-none-any.whl

dataflow/environment.py

@@ -1,18 +1,21 @@
-import os, shutil, subprocess, datetime
+import os, shutil, subprocess, datetime, yaml, re
 from .models.environment import JobLogs, Environment
 import json, asyncio, pkg_resources
 from sqlalchemy.orm import Session
 from .configuration import ConfigurationManager
+from .utils.logger import CustomLogger
 
 class EnvironmentManager:
     def __init__(self):
         """Initialize the EnvironmentManager"""
         self.config = ConfigurationManager('/dataflow/app/config/dataflow.cfg')
-        self.published_env_path = self.config.get_config_value('paths', 'published_env_path')
-        self.draft_env_path = self.config.get_config_value('paths', 'drafts_env_path')
+        self.env_base_path = self.config.get_config_value('paths', 'env_path')
         self.env_logs_path = self.config.get_config_value('paths', 'env_logs_path')
+        self.env_version_path = self.config.get_config_value('paths', 'env_versions_path')
+        os.makedirs(self.env_version_path, exist_ok=True)
+        self.logger = CustomLogger().get_logger(__name__)
     
-    async def create_env(self, env_name, py_version, py_requirements, status, base_env_id, env_version=None, user_name=None, db:Session=None):
+    async def create_env(self, env_name, py_version, pip_libraries, conda_libraries, status, env_version='1', user_name=None, db:Session=None):
         """
         Creates a conda environment with specified Python version and packages.
         
@@ -26,47 +29,56 @@ class EnvironmentManager:
             db (Session): Database session (optional, will create if None)
             
         Returns:
-            str: Build status ('success' or 'fail')
+            str: Build status ('success' or 'failed')
         """
         # Set up logging
         log_file_location = None
         if db:
             log_file_location = self._setup_logging(env_name, env_version, user_name, db)
 
+        # Create the conda environment YAML file
+        yaml_path = os.path.join(self.env_version_path, f"{env_name}_v{env_version}.yaml")
+        self.create_conda_yaml(
+            yaml_path=yaml_path,
+            env_name=env_name,
+            python_version=py_version,
+            conda_packages=conda_libraries,
+            pip_packages=pip_libraries
+        )
+
         if status == "published":
             return await self._execute_env_operation(
                 env_name=env_name,
-                py_version=py_version,
-                py_requirements=py_requirements,
                 status="published",
-                env_version=None,
-                mode="create"
+                mode="create",
+                yaml_file_path=yaml_path,
+                version=int(env_version)
             )
         elif status == "draft":
-            # Build the environment
+            mode = "create" if env_version == '1' else "update"
             build_status = await self._execute_env_operation(
                 env_name=env_name,
-                py_version=py_version,
-                py_requirements=py_requirements,
                 status=status,
-                env_version=env_version,
+                mode=mode,
+                yaml_file_path=yaml_path,
                 log_file_location=log_file_location,
-                mode="create"
+                version=int(env_version)
             )
             
             # Update job log status if db was provided
             if db and log_file_location:
                 log_file_name = os.path.basename(log_file_location)
                 await self._update_job_status(log_file_name, build_status, log_file_location, db)
-                updated_py_requirements = self.update_library_versions(py_requirements, os.path.join(self.draft_env_path, env_name, f"{env_name}_v{env_version}"))
-                self.update_environment_db(env_name, env_version, updated_py_requirements, base_env_id, py_version, db)
+                pip_libraries, conda_libraries = self.update_library_versions(yaml_path)
+                self.update_environment_db(env_name, env_version, pip_libraries, conda_libraries, build_status, db)
 
             return build_status
         
         else:
+            self.logger.error(f"Invalid status '{status}' provided for environment creation.")
             raise ValueError("Invalid status. Use 'draft' or 'published'.")
     
-    async def clone_env(self, source_path, target_env_name, libraries, py_version, user_name=None, db: Session=None):
+    async def clone_env(self, source_path, env_name, pip_libraries, conda_libraries, user_name, db=None):
         """
         Clones an existing conda environment.
         
@@ -79,56 +91,95 @@ class EnvironmentManager:
             db (Session): Database session (optional, will create if None)
             
         Returns:
-            str: Build status ('success' or 'fail')
+            str: Build status ('success' or 'failed')
         """
         # Set up logging
         log_file_location = None
         if db:
-            log_file_location = self._setup_logging(target_env_name, "1", user_name, db)
+            log_file_location = self._setup_logging(env_name, "1", user_name, db)
+
+        yaml_path = f"{self.env_version_path}/{env_name}_v1.yaml"
         
         # Perform the clone operation
         clone_status = await self._execute_env_operation(
-            env_name=target_env_name,
-            source_path=source_path,
+            env_name=env_name,
             status="draft",
-            env_version="1",
+            mode="clone",
+            yaml_file_path=yaml_path,
+            source_path=source_path,
             log_file_location=log_file_location,
-            mode="clone"
+            version=1           
         )
         
         # Update job log status if db was provided
         if db and log_file_location:
             log_file_name = os.path.basename(log_file_location)
             await self._update_job_status(log_file_name, clone_status, log_file_location, db)
-            self.update_environment_db(env_short_name=target_env_name, version="1", libraries=libraries, base_env_id=None, py_version=py_version, db=db)
+            self.update_environment_db(
+                env_short_name=env_name, 
+                version="1", 
+                pip_libraries=pip_libraries, 
+                conda_libraries=conda_libraries,
+                status=clone_status, 
+                db=db
+            )
             
         return clone_status
     
-    async def create_published_env(self, env_name, py_version, py_requirements):
+    async def revert_env(self, env_name, curr_version, revert_version, new_version, user_name, db: Session):
         """
-        Creates a published conda environment.
+        Reverts an environment to a previous version.
         
         Args:
             env_name (str): Name of the environment
-            py_version (str): Python version to use
-            py_requirements (list): List of packages to install
-            status (str): Environment status ('draft' or 'published')
-            env_version (str): Version of the environment (for draft environments)
-            user_name (str): Username who initiated the creation
+            version (str): Version to revert to
+            db (Session): Database session
             
         Returns:
-            str: Build status ('success' or 'fail')
+            str: Build status ('success' or 'failed')
         """
-        return self._execute_env_operation(
-            env_name=env_name,
-            py_version=py_version,
-            py_requirements=py_requirements,
-            status="published",
-            env_version=None,
-            mode="create"
-        )
+        try:            
+            # Get the YAML file for the specified version
+            old_yaml_path = f"{self.env_version_path}/{env_name}_v{revert_version}.yaml"
+            new_yaml_path = f"{self.env_version_path}/{env_name}_v{new_version}.yaml"
+            if not os.path.exists(old_yaml_path):
+                raise FileNotFoundError(f"YAML file for version {revert_version} does not exist.")
+            
+            os.symlink(old_yaml_path, new_yaml_path)
+
+            log_file_location = None
+            if db:
+                log_file_location = self._setup_logging(env_name, new_version, user_name, db)
+            
+            # Execute the revert operation
+            revert_status = await self._execute_env_operation(
+                env_name=env_name,
+                status="draft",
+                mode="update",
+                yaml_file_path=new_yaml_path,
+                log_file_location=log_file_location,
+                version=int(new_version)
+            )
+            
+            log_file_name = os.path.basename(log_file_location)
+            await self._update_job_status(log_file_name, revert_status, log_file_location, db)
+            pip_libraries, conda_libraries = self.update_library_versions(new_yaml_path)
+            self.update_environment_db(env_name, new_version, pip_libraries, conda_libraries, revert_status, db)
         
-    async def _execute_env_operation(self, env_name: str, status: str, mode: str, env_version: str = None, py_version: str = None, py_requirements=None, source_path=None, log_file_location=None):
+        except Exception as e:
+            self.logger.error(f"Failed to revert environment {env_name}: {e}")
+            return "failed"
+        
+    async def _execute_env_operation(
+        self, 
+        env_name: str, 
+        status: str, 
+        mode: str, 
+        yaml_file_path: str, 
+        version: int,
+        source_path=None, 
+        log_file_location=None,
+    ):
         """
         Executes environment operations (create or clone).
         
@@ -143,34 +194,32 @@ class EnvironmentManager:
             log_file_location (str): Path to log file
             
         Returns:
-            str: Build status ('success' or 'fail')
+            str: Build status ('success' or 'failed')
         """
+        self.logger.info(f"Executing environment operation: {env_name}, Status: {status}, Mode: {mode}")
         status = status.lower()
-        if status == "published":
-            env_base_path = self.config.get_config_value('paths', 'published_env_path')
-            conda_env_path = os.path.join(env_base_path, env_name)
-        else:
-            env_base_path = self.config.get_config_value('paths', 'drafts_env_path')
-            conda_env_path = os.path.join(env_base_path, env_name, f"{env_name}_v{env_version}")
+        conda_env_path = os.path.join(self.env_base_path, env_name)
 
         try:
-            if not os.path.exists(conda_env_path):
-                os.makedirs(conda_env_path, exist_ok=True)
+            if os.path.exists(conda_env_path) and mode == "create":
+                raise FileExistsError(f"Environment '{env_name}' already exists at {conda_env_path}.")
+            
+            os.makedirs(conda_env_path, exist_ok=True)
 
-            if mode == "create":
-                # Convert requirements list to comma-separated string
-                if isinstance(py_requirements, list):
-                    py_requirements = ",".join(py_requirements)
-                
+            if mode == "create":                
                 create_env_script_path = pkg_resources.resource_filename('dataflow', 'scripts/create_environment.sh')
-                command = ["bash", create_env_script_path, py_requirements, conda_env_path, py_version]
-                
+                command = ["bash", create_env_script_path, yaml_file_path, conda_env_path]
+
+            elif mode == "update":
+                update_env_script_path = pkg_resources.resource_filename('dataflow', 'scripts/update_environment.sh')
+                command = ["bash", update_env_script_path, yaml_file_path, conda_env_path]
+            
             elif mode == "clone":
                 clone_env_script_path = pkg_resources.resource_filename('dataflow', 'scripts/clone_environment.sh')
                 command = ["bash", clone_env_script_path, source_path, conda_env_path]
                 
             else:
-                raise ValueError("Invalid mode. Use 'create' or 'clone'.")
+                raise ValueError("Invalid mode. Use 'create', 'update', or 'clone'.")
 
             process = await asyncio.create_subprocess_exec(
                 *command, 
@@ -184,6 +233,16 @@ class EnvironmentManager:
             with open(log_file_location, "a") as log_file:
                 success_detected = False
                 try:
+                    # Write an initial log entry to indicate the operation has started
+                    start_message = {
+                        "timestamp": self.format_timestamp(),
+                        "type": "log",
+                        "content": f"Starting environment {mode} operation for {env_name}"
+                    }
+                    log_file.write(json.dumps(start_message) + "\n")
+                    log_file.flush()
+                    
+                    # Process stdout line by line
                     while True:
                         line = await process.stdout.readline()
                         if not line:
@@ -213,7 +272,7 @@ class EnvironmentManager:
                         }
                         log_file.write(json.dumps(error_message_dict) + "\n")
 
-                    final_build_status = "fail" if process.returncode != 0 else "success"
+                    final_build_status = "failed" if process.returncode != 0 else "success"
 
                 except asyncio.CancelledError:
                     process.kill()
@@ -224,27 +283,24 @@ class EnvironmentManager:
                         "content": msg_content
                     }
                     log_file.write(json.dumps(cancellation_message) + "\n")
-                    final_build_status = "fail"
+                    final_build_status = "failed"
                 
                 finally:
-                    if final_build_status == "success" and status == "draft":
-                        symlink_path = os.path.join(env_base_path, env_name, "default")
-                        self.update_symlink(symlink_path, conda_env_path)
-                    elif final_build_status != "success":
+                    if final_build_status != "success" and version == 1:
                         if os.path.exists(conda_env_path):
                             shutil.rmtree(conda_env_path)
                 
             return final_build_status
         
         except OSError as e:
-            print(f"OS error while operating on {conda_env_path}: {e}")
-            return "fail"
+            self.logger.error(f"OS error while operating on {conda_env_path}: {e}")
+            return "failed"
         except subprocess.CalledProcessError as e:
-            print(f"Subprocess error during environment operation: {e}")
-            return "fail"
+            self.logger.error(f"Subprocess error during environment operation: {e}")
+            return "failed"
         except Exception as e:
-            print(f"Unexpected error during environment operation for {env_name}: {e}")
-            return "fail"
+            self.logger.error(f"Unexpected error during environment operation for {env_name}: {e}")
+            return "failed"
     
     def _setup_logging(self, env_name: str, env_version: str, user_name: str, db: Session):
         """
@@ -281,7 +337,7 @@ class EnvironmentManager:
         Args:
             db (Session): Database session
             log_file_name (str): Name of the log file
-            build_status (str): Build status ('success' or 'fail') 
+            build_status (str): Build status ('success' or 'failed') 
             log_file_location (str): Path to the log file
         """
         attempts = 3
@@ -306,7 +362,7 @@ class EnvironmentManager:
                 if attempts > 0:
                     await asyncio.sleep(retry_delay)
                 else:
-                    print(f"Failed to update job log after multiple attempts: {e}")
+                    self.logger.error(f"Failed to update job log after multiple attempts: {e}")
     
     def create_job_entry(self, user_name: str, db: Session, log_file_name: str, log_file_location: str):
         """
@@ -325,8 +381,9 @@ class EnvironmentManager:
 
         if job:
             if job.status == "success":
+                self.logger.error(f"Job with log_file_name '{log_file_name}' already completed successfully.")
                 raise ValueError(f"Job with log_file_name '{log_file_name}' already completed successfully.")
-            if job.status == "fail":
+            if job.status == "failed":
                 job.created_at = datetime.datetime.now() 
                 job.status = "in_progress"
         else:
@@ -349,7 +406,7 @@ class EnvironmentManager:
         Args:
             db (Session): Database session
             log_file_name (str): Name of the log file
-            final_build_status (str): Final status of the build ('success' or 'fail')
+            final_build_status (str): Final status of the build ('success' or 'failed')
         """
         try:
             job_record = db.query(JobLogs).filter(JobLogs.log_file_name == log_file_name).first()
@@ -358,24 +415,12 @@ class EnvironmentManager:
                 job_record.status = final_build_status
                 db.commit()
             else:
+                self.logger.error(f"No job log found for file: {log_file_name}")
                 raise ValueError(f"No job log found for file: {log_file_name}")
         except Exception as e:
+            self.logger.error(f"Failed to update job log for {log_file_name}: {e}")
             db.rollback()
             raise
-            
-    def update_symlink(self, symlink_path, conda_env_path):
-        """
-        Creates or updates the symlink to point to the default version.
-        """
-        symlink_dir = os.path.dirname(symlink_path)
-        if not os.path.exists(symlink_dir):
-            os.makedirs(symlink_dir, exist_ok=True)
-
-        # If symlink exists, remove it before updating
-        if os.path.islink(symlink_path):
-            os.remove(symlink_path)
-
-        subprocess.run(["ln", "-sf", conda_env_path, symlink_path], check=True)
 
     def format_timestamp(self):
         """
@@ -386,73 +431,214 @@ class EnvironmentManager:
         """
         return datetime.datetime.now().strftime("%b %d  %I:%M:%S %p")
     
-    def update_environment_db(self, env_short_name, version, libraries, base_env_id, py_version, db: Session):
+    def update_environment_db(self, env_short_name, version, pip_libraries, conda_libraries, status, db: Session):
         """
         Updates the environment table with the new version and libraries.
         """
         try:
-            if isinstance(libraries, list):
-                libraries = ", ".join(libraries)
+            if isinstance(pip_libraries, list):
+                pip_libraries = ", ".join(pip_libraries)
+            if isinstance(conda_libraries, list):
+                conda_libraries = ", ".join(conda_libraries)
             current_env = db.query(Environment).filter(Environment.short_name == env_short_name).first()
-            status = "Draft" if current_env and current_env.status == "Saved" else current_env.status
-            db.query(Environment).filter(Environment.short_name == env_short_name).update({"version": version, "py_requirements": libraries,"base_image_id": base_env_id,"py_version": py_version,"status": status})
+            if not current_env:
+                raise ValueError(f"Environment with short name '{env_short_name}' does not exist.")
+            
+            env_status = "Draft" if status == "success" else "Failed"
+
+            db.query(Environment).filter(
+                Environment.short_name == env_short_name
+            ).update({"version": version, "pip_libraries": pip_libraries, "conda_libraries": conda_libraries, "status": env_status})
             db.commit()
 
         except Exception as e:
+            self.logger.error(f"Failed to update environment {env_short_name} in database: {e}")
             db.rollback()
             raise
 
-    
-    def update_library_versions(self, libraries: list, conda_env_path: str) -> list:
+    def update_library_versions(self, yaml_path: str):
         """
-        Updates libraries without version specifications by getting their actual installed versions.
+        Updates libraries without version specifications by getting their actual installed versions from a conda YAML file.
         
         Args:
-            libraries (list): List of library requirements, some may not have version specs.
-            conda_env_path (str): Path to the conda environment where libraries are installed.
+            yaml_path (str): Path to the conda environment YAML file.
             
         Returns:
-            list: Updated list of libraries with version specifications.
+            tuple: Updated lists of (pip_libraries, conda_libraries) with version specifications.
         """
         try:
-            pip_freeze_cmd = f"{conda_env_path}/bin/pip freeze"
-            result = subprocess.run(
-                pip_freeze_cmd, 
-                shell=True, 
-                capture_output=True, 
-                text=True,
-                check=True
-            )
+            # Define default conda packages to ignore
+            default_conda_packages = {
+                "_libgcc_mutex", "_openmp_mutex", "bzip2", "ca-certificates", 
+                "ld_impl_linux-64", "libexpat", "libffi", "libgcc", "libgcc-ng", 
+                "libgomp", "liblzma", "libnsl", "libsqlite", "libuuid", "libxcrypt", 
+                "libzlib", "ncurses", "openssl", "readline", "setuptools", "tk", 
+                "tzdata", "wheel", "libstdcxx-ng", "python"
+            }
             
-            installed_versions = {}
-            for line in result.stdout.splitlines():
-                if "==" in line:
-                    lib_name, version = line.split("==", 1)
-                    installed_versions[lib_name.lower()] = version
+            # Read the YAML file
+            with open(yaml_path, 'r') as f:
+                yaml_content = yaml.safe_load(f)
             
-            # Update libraries without version specs
-            updated_libraries = []
-            for lib in libraries:
-                # Skip libraries that are python version specifications
-                if lib.lower().startswith("python=="):
-                    continue
-                    
-                if "==" not in lib:
-                    lib_name = lib.strip()
-                    lib_name_lower = lib_name.lower()
+            # Extract conda and pip dependencies
+            dependencies = yaml_content.get('dependencies', [])
+            
+            # Process conda libraries
+            conda_libraries = []
+            pip_libraries = []
+            
+            for dep in dependencies:
+                if isinstance(dep, str):
+                    if dep.startswith("python="):
+                        continue
                     
-                    if lib_name_lower in installed_versions:
-                        updated_libraries.append(f"{lib_name}=={installed_versions[lib_name_lower]}")
-                    else:
-                        updated_libraries.append(lib)
-                else:
-                    updated_libraries.append(lib)
+                    parts = dep.split('=')
+                    package_name = parts[0].strip()
                     
-            return updated_libraries
+                    if package_name.lower() not in default_conda_packages:
+                        if len(parts) >= 2:
+                            package_with_version = f"{package_name}={parts[1]}"
+                            conda_libraries.append(package_with_version)
+                        else:
+                            # No version specified, keep as is
+                            conda_libraries.append(dep)
+                
+                elif isinstance(dep, dict) and 'pip' in dep:
+                    # This is the pip section
+                    for pip_pkg in dep['pip']:
+                        pip_libraries.append(pip_pkg)
+            
+            return pip_libraries, conda_libraries
+            
+        except Exception as e:
+            self.logger.error(f"Error reading YAML file and extracting libraries: {str(e)}")
+            return [], []
             
         except subprocess.CalledProcessError as e:
-            print(f"Error running pip freeze: {e.stderr}")
-            return libraries
+            self.logger.error(f"Error running pip freeze: {e.stderr}")
+            return pip_libraries
+        except Exception as e:
+            self.logger.error(f"Error updating library versions: {str(e)}")
+            return pip_libraries
+
+    def create_conda_yaml(self, yaml_path, env_name, python_version, conda_packages, pip_packages):
+        """
+        Creates a conda environment YAML file with specified packages and channels.
+        
+        Args:
+            yaml_path (str): Path where to save the YAML file
+            env_name (str): Name of the conda environment
+            python_version (str): Python version to use
+            conda_channels (list): List of conda channels
+            conda_packages (list): List of conda packages to install
+            pip_packages (list): List of pip packages to install
+        """
+        try:
+            # Create the environment specification
+            env_spec = {
+                "name": env_name,
+                "channels": ["conda-forge", "defaults"],
+                "dependencies": [
+                    f"python={python_version}"
+                ]
+            }
+            
+            # Add conda packages
+            if conda_packages and len(conda_packages) > 0:
+                env_spec["dependencies"].extend(conda_packages)
+
+            pip_pattern = re.compile(r"^pip([=]{1,2}.*)?$")  # matches pip, pip=..., pip==...
+            pip_found = any(pip_pattern.match(pkg.strip()) for pkg in conda_packages)
+            
+            # if pip is not already included in conda packages, add it
+            if not pip_found:
+                env_spec["dependencies"].append("pip")
+        
+            # Add pip packages if any
+            if pip_packages and len(pip_packages) > 0:
+                pip_section = {
+                    "pip": pip_packages
+                }
+                env_spec["dependencies"].append(pip_section)
+                
+            with open(yaml_path, 'w') as yaml_file:
+                yaml.dump(env_spec, yaml_file, default_flow_style=False)
+                
+            return yaml_path
+            
         except Exception as e:
-            print(f"Error updating library versions: {str(e)}")
-            return libraries
+            self.logger.error(f"Failed to create conda environment YAML file: {str(e)}")
+            raise Exception(f"Failed to create conda environment YAML file: {str(e)}")
+       
+    def format_py_requirements(self, env):
+        """
+        Format pip and conda libraries into a standardized list of dictionaries
+        sorted alphabetically by library name.
+        
+        Args:
+            env: Environment object containing pip_libraries and conda_libraries strings
+            
+        Returns:
+            list: List of dictionaries with format [{"name":"lib_name", "version":"version", "manager":"pip|conda"}, ...]
+        """
+        py_requirements = []
+        
+        # process libraries, handle both '==' and '=' version specifications
+        if env.pip_libraries:
+            for lib in env.pip_libraries.split(','):
+                lib = lib.strip()
+                if not lib:
+                    continue
+                    
+                if '==' in lib:
+                    name, version = lib.split('==', 1)
+                    py_requirements.append({
+                        "name": name.strip(),
+                        "version": version.strip(),
+                        "manager": "pip"
+                    })
+                elif '=' in lib:
+                    name, version = lib.split('=', 1)
+                    py_requirements.append({
+                        "name": name.strip(),
+                        "version": version.strip(),
+                        "manager": "pip"
+                    })
+                else:
+                    py_requirements.append({
+                        "name": lib,
+                        "version": "",
+                        "manager": "pip"
+                    })
+        
+        if env.conda_libraries:
+            for lib in env.conda_libraries.split(','):
+                lib = lib.strip()
+                if not lib:
+                    continue
+                    
+                if '==' in lib:
+                    name, version = lib.split('==', 1)
+                    py_requirements.append({
+                        "name": name.strip(),
+                        "version": version.strip(),
+                        "manager": "conda"
+                    })
+                elif '=' in lib:
+                    name, version = lib.split('=', 1)
+                    py_requirements.append({
+                        "name": name.strip(),
+                        "version": version.strip(),
+                        "manager": "conda"
+                    })
+                else:
+                    py_requirements.append({
+                        "name": lib,
+                        "version": "",
+                        "manager": "conda"
+                    })
+
+        # sort the requirements list alphabetically by name
+        py_requirements.sort(key=lambda x: x["name"].lower())
+        
+        return py_requirements

dataflow/models/environment.py

@@ -15,22 +15,21 @@ class EnvironmentAttributes(Base):
     enabled = Column(Boolean, default=True)
     version = Column(String, default=0)
     is_latest = Column(Boolean, default=True)
-    base_image_id = Column(Integer, default=None)
+    base_env_id = Column(Integer, default=None)
     short_name = Column(String(5))
     status = Column(String, default="Saved")
     icon = Column(String)
     py_version = Column(String)
     r_version = Column(String)
-    py_requirements = Column(Text)
+    pip_libraries = Column(Text)
+    conda_libraries = Column(Text)
     r_requirements = Column(Text)
-    py_requirements_compiled = Column(Text)
-    r_requirements_compiled = Column(Text)
     created_date = Column(DateTime, server_default=func.now())
     created_by = Column(String)
 
 
 
-class Environment(EnvironmentAttributes):
+class Environment(EnvironmentAttributes): 
     __tablename__ = 'ENVIRONMENT'
 
     id = Column(Integer, primary_key=True, autoincrement=True)
@@ -43,7 +42,7 @@ class ArchivedEnvironment(EnvironmentAttributes):
     __tablename__ = 'ARCHIVED_ENVIRONMENT'
 
     id = Column(Integer, primary_key=True, autoincrement=True)
-    original_env_id = Column(Integer, ForeignKey('ENVIRONMENT.id'))
+    original_env_id = Column(Integer, ForeignKey('ENVIRONMENT.id', ondelete='CASCADE'))
     is_latest = Column(Boolean, default=False) 
 
     # Relationship with Environment

dataflow/models/environment_status.py

@@ -10,7 +10,7 @@ class EnvironmentStatus(Base):
 
     __tablename__='ENVIRONMENT_STATUS'
 
-    id = Column(Integer, ForeignKey('ENVIRONMENT.id'), primary_key=True, nullable=False)
+    id = Column(Integer, ForeignKey('ENVIRONMENT.id', ondelete='CASCADE'), primary_key=True, nullable=False)
     status = Column(String, nullable=False)
     comment = Column(String)
     status_changed_date = Column(DateTime, server_default=func.now(), nullable=False)

dataflow/scripts/clone_environment.sh

@@ -4,12 +4,16 @@ set -e
 source_env_name=$1
 target_env_path=$2
 
-# Use an isolated conda package cache to avoid concurrency issues
-export CONDA_PKGS_DIRS=$(mktemp -d)
-# to delete conda package cache after script finishes
-trap 'rm -rf "$CONDA_PKGS_DIRS"' EXIT
+# Extract just the env name (basename) from the target path
+env_name=$(basename "$target_env_path")
+
+# Set unique cache dir per environment
+export CONDA_PKGS_DIRS="/dataflow/envs/cache/${env_name}"
+mkdir -p "$CONDA_PKGS_DIRS"
 
 # 1. Cloning conda env
 conda create --clone ${source_env_name} --prefix ${target_env_path} --yes
 
+conda env export --prefix "$conda_env_path" > "$yaml_file_path"
+
 echo "Environment Creation Successful"

dataflow/scripts/create_environment.sh

@@ -1,17 +1,30 @@
 #!/bin/bash
+# filepath: /home/hari/dbo/dataflow-core/dataflow/scripts/create_environment.sh
 set -e
 
-IFS=',' read -r -a  libraries <<< $1
+# Accept new parameters
+yaml_file_path=$1
 conda_env_path=$2
-py_version=$3
 
-# Use an isolated conda package cache to avoid concurrency issues
-export CONDA_PKGS_DIRS=$(mktemp -d)
-# to delete conda package cache after script finishes
-trap 'rm -rf "$CONDA_PKGS_DIRS"' EXIT
+# Validate inputs
+if [ -z "$yaml_file_path" ] || [ -z "$conda_env_path" ]; then
+    echo "Error: Missing required parameters"
+    exit 1
+fi
 
-# 1. Creating conda environment
-conda create --prefix ${conda_env_path} --yes python=${py_version} 
+if [ ! -f "$yaml_file_path" ]; then
+    echo "Error: YAML file does not exist: $yaml_file_path"
+    exit 1
+fi
 
-# 2. Install user libraries
-${conda_env_path}/bin/pip install --root-user-action ignore ${libraries[@]}      
+# Extract just the env name (basename) from the target path
+env_name=$(basename "$conda_env_path")
+
+# Set unique cache dir per environment
+export CONDA_PKGS_DIRS="/dataflow/envs/cache/${env_name}"
+mkdir -p "$CONDA_PKGS_DIRS"
+
+# Create the conda environment from the YAML file
+conda env create --file "$yaml_file_path" --prefix "$conda_env_path" --yes
+
+conda env export --prefix "$conda_env_path" > "$yaml_file_path"

dataflow/scripts/update_environment.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# filepath: /home/hari/dbo/dataflow-core/dataflow/scripts/update_environment.sh
+set -e
+
+# Accept parameters
+yaml_file_path=$1
+conda_env_path=$2
+
+# Validate inputs
+if [ -z "$yaml_file_path" ] || [ -z "$conda_env_path" ]; then
+    echo "Error: Missing required parameters"
+    exit 1
+fi
+
+if [ ! -f "$yaml_file_path" ]; then
+    echo "Error: YAML file does not exist: $yaml_file_path"
+    exit 1
+fi
+
+if [ ! -d "$conda_env_path" ]; then
+    echo "Error: Conda environment does not exist at: $conda_env_path"
+    exit 1
+fi
+
+# Extract just the env name (basename) from the target path
+env_name=$(basename "$conda_env_path")
+
+# Set unique cache dir per environment
+export CONDA_PKGS_DIRS="/dataflow/envs/cache/${env_name}"
+mkdir -p "$CONDA_PKGS_DIRS"
+
+# Update the conda environment using the YAML file
+conda env update --prefix "$conda_env_path" --file "$yaml_file_path" --prune
+
+if [ ! -L "$yaml_file_path" ]; then
+    conda env export --prefix "$conda_env_path" > "$yaml_file_path"
+fi

{dataflow_core-2.1.2.dist-info → dataflow_core-2.1.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataflow-core
-Version: 2.1.2
+Version: 2.1.4
 Summary: Dataflow core package
 Author: Dataflow
 Author-email: 

{dataflow_core-2.1.2.dist-info → dataflow_core-2.1.4.dist-info}/RECORD

@@ -7,13 +7,13 @@ dataflow/configuration.py,sha256=7To6XwH1eESiYp39eqPcswXWwrdBUdPF6xN6WnazOF0,663
 dataflow/database_manager.py,sha256=tJHMuOZ9Muskrh9t4uLRlTuFU0VkHAzoHlGP5DORIC4,899
 dataflow/dataflow.py,sha256=-UYZst7EO1GgaOjlAkKu-tu7RC6XsgadGeDp1MOvZiA,7439
 dataflow/db.py,sha256=5UwE4w5Vn9RqFIhr8ARlu2haZX-PtHDLRPjmn5BG2m8,1649
-dataflow/environment.py,sha256=04LwJwAjciBOz-EJJGzBA_BSQRKf587TJ30R8cDzvhg,19582
+dataflow/environment.py,sha256=eAWYhQFM7CA3o-CA3qTWbkE8ZHZ-Jo2qvSkDQ4dKqmY,27238
 dataflow/models/__init__.py,sha256=QMLiKj8BMhfScWMm8kgHkMjwAlFeg5Cym3_AI1NvBUA,783
 dataflow/models/app_types.py,sha256=yE_ZB13lhpK7AZ7PyBwnQlf0RlIHYs_-vdMKx7_RMlY,379
 dataflow/models/blacklist_library.py,sha256=B2oi3Z8GcR_glhLAyinFk0W8c9txXvm3uOER6dY-q7I,991
 dataflow/models/connection.py,sha256=_VJL3KuIrm8t4lJmtunIL3-AXF9Yvi5wUolzdR3tE0E,1017
-dataflow/models/environment.py,sha256=HQW1L0qKksFR0qeQQbwcPjd-zcZwS1PEvzMOFrKgnpE,2142
-dataflow/models/environment_status.py,sha256=GnoAKI8GdCTqTq4HLhx16K2k--LMpjeJuRRRjGfAXoA,516
+dataflow/models/environment.py,sha256=Vg-4vQe_cHIfOX-kPJWd6SIEapPJcgxoCITgdRrqT_o,2107
+dataflow/models/environment_status.py,sha256=lvPDNUsUoTW9D97B07aKqJQHRKp4LvPM28pQDMPH1ac,536
 dataflow/models/git_ssh.py,sha256=W15SDypxzGOz_aZkHEnVZ6DIMVsjAsbSIXVIEt2mPYU,694
 dataflow/models/pinned_projects.py,sha256=rkpPX_f2U9HjmrRo7_K8rnZIeXuQKGq6hYTrtLmu21c,566
 dataflow/models/project_details.py,sha256=94wTygXv9iGB0w8g_6vtkB5ZqIzpEv1W9uWwCA4hM0Y,1078
@@ -29,15 +29,16 @@ dataflow/models/user.py,sha256=PT-zwZj7NWUubIj_7EY2EsjduMbI_42EyMclWMLESGk,1073
 dataflow/models/user_environment.py,sha256=yI9NutULcLiwlycuEin6ROe6o1Sjdv_sgw2MEkJFeYg,568
 dataflow/models/user_team.py,sha256=r_fmKvf6JuGgiiI9TXWjVG2QZ3WOvDrOwYWVQ3r8oWo,659
 dataflow/models/variables.py,sha256=Sinvv3zFYni5i_GrL69cVfhCh4tOOaIHiEzWYRJ-i10,1132
-dataflow/scripts/clone_environment.sh,sha256=PJfMWPgiWSkvY-x98WmjeAkRREjC9824JzTazHe2iQQ,390
-dataflow/scripts/create_environment.sh,sha256=ams50MD1r53cHRYDfpAvmEAMsCaCFvlL0cmnRVXhFgY,496
+dataflow/scripts/clone_environment.sh,sha256=xWJBw9z1W1rztrzLXYro3UtEdFuBSqNrB83y45zqFfE,487
+dataflow/scripts/create_environment.sh,sha256=3FHgNplJuEZvyTsLqlCJNX9oyfXgsfqn80VZk2xtvso,828
+dataflow/scripts/update_environment.sh,sha256=2dtn2xlNi6frpig-sqlGE1_IKRbbkqYOCpf_qyMKKII,992
 dataflow/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dataflow/utils/aws_secrets_manager.py,sha256=A_fNs9VNah9dDdl9NhqizJamYU7xr2v_GXlw9InEDFk,2380
 dataflow/utils/get_current_user.py,sha256=akjcUyTpmMdAZj9LFGSTs76hjBRjltNk9hLUqC_BdkA,1140
 dataflow/utils/json_handler.py,sha256=5_7WdypegRBDe2HSqBXyrJAdd92wsha8qRcmQvCj1TA,782
 dataflow/utils/logger.py,sha256=7BFrOq5Oiqn8P4XZbgJzMP5O07d2fpdECbbfsjrUuHw,1213
-dataflow_core-2.1.2.dist-info/METADATA,sha256=4eTzui9zX33CbX6cQdj1p2uzZIq6OMD-ex_mAWzEY2E,301
-dataflow_core-2.1.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dataflow_core-2.1.2.dist-info/entry_points.txt,sha256=ppj_EIbYrJJwCPg1kfdsZk5q1N-Ejfis1neYrnjhO8o,117
-dataflow_core-2.1.2.dist-info/top_level.txt,sha256=SZsUOpSCK9ntUy-3Tusxzf5A2e8ebwD8vouPb1dPt_8,23
-dataflow_core-2.1.2.dist-info/RECORD,,
+dataflow_core-2.1.4.dist-info/METADATA,sha256=ImIZofq3kQ0DOVQxA4YPc2Tcy3Q5_bRxO_A7TzE0X7E,301
+dataflow_core-2.1.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dataflow_core-2.1.4.dist-info/entry_points.txt,sha256=ppj_EIbYrJJwCPg1kfdsZk5q1N-Ejfis1neYrnjhO8o,117
+dataflow_core-2.1.4.dist-info/top_level.txt,sha256=SZsUOpSCK9ntUy-3Tusxzf5A2e8ebwD8vouPb1dPt_8,23
+dataflow_core-2.1.4.dist-info/RECORD,,