dataflow-core 2.1.10__py3-none-any.whl → 2.1.11__py3-none-any.whl

dataflow/environment.py

@@ -1,5 +1,5 @@
 import os, shutil, subprocess, datetime, yaml, re
-from .models.environment import JobLogs, Environment
+from .models.environment import JobLogs, Environment, LocalEnvironment
 import json, asyncio, pkg_resources
 from sqlalchemy.orm import Session
 from .configuration import ConfigurationManager
@@ -12,6 +12,7 @@ class EnvironmentManager:
         self.env_base_path = self.config.get_config_value('paths', 'env_path')
         self.env_logs_path = self.config.get_config_value('paths', 'env_logs_path')
         self.env_version_path = self.config.get_config_value('paths', 'env_versions_path')
+        self.local_env_logs_path = self.config.get_config_value('paths', 'local_env_logs_path')
         os.makedirs(self.env_version_path, exist_ok=True)
         self.logger = CustomLogger().get_logger(__name__)
     
@@ -78,7 +79,7 @@ class EnvironmentManager:
             self.logger.error(f"Invalid status '{status}' provided for environment creation.")
             raise ValueError("Invalid status. Use 'draft' or 'published'.")
     
-    async def clone_env(self, source_path, env_name, pip_libraries, conda_libraries, user_name, db=None):
+    async def clone_env(self, source_path, env_name, pip_libraries, conda_libraries, user_name, db=None, local_clone=False):
         """
         Clones an existing conda environment.
         
@@ -96,7 +97,10 @@ class EnvironmentManager:
         # Set up logging
         log_file_location = None
         if db:
-            log_file_location = self._setup_logging(env_name, "1", user_name, db)
+            if local_clone:
+                log_file_location = os.path.join(self.local_env_logs_path, f"{env_name}.log")
+            else:
+                log_file_location = self._setup_logging(env_name, "1", user_name, db)
 
         yaml_path = f"{self.env_version_path}/{env_name}_v1.yaml"
         
@@ -104,7 +108,7 @@ class EnvironmentManager:
         clone_status = await self._execute_env_operation(
             env_name=env_name,
             status="draft",
-            mode="clone",
+            mode="local_clone" if local_clone else "clone",
             yaml_file_path=yaml_path,
             source_path=source_path,
             log_file_location=log_file_location,
@@ -113,16 +117,22 @@ class EnvironmentManager:
         
         # Update job log status if db was provided
         if db and log_file_location:
-            log_file_name = os.path.basename(log_file_location)
-            await self._update_job_status(log_file_name, clone_status, log_file_location, db)
-            self.update_environment_db(
-                env_short_name=env_name, 
-                version="1", 
-                pip_libraries=pip_libraries, 
-                conda_libraries=conda_libraries,
-                status=clone_status, 
-                db=db
-            )
+            if local_clone:
+                db.query(LocalEnvironment).filter(
+                    LocalEnvironment.name == env_name
+                ).update({"status": clone_status.title()})
+                db.commit()
+            else:
+                log_file_name = os.path.basename(log_file_location)
+                await self._update_job_status(log_file_name, clone_status, log_file_location, db)
+                self.update_environment_db(
+                    env_short_name=env_name, 
+                    version="1", 
+                    pip_libraries=pip_libraries, 
+                    conda_libraries=conda_libraries,
+                    status=clone_status, 
+                    db=db
+                )
             
         return clone_status
     
@@ -198,13 +208,15 @@ class EnvironmentManager:
         """
         self.logger.info(f"Executing environment operation: {env_name}, Status: {status}, Mode: {mode}")
         status = status.lower()
-        conda_env_path = os.path.join(self.env_base_path, env_name)
+        if mode == "local_clone":
+            conda_env_path = os.path.join(os.getenv("CONDA_ENVS_PATH"), env_name)
+        else:
+            os.makedirs(self.env_base_path, exist_ok=True)
+            conda_env_path = os.path.join(self.env_base_path, env_name)
 
         try:
             if os.path.exists(conda_env_path) and mode == "create":
                 raise FileExistsError(f"Environment '{env_name}' already exists at {conda_env_path}.")
-            
-            os.makedirs(conda_env_path, exist_ok=True)
 
             if mode == "create":                
                 create_env_script_path = pkg_resources.resource_filename('dataflow', 'scripts/create_environment.sh')
@@ -216,8 +228,13 @@ class EnvironmentManager:
             
             elif mode == "clone":
                 clone_env_script_path = pkg_resources.resource_filename('dataflow', 'scripts/clone_environment.sh')
-                command = ["bash", clone_env_script_path, source_path, conda_env_path]
-                
+                command = ["bash", clone_env_script_path, source_path, conda_env_path, yaml_file_path]
+            
+            elif mode == "local_clone":
+                conda_cmd = f"conda create --name {env_name} --clone {source_path} --yes"
+                command = ["su", "-", os.environ.get("NB_USER"), "-c", conda_cmd]
+                self.logger.info(f"Cloning environment locally with command: {' '.join(command)}")
+
             else:
                 raise ValueError("Invalid mode. Use 'create', 'update', or 'clone'.")
 

dataflow/models/environment.py

@@ -2,7 +2,8 @@ from sqlalchemy import Column, Integer, String, Boolean, Text, ForeignKey, DateT
 from sqlalchemy.orm import relationship
 from sqlalchemy.sql import func
 from datetime import datetime, timezone
-from dataflow.db import Base, Local_Base
+from dataflow.db import Base
+from enum import Enum
 
 class EnvironmentAttributes(Base):
     """
@@ -62,14 +63,20 @@ class JobLogs(Base):
     created_by = Column(String)
 
 
-class LocalEnvironment(Local_Base):
+class LocalEnvironment(Base):
     __tablename__ = "LOCAL_ENVIRONMENT"
 
     id = Column(Integer, primary_key=True, autoincrement=True)
-    name = Column(String, nullable=False)
+    name = Column(String, nullable=False, index=True)
+    user_name = Column(String, ForeignKey('USER.user_name', ondelete='CASCADE'), nullable=False, index=True)
     py_version = Column(String)
     pip_libraries = Column(Text)
     conda_libraries = Column(Text)
     status = Column(String, default="Created")
+    cloned_from = Column(String, ForeignKey('ENVIRONMENT.short_name', ondelete='SET NULL'), nullable=True)
     updated_at = Column(DateTime, default=datetime.now(timezone.utc), onupdate=datetime.now(timezone.utc))
     need_refresh = Column(Boolean, default=False)
+
+class EnvType(str, Enum):
+    dataflow = "dataflow"
+    local = "local"

dataflow/scripts/clone_environment.sh

@@ -3,6 +3,7 @@ set -e
 
 source_env_name=$1
 target_env_path=$2
+yaml_file_path=$3
 
 # Extract just the env name (basename) from the target path
 env_name=$(basename "$target_env_path")
@@ -14,6 +15,6 @@ mkdir -p "$CONDA_PKGS_DIRS"
 # 1. Cloning conda env
 conda create --clone ${source_env_name} --prefix ${target_env_path} --yes
 
-conda env export --prefix "$conda_env_path" > "$yaml_file_path"
+conda env export --prefix "$target_env_path" > "$yaml_file_path"
 
 echo "Environment Creation Successful"

{dataflow_core-2.1.10.dist-info → dataflow_core-2.1.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataflow-core
-Version: 2.1.10
+Version: 2.1.11
 Summary: Dataflow core package
 Author: Dataflow
 Author-email: 

{dataflow_core-2.1.10.dist-info → dataflow_core-2.1.11.dist-info}/RECORD

@@ -7,13 +7,13 @@ dataflow/configuration.py,sha256=7To6XwH1eESiYp39eqPcswXWwrdBUdPF6xN6WnazOF0,663
 dataflow/database_manager.py,sha256=tJHMuOZ9Muskrh9t4uLRlTuFU0VkHAzoHlGP5DORIC4,899
 dataflow/dataflow.py,sha256=A66OFXgJCxW18Dk_BnasvR82LrPXQ2iB-QMTyYJJ1cc,13100
 dataflow/db.py,sha256=73ojGqpCTRVTlPszD73Ozhjih_BI2KTHmazqxxL6iWk,3780
-dataflow/environment.py,sha256=05F-dBRyZu2mr26vuiuJU13pfgtRpA9yW69bk83HGUw,27239
+dataflow/environment.py,sha256=qiyuRRPpVLVWiYccRHXnGyWMr_ZBPWzixAyDtAzxQYE,28277
 dataflow/models/__init__.py,sha256=WnlLd-0T3HYtJloDms1a58lN66WzBIbBSTReDXD6HaQ,892
 dataflow/models/app_types.py,sha256=yE_ZB13lhpK7AZ7PyBwnQlf0RlIHYs_-vdMKx7_RMlY,379
 dataflow/models/blacklist_library.py,sha256=B2oi3Z8GcR_glhLAyinFk0W8c9txXvm3uOER6dY-q7I,991
 dataflow/models/connection.py,sha256=_VJL3KuIrm8t4lJmtunIL3-AXF9Yvi5wUolzdR3tE0E,1017
 dataflow/models/dataflow_zone.py,sha256=yFCvQXos5M1cU7ksbVSO338_RkT3hkdw2wr3kCJ_rec,769
-dataflow/models/environment.py,sha256=300CUwx7d7bjadrMsTch7NSRYLCuf9AK9WVzlT26CAc,2616
+dataflow/models/environment.py,sha256=3W-Pvkuiufjw0MWy9F6uWSd0HDPjRNNJe3TnhpHxcGg,2920
 dataflow/models/environment_status.py,sha256=lvPDNUsUoTW9D97B07aKqJQHRKp4LvPM28pQDMPH1ac,536
 dataflow/models/git_ssh.py,sha256=W15SDypxzGOz_aZkHEnVZ6DIMVsjAsbSIXVIEt2mPYU,694
 dataflow/models/pinned_projects.py,sha256=I-XMQq7__XJJi2lyOdEvQEfhPRz8D6KHA6Cbavbf05o,606
@@ -34,7 +34,7 @@ dataflow/schemas/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,
 dataflow/schemas/connection.py,sha256=1bKTSEPW1QwcFQe35QCIr-S5DTMr34WJCjSKhxMpPek,2449
 dataflow/schemas/git_ssh.py,sha256=N1O7HM6ZbygIBZn2rKvNR0e7IM3ZJMAH6aJtjaghDr0,1283
 dataflow/schemas/secret.py,sha256=dAN_2IBTjnjf4sT6cyPVd_Zcdaw8s_EMG4aodMoCwXU,1165
-dataflow/scripts/clone_environment.sh,sha256=xWJBw9z1W1rztrzLXYro3UtEdFuBSqNrB83y45zqFfE,487
+dataflow/scripts/clone_environment.sh,sha256=Qy0GylsA3kUVUL_L1MirxIWujOFhT1tikKqXNtCTWd4,506
 dataflow/scripts/create_environment.sh,sha256=3FHgNplJuEZvyTsLqlCJNX9oyfXgsfqn80VZk2xtvso,828
 dataflow/scripts/update_environment.sh,sha256=2dtn2xlNi6frpig-sqlGE1_IKRbbkqYOCpf_qyMKKII,992
 dataflow/secrets_manager/__init__.py,sha256=idGqIDtYl0De2WIK9Obl-N7SDPSYtVM0D-wXfZjCiy4,559
@@ -48,8 +48,8 @@ dataflow/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dataflow/utils/exceptions.py,sha256=8GRFoYZ5dPGQckVm2znaHpPi0ZAs69fK-RGKukEsapk,4432
 dataflow/utils/get_current_user.py,sha256=4nSO3SPVMZhW-MsIgxR3f9ZzrFaIZIuyrM6hvfyE7PQ,1202
 dataflow/utils/logger.py,sha256=7BFrOq5Oiqn8P4XZbgJzMP5O07d2fpdECbbfsjrUuHw,1213
-dataflow_core-2.1.10.dist-info/METADATA,sha256=kzjEwzhHHXfO29gQPh5S30jzY4SZRwEySKXD7RZmKDs,370
-dataflow_core-2.1.10.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dataflow_core-2.1.10.dist-info/entry_points.txt,sha256=ppj_EIbYrJJwCPg1kfdsZk5q1N-Ejfis1neYrnjhO8o,117
-dataflow_core-2.1.10.dist-info/top_level.txt,sha256=SZsUOpSCK9ntUy-3Tusxzf5A2e8ebwD8vouPb1dPt_8,23
-dataflow_core-2.1.10.dist-info/RECORD,,
+dataflow_core-2.1.11.dist-info/METADATA,sha256=5ascZ5HFZFObYNT9mPFD1jrOrxoeFhUbyZX3kkjydzI,370
+dataflow_core-2.1.11.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dataflow_core-2.1.11.dist-info/entry_points.txt,sha256=ppj_EIbYrJJwCPg1kfdsZk5q1N-Ejfis1neYrnjhO8o,117
+dataflow_core-2.1.11.dist-info/top_level.txt,sha256=SZsUOpSCK9ntUy-3Tusxzf5A2e8ebwD8vouPb1dPt_8,23
+dataflow_core-2.1.11.dist-info/RECORD,,