dataflow-core 2.1.10__tar.gz → 2.1.11__tar.gz

{dataflow_core-2.1.10 → dataflow_core-2.1.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataflow-core
-Version: 2.1.10
+Version: 2.1.11
 Summary: Dataflow core package
 Author: Dataflow
 Author-email: 

dataflow_core-2.1.11/README.md

@@ -0,0 +1 @@
+# Dataflow-Core

{dataflow_core-2.1.10 → dataflow_core-2.1.11}/dataflow/environment.py

@@ -1,5 +1,5 @@
 import os, shutil, subprocess, datetime, yaml, re
-from .models.environment import JobLogs, Environment
+from .models.environment import JobLogs, Environment, LocalEnvironment
 import json, asyncio, pkg_resources
 from sqlalchemy.orm import Session
 from .configuration import ConfigurationManager
@@ -12,6 +12,7 @@ class EnvironmentManager:
         self.env_base_path = self.config.get_config_value('paths', 'env_path')
         self.env_logs_path = self.config.get_config_value('paths', 'env_logs_path')
         self.env_version_path = self.config.get_config_value('paths', 'env_versions_path')
+        self.local_env_logs_path = self.config.get_config_value('paths', 'local_env_logs_path')
         os.makedirs(self.env_version_path, exist_ok=True)
         self.logger = CustomLogger().get_logger(__name__)
     
@@ -78,7 +79,7 @@ class EnvironmentManager:
             self.logger.error(f"Invalid status '{status}' provided for environment creation.")
             raise ValueError("Invalid status. Use 'draft' or 'published'.")
     
-    async def clone_env(self, source_path, env_name, pip_libraries, conda_libraries, user_name, db=None):
+    async def clone_env(self, source_path, env_name, pip_libraries, conda_libraries, user_name, db=None, local_clone=False):
         """
         Clones an existing conda environment.
         
@@ -96,7 +97,10 @@ class EnvironmentManager:
         # Set up logging
         log_file_location = None
         if db:
-            log_file_location = self._setup_logging(env_name, "1", user_name, db)
+            if local_clone:
+                log_file_location = os.path.join(self.local_env_logs_path, f"{env_name}.log")
+            else:
+                log_file_location = self._setup_logging(env_name, "1", user_name, db)
 
         yaml_path = f"{self.env_version_path}/{env_name}_v1.yaml"
         
@@ -104,7 +108,7 @@ class EnvironmentManager:
         clone_status = await self._execute_env_operation(
             env_name=env_name,
             status="draft",
-            mode="clone",
+            mode="local_clone" if local_clone else "clone",
             yaml_file_path=yaml_path,
             source_path=source_path,
             log_file_location=log_file_location,
@@ -113,16 +117,22 @@ class EnvironmentManager:
         
         # Update job log status if db was provided
         if db and log_file_location:
-            log_file_name = os.path.basename(log_file_location)
-            await self._update_job_status(log_file_name, clone_status, log_file_location, db)
-            self.update_environment_db(
-                env_short_name=env_name, 
-                version="1", 
-                pip_libraries=pip_libraries, 
-                conda_libraries=conda_libraries,
-                status=clone_status, 
-                db=db
-            )
+            if local_clone:
+                db.query(LocalEnvironment).filter(
+                    LocalEnvironment.name == env_name
+                ).update({"status": clone_status.title()})
+                db.commit()
+            else:
+                log_file_name = os.path.basename(log_file_location)
+                await self._update_job_status(log_file_name, clone_status, log_file_location, db)
+                self.update_environment_db(
+                    env_short_name=env_name, 
+                    version="1", 
+                    pip_libraries=pip_libraries, 
+                    conda_libraries=conda_libraries,
+                    status=clone_status, 
+                    db=db
+                )
             
         return clone_status
     
@@ -198,13 +208,15 @@ class EnvironmentManager:
         """
         self.logger.info(f"Executing environment operation: {env_name}, Status: {status}, Mode: {mode}")
         status = status.lower()
-        conda_env_path = os.path.join(self.env_base_path, env_name)
+        if mode == "local_clone":
+            conda_env_path = os.path.join(os.getenv("CONDA_ENVS_PATH"), env_name)
+        else:
+            os.makedirs(self.env_base_path, exist_ok=True)
+            conda_env_path = os.path.join(self.env_base_path, env_name)
 
         try:
             if os.path.exists(conda_env_path) and mode == "create":
                 raise FileExistsError(f"Environment '{env_name}' already exists at {conda_env_path}.")
-            
-            os.makedirs(conda_env_path, exist_ok=True)
 
             if mode == "create":                
                 create_env_script_path = pkg_resources.resource_filename('dataflow', 'scripts/create_environment.sh')
@@ -216,8 +228,13 @@ class EnvironmentManager:
             
             elif mode == "clone":
                 clone_env_script_path = pkg_resources.resource_filename('dataflow', 'scripts/clone_environment.sh')
-                command = ["bash", clone_env_script_path, source_path, conda_env_path]
-                
+                command = ["bash", clone_env_script_path, source_path, conda_env_path, yaml_file_path]
+            
+            elif mode == "local_clone":
+                conda_cmd = f"conda create --name {env_name} --clone {source_path} --yes"
+                command = ["su", "-", os.environ.get("NB_USER"), "-c", conda_cmd]
+                self.logger.info(f"Cloning environment locally with command: {' '.join(command)}")
+
             else:
                 raise ValueError("Invalid mode. Use 'create', 'update', or 'clone'.")
 

{dataflow_core-2.1.10 → dataflow_core-2.1.11}/dataflow/models/environment.py

@@ -2,7 +2,8 @@ from sqlalchemy import Column, Integer, String, Boolean, Text, ForeignKey, DateT
 from sqlalchemy.orm import relationship
 from sqlalchemy.sql import func
 from datetime import datetime, timezone
-from dataflow.db import Base, Local_Base
+from dataflow.db import Base
+from enum import Enum
 
 class EnvironmentAttributes(Base):
     """
@@ -62,14 +63,20 @@ class JobLogs(Base):
     created_by = Column(String)
 
 
-class LocalEnvironment(Local_Base):
+class LocalEnvironment(Base):
     __tablename__ = "LOCAL_ENVIRONMENT"
 
     id = Column(Integer, primary_key=True, autoincrement=True)
-    name = Column(String, nullable=False)
+    name = Column(String, nullable=False, index=True)
+    user_name = Column(String, ForeignKey('USER.user_name', ondelete='CASCADE'), nullable=False, index=True)
     py_version = Column(String)
     pip_libraries = Column(Text)
     conda_libraries = Column(Text)
     status = Column(String, default="Created")
+    cloned_from = Column(String, ForeignKey('ENVIRONMENT.short_name', ondelete='SET NULL'), nullable=True)
     updated_at = Column(DateTime, default=datetime.now(timezone.utc), onupdate=datetime.now(timezone.utc))
     need_refresh = Column(Boolean, default=False)
+
+class EnvType(str, Enum):
+    dataflow = "dataflow"
+    local = "local"

{dataflow_core-2.1.10 → dataflow_core-2.1.11}/dataflow/scripts/clone_environment.sh

@@ -3,6 +3,7 @@ set -e
 
 source_env_name=$1
 target_env_path=$2
+yaml_file_path=$3
 
 # Extract just the env name (basename) from the target path
 env_name=$(basename "$target_env_path")
@@ -14,6 +15,6 @@ mkdir -p "$CONDA_PKGS_DIRS"
 # 1. Cloning conda env
 conda create --clone ${source_env_name} --prefix ${target_env_path} --yes
 
-conda env export --prefix "$conda_env_path" > "$yaml_file_path"
+conda env export --prefix "$target_env_path" > "$yaml_file_path"
 
 echo "Environment Creation Successful"

{dataflow_core-2.1.10 → dataflow_core-2.1.11}/dataflow_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataflow-core
-Version: 2.1.10
+Version: 2.1.11
 Summary: Dataflow core package
 Author: Dataflow
 Author-email: 

{dataflow_core-2.1.10 → dataflow_core-2.1.11}/dataflow_core.egg-info/SOURCES.txt

@@ -1,3 +1,4 @@
+README.md
 setup.py
 authenticator/__init__.py
 authenticator/dataflowairflowauthenticator.py

{dataflow_core-2.1.10 → dataflow_core-2.1.11}/setup.py

@@ -14,7 +14,7 @@ class PostInstall(install):
 
 setup(
     name="dataflow-core",
-    version="2.1.10",
+    version="2.1.11",
     packages=find_packages(include=["dataflow", "dataflow.*", "authenticator", "authenticator.*"]),
     include_package_data=True,
     package_data={