dataflow-conda-plugin 0.1.3rc1__tar.gz → 0.1.4__tar.gz

{dataflow_conda_plugin-0.1.3rc1 → dataflow_conda_plugin-0.1.4}/PKG-INFO

@@ -1,3 +1,3 @@
 Metadata-Version: 2.4
 Name: dataflow-conda-plugin
-Version: 0.1.3rc1
+Version: 0.1.4

{dataflow_conda_plugin-0.1.3rc1 → dataflow_conda_plugin-0.1.4}/dataflow_conda_plugin.egg-info/PKG-INFO

@@ -1,3 +1,3 @@
 Metadata-Version: 2.4
 Name: dataflow-conda-plugin
-Version: 0.1.3rc1
+Version: 0.1.4

dataflow_conda_plugin-0.1.4/plugin/plugin.py

@@ -0,0 +1,161 @@
+import subprocess, sys, pkg_resources, os, requests
+from conda import plugins
+from conda.base.context import context
+from dataflow.utils.logger import CustomLogger
+from dataflow.configuration import ConfigurationManager
+
+logger = CustomLogger().get_logger(__name__)
+dataflow_config = ConfigurationManager('/dataflow/app/auth_config/dataflow_auth.cfg')
+
+env_api = dataflow_config.get_config_value('auth', 'env_api')
+
+def is_local_environment(target_prefix):
+    """Check if the environment is a local user environment."""
+    return (
+        os.environ.get('HOSTNAME') is not None and 
+        target_prefix and 
+        target_prefix.startswith('/home/jovyan')
+    )
+
+def save_environment(env_name: str, status: str):
+    """Save environment information via API."""
+    try:
+        response = requests.post(
+            env_api,
+            params={
+                "env_name": env_name,
+                "status": status
+            },
+            timeout=30
+        )
+        
+        if response.status_code in [200, 201]:
+            logger.info(f"Environment '{env_name}' saved successfully")
+        else:
+            logger.error(f"Error saving environment: {response.status_code} - {response.text}")
+            print("Error saving environment! Please try again after deleting the environment")
+            
+    except requests.exceptions.RequestException as e:
+        logger.error(f"Network error saving environment: {str(e)}")
+        print("Error saving environment! Please check your connection and try again")
+    except Exception as e:
+        logger.error(f"Unexpected error saving environment: {str(e)}")
+        print("Error saving environment! Please try again after deleting the environment")
+
+def install_deps(command: str):
+    """Install dataflow dependencies."""
+    target_prefix = context.target_prefix
+    args = context._argparse_args
+    env_name = os.path.basename(target_prefix) if target_prefix else None
+
+    should_save_to_db = is_local_environment(target_prefix) and env_name
+
+    try:
+        if (args.get('clone') is not None):
+            if should_save_to_db:
+                save_environment(env_name, "Success")
+            return
+        
+        if env_name and should_save_to_db:
+            save_environment(env_name, "Success")
+        
+        install_dataflow_deps = pkg_resources.resource_filename('plugin', 'scripts/install_dataflow_deps.sh')
+        process = subprocess.Popen(
+            ["bash", install_dataflow_deps, target_prefix],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+            bufsize=1
+        )
+        
+        for line in iter(process.stdout.readline, ''):
+            print(line, end='')
+            sys.stdout.flush()
+        
+        return_code = process.wait()
+        if return_code != 0:
+            print(f"Error in creating environment!!")
+            if should_save_to_db and env_name:
+                save_environment(env_name, "Failed")
+        else:
+            if env_name and should_save_to_db:
+                save_environment(env_name, "Success")
+
+    except Exception as e:
+        print(f"An unexpected error occurred: {str(e)}\nPlease delete the environment and try again.")
+        logger.error(f"Error installing dependencies: {str(e)}")
+        if should_save_to_db and env_name:
+            save_environment(env_name, "Failed")
+
+def remove_environment(env_name: str):
+    """Remove environment information via API."""
+    try:
+        response = requests.delete(
+            env_api,
+            params={"env_name": env_name},
+            timeout=30
+        )
+        
+        if response.status_code == 200:
+            logger.info(f"Environment '{env_name}' removed successfully")
+        elif response.status_code == 404:
+            logger.warning(f"Environment '{env_name}' not found")
+        else:
+            logger.error(f"Error removing environment: {response.status_code} - {response.text}")
+            print("Error removing environment! Please delete from the dataflow environment page")
+            
+    except requests.exceptions.RequestException as e:
+        logger.error(f"Network error removing environment: {str(e)}")
+        print("Error removing environment! Please delete from the dataflow environment page")
+    except Exception as e:
+        logger.error(f"Unexpected error removing environment: {str(e)}")
+        print("Error removing environment! Please delete from the dataflow environment page")
+
+def mark_environment_for_refresh(env_name: str):
+    """Mark environment for refresh via API."""
+    try:
+        response = requests.put(
+            env_api,
+            params={"env_name": env_name},
+            json={"need_refresh": True},
+            timeout=30
+        )
+        
+        if response.status_code == 200:
+            logger.info(f"Environment '{env_name}' marked for refresh")
+        else:
+            logger.error(f"Error marking environment for refresh: {response.status_code} - {response.text}")
+
+    except requests.exceptions.RequestException as e:
+        logger.error(f"Network error marking environment for refresh: {str(e)}")
+    except Exception as e:
+        logger.error(f"Unexpected error marking environment for refresh: {str(e)}")
+
+def package_operations(command: str):
+    """Track conda install/remove/update commands for packages and update libraries in database."""
+    target_prefix = context.target_prefix
+    env_name = os.path.basename(target_prefix) if target_prefix else None
+
+    # to catch env removal
+    if not os.path.exists(target_prefix):
+        if is_local_environment(target_prefix) and env_name:
+            remove_environment(env_name)
+        return
+
+    should_update_db = is_local_environment(target_prefix) and env_name
+
+    if should_update_db:
+        mark_environment_for_refresh(env_name)
+
+@plugins.hookimpl
+def conda_post_commands():
+    yield plugins.CondaPostCommand(
+        name=f"install_deps_post_command",
+        action=install_deps,
+        run_for={"create", "env_create"},
+    )
+    yield plugins.CondaPostCommand(
+        name=f"package_operations_post_command",
+        action=package_operations,
+        run_for={"install", "remove", "update"},
+    )

{dataflow_conda_plugin-0.1.3rc1 → dataflow_conda_plugin-0.1.4}/plugin/scripts/install_dataflow_deps.sh

@@ -4,12 +4,12 @@ conda_env_path=$1
 
 py_version=$(${conda_env_path}/bin/python -c "import sys; print('.'.join(map(str, sys.version_info[:2])))")
                       
-${conda_env_path}/bin/pip install dash==3.0.3 dash-renderer==1.9.1 plotly==6.0.1 typing==3.7.4.3 streamlit==1.45.1 ipython==9.2.0 ipykernel==6.29.5 ipython-sql==0.4.1 jupysql==0.10.14 psycopg2-binary==2.9.10 cryptography==44.0.3 dataflow-core dataflow-dbt
+${conda_env_path}/bin/pip install dash==3.0.3 dash-renderer==1.9.1 plotly==6.0.1 typing==3.7.4.3 streamlit==1.45.1 ipython==9.2.0 ipykernel==6.29.5 ipython-sql==0.4.1 jupysql==0.10.14 psycopg2-binary==2.9.10 cryptography==44.0.3 dataflow-core==2.1.11 dataflow-dbt==0.0.3
 
 # 3. Install Dataflow Airflow to a separate path in environment 
 ${conda_env_path}/bin/pip install \
     --force-reinstall --root-user-action ignore \
-    --no-warn-conflicts dataflow-airflow \
+    --no-warn-conflicts dataflow-airflow==2.10.7 \
     --target ${conda_env_path}/bin/airflow-libraries/
 
 files=(

{dataflow_conda_plugin-0.1.3rc1 → dataflow_conda_plugin-0.1.4}/setup.py

@@ -14,7 +14,7 @@ class PostInstall(install):
 
 setup(
     name="dataflow-conda-plugin",
-    version='0.1.3rc1',
+    version="0.1.4",
     entry_points={"conda": ["dataflow-conda-plugin = plugin.plugin"]},
     packages=find_packages(include=["plugin"]),
     package_data={'plugin': ['scripts/*.sh']},

dataflow_conda_plugin-0.1.3rc1/plugin/plugin.py

@@ -1,153 +0,0 @@
-import subprocess, sys, pkg_resources, os
-from conda import plugins
-from conda.base.context import context
-from dataflow.models import LocalEnvironment
-from dataflow.db import get_local_db
-from datetime import datetime, timezone
-from dataflow.utils.logger import CustomLogger
-
-logger = CustomLogger().get_logger(__name__)
-
-def is_local_environment(target_prefix):
-    """Check if the environment is a local user environment."""
-    return (
-        os.environ.get('HOSTNAME') is not None and 
-        target_prefix and 
-        target_prefix.startswith('/home/jovyan')
-    )
-
-def save_environment_to_db(env_name: str, status: str = "Created"):
-    """Save environment information to LocalEnvironment table."""
-    try:
-        db_generator = get_local_db()
-        db = next(db_generator)
-        
-        # Check if environment already exists
-        existing_env = db.query(LocalEnvironment).filter_by(name=env_name).first()
-        if existing_env:
-            # Update status if environment exists
-            existing_env.status = status
-            existing_env.updated_at = datetime.now(timezone.utc)
-            db.commit()
-            return
-        
-        # Create new LocalEnvironment record
-        local_env = LocalEnvironment(
-            name=env_name,
-            status=status,
-            updated_at=datetime.now(timezone.utc)
-        )
-        
-        db.add(local_env)
-        db.commit()
-        
-    except Exception as e:
-        print("Error saving environment! Please try again after deleting the environment")
-        logger.error(f"Error saving environment to database: {str(e)}")
-    finally:
-        db_generator.close()
-
-def install_deps(command: str):
-    """Install dataflow dependencies."""
-    target_prefix = context.target_prefix
-    args = context._argparse_args
-    env_name = os.path.basename(target_prefix) if target_prefix else None
-
-    should_save_to_db = is_local_environment(target_prefix) and env_name
-
-    try:
-        if (args.get('clone') is not None):
-            if should_save_to_db:
-                save_environment_to_db(env_name, "Created")
-            return
-        
-        install_dataflow_deps = pkg_resources.resource_filename('plugin', 'scripts/install_dataflow_deps.sh')
-        process = subprocess.Popen(
-            ["bash", install_dataflow_deps, target_prefix],
-            stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
-            text=True,
-            bufsize=1
-        )
-        
-        for line in iter(process.stdout.readline, ''):
-            print(line, end='')
-            sys.stdout.flush()
-        
-        return_code = process.wait()
-        if return_code != 0:
-            print(f"Error in creating environment!!")
-            if should_save_to_db and env_name:
-                save_environment_to_db(env_name, "Failed")
-        else:
-            if env_name and should_save_to_db:
-                save_environment_to_db(env_name, "Created")
-
-    except Exception as e:
-        print(f"An unexpected error occurred: {str(e)}\nPlease delete the environment and try again.")
-        logger.error(f"Error installing dependencies: {str(e)}")
-        if should_save_to_db and env_name:
-            save_environment_to_db(env_name, "Failed")
-
-def remove_environment_from_db(env_name: str):
-    """Remove environment information from LocalEnvironment table."""
-    try:
-        db_generator = get_local_db()
-        db = next(db_generator)
-        
-        # Find and delete the environment
-        existing_env = db.query(LocalEnvironment).filter_by(name=env_name).first()
-        if existing_env:
-            db.delete(existing_env)
-            db.commit()
-        else:
-            logger.warning(f"Environment '{env_name}' not found in database")
-
-    except Exception as e:
-        print(f"Error removing environment! Please delete from the dataflow enviornment page")
-        logger.error(f"Error removing environment from database: {str(e)}")
-    finally:
-        db_generator.close()
-
-def package_operations(command: str):
-    """Track conda install/remove/update commands for packages and update libraries in database."""
-    target_prefix = context.target_prefix
-    env_name = os.path.basename(target_prefix) if target_prefix else None
-
-    # to catch env removal
-    if not os.path.exists(target_prefix):
-        if is_local_environment(target_prefix) and env_name:
-            remove_environment_from_db(env_name)
-        return
-
-    should_update_db = is_local_environment(target_prefix) and env_name
-
-    if should_update_db:
-        try:
-            db_generator = get_local_db()
-            db = next(db_generator)
-            
-            # Find the environment and set need_refresh to True
-            existing_env = db.query(LocalEnvironment).filter_by(name=env_name).first()
-            if existing_env:
-                existing_env.need_refresh = True
-                existing_env.updated_at = datetime.now(timezone.utc)
-                db.commit()
-            
-        except Exception as e:
-            logger.error(f"Error updating need_refresh in database: {str(e)}")
-        finally:
-            db_generator.close()
-
-@plugins.hookimpl
-def conda_post_commands():
-    yield plugins.CondaPostCommand(
-        name=f"install_deps_post_command",
-        action=install_deps,
-        run_for={"create", "env_create"},
-    )
-    yield plugins.CondaPostCommand(
-        name=f"package_operations_post_command",
-        action=package_operations,
-        run_for={"install", "remove", "update"},
-    )