dataflow-conda-plugin 0.1.3rc1__py3-none-any.whl → 0.1.4__py3-none-any.whl

{dataflow_conda_plugin-0.1.3rc1.dist-info → dataflow_conda_plugin-0.1.4.dist-info}/METADATA

@@ -1,3 +1,3 @@
 Metadata-Version: 2.4
 Name: dataflow-conda-plugin
-Version: 0.1.3rc1
+Version: 0.1.4

dataflow_conda_plugin-0.1.4.dist-info/RECORD

@@ -0,0 +1,8 @@
+plugin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+plugin/plugin.py,sha256=83cYuYaTDyYNPY_mWgn_isJaBb8G3kuUbzZtM7Yx15Q,6209
+plugin/scripts/install_dataflow_deps.sh,sha256=hp_v6h5Ikh9K4kp80-le8BNhjnP6znyQdKlw5-bC54o,1122
+dataflow_conda_plugin-0.1.4.dist-info/METADATA,sha256=itEkiWPFhSW-67tRwEFsxgAUsr2OHYzBbZpCXKdYAUI,65
+dataflow_conda_plugin-0.1.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dataflow_conda_plugin-0.1.4.dist-info/entry_points.txt,sha256=Vk2GKuQjr-5WV4NDKyPsnF37p3LUoAgPYOSMHxnpac8,46
+dataflow_conda_plugin-0.1.4.dist-info/top_level.txt,sha256=Io_dflkI6h0vZSGOzZxx4e76CDTSANVfu4v4tVY6zsA,7
+dataflow_conda_plugin-0.1.4.dist-info/RECORD,,

plugin/plugin.py

@@ -1,12 +1,13 @@
-import subprocess, sys, pkg_resources, os
+import subprocess, sys, pkg_resources, os, requests
 from conda import plugins
 from conda.base.context import context
-from dataflow.models import LocalEnvironment
-from dataflow.db import get_local_db
-from datetime import datetime, timezone
 from dataflow.utils.logger import CustomLogger
+from dataflow.configuration import ConfigurationManager
 
 logger = CustomLogger().get_logger(__name__)
+dataflow_config = ConfigurationManager('/dataflow/app/auth_config/dataflow_auth.cfg')
+
+env_api = dataflow_config.get_config_value('auth', 'env_api')
 
 def is_local_environment(target_prefix):
     """Check if the environment is a local user environment."""
@@ -16,36 +17,30 @@ def is_local_environment(target_prefix):
         target_prefix.startswith('/home/jovyan')
     )
 
-def save_environment_to_db(env_name: str, status: str = "Created"):
-    """Save environment information to LocalEnvironment table."""
+def save_environment(env_name: str, status: str):
+    """Save environment information via API."""
     try:
-        db_generator = get_local_db()
-        db = next(db_generator)
-        
-        # Check if environment already exists
-        existing_env = db.query(LocalEnvironment).filter_by(name=env_name).first()
-        if existing_env:
-            # Update status if environment exists
-            existing_env.status = status
-            existing_env.updated_at = datetime.now(timezone.utc)
-            db.commit()
-            return
-        
-        # Create new LocalEnvironment record
-        local_env = LocalEnvironment(
-            name=env_name,
-            status=status,
-            updated_at=datetime.now(timezone.utc)
+        response = requests.post(
+            env_api,
+            params={
+                "env_name": env_name,
+                "status": status
+            },
+            timeout=30
         )
         
-        db.add(local_env)
-        db.commit()
-        
+        if response.status_code in [200, 201]:
+            logger.info(f"Environment '{env_name}' saved successfully")
+        else:
+            logger.error(f"Error saving environment: {response.status_code} - {response.text}")
+            print("Error saving environment! Please try again after deleting the environment")
+            
+    except requests.exceptions.RequestException as e:
+        logger.error(f"Network error saving environment: {str(e)}")
+        print("Error saving environment! Please check your connection and try again")
     except Exception as e:
+        logger.error(f"Unexpected error saving environment: {str(e)}")
         print("Error saving environment! Please try again after deleting the environment")
-        logger.error(f"Error saving environment to database: {str(e)}")
-    finally:
-        db_generator.close()
 
 def install_deps(command: str):
     """Install dataflow dependencies."""
@@ -58,9 +53,12 @@ def install_deps(command: str):
     try:
         if (args.get('clone') is not None):
             if should_save_to_db:
-                save_environment_to_db(env_name, "Created")
+                save_environment(env_name, "Success")
             return
         
+        if env_name and should_save_to_db:
+            save_environment(env_name, "Success")
+        
         install_dataflow_deps = pkg_resources.resource_filename('plugin', 'scripts/install_dataflow_deps.sh')
         process = subprocess.Popen(
             ["bash", install_dataflow_deps, target_prefix],
@@ -78,36 +76,60 @@ def install_deps(command: str):
         if return_code != 0:
             print(f"Error in creating environment!!")
             if should_save_to_db and env_name:
-                save_environment_to_db(env_name, "Failed")
+                save_environment(env_name, "Failed")
         else:
             if env_name and should_save_to_db:
-                save_environment_to_db(env_name, "Created")
+                save_environment(env_name, "Success")
 
     except Exception as e:
         print(f"An unexpected error occurred: {str(e)}\nPlease delete the environment and try again.")
         logger.error(f"Error installing dependencies: {str(e)}")
         if should_save_to_db and env_name:
-            save_environment_to_db(env_name, "Failed")
+            save_environment(env_name, "Failed")
+
+def remove_environment(env_name: str):
+    """Remove environment information via API."""
+    try:
+        response = requests.delete(
+            env_api,
+            params={"env_name": env_name},
+            timeout=30
+        )
+        
+        if response.status_code == 200:
+            logger.info(f"Environment '{env_name}' removed successfully")
+        elif response.status_code == 404:
+            logger.warning(f"Environment '{env_name}' not found")
+        else:
+            logger.error(f"Error removing environment: {response.status_code} - {response.text}")
+            print("Error removing environment! Please delete from the dataflow environment page")
+            
+    except requests.exceptions.RequestException as e:
+        logger.error(f"Network error removing environment: {str(e)}")
+        print("Error removing environment! Please delete from the dataflow environment page")
+    except Exception as e:
+        logger.error(f"Unexpected error removing environment: {str(e)}")
+        print("Error removing environment! Please delete from the dataflow environment page")
 
-def remove_environment_from_db(env_name: str):
-    """Remove environment information from LocalEnvironment table."""
+def mark_environment_for_refresh(env_name: str):
+    """Mark environment for refresh via API."""
     try:
-        db_generator = get_local_db()
-        db = next(db_generator)
+        response = requests.put(
+            env_api,
+            params={"env_name": env_name},
+            json={"need_refresh": True},
+            timeout=30
+        )
         
-        # Find and delete the environment
-        existing_env = db.query(LocalEnvironment).filter_by(name=env_name).first()
-        if existing_env:
-            db.delete(existing_env)
-            db.commit()
+        if response.status_code == 200:
+            logger.info(f"Environment '{env_name}' marked for refresh")
         else:
-            logger.warning(f"Environment '{env_name}' not found in database")
+            logger.error(f"Error marking environment for refresh: {response.status_code} - {response.text}")
 
+    except requests.exceptions.RequestException as e:
+        logger.error(f"Network error marking environment for refresh: {str(e)}")
     except Exception as e:
-        print(f"Error removing environment! Please delete from the dataflow enviornment page")
-        logger.error(f"Error removing environment from database: {str(e)}")
-    finally:
-        db_generator.close()
+        logger.error(f"Unexpected error marking environment for refresh: {str(e)}")
 
 def package_operations(command: str):
     """Track conda install/remove/update commands for packages and update libraries in database."""
@@ -117,27 +139,13 @@ def package_operations(command: str):
     # to catch env removal
     if not os.path.exists(target_prefix):
         if is_local_environment(target_prefix) and env_name:
-            remove_environment_from_db(env_name)
+            remove_environment(env_name)
         return
 
     should_update_db = is_local_environment(target_prefix) and env_name
 
     if should_update_db:
-        try:
-            db_generator = get_local_db()
-            db = next(db_generator)
-            
-            # Find the environment and set need_refresh to True
-            existing_env = db.query(LocalEnvironment).filter_by(name=env_name).first()
-            if existing_env:
-                existing_env.need_refresh = True
-                existing_env.updated_at = datetime.now(timezone.utc)
-                db.commit()
-            
-        except Exception as e:
-            logger.error(f"Error updating need_refresh in database: {str(e)}")
-        finally:
-            db_generator.close()
+        mark_environment_for_refresh(env_name)
 
 @plugins.hookimpl
 def conda_post_commands():

plugin/scripts/install_dataflow_deps.sh

@@ -4,12 +4,12 @@ conda_env_path=$1
 
 py_version=$(${conda_env_path}/bin/python -c "import sys; print('.'.join(map(str, sys.version_info[:2])))")
                       
-${conda_env_path}/bin/pip install dash==3.0.3 dash-renderer==1.9.1 plotly==6.0.1 typing==3.7.4.3 streamlit==1.45.1 ipython==9.2.0 ipykernel==6.29.5 ipython-sql==0.4.1 jupysql==0.10.14 psycopg2-binary==2.9.10 cryptography==44.0.3 dataflow-core dataflow-dbt
+${conda_env_path}/bin/pip install dash==3.0.3 dash-renderer==1.9.1 plotly==6.0.1 typing==3.7.4.3 streamlit==1.45.1 ipython==9.2.0 ipykernel==6.29.5 ipython-sql==0.4.1 jupysql==0.10.14 psycopg2-binary==2.9.10 cryptography==44.0.3 dataflow-core==2.1.11 dataflow-dbt==0.0.3
 
 # 3. Install Dataflow Airflow to a separate path in environment 
 ${conda_env_path}/bin/pip install \
     --force-reinstall --root-user-action ignore \
-    --no-warn-conflicts dataflow-airflow \
+    --no-warn-conflicts dataflow-airflow==2.10.7 \
     --target ${conda_env_path}/bin/airflow-libraries/
 
 files=(

dataflow_conda_plugin-0.1.3rc1.dist-info/RECORD

@@ -1,8 +0,0 @@
-plugin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-plugin/plugin.py,sha256=ubvJ7Zu_iMfOxVGHNgU_OtrXgx8zc7bCUsv1NJ_tda8,5516
-plugin/scripts/install_dataflow_deps.sh,sha256=sbr4tbtjCE4nSKnn-Net4e73ZGGi1WWqkzQrU7_nZJI,1099
-dataflow_conda_plugin-0.1.3rc1.dist-info/METADATA,sha256=VSNCzwWj974IkvGOUvfVs5Lrj-fAQmDvWsbCiCNyLMk,68
-dataflow_conda_plugin-0.1.3rc1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dataflow_conda_plugin-0.1.3rc1.dist-info/entry_points.txt,sha256=Vk2GKuQjr-5WV4NDKyPsnF37p3LUoAgPYOSMHxnpac8,46
-dataflow_conda_plugin-0.1.3rc1.dist-info/top_level.txt,sha256=Io_dflkI6h0vZSGOzZxx4e76CDTSANVfu4v4tVY6zsA,7
-dataflow_conda_plugin-0.1.3rc1.dist-info/RECORD,,