dataflow-core 2.0.5__py3-none-any.whl → 2.0.6__py3-none-any.whl

dataflow/dataflow.py

@@ -2,21 +2,21 @@ import os, requests
 from .models.database import DatabaseManager
 from sqlalchemy.inspection import inspect
 from .utils.aws_secrets_manager import SecretsManagerClient
-import json
+import json, asyncio, pkg_resources
 from authenticator.package.configuration import ConfigurationManager
 
 
 class Dataflow:
     def __init__(self):
-        self.dataflow_config = ConfigurationManager('/dataflow/app/auth_config/dataflow_auth.cfg')
-        self.auth_api = self.dataflow_config.get_config_value('auth', 'ui_auth_api')
         self.secrets_manager = SecretsManagerClient('us-east-1')
 
     def auth(self, session_id: str):
         """Retrieve user information from the auth API."""
         try:
+            dataflow_config = ConfigurationManager('/dataflow/app/auth_config/dataflow_auth.cfg')
+            auth_api = dataflow_config.get_config_value('auth', 'ui_auth_api')
             response = requests.get(
-                self.auth_api,
+                auth_api,
                 cookies={"dataflow_session": session_id, "jupyterhub-hub-login": ""}
             )
             
@@ -86,3 +86,42 @@ class Dataflow:
         
         except Exception as e:
             return None
+        
+    async def create_env(self, env_name, py_version, py_requirements, status, env_version=None):
+        """
+        Creates a conda environment at the specified path and installs libraries in one command.
+        """
+        config = ConfigurationManager('/dataflow/app/config/dataflow.cfg')
+        status = status.lower()
+        if status == "published":
+            env_base_path = config.get_config_value('paths', 'published_env_path')
+            conda_env_path = os.path.join(env_base_path, env_name)
+        else:
+            env_base_path = config.get_config_value('paths', 'drafts_env_path')
+            conda_env_path = os.path.join(env_base_path, env_name, f"{env_name}_v{env_version}")
+        try:
+            if not os.path.exists(conda_env_path):
+                os.makedirs(conda_env_path, exist_ok=True)
+
+            py_requirements = ",".join(py_requirements) 
+
+            script_path = pkg_resources.resource_filename('dataflow', 'scripts/create_environment.sh')
+    
+            # Make the script executable
+            os.chmod(script_path, 0o755)
+            
+            # Prepare command with arguments
+            command = ["bash", script_path, py_requirements, conda_env_path, py_version]
+
+            process = await asyncio.create_subprocess_exec(
+                *command, 
+                stdout=asyncio.subprocess.PIPE, 
+                stderr=asyncio.subprocess.PIPE
+            )
+            
+            return process
+        except OSError as e:
+            print(f"OS error while creating {conda_env_path}: {e}")
+        except Exception as e:
+            print(f"Unexpected error while creating {conda_env_path}: {e}")
+            return {"error": str(e)}

dataflow/scripts/create_environment.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+set -e
+
+IFS=',' read -r -a  libraries <<< $1
+conda_env_path=$2
+py_version=$3
+
+# Use an isolated conda package cache to avoid concurrency issues
+export CONDA_PKGS_DIRS=$(mktemp -d)
+# to delete conda package cache after script finishes
+trap 'rm -rf "$CONDA_PKGS_DIRS"' EXIT
+
+# 1. Creating conda environment
+conda create --prefix ${conda_env_path} --yes python=${py_version}                         
+${conda_env_path}/bin/pip install --root-user-action ignore ${libraries[@]}      
+
+# 3. Install Dataflow Airflow to a separate path in environment 
+${conda_env_path}/bin/pip install \
+    --force-reinstall --root-user-action ignore \
+    --no-warn-conflicts dataflow-airflow==2.10.3 \
+    --target ${conda_env_path}/bin/airflow-libraries/ \
+    --constraint https://raw.githubusercontent.com//apache/airflow/constraints-2.10.3/constraints-${py_version}.txt || true 
+
+files=(
+    ${conda_env_path}/lib/python${py_version}/site-packages/dbt/config/profile.py 
+    ${conda_env_path}/lib/python${py_version}/site-packages/dbt/task/debug.py
+)
+for file in ${files[@]}
+do      
+    awk '{gsub("from dbt.clients.yaml_helper import load_yaml_text", "from dbt.dataflow_config.secrets_manager import load_yaml_text"); print}' $file > temp 
+    mv temp $file
+done
+
+echo "Environment Creation Successful"

{dataflow_core-2.0.5.dist-info → dataflow_core-2.0.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dataflow-core
-Version: 2.0.5
+Version: 2.0.6
 Summary: Dataflow core package
 Home-page: UNKNOWN
 Author: Dataflow

{dataflow_core-2.0.5.dist-info → dataflow_core-2.0.6.dist-info}/RECORD

@@ -10,15 +10,16 @@ authenticator/package/models/session.py,sha256=j6PhbrTMJxEkeDT4Vf5SqGtM_LI_vZy9O
 authenticator/package/models/user.py,sha256=IYogp_vt0yDBG5i936uNPjgTis77VYPzITn9XpQUIyw,788
 dataflow/__init__.py,sha256=WTRg8HMpMWSgxYJ9ZGVldx4k07fAbta3mBmZ1hG9mWE,30
 dataflow/configuration.py,sha256=7To6XwH1eESiYp39eqPcswXWwrdBUdPF6xN6WnazOF0,663
-dataflow/dataflow.py,sha256=8jLshxmxVtBTF6LYIGD9RJyB1mujo0tNr8vjHE04JNE,3418
+dataflow/dataflow.py,sha256=viEKxu1bVFtenbPx8etCb7Gf-Hbwv0KvkE7occVbX8I,5136
 dataflow/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dataflow/models/database.py,sha256=y09pqnglsBBtDZlyhvqDAlpUSFovwAzBAi6jOYl_XNk,896
 dataflow/models/session.py,sha256=C9crPh6ZDFuL27hZ_zhUXDZZ0ZiIDE8ZD19O_4WPw-I,488
 dataflow/models/user.py,sha256=IYogp_vt0yDBG5i936uNPjgTis77VYPzITn9XpQUIyw,788
+dataflow/scripts/create_environment.sh,sha256=r4cKRTMf1G9pXx1fqMpzkvd-ROzUH0Qp16f9t84Nh5c,1300
 dataflow/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dataflow/utils/aws_secrets_manager.py,sha256=FqHm3YRynv580FpFsS0RfI1MSGY5aq-V7t4blpiYsS4,2588
-dataflow_core-2.0.5.dist-info/METADATA,sha256=L8jAsmbkSv0xT9rh2Ac2t3dHuI0kJMLhVDXcnUWYbyk,317
-dataflow_core-2.0.5.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-dataflow_core-2.0.5.dist-info/entry_points.txt,sha256=lDLG2MMWlKfkqsVWFghF7sx-yEvM2xqMmHE7rMTysE4,118
-dataflow_core-2.0.5.dist-info/top_level.txt,sha256=SZsUOpSCK9ntUy-3Tusxzf5A2e8ebwD8vouPb1dPt_8,23
-dataflow_core-2.0.5.dist-info/RECORD,,
+dataflow_core-2.0.6.dist-info/METADATA,sha256=BCXu55SZS71k7ZQ2ilfFn_nimogjDv0SDKtrDyje5iw,317
+dataflow_core-2.0.6.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+dataflow_core-2.0.6.dist-info/entry_points.txt,sha256=lDLG2MMWlKfkqsVWFghF7sx-yEvM2xqMmHE7rMTysE4,118
+dataflow_core-2.0.6.dist-info/top_level.txt,sha256=SZsUOpSCK9ntUy-3Tusxzf5A2e8ebwD8vouPb1dPt_8,23
+dataflow_core-2.0.6.dist-info/RECORD,,