dataflow-core 2.0.3__tar.gz → 2.0.6__tar.gz

dataflow-core-2.0.6/PKG-INFO

@@ -0,0 +1,10 @@
+Metadata-Version: 1.0
+Name: dataflow-core
+Version: 2.0.6
+Summary: Dataflow core package
+Home-page: UNKNOWN
+Author: Dataflow
+Author-email: UNKNOWN
+License: UNKNOWN
+Description: UNKNOWN
+Platform: UNKNOWN

{dataflow_core-2.0.3 → dataflow-core-2.0.6}/authenticator/dataflowairflowauthenticator.py

@@ -1,34 +1,19 @@
-from .package.configuration import ConfigurationManager
 from .package.models.database import DatabaseManager
-from .package.models import (
-    user as m_user,
-    session as m_session
-    )
+from dataflow.dataflow import Dataflow
 
 from typing import Any, Callable
 from airflow.www.security import FabAirflowSecurityManagerOverride
+from airflow.configuration import conf
 
 class DataflowAirflowAuthenticator(FabAirflowSecurityManagerOverride):
     def __init__(self, wsgi_app: Callable) -> None:
         self.wsgi_app = wsgi_app
+        self.dataflow = Dataflow()
+        self.airflow_database_url = conf.get("database", "sql_alchemy_conn")
 
-        # Dataflow database configuration
-        self.dataflow_config = ConfigurationManager('/dataflow/app/config/dataflow.cfg')
-        self.dataflow_database_url = self.dataflow_config.get_config_value('database', 'database_url')
-
-        self.dataflow_db_instance = DatabaseManager(self.dataflow_database_url)
-        self.dataflow_db = next(self.dataflow_db_instance.get_session())
-
-        # Airflow database configuration
-        self.airflow_config = ConfigurationManager('airflow.cfg')
-        self.airflow_database_url = self.airflow_config.get_config_value('database', 'sql_alchemy_conn')
-        
         self.airflow_db_instance = DatabaseManager(self.airflow_database_url)
         self.airflow_db = next(self.airflow_db_instance.get_session())
 
-        m_user.Base.metadata.create_all(bind=self.dataflow_db_instance.get_engine())
-        m_session.Base.metadata.create_all(bind=self.dataflow_db_instance.get_engine())
-
     def __call__(self, environ: dict, start_response: Callable) -> Any:
 
         path = environ.get('PATH_INFO', '')
@@ -51,39 +36,28 @@ class DataflowAirflowAuthenticator(FabAirflowSecurityManagerOverride):
             user_session_id = user_session_id.split('=')[1]
 
             # Retrieving user details
-            user_data = self.find_dataflow_user(user_session_id)
+            user_data = self.dataflow.auth(user_session_id)
 
             if user_data is None:
                 raise Exception("No user found for the dataflow_session id")
             
-            user = self.find_user(user_data.user_name)
+            user = self.find_user(user_data["user_name"])
 
             if not user:
-                user_role = self.find_role(user_data.role.title())
-                user = self.add_user(username=user_data.user_name, first_name=self.not_none(user_data.first_name), last_name=self.not_none(user_data.last_name), email=self.not_none(user_data.email), role=user_role, password=self.not_none(user_data.password))
+                user_role = self.find_role(user_data["role"].title())
+                user = self.add_user(
+                    username=user_data["user_name"],
+                    first_name=user_data.get("first_name", ""),
+                    last_name=user_data.get("last_name", ""),
+                    email=user_data.get("email", ""),
+                    role=user_role
+                )
 
             environ['REMOTE_USER'] = user.username
-            self.write_user_id(user_data.user_id)
             return self.wsgi_app(environ, start_response)
 
         except Exception as e:
             return self.wsgi_app(environ, start_response)
-    
-    def not_none(self, value):
-        return value if value is not None else ""
-    
-    def find_dataflow_user(self, user_session_id):
-        """Find user by session_id in dataflow database."""
-        query = self.dataflow_db.query(m_session.Session_table)
-        session = query.filter(m_session.Session_table.session_id == user_session_id).first()
-        if session is None:
-            return None
-
-        user_data = self.dataflow_db.query(m_user.User).filter(m_user.User.user_id == session.user_id).first()
-        if user_data is None:
-            return None
-        
-        return user_data
 
     def find_user(self, username=None):
         """Find user by username or email."""
@@ -106,16 +80,4 @@ class DataflowAirflowAuthenticator(FabAirflowSecurityManagerOverride):
         self.airflow_db.add(user)
         self.airflow_db.commit()
         return user
-
-    def write_user_id(self, user_id):
-        """
-        Write the given user_id to a file named dataflow_user_id.txt.
-        
-        Args:
-            user_id (str): The user ID to be written to the file.
-        """
-        file_name = 'dataflow_user_id.txt'
-        with open(file_name, 'w') as file:
-            file.write(str(user_id))
-
         

dataflow-core-2.0.6/authenticator/dataflowsupersetauthenticator.py

@@ -0,0 +1,48 @@
+from flask import redirect, request
+from flask_appbuilder.security.views import AuthDBView
+from superset.security import SupersetSecurityManager
+from flask_appbuilder.security.views import expose
+from flask_login import login_user
+from .package.configuration import ConfigurationManager
+from .package.models.database import DatabaseManager
+from .package.models import (
+    user as m_user,
+    session as m_session
+    )
+from dataflow.dataflow import Dataflow
+
+class CustomAuthDBView(AuthDBView):
+    def __init__(self):
+        self.dataflow = Dataflow()
+
+    @expose('/login/', methods=['GET'])
+    def login(self):
+        try:
+            session_id = request.cookies.get('dataflow_session')
+            
+            user_details = self.dataflow.auth(session_id)
+            user = self.appbuilder.sm.find_user(username=user_details['user_name'])
+            if user:
+                login_user(user, remember=False)
+            else:
+                user = self.appbuilder.sm.add_user(
+                    username=user_details['user_name'], 
+                    first_name=user_details.get("first_name", ""),
+                    last_name=user_details.get("last_name", ""), 
+                    email=user_details.get("email", ""), 
+                    role=self.appbuilder.sm.find_role('Admin'), 
+                    password=""
+                )
+                if user:
+                    login_user(user, remember=False)
+                    
+            return redirect(self.appbuilder.get_url_for_index)
+
+        except Exception as e:
+            return super().login()
+
+
+class CustomSecurityManager(SupersetSecurityManager):
+    authdbview = CustomAuthDBView
+    def __init__(self, appbuilder):
+        super(CustomSecurityManager, self).__init__(appbuilder)

dataflow-core-2.0.6/dataflow/__init__.py

@@ -0,0 +1 @@
+from .dataflow import Dataflow

dataflow-core-2.0.6/dataflow/dataflow.py

@@ -0,0 +1,127 @@
+import os, requests
+from .models.database import DatabaseManager
+from sqlalchemy.inspection import inspect
+from .utils.aws_secrets_manager import SecretsManagerClient
+import json, asyncio, pkg_resources
+from authenticator.package.configuration import ConfigurationManager
+
+
+class Dataflow:
+    def __init__(self):
+        self.secrets_manager = SecretsManagerClient('us-east-1')
+
+    def auth(self, session_id: str):
+        """Retrieve user information from the auth API."""
+        try:
+            dataflow_config = ConfigurationManager('/dataflow/app/auth_config/dataflow_auth.cfg')
+            auth_api = dataflow_config.get_config_value('auth', 'ui_auth_api')
+            response = requests.get(
+                auth_api,
+                cookies={"dataflow_session": session_id, "jupyterhub-hub-login": ""}
+            )
+            
+            if response.status_code != 200:
+                return response.json()
+            
+            user_data = response.json()
+            user_dict = {
+                "user_name": user_data["user_name"], 
+                "first_name": user_data["first_name"],
+                "last_name": user_data["last_name"] if user_data.get("last_name") else "",
+                "email": user_data["email"],
+                "role": user_data["role"]
+            }
+            return user_dict
+                  
+        except Exception as e:
+            return e
+    
+    def variable(self, variable_name: str):
+        """Get variable value from secrets manager."""
+        try:
+            host_name = os.environ["HOSTNAME"]
+            user_name = host_name.replace("jupyter-","")
+            
+            vault_path = "variables"
+            variable_data =  self.secrets_manager.get_secret_by_key(vault_path, user_name, variable_name)
+            return variable_data['value']
+            
+        except Exception as e:
+            return None
+        
+    def connection(self, conn_id: str):
+        """Get connection details from secrets manager."""
+        try:
+            host_name = os.environ["HOSTNAME"]
+            user_name=host_name.replace("jupyter-","")
+            
+            vault_path = "connections"
+            secret = self.secrets_manager.get_secret_by_key(vault_path, user_name, conn_id)
+
+            conn_type = secret['conn_type'].lower()
+            username = secret['login']
+            password = secret.get('password', '')
+            host = secret['host']
+            port = secret['port']
+            database = secret.get('schemas', '')
+
+            user_info = f"{username}:{password}@" if password else f"{username}@"
+            db_info = f"/{database}" if database else ""
+
+            connection_string = f"{conn_type}://{user_info}{host}:{port}{db_info}"
+
+            extra = secret.get('extra', '')
+            if extra:
+                try:
+                    extra_params = json.loads(extra)
+                    if extra_params:
+                        extra_query = "&".join(f"{key}={value}" for key, value in extra_params.items())
+                        connection_string += f"?{extra_query}"
+                except json.JSONDecodeError:
+                    # If 'extra' is not valid JSON, skip adding extra parameters
+                    pass
+
+            connection_instance = DatabaseManager(connection_string)
+            return next(connection_instance.get_session())
+        
+        except Exception as e:
+            return None
+        
+    async def create_env(self, env_name, py_version, py_requirements, status, env_version=None):
+        """
+        Creates a conda environment at the specified path and installs libraries in one command.
+        """
+        config = ConfigurationManager('/dataflow/app/config/dataflow.cfg')
+        status = status.lower()
+        if status == "published":
+            env_base_path = config.get_config_value('paths', 'published_env_path')
+            conda_env_path = os.path.join(env_base_path, env_name)
+        else:
+            env_base_path = config.get_config_value('paths', 'drafts_env_path')
+            conda_env_path = os.path.join(env_base_path, env_name, f"{env_name}_v{env_version}")
+        try:
+            if not os.path.exists(conda_env_path):
+                os.makedirs(conda_env_path, exist_ok=True)
+
+            py_requirements = ",".join(py_requirements) 
+
+            script_path = pkg_resources.resource_filename('dataflow', 'scripts/create_environment.sh')
+    
+            # Make the script executable
+            os.chmod(script_path, 0o755)
+            
+            # Prepare command with arguments
+            command = ["bash", script_path, py_requirements, conda_env_path, py_version]
+
+            process = await asyncio.create_subprocess_exec(
+                *command, 
+                stdout=asyncio.subprocess.PIPE, 
+                stderr=asyncio.subprocess.PIPE
+            )
+            
+            return process
+        except OSError as e:
+            print(f"OS error while creating {conda_env_path}: {e}")
+        except Exception as e:
+            print(f"Unexpected error while creating {conda_env_path}: {e}")
+            return {"error": str(e)}

dataflow-core-2.0.6/dataflow_core.egg-info/PKG-INFO

@@ -0,0 +1,10 @@
+Metadata-Version: 1.0
+Name: dataflow-core
+Version: 2.0.6
+Summary: Dataflow core package
+Home-page: UNKNOWN
+Author: Dataflow
+Author-email: UNKNOWN
+License: UNKNOWN
+Description: UNKNOWN
+Platform: UNKNOWN

{dataflow_core-2.0.3 → dataflow-core-2.0.6}/dataflow_core.egg-info/entry_points.txt

@@ -1,2 +1,3 @@
 [jupyterhub.authenticators]
 dataflow_authenticator = authenticator.dataflowhubauthenticator:DataflowHubAuthenticator
+

{dataflow_core-2.0.3 → dataflow-core-2.0.6}/dataflow_core.egg-info/requires.txt

@@ -2,3 +2,4 @@ sqlalchemy
 boto3
 psycopg2-binary
 pymysql
+requests

{dataflow_core-2.0.3 → dataflow-core-2.0.6}/setup.py

@@ -2,15 +2,18 @@ from setuptools import setup, find_packages
 
 setup(
     name="dataflow-core",
-    version="2.0.3",
-    packages=find_packages(exclude=["tests", "tests.*"]),
+    version="2.0.6",
+    packages=find_packages(include=["dataflow", "dataflow.*", "authenticator", "authenticator.*"]),
     include_package_data=True,
-    package_data={},
+    package_data={
+        "dataflow": ["scripts/*.sh"],
+    },
     install_requires=[
         'sqlalchemy',
         'boto3',
         'psycopg2-binary',
-        'pymysql'
+        'pymysql',
+        'requests'
     ],
     author="Dataflow",
     author_email="",

dataflow_core-2.0.3/PKG-INFO

@@ -1,10 +0,0 @@
-Metadata-Version: 2.1
-Name: dataflow-core
-Version: 2.0.3
-Summary: Dataflow core package
-Author: Dataflow
-Author-email: 
-Requires-Dist: sqlalchemy
-Requires-Dist: boto3
-Requires-Dist: psycopg2-binary
-Requires-Dist: pymysql

dataflow_core-2.0.3/authenticator/dataflowsupersetauthenticator.py

@@ -1,57 +0,0 @@
-from flask import redirect, request
-from flask_appbuilder.security.views import AuthDBView
-from superset.security import SupersetSecurityManager
-from flask_appbuilder.security.views import expose
-from flask_login import login_user
-from .package.configuration import ConfigurationManager
-from .package.models.database import DatabaseManager
-from .package.models import (
-    user as m_user,
-    session as m_session
-    )
-
-class CustomAuthDBView(AuthDBView):
-    def __init__(self):
-        self.dataflow_config = ConfigurationManager('/dataflow/app/config/dataflow.cfg')
-        self.dataflow_database_url = self.dataflow_config.get_config_value('database', 'database_url')
-        self.dataflow_db_instance = DatabaseManager(self.dataflow_database_url)
-        self.dataflow_db = next(self.dataflow_db_instance.get_session())
-        m_user.Base.metadata.create_all(bind=self.dataflow_db_instance.get_engine())
-        m_session.Base.metadata.create_all(bind=self.dataflow_db_instance.get_engine())
-
-    def _get_user_id_from_session(self, session_id):
-        query = self.dataflow_db.query(m_session.Session_table).filter(m_session.Session_table.session_id == session_id).first()
-        return query.user_id if query!=None else None
-
-    def _get_user_details_from_user_table(self, user_id):
-        user_details = self.dataflow_db.query(m_user.User).filter(m_user.User.user_id == user_id).first()
-        return user_details if user_details!=None else None
-    
-    def not_none(self, value):
-        return value if value is not None else ""
-
-    @expose('/login/', methods=['GET'])
-    def login(self):
-        try:
-            session_id = request.cookies.get('dataflow_session')
-            
-            user_id = self._get_user_id_from_session(session_id)
-            user_details = self._get_user_details_from_user_table(user_id)
-            user = self.appbuilder.sm.find_user(username=user_details.user_name)
-            if user:
-                login_user(user, remember=False)
-            else:
-                user = self.appbuilder.sm.add_user(username=self.not_none(user_details.user_name), first_name=self.not_none(user_details.first_name), last_name=self.not_none(user_details.last_name), email=self.not_none(user_details.email), role=self.appbuilder.sm.find_role('Admin'), password=self.not_none(user_details.password))
-                if user:
-                    login_user(user, remember=False)
-                    
-            return redirect(self.appbuilder.get_url_for_index)
-
-        except Exception as e:
-            return super().login()
-
-
-class CustomSecurityManager(SupersetSecurityManager):
-    authdbview = CustomAuthDBView
-    def __init__(self, appbuilder):
-        super(CustomSecurityManager, self).__init__(appbuilder)

dataflow_core-2.0.3/dataflow/dataflow.py

@@ -1,93 +0,0 @@
-import os
-from .configuration import ConfigurationManager
-from .models import (
-    session as m_session,
-    user as m_user,
-)
-from .models.database import DatabaseManager
-from sqlalchemy.inspection import inspect
-from .utils.aws_secrets_manager import SecretsManagerClient
-import json
-
-
-class Dataflow:
-    def __init__(self):
-        self.dataflow_config = ConfigurationManager('/dataflow/app/config/dataflow.cfg')
-        self.dataflow_database_url = self.dataflow_config.get_config_value('database', 'database_url')
-
-        self.dataflow_db_instance = DatabaseManager(self.dataflow_database_url)
-        self.dataflow_db = next(self.dataflow_db_instance.get_session())
-
-        self.secrets_manager = SecretsManagerClient('us-east-1')
-
-        m_user.Base.metadata.create_all(bind=self.dataflow_db_instance.get_engine())
-        m_session.Base.metadata.create_all(bind=self.dataflow_db_instance.get_engine())
-
-    def auth(self, session_id: str):
-        """Find user by session_id in dataflow database."""
-        try:
-            query = self.dataflow_db.query(m_session.Session_table)
-            session = query.filter(m_session.Session_table.session_id == session_id).first()
-            if session is None:
-                return False
-
-            user_data = self.dataflow_db.query(m_user.User).filter(m_user.User.user_id == session.user_id).first()
-            if user_data is None:
-                return False
-            
-            user_dict = {"user_name": user_data.user_name, "name": f"{user_data.first_name} {user_data.last_name}", "email": user_data.email, "role": user_data.role}
-            return user_dict
-        
-        except Exception as e:
-            return False
-    
-    def variable(self, variable_name: str):
-        """Get variable value from secrets manager."""
-        try:
-            host_name = os.environ["HOSTNAME"]
-            user_name = host_name.replace("jupyter-","")
-            
-            vault_path = "variables"
-            variable_data =  self.secrets_manager.get_secret_by_key(vault_path, user_name, variable_name)
-            return variable_data['value']
-            
-        except Exception as e:
-            return None
-        
-    def connection(self, conn_id: str):
-        """Get connection details from secrets manager."""
-        try:
-            host_name = os.environ["HOSTNAME"]
-            user_name=host_name.replace("jupyter-","")
-            
-            vault_path = "connections"
-            secret = self.secrets_manager.get_secret_by_key(vault_path, user_name, conn_id)
-
-            conn_type = secret['conn_type'].lower()
-            username = secret['login']
-            password = secret.get('password', '')
-            host = secret['host']
-            port = secret['port']
-            database = secret.get('schemas', '')
-
-            user_info = f"{username}:{password}@" if password else f"{username}@"
-            db_info = f"/{database}" if database else ""
-
-            connection_string = f"{conn_type}://{user_info}{host}:{port}{db_info}"
-
-            extra = secret.get('extra', '')
-            if extra:
-                try:
-                    extra_params = json.loads(extra)
-                    if extra_params:
-                        extra_query = "&".join(f"{key}={value}" for key, value in extra_params.items())
-                        connection_string += f"?{extra_query}"
-                except json.JSONDecodeError:
-                    # If 'extra' is not valid JSON, skip adding extra parameters
-                    pass
-
-            connection_instance = DatabaseManager(connection_string)
-            return next(connection_instance.get_session())
-        
-        except Exception as e:
-            return None

dataflow_core-2.0.3/dataflow_core.egg-info/PKG-INFO

@@ -1,10 +0,0 @@
-Metadata-Version: 2.1
-Name: dataflow-core
-Version: 2.0.3
-Summary: Dataflow core package
-Author: Dataflow
-Author-email: 
-Requires-Dist: sqlalchemy
-Requires-Dist: boto3
-Requires-Dist: psycopg2-binary
-Requires-Dist: pymysql