dataflow-core 2.0.5__tar.gz → 2.0.7__tar.gz

dataflow_core-2.0.7/PKG-INFO

@@ -0,0 +1,14 @@
+Metadata-Version: 2.4
+Name: dataflow-core
+Version: 2.0.7
+Summary: Dataflow core package
+Author: Dataflow
+Author-email: 
+Requires-Dist: sqlalchemy
+Requires-Dist: boto3
+Requires-Dist: psycopg2-binary
+Requires-Dist: pymysql
+Requires-Dist: requests
+Dynamic: author
+Dynamic: requires-dist
+Dynamic: summary

{dataflow-core-2.0.5 → dataflow_core-2.0.7}/authenticator/dataflowhubauthenticator.py

@@ -65,7 +65,23 @@ class DataflowHubAuthenticator(Authenticator):
                 self.db.refresh(db_item)
 
             expires = datetime.now(timezone.utc) + timedelta(days=365)
-            handler.set_cookie("dataflow_session", session_id, expires=expires)
+            host = handler.request.host
+            parts = host.split('.')
+            if len(parts) >= 2:
+                domain =  '.'.join(parts[-2:])
+            else:
+                domain =  host 
+            base_domain = f".{domain}" 
+            handler.set_cookie(
+                "dataflow_session",
+                session_id,
+                domain=base_domain, 
+                path="/",
+                expires=expires,
+                secure=True,            
+                httponly=True,
+                samesite="None"          
+            )
             user_dict = {"name": username, "session_id": session_id}
             return user_dict
 

{dataflow-core-2.0.5 → dataflow_core-2.0.7}/dataflow/dataflow.py

@@ -2,21 +2,21 @@ import os, requests
 from .models.database import DatabaseManager
 from sqlalchemy.inspection import inspect
 from .utils.aws_secrets_manager import SecretsManagerClient
-import json
+import json, asyncio, pkg_resources
 from authenticator.package.configuration import ConfigurationManager
 
 
 class Dataflow:
     def __init__(self):
-        self.dataflow_config = ConfigurationManager('/dataflow/app/auth_config/dataflow_auth.cfg')
-        self.auth_api = self.dataflow_config.get_config_value('auth', 'ui_auth_api')
         self.secrets_manager = SecretsManagerClient('us-east-1')
 
     def auth(self, session_id: str):
         """Retrieve user information from the auth API."""
         try:
+            dataflow_config = ConfigurationManager('/dataflow/app/auth_config/dataflow_auth.cfg')
+            auth_api = dataflow_config.get_config_value('auth', 'ui_auth_api')
             response = requests.get(
-                self.auth_api,
+                auth_api,
                 cookies={"dataflow_session": session_id, "jupyterhub-hub-login": ""}
             )
             
@@ -86,3 +86,42 @@ class Dataflow:
         
         except Exception as e:
             return None
+        
+    async def create_env(self, env_name, py_version, py_requirements, status, env_version=None):
+        """
+        Creates a conda environment at the specified path and installs libraries in one command.
+        """
+        config = ConfigurationManager('/dataflow/app/config/dataflow.cfg')
+        status = status.lower()
+        if status == "published":
+            env_base_path = config.get_config_value('paths', 'published_env_path')
+            conda_env_path = os.path.join(env_base_path, env_name)
+        else:
+            env_base_path = config.get_config_value('paths', 'drafts_env_path')
+            conda_env_path = os.path.join(env_base_path, env_name, f"{env_name}_v{env_version}")
+        try:
+            if not os.path.exists(conda_env_path):
+                os.makedirs(conda_env_path, exist_ok=True)
+
+            py_requirements = ",".join(py_requirements) 
+
+            script_path = pkg_resources.resource_filename('dataflow', 'scripts/create_environment.sh')
+    
+            # Make the script executable
+            os.chmod(script_path, 0o755)
+            
+            # Prepare command with arguments
+            command = ["bash", script_path, py_requirements, conda_env_path, py_version]
+
+            process = await asyncio.create_subprocess_exec(
+                *command, 
+                stdout=asyncio.subprocess.PIPE, 
+                stderr=asyncio.subprocess.PIPE
+            )
+            
+            return process
+        except OSError as e:
+            print(f"OS error while creating {conda_env_path}: {e}")
+        except Exception as e:
+            print(f"Unexpected error while creating {conda_env_path}: {e}")
+            return {"error": str(e)}

dataflow_core-2.0.7/dataflow/scripts/create_environment.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+set -e
+
+IFS=',' read -r -a  libraries <<< $1
+conda_env_path=$2
+py_version=$3
+
+# Use an isolated conda package cache to avoid concurrency issues
+export CONDA_PKGS_DIRS=$(mktemp -d)
+# to delete conda package cache after script finishes
+trap 'rm -rf "$CONDA_PKGS_DIRS"' EXIT
+
+# 1. Creating conda environment
+conda create --prefix ${conda_env_path} --yes python=${py_version}                         
+${conda_env_path}/bin/pip install --root-user-action ignore ${libraries[@]}      
+
+# 3. Install Dataflow Airflow to a separate path in environment 
+${conda_env_path}/bin/pip install \
+    --force-reinstall --root-user-action ignore \
+    --no-warn-conflicts dataflow-airflow==2.10.5 \
+    --target ${conda_env_path}/bin/airflow-libraries/ \
+    --constraint https://raw.githubusercontent.com//apache/airflow/constraints-2.10.5/constraints-${py_version}.txt || true 
+
+files=(
+    ${conda_env_path}/lib/python${py_version}/site-packages/dbt/config/profile.py 
+    ${conda_env_path}/lib/python${py_version}/site-packages/dbt/task/debug.py
+)
+for file in ${files[@]}
+do      
+    awk '{gsub("from dbt.clients.yaml_helper import load_yaml_text", "from dbt.dataflow_config.secrets_manager import load_yaml_text"); print}' $file > temp 
+    mv temp $file
+done
+
+echo "Environment Creation Successful"

dataflow_core-2.0.7/dataflow_core.egg-info/PKG-INFO

@@ -0,0 +1,14 @@
+Metadata-Version: 2.4
+Name: dataflow-core
+Version: 2.0.7
+Summary: Dataflow core package
+Author: Dataflow
+Author-email: 
+Requires-Dist: sqlalchemy
+Requires-Dist: boto3
+Requires-Dist: psycopg2-binary
+Requires-Dist: pymysql
+Requires-Dist: requests
+Dynamic: author
+Dynamic: requires-dist
+Dynamic: summary

{dataflow-core-2.0.5 → dataflow_core-2.0.7}/dataflow_core.egg-info/SOURCES.txt

@@ -15,8 +15,7 @@ dataflow/configuration.py
 dataflow/dataflow.py
 dataflow/models/__init__.py
 dataflow/models/database.py
-dataflow/models/session.py
-dataflow/models/user.py
+dataflow/scripts/create_environment.sh
 dataflow/utils/__init__.py
 dataflow/utils/aws_secrets_manager.py
 dataflow_core.egg-info/PKG-INFO

{dataflow-core-2.0.5 → dataflow_core-2.0.7}/dataflow_core.egg-info/entry_points.txt

@@ -1,3 +1,2 @@
 [jupyterhub.authenticators]
 dataflow_authenticator = authenticator.dataflowhubauthenticator:DataflowHubAuthenticator
-

{dataflow-core-2.0.5 → dataflow_core-2.0.7}/setup.py

@@ -2,10 +2,12 @@ from setuptools import setup, find_packages
 
 setup(
     name="dataflow-core",
-    version="2.0.5",
-    packages=find_packages(exclude=["tests", "tests.*"]),
+    version="2.0.7",
+    packages=find_packages(include=["dataflow", "dataflow.*", "authenticator", "authenticator.*"]),
     include_package_data=True,
-    package_data={},
+    package_data={
+        "dataflow": ["scripts/*.sh"],
+    },
     install_requires=[
         'sqlalchemy',
         'boto3',
@@ -21,4 +23,4 @@ setup(
             'dataflow_authenticator = authenticator.dataflowhubauthenticator:DataflowHubAuthenticator',
         ],
     },
-)
+)

dataflow-core-2.0.5/PKG-INFO

@@ -1,10 +0,0 @@
-Metadata-Version: 1.0
-Name: dataflow-core
-Version: 2.0.5
-Summary: Dataflow core package
-Home-page: UNKNOWN
-Author: Dataflow
-Author-email: UNKNOWN
-License: UNKNOWN
-Description: UNKNOWN
-Platform: UNKNOWN

dataflow-core-2.0.5/dataflow/models/session.py

@@ -1,17 +0,0 @@
-"""models.py"""
-from sqlalchemy import Column, Integer, String
-from sqlalchemy.ext.declarative import declarative_base
-
-#instance for create declarative base
-Base=declarative_base()
-
-class Session_table(Base):
-    """
-    Table SESSIONS
-    """
-
-    __tablename__='SESSION'
-
-    id = Column(Integer, primary_key=True, index=True, unique=True, nullable=False, autoincrement=True)
-    session_id = Column(String, unique=True, nullable=False)
-    user_id = Column(String, nullable=False)    

dataflow-core-2.0.5/dataflow/models/user.py

@@ -1,23 +0,0 @@
-"""models.py"""
-from sqlalchemy import Column, Integer, String, LargeBinary, Enum
-from sqlalchemy.ext.declarative import declarative_base
-
-#instance for create declarative base
-Base=declarative_base()
-
-class User(Base):
-    """
-    Table USER
-    """
-
-    __tablename__='USER'
-
-    user_id = Column(Integer, primary_key=True, index=True, autoincrement=True, nullable=False)
-    user_name = Column(String, unique=True, nullable=False)
-    first_name = Column(String)
-    last_name = Column(String)
-    email = Column(String, unique=True)
-    role = Column(Enum('admin', 'user', name='role_field'), nullable=False)
-    image = Column(LargeBinary)
-    active = Column(Enum('N', 'Y', name='active_field'), nullable=False, server_default=str("N"))
-    password = Column(String, nullable=False)

dataflow-core-2.0.5/dataflow_core.egg-info/PKG-INFO

@@ -1,10 +0,0 @@
-Metadata-Version: 1.0
-Name: dataflow-core
-Version: 2.0.5
-Summary: Dataflow core package
-Home-page: UNKNOWN
-Author: Dataflow
-Author-email: UNKNOWN
-License: UNKNOWN
-Description: UNKNOWN
-Platform: UNKNOWN