dataflow-core 2.0.5__py3-none-any.whl → 2.0.7__py3-none-any.whl

authenticator/dataflowhubauthenticator.py

@@ -65,7 +65,23 @@ class DataflowHubAuthenticator(Authenticator):
                 self.db.refresh(db_item)
 
             expires = datetime.now(timezone.utc) + timedelta(days=365)
-            handler.set_cookie("dataflow_session", session_id, expires=expires)
+            host = handler.request.host
+            parts = host.split('.')
+            if len(parts) >= 2:
+                domain =  '.'.join(parts[-2:])
+            else:
+                domain =  host 
+            base_domain = f".{domain}" 
+            handler.set_cookie(
+                "dataflow_session",
+                session_id,
+                domain=base_domain, 
+                path="/",
+                expires=expires,
+                secure=True,            
+                httponly=True,
+                samesite="None"          
+            )
             user_dict = {"name": username, "session_id": session_id}
             return user_dict
 

dataflow/dataflow.py

@@ -2,21 +2,21 @@ import os, requests
 from .models.database import DatabaseManager
 from sqlalchemy.inspection import inspect
 from .utils.aws_secrets_manager import SecretsManagerClient
-import json
+import json, asyncio, pkg_resources
 from authenticator.package.configuration import ConfigurationManager
 
 
 class Dataflow:
     def __init__(self):
-        self.dataflow_config = ConfigurationManager('/dataflow/app/auth_config/dataflow_auth.cfg')
-        self.auth_api = self.dataflow_config.get_config_value('auth', 'ui_auth_api')
         self.secrets_manager = SecretsManagerClient('us-east-1')
 
     def auth(self, session_id: str):
         """Retrieve user information from the auth API."""
         try:
+            dataflow_config = ConfigurationManager('/dataflow/app/auth_config/dataflow_auth.cfg')
+            auth_api = dataflow_config.get_config_value('auth', 'ui_auth_api')
             response = requests.get(
-                self.auth_api,
+                auth_api,
                 cookies={"dataflow_session": session_id, "jupyterhub-hub-login": ""}
             )
             
@@ -86,3 +86,42 @@ class Dataflow:
         
         except Exception as e:
             return None
+        
+    async def create_env(self, env_name, py_version, py_requirements, status, env_version=None):
+        """
+        Creates a conda environment at the specified path and installs libraries in one command.
+        """
+        config = ConfigurationManager('/dataflow/app/config/dataflow.cfg')
+        status = status.lower()
+        if status == "published":
+            env_base_path = config.get_config_value('paths', 'published_env_path')
+            conda_env_path = os.path.join(env_base_path, env_name)
+        else:
+            env_base_path = config.get_config_value('paths', 'drafts_env_path')
+            conda_env_path = os.path.join(env_base_path, env_name, f"{env_name}_v{env_version}")
+        try:
+            if not os.path.exists(conda_env_path):
+                os.makedirs(conda_env_path, exist_ok=True)
+
+            py_requirements = ",".join(py_requirements) 
+
+            script_path = pkg_resources.resource_filename('dataflow', 'scripts/create_environment.sh')
+    
+            # Make the script executable
+            os.chmod(script_path, 0o755)
+            
+            # Prepare command with arguments
+            command = ["bash", script_path, py_requirements, conda_env_path, py_version]
+
+            process = await asyncio.create_subprocess_exec(
+                *command, 
+                stdout=asyncio.subprocess.PIPE, 
+                stderr=asyncio.subprocess.PIPE
+            )
+            
+            return process
+        except OSError as e:
+            print(f"OS error while creating {conda_env_path}: {e}")
+        except Exception as e:
+            print(f"Unexpected error while creating {conda_env_path}: {e}")
+            return {"error": str(e)}

dataflow/scripts/create_environment.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+set -e
+
+IFS=',' read -r -a  libraries <<< $1
+conda_env_path=$2
+py_version=$3
+
+# Use an isolated conda package cache to avoid concurrency issues
+export CONDA_PKGS_DIRS=$(mktemp -d)
+# to delete conda package cache after script finishes
+trap 'rm -rf "$CONDA_PKGS_DIRS"' EXIT
+
+# 1. Creating conda environment
+conda create --prefix ${conda_env_path} --yes python=${py_version}                         
+${conda_env_path}/bin/pip install --root-user-action ignore ${libraries[@]}      
+
+# 3. Install Dataflow Airflow to a separate path in environment 
+${conda_env_path}/bin/pip install \
+    --force-reinstall --root-user-action ignore \
+    --no-warn-conflicts dataflow-airflow==2.10.5 \
+    --target ${conda_env_path}/bin/airflow-libraries/ \
+    --constraint https://raw.githubusercontent.com//apache/airflow/constraints-2.10.5/constraints-${py_version}.txt || true 
+
+files=(
+    ${conda_env_path}/lib/python${py_version}/site-packages/dbt/config/profile.py 
+    ${conda_env_path}/lib/python${py_version}/site-packages/dbt/task/debug.py
+)
+for file in ${files[@]}
+do      
+    awk '{gsub("from dbt.clients.yaml_helper import load_yaml_text", "from dbt.dataflow_config.secrets_manager import load_yaml_text"); print}' $file > temp 
+    mv temp $file
+done
+
+echo "Environment Creation Successful"

{dataflow_core-2.0.5.dist-info → dataflow_core-2.0.7.dist-info}/METADATA

@@ -1,18 +1,14 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: dataflow-core
-Version: 2.0.5
+Version: 2.0.7
 Summary: Dataflow core package
-Home-page: UNKNOWN
 Author: Dataflow
-Author-email: UNKNOWN
-License: UNKNOWN
-Platform: UNKNOWN
+Author-email: 
 Requires-Dist: sqlalchemy
 Requires-Dist: boto3
 Requires-Dist: psycopg2-binary
 Requires-Dist: pymysql
 Requires-Dist: requests
-
-UNKNOWN
-
-
+Dynamic: author
+Dynamic: requires-dist
+Dynamic: summary

{dataflow_core-2.0.5.dist-info → dataflow_core-2.0.7.dist-info}/RECORD

@@ -1,6 +1,6 @@
 authenticator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 authenticator/dataflowairflowauthenticator.py,sha256=PcGlL2cq5EA9RdtClyrOupvNXxk6h54UPQcQ-g4VQtA,3117
-authenticator/dataflowhubauthenticator.py,sha256=UgjZiXSP-hlbciyJVb-QIgUjS3_EgX0WIvbhD563rVw,2918
+authenticator/dataflowhubauthenticator.py,sha256=m7ef84WfAjQ0IM8bg-1JhPs75rMNwKq3B0ZB5XF-07w,3403
 authenticator/dataflowsupersetauthenticator.py,sha256=M2pNpIdmWwzJVuMQ6trXLWQT4HpcYT84bC93uOSm0fQ,1772
 authenticator/package/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 authenticator/package/configuration.py,sha256=7To6XwH1eESiYp39eqPcswXWwrdBUdPF6xN6WnazOF0,663
@@ -10,15 +10,14 @@ authenticator/package/models/session.py,sha256=j6PhbrTMJxEkeDT4Vf5SqGtM_LI_vZy9O
 authenticator/package/models/user.py,sha256=IYogp_vt0yDBG5i936uNPjgTis77VYPzITn9XpQUIyw,788
 dataflow/__init__.py,sha256=WTRg8HMpMWSgxYJ9ZGVldx4k07fAbta3mBmZ1hG9mWE,30
 dataflow/configuration.py,sha256=7To6XwH1eESiYp39eqPcswXWwrdBUdPF6xN6WnazOF0,663
-dataflow/dataflow.py,sha256=8jLshxmxVtBTF6LYIGD9RJyB1mujo0tNr8vjHE04JNE,3418
+dataflow/dataflow.py,sha256=viEKxu1bVFtenbPx8etCb7Gf-Hbwv0KvkE7occVbX8I,5136
 dataflow/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dataflow/models/database.py,sha256=y09pqnglsBBtDZlyhvqDAlpUSFovwAzBAi6jOYl_XNk,896
-dataflow/models/session.py,sha256=C9crPh6ZDFuL27hZ_zhUXDZZ0ZiIDE8ZD19O_4WPw-I,488
-dataflow/models/user.py,sha256=IYogp_vt0yDBG5i936uNPjgTis77VYPzITn9XpQUIyw,788
+dataflow/scripts/create_environment.sh,sha256=nnE2295DCflUGz11RR0k6-TulQlpoXLqEpKZ9QU5t6g,1300
 dataflow/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dataflow/utils/aws_secrets_manager.py,sha256=FqHm3YRynv580FpFsS0RfI1MSGY5aq-V7t4blpiYsS4,2588
-dataflow_core-2.0.5.dist-info/METADATA,sha256=L8jAsmbkSv0xT9rh2Ac2t3dHuI0kJMLhVDXcnUWYbyk,317
-dataflow_core-2.0.5.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-dataflow_core-2.0.5.dist-info/entry_points.txt,sha256=lDLG2MMWlKfkqsVWFghF7sx-yEvM2xqMmHE7rMTysE4,118
-dataflow_core-2.0.5.dist-info/top_level.txt,sha256=SZsUOpSCK9ntUy-3Tusxzf5A2e8ebwD8vouPb1dPt_8,23
-dataflow_core-2.0.5.dist-info/RECORD,,
+dataflow_core-2.0.7.dist-info/METADATA,sha256=em6wCVL-OURnlA9c7tHV_25vCM6fsrEVuRCOuAtIKBo,301
+dataflow_core-2.0.7.dist-info/WHEEL,sha256=wXxTzcEDnjrTwFYjLPcsW_7_XihufBwmpiBeiXNBGEA,91
+dataflow_core-2.0.7.dist-info/entry_points.txt,sha256=ppj_EIbYrJJwCPg1kfdsZk5q1N-Ejfis1neYrnjhO8o,117
+dataflow_core-2.0.7.dist-info/top_level.txt,sha256=SZsUOpSCK9ntUy-3Tusxzf5A2e8ebwD8vouPb1dPt_8,23
+dataflow_core-2.0.7.dist-info/RECORD,,

{dataflow_core-2.0.5.dist-info → dataflow_core-2.0.7.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.45.1)
+Generator: setuptools (80.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{dataflow_core-2.0.5.dist-info → dataflow_core-2.0.7.dist-info}/entry_points.txt

@@ -1,3 +1,2 @@
 [jupyterhub.authenticators]
 dataflow_authenticator = authenticator.dataflowhubauthenticator:DataflowHubAuthenticator
-

dataflow/models/session.py

@@ -1,17 +0,0 @@
-"""models.py"""
-from sqlalchemy import Column, Integer, String
-from sqlalchemy.ext.declarative import declarative_base
-
-#instance for create declarative base
-Base=declarative_base()
-
-class Session_table(Base):
-    """
-    Table SESSIONS
-    """
-
-    __tablename__='SESSION'
-
-    id = Column(Integer, primary_key=True, index=True, unique=True, nullable=False, autoincrement=True)
-    session_id = Column(String, unique=True, nullable=False)
-    user_id = Column(String, nullable=False)    

dataflow/models/user.py

@@ -1,23 +0,0 @@
-"""models.py"""
-from sqlalchemy import Column, Integer, String, LargeBinary, Enum
-from sqlalchemy.ext.declarative import declarative_base
-
-#instance for create declarative base
-Base=declarative_base()
-
-class User(Base):
-    """
-    Table USER
-    """
-
-    __tablename__='USER'
-
-    user_id = Column(Integer, primary_key=True, index=True, autoincrement=True, nullable=False)
-    user_name = Column(String, unique=True, nullable=False)
-    first_name = Column(String)
-    last_name = Column(String)
-    email = Column(String, unique=True)
-    role = Column(Enum('admin', 'user', name='role_field'), nullable=False)
-    image = Column(LargeBinary)
-    active = Column(Enum('N', 'Y', name='active_field'), nullable=False, server_default=str("N"))
-    password = Column(String, nullable=False)