dataflow-core 2.1.5__py3-none-any.whl → 2.1.7__py3-none-any.whl

authenticator/dataflowhubauthenticator.py

@@ -136,7 +136,6 @@ class DataflowBaseAuthenticator(Authenticator):
                 last_name=last_name or "",
                 email=email,
                 role_id=role_id,
-                active='Y',
                 password='user@123',
             )
             

dataflow/db.py

@@ -1,14 +1,20 @@
+import dataflow
 from .database_manager import DatabaseManager
 from .configuration import ConfigurationManager
 from sqlalchemy.orm import declarative_base
 from sqlalchemy.exc import SQLAlchemyError
 from .utils.logger import CustomLogger
+import os
+import pwd
+import grp
+from urllib.parse import urlparse
 
 logger = CustomLogger().get_logger(__name__)
 
 dataflow_config = ConfigurationManager('/dataflow/app/config/dataflow.cfg')
+dataflow_local_config = ConfigurationManager('/dataflow/app/auth_config/dataflow_auth.cfg')
 db_url = dataflow_config.get_config_value('database', 'database_url')
-local_db_url = dataflow_config.get_config_value('database', 'single_user_db_url')
+local_db_url = dataflow_local_config.get_config_value('database', 'single_user_db_url')
 
 db_manager = DatabaseManager(db_url)
 local_db_manager = None
@@ -23,10 +29,44 @@ def create_tables(local_db=False):
     """
     try:
         if local_db:
-            global local_db_manager
-            if local_db_manager is None:
-                local_db_manager = DatabaseManager(local_db_url) 
-            Local_Base.metadata.create_all(bind=local_db_manager.get_engine())
+            if local_db_url and local_db_url.startswith('sqlite:///'):
+                parsed_url = urlparse(local_db_url)
+                db_file_path = parsed_url.path
+                db_dir = os.path.dirname(db_file_path)
+                logger.info(f"Creating local database at: {db_file_path}")
+                
+                if db_dir and not os.path.exists(db_dir):
+                    os.makedirs(db_dir, exist_ok=True)
+                    logger.info(f"Created directory for local database: {db_dir}")
+                
+                # Set ownership for database directory and file
+                nb_user = os.environ.get("NB_USER")
+                if nb_user:
+                    try:
+                        uid = pwd.getpwnam(nb_user).pw_uid
+                        gid = grp.getgrnam(nb_user).gr_gid
+                        
+                        # Set ownership for database directory
+                        if os.path.exists(db_dir):
+                            os.chown(db_dir, uid, gid)
+                        
+                        global local_db_manager
+                        if local_db_manager is None:
+                            local_db_manager = DatabaseManager(local_db_url) 
+                        Local_Base.metadata.create_all(bind=local_db_manager.get_engine())
+                        
+                        if os.path.exists(db_file_path):
+                            os.chown(db_file_path, uid, gid)
+                        logger.info(f"Ownership set for local database files")
+                    except (KeyError, OSError) as e:
+                        logger.warning(f"Could not set ownership for database files: {e}")
+                else:
+                    logger.error("NB_USER environment variable is not set. Cannot set ownership for local database files.")
+                    raise ValueError("Can only use local database with a valid NB_USER environment variable")
+            else:
+                logger.error("Local database URL must be a valid SQLite URL starting with 'sqlite:///'")
+                raise ValueError("Local database URL must be a valid SQLite URL starting with 'sqlite:///'")
+            
         else:
             Base.metadata.create_all(bind=db_manager.get_engine())
         logger.info("Database tables created successfully")

dataflow/models/__init__.py

@@ -3,7 +3,7 @@
 from .role import Role
 from .user import User
 from .team import Team
-from .environment import Environment
+from .environment import (Environment, LocalEnvironment, ArchivedEnvironment, JobLogs)
 from .project_details import ProjectDetails
 from .recent_projects import RecentProjects
 from .pinned_projects import PinnedProject
@@ -11,8 +11,9 @@ from .app_types import AppType
 from .blacklist_library import BlacklistedLibrary
 from .environment_status import EnvironmentStatus
 from .session import Session
-from .server_config import ServerConfig
-from .runtime import RuntimeZone
+from .server_config import ServerConfig, CustomServerConfig
+from .dataflow_zone import DataflowZone
+from .role_zone import RoleZone
 from .environment_status import EnvironmentStatus
 from .user_team import UserTeam
 from .role_server import RoleServer

dataflow/models/dataflow_zone.py

@@ -0,0 +1,19 @@
+from sqlalchemy import Column, Integer, String, Boolean
+from sqlalchemy.orm import relationship
+from dataflow.db import Base
+
+class DataflowZone(Base):
+    __tablename__ = "DATAFLOW_ZONE"
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    slug = Column(String, unique=True, nullable=False)
+    display_name = Column(String, nullable=False)
+    is_runtime = Column(Boolean, default=False)
+    subdomain = Column(String)
+    spark_enabled = Column(Boolean, default=False)
+    display_order = Column(Integer, default=0)
+
+    role_zone_assocs = relationship("RoleZone", back_populates="zone")
+
+    def __repr__(self):
+        return f"<DataflowZone(id={self.id}, slug='{self.slug}', display_name='{self.display_name}', display_order={self.display_order})>"

dataflow/models/environment.py

@@ -1,8 +1,8 @@
 from sqlalchemy import Column, Integer, String, Boolean, Text, ForeignKey, DateTime
 from sqlalchemy.orm import relationship
 from sqlalchemy.sql import func
-from datetime import datetime
-from dataflow.db import Base
+from datetime import datetime, timezone
+from dataflow.db import Base, Local_Base
 
 class EnvironmentAttributes(Base):
     """
@@ -61,3 +61,15 @@ class JobLogs(Base):
     status = Column(String)
     created_by = Column(String)
 
+
+class LocalEnvironment(Local_Base):
+    __tablename__ = "LOCAL_ENVIRONMENT"
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    name = Column(String, nullable=False)
+    py_version = Column(String)
+    pip_libraries = Column(Text)
+    conda_libraries = Column(Text)
+    status = Column(String, default="Created")
+    updated_at = Column(DateTime, default=datetime.now(timezone.utc), onupdate=datetime.now(timezone.utc))
+    need_refresh = Column(Boolean, default=False)

dataflow/models/pinned_projects.py

@@ -6,8 +6,8 @@ class PinnedProject(Base):
     __tablename__ = "PINNED_PROJECT"
 
     id = Column(Integer, primary_key=True)
-    user_id = Column(Integer, ForeignKey('USER.user_id'), index=True)
-    project_id = Column(Integer, ForeignKey('PROJECT_DETAIL.project_id'), index=True)
+    user_id = Column(Integer, ForeignKey('USER.user_id', ondelete="CASCADE"), index=True)
+    project_id = Column(Integer, ForeignKey('PROJECT_DETAIL.project_id', ondelete="CASCADE"), index=True)
     pinned_at = Column(DateTime, default=datetime.utcnow)
 
     __table_args__ = (

dataflow/models/project_details.py

@@ -10,7 +10,7 @@ class ProjectDetails(Base):
     git_url = Column(String)
     git_branch = Column(String, nullable=True)
     git_folder = Column(String, nullable=True)
-    type = Column(String, ForeignKey('RUNTIME_APP_TYPE.name'), nullable=False)
+    type = Column(String, ForeignKey('RUNTIME_APP_TYPE.name', ondelete="CASCADE"), nullable=False)
     slug = Column(String, nullable=False, unique=True)
     runtime = Column(String, nullable=False)
     py_env = Column(String, nullable=True)

dataflow/models/recent_projects.py

@@ -5,5 +5,5 @@ class RecentProjects(Base):
     __tablename__ = "RECENT_PROJECT"
 
     id = Column(Integer, primary_key=True, autoincrement=True)
-    project_id = Column(Integer, ForeignKey('PROJECT_DETAIL.project_id'), nullable=False)
+    project_id = Column(Integer, ForeignKey('PROJECT_DETAIL.project_id', ondelete="CASCADE"), nullable=False)
 

dataflow/models/role.py

@@ -2,6 +2,12 @@
 from sqlalchemy import Column, Integer, String, Enum
 from sqlalchemy.orm import relationship
 from dataflow.db import Base
+import enum
+
+class BaseRoleField(enum.Enum):
+    admin = "admin"
+    user = "user"
+    applicant = "applicant"
 
 class Role(Base):
     """
@@ -13,7 +19,11 @@ class Role(Base):
     id = Column(Integer, primary_key=True, index=True, autoincrement=True, nullable=False)
     name = Column(String, unique=True, nullable=False)
     description = Column(String, nullable=True)
-    base_role = Column(Enum('admin', 'user', 'applicant', name='base_role_field'), default='user', nullable=False)
+    base_role = Column(Enum(BaseRoleField), nullable=False, default=BaseRoleField.user)
 
     users = relationship("User", back_populates="role_details", cascade="all, delete-orphan")
-    role_server_assocs = relationship("RoleServer", back_populates="role")
+    role_server_assocs = relationship("RoleServer", back_populates="role")
+    role_zone_assocs = relationship("RoleZone", back_populates="role")
+
+    def __repr__(self):
+        return f"<Role(id={self.id}, name='{self.name}', base_role='{self.base_role}')>"

dataflow/models/role_zone.py

@@ -0,0 +1,17 @@
+from typing import Dict, List, Optional
+from sqlalchemy import Column, Integer, ForeignKey, UniqueConstraint, Boolean
+from sqlalchemy.orm import relationship
+from dataflow.db import Base
+
+class RoleZone(Base):
+    __tablename__ = 'ROLE_ZONE'
+
+    role_id = Column(Integer, ForeignKey('ROLE.id', ondelete="CASCADE"), primary_key=True)
+    zone_id = Column(Integer, ForeignKey('DATAFLOW_ZONE.id', ondelete="CASCADE"), primary_key=True)
+    is_default = Column(Boolean, default=False, nullable=False)
+
+    role = relationship("Role", back_populates="role_zone_assocs")
+    zone = relationship("DataflowZone", back_populates="role_zone_assocs")
+
+    def __repr__(self):
+        return f"<RoleZone(role_id={self.role_id}, zone_id={self.zone_id}, is_default={self.is_default})>"

dataflow/models/server_config.py

@@ -24,7 +24,7 @@ class CustomServerConfig(Base):
     __tablename__ = "CUSTOM_SERVER"
 
     id = Column(Integer, primary_key=True, autoincrement=True)
-    base_server_id = Column(Integer, ForeignKey(ServerConfig.id), nullable=False)
+    base_server_id = Column(Integer, ForeignKey(ServerConfig.id, ondelete="CASCADE"), nullable=False)
     display_name = Column(String, nullable=False, unique=True, index=True)
     description = Column(Text, nullable=True)
     

dataflow/models/user.py

@@ -1,5 +1,5 @@
 """models.py"""
-from sqlalchemy import Column, Integer, String, Boolean, LargeBinary, Enum, ForeignKey
+from sqlalchemy import Column, Integer, String, Boolean, LargeBinary, ForeignKey
 from sqlalchemy.orm import relationship
 from dataflow.db import Base
 
@@ -18,9 +18,10 @@ class User(Base):
     role_id = Column(Integer, ForeignKey('ROLE.id'), nullable=False)
     image = Column(LargeBinary)
     image_url = Column(String, nullable=True)
-    active = Column(Enum('N', 'Y', name='active_field'), nullable=False, server_default=str("N"))
+    active = Column(Boolean, nullable=False, default=True)
     password = Column(String, nullable=False)
-    active_env = Column(String, ForeignKey('ENVIRONMENT.short_name')) 
+    active_env = Column(String)
+    active_env_type = Column(String, nullable=True)
     current_server = Column(String)
     show_server_page = Column(Boolean, default = True)
 

dataflow/utils/get_current_user.py

@@ -26,9 +26,11 @@ def get_user_from_session(session_id: str, db: Session):
         raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="User not found")
 
     base_role = user.role_details.base_role
+    role_id = user.role_details.id
     role_name = user.role_details.name
     user.base_role = base_role
     user.role = role_name
+    user.role_id = role_id
     
     return user
 

{dataflow_core-2.1.5.dist-info → dataflow_core-2.1.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataflow-core
-Version: 2.1.5
+Version: 2.1.7
 Summary: Dataflow core package
 Author: Dataflow
 Author-email: 

{dataflow_core-2.1.5.dist-info → dataflow_core-2.1.7.dist-info}/RECORD

@@ -1,31 +1,32 @@
 authenticator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 authenticator/dataflowairflowauthenticator.py,sha256=gEdCiL2yJQ7lYvAwbrjcAkccVMfehoMJldw9eU7cc2s,2243
-authenticator/dataflowhubauthenticator.py,sha256=-wFEPEQfgCgtghC0Eo9e18B-OU1JsPKam0tbQaYCg1s,13563
+authenticator/dataflowhubauthenticator.py,sha256=Jioo4a9ufDa-jZFL8GCvKnLZkOTVIeU0E7QbWqyPmoQ,13535
 authenticator/dataflowsupersetauthenticator.py,sha256=NkAmDaIc-ui-qEolu4xz_UY7P_2g8111hwNjPvAOW1Q,2839
 dataflow/__init__.py,sha256=WTRg8HMpMWSgxYJ9ZGVldx4k07fAbta3mBmZ1hG9mWE,30
 dataflow/configuration.py,sha256=7To6XwH1eESiYp39eqPcswXWwrdBUdPF6xN6WnazOF0,663
 dataflow/database_manager.py,sha256=tJHMuOZ9Muskrh9t4uLRlTuFU0VkHAzoHlGP5DORIC4,899
 dataflow/dataflow.py,sha256=-UYZst7EO1GgaOjlAkKu-tu7RC6XsgadGeDp1MOvZiA,7439
-dataflow/db.py,sha256=5UwE4w5Vn9RqFIhr8ARlu2haZX-PtHDLRPjmn5BG2m8,1649
+dataflow/db.py,sha256=73ojGqpCTRVTlPszD73Ozhjih_BI2KTHmazqxxL6iWk,3780
 dataflow/environment.py,sha256=05F-dBRyZu2mr26vuiuJU13pfgtRpA9yW69bk83HGUw,27239
-dataflow/models/__init__.py,sha256=QMLiKj8BMhfScWMm8kgHkMjwAlFeg5Cym3_AI1NvBUA,783
+dataflow/models/__init__.py,sha256=WnlLd-0T3HYtJloDms1a58lN66WzBIbBSTReDXD6HaQ,892
 dataflow/models/app_types.py,sha256=yE_ZB13lhpK7AZ7PyBwnQlf0RlIHYs_-vdMKx7_RMlY,379
 dataflow/models/blacklist_library.py,sha256=B2oi3Z8GcR_glhLAyinFk0W8c9txXvm3uOER6dY-q7I,991
 dataflow/models/connection.py,sha256=_VJL3KuIrm8t4lJmtunIL3-AXF9Yvi5wUolzdR3tE0E,1017
-dataflow/models/environment.py,sha256=Vg-4vQe_cHIfOX-kPJWd6SIEapPJcgxoCITgdRrqT_o,2107
+dataflow/models/dataflow_zone.py,sha256=yFCvQXos5M1cU7ksbVSO338_RkT3hkdw2wr3kCJ_rec,769
+dataflow/models/environment.py,sha256=300CUwx7d7bjadrMsTch7NSRYLCuf9AK9WVzlT26CAc,2616
 dataflow/models/environment_status.py,sha256=lvPDNUsUoTW9D97B07aKqJQHRKp4LvPM28pQDMPH1ac,536
 dataflow/models/git_ssh.py,sha256=W15SDypxzGOz_aZkHEnVZ6DIMVsjAsbSIXVIEt2mPYU,694
-dataflow/models/pinned_projects.py,sha256=rkpPX_f2U9HjmrRo7_K8rnZIeXuQKGq6hYTrtLmu21c,566
-dataflow/models/project_details.py,sha256=94wTygXv9iGB0w8g_6vtkB5ZqIzpEv1W9uWwCA4hM0Y,1078
+dataflow/models/pinned_projects.py,sha256=I-XMQq7__XJJi2lyOdEvQEfhPRz8D6KHA6Cbavbf05o,606
+dataflow/models/project_details.py,sha256=iNb95L3UGjcC_Ws6FiUTDANCUFhYWpnsKraS6rmCyRU,1098
 dataflow/models/recent_project_studio.py,sha256=m12KGCsv453C1ijHjfVD8E7cJ7Og_0N8uc7_9VlfkYw,812
-dataflow/models/recent_projects.py,sha256=QqDlk3ll7tBaQl5hqvRarlB9_SUBuN44muLIuTVbPe0,301
-dataflow/models/role.py,sha256=_I5F2TFox_k2-LGgjVuO2PIW9-gpwpoX8Te7m024A8k,693
+dataflow/models/recent_projects.py,sha256=OFd5MSRXVRHs9UbvUNoJBBnh9rgsJ0lwE23wm5_Hc5w,321
+dataflow/models/role.py,sha256=0fgLjCx7aETRwCV5SW0-4PQBWzWddWO9a_ObJA14VOY,962
 dataflow/models/role_server.py,sha256=mMcfjsGX1cY8hOAOBBmrZgw8ozdfuvjKJoBlR6F0Kdc,689
-dataflow/models/runtime.py,sha256=OiuBfZTMg81U10GS00DxfhiAmHlcyQUw5LBR8RaPl7s,415
-dataflow/models/server_config.py,sha256=GTMtQfgtuvKUbxV16VhEpKGhYoNISFLRWdUPqBJmYbM,1365
+dataflow/models/role_zone.py,sha256=uH8JheWyzUWcRECRzYuzdphUAUXeU5zRVA1Bhuz3h9w,772
+dataflow/models/server_config.py,sha256=8ocKT8tPen9tedO8BLPEfkWxUEIHaqvA2L-qEhrFND0,1385
 dataflow/models/session.py,sha256=c8TI6qXsM8utzp5vSQtAOXJSbgasnyu-a0qSAvA-rWs,459
 dataflow/models/team.py,sha256=fjkqF0N4PSwSYTgHjEQl9wuC7yumd0iOb5nNFePI6q4,488
-dataflow/models/user.py,sha256=PT-zwZj7NWUubIj_7EY2EsjduMbI_42EyMclWMLESGk,1073
+dataflow/models/user.py,sha256=qFvkU6hZQzv-QWCC8nDKonnpl2Z2YEqMni5iUVQpMB0,1041
 dataflow/models/user_environment.py,sha256=yI9NutULcLiwlycuEin6ROe6o1Sjdv_sgw2MEkJFeYg,568
 dataflow/models/user_team.py,sha256=r_fmKvf6JuGgiiI9TXWjVG2QZ3WOvDrOwYWVQ3r8oWo,659
 dataflow/models/variables.py,sha256=Sinvv3zFYni5i_GrL69cVfhCh4tOOaIHiEzWYRJ-i10,1132
@@ -34,11 +35,11 @@ dataflow/scripts/create_environment.sh,sha256=3FHgNplJuEZvyTsLqlCJNX9oyfXgsfqn80
 dataflow/scripts/update_environment.sh,sha256=2dtn2xlNi6frpig-sqlGE1_IKRbbkqYOCpf_qyMKKII,992
 dataflow/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dataflow/utils/aws_secrets_manager.py,sha256=A_fNs9VNah9dDdl9NhqizJamYU7xr2v_GXlw9InEDFk,2380
-dataflow/utils/get_current_user.py,sha256=akjcUyTpmMdAZj9LFGSTs76hjBRjltNk9hLUqC_BdkA,1140
+dataflow/utils/get_current_user.py,sha256=4nSO3SPVMZhW-MsIgxR3f9ZzrFaIZIuyrM6hvfyE7PQ,1202
 dataflow/utils/json_handler.py,sha256=5_7WdypegRBDe2HSqBXyrJAdd92wsha8qRcmQvCj1TA,782
 dataflow/utils/logger.py,sha256=7BFrOq5Oiqn8P4XZbgJzMP5O07d2fpdECbbfsjrUuHw,1213
-dataflow_core-2.1.5.dist-info/METADATA,sha256=8UhovnGu3NJ6b4wK-xBK7QBkr17y3O96P_RGTJHf8XY,301
-dataflow_core-2.1.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dataflow_core-2.1.5.dist-info/entry_points.txt,sha256=ppj_EIbYrJJwCPg1kfdsZk5q1N-Ejfis1neYrnjhO8o,117
-dataflow_core-2.1.5.dist-info/top_level.txt,sha256=SZsUOpSCK9ntUy-3Tusxzf5A2e8ebwD8vouPb1dPt_8,23
-dataflow_core-2.1.5.dist-info/RECORD,,
+dataflow_core-2.1.7.dist-info/METADATA,sha256=CeAiZQe1R8cUKi06Zzv2YBoY4lcfzrRUPEUJG73gMZA,301
+dataflow_core-2.1.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dataflow_core-2.1.7.dist-info/entry_points.txt,sha256=ppj_EIbYrJJwCPg1kfdsZk5q1N-Ejfis1neYrnjhO8o,117
+dataflow_core-2.1.7.dist-info/top_level.txt,sha256=SZsUOpSCK9ntUy-3Tusxzf5A2e8ebwD8vouPb1dPt_8,23
+dataflow_core-2.1.7.dist-info/RECORD,,

dataflow/models/runtime.py

@@ -1,11 +0,0 @@
-from sqlalchemy import Column, Integer, String, Boolean
-from dataflow.db import Base
-
-class RuntimeZone(Base):
-    __tablename__ = "RUNTIME_ZONE"
-
-    id = Column(Integer, primary_key=True, autoincrement=True, unique=True)
-    name = Column(String, unique=True, nullable=False)
-    display_name = Column(String, nullable=False)
-    display_order = Column(Integer)
-    spark_enabled = Column(Boolean, default=False)