dataflow-core 2.1.5__tar.gz → 2.1.7__tar.gz

{dataflow_core-2.1.5 → dataflow_core-2.1.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataflow-core
-Version: 2.1.5
+Version: 2.1.7
 Summary: Dataflow core package
 Author: Dataflow
 Author-email: 

{dataflow_core-2.1.5 → dataflow_core-2.1.7}/authenticator/dataflowhubauthenticator.py

@@ -136,7 +136,6 @@ class DataflowBaseAuthenticator(Authenticator):
                 last_name=last_name or "",
                 email=email,
                 role_id=role_id,
-                active='Y',
                 password='user@123',
             )
             

dataflow_core-2.1.7/dataflow/db.py

@@ -0,0 +1,91 @@
+import dataflow
+from .database_manager import DatabaseManager
+from .configuration import ConfigurationManager
+from sqlalchemy.orm import declarative_base
+from sqlalchemy.exc import SQLAlchemyError
+from .utils.logger import CustomLogger
+import os
+import pwd
+import grp
+from urllib.parse import urlparse
+
+logger = CustomLogger().get_logger(__name__)
+
+dataflow_config = ConfigurationManager('/dataflow/app/config/dataflow.cfg')
+dataflow_local_config = ConfigurationManager('/dataflow/app/auth_config/dataflow_auth.cfg')
+db_url = dataflow_config.get_config_value('database', 'database_url')
+local_db_url = dataflow_local_config.get_config_value('database', 'single_user_db_url')
+
+db_manager = DatabaseManager(db_url)
+local_db_manager = None
+
+Base = declarative_base()
+Local_Base = declarative_base()
+
+def create_tables(local_db=False):
+    """
+    Create all tables in the database.
+    This is called at the start of the application.
+    """
+    try:
+        if local_db:
+            if local_db_url and local_db_url.startswith('sqlite:///'):
+                parsed_url = urlparse(local_db_url)
+                db_file_path = parsed_url.path
+                db_dir = os.path.dirname(db_file_path)
+                logger.info(f"Creating local database at: {db_file_path}")
+                
+                if db_dir and not os.path.exists(db_dir):
+                    os.makedirs(db_dir, exist_ok=True)
+                    logger.info(f"Created directory for local database: {db_dir}")
+                
+                # Set ownership for database directory and file
+                nb_user = os.environ.get("NB_USER")
+                if nb_user:
+                    try:
+                        uid = pwd.getpwnam(nb_user).pw_uid
+                        gid = grp.getgrnam(nb_user).gr_gid
+                        
+                        # Set ownership for database directory
+                        if os.path.exists(db_dir):
+                            os.chown(db_dir, uid, gid)
+                        
+                        global local_db_manager
+                        if local_db_manager is None:
+                            local_db_manager = DatabaseManager(local_db_url) 
+                        Local_Base.metadata.create_all(bind=local_db_manager.get_engine())
+                        
+                        if os.path.exists(db_file_path):
+                            os.chown(db_file_path, uid, gid)
+                        logger.info(f"Ownership set for local database files")
+                    except (KeyError, OSError) as e:
+                        logger.warning(f"Could not set ownership for database files: {e}")
+                else:
+                    logger.error("NB_USER environment variable is not set. Cannot set ownership for local database files.")
+                    raise ValueError("Can only use local database with a valid NB_USER environment variable")
+            else:
+                logger.error("Local database URL must be a valid SQLite URL starting with 'sqlite:///'")
+                raise ValueError("Local database URL must be a valid SQLite URL starting with 'sqlite:///'")
+            
+        else:
+            Base.metadata.create_all(bind=db_manager.get_engine())
+        logger.info("Database tables created successfully")
+    except SQLAlchemyError as e:
+        error_message = f"Failed to create tables: {str(e)}"
+        logger.error(error_message)
+        raise e
+    
+def get_local_db():
+    """
+    Get a local database session.
+    
+    Yields:
+        Session: Local database session
+    """
+    global local_db_manager
+    if local_db_manager is None:
+        local_db_manager = DatabaseManager(local_db_url)
+    yield from local_db_manager.get_session()
+
+def get_db():
+    yield from db_manager.get_session()

{dataflow_core-2.1.5 → dataflow_core-2.1.7}/dataflow/models/__init__.py

@@ -3,7 +3,7 @@
 from .role import Role
 from .user import User
 from .team import Team
-from .environment import Environment
+from .environment import (Environment, LocalEnvironment, ArchivedEnvironment, JobLogs)
 from .project_details import ProjectDetails
 from .recent_projects import RecentProjects
 from .pinned_projects import PinnedProject
@@ -11,8 +11,9 @@ from .app_types import AppType
 from .blacklist_library import BlacklistedLibrary
 from .environment_status import EnvironmentStatus
 from .session import Session
-from .server_config import ServerConfig
-from .runtime import RuntimeZone
+from .server_config import ServerConfig, CustomServerConfig
+from .dataflow_zone import DataflowZone
+from .role_zone import RoleZone
 from .environment_status import EnvironmentStatus
 from .user_team import UserTeam
 from .role_server import RoleServer

dataflow_core-2.1.7/dataflow/models/dataflow_zone.py

@@ -0,0 +1,19 @@
+from sqlalchemy import Column, Integer, String, Boolean
+from sqlalchemy.orm import relationship
+from dataflow.db import Base
+
+class DataflowZone(Base):
+    __tablename__ = "DATAFLOW_ZONE"
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    slug = Column(String, unique=True, nullable=False)
+    display_name = Column(String, nullable=False)
+    is_runtime = Column(Boolean, default=False)
+    subdomain = Column(String)
+    spark_enabled = Column(Boolean, default=False)
+    display_order = Column(Integer, default=0)
+
+    role_zone_assocs = relationship("RoleZone", back_populates="zone")
+
+    def __repr__(self):
+        return f"<DataflowZone(id={self.id}, slug='{self.slug}', display_name='{self.display_name}', display_order={self.display_order})>"

{dataflow_core-2.1.5 → dataflow_core-2.1.7}/dataflow/models/environment.py

@@ -1,8 +1,8 @@
 from sqlalchemy import Column, Integer, String, Boolean, Text, ForeignKey, DateTime
 from sqlalchemy.orm import relationship
 from sqlalchemy.sql import func
-from datetime import datetime
-from dataflow.db import Base
+from datetime import datetime, timezone
+from dataflow.db import Base, Local_Base
 
 class EnvironmentAttributes(Base):
     """
@@ -61,3 +61,15 @@ class JobLogs(Base):
     status = Column(String)
     created_by = Column(String)
 
+
+class LocalEnvironment(Local_Base):
+    __tablename__ = "LOCAL_ENVIRONMENT"
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    name = Column(String, nullable=False)
+    py_version = Column(String)
+    pip_libraries = Column(Text)
+    conda_libraries = Column(Text)
+    status = Column(String, default="Created")
+    updated_at = Column(DateTime, default=datetime.now(timezone.utc), onupdate=datetime.now(timezone.utc))
+    need_refresh = Column(Boolean, default=False)

{dataflow_core-2.1.5 → dataflow_core-2.1.7}/dataflow/models/pinned_projects.py

@@ -6,8 +6,8 @@ class PinnedProject(Base):
     __tablename__ = "PINNED_PROJECT"
 
     id = Column(Integer, primary_key=True)
-    user_id = Column(Integer, ForeignKey('USER.user_id'), index=True)
-    project_id = Column(Integer, ForeignKey('PROJECT_DETAIL.project_id'), index=True)
+    user_id = Column(Integer, ForeignKey('USER.user_id', ondelete="CASCADE"), index=True)
+    project_id = Column(Integer, ForeignKey('PROJECT_DETAIL.project_id', ondelete="CASCADE"), index=True)
     pinned_at = Column(DateTime, default=datetime.utcnow)
 
     __table_args__ = (

{dataflow_core-2.1.5 → dataflow_core-2.1.7}/dataflow/models/project_details.py

@@ -10,7 +10,7 @@ class ProjectDetails(Base):
     git_url = Column(String)
     git_branch = Column(String, nullable=True)
     git_folder = Column(String, nullable=True)
-    type = Column(String, ForeignKey('RUNTIME_APP_TYPE.name'), nullable=False)
+    type = Column(String, ForeignKey('RUNTIME_APP_TYPE.name', ondelete="CASCADE"), nullable=False)
     slug = Column(String, nullable=False, unique=True)
     runtime = Column(String, nullable=False)
     py_env = Column(String, nullable=True)

{dataflow_core-2.1.5 → dataflow_core-2.1.7}/dataflow/models/recent_projects.py

@@ -5,5 +5,5 @@ class RecentProjects(Base):
     __tablename__ = "RECENT_PROJECT"
 
     id = Column(Integer, primary_key=True, autoincrement=True)
-    project_id = Column(Integer, ForeignKey('PROJECT_DETAIL.project_id'), nullable=False)
+    project_id = Column(Integer, ForeignKey('PROJECT_DETAIL.project_id', ondelete="CASCADE"), nullable=False)
 

{dataflow_core-2.1.5 → dataflow_core-2.1.7}/dataflow/models/role.py

@@ -2,6 +2,12 @@
 from sqlalchemy import Column, Integer, String, Enum
 from sqlalchemy.orm import relationship
 from dataflow.db import Base
+import enum
+
+class BaseRoleField(enum.Enum):
+    admin = "admin"
+    user = "user"
+    applicant = "applicant"
 
 class Role(Base):
     """
@@ -13,7 +19,11 @@ class Role(Base):
     id = Column(Integer, primary_key=True, index=True, autoincrement=True, nullable=False)
     name = Column(String, unique=True, nullable=False)
     description = Column(String, nullable=True)
-    base_role = Column(Enum('admin', 'user', 'applicant', name='base_role_field'), default='user', nullable=False)
+    base_role = Column(Enum(BaseRoleField), nullable=False, default=BaseRoleField.user)
 
     users = relationship("User", back_populates="role_details", cascade="all, delete-orphan")
-    role_server_assocs = relationship("RoleServer", back_populates="role")
+    role_server_assocs = relationship("RoleServer", back_populates="role")
+    role_zone_assocs = relationship("RoleZone", back_populates="role")
+
+    def __repr__(self):
+        return f"<Role(id={self.id}, name='{self.name}', base_role='{self.base_role}')>"

dataflow_core-2.1.7/dataflow/models/role_zone.py

@@ -0,0 +1,17 @@
+from typing import Dict, List, Optional
+from sqlalchemy import Column, Integer, ForeignKey, UniqueConstraint, Boolean
+from sqlalchemy.orm import relationship
+from dataflow.db import Base
+
+class RoleZone(Base):
+    __tablename__ = 'ROLE_ZONE'
+
+    role_id = Column(Integer, ForeignKey('ROLE.id', ondelete="CASCADE"), primary_key=True)
+    zone_id = Column(Integer, ForeignKey('DATAFLOW_ZONE.id', ondelete="CASCADE"), primary_key=True)
+    is_default = Column(Boolean, default=False, nullable=False)
+
+    role = relationship("Role", back_populates="role_zone_assocs")
+    zone = relationship("DataflowZone", back_populates="role_zone_assocs")
+
+    def __repr__(self):
+        return f"<RoleZone(role_id={self.role_id}, zone_id={self.zone_id}, is_default={self.is_default})>"

{dataflow_core-2.1.5 → dataflow_core-2.1.7}/dataflow/models/server_config.py

@@ -24,7 +24,7 @@ class CustomServerConfig(Base):
     __tablename__ = "CUSTOM_SERVER"
 
     id = Column(Integer, primary_key=True, autoincrement=True)
-    base_server_id = Column(Integer, ForeignKey(ServerConfig.id), nullable=False)
+    base_server_id = Column(Integer, ForeignKey(ServerConfig.id, ondelete="CASCADE"), nullable=False)
     display_name = Column(String, nullable=False, unique=True, index=True)
     description = Column(Text, nullable=True)
     

{dataflow_core-2.1.5 → dataflow_core-2.1.7}/dataflow/models/user.py

@@ -1,5 +1,5 @@
 """models.py"""
-from sqlalchemy import Column, Integer, String, Boolean, LargeBinary, Enum, ForeignKey
+from sqlalchemy import Column, Integer, String, Boolean, LargeBinary, ForeignKey
 from sqlalchemy.orm import relationship
 from dataflow.db import Base
 
@@ -18,9 +18,10 @@ class User(Base):
     role_id = Column(Integer, ForeignKey('ROLE.id'), nullable=False)
     image = Column(LargeBinary)
     image_url = Column(String, nullable=True)
-    active = Column(Enum('N', 'Y', name='active_field'), nullable=False, server_default=str("N"))
+    active = Column(Boolean, nullable=False, default=True)
     password = Column(String, nullable=False)
-    active_env = Column(String, ForeignKey('ENVIRONMENT.short_name')) 
+    active_env = Column(String)
+    active_env_type = Column(String, nullable=True)
     current_server = Column(String)
     show_server_page = Column(Boolean, default = True)
 

{dataflow_core-2.1.5 → dataflow_core-2.1.7}/dataflow/utils/get_current_user.py

@@ -26,9 +26,11 @@ def get_user_from_session(session_id: str, db: Session):
         raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="User not found")
 
     base_role = user.role_details.base_role
+    role_id = user.role_details.id
     role_name = user.role_details.name
     user.base_role = base_role
     user.role = role_name
+    user.role_id = role_id
     
     return user
 

{dataflow_core-2.1.5 → dataflow_core-2.1.7}/dataflow_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataflow-core
-Version: 2.1.5
+Version: 2.1.7
 Summary: Dataflow core package
 Author: Dataflow
 Author-email: 

{dataflow_core-2.1.5 → dataflow_core-2.1.7}/dataflow_core.egg-info/SOURCES.txt

@@ -14,6 +14,7 @@ dataflow/models/__init__.py
 dataflow/models/app_types.py
 dataflow/models/blacklist_library.py
 dataflow/models/connection.py
+dataflow/models/dataflow_zone.py
 dataflow/models/environment.py
 dataflow/models/environment_status.py
 dataflow/models/git_ssh.py
@@ -23,7 +24,7 @@ dataflow/models/recent_project_studio.py
 dataflow/models/recent_projects.py
 dataflow/models/role.py
 dataflow/models/role_server.py
-dataflow/models/runtime.py
+dataflow/models/role_zone.py
 dataflow/models/server_config.py
 dataflow/models/session.py
 dataflow/models/team.py

{dataflow_core-2.1.5 → dataflow_core-2.1.7}/setup.py

@@ -14,7 +14,7 @@ class PostInstall(install):
 
 setup(
     name="dataflow-core",
-    version="2.1.5",
+    version="2.1.7",
     packages=find_packages(include=["dataflow", "dataflow.*", "authenticator", "authenticator.*"]),
     include_package_data=True,
     package_data={

dataflow_core-2.1.5/dataflow/db.py

@@ -1,51 +0,0 @@
-from .database_manager import DatabaseManager
-from .configuration import ConfigurationManager
-from sqlalchemy.orm import declarative_base
-from sqlalchemy.exc import SQLAlchemyError
-from .utils.logger import CustomLogger
-
-logger = CustomLogger().get_logger(__name__)
-
-dataflow_config = ConfigurationManager('/dataflow/app/config/dataflow.cfg')
-db_url = dataflow_config.get_config_value('database', 'database_url')
-local_db_url = dataflow_config.get_config_value('database', 'single_user_db_url')
-
-db_manager = DatabaseManager(db_url)
-local_db_manager = None
-
-Base = declarative_base()
-Local_Base = declarative_base()
-
-def create_tables(local_db=False):
-    """
-    Create all tables in the database.
-    This is called at the start of the application.
-    """
-    try:
-        if local_db:
-            global local_db_manager
-            if local_db_manager is None:
-                local_db_manager = DatabaseManager(local_db_url) 
-            Local_Base.metadata.create_all(bind=local_db_manager.get_engine())
-        else:
-            Base.metadata.create_all(bind=db_manager.get_engine())
-        logger.info("Database tables created successfully")
-    except SQLAlchemyError as e:
-        error_message = f"Failed to create tables: {str(e)}"
-        logger.error(error_message)
-        raise e
-    
-def get_local_db():
-    """
-    Get a local database session.
-    
-    Yields:
-        Session: Local database session
-    """
-    global local_db_manager
-    if local_db_manager is None:
-        local_db_manager = DatabaseManager(local_db_url)
-    yield from local_db_manager.get_session()
-
-def get_db():
-    yield from db_manager.get_session()

dataflow_core-2.1.5/dataflow/models/runtime.py

@@ -1,11 +0,0 @@
-from sqlalchemy import Column, Integer, String, Boolean
-from dataflow.db import Base
-
-class RuntimeZone(Base):
-    __tablename__ = "RUNTIME_ZONE"
-
-    id = Column(Integer, primary_key=True, autoincrement=True, unique=True)
-    name = Column(String, unique=True, nullable=False)
-    display_name = Column(String, nullable=False)
-    display_order = Column(Integer)
-    spark_enabled = Column(Boolean, default=False)