dataflow-core 2.1.14rc1__py3-none-any.whl → 2.1.15rc2__py3-none-any.whl

dataflow/models/organization.py

@@ -0,0 +1,79 @@
+from sqlalchemy import (
+    Column, Integer, String, Enum, DateTime, ForeignKey, Index
+)
+import uuid, enum
+from sqlalchemy.dialects.postgresql import JSONB, UUID
+from sqlalchemy.sql import func
+from sqlalchemy.orm import relationship
+from datetime import datetime
+from dataflow.db import Base
+
+
+class Organization(Base):
+    """
+    Organization model for the database.
+    """
+    __tablename__ = "ORGANIZATION"
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    uid = Column(UUID(as_uuid=True), default=uuid.uuid4, nullable=False, unique=True) 
+    name = Column(String(255), nullable=False, unique=True)
+    invite_code = Column(String(64), nullable=False, unique=True)
+    email_domain = Column(String(255), nullable=False, unique=True)
+    spark_enabled_zones = Column(JSONB, default=func.json([]))  # List of zone IDs where Spark is enabled
+    created_at = Column(DateTime, default=datetime.utcnow)
+
+    # Association object link
+    org_user_assocs = relationship("OrganizationUser", back_populates="organization", cascade="all, delete-orphan")
+    custom_servers = relationship("CustomServerConfig")
+    onboarding_requests = relationship("UserOnboarding", back_populates="organization", cascade="all, delete-orphan")
+    servers = relationship("ServerConfig", secondary="ORGANIZATION_SERVER", back_populates="organizations")
+    apps = relationship("AppType", secondary="ORGANIZATION_APP_TYPE", back_populates="organizations")
+    roles = relationship("Role", cascade="all, delete-orphan")
+    environments = relationship("Environment", back_populates="organization")
+
+class OnboardingStatus(enum.Enum):
+    pending = 'pending'
+    rejected = 'rejected'
+    accepted = 'accepted'
+
+class OrganizationOnboarding(Base):
+    __tablename__ = 'ORGANIZATION_ONBOARDING'
+    # This prevents an org from having more than one active ('pending' or 'accepted') application
+    # while allowing multiple 'rejected' entries.
+    __table_args__ = (
+        Index(
+            'idx_pending_org_application',
+            'name',
+            unique=True,
+            postgresql_where=Column('status').in_([
+                OnboardingStatus.pending.value,
+                OnboardingStatus.accepted.value
+            ])
+        ),
+    )
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+
+    name = Column(String(255), nullable=False)
+    age = Column(Integer, nullable=True)
+    domain = Column(String(255), nullable=True)
+    no_of_employees = Column(String(50), nullable=True)
+    address = Column(String(500), nullable=True)
+
+    admin_first_name = Column(String(100), nullable=False)
+    admin_last_name = Column(String(100), nullable=True)
+    admin_designation = Column(String(100), nullable=False)
+    admin_email = Column(String(255), nullable=False, unique=True)
+    admin_username = Column(String(100), nullable=False, unique=True)
+    admin_password = Column(String(255), nullable=False)
+
+    # New fields
+    discovery_source = Column(String(255), nullable=True)
+    additional_info = Column(String(1000), nullable=True)
+    size_of_data = Column(String(100), nullable=True)
+
+    user_id = Column(Integer, ForeignKey('USER.user_id'), nullable=False)
+    status = Column(Enum(OnboardingStatus), default=OnboardingStatus.pending, nullable=False)
+
+    user = relationship("User", back_populates="organization_onboarding")

dataflow/models/project_details.py

@@ -1,23 +1,25 @@
-from sqlalchemy import Column, String, Enum, DateTime, Integer, func, ForeignKey
+from sqlalchemy import Column, String, Enum, DateTime, Integer, func, ForeignKey, UniqueConstraint
 from sqlalchemy.orm import relationship
 from dataflow.db import Base
 
 class ProjectDetails(Base):
     __tablename__ = "PROJECT_DETAIL"
+    __table_args__ = (UniqueConstraint('org_id', 'slug', name='uq_project_org_slug'),)
     
     project_id = Column(Integer, primary_key=True, autoincrement=True)
     project_name = Column(String, nullable=False)
     git_url = Column(String)
     git_branch = Column(String, nullable=True)
     git_folder = Column(String, nullable=True)
-    type = Column(String, ForeignKey('RUNTIME_APP_TYPE.name', ondelete="CASCADE"), nullable=False)
-    slug = Column(String, nullable=False, unique=True)
+    type = Column(String, ForeignKey('APP_TYPE.name'), nullable=False)
+    slug = Column(String, nullable=False)
     runtime = Column(String, nullable=False)
-    py_env = Column(String, nullable=True)
+    py_env = Column(Integer, nullable=True)
     launch_url = Column(String, nullable=True)  
     status = Column(Enum("pending", "created" ,"deployed", "stopped", "failed", name="deployment_status"), default="created")
     last_deployed = Column(DateTime, nullable=True)
     created_at = Column(DateTime, nullable=False, server_default=func.now())
     created_by = Column(String, nullable=False)
+    org_id = Column(Integer, ForeignKey('ORGANIZATION.id', ondelete='CASCADE'), nullable=False)
 
-    app_type = relationship("AppType")
+    app_type = relationship("AppType")

dataflow/models/role.py

@@ -1,5 +1,5 @@
 """models.py"""
-from sqlalchemy import Column, Integer, String, Enum
+from sqlalchemy import Column, Integer, String, Enum, ForeignKey, UniqueConstraint
 from sqlalchemy.orm import relationship
 from dataflow.db import Base
 import enum
@@ -11,19 +11,25 @@ class BaseRoleField(enum.Enum):
 
 class Role(Base):
     """
-    Table Role
+    Table ROLE
     """
 
     __tablename__='ROLE'
+    __table_args__ = (
+        UniqueConstraint('name', 'org_id', name='uq_role_name_org'),
+    )
 
     id = Column(Integer, primary_key=True, index=True, autoincrement=True, nullable=False)
-    name = Column(String, unique=True, nullable=False)
+    name = Column(String, nullable=False)
+    org_id = Column(Integer, ForeignKey('ORGANIZATION.id'))
     description = Column(String, nullable=True)
     base_role = Column(Enum(BaseRoleField), nullable=False, default=BaseRoleField.user)
 
-    users = relationship("User", back_populates="role_details", cascade="all, delete-orphan")
-    role_server_assocs = relationship("RoleServer", back_populates="role")
+    # Relationships
     role_zone_assocs = relationship("RoleZone", back_populates="role")
+    org_user_assocs = relationship("OrganizationUser", back_populates="role", cascade="all, delete-orphan")
+    organization = relationship("Organization", back_populates="roles")
+    servers = relationship("CustomServerConfig", secondary="ROLE_SERVER", back_populates="roles")
 
     def __repr__(self):
         return f"<Role(id={self.id}, name='{self.name}', base_role='{self.base_role}')>"

dataflow/models/role_server.py

@@ -5,10 +5,7 @@ from dataflow.db import Base
 
 class RoleServer(Base):
     __tablename__ = 'ROLE_SERVER'
-    __table_args__ = (UniqueConstraint('role_id', 'server_id', name='_role_server_uc'),)
+    __table_args__ = (UniqueConstraint('role_id', 'custom_server_id', name='_role_server_uc'),)
 
     role_id = Column(Integer, ForeignKey('ROLE.id', ondelete="CASCADE"), nullable=False, primary_key=True)
-    server_id = Column(Integer, ForeignKey('CUSTOM_SERVER.id', ondelete="CASCADE"), nullable=False, primary_key=True)
-
-    role = relationship("Role", back_populates="role_server_assocs")
-    server = relationship("CustomServerConfig", back_populates="role_server_assocs")
+    custom_server_id = Column(Integer, ForeignKey('CUSTOM_SERVER.id', ondelete="CASCADE"), nullable=False, primary_key=True)

dataflow/models/server_config.py

@@ -19,15 +19,19 @@ class ServerConfig(Base):
     description = Column(Text, nullable=True)
     kubespawner_override = Column(JSONB, default=func.json({}))
 
+    # Relationships
+    organizations = relationship("Organization", secondary="ORGANIZATION_SERVER", back_populates="servers")
 
 class CustomServerConfig(Base):
     __tablename__ = "CUSTOM_SERVER"
 
     id = Column(Integer, primary_key=True, autoincrement=True)
     base_server_id = Column(Integer, ForeignKey(ServerConfig.id, ondelete="CASCADE"), nullable=False)
+    org_id = Column(Integer, ForeignKey('ORGANIZATION.id'), nullable=False)
     display_name = Column(String, nullable=False, unique=True, index=True)
     description = Column(Text, nullable=True)
     
     # Relationship to the server_config table
     server_config = relationship(ServerConfig)
-    role_server_assocs = relationship("RoleServer", back_populates="server", cascade="all, delete-orphan")
+    organization = relationship("Organization", back_populates="custom_servers")
+    roles = relationship("Role", secondary="ROLE_SERVER", back_populates="servers")

dataflow/models/team.py

@@ -1,5 +1,5 @@
 """models.py"""
-from sqlalchemy import Column, Integer, String
+from sqlalchemy import Column, Integer, String, ForeignKey, UniqueConstraint
 from sqlalchemy.orm import relationship
 from dataflow.db import Base
 
@@ -9,9 +9,15 @@ class Team(Base):
     """
 
     __tablename__='TEAM'
+    __table_args__ = (
+        UniqueConstraint('team_name', 'org_id', name='uc_team_name_org_id'),
+    )
 
     team_id = Column(Integer, primary_key=True, index=True, autoincrement=True, nullable=False)
-    team_name = Column(String, unique=True, nullable=False)
+    team_name = Column(String, nullable=False)
+    org_id = Column(Integer, ForeignKey('ORGANIZATION.id', ondelete="CASCADE"), nullable=False)
     description = Column(String, nullable=True)
-    
-    user_team_assocs = relationship("UserTeam", back_populates="team")
+
+    # relationships
+    users = relationship("User", secondary="USER_TEAM", back_populates="teams")
+    organization = relationship("Organization")

dataflow/models/user.py

@@ -1,31 +1,68 @@
 """models.py"""
-from sqlalchemy import Column, Integer, String, Boolean, LargeBinary, ForeignKey
+from sqlalchemy import Column, Integer, String, Boolean, LargeBinary, Enum, ForeignKey, DateTime, func
+from sqlalchemy.dialects.postgresql import ENUM
+from sqlalchemy import Index
 from sqlalchemy.orm import relationship
 from dataflow.db import Base
+import enum
 
 class User(Base):
     """
     Table USER
     """
-
-    __tablename__='USER'
+    __tablename__ = 'USER'
 
     user_id = Column(Integer, primary_key=True, index=True, autoincrement=True, nullable=False)
     user_name = Column(String, unique=True, nullable=False)
     first_name = Column(String)
     last_name = Column(String)
     email = Column(String, unique=True)
-    role_id = Column(Integer, ForeignKey('ROLE.id'), nullable=False)
     image = Column(LargeBinary)
     image_url = Column(String, nullable=True)
     active = Column(Boolean, nullable=False, default=True)
     password = Column(String, nullable=False)
-    active_env = Column(String)
-    active_env_type = Column(String, nullable=True)
-    current_server = Column(String)
-    show_server_page = Column(Boolean, default = True)
-    monthly_allocation = Column(Integer, nullable=True, default=0)
+    active_org_id = Column(Integer, ForeignKey('ORGANIZATION.id'))
+
+    # Relationships
+    org_user_assocs = relationship("OrganizationUser", back_populates="user", cascade="all, delete-orphan")
+    teams = relationship("Team", secondary="USER_TEAM", back_populates="users")
+    onboarding_requests = relationship("UserOnboarding", back_populates="user", cascade="all, delete-orphan")
+    organization_onboarding = relationship("OrganizationOnboarding", back_populates="user", cascade="all, delete-orphan")
+
+
+class OnboardingStatus(enum.Enum):
+    pending = 'pending'
+    rejected = 'rejected'
+    accepted = 'accepted'
+
+class UserOnboarding(Base):
+    """
+    SQLAlchemy model for the "USER_ONBOARDING" table.
+    This table stores user applications to organizations.
+    """
+    __tablename__ = "USER_ONBOARDING"
+    # This prevents a user from having more than one active ('pending' or 'accepted') application
+    # for a given organization, while allowing multiple 'rejected' entries.
+    __table_args__ = (
+        Index(
+            'idx_pending_user_org_application',
+            'user_id',
+            'org_id',
+            unique=True,
+            postgresql_where=Column('status').in_([
+                OnboardingStatus.pending.value,
+                OnboardingStatus.accepted.value
+            ])
+        ),
+    )
 
-    role_details = relationship("Role")
+    id = Column(Integer, primary_key=True, autoincrement=True, nullable=False)
+    user_id = Column(Integer, ForeignKey("USER.user_id", ondelete="CASCADE"), nullable=False)
+    org_id = Column(Integer, ForeignKey("ORGANIZATION.id", ondelete="CASCADE"), nullable=False)
+    status = Column(Enum(OnboardingStatus, name='onboarding_status'), nullable=False, default=OnboardingStatus.pending.value)
+    created_at = Column(DateTime, default=func.now(), nullable=False)
+    updated_at = Column(DateTime, default=func.now(), onupdate=func.now(), nullable=False)
 
-    user_team_assocs = relationship("UserTeam", back_populates="user")
+    # Relationships
+    user = relationship("User", back_populates="onboarding_requests")
+    organization = relationship("Organization", back_populates="onboarding_requests")

dataflow/models/user_team.py

@@ -8,7 +8,4 @@ class UserTeam(Base):
     __table_args__ = (UniqueConstraint('user_id', 'team_id', name='_user_team_uc'),)
 
     user_id = Column(Integer, ForeignKey('USER.user_id', ondelete="CASCADE"), nullable=False, primary_key=True)
-    team_id = Column(Integer, ForeignKey('TEAM.team_id', ondelete="CASCADE"), nullable=False, primary_key=True)
-
-    user = relationship("User", back_populates="user_team_assocs")
-    team = relationship("Team", back_populates="user_team_assocs")
+    team_id = Column(Integer, ForeignKey('TEAM.team_id', ondelete="CASCADE"), nullable=False, primary_key=True)

dataflow/models/variables.py

@@ -14,7 +14,8 @@ class Variable(Base):
     __tablename__ = 'VARIABLE'
 
     id = Column(Integer, primary_key=True, index=True, autoincrement=True, nullable=False)
-    key = Column(String, nullable=False)
+    key = Column(String, index=True, nullable=False)
+    org_id = Column(Integer, ForeignKey("ORGANIZATION.id"), index=True, nullable=False)
     value = Column(Text, nullable=False)
     type = Column(String, nullable=False)
     description = Column(Text, nullable=True)
@@ -30,5 +31,5 @@ class Variable(Base):
 
     __table_args__ = (
         CheckConstraint(type.in_(['variable', 'secret']), name='check_variable_type'),
-        UniqueConstraint('key', 'runtime', 'slug', 'created_by', name='unique_key'),
+        UniqueConstraint('key', 'org_id', 'runtime', 'slug', 'created_by', name='unique_key'),
     )

dataflow/secrets_manager/factory.py

@@ -3,6 +3,7 @@ import os
 from .interface import SecretManager
 from .providers.aws_manager import AWSSecretsManager
 from .providers.azure_manager import AzureKeyVault
+from .providers.gcp_manager import GCPSecretsManager
 from ..configuration import ConfigurationManager
 
 # A custom exception for clear error messages
@@ -17,10 +18,6 @@ def get_secret_manager() -> SecretManager:
     to determine which cloud provider's secret manager to instantiate.
     """
     try:
-        # dataflow_config = None
-        # if os.getenv('HOSTNAME'):
-        #     dataflow_config = ConfigurationManager('/dataflow/app/auth_config/dataflow_auth.cfg')
-        # else:
         dataflow_config = ConfigurationManager('/dataflow/app/config/dataflow.cfg')
     except Exception as e:
         raise SecretProviderError(
@@ -49,11 +46,20 @@ def get_secret_manager() -> SecretManager:
             )
         return AzureKeyVault(vault_url=vault_url)
 
-    # You can easily add more providers here in the future
-    # elif provider == "gcp":
-    #     return GCPSecretManager()
+    elif provider == "gcp":
+        project_id = dataflow_config.get_config_value('cloudProvider', 'gcp_project_id')
+        region = dataflow_config.get_config_value('cloudProvider', 'gcp_region')
+        if not project_id:
+            raise SecretProviderError(
+                "GCP_PROJECT_ID must be set when using the GCP provider."
+            )
+        if not region:
+            raise SecretProviderError(
+                "GCP_REGION must be set when using the GCP provider."
+            )
+        return GCPSecretsManager(project_id=project_id, region=region)
 
     else:
         raise SecretProviderError(
-            f"Unsupported secret provider: '{provider}'. Supported providers are: aws, azure."
+            f"Unsupported secret provider: '{provider}'. Supported providers are: aws, azure and gcp"
         )