dataflow-core 2.1.15rc2__tar.gz → 2.1.15rc3__tar.gz

{dataflow_core-2.1.15rc2 → dataflow_core-2.1.15rc3}/PKG-INFO

@@ -1,10 +1,11 @@
 Metadata-Version: 2.4
 Name: dataflow-core
-Version: 2.1.15rc2
+Version: 2.1.15rc3
 Summary: Dataflow core package
 Author: Dataflow
 Author-email: 
 Requires-Dist: sqlalchemy
+Requires-Dist: alembic
 Requires-Dist: boto3
 Requires-Dist: psycopg2-binary
 Requires-Dist: pymysql

{dataflow_core-2.1.15rc2 → dataflow_core-2.1.15rc3}/dataflow/models/__init__.py

@@ -3,7 +3,7 @@
 from .role import Role
 from .user import User, UserOnboarding, OnboardingStatus
 from .team import Team
-from .environment import (Environment, LocalEnvironment, ArchivedEnvironment, JobLogs)
+from .environment import (Environment, LocalEnvironment, ArchivedEnvironment, JobLogs, PipSource)
 from .project_details import ProjectDetails
 from .recent_projects import RecentProjects
 from .pinned_projects import PinnedProject

{dataflow_core-2.1.15rc2 → dataflow_core-2.1.15rc3}/dataflow/models/app_types.py

@@ -6,10 +6,10 @@ class AppType(Base):
     __tablename__ = "APP_TYPE"
     
     id = Column(Integer, primary_key=True, autoincrement=True, unique=True)
-    name = Column(String, unique=True, nullable=True)
+    name = Column(String, unique=True, nullable=False)
     display_name = Column(String, nullable=False)
     code_based = Column(Boolean, nullable=False)
-    studio = Column(Boolean, nullable=False, default=False)
-    runtime = Column(Boolean, nullable=False, default=False)
+    studio = Column(Boolean, nullable=False, default=False, server_default='false')
+    runtime = Column(Boolean, nullable=False, default=False, server_default='false')
 
     organizations = relationship("Organization", secondary="ORGANIZATION_APP_TYPE", back_populates="apps")

{dataflow_core-2.1.15rc2 → dataflow_core-2.1.15rc3}/dataflow/models/connection.py

@@ -15,11 +15,11 @@ class Connection(Base):
     conn_type = Column(String, nullable=False)
     runtime = Column(String, nullable=True)
     slug = Column(String, nullable=True)
-    status = Column(Boolean, default=False)
+    status = Column(Boolean, default=False, server_default='false')
     created_by = Column(String, nullable=True)
     created_at = Column(DateTime(timezone=True), server_default=func.now())
     updated_at = Column(DateTime(timezone=True), onupdate=func.now())
-    is_active = Column(Boolean, default=True)
+    is_active = Column(Boolean, default=True, server_default='true')
     
     __table_args__ = (
         UniqueConstraint('conn_id', 'org_id', 'runtime', 'slug', 'is_active', 'created_by', name='uq_active_conn_with_runtime_slug'),

{dataflow_core-2.1.15rc2 → dataflow_core-2.1.15rc3}/dataflow/models/dataflow_zone.py

@@ -8,9 +8,9 @@ class DataflowZone(Base):
     id = Column(Integer, primary_key=True, autoincrement=True)
     slug = Column(String, unique=True, nullable=False)
     display_name = Column(String, nullable=False)
-    is_runtime = Column(Boolean, default=False) 
+    is_runtime = Column(Boolean, default=False, server_default='false')
     subdomain = Column(String)
-    display_order = Column(Integer, default=0)
+    display_order = Column(Integer, default=0, server_default='0')
 
     role_zone_assocs = relationship("RoleZone", back_populates="zone")
 

{dataflow_core-2.1.15rc2 → dataflow_core-2.1.15rc3}/dataflow/models/environment.py

@@ -1,5 +1,8 @@
-from sqlalchemy import Column, Integer, String, Boolean, Text, ForeignKey, DateTime, UniqueConstraint
-from sqlalchemy.orm import relationship
+from sqlalchemy import (
+    Column, Integer, String, Boolean, Text, 
+    ForeignKey, DateTime, UniqueConstraint, CheckConstraint
+)
+from sqlalchemy.orm import relationship, Session
 from sqlalchemy.sql import func
 from datetime import datetime, timezone
 from dataflow.db import Base
@@ -11,14 +14,14 @@ class EnvironmentAttributes(Base):
     """
     __abstract__ = True 
 
-    name = Column(String)
+    name = Column(String, nullable=False)
     url = Column(String)
-    enabled = Column(Boolean, default=True)
-    version = Column(String, default=0)
-    is_latest = Column(Boolean, default=True)
-    base_env_id = Column(Integer, nullable=True)
+    enabled = Column(Boolean, default=True, server_default='true')
+    version = Column(String, default=0, server_default='0')
+    is_latest = Column(Boolean, default=True, server_default='true')
+    base_env_id = Column(Integer, default=None)
     short_name = Column(String(5))
-    status = Column(String, default="Saved")
+    status = Column(String, default="Saved", server_default="Saved")
     icon = Column(String)
     py_version = Column(String)
     r_version = Column(String)
@@ -52,7 +55,7 @@ class JobLogs(Base):
     __table_args__ = (UniqueConstraint('log_file_name', 'org_id', name='_job_log_file_org_uc'),)
 
     id = Column(Integer, primary_key=True, index=True)
-    created_at = Column(DateTime, default=datetime.now)
+    created_at = Column(DateTime, default=datetime.now, server_default=func.now())
     completed_at = Column(DateTime, nullable=True)
     log_file_name = Column(String, nullable=False)
     log_file_location = Column(String, nullable=False)
@@ -71,11 +74,52 @@ class LocalEnvironment(Base):
     py_version = Column(String)
     pip_libraries = Column(Text)
     conda_libraries = Column(Text)
-    status = Column(String, default="Created")
+    status = Column(String, default="Created", server_default="Created")
     cloned_from = Column(String, nullable=True)
     updated_at = Column(DateTime, default=datetime.now(timezone.utc), onupdate=datetime.now(timezone.utc))
-    need_refresh = Column(Boolean, default=False)
+    need_refresh = Column(Boolean, default=False, server_default='false')
 
 class EnvType(str, Enum):
     dataflow = "dataflow"
     local = "local"
+
+class PipSource(Base):
+    __tablename__ = "PIP_SOURCE"
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+
+    org_id = Column(Integer, ForeignKey("ORGANIZATION.id", ondelete="CASCADE"), nullable=False, index=True)
+    user_name = Column(String, ForeignKey("USER.user_name", ondelete="CASCADE"), nullable=True, index=True)
+
+    name = Column(String, nullable=False)
+    url = Column(String, nullable=False)
+    is_index = Column(Boolean, default=False, nullable=False, server_default='false')
+
+    created_at = Column(DateTime, default=datetime.now(timezone.utc), nullable=False)
+    updated_at = Column(DateTime, default=datetime.now(timezone.utc), onupdate=datetime.now(timezone.utc), nullable=False)
+
+    __table_args__ = (
+        UniqueConstraint("org_id", "name", "user_name", name="uq_pip_source_per_user_org"),
+        CheckConstraint("NOT (is_index = TRUE AND user_name IS NOT NULL)", name="check_no_user_index_url"),
+    )
+
+    @classmethod
+    def get_org_sources(cls, session: Session, org_id: int):
+        """
+        Returns all sources for the given org (org-level).
+        """
+        return session.query(cls).filter(
+            cls.org_id == org_id,
+            cls.user_name == None
+        ).all()
+
+    @classmethod
+    def get_user_sources(cls, session: Session, org_id: int, user_name: str):
+        """
+        Returns merged sources for a user in an org (org-level + user-level personal sources).
+        """
+        return session.query(cls).filter(
+            cls.org_id == org_id,
+            ((cls.user_name == None) | (cls.user_name == user_name))
+        ).all()
+

{dataflow_core-2.1.15rc2 → dataflow_core-2.1.15rc3}/dataflow/models/org_associations.py

@@ -13,12 +13,12 @@ class OrganizationUser(Base):
 
     org_id = Column(Integer, ForeignKey('ORGANIZATION.id', ondelete="CASCADE"), primary_key=True, nullable=False)
     user_id = Column(Integer, ForeignKey('USER.user_id', ondelete="CASCADE"), primary_key=True, nullable=False)
-    role_id = Column(Integer, ForeignKey('ROLE.id', ondelete="SET NULL"))
+    role_id = Column(Integer, ForeignKey('ROLE.id', ondelete="SET NULL"), nullable=False)
     active_env_short_name = Column(String, nullable=True)
     active_env_type = Column(Enum(EnvType), nullable=True)
     active_server_id = Column(Integer, ForeignKey('CUSTOM_SERVER.id', ondelete="SET NULL"))
-    show_server_page = Column(Boolean, default=True)
-    monthly_allocation = Column(Integer, default=0)
+    show_server_page = Column(Boolean, default = True, server_default='true')
+    monthly_allocation = Column(Integer, nullable=True, default=0, server_default='0')
 
     # Relationships
     user = relationship("User", back_populates="org_user_assocs")

{dataflow_core-2.1.15rc2 → dataflow_core-2.1.15rc3}/dataflow/models/organization.py

@@ -1,5 +1,5 @@
 from sqlalchemy import (
-    Column, Integer, String, Enum, DateTime, ForeignKey, Index
+    Column, Integer, String, Enum, DateTime, ForeignKey, Index, text
 )
 import uuid, enum
 from sqlalchemy.dialects.postgresql import JSONB, UUID
@@ -16,12 +16,12 @@ class Organization(Base):
     __tablename__ = "ORGANIZATION"
 
     id = Column(Integer, primary_key=True, autoincrement=True)
-    uid = Column(UUID(as_uuid=True), default=uuid.uuid4, nullable=False, unique=True) 
+    uid = Column(UUID(as_uuid=True), default=uuid.uuid4, nullable=False, unique=True, server_default=text("gen_random_uuid()"))
     name = Column(String(255), nullable=False, unique=True)
     invite_code = Column(String(64), nullable=False, unique=True)
     email_domain = Column(String(255), nullable=False, unique=True)
-    spark_enabled_zones = Column(JSONB, default=func.json([]))  # List of zone IDs where Spark is enabled
-    created_at = Column(DateTime, default=datetime.utcnow)
+    spark_enabled_zones = Column(JSONB, default=func.json([]), server_default=text("'[]'::jsonb"))  # List of zone IDs where Spark is enabled
+    created_at = Column(DateTime, default=datetime.utcnow, server_default=func.now())
 
     # Association object link
     org_user_assocs = relationship("OrganizationUser", back_populates="organization", cascade="all, delete-orphan")
@@ -68,7 +68,6 @@ class OrganizationOnboarding(Base):
     admin_username = Column(String(100), nullable=False, unique=True)
     admin_password = Column(String(255), nullable=False)
 
-    # New fields
     discovery_source = Column(String(255), nullable=True)
     additional_info = Column(String(1000), nullable=True)
     size_of_data = Column(String(100), nullable=True)

{dataflow_core-2.1.15rc2 → dataflow_core-2.1.15rc3}/dataflow/models/pinned_projects.py

@@ -1,4 +1,4 @@
-from sqlalchemy import Column, Integer, ForeignKey, DateTime, UniqueConstraint
+from sqlalchemy import Column, Integer, ForeignKey, DateTime, UniqueConstraint, func
 from dataflow.db import Base
 from datetime import datetime
 
@@ -8,7 +8,7 @@ class PinnedProject(Base):
     id = Column(Integer, primary_key=True)
     user_id = Column(Integer, ForeignKey('USER.user_id', ondelete="CASCADE"), index=True)
     project_id = Column(Integer, ForeignKey('PROJECT_DETAIL.project_id', ondelete="CASCADE"), index=True)
-    pinned_at = Column(DateTime, default=datetime.utcnow)
+    pinned_at = Column(DateTime, default=datetime.utcnow, server_default=func.now())
 
     __table_args__ = (
         UniqueConstraint("user_id", "project_id", name="uix_user_project"),

{dataflow_core-2.1.15rc2 → dataflow_core-2.1.15rc3}/dataflow/models/project_details.py

@@ -16,10 +16,11 @@ class ProjectDetails(Base):
     runtime = Column(String, nullable=False)
     py_env = Column(Integer, nullable=True)
     launch_url = Column(String, nullable=True)  
-    status = Column(Enum("pending", "created" ,"deployed", "stopped", "failed", name="deployment_status"), default="created")
+    status = Column(Enum("pending", "created" ,"deployed", "stopped", "failed", name="deployment_status"), default="created", server_default="created")
     last_deployed = Column(DateTime, nullable=True)
     created_at = Column(DateTime, nullable=False, server_default=func.now())
     created_by = Column(String, nullable=False)
     org_id = Column(Integer, ForeignKey('ORGANIZATION.id', ondelete='CASCADE'), nullable=False)
+    airflow_config_file = Column(String, nullable=True)
 
     app_type = relationship("AppType")

{dataflow_core-2.1.15rc2 → dataflow_core-2.1.15rc3}/dataflow/models/recent_project_studio.py

@@ -12,7 +12,7 @@ class RecentProjectStudio(Base):
     project_name = Column(String, nullable=False)
     project_path = Column(String, nullable=False)
     last_opened_date = Column(DateTime, server_default=func.now(), nullable=False)
-    remember = Column(Boolean, default=False)
+    remember = Column(Boolean, default=False, server_default='false')
 
     __table_args__ = (
         UniqueConstraint(user_name, project_path, app_name, name='user_name_project_path_app_name_unique'),

{dataflow_core-2.1.15rc2 → dataflow_core-2.1.15rc3}/dataflow/models/role.py

@@ -23,7 +23,7 @@ class Role(Base):
     name = Column(String, nullable=False)
     org_id = Column(Integer, ForeignKey('ORGANIZATION.id'))
     description = Column(String, nullable=True)
-    base_role = Column(Enum(BaseRoleField), nullable=False, default=BaseRoleField.user)
+    base_role = Column(Enum(BaseRoleField), nullable=False, default=BaseRoleField.user, server_default=BaseRoleField.user.value)
 
     # Relationships
     role_zone_assocs = relationship("RoleZone", back_populates="role")

{dataflow_core-2.1.15rc2 → dataflow_core-2.1.15rc3}/dataflow/models/role_zone.py

@@ -1,14 +1,19 @@
 from typing import Dict, List, Optional
-from sqlalchemy import Column, Integer, ForeignKey, UniqueConstraint, Boolean
+from sqlalchemy import Column, Integer, ForeignKey, UniqueConstraint, Boolean, Index
 from sqlalchemy.orm import relationship
 from dataflow.db import Base
 
 class RoleZone(Base):
     __tablename__ = 'ROLE_ZONE'
-
+    
     role_id = Column(Integer, ForeignKey('ROLE.id', ondelete="CASCADE"), primary_key=True)
     zone_id = Column(Integer, ForeignKey('DATAFLOW_ZONE.id', ondelete="CASCADE"), primary_key=True)
-    is_default = Column(Boolean, default=False, nullable=False)
+    is_default = Column(Boolean, default=False, nullable=False, server_default='false')
+    
+    __table_args__ = (
+        Index('idx_role_runtime_default', 'role_id', unique=True, 
+              postgresql_where=is_default.is_(True)),
+    )
 
     role = relationship("Role", back_populates="role_zone_assocs")
     zone = relationship("DataflowZone", back_populates="role_zone_assocs")

{dataflow_core-2.1.15rc2 → dataflow_core-2.1.15rc3}/dataflow/models/server_config.py

@@ -1,4 +1,4 @@
-from sqlalchemy import Column, Integer, String, Boolean, Text, ForeignKey
+from sqlalchemy import Column, Integer, String, Boolean, Text, ForeignKey, text
 from sqlalchemy.dialects.postgresql import JSONB 
 from sqlalchemy.sql import func
 from sqlalchemy.orm import relationship
@@ -14,10 +14,10 @@ class ServerConfig(Base):
     ram = Column(String, nullable=False)
     cpu = Column(String, nullable=False)
     gpu = Column(String)
-    default = Column(Boolean, default=False)
-    tags = Column(JSONB, default=func.json([]))
+    default = Column(Boolean, default=False, server_default='false')
+    tags = Column(JSONB, default=func.json([]), server_default=text("'[]'::jsonb"))
     description = Column(Text, nullable=True)
-    kubespawner_override = Column(JSONB, default=func.json({}))
+    kubespawner_override = Column(JSONB, default=func.json({}), server_default=text("'{}'::jsonb"))
 
     # Relationships
     organizations = relationship("Organization", secondary="ORGANIZATION_SERVER", back_populates="servers")
@@ -27,7 +27,7 @@ class CustomServerConfig(Base):
 
     id = Column(Integer, primary_key=True, autoincrement=True)
     base_server_id = Column(Integer, ForeignKey(ServerConfig.id, ondelete="CASCADE"), nullable=False)
-    org_id = Column(Integer, ForeignKey('ORGANIZATION.id'), nullable=False)
+    org_id = Column(Integer, ForeignKey('ORGANIZATION.id', ondelete="CASCADE"), nullable=False)
     display_name = Column(String, nullable=False, unique=True, index=True)
     description = Column(Text, nullable=True)
     

{dataflow_core-2.1.15rc2 → dataflow_core-2.1.15rc3}/dataflow/models/user.py

@@ -19,7 +19,7 @@ class User(Base):
     email = Column(String, unique=True)
     image = Column(LargeBinary)
     image_url = Column(String, nullable=True)
-    active = Column(Boolean, nullable=False, default=True)
+    active = Column(Boolean, nullable=False, default=True, server_default='true')
     password = Column(String, nullable=False)
     active_org_id = Column(Integer, ForeignKey('ORGANIZATION.id'))
 
@@ -59,9 +59,9 @@ class UserOnboarding(Base):
     id = Column(Integer, primary_key=True, autoincrement=True, nullable=False)
     user_id = Column(Integer, ForeignKey("USER.user_id", ondelete="CASCADE"), nullable=False)
     org_id = Column(Integer, ForeignKey("ORGANIZATION.id", ondelete="CASCADE"), nullable=False)
-    status = Column(Enum(OnboardingStatus, name='onboarding_status'), nullable=False, default=OnboardingStatus.pending.value)
-    created_at = Column(DateTime, default=func.now(), nullable=False)
-    updated_at = Column(DateTime, default=func.now(), onupdate=func.now(), nullable=False)
+    status = Column(Enum(OnboardingStatus, name='onboarding_status'), nullable=False, default=OnboardingStatus.pending.value, server_default='pending')
+    created_at = Column(DateTime, default=func.now(), nullable=False, server_default=func.now())
+    updated_at = Column(DateTime, default=func.now(), onupdate=func.now(), nullable=False, server_default=func.now())
 
     # Relationships
     user = relationship("User", back_populates="onboarding_requests")

{dataflow_core-2.1.15rc2 → dataflow_core-2.1.15rc3}/dataflow/models/variables.py

@@ -25,9 +25,10 @@ class Variable(Base):
     created_at = Column(DateTime, server_default=func.now())
     updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
     created_by = Column(String, ForeignKey('USER.user_name'), nullable=True)
-    is_active = Column(Boolean, default=True, nullable=False)
+    is_active = Column(Boolean, default=True, nullable=False, server_default='true')
     datatype = Column(Enum(DataType, name="data_type"), nullable=False)
-    set_as_env = Column(Boolean, default=False, nullable=False)
+    set_as_env = Column(Boolean, default=False, nullable=False, server_default='false')
+
 
     __table_args__ = (
         CheckConstraint(type.in_(['variable', 'secret']), name='check_variable_type'),

{dataflow_core-2.1.15rc2 → dataflow_core-2.1.15rc3}/dataflow_core.egg-info/PKG-INFO

@@ -1,10 +1,11 @@
 Metadata-Version: 2.4
 Name: dataflow-core
-Version: 2.1.15rc2
+Version: 2.1.15rc3
 Summary: Dataflow core package
 Author: Dataflow
 Author-email: 
 Requires-Dist: sqlalchemy
+Requires-Dist: alembic
 Requires-Dist: boto3
 Requires-Dist: psycopg2-binary
 Requires-Dist: pymysql

{dataflow_core-2.1.15rc2 → dataflow_core-2.1.15rc3}/dataflow_core.egg-info/SOURCES.txt

@@ -59,4 +59,8 @@ dataflow_core.egg-info/SOURCES.txt
 dataflow_core.egg-info/dependency_links.txt
 dataflow_core.egg-info/entry_points.txt
 dataflow_core.egg-info/requires.txt
-dataflow_core.egg-info/top_level.txt
+dataflow_core.egg-info/top_level.txt
+dfmigration/__init__.py
+dfmigration/env.py
+dfmigration/versions/001_initial_baseline_migration.py
+dfmigration/versions/__init__.py

{dataflow_core-2.1.15rc2 → dataflow_core-2.1.15rc3}/dataflow_core.egg-info/requires.txt

@@ -1,4 +1,5 @@
 sqlalchemy
+alembic
 boto3
 psycopg2-binary
 pymysql

{dataflow_core-2.1.15rc2 → dataflow_core-2.1.15rc3}/dataflow_core.egg-info/top_level.txt

@@ -1,2 +1,3 @@
 authenticator
 dataflow
+dfmigration

dataflow_core-2.1.15rc3/dfmigration/env.py

@@ -0,0 +1,45 @@
+from sqlalchemy import engine_from_config, pool
+from alembic import context
+import os
+
+config = context.config
+target_metadata = None
+
+def get_url():
+    return os.getenv('DATABASE_URL')
+
+def run_migrations_offline() -> None:
+    url = get_url()
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+def run_migrations_online() -> None:
+    configuration = config.get_section(config.config_ini_section) or {}
+    configuration['sqlalchemy.url'] = get_url()
+    
+    connectable = engine_from_config(
+        configuration,
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection, 
+            target_metadata=target_metadata
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

dataflow_core-2.1.15rc3/dfmigration/versions/001_initial_baseline_migration.py

@@ -0,0 +1,20 @@
+"""Initial baseline migration
+
+Revision ID: 001
+Revises: None
+Create Date: 2025-09-10 15:00:00.000000
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+revision = '001'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+def upgrade() -> None:
+    pass
+
+def downgrade() -> None:
+    pass

{dataflow_core-2.1.15rc2 → dataflow_core-2.1.15rc3}/setup.py

@@ -14,14 +14,15 @@ class PostInstall(install):
 
 setup(
     name="dataflow-core",
-    version="2.1.15rc2",
-    packages=find_packages(include=["dataflow", "dataflow.*", "authenticator", "authenticator.*"]),
+    version="2.1.15rc3",
+    packages=find_packages(include=["dataflow", "dataflow.*", "authenticator", "authenticator.*", "dfmigration", "dfmigration.*"]),
     include_package_data=True,
     package_data={
         "dataflow": ["scripts/*.sh"],
     },
     install_requires=[
         'sqlalchemy',
+        'alembic',
         'boto3',
         'psycopg2-binary',
         'pymysql',