dataflow-core 2.1.30rc12__tar.gz → 2.1.32__tar.gz

{dataflow_core-2.1.30rc12 → dataflow_core-2.1.32}/PKG-INFO

@@ -1,14 +1,16 @@
 Metadata-Version: 2.4
 Name: dataflow-core
-Version: 2.1.30rc12
+Version: 2.1.32
 Summary: Dataflow core package
 Author: Dataflow
 Author-email: 
-Requires-Dist: sqlalchemy
+Requires-Dist: sqlalchemy<2.1
 Requires-Dist: alembic
 Requires-Dist: boto3
 Requires-Dist: psycopg2-binary
 Requires-Dist: pymysql
+Requires-Dist: pymssql<3,>=2.2.8
+Requires-Dist: sqlalchemy-bigquery<2,>=1.12.1
 Requires-Dist: requests
 Requires-Dist: azure-identity
 Requires-Dist: azure-keyvault-secrets

{dataflow_core-2.1.30rc12 → dataflow_core-2.1.32}/dataflow/environment.py

@@ -1,5 +1,5 @@
 import os, shutil, subprocess, datetime, yaml, re
-from .models.environment import JobLogs, Environment, LocalEnvironment
+from .models.environment import JobLogs, Environment, LocalEnvironment, ArchivedEnvironment
 import json, asyncio, pkg_resources
 from sqlalchemy.orm import Session
 from .configuration import ConfigurationManager
@@ -74,7 +74,11 @@ class EnvironmentManager:
             if db and log_file_location:
                 log_file_name = os.path.basename(log_file_location)
                 await self._update_job_status(log_file_name, build_status, log_file_location, db)
-                pip_libraries, conda_libraries = self.update_library_versions(yaml_path)
+                if build_status == "success":
+                    env_path = os.path.join(self.env_build_path, env_name)
+                    pip_libraries, conda_libraries = self.update_library_versions(env_path)
+                else:
+                    pip_libraries, conda_libraries = None, None
                 self.update_environment_db(env_name, env_version, pip_libraries, conda_libraries, build_status, db)
 
             return build_status
@@ -164,7 +168,13 @@ class EnvironmentManager:
             os.symlink(old_yaml_path, new_yaml_path)
             os.symlink(old_squashfs_path, new_squashfs_path)
 
-            pip_libraries, conda_libraries = self.update_library_versions(new_yaml_path)
+            archived = db.query(ArchivedEnvironment).filter(
+                ArchivedEnvironment.short_name == env_name,
+                ArchivedEnvironment.version == str(revert_version),
+                ArchivedEnvironment.org_id == self.org_id
+            ).first()
+            pip_libraries = archived.pip_libraries if archived else None
+            conda_libraries = archived.conda_libraries if archived else None
 
             self.update_environment_db(env_name, new_version, pip_libraries, conda_libraries, "success", db)
             return "success"
@@ -172,30 +182,180 @@ class EnvironmentManager:
         except Exception as e:
             self.logger.critical(f"Failed to revert environment {env_name}: {e}", exc_info=True, extra={"status_code": 500, "environment": env_name})
             return "failed"
-        
+
+    async def build_env_from_upper(self, env_name, base_short_name, base_version, base_created_by,
+                                    tar_path, py_version, env_version, user_name=None, db: Session = None):
+        """Builds a new squashfs environment from a captured overlay upperdir.
+
+        Reconstructs the overlay a user's activated environment ran with at
+        runtime (the base environment's squashfs as the read-only lowerdir,
+        plus a tar of the writable upperdir capturing their pip/conda
+        modifications) and repackages the merged result as a new versioned
+        squashfs. This is the "Save as Draft" build path -- as opposed to
+        create_env, which builds a fresh conda environment from a YAML spec.
+
+        A conda env bakes its absolute prefix into shebangs, conda-meta and
+        .pth files, so build_from_upper.sh branches on whether the prefix
+        changes: same prefix ("Save") squashes the merged tree verbatim, a new
+        prefix ("Save As" under a new short name) clones it first so conda
+        rewrites those paths.
+
+        Args:
+            env_name (str): Short name for the new/target environment
+            base_short_name (str): Short name of the activated (base) environment
+            base_version (str): Version of the activated (base) environment
+            base_created_by (str): created_by of the base environment ("Dataflow"
+                for system environments, which live under the shared "dataflow"
+                sub path instead of an org-scoped one)
+            tar_path (str): Path to the tarred overlay upperdir
+            py_version (str): Python version, recorded in the conda YAML
+            env_version (str): Version to assign to the new environment
+            user_name (str): Username who initiated the save
+            db (Session): Database session
+
+        Returns:
+            str: Build status ('success' or 'failed')
+        """
+        log_file_location = None
+        if db:
+            log_file_location = self._setup_logging(env_name, env_version, user_name, db)
+
+        env_root_path = self.config.get_config_value('paths', 'env_path')
+        env_mount_root = self.config.get_config_value('paths', 'env_mount_path')
+        if base_created_by == "Dataflow":
+            base_squashfs_path = os.path.join(env_root_path, "dataflow", "python_envs", f"{base_short_name}_v{base_version}.squashfs")
+            base_sub_path = "dataflow"
+        else:
+            base_squashfs_path = os.path.join(self.env_base_path, f"{base_short_name}_v{base_version}.squashfs")
+            base_sub_path = self.env_sub_path
+
+        # Prefixes must match what dataflow-init mounts the squashfs at, since
+        # that is the path baked into the environment's own files:
+        # <env_mount_path>/{dataflow|org_id}/<short_name>.
+        base_prefix = os.path.join(env_mount_root, base_sub_path, base_short_name)
+        target_prefix = os.path.join(self.env_build_path, env_name)
+
+        yaml_path = os.path.join(self.env_version_path, f"{env_name}_v{env_version}.yaml")
+        scratch_dir = os.path.join(self.env_build_path, f"{env_name}_v{env_version}_scratch")
+
+        build_status = await self._execute_env_operation(
+            env_name=env_name,
+            status="draft",
+            mode="from_upper",
+            yaml_file_path=yaml_path,
+            log_file_location=log_file_location,
+            env_version=int(env_version),
+            py_version=py_version,
+            base_squashfs_path=base_squashfs_path,
+            tar_path=tar_path,
+            scratch_dir=scratch_dir,
+            base_prefix=base_prefix,
+            target_prefix=target_prefix,
+        )
+
+        # Update job log status and library versions if db was provided
+        if db and log_file_location:
+            log_file_name = os.path.basename(log_file_location)
+            await self._update_job_status(log_file_name, build_status, log_file_location, db)
+
+            # build_from_upper.sh deliberately leaves the overlay mounted (and
+            # any clone in place) on success so we can introspect the built
+            # environment here before tearing it down -- unlike create_env's
+            # build prefix, base_prefix is a mount, not a plain directory, so
+            # it must be read before cleanup. On failure nothing was left
+            # behind (or it was torn down by the script's own failure trap),
+            # so the path is just an empty directory -- reading it would
+            # silently report zero packages and blank out this version's known
+            # library list. Leave pip_libraries/conda_libraries as None
+            # (update_environment_db then skips those columns) so a failed
+            # build doesn't clobber whatever the row already had.
+            if build_status == "success":
+                pip_libraries, conda_libraries = self.update_library_versions(target_prefix)
+                self.create_conda_yaml(
+                    yaml_path=yaml_path,
+                    env_name=env_name,
+                    python_version=py_version,
+                    conda_packages=conda_libraries,
+                    pip_packages=pip_libraries,
+                )
+            else:
+                pip_libraries, conda_libraries = None, None
+            self.update_environment_db(env_name, env_version, pip_libraries, conda_libraries, build_status, db)
+
+        self._cleanup_overlay_scratch(scratch_dir, base_prefix, target_prefix)
+
+        # The tar is a one-shot input; remove it regardless of outcome so a
+        # retry doesn't silently reuse a stale/partial capture, and so it
+        # doesn't linger on the user's NFS home.
+        try:
+            if os.path.exists(tar_path):
+                os.remove(tar_path)
+        except OSError as e:
+            self.logger.error(f"Failed to remove upper-layer tar {tar_path}: {e}")
+
+        return build_status
+
+    def _cleanup_overlay_scratch(self, scratch_dir: str, base_prefix: str, target_prefix: str):
+        """Unmounts the overlay left mounted by build_from_upper.sh and removes
+        its scratch directory, plus the clone it may have built at
+        target_prefix. Safe to call even if nothing ended up mounted (e.g. the
+        build failed before the overlay mount step).
+
+        Args:
+            scratch_dir (str): The lower/upper/work scratch directory passed to
+                build_from_upper.sh
+            base_prefix (str): The base environment's prefix, where the merged
+                overlay was mounted
+            target_prefix (str): The new environment's prefix; equal to
+                base_prefix on a version bump, otherwise the cloned env
+        """
+        lower_dir = os.path.join(scratch_dir, "lower")
+        for mount_dir in (base_prefix, lower_dir):
+            try:
+                if subprocess.run(["mountpoint", "-q", mount_dir]).returncode == 0:
+                    subprocess.run(["umount", mount_dir], check=True)
+            except Exception as e:
+                self.logger.error(f"Failed to unmount {mount_dir}: {e}")
+        shutil.rmtree(scratch_dir, ignore_errors=True)
+        shutil.rmtree(target_prefix, ignore_errors=True)
+
     async def _execute_env_operation(
-        self, 
-        env_name: str, 
-        status: str, 
-        mode: str, 
-        yaml_file_path: str, 
+        self,
+        env_name: str,
+        status: str,
+        mode: str,
+        yaml_file_path: str,
         env_version: int,
-        source_path=None, 
+        source_path=None,
         log_file_location=None,
-        py_version=None
+        py_version=None,
+        base_squashfs_path=None,
+        tar_path=None,
+        scratch_dir=None,
+        base_prefix=None,
+        target_prefix=None,
     ):
-        """Executes environment operations (create or clone).
-        
+        """Executes environment operations (create, clone, or from_upper).
+
         Args:
             env_name (str): Name of the environment
             status (str): Environment status ('draft' or 'published')
-            mode (str): Operation mode ('create' or 'clone')
+            mode (str): Operation mode ('create', 'clone', 'local_clone', or 'from_upper')
             env_version (str): Version of the environment (for draft environments)
             py_version (str): Python version to use (for create mode)
             py_requirements (list): List of packages to install (for create mode)
             source_path (str): Path to source environment (for clone mode)
             log_file_location (str): Path to log file
-            
+            base_squashfs_path (str): Path to the base squashfs to loop-mount as the
+                overlay lowerdir (for from_upper mode)
+            tar_path (str): Path to the tarred overlay upperdir (for from_upper mode)
+            scratch_dir (str): Scratch directory for the lower/upper/work
+                mounts (for from_upper mode)
+            base_prefix (str): Prefix the base environment was built at, where
+                the merged overlay gets mounted (for from_upper mode)
+            target_prefix (str): Prefix the new environment will be mounted at;
+                differing from base_prefix triggers the clone (for from_upper mode)
+
         Returns:
             str: Build status ('success' or 'failed')
         """
@@ -218,14 +378,18 @@ class EnvironmentManager:
             elif mode == "clone":
                 clone_env_script_path = pkg_resources.resource_filename('dataflow', 'scripts/clone_environment.sh')
                 command = ["bash", clone_env_script_path, source_path, os.path.join(self.env_build_path, env_name), os.path.join(self.env_base_path, f"{env_name}_v{env_version}.squashfs"), yaml_file_path]
-            
+
             elif mode == "local_clone":
                 conda_cmd = f"conda create --name {env_name} --clone {source_path} --yes"
                 command = ["su", "-", os.environ.get("NB_USER"), "-c", conda_cmd]
                 self.logger.info(f"Cloning environment locally with command: {' '.join(command)}")
 
+            elif mode == "from_upper":
+                build_from_upper_script_path = pkg_resources.resource_filename('dataflow', 'scripts/build_from_upper.sh')
+                command = ["bash", build_from_upper_script_path, base_squashfs_path, tar_path, conda_env_path, scratch_dir, base_prefix, target_prefix]
+
             else:
-                raise ValueError("Invalid mode. Use 'create' or 'clone'.")
+                raise ValueError("Invalid mode. Use 'create', 'clone', 'local_clone', or 'from_upper'.")
 
             process = await asyncio.create_subprocess_exec(
                 *command, 
@@ -439,8 +603,11 @@ class EnvironmentManager:
         Args:
             env_short_name (str): Short name of the environment
             version (str): Version of the environment
-            pip_libraries (list): List of pip libraries
-            conda_libraries (list): List of conda libraries
+            pip_libraries (list or None): List of pip libraries, or None to leave the
+                column untouched (e.g. a failed build never actually introspected
+                the new version's packages -- don't clobber whatever was there)
+            conda_libraries (list or None): List of conda libraries, or None to leave
+                the column untouched
             status (str): Build status ('success' or 'failed')
             db (Session): Database session
         """
@@ -452,13 +619,19 @@ class EnvironmentManager:
             current_env = db.query(Environment).filter(Environment.short_name == env_short_name).first()
             if not current_env:
                 raise ValueError(f"Environment with short name '{env_short_name}' does not exist.")
-            
+
             env_status = "Draft" if status == "success" else "Failed"
 
+            update_values = {"version": version, "status": env_status}
+            if pip_libraries is not None:
+                update_values["pip_libraries"] = pip_libraries
+            if conda_libraries is not None:
+                update_values["conda_libraries"] = conda_libraries
+
             db.query(Environment).filter(
                 Environment.short_name == env_short_name,
                 Environment.org_id == self.org_id if self.org_id is not None else Environment.org_id.is_(None)
-            ).update({"version": version, "pip_libraries": pip_libraries, "conda_libraries": conda_libraries, "status": env_status})
+            ).update(update_values)
             db.commit()
 
         except Exception as e:
@@ -466,63 +639,46 @@ class EnvironmentManager:
             db.rollback()
             raise
 
-    def update_library_versions(self, yaml_path: str):
+    def update_library_versions(self, env_path: str):
         """
-        Updates libraries without version specifications by getting their actual installed versions from a conda YAML file.
-        
-        Args:
-            yaml_path (str): Path to the conda environment YAML file.
-            
+        Reads pip and conda packages from a built environment directory.
+
+        Uses pip freeze via the env's own Python binary so packages installed
+        with --target (e.g. dataflow-airflow via airflow_custom.pth) are included.
+        Runs conda list for conda-channel packages.
+
         Returns:
-            tuple: Updated lists of (pip_libraries, conda_libraries) with version specifications.
+            tuple: (pip_libraries, conda_libraries) as lists of "name==version" / "name=version" strings.
         """
+        pip_libraries = []
         try:
-            # Define default conda packages to ignore
-            default_conda_packages = {
-                "_libgcc_mutex", "_openmp_mutex", "bzip2", "ca-certificates", 
-                "ld_impl_linux-64", "libexpat", "libffi", "libgcc", "libgcc-ng", 
-                "libgomp", "liblzma", "libnsl", "libsqlite", "libuuid", "libxcrypt", 
-                "libzlib", "ncurses", "openssl", "readline", "setuptools", "tk", 
-                "tzdata", "wheel", "libstdcxx-ng", "python"
-            }
-            
-            # Read the YAML file
-            with open(yaml_path, 'r') as f:
-                yaml_content = yaml.safe_load(f)
-            
-            # Extract conda and pip dependencies
-            dependencies = yaml_content.get('dependencies', [])
-            
-            # Process conda libraries
-            conda_libraries = []
-            pip_libraries = []
-            
-            for dep in dependencies:
-                if isinstance(dep, str):
-                    if dep.startswith("python="):
-                        continue
-                    
-                    parts = dep.split('=')
-                    package_name = parts[0].strip()
-                    
-                    if package_name.lower() not in default_conda_packages:
-                        if len(parts) >= 2:
-                            package_with_version = f"{package_name}={parts[1]}"
-                            conda_libraries.append(package_with_version)
-                        else:
-                            # No version specified, keep as is
-                            conda_libraries.append(dep)
-                
-                elif isinstance(dep, dict) and 'pip' in dep:
-                    # This is the pip section
-                    for pip_pkg in dep['pip']:
-                        pip_libraries.append(pip_pkg)
-            
-            return pip_libraries, conda_libraries
-            
+            result = subprocess.run(
+                [f"{env_path}/bin/python", "-m", "pip", "freeze", "--disable-pip-version-check"],
+                capture_output=True, text=True, timeout=120
+            )
+            if result.returncode == 0:
+                for line in result.stdout.strip().splitlines():
+                    line = line.strip()
+                    if line and not line.startswith("#") and not line.startswith("-"):
+                        pip_libraries.append(line)
+            else:
+                self.logger.error(f"pip freeze failed for {env_path}: {result.stderr.strip()}")
         except Exception as e:
-            self.logger.critical(f"Error reading YAML file and extracting libraries: {str(e)}", exc_info=True, extra={"status_code": 500})
-            return [], []
+            self.logger.error(f"Failed to get pip packages for {env_path}: {e}")
+
+        conda_libraries = []
+        try:
+            result = subprocess.run(
+                ["conda", "list", "--prefix", env_path, "--json"],
+                capture_output=True, text=True, check=True
+            )
+            for pkg in json.loads(result.stdout):
+                if pkg.get("channel") not in ("pypi", "<develop>") and pkg["name"] != "python":
+                    conda_libraries.append(f"{pkg['name']}={pkg['version']}")
+        except Exception as e:
+            self.logger.error(f"Failed to get conda packages for {env_path}: {e}")
+
+        return pip_libraries, conda_libraries
 
     def create_conda_yaml(self, yaml_path, env_name, python_version, conda_packages, pip_packages):
         """

{dataflow_core-2.1.30rc12 → dataflow_core-2.1.32}/dataflow/models/__init__.py

@@ -36,3 +36,4 @@ from .active_server_view import ActiveServerView
 from .inference_service import InferenceService, InferenceStatus
 from .onboarding_project import OnboardingProject
 from .ai_provider import AIProvider, AIModel, AIUserProviderKey
+from .dbml_diagram import Diagram

dataflow_core-2.1.32/dataflow/models/dbml_diagram.py

@@ -0,0 +1,58 @@
+from sqlalchemy import Column, Integer, String, ForeignKey, Text, DateTime, func, Enum, Index, text
+from sqlalchemy.dialects.postgresql import JSONB, UUID
+from dataflow.db import Base
+import uuid
+
+
+class Diagram(Base):
+
+    """TABLE 'DBML_DIAGRAM'
+
+    Stores a user-authored database ER diagram. The DBML source is the single source of
+    truth for the schema itself (tables, columns, relationships, header colours); the
+    layout column only holds presentation state that DBML has no syntax for, i.e. where
+    each table sits on the canvas and the last viewport.
+
+    Attributes:
+        id (int): Primary key for the diagram entry.
+        uid (UUID): Public identifier used in URLs, so the internal id is never exposed.
+        name (str): Display name of the diagram, unique per user within an organization.
+        description (str): Optional description of the diagram.
+        org_id (int): Foreign key referencing the ORGANIZATION table.
+        user_id (int): Foreign key referencing the USER table (the owner).
+        dbml_source (str): The DBML document defining the schema.
+        layout (dict): Canvas presentation state. Shape:
+            {"tables": {"public.users": {"x": 40, "y": 120}}, "viewport": {"x": 0, "y": 0, "zoom": 1}}
+        visibility (str): 'PRIVATE' or 'ORG'. Reserved for org sharing; currently always PRIVATE.
+        created_at (datetime): Timestamp of when the diagram was created.
+        updated_at (datetime): Timestamp of the last update to the diagram.
+        deleted_at (datetime): Soft delete timestamp; NULL while the diagram is active.
+
+    Constraints:
+        Index: Partial unique index ensuring a user cannot have two active diagrams
+            with the same name in the same organization.
+    """
+
+    __tablename__ = 'DBML_DIAGRAM'
+
+    id = Column(Integer, primary_key=True, autoincrement=True, nullable=False)
+    uid = Column(UUID(as_uuid=True), default=uuid.uuid4, nullable=False, unique=True, server_default=text("gen_random_uuid()"))
+    name = Column(String, nullable=False)
+    description = Column(Text, nullable=True)
+    org_id = Column(Integer, ForeignKey("ORGANIZATION.id", ondelete="CASCADE"), index=True, nullable=False)
+    user_id = Column(Integer, ForeignKey("USER.user_id", ondelete="CASCADE"), index=True, nullable=False)
+    dbml_source = Column(Text, nullable=False, server_default='')
+    layout = Column(JSONB, nullable=True)
+    visibility = Column(Enum("PRIVATE", "ORG", name="diagram_visibility"), default="PRIVATE", server_default="PRIVATE", nullable=False)
+    created_at = Column(DateTime(timezone=True), server_default=func.now())
+    updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
+    deleted_at = Column(DateTime(timezone=True), nullable=True)
+
+    __table_args__ = (
+        Index(
+            'uq_active_diagram_name',
+            'name', 'user_id', 'org_id',
+            unique=True,
+            postgresql_where=deleted_at.is_(None)
+        ),
+    )

{dataflow_core-2.1.30rc12 → dataflow_core-2.1.32}/dataflow/models/environment.py

@@ -1,5 +1,5 @@
 from sqlalchemy import (
-    Column, Integer, String, Boolean, Text, 
+    Column, Integer, BigInteger, String, Boolean, Text,
     ForeignKey, DateTime, UniqueConstraint, CheckConstraint, Index
 )
 from sqlalchemy.orm import relationship, Session
@@ -181,6 +181,20 @@ class LocalEnvironment(Base):
         cloned_from (str): Source from which the environment was cloned.
         updated_at (datetime): Timestamp of when the local environment was last updated.
         need_refresh (bool): Indicates if the local environment needs a refresh.
+        base_env_id (int): ID of the base (squashfs) ENVIRONMENT this workset's delta sits on.
+        base_version (str): Version of the base environment the delta is keyed to, mirrors the
+            on-disk `_worksets/{short_name}_v{version}/` directory.
+        pip_diff (str): Precomputed package diff (added/removed/modified) captured at the last
+            checkpoint, serialized as JSON. Computed while the env is live so it never needs to
+            be re-derived from the (otherwise meaningless without its base) delta tar.
+        workset_path (str): Path to the persisted delta directory, relative to the user's jovyan
+            home, e.g. `.dataflow/envs/_worksets/{short_name}_v{version}`.
+        delta_size_bytes (int): Size in bytes of the last checkpointed delta.tar, for UI display
+            and retention/cleanup decisions.
+        is_workset (bool): True for auto-persisted workset shadow rows (one per user/base
+            env/version, tracking the overlay upperdir delta), as opposed to genuine
+            user-created local/conda environments which also live in this table. Queries that
+            list a user's real local environments must filter is_workset = false.
     """
 
     __tablename__ = "LOCAL_ENVIRONMENT"
@@ -197,6 +211,13 @@ class LocalEnvironment(Base):
     cloned_from = Column(String, nullable=True)
     updated_at = Column(DateTime, default=datetime.now(timezone.utc), onupdate=datetime.now(timezone.utc))
     need_refresh = Column(Boolean, default=False, server_default='false')
+    # Auto-persisted private-environment workset tracking (overlay upperdir delta).
+    base_env_id = Column(Integer, nullable=True)
+    base_version = Column(String, nullable=True)
+    pip_diff = Column(Text, nullable=True)
+    workset_path = Column(String, nullable=True)
+    delta_size_bytes = Column(BigInteger, nullable=True)
+    is_workset = Column(Boolean, default=False, server_default='false', nullable=False)
 
 class EnvType(str, Enum):
 

{dataflow_core-2.1.30rc12 → dataflow_core-2.1.32}/dataflow/models/project_details.py

@@ -36,7 +36,13 @@ class ProjectDetails(Base):
         exposed_port (int): Port exposed by the custom image.
         private_repo (bool): Whether the image repository is private.
         registry_secret_id (int): Foreign key referencing the VARIABLE table for registry credentials.
-    
+        source_type (str): Where the project's code comes from: GIT (cloned from git_url) or STUDIO
+            (synced from the creator's Studio folder into the runtime's projects volume).
+        studio_source_path (str): For STUDIO projects, the source folder relative to the creator's
+            /home/jovyan.
+        last_synced_at (datetime): For STUDIO projects, when files were last synced from Studio.
+        synced_by (str): For STUDIO projects, the user who ran the last sync.
+
     Relationships:
         app_type: Relationship to the AppType model.
     """
@@ -87,6 +93,15 @@ class ProjectDetails(Base):
     resource_config = Column(JSONB, nullable=True)
     cost_incurred = Column(JSONB, nullable=True)
     runtime_cost_state = Column(JSONB, nullable=True)
+    source_type = Column(
+        Enum("GIT", "STUDIO", name="project_source_type"),
+        nullable=False,
+        default="GIT",
+        server_default="GIT",
+    )
+    studio_source_path = Column(String, nullable=True)
+    last_synced_at = Column(DateTime(timezone=True), nullable=True)
+    synced_by = Column(String, nullable=True)
 
     __table_args__ = (
         Index(

dataflow_core-2.1.32/dataflow/scripts/build_from_upper.sh

@@ -0,0 +1,103 @@
+#!/bin/bash
+
+set -e
+
+# Reconstructs the overlayfs a user's activated environment ran with at
+# runtime (read-only base squashfs as lowerdir + a captured upperdir tar of
+# their pip/conda modifications), then repackages the merged result as a new
+# squashfs. Requires CAP_SYS_ADMIN (loop + overlay mounts) and CAP_MKNOD
+# (recreating whiteout char-devices on tar extract) -- run this in a
+# privileged container.
+
+base_squashfs_path=$1
+tar_path=$2
+squashfs_file_path=$3
+scratch_dir=$4
+base_prefix=$5
+target_prefix=$6
+
+if [ -z "$base_squashfs_path" ] || [ -z "$tar_path" ] || [ -z "$squashfs_file_path" ] \
+    || [ -z "$scratch_dir" ] || [ -z "$base_prefix" ] || [ -z "$target_prefix" ]; then
+    echo "Error: Missing required parameters"
+    exit 1
+fi
+
+if [ ! -f "$base_squashfs_path" ]; then
+    echo "Error: base squashfs file does not exist: $base_squashfs_path"
+    exit 1
+fi
+
+if [ ! -f "$tar_path" ]; then
+    echo "Error: upper tar file does not exist: $tar_path"
+    exit 1
+fi
+
+squashfs_file_name="$(basename "${squashfs_file_path}")"
+env_name_with_version="${squashfs_file_name%.squashfs}"
+
+lower_dir="${scratch_dir}/lower"
+upper_dir="${scratch_dir}/upper"
+work_dir="${scratch_dir}/work"
+merged_dir="${base_prefix}"
+
+# On failure, unwind whatever mounts were set up so we don't leak them. On
+# success we deliberately leave lower/merged mounted, and leave a cloned
+# target prefix in place -- the caller reads them (pip/conda introspection)
+# before tearing everything down.
+cleanup_on_failure() {
+    mountpoint -q "$merged_dir" 2>/dev/null && umount "$merged_dir" 2>/dev/null
+    mountpoint -q "$lower_dir" 2>/dev/null && umount "$lower_dir" 2>/dev/null
+    if [ "$target_prefix" != "$base_prefix" ]; then
+        rm -rf "$target_prefix"
+    fi
+    exit 1
+}
+trap cleanup_on_failure ERR
+
+rm -rf "$scratch_dir"
+mkdir -p "$lower_dir" "$upper_dir" "$work_dir"
+
+# Defensive: a leftover mount here would silently become the clone source.
+mountpoint -q "$merged_dir" 2>/dev/null && umount "$merged_dir"
+rm -rf "$merged_dir"
+mkdir -p "$merged_dir"
+
+# lowerdir: loop-mount the base squashfs read-only -- identical to the
+# lowerdir the user's pod ran against at runtime.
+mount -t squashfs -o loop,ro "$base_squashfs_path" "$lower_dir"
+
+# upperdir: extract the captured delta. --xattrs is mandatory -- an
+# opaque-directory whiteout (a directory the user deleted and recreated) is
+# recorded as the trusted.overlay.opaque xattr, not a regular tar entry, and
+# silently vanishes on reconstruction without it. Character-device whiteout
+# markers for deleted files require CAP_MKNOD to recreate on extract.
+tar --xattrs --xattrs-include='*' --acls -xf "$tar_path" -C "$upper_dir"
+
+# merged: reconstruct the overlay exactly as it was on the user's pod.
+mount -t overlay overlay -o "lowerdir=${lower_dir},upperdir=${upper_dir},workdir=${work_dir}" "$merged_dir"
+
+if [ "$base_prefix" = "$target_prefix" ]; then
+    # Same prefix: the merged view is already correct for where it will be
+    # mounted. Squash it as-is.
+    squashfs_source="$merged_dir"
+else
+    echo "${base_prefix} to ${target_prefix}; cloning environment"
+
+    export PIP_NO_CACHE_DIR=1
+    export CONDA_PKGS_DIRS="${scratch_dir}/pkgs/${env_name_with_version}/"
+    mkdir -p "$CONDA_PKGS_DIRS"
+    conda config --set always_copy true
+
+    mkdir -p "$(dirname "$target_prefix")"
+    conda create --clone "$merged_dir" --prefix "$target_prefix" --yes
+
+    squashfs_source="$target_prefix"
+fi
+
+# Build the new squashfs from the merged (base + user modifications) view, or
+# from the prefix-corrected clone of it.
+mksquashfs "$squashfs_source" "${scratch_dir}/${squashfs_file_name}" -comp zstd -Xcompression-level 4 -force-uid 1000 -force-gid 1000
+mkdir -p "$(dirname "$squashfs_file_path")"
+cp -f "${scratch_dir}/${squashfs_file_name}" "$squashfs_file_path"
+
+echo "Environment Creation Successful"

{dataflow_core-2.1.30rc12 → dataflow_core-2.1.32}/dataflow/scripts/clone_environment.sh

@@ -23,7 +23,7 @@ conda create --clone ${source_env_path} --prefix ${env_build_path} --yes
 conda env export --prefix "${env_build_path}" > "$yaml_file_path"
 
 # 2. Create squashfs file from the cloned environment
-mksquashfs "${env_build_path}" "${squashfs_file_name}" -comp zstd -Xcompression-level 4
+mksquashfs "${env_build_path}" "${squashfs_file_name}" -comp zstd -Xcompression-level 4 -force-uid 1000 -force-gid 1000
 cp -f "${squashfs_file_name}" "${squashfs_file_path}"
 # 3. Cleanup the temporary environment directory
 rm -rf "${env_build_path}"